mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-09-03 02:33:53 +00:00
Code Issues 1 Projects Releases Wiki Activity

validate:launcher: Rework expected-issues data format

Browse source 
Instead of having the issues centered on the test classes, they
are now focusing on the "bug".

And harmise names on `expected_issue` not `expected_failures`
This commit is contained in:
Thibault Saunier 2019-03-18 16:52:11 -03:00 committed by Thibault Saunier
parent 6a4639352b
commit ac7efe9500
3 changed files with 103 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
validate/launcher/apps/gstvalidate.py
View file

@ -301,7 +301,7 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator):
fname = self.get_fname( fname = self.get_fname(
scenario, protocol=mediainfo.get_protocol(), name=description["name"]) scenario, protocol=mediainfo.get_protocol(), name=description["name"])
expected_failures = extra_data.get("expected-failures") expected_issues = extra_data.get("expected-issues")
extra_env_vars = extra_data.get("extra_env_vars") extra_env_vars = extra_data.get("extra_env_vars")
test = GstValidateLaunchTest(fname, test = GstValidateLaunchTest(fname,
self.test_manager.options, self.test_manager.options,
@ -309,7 +309,7 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator):
pipeline_desc, pipeline_desc,
scenario=scenario, scenario=scenario,
media_descriptor=mediainfo, media_descriptor=mediainfo,
expected_failures=expected_failures, expected_issues=expected_issues,
extra_env_variables=extra_env_vars) extra_env_variables=extra_env_vars)
if extra_data.get('config_file'): if extra_data.get('config_file'):
test.add_validate_config(extra_data['config_file']) test.add_validate_config(extra_data['config_file'])
@ -497,7 +497,7 @@ class GstValidateLaunchTest(GstValidateTest):
def __init__(self, classname, options, reporter, pipeline_desc, def __init__(self, classname, options, reporter, pipeline_desc,
timeout=DEFAULT_TIMEOUT, scenario=None, timeout=DEFAULT_TIMEOUT, scenario=None,
media_descriptor=None, duration=0, hard_timeout=None, media_descriptor=None, duration=0, hard_timeout=None,
extra_env_variables=None, expected_failures=None): extra_env_variables=None, expected_issues=None):
extra_env_variables = extra_env_variables or {} extra_env_variables = extra_env_variables or {}
@ -516,7 +516,7 @@ class GstValidateLaunchTest(GstValidateTest):
hard_timeout=hard_timeout, hard_timeout=hard_timeout,
media_descriptor=media_descriptor, media_descriptor=media_descriptor,
extra_env_variables=extra_env_variables, extra_env_variables=extra_env_variables,
expected_failures=expected_failures) expected_issues=expected_issues)
self.pipeline_desc = pipeline_desc self.pipeline_desc = pipeline_desc
self.media_descriptor = media_descriptor self.media_descriptor = media_descriptor
@ -534,7 +534,7 @@ class GstValidateMediaCheckTest(GstValidateTest):
def __init__(self, classname, options, reporter, media_descriptor, def __init__(self, classname, options, reporter, media_descriptor,
uri, minfo_path, timeout=DEFAULT_TIMEOUT, uri, minfo_path, timeout=DEFAULT_TIMEOUT,
extra_env_variables=None, extra_env_variables=None,
expected_failures=None): expected_issues=None):
extra_env_variables = extra_env_variables or {} extra_env_variables = extra_env_variables or {}
super( super(
@ -543,7 +543,7 @@ class GstValidateMediaCheckTest(GstValidateTest):
timeout=timeout, timeout=timeout,
media_descriptor=media_descriptor, media_descriptor=media_descriptor,
extra_env_variables=extra_env_variables, extra_env_variables=extra_env_variables,
expected_failures=expected_failures) expected_issues=expected_issues)
self._uri = uri self._uri = uri
self._media_info_path = minfo_path self._media_info_path = minfo_path
@ -564,7 +564,7 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa
timeout=DEFAULT_TIMEOUT, timeout=DEFAULT_TIMEOUT,
scenario=None, scenario=None,
extra_env_variables=None, extra_env_variables=None,
expected_failures=None): expected_issues=None):
Loggable.__init__(self) Loggable.__init__(self)
extra_env_variables = extra_env_variables or {} extra_env_variables = extra_env_variables or {}
@ -587,7 +587,7 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa
scenario=scenario, scenario=scenario,
media_descriptor=media_descriptor, media_descriptor=media_descriptor,
extra_env_variables=None, extra_env_variables=None,
expected_failures=expected_failures) expected_issues=expected_issues)
extra_env_variables = extra_env_variables or {} extra_env_variables = extra_env_variables or {}
GstValidateEncodingTestInterface.__init__( GstValidateEncodingTestInterface.__init__(

181
validate/launcher/baseclasses.py
View file

@ -82,7 +82,7 @@ class Test(Loggable):
def __init__(self, application_name, classname, options, def __init__(self, application_name, classname, options,
reporter, duration=0, timeout=DEFAULT_TIMEOUT, reporter, duration=0, timeout=DEFAULT_TIMEOUT,
hard_timeout=None, extra_env_variables=None, hard_timeout=None, extra_env_variables=None,
expected_failures=None, is_parallel=True, expected_issues=None, is_parallel=True,
workdir=None): workdir=None):
""" """
@timeout: The timeout during which the value return by get_current_value @timeout: The timeout during which the value return by get_current_value
@ -109,12 +109,12 @@ class Test(Loggable):
self.duration = duration self.duration = duration
self.stack_trace = None self.stack_trace = None
self._uuid = None self._uuid = None
if expected_failures is None: if expected_issues is None:
self.expected_failures = [] self.expected_issues = []
elif not isinstance(expected_failures, list): elif not isinstance(expected_issues, list):
self.expected_failures = [expected_failures] self.expected_issues = [expected_issues]
else: else:
self.expected_failures = expected_failures self.expected_issues = expected_issues
extra_env_variables = extra_env_variables or {} extra_env_variables = extra_env_variables or {}
self.extra_env_variables = extra_env_variables self.extra_env_variables = extra_env_variables
@ -127,23 +127,30 @@ class Test(Loggable):
self.clean() self.clean()
def _generate_known_issues(self): def _generate_expected_issues(self):
return '' return ''
def generate_known_issues(self): def generate_expected_issues(self):
res = '%s"%s": [' % (" " * 4, self.classname) res = '%s"FIXME \'%s\' issues [REPORT A BUG ' % (" " * 4, self.classname) \
+ 'in https://gitlab.freedesktop.org/gstreamer/ '\
+ 'or use a proper bug description]": {'
res += """
"tests": [
"%s"
],
"issues": [""" % (self.classname)
retcode = self.process.returncode if self.process else 0 retcode = self.process.returncode if self.process else 0
if retcode != 0: if retcode != 0:
signame = EXITING_SIGNALS.get(retcode) signame = EXITING_SIGNALS.get(retcode)
val = "'" + signame + "'" if signame else retcode val = "'" + signame + "'" if signame else retcode
res += """\n { res += """\n {
'bug': 'FIXME - REPORT A BUG in https://gitlab.freedesktop.org/gstreamer/ ? (or remove this line)', '%s': %s,
'%s': %s, 'sometimes': True,
'sometimes': True, },""" % ("signame" if signame else "returncode", val)
},""" % ("signame" if signame else "returncode", val)
res += self._generate_known_issues() res += self._generate_expected_issues()
res += "\n%s],\n" % (" " * 4) res += "\n%s],\n%s},\n" % (" " * 8, " " * 4)
return res return res
@ -694,7 +701,7 @@ class GstValidateTest(Test):
options, reporter, duration=0, options, reporter, duration=0,
timeout=DEFAULT_TIMEOUT, scenario=None, hard_timeout=None, timeout=DEFAULT_TIMEOUT, scenario=None, hard_timeout=None,
media_descriptor=None, extra_env_variables=None, media_descriptor=None, extra_env_variables=None,
expected_failures=None, workdir=None): expected_issues=None, workdir=None):
extra_env_variables = extra_env_variables or {} extra_env_variables = extra_env_variables or {}
@ -738,7 +745,7 @@ class GstValidateTest(Test):
timeout=timeout, timeout=timeout,
hard_timeout=hard_timeout, hard_timeout=hard_timeout,
extra_env_variables=extra_env_variables, extra_env_variables=extra_env_variables,
expected_failures=expected_failures, expected_issues=expected_issues,
workdir=workdir) workdir=workdir)
# defines how much the process can be outside of the configured # defines how much the process can be outside of the configured
@ -856,32 +863,32 @@ class GstValidateTest(Test):
return value return value
def report_matches_expected_failure(self, report, expected_failure): def report_matches_expected_issues(self, report, expected_issues):
for key in ['bug', 'bugs', 'sometimes']: for key in ['bug', 'bugs', 'sometimes']:
if key in expected_failure: if key in expected_issues:
del expected_failure[key] del expected_issues[key]
for key, value in list(report.items()): for key, value in list(report.items()):
if key in expected_failure: if key in expected_issues:
if not re.findall(expected_failure[key], str(value)): if not re.findall(expected_issues[key], str(value)):
return False return False
expected_failure.pop(key) expected_issues.pop(key)
return not bool(expected_failure) return not bool(expected_issues)
def check_reported_issues(self): def check_reported_issues(self):
ret = [] ret = []
expected_failures = copy.deepcopy(self.expected_failures) expected_issues = copy.deepcopy(self.expected_issues)
expected_retcode = [0] expected_retcode = [0]
for report in self.reports: for report in self.reports:
found = None found = None
for expected_failure in expected_failures: for expected_issue in expected_issues:
if self.report_matches_expected_failure(report, if self.report_matches_expected_issues(report,
expected_failure.copy()): expected_issue.copy()):
found = expected_failure found = expected_issue
break break
if found is not None: if found is not None:
expected_failures.remove(found) expected_issues.remove(found)
if report['level'] == 'critical': if report['level'] == 'critical':
if found.get('sometimes', True) and isinstance(expected_retcode, list): if found.get('sometimes', True) and isinstance(expected_retcode, list):
expected_retcode.append(18) expected_retcode.append(18)
@ -891,13 +898,13 @@ class GstValidateTest(Test):
ret.append(report) ret.append(report)
if not ret: if not ret:
return None, expected_failures, expected_retcode return None, expected_issues, expected_retcode
return ret, expected_failures, expected_retcode return ret, expected_issues, expected_retcode
def check_expected_traceback(self, expected_failure): def check_expected_traceback(self, expected_issues):
msg = None msg = None
expected_symbols = expected_failure.get('stacktrace_symbols') expected_symbols = expected_issues.get('stacktrace_symbols')
if expected_symbols: if expected_symbols:
trace_gatherer = BackTraceGenerator.get_default() trace_gatherer = BackTraceGenerator.get_default()
stack_trace = trace_gatherer.get_trace(self) stack_trace = trace_gatherer.get_trace(self)
@ -947,10 +954,10 @@ class GstValidateTest(Test):
self.debug("%s returncode: %s", self, self.process.returncode) self.debug("%s returncode: %s", self, self.process.returncode)
self.criticals, not_found_expected_failures, expected_returncode = self.check_reported_issues() self.criticals, not_found_expected_issues, expected_returncode = self.check_reported_issues()
expected_timeout = None expected_timeout = None
expected_signal = None expected_signal = None
for i, f in enumerate(not_found_expected_failures): for i, f in enumerate(not_found_expected_issues):
returncode = f.get('returncode', []) returncode = f.get('returncode', [])
if not isinstance(returncode, list): if not isinstance(returncode, list):
returncode = [returncode] returncode = [returncode]
@ -970,8 +977,8 @@ class GstValidateTest(Test):
elif f.get("timeout"): elif f.get("timeout"):
expected_timeout = f expected_timeout = f
not_found_expected_failures = [f for f in not_found_expected_failures not_found_expected_issues = [f for f in not_found_expected_issues
if not f.get('returncode') and not f.get('signame')] if not f.get('returncode') and not f.get('signame')]
msg = "" msg = ""
result = Result.PASSED result = Result.PASSED
@ -982,17 +989,19 @@ class GstValidateTest(Test):
result = Result.FAILED result = Result.FAILED
msg = signal_fault_info[0] msg = signal_fault_info[0]
elif expected_timeout: elif expected_timeout:
not_found_expected_failures.remove(expected_timeout) not_found_expected_issues.remove(expected_timeout)
result, msg = self.check_expected_timeout(expected_timeout) result, msg = self.check_expected_timeout(expected_timeout)
else: else:
return return
elif self.process.returncode in EXITING_SIGNALS: elif self.process.returncode in EXITING_SIGNALS:
msg = "Application exited with signal %s" % (EXITING_SIGNALS[self.process.returncode]) msg = "Application exited with signal %s" % (
EXITING_SIGNALS[self.process.returncode])
if self.process.returncode not in expected_returncode: if self.process.returncode not in expected_returncode:
result = Result.FAILED result = Result.FAILED
else: else:
if expected_signal: if expected_signal:
stack_msg, stack_res = self.check_expected_traceback(expected_signal) stack_msg, stack_res = self.check_expected_traceback(
expected_signal)
if not stack_res: if not stack_res:
msg += stack_msg msg += stack_msg
result = Result.FAILED result = Result.FAILED
@ -1011,45 +1020,42 @@ class GstValidateTest(Test):
for c in self.criticals]) for c in self.criticals])
result = Result.FAILED result = Result.FAILED
if not_found_expected_failures: if not_found_expected_issues:
mandatory_failures = [f for f in not_found_expected_failures mandatory_failures = [f for f in not_found_expected_issues
if not f.get('sometimes', True)] if not f.get('sometimes', True)]
if mandatory_failures: if mandatory_failures:
msg += " (Expected errors not found: %s) " % mandatory_failures msg += " (Expected errors not found: %s) " % mandatory_failures
result = Result.FAILED result = Result.FAILED
elif self.expected_failures: elif self.expected_issues:
msg += ' %s(Expected errors occured: %s)%s' % (Colors.OKBLUE, msg += ' %s(Expected errors occured: %s)%s' % (Colors.OKBLUE,
self.expected_failures, self.expected_issues,
Colors.ENDC) Colors.ENDC)
self.set_result(result, msg.strip()) self.set_result(result, msg.strip())
def _generate_known_issues(self): def _generate_expected_issues(self):
res = "" res = ""
self.criticals = self.criticals or [] self.criticals = self.criticals or []
if self.result == Result.TIMEOUT: if self.result == Result.TIMEOUT:
res += """ { res += """ {
'bug': 'FIXME - REPORT A BUG in https://gitlab.freedesktop.org/gstreamer/ ? (or remove this line)', 'timeout': True,
'timeout': True, 'sometimes': True,
'sometimes': True, },"""
},"""
for report in self.criticals: for report in self.criticals:
res += "\n%s{" % (" " * 8) res += "\n%s{" % (" " * 12)
res += '\n%s"bug": "FIXME - REPORT A BUG in https://gitlab.freedesktop.org/gstreamer/ ? (or remove this line)",' % (
" " * 12,)
for key, value in report.items(): for key, value in report.items():
if key == "type": if key == "type":
continue continue
if value is None: if value is None:
continue continue
res += '\n%s%s"%s": "%s",' % ( res += '\n%s%s"%s": "%s",' % (
" " * 12, "# " if key == "details" else "", " " * 16, "# " if key == "details" else "",
key, value.replace('\n', '\\n')) key, value.replace('\n', '\\n'))
res += "\n%s}," % (" " * 8) res += "\n%s}," % (" " * 12)
return res return res
@ -1252,7 +1258,7 @@ class TestsManager(Loggable):
self._generators = [] self._generators = []
self.check_testslist = True self.check_testslist = True
self.all_tests = None self.all_tests = None
self.expected_failures = {} self.expected_issues = {}
self.blacklisted_tests = [] self.blacklisted_tests = []
def init(self): def init(self):
@ -1265,23 +1271,31 @@ class TestsManager(Loggable):
regex = re.compile(classname) regex = re.compile(classname)
return [test for test in self.list_tests() if regex.findall(test.classname)] return [test for test in self.list_tests() if regex.findall(test.classname)]
def add_expected_issues(self, expected_failures): def add_expected_issues(self, expected_issues):
expected_failures_re = {} for bugid, failure_def in list(expected_issues.items()):
for test_name_regex, failures in list(expected_failures.items()): tests_regexes = []
regex = re.compile(test_name_regex) for test_name_regex in failure_def['tests']:
expected_failures_re[regex] = failures regex = re.compile(test_name_regex)
for test in self.tests: tests_regexes.append(regex)
if regex.findall(test.classname): for test in self.tests:
test.expected_failures.extend(failures) if regex.findall(test.classname):
test.expected_issues.extend(failure_def['issues'])
self.debug("%s added expected issues from %s" % (
test.classname, bugid))
failure_def['tests'] = tests_regexes
self.expected_failures.update(expected_failures_re) self.expected_issues.update(expected_issues)
def add_test(self, test): def add_test(self, test):
if test.generator is None: if test.generator is None:
test.classname = self.loading_testsuite + '.' + test.classname test.classname = self.loading_testsuite + '.' + test.classname
for regex, failures in list(self.expected_failures.items()):
if regex.findall(test.classname): for bugid, failure_def in list(self.expected_issues.items()):
test.expected_failures.extend(failures) for regex in failure_def['tests']:
if regex.findall(test.classname):
test.expected_issues.extend(failure_def['issues'])
self.debug("%s added expected issues from %s" % (
test.classname, bugid))
if self._is_test_wanted(test): if self._is_test_wanted(test):
if test not in self.tests: if test not in self.tests:
@ -1365,24 +1379,15 @@ class TestsManager(Loggable):
return True return True
def check_expected_failures(self): def check_expected_issues(self):
if not self.expected_failures or not self.options.check_bugs_status: if not self.expected_issues or not self.options.check_bugs_status:
return True return True
bugs_definitions = defaultdict(list) bugs_definitions = defaultdict(list)
for regex, failures in list(self.expected_failures.items()): for bug, failure_def in list(self.expected_issues.items()):
for failure in failures: tests_names = '|'.join(
bugs = failure.get('bug') [regex.pattern for regex in failure_def['tests']])
if not bugs: bugs_definitions[tests_names].extend([bug])
printc('+ %s:\n --> no bug reported associated with %s\n' % (
regex.pattern, failure), Colors.WARNING)
continue
if not isinstance(bugs, list):
bugs = [bugs]
cbugs = bugs_definitions.get(regex.pattern, [])
bugs.extend([b for b in bugs if b not in cbugs])
bugs_definitions[regex.pattern].extend(bugs)
return check_bugs_resolution(bugs_definitions.items()) return check_bugs_resolution(bugs_definitions.items())
@ -1680,7 +1685,7 @@ class _TestsLauncher(Loggable):
if not tester.set_blacklists(): if not tester.set_blacklists():
return False return False
if not tester.check_expected_failures(): if not tester.check_expected_issues():
return False return False
if self.options.check_bugs_status: if self.options.check_bugs_status:
@ -1932,7 +1937,7 @@ class _TestsLauncher(Loggable):
all_known_issues = "" all_known_issues = ""
for test in self.tests: for test in self.tests:
if test.result not in [Result.PASSED, Result.NOT_RUN]: if test.result not in [Result.PASSED, Result.NOT_RUN]:
known_issues = test.generate_known_issues() known_issues = test.generate_expected_issues()
if known_issues: if known_issues:
all_known_issues += known_issues all_known_issues += known_issues
if all_known_issues: if all_known_issues:

4
validate/tests/launcher_tests/test_validate.py
View file

@ -29,13 +29,13 @@ def get_pipelines(test_manager):
"audiotestsrc ! audio/x-raw,channels=2,channel-mask='(bitmask)0x67' " "audiotestsrc ! audio/x-raw,channels=2,channel-mask='(bitmask)0x67' "
"! audioconvert ! capsfilter caps=audio/x-raw,channels=6,channel-mask='(bitmask)0x32' " "! audioconvert ! capsfilter caps=audio/x-raw,channels=6,channel-mask='(bitmask)0x32' "
" name=capsfilter ! fakesink", " name=capsfilter ! fakesink",
{"expected-failures": [ {"expected-issues": [
{'returncode': 18}, {'returncode': 18},
{'level': 'critical', 'summary': 'a NOT NEGOTIATED message has been posted on the bus.', {'level': 'critical', 'summary': 'a NOT NEGOTIATED message has been posted on the bus.',
'details': r'.*Caps negotiation failed at pad.*capsfilter:sink.*as it refused caps:.*'}]}), 'details': r'.*Caps negotiation failed at pad.*capsfilter:sink.*as it refused caps:.*'}]}),
("not_negotiated.caps_query_failure", ("not_negotiated.caps_query_failure",
"\( \( audiotestsrc \) ! input-selector name=i \) ! capsfilter name=capsfilter caps=video/x-raw ! fakesink", "\( \( audiotestsrc \) ! input-selector name=i \) ! capsfilter name=capsfilter caps=video/x-raw ! fakesink",
{"expected-failures": [ {"expected-issues": [
{'returncode': 18}, {'returncode': 18},
{'level': 'critical', 'summary': 'a NOT NEGOTIATED message has been posted on the bus.', {'level': 'critical', 'summary': 'a NOT NEGOTIATED message has been posted on the bus.',
'details': 'Caps negotiation failed starting from pad \'capsfilter:sink\' as the ' 'details': 'Caps negotiation failed starting from pad \'capsfilter:sink\' as the '