From a77825ecbc5ebcaaf9740c0f61b32d28580e24d5 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 17 Mar 2019 14:39:38 -0300 Subject: [PATCH] validate:launcher: Allow referencing known issue by exiting signal names --- validate/launcher/baseclasses.py | 118 +++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 36 deletions(-) diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index 6038199140..c4e2131c61 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -69,6 +69,7 @@ EXITING_SIGNALS = dict([(-getattr(signal, s), s) for s in [ 'SIGQUIT', 'SIGILL', 'SIGABRT', 'SIGFPE', 'SIGSEGV', 'SIGBUS', 'SIGSYS', 'SIGTRAP', 'SIGXCPU', 'SIGXFSZ', 'SIGIOT'] if hasattr(signal, s)]) EXITING_SIGNALS.update({139: "SIGSEGV"}) +EXITING_SIGNALS.update({(v, k) for k, v in EXITING_SIGNALS.items()}) class Test(Loggable): @@ -123,8 +124,25 @@ class Test(Loggable): self.clean() + def _generate_known_issues(self): + return '' + def generate_known_issues(self): - return None + res = '%s"%s": [' % (" " * 4, self.classname) + + retcode = self.process.returncode if self.process else 0 + if retcode != 0: + signame = EXITING_SIGNALS.get(retcode) + val = "'" + signame + "'" if signame else retcode + res += """\n { + 'bug': 'FIXME - REPORT A BUG in https://gitlab.freedesktop.org/gstreamer/ ? (or remove this line)', + '%s': %s, + 'sometimes': True, + },""" % ("signame" if signame else "returncode", val) + res += self._generate_known_issues() + res += "\n%s],\n" % (" " * 4) + + return res def clean(self): self.kill_subprocess() @@ -872,17 +890,9 @@ class GstValidateTest(Test): return ret, expected_failures, expected_retcode - def check_expected_timeout(self, expected_timeout): - msg = "Expected timeout happened. " - result = Result.PASSED - message = expected_timeout.get('message') - if message: - if not re.findall(message, self.message): - result = Result.FAILED - msg = "Expected timeout message: %s got %s " % ( - message, self.message) - - expected_symbols = expected_timeout.get('stacktrace_symbols') + def check_expected_traceback(self, expected_failure): + msg = None + expected_symbols = expected_failure.get('stacktrace_symbols') if expected_symbols: trace_gatherer = BackTraceGenerator.get_default() stack_trace = trace_gatherer.get_trace(self) @@ -894,11 +904,29 @@ class GstValidateTest(Test): not_found_symbols = [s for s in expected_symbols if s not in stack_trace] if not_found_symbols: - result = Result.TIMEOUT - msg = "Expected symbols '%s' not found in stack trace " % ( + msg = " Expected symbols '%s' not found in stack trace " % ( not_found_symbols) + + return msg, False else: - msg += "No stack trace available, could not verify symbols " + msg += " No stack trace available, could not verify symbols " + + return msg, True + + def check_expected_timeout(self, expected_timeout): + msg = "Expected timeout happened. " + result = Result.PASSED + message = expected_timeout.get('message') + if message: + if not re.findall(message, self.message): + result = Result.FAILED + msg = "Expected timeout message: %s got %s " % ( + message, self.message) + + stack_msg, stack_res = self.check_expected_traceback(expected_timeout) + if not stack_res: + result = Result.TIMEOUT + msg += stack_msg return result, msg @@ -915,20 +943,30 @@ class GstValidateTest(Test): self.debug("%s returncode: %s", self, self.process.returncode) self.criticals, not_found_expected_failures, expected_returncode = self.check_reported_issues() - expected_timeout = None + expected_signal = None for i, f in enumerate(not_found_expected_failures): - if len(f) == 1 and f.get("returncode"): - returncode = f['returncode'] - if not isinstance(expected_returncode, list): - returncode = [expected_returncode] + returncode = f.get('returncode', []) + if not isinstance(returncode, list): + returncode = [returncode] + + if f.get('signame'): + signames = f['signame'] + if not isinstance(signames, list): + signames = [signames] + + returncode = [EXITING_SIGNALS[signame] for signame in signames] + + if returncode: if 'sometimes' in f: returncode.append(0) + expected_returncode = returncode + expected_signal = f elif f.get("timeout"): expected_timeout = f not_found_expected_failures = [f for f in not_found_expected_failures - if not f.get('returncode')] + if not f.get('returncode') and not f.get('signame')] msg = "" result = Result.PASSED @@ -944,10 +982,15 @@ class GstValidateTest(Test): else: return elif self.process.returncode in EXITING_SIGNALS: - result = Result.FAILED - msg = "Application exited with signal %s" % ( - EXITING_SIGNALS[self.process.returncode] - ) + msg = "Application exited with signal %s" % (EXITING_SIGNALS[self.process.returncode]) + if self.process.returncode not in expected_returncode: + result = Result.FAILED + else: + if expected_signal: + stack_msg, stack_res = self.check_expected_traceback(expected_signal) + if not stack_res: + msg += stack_msg + result = Result.FAILED self.add_stack_trace_to_logfile() elif self.process.returncode == VALGRIND_ERROR_CODE: msg = "Valgrind reported errors " @@ -968,19 +1011,18 @@ class GstValidateTest(Test): if not f.get('sometimes', True)] if mandatory_failures: - msg += "(Expected errors not found: %s) " % mandatory_failures + msg += " (Expected errors not found: %s) " % mandatory_failures result = Result.FAILED elif self.expected_failures: - msg += '%s(Expected errors occured: %s)%s' % (Colors.OKBLUE, + msg += ' %s(Expected errors occured: %s)%s' % (Colors.OKBLUE, self.expected_failures, Colors.ENDC) self.set_result(result, msg.strip()) - def generate_known_issues(self): - if not self.criticals and self.result != Result.TIMEOUT: - return None - res = '%s"%s": [' % (" " * 4, self.classname) + def _generate_known_issues(self): + res = "" + self.criticals = self.criticals or [] if self.result == Result.TIMEOUT: res += """ { 'bug': 'FIXME - REPORT A BUG in https://gitlab.freedesktop.org/gstreamer/ ? (or remove this line)', @@ -996,13 +1038,14 @@ class GstValidateTest(Test): for key, value in report.items(): if key == "type": continue + if value is None: + continue res += '\n%s%s"%s": "%s",' % ( " " * 12, "# " if key == "details" else "", key, value.replace('\n', '\\n')) res += "\n%s}," % (" " * 8) - res += "\n%s],\n" % (" " * 4) return res def get_valgrind_suppressions(self): @@ -1857,21 +1900,24 @@ class _TestsLauncher(Loggable): if self.options.forever: r = 1 while True: - printc("Running iteration %d" % r, title=True) + printc("-> Iteration %d" % r, end='') if not self._run_tests(): break r += 1 self.clean_tests() + printc("OK", Colors.OKGREEN) return False elif self.options.n_runs: res = True for r in range(self.options.n_runs): - t = "Running iteration %d" % r - print("%s\n%s\n%s\n" % ("=" * len(t), t, "=" * len(t))) + printc("-> Iteration %d" % r) if not self._run_tests(): res = False + printc("ERROR", Colors.FAIL) + else: + printc("OK", Colors.OKGREEN) self.clean_tests() return res @@ -1880,7 +1926,7 @@ class _TestsLauncher(Loggable): finally: all_known_issues = "" for test in self.tests: - if test.result != Result.PASSED: + if test.result not in [Result.PASSED, Result.NOT_RUN]: known_issues = test.generate_known_issues() if known_issues: all_known_issues += known_issues