From 03548cd63d3fe13628bd299963b31a2a5adcf6c3 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 2 Sep 2016 17:37:24 -0300 Subject: [PATCH] validate:launcher: Allow specifying expected tests errors In the future instead of blacklisting tests we should define what error is expected, and this way when the bug is closed, we will notice, also, it will allow us to check GstValidate error reporting itself. --- validate/launcher/apps/gstvalidate.py | 21 ++-- validate/launcher/baseclasses.py | 142 +++++++++++++++++++++----- 2 files changed, 131 insertions(+), 32 deletions(-) diff --git a/validate/launcher/apps/gstvalidate.py b/validate/launcher/apps/gstvalidate.py index 5e6c2b9ede..02f394cee6 100644 --- a/validate/launcher/apps/gstvalidate.py +++ b/validate/launcher/apps/gstvalidate.py @@ -225,12 +225,14 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator): fname = self.get_fname(scenario, protocol=mediainfo.get_protocol(), name=name) + expected_failures = extra_datas.get("expected-failures") self.add_test(GstValidateLaunchTest(fname, self.test_manager.options, self.test_manager.reporter, pipeline_desc, scenario=scenario, - media_descriptor=mediainfo) + media_descriptor=mediainfo, + expected_failures=expected_failures) ) @@ -373,7 +375,7 @@ class GstValidateLaunchTest(GstValidateTest): def __init__(self, classname, options, reporter, pipeline_desc, timeout=DEFAULT_TIMEOUT, scenario=None, media_descriptor=None, duration=0, hard_timeout=None, - extra_env_variables=None): + extra_env_variables=None, expected_failures=None): extra_env_variables = extra_env_variables or {} @@ -398,7 +400,8 @@ class GstValidateLaunchTest(GstValidateTest): timeout=timeout, hard_timeout=hard_timeout, media_descriptor=media_descriptor, - extra_env_variables=extra_env_variables) + extra_env_variables=extra_env_variables, + expected_failures=expected_failures) self.pipeline_desc = pipeline_desc self.media_descriptor = media_descriptor @@ -415,7 +418,8 @@ class GstValidateMediaCheckTest(GstValidateTest): def __init__(self, classname, options, reporter, media_descriptor, uri, minfo_path, timeout=DEFAULT_TIMEOUT, - extra_env_variables=None): + extra_env_variables=None, + expected_failures=None): extra_env_variables = extra_env_variables or {} super( @@ -423,7 +427,8 @@ class GstValidateMediaCheckTest(GstValidateTest): options, reporter, timeout=timeout, media_descriptor=media_descriptor, - extra_env_variables=extra_env_variables) + extra_env_variables=extra_env_variables, + expected_failures=expected_failures) self._uri = uri self._media_info_path = minfo_path @@ -440,7 +445,8 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa combination, uri, media_descriptor, timeout=DEFAULT_TIMEOUT, scenario=None, - extra_env_variables=None): + extra_env_variables=None, + expected_failures=None): Loggable.__init__(self) extra_env_variables = extra_env_variables or {} @@ -468,7 +474,8 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa timeout=timeout, scenario=scenario, media_descriptor=media_descriptor, - extra_env_variables=None) + extra_env_variables=None, + expected_failures=expected_failures) extra_env_variables = extra_env_variables or {} GstValidateEncodingTestInterface.__init__( diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index ebdb6eaeb8..6aeaa62aff 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -23,6 +23,7 @@ import json import os import sys import re +import copy import SocketServer import struct import time @@ -56,7 +57,8 @@ class Test(Loggable): def __init__(self, application_name, classname, options, reporter, duration=0, timeout=DEFAULT_TIMEOUT, - hard_timeout=None, extra_env_variables=None): + hard_timeout=None, extra_env_variables=None, + expected_failures=None): """ @timeout: The timeout during which the value return by get_current_value keeps being exactly equal @@ -75,6 +77,12 @@ class Test(Loggable): self.thread = None self.queue = None self.duration = duration + if expected_failures is None: + self.expected_failures = [] + elif not isinstance(expected_failures, list): + self.expected_failures = [expected_failures] + else: + self.expected_failures = expected_failures extra_env_variables = extra_env_variables or {} self.extra_env_variables = extra_env_variables @@ -82,6 +90,7 @@ class Test(Loggable): self.clean() def clean(self): + self.kill_subprocess() self.message = "" self.error_str = "" self.time_taken = 0.0 @@ -484,7 +493,8 @@ class GstValidateTest(Test): def __init__(self, application_name, classname, options, reporter, duration=0, timeout=DEFAULT_TIMEOUT, scenario=None, hard_timeout=None, - media_descriptor=None, extra_env_variables=None): + media_descriptor=None, extra_env_variables=None, + expected_failures=None): extra_env_variables = extra_env_variables or {} @@ -506,7 +516,7 @@ class GstValidateTest(Test): self.reports = [] self.position = -1 - self.duration = -1 + self.media_duration = -1 self.speed = 1.0 self.actions_infos = [] self.media_descriptor = media_descriptor @@ -524,7 +534,8 @@ class GstValidateTest(Test): duration=duration, timeout=timeout, hard_timeout=hard_timeout, - extra_env_variables=extra_env_variables) + extra_env_variables=extra_env_variables, + expected_failures=expected_failures) # defines how much the process can be outside of the configured # segment / seek @@ -553,7 +564,7 @@ class GstValidateTest(Test): def set_position(self, position, duration, speed=None): self.position = position - self.duration = duration + self.media_duration = duration if speed: self.speed = speed @@ -584,9 +595,11 @@ class GstValidateTest(Test): Test.test_start(self, queue) def test_end(self): - Test.test_end(self) + res = Test.test_end(self) self.stop_server() + return res + def get_override_file(self, media_descriptor): if media_descriptor: if media_descriptor.get_path(): @@ -665,6 +678,11 @@ class GstValidateTest(Test): def clean(self): Test.clean(self) self._sent_eos_pos = None + self.reports = [] + self.position = -1 + self.media_duration = -1 + self.speed = 1.0 + self.actions_infos = [] def build_arguments(self): super(GstValidateTest, self).build_arguments() @@ -686,16 +704,44 @@ class GstValidateTest(Test): return value - def get_validate_criticals_errors(self): + def report_matches_expected_failure(self, report, expected_failure): + for key in ['bug', 'sometimes']: + if key in expected_failure: + del expected_failure[key] + for key, value in report.items(): + if key in expected_failure: + if not re.findall(expected_failure[key], value): + return False + expected_failure.pop(key) + + return not bool(expected_failure) + + def check_reported_issues(self): ret = [] + expected_failures = copy.deepcopy(self.expected_failures) + expected_retcode = [0] for report in self.reports: - if report['level'] == 'critical': + found = None + for expected_failure in expected_failures: + if self.report_matches_expected_failure(report, + expected_failure.copy()): + found = expected_failure + break + + if found is not None: + expected_failures.remove(found) + if report['level'] == 'critical': + if found.get('sometimes') and isinstance(expected_retcode, list): + expected_retcode.append(18) + else: + expected_retcode = [18] + elif report['level'] == 'critical': ret.append(report['summary']) if not ret: - return None + return None, expected_failures, expected_retcode - return str(ret) + return ret, expected_failures, expected_retcode def check_results(self): if self.result is Result.FAILED or self.result is Result.PASSED or self.result is Result.TIMEOUT: @@ -703,23 +749,53 @@ class GstValidateTest(Test): self.debug("%s returncode: %s", self, self.process.returncode) - criticals = self.get_validate_criticals_errors() + criticals, not_found_expected_failures, expected_returncode = self.check_reported_issues() + + returncode_index = None + for i, f in enumerate(not_found_expected_failures): + if len(f) == 1 and f.get("returncode"): + returncode = f['returncode'] + if not isinstance(expected_returncode, list): + returncode = [expected_returncode] + if 'sometimes' in f: + returncode.append(0) + returncode_index = i + break + + not_found_expected_failures = [f for f in not_found_expected_failures + if not f.get('returncode')] + + msg = "" + result = Result.PASSED if self.process.returncode == 139: - # FIXME Reimplement something like that if needed - # self.get_backtrace("SEGFAULT") - self.set_result(Result.FAILED, - "Application segfaulted", - "segfault") + result = Result.FAILED + msg = "Application segfaulted " elif self.process.returncode == VALGRIND_ERROR_CODE: - self.set_result(Result.FAILED, "Valgrind reported errors") - elif criticals or self.process.returncode != 0: - if criticals is None: - criticals = "No criticals" - self.set_result(Result.FAILED, - "Application returned %s (issues: %s)" - % (self.process.returncode, criticals)) - else: - self.set_result(Result.PASSED) + msg = "Valgrind reported errors " + result = Result.FAILED + elif self.process.returncode not in expected_returncode: + msg = "Application returned %s " % self.process.returncode + if expected_returncode != 0: + msg += "(expected %s) " % expected_returncode + result = Result.FAILED + + if criticals: + msg += "(critical errors: [%s]) " % ', '.join(criticals) + result = Result.FAILED + + if not_found_expected_failures: + mandatory_failures = [f for f in not_found_expected_failures + if not f.get('sometimes')] + + if mandatory_failures: + msg += "(Expected errors not found: %s) " % mandatory_failures + result = Result.FAILED + elif self.expected_failures: + msg += '%s(Expected errors occured: %s)%s' % (Colors.OKBLUE, + self.expected_failures, + Colors.ENDC) + + self.set_result(result, msg.strip()) def get_valgrind_suppression_file(self, subdir, name): p = get_data_file(subdir, name) @@ -892,6 +968,7 @@ class TestsManager(Loggable): self.starting_test_num = 0 self.check_testslist = True self.all_tests = None + self.expected_failures = {} def init(self): return False @@ -899,7 +976,22 @@ class TestsManager(Loggable): def list_tests(self): return sorted(list(self.tests)) + def add_expected_issues(self, expected_failures): + expected_failures_re = {} + for test_name_regex, failures in expected_failures.items(): + regex = re.compile(test_name_regex) + expected_failures_re[regex] = failures + for test in self.tests: + if regex.findall(test.classname): + test.expected_failures.extend(failures) + + self.expected_failures.update(expected_failures_re) + def add_test(self, test): + for regex, failures in self.expected_failures.items(): + if regex.findall(test.classname): + test.expected_failures.extend(failures) + if self._is_test_wanted(test): if test not in self.tests: self.tests.append(test)