mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-12-20 07:16:55 +00:00
Code Issues 1 Projects Releases Wiki Activity

validate:launcher: Allow specifying expected tests errors

Browse source 
In the future instead of blacklisting tests we should define
what error is expected, and this way when the bug is closed,
we will notice, also, it will allow us to check GstValidate
error reporting itself.
This commit is contained in:
Thibault Saunier 2016-09-02 17:37:24 -03:00
parent 2fff14e469
commit 03548cd63d
2 changed files with 131 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
validate/launcher/apps/gstvalidate.py
View file

@ -225,12 +225,14 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator):
fname = self.get_fname(scenario, protocol=mediainfo.get_protocol(), name=name)
expected_failures = extra_datas.get("expected-failures")
self.add_test(GstValidateLaunchTest(fname,
self.test_manager.options,
self.test_manager.reporter,
pipeline_desc,
scenario=scenario,
media_descriptor=mediainfo)
media_descriptor=mediainfo,
expected_failures=expected_failures)
)
@ -373,7 +375,7 @@ class GstValidateLaunchTest(GstValidateTest):
def __init__(self, classname, options, reporter, pipeline_desc,
timeout=DEFAULT_TIMEOUT, scenario=None,
media_descriptor=None, duration=0, hard_timeout=None,
extra_env_variables=None):
extra_env_variables=None, expected_failures=None):
extra_env_variables = extra_env_variables or {}
@ -398,7 +400,8 @@ class GstValidateLaunchTest(GstValidateTest):
timeout=timeout,
hard_timeout=hard_timeout,
media_descriptor=media_descriptor,
extra_env_variables=extra_env_variables)
extra_env_variables=extra_env_variables,
expected_failures=expected_failures)
self.pipeline_desc = pipeline_desc
self.media_descriptor = media_descriptor
@ -415,7 +418,8 @@ class GstValidateMediaCheckTest(GstValidateTest):
def __init__(self, classname, options, reporter, media_descriptor,
uri, minfo_path, timeout=DEFAULT_TIMEOUT,
extra_env_variables=None):
extra_env_variables=None,
expected_failures=None):
extra_env_variables = extra_env_variables or {}
super(
@ -423,7 +427,8 @@ class GstValidateMediaCheckTest(GstValidateTest):
options, reporter,
timeout=timeout,
media_descriptor=media_descriptor,
extra_env_variables=extra_env_variables)
extra_env_variables=extra_env_variables,
expected_failures=expected_failures)
self._uri = uri
self._media_info_path = minfo_path
@ -440,7 +445,8 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa
combination, uri, media_descriptor,
timeout=DEFAULT_TIMEOUT,
scenario=None,
extra_env_variables=None):
extra_env_variables=None,
expected_failures=None):
Loggable.__init__(self)
extra_env_variables = extra_env_variables or {}
@ -468,7 +474,8 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa
timeout=timeout,
scenario=scenario,
media_descriptor=media_descriptor,
extra_env_variables=None)
extra_env_variables=None,
expected_failures=expected_failures)
extra_env_variables = extra_env_variables or {}
GstValidateEncodingTestInterface.__init__(

140
validate/launcher/baseclasses.py
View file

@ -23,6 +23,7 @@ import json
import os
import sys
import re
import copy
import SocketServer
import struct
import time
@ -56,7 +57,8 @@ class Test(Loggable):
def __init__(self, application_name, classname, options,
reporter, duration=0, timeout=DEFAULT_TIMEOUT,
hard_timeout=None, extra_env_variables=None):
hard_timeout=None, extra_env_variables=None,
expected_failures=None):
"""
@timeout: The timeout during which the value return by get_current_value
keeps being exactly equal
@ -75,6 +77,12 @@ class Test(Loggable):
self.thread = None
self.queue = None
self.duration = duration
if expected_failures is None:
self.expected_failures = []
elif not isinstance(expected_failures, list):
self.expected_failures = [expected_failures]
else:
self.expected_failures = expected_failures
extra_env_variables = extra_env_variables or {}
self.extra_env_variables = extra_env_variables
@ -82,6 +90,7 @@ class Test(Loggable):
self.clean()
def clean(self):
self.kill_subprocess()
self.message = ""
self.error_str = ""
self.time_taken = 0.0
@ -484,7 +493,8 @@ class GstValidateTest(Test):
def __init__(self, application_name, classname,
options, reporter, duration=0,
timeout=DEFAULT_TIMEOUT, scenario=None, hard_timeout=None,
media_descriptor=None, extra_env_variables=None):
media_descriptor=None, extra_env_variables=None,
expected_failures=None):
extra_env_variables = extra_env_variables or {}
@ -506,7 +516,7 @@ class GstValidateTest(Test):
self.reports = []
self.position = -1
self.duration = -1
self.media_duration = -1
self.speed = 1.0
self.actions_infos = []
self.media_descriptor = media_descriptor
@ -524,7 +534,8 @@ class GstValidateTest(Test):
duration=duration,
timeout=timeout,
hard_timeout=hard_timeout,
extra_env_variables=extra_env_variables)
extra_env_variables=extra_env_variables,
expected_failures=expected_failures)
# defines how much the process can be outside of the configured
# segment / seek
@ -553,7 +564,7 @@ class GstValidateTest(Test):
def set_position(self, position, duration, speed=None):
self.position = position
self.duration = duration
self.media_duration = duration
if speed:
self.speed = speed
@ -584,9 +595,11 @@ class GstValidateTest(Test):
Test.test_start(self, queue)
def test_end(self):
Test.test_end(self)
res = Test.test_end(self)
self.stop_server()
return res
def get_override_file(self, media_descriptor):
if media_descriptor:
if media_descriptor.get_path():
@ -665,6 +678,11 @@ class GstValidateTest(Test):
def clean(self):
Test.clean(self)
self._sent_eos_pos = None
self.reports = []
self.position = -1
self.media_duration = -1
self.speed = 1.0
self.actions_infos = []
def build_arguments(self):
super(GstValidateTest, self).build_arguments()
@ -686,16 +704,44 @@ class GstValidateTest(Test):
return value
def get_validate_criticals_errors(self):
def report_matches_expected_failure(self, report, expected_failure):
for key in ['bug', 'sometimes']:
if key in expected_failure:
del expected_failure[key]
for key, value in report.items():
if key in expected_failure:
if not re.findall(expected_failure[key], value):
return False
expected_failure.pop(key)
return not bool(expected_failure)
def check_reported_issues(self):
ret = []
expected_failures = copy.deepcopy(self.expected_failures)
expected_retcode = [0]
for report in self.reports:
found = None
for expected_failure in expected_failures:
if self.report_matches_expected_failure(report,
expected_failure.copy()):
found = expected_failure
break
if found is not None:
expected_failures.remove(found)
if report['level'] == 'critical':
if found.get('sometimes') and isinstance(expected_retcode, list):
expected_retcode.append(18)
else:
expected_retcode = [18]
elif report['level'] == 'critical':
ret.append(report['summary'])
if not ret:
return None
return None, expected_failures, expected_retcode
return str(ret)
return ret, expected_failures, expected_retcode
def check_results(self):
if self.result is Result.FAILED or self.result is Result.PASSED or self.result is Result.TIMEOUT:
@ -703,23 +749,53 @@ class GstValidateTest(Test):
self.debug("%s returncode: %s", self, self.process.returncode)
criticals = self.get_validate_criticals_errors()
criticals, not_found_expected_failures, expected_returncode = self.check_reported_issues()
returncode_index = None
for i, f in enumerate(not_found_expected_failures):
if len(f) == 1 and f.get("returncode"):
returncode = f['returncode']
if not isinstance(expected_returncode, list):
returncode = [expected_returncode]
if 'sometimes' in f:
returncode.append(0)
returncode_index = i
break
not_found_expected_failures = [f for f in not_found_expected_failures
if not f.get('returncode')]
msg = ""
result = Result.PASSED
if self.process.returncode == 139:
# FIXME Reimplement something like that if needed
# self.get_backtrace("SEGFAULT")
self.set_result(Result.FAILED,
"Application segfaulted",
"segfault")
result = Result.FAILED
msg = "Application segfaulted "
elif self.process.returncode == VALGRIND_ERROR_CODE:
self.set_result(Result.FAILED, "Valgrind reported errors")
elif criticals or self.process.returncode != 0:
if criticals is None:
criticals = "No criticals"
self.set_result(Result.FAILED,
"Application returned %s (issues: %s)"
% (self.process.returncode, criticals))
else:
self.set_result(Result.PASSED)
msg = "Valgrind reported errors "
result = Result.FAILED
elif self.process.returncode not in expected_returncode:
msg = "Application returned %s " % self.process.returncode
if expected_returncode != 0:
msg += "(expected %s) " % expected_returncode
result = Result.FAILED
if criticals:
msg += "(critical errors: [%s]) " % ', '.join(criticals)
result = Result.FAILED
if not_found_expected_failures:
mandatory_failures = [f for f in not_found_expected_failures
if not f.get('sometimes')]
if mandatory_failures:
msg += "(Expected errors not found: %s) " % mandatory_failures
result = Result.FAILED
elif self.expected_failures:
msg += '%s(Expected errors occured: %s)%s' % (Colors.OKBLUE,
self.expected_failures,
Colors.ENDC)
self.set_result(result, msg.strip())
def get_valgrind_suppression_file(self, subdir, name):
p = get_data_file(subdir, name)
@ -892,6 +968,7 @@ class TestsManager(Loggable):
self.starting_test_num = 0
self.check_testslist = True
self.all_tests = None
self.expected_failures = {}
def init(self):
return False
@ -899,7 +976,22 @@ class TestsManager(Loggable):
def list_tests(self):
return sorted(list(self.tests))
def add_expected_issues(self, expected_failures):
expected_failures_re = {}
for test_name_regex, failures in expected_failures.items():
regex = re.compile(test_name_regex)
expected_failures_re[regex] = failures
for test in self.tests:
if regex.findall(test.classname):
test.expected_failures.extend(failures)
self.expected_failures.update(expected_failures_re)
def add_test(self, test):
for regex, failures in self.expected_failures.items():
if regex.findall(test.classname):
test.expected_failures.extend(failures)
if self._is_test_wanted(test):
if test not in self.tests:
self.tests.append(test)