mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-12-24 09:10:36 +00:00
validate: launcher: Namespace test name with the testsuite name
Also allowing users to pass test names directly
This commit is contained in:
Thibault Saunier
parent
f9ef2bc56a
commit
2cf93f491b
4 changed files with 56 additions and 38 deletions
|
|
@ -161,8 +161,6 @@ class MesonTestsManager(TestsManager):
|
|||
if test.suite:
|
||||
name = '.'.join(test.suite) + '.' + name
|
||||
|
||||
name = self.name + '.' + name
|
||||
|
||||
return name.replace('..', '.').replace(' ', '-')
|
||||
|
||||
def list_tests(self):
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ class GstValidateMediaCheckTestsGenerator(GstValidateTestsGenerator):
|
|||
except KeyError:
|
||||
timeout = DEFAULT_TIMEOUT
|
||||
|
||||
classname = "validate.%s.media_check.%s" % (protocol,
|
||||
os.path.basename(url2path(uri)).replace(".", "_"))
|
||||
classname = "%s.media_check.%s" % (protocol,
|
||||
os.path.basename(url2path(uri)).replace(".", "_"))
|
||||
self.add_test(GstValidateMediaCheckTest(classname,
|
||||
self.test_manager.options,
|
||||
self.test_manager.reporter,
|
||||
|
|
@ -113,10 +113,10 @@ class GstValidateTranscodingTestsGenerator(GstValidateTestsGenerator):
|
|||
continue
|
||||
|
||||
for comb in self.test_manager.get_encoding_formats():
|
||||
classname = "validate.%s.transcode.to_%s.%s" % (mediainfo.media_descriptor.get_protocol(),
|
||||
str(comb).replace(
|
||||
' ', '_'),
|
||||
mediainfo.media_descriptor.get_clean_name())
|
||||
classname = "%s.transcode.to_%s.%s" % (mediainfo.media_descriptor.get_protocol(),
|
||||
str(comb).replace(
|
||||
' ', '_'),
|
||||
mediainfo.media_descriptor.get_clean_name())
|
||||
self.add_test(GstValidateTranscodingTest(classname,
|
||||
self.test_manager.options,
|
||||
self.test_manager.reporter,
|
||||
|
|
@ -199,9 +199,9 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator):
|
|||
protocol_str = ""
|
||||
|
||||
if scenario is not None and scenario.name.lower() != "none":
|
||||
return "%s.%s%s.%s" % ("validate", protocol_str, name, scenario.name)
|
||||
return "%s%s.%s" % (protocol_str, name, scenario.name)
|
||||
|
||||
return ("%s.%s.%s.%s" % ("validate", protocol_str, self.name, name)).replace("..", ".")
|
||||
return ("%s.%s.%s" % (protocol_str, self.name, name)).replace("..", ".")
|
||||
|
||||
def generate_tests(self, uri_minfo_special_scenarios, scenarios):
|
||||
if self._valid_scenarios is None:
|
||||
|
|
@ -1002,60 +1002,60 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
|
|||
def register_default_blacklist(self):
|
||||
self.set_default_blacklist([
|
||||
# hls known issues
|
||||
("validate.hls.playback.seek_with_stop.*",
|
||||
("hls.playback.seek_with_stop.*",
|
||||
"https://bugzilla.gnome.org/show_bug.cgi?id=753689"),
|
||||
|
||||
# dash known issues
|
||||
("validate.dash.media_check.*",
|
||||
("dash.media_check.*",
|
||||
"Caps are different depending on selected bitrates, etc"),
|
||||
|
||||
# Matroska/WEBM known issues:
|
||||
("validate.*.reverse_playback.*webm$",
|
||||
("*.reverse_playback.*webm$",
|
||||
"https://bugzilla.gnome.org/show_bug.cgi?id=679250"),
|
||||
("validate.*.reverse_playback.*mkv$",
|
||||
("*.reverse_playback.*mkv$",
|
||||
"https://bugzilla.gnome.org/show_bug.cgi?id=679250"),
|
||||
("validate.http.playback.seek_with_stop.*webm",
|
||||
("http.playback.seek_with_stop.*webm",
|
||||
"matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"),
|
||||
("validate.http.playback.seek_with_stop.*mkv",
|
||||
("http.playback.seek_with_stop.*mkv",
|
||||
"matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"),
|
||||
|
||||
# MPEG TS known issues:
|
||||
('(?i)validate.*.playback.reverse_playback.*(?:_|.)(?:|m)ts$',
|
||||
('(?i)*playback.reverse_playback.*(?:_|.)(?:|m)ts$',
|
||||
"https://bugzilla.gnome.org/show_bug.cgi?id=702595"),
|
||||
|
||||
# Fragmented MP4 disabled tests:
|
||||
('validate.*.playback..*seek.*.fragmented_nonseekable_sink_mp4',
|
||||
('*.playback..*seek.*.fragmented_nonseekable_sink_mp4',
|
||||
"Seeking on fragmented files without indexes isn't implemented"),
|
||||
('validate.*.playback.reverse_playback.fragmented_nonseekable_sink_mp4',
|
||||
('*.playback.reverse_playback.fragmented_nonseekable_sink_mp4',
|
||||
"Seeking on fragmented files without indexes isn't implemented"),
|
||||
|
||||
# HTTP known issues:
|
||||
("validate.http.*scrub_forward_seeking.*",
|
||||
("http.*scrub_forward_seeking.*",
|
||||
"This is not stable enough for now."),
|
||||
("validate.http.playback.change_state_intensive.raw_video_mov",
|
||||
("http.playback.change_state_intensive.raw_video_mov",
|
||||
"This is not stable enough for now. (flow return from pad push doesn't match expected value)"),
|
||||
|
||||
# MXF known issues"
|
||||
(".*reverse_playback.*mxf",
|
||||
("*reverse_playback.*mxf",
|
||||
"Reverse playback is not handled in MXF"),
|
||||
("validate\.file\.transcode.*mxf",
|
||||
("file\.transcode.*mxf",
|
||||
"FIXME: Transcoding and mixing tests need to be tested"),
|
||||
|
||||
# WMV known issues"
|
||||
(".*reverse_playback.*wmv",
|
||||
("*reverse_playback.*wmv",
|
||||
"Reverse playback is not handled in wmv"),
|
||||
(".*reverse_playback.*asf",
|
||||
"Reverse playback is not handled in asf"),
|
||||
|
||||
# ogg known issues
|
||||
("validate.http.playback.seek.*vorbis_theora_1_ogg",
|
||||
("http.playback.seek.*vorbis_theora_1_ogg",
|
||||
"https://bugzilla.gnome.org/show_bug.cgi?id=769545"),
|
||||
# RTSP known issues
|
||||
('validate.rtsp.playback.reverse.*',
|
||||
('rtsp.playback.reverse.*',
|
||||
'https://bugzilla.gnome.org/show_bug.cgi?id=626811'),
|
||||
('validate.rtsp.playback.seek_with_stop.*',
|
||||
('rtsp.playback.seek_with_stop.*',
|
||||
'https://bugzilla.gnome.org/show_bug.cgi?id=784298'),
|
||||
('validate.rtsp.playback.fast_*',
|
||||
('rtsp.playback.fast_*',
|
||||
'https://bugzilla.gnome.org/show_bug.cgi?id=754575'),
|
||||
])
|
||||
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ class Test(Loggable):
|
|||
self.extra_env_variables = extra_env_variables
|
||||
self.optional = False
|
||||
self.is_parallel = is_parallel
|
||||
self.generator = None
|
||||
|
||||
self.clean()
|
||||
|
||||
|
|
@ -1040,6 +1041,7 @@ class TestsManager(Loggable):
|
|||
""" A class responsible for managing tests. """
|
||||
|
||||
name = "base"
|
||||
loading_testsuite = None
|
||||
|
||||
def __init__(self):
|
||||
|
||||
|
|
@ -1080,6 +1082,8 @@ class TestsManager(Loggable):
|
|||
self.expected_failures.update(expected_failures_re)
|
||||
|
||||
def add_test(self, test):
|
||||
if test.generator is None:
|
||||
test.classname = self.loading_testsuite + '.' + test.classname
|
||||
for regex, failures in list(self.expected_failures.items()):
|
||||
if regex.findall(test.classname):
|
||||
test.expected_failures.extend(failures)
|
||||
|
|
@ -1103,10 +1107,11 @@ class TestsManager(Loggable):
|
|||
"""
|
||||
@generators: A list of, or one single #TestsGenerator to be used to generate tests
|
||||
"""
|
||||
if isinstance(generators, list):
|
||||
self._generators.extend(generators)
|
||||
else:
|
||||
self._generators.append(generators)
|
||||
if not isinstance(generators, list):
|
||||
generators = [generators]
|
||||
self._generators.extend(generators)
|
||||
for generator in generators:
|
||||
generator.testsuite = self.loading_testsuite
|
||||
|
||||
self._generators = list(set(self._generators))
|
||||
|
||||
|
|
@ -1122,7 +1127,10 @@ class TestsManager(Loggable):
|
|||
self.blacklisted_tests_patterns.append(re.compile(pattern))
|
||||
|
||||
def set_default_blacklist(self, default_blacklist):
|
||||
self.blacklisted_tests += default_blacklist
|
||||
for test_regex, reason in default_blacklist:
|
||||
if not test_regex.startswith(self.loading_testsuite + '.'):
|
||||
test_regex = self.loading_testsuite + '.' + test_regex
|
||||
self.blacklisted_tests.append((test_regex, reason))
|
||||
|
||||
def add_options(self, parser):
|
||||
""" Add more arguments. """
|
||||
|
|
@ -1330,6 +1338,7 @@ class TestsGenerator(Loggable):
|
|||
Loggable.__init__(self)
|
||||
self.name = name
|
||||
self.test_manager = test_manager
|
||||
self.testsuite = None
|
||||
self._tests = {}
|
||||
for test in tests:
|
||||
self._tests[test.classname] = test
|
||||
|
|
@ -1341,6 +1350,8 @@ class TestsGenerator(Loggable):
|
|||
return list(self._tests.values())
|
||||
|
||||
def add_test(self, test):
|
||||
test.generator = self
|
||||
test.classname = self.testsuite + '.' + test.classname
|
||||
self._tests[test.classname] = test
|
||||
|
||||
|
||||
|
|
@ -1438,8 +1449,13 @@ class _TestsLauncher(Loggable):
|
|||
|
||||
module = loaded_module[0]
|
||||
if not loaded_module[0]:
|
||||
printc("Could not load testsuite: %s, reasons: %s" % (
|
||||
testsuite, loaded_module[1]), Colors.FAIL)
|
||||
if "." in testsuite:
|
||||
self.options.testsuites.append(testsuite.split('.')[0])
|
||||
self.info("%s looks like a test name, trying that" % testsuite)
|
||||
self.options.wanted_tests.append(testsuite)
|
||||
else:
|
||||
printc("Could not load testsuite: %s, reasons: %s" % (
|
||||
testsuite, loaded_module[1]), Colors.FAIL)
|
||||
continue
|
||||
|
||||
testsuites.append(module)
|
||||
|
|
@ -1465,10 +1481,14 @@ class _TestsLauncher(Loggable):
|
|||
continue
|
||||
|
||||
if self.options.user_paths:
|
||||
TestsManager.loading_testsuite = tester.name
|
||||
tester.register_defaults()
|
||||
loaded = True
|
||||
elif testsuite.setup_tests(tester, self.options):
|
||||
loaded = True
|
||||
else:
|
||||
TestsManager.loading_testsuite = testsuite.__name__
|
||||
if testsuite.setup_tests(tester, self.options):
|
||||
loaded = True
|
||||
TestsManager.loading_testsuite = None
|
||||
|
||||
if not loaded:
|
||||
printc("Could not load testsuite: %s"
|
||||
|
|
|
|||
|
|
@ -353,7 +353,7 @@ def main(libsdir):
|
|||
prog='gst-validate-launcher', description=_help_message)
|
||||
|
||||
parser.add_argument('testsuites', metavar='N', nargs='*',
|
||||
help="""Lets you specify a file where the testsuite to execute is defined.
|
||||
help="""Lets you specify a test to run, a testsuite name or a file where the testsuite to execute is defined.
|
||||
|
||||
In the module if you want to work with a specific test manager(s) (for example,
|
||||
'ges' or 'validate'), you should define the TEST_MANAGER variable in the
|
||||
|
|
|
|||
Loading…
Reference in a new issue