From 2cf93f491bd7a81ee50ce86bfcd690be1af0d01c Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 13 Jul 2017 16:43:32 -0400 Subject: [PATCH] validate: launcher: Namespace test name with the testsuite name Also allowing users to pass test names directly --- validate/launcher/apps/gstcheck.py | 2 -- validate/launcher/apps/gstvalidate.py | 52 +++++++++++++-------------- validate/launcher/baseclasses.py | 38 +++++++++++++++----- validate/launcher/main.py | 2 +- 4 files changed, 56 insertions(+), 38 deletions(-) diff --git a/validate/launcher/apps/gstcheck.py b/validate/launcher/apps/gstcheck.py index b3eea9aff6..479e4757e2 100644 --- a/validate/launcher/apps/gstcheck.py +++ b/validate/launcher/apps/gstcheck.py @@ -161,8 +161,6 @@ class MesonTestsManager(TestsManager): if test.suite: name = '.'.join(test.suite) + '.' + name - name = self.name + '.' + name - return name.replace('..', '.').replace(' ', '-') def list_tests(self): diff --git a/validate/launcher/apps/gstvalidate.py b/validate/launcher/apps/gstvalidate.py index 0c102f9935..74bf12931d 100644 --- a/validate/launcher/apps/gstvalidate.py +++ b/validate/launcher/apps/gstvalidate.py @@ -87,8 +87,8 @@ class GstValidateMediaCheckTestsGenerator(GstValidateTestsGenerator): except KeyError: timeout = DEFAULT_TIMEOUT - classname = "validate.%s.media_check.%s" % (protocol, - os.path.basename(url2path(uri)).replace(".", "_")) + classname = "%s.media_check.%s" % (protocol, + os.path.basename(url2path(uri)).replace(".", "_")) self.add_test(GstValidateMediaCheckTest(classname, self.test_manager.options, self.test_manager.reporter, @@ -113,10 +113,10 @@ class GstValidateTranscodingTestsGenerator(GstValidateTestsGenerator): continue for comb in self.test_manager.get_encoding_formats(): - classname = "validate.%s.transcode.to_%s.%s" % (mediainfo.media_descriptor.get_protocol(), - str(comb).replace( - ' ', '_'), - mediainfo.media_descriptor.get_clean_name()) + classname = "%s.transcode.to_%s.%s" % (mediainfo.media_descriptor.get_protocol(), + str(comb).replace( + ' ', '_'), + mediainfo.media_descriptor.get_clean_name()) self.add_test(GstValidateTranscodingTest(classname, self.test_manager.options, self.test_manager.reporter, @@ -199,9 +199,9 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator): protocol_str = "" if scenario is not None and scenario.name.lower() != "none": - return "%s.%s%s.%s" % ("validate", protocol_str, name, scenario.name) + return "%s%s.%s" % (protocol_str, name, scenario.name) - return ("%s.%s.%s.%s" % ("validate", protocol_str, self.name, name)).replace("..", ".") + return ("%s.%s.%s" % (protocol_str, self.name, name)).replace("..", ".") def generate_tests(self, uri_minfo_special_scenarios, scenarios): if self._valid_scenarios is None: @@ -1002,60 +1002,60 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") def register_default_blacklist(self): self.set_default_blacklist([ # hls known issues - ("validate.hls.playback.seek_with_stop.*", + ("hls.playback.seek_with_stop.*", "https://bugzilla.gnome.org/show_bug.cgi?id=753689"), # dash known issues - ("validate.dash.media_check.*", + ("dash.media_check.*", "Caps are different depending on selected bitrates, etc"), # Matroska/WEBM known issues: - ("validate.*.reverse_playback.*webm$", + ("*.reverse_playback.*webm$", "https://bugzilla.gnome.org/show_bug.cgi?id=679250"), - ("validate.*.reverse_playback.*mkv$", + ("*.reverse_playback.*mkv$", "https://bugzilla.gnome.org/show_bug.cgi?id=679250"), - ("validate.http.playback.seek_with_stop.*webm", + ("http.playback.seek_with_stop.*webm", "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"), - ("validate.http.playback.seek_with_stop.*mkv", + ("http.playback.seek_with_stop.*mkv", "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"), # MPEG TS known issues: - ('(?i)validate.*.playback.reverse_playback.*(?:_|.)(?:|m)ts$', + ('(?i)*playback.reverse_playback.*(?:_|.)(?:|m)ts$', "https://bugzilla.gnome.org/show_bug.cgi?id=702595"), # Fragmented MP4 disabled tests: - ('validate.*.playback..*seek.*.fragmented_nonseekable_sink_mp4', + ('*.playback..*seek.*.fragmented_nonseekable_sink_mp4', "Seeking on fragmented files without indexes isn't implemented"), - ('validate.*.playback.reverse_playback.fragmented_nonseekable_sink_mp4', + ('*.playback.reverse_playback.fragmented_nonseekable_sink_mp4', "Seeking on fragmented files without indexes isn't implemented"), # HTTP known issues: - ("validate.http.*scrub_forward_seeking.*", + ("http.*scrub_forward_seeking.*", "This is not stable enough for now."), - ("validate.http.playback.change_state_intensive.raw_video_mov", + ("http.playback.change_state_intensive.raw_video_mov", "This is not stable enough for now. (flow return from pad push doesn't match expected value)"), # MXF known issues" - (".*reverse_playback.*mxf", + ("*reverse_playback.*mxf", "Reverse playback is not handled in MXF"), - ("validate\.file\.transcode.*mxf", + ("file\.transcode.*mxf", "FIXME: Transcoding and mixing tests need to be tested"), # WMV known issues" - (".*reverse_playback.*wmv", + ("*reverse_playback.*wmv", "Reverse playback is not handled in wmv"), (".*reverse_playback.*asf", "Reverse playback is not handled in asf"), # ogg known issues - ("validate.http.playback.seek.*vorbis_theora_1_ogg", + ("http.playback.seek.*vorbis_theora_1_ogg", "https://bugzilla.gnome.org/show_bug.cgi?id=769545"), # RTSP known issues - ('validate.rtsp.playback.reverse.*', + ('rtsp.playback.reverse.*', 'https://bugzilla.gnome.org/show_bug.cgi?id=626811'), - ('validate.rtsp.playback.seek_with_stop.*', + ('rtsp.playback.seek_with_stop.*', 'https://bugzilla.gnome.org/show_bug.cgi?id=784298'), - ('validate.rtsp.playback.fast_*', + ('rtsp.playback.fast_*', 'https://bugzilla.gnome.org/show_bug.cgi?id=754575'), ]) diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index 33021b0379..5db511a275 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -105,6 +105,7 @@ class Test(Loggable): self.extra_env_variables = extra_env_variables self.optional = False self.is_parallel = is_parallel + self.generator = None self.clean() @@ -1040,6 +1041,7 @@ class TestsManager(Loggable): """ A class responsible for managing tests. """ name = "base" + loading_testsuite = None def __init__(self): @@ -1080,6 +1082,8 @@ class TestsManager(Loggable): self.expected_failures.update(expected_failures_re) def add_test(self, test): + if test.generator is None: + test.classname = self.loading_testsuite + '.' + test.classname for regex, failures in list(self.expected_failures.items()): if regex.findall(test.classname): test.expected_failures.extend(failures) @@ -1103,10 +1107,11 @@ class TestsManager(Loggable): """ @generators: A list of, or one single #TestsGenerator to be used to generate tests """ - if isinstance(generators, list): - self._generators.extend(generators) - else: - self._generators.append(generators) + if not isinstance(generators, list): + generators = [generators] + self._generators.extend(generators) + for generator in generators: + generator.testsuite = self.loading_testsuite self._generators = list(set(self._generators)) @@ -1122,7 +1127,10 @@ class TestsManager(Loggable): self.blacklisted_tests_patterns.append(re.compile(pattern)) def set_default_blacklist(self, default_blacklist): - self.blacklisted_tests += default_blacklist + for test_regex, reason in default_blacklist: + if not test_regex.startswith(self.loading_testsuite + '.'): + test_regex = self.loading_testsuite + '.' + test_regex + self.blacklisted_tests.append((test_regex, reason)) def add_options(self, parser): """ Add more arguments. """ @@ -1330,6 +1338,7 @@ class TestsGenerator(Loggable): Loggable.__init__(self) self.name = name self.test_manager = test_manager + self.testsuite = None self._tests = {} for test in tests: self._tests[test.classname] = test @@ -1341,6 +1350,8 @@ class TestsGenerator(Loggable): return list(self._tests.values()) def add_test(self, test): + test.generator = self + test.classname = self.testsuite + '.' + test.classname self._tests[test.classname] = test @@ -1438,8 +1449,13 @@ class _TestsLauncher(Loggable): module = loaded_module[0] if not loaded_module[0]: - printc("Could not load testsuite: %s, reasons: %s" % ( - testsuite, loaded_module[1]), Colors.FAIL) + if "." in testsuite: + self.options.testsuites.append(testsuite.split('.')[0]) + self.info("%s looks like a test name, trying that" % testsuite) + self.options.wanted_tests.append(testsuite) + else: + printc("Could not load testsuite: %s, reasons: %s" % ( + testsuite, loaded_module[1]), Colors.FAIL) continue testsuites.append(module) @@ -1465,10 +1481,14 @@ class _TestsLauncher(Loggable): continue if self.options.user_paths: + TestsManager.loading_testsuite = tester.name tester.register_defaults() loaded = True - elif testsuite.setup_tests(tester, self.options): - loaded = True + else: + TestsManager.loading_testsuite = testsuite.__name__ + if testsuite.setup_tests(tester, self.options): + loaded = True + TestsManager.loading_testsuite = None if not loaded: printc("Could not load testsuite: %s" diff --git a/validate/launcher/main.py b/validate/launcher/main.py index e4eb68e7f9..2cc28608ab 100644 --- a/validate/launcher/main.py +++ b/validate/launcher/main.py @@ -353,7 +353,7 @@ def main(libsdir): prog='gst-validate-launcher', description=_help_message) parser.add_argument('testsuites', metavar='N', nargs='*', - help="""Lets you specify a file where the testsuite to execute is defined. + help="""Lets you specify a test to run, a testsuite name or a file where the testsuite to execute is defined. In the module if you want to work with a specific test manager(s) (for example, 'ges' or 'validate'), you should define the TEST_MANAGER variable in the