validate: launcher: Namespace test name with the testsuite name

Also allowing users to pass test names directly

validate/launcher/apps/gstcheck.py

@@ -161,8 +161,6 @@ class MesonTestsManager(TestsManager):
         if test.suite:
             name = '.'.join(test.suite) + '.' + name
 
-        name = self.name + '.' + name
-
         return name.replace('..', '.').replace(' ', '-')
 
     def list_tests(self):

validate/launcher/apps/gstvalidate.py

@@ -87,8 +87,8 @@ class GstValidateMediaCheckTestsGenerator(GstValidateTestsGenerator):
             except KeyError:
                 timeout = DEFAULT_TIMEOUT
 
-            classname = "validate.%s.media_check.%s" % (protocol,
+            classname = "%s.media_check.%s" % (protocol,
-                                                        os.path.basename(url2path(uri)).replace(".", "_"))
+                                               os.path.basename(url2path(uri)).replace(".", "_"))
             self.add_test(GstValidateMediaCheckTest(classname,
                                                     self.test_manager.options,
                                                     self.test_manager.reporter,
@@ -113,10 +113,10 @@ class GstValidateTranscodingTestsGenerator(GstValidateTestsGenerator):
                 continue
 
             for comb in self.test_manager.get_encoding_formats():
-                classname = "validate.%s.transcode.to_%s.%s" % (mediainfo.media_descriptor.get_protocol(),
+                classname = "%s.transcode.to_%s.%s" % (mediainfo.media_descriptor.get_protocol(),
-                                                                str(comb).replace(
+                                                       str(comb).replace(
-                                                                    ' ', '_'),
+                                                       ' ', '_'),
-                                                                mediainfo.media_descriptor.get_clean_name())
+                                                       mediainfo.media_descriptor.get_clean_name())
                 self.add_test(GstValidateTranscodingTest(classname,
                                                          self.test_manager.options,
                                                          self.test_manager.reporter,
@@ -199,9 +199,9 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator):
             protocol_str = ""
 
         if scenario is not None and scenario.name.lower() != "none":
-            return "%s.%s%s.%s" % ("validate", protocol_str, name, scenario.name)
+            return "%s%s.%s" % (protocol_str, name, scenario.name)
 
-        return ("%s.%s.%s.%s" % ("validate", protocol_str, self.name, name)).replace("..", ".")
+        return ("%s.%s.%s" % (protocol_str, self.name, name)).replace("..", ".")
 
     def generate_tests(self, uri_minfo_special_scenarios, scenarios):
         if self._valid_scenarios is None:
@@ -1002,60 +1002,60 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
     def register_default_blacklist(self):
         self.set_default_blacklist([
             # hls known issues
-            ("validate.hls.playback.seek_with_stop.*",
+            ("hls.playback.seek_with_stop.*",
              "https://bugzilla.gnome.org/show_bug.cgi?id=753689"),
 
             # dash known issues
-            ("validate.dash.media_check.*",
+            ("dash.media_check.*",
              "Caps are different depending on selected bitrates, etc"),
 
             # Matroska/WEBM known issues:
-            ("validate.*.reverse_playback.*webm$",
+            ("*.reverse_playback.*webm$",
              "https://bugzilla.gnome.org/show_bug.cgi?id=679250"),
-            ("validate.*.reverse_playback.*mkv$",
+            ("*.reverse_playback.*mkv$",
              "https://bugzilla.gnome.org/show_bug.cgi?id=679250"),
-            ("validate.http.playback.seek_with_stop.*webm",
+            ("http.playback.seek_with_stop.*webm",
              "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"),
-            ("validate.http.playback.seek_with_stop.*mkv",
+            ("http.playback.seek_with_stop.*mkv",
              "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"),
 
             # MPEG TS known issues:
-            ('(?i)validate.*.playback.reverse_playback.*(?:_|.)(?:|m)ts$',
+            ('(?i)*playback.reverse_playback.*(?:_|.)(?:|m)ts$',
              "https://bugzilla.gnome.org/show_bug.cgi?id=702595"),
 
             # Fragmented MP4 disabled tests:
-            ('validate.*.playback..*seek.*.fragmented_nonseekable_sink_mp4',
+            ('*.playback..*seek.*.fragmented_nonseekable_sink_mp4',
              "Seeking on fragmented files without indexes isn't implemented"),
-            ('validate.*.playback.reverse_playback.fragmented_nonseekable_sink_mp4',
+            ('*.playback.reverse_playback.fragmented_nonseekable_sink_mp4',
              "Seeking on fragmented files without indexes isn't implemented"),
 
             # HTTP known issues:
-            ("validate.http.*scrub_forward_seeking.*",
+            ("http.*scrub_forward_seeking.*",
              "This is not stable enough for now."),
-            ("validate.http.playback.change_state_intensive.raw_video_mov",
+            ("http.playback.change_state_intensive.raw_video_mov",
              "This is not stable enough for now. (flow return from pad push doesn't match expected value)"),
 
             # MXF known issues"
-            (".*reverse_playback.*mxf",
+            ("*reverse_playback.*mxf",
              "Reverse playback is not handled in MXF"),
-            ("validate\.file\.transcode.*mxf",
+            ("file\.transcode.*mxf",
              "FIXME: Transcoding and mixing tests need to be tested"),
 
             # WMV known issues"
-            (".*reverse_playback.*wmv",
+            ("*reverse_playback.*wmv",
              "Reverse playback is not handled in wmv"),
             (".*reverse_playback.*asf",
              "Reverse playback is not handled in asf"),
 
             # ogg known issues
-            ("validate.http.playback.seek.*vorbis_theora_1_ogg",
+            ("http.playback.seek.*vorbis_theora_1_ogg",
              "https://bugzilla.gnome.org/show_bug.cgi?id=769545"),
             # RTSP known issues
-            ('validate.rtsp.playback.reverse.*',
+            ('rtsp.playback.reverse.*',
              'https://bugzilla.gnome.org/show_bug.cgi?id=626811'),
-            ('validate.rtsp.playback.seek_with_stop.*',
+            ('rtsp.playback.seek_with_stop.*',
              'https://bugzilla.gnome.org/show_bug.cgi?id=784298'),
-            ('validate.rtsp.playback.fast_*',
+            ('rtsp.playback.fast_*',
              'https://bugzilla.gnome.org/show_bug.cgi?id=754575'),
         ])
 

validate/launcher/baseclasses.py

@@ -105,6 +105,7 @@ class Test(Loggable):
         self.extra_env_variables = extra_env_variables
         self.optional = False
         self.is_parallel = is_parallel
+        self.generator = None
 
         self.clean()
 
@@ -1040,6 +1041,7 @@ class TestsManager(Loggable):
     """ A class responsible for managing tests. """
 
     name = "base"
+    loading_testsuite = None
 
     def __init__(self):
 
@@ -1080,6 +1082,8 @@ class TestsManager(Loggable):
         self.expected_failures.update(expected_failures_re)
 
     def add_test(self, test):
+        if test.generator is None:
+            test.classname = self.loading_testsuite + '.' + test.classname
         for regex, failures in list(self.expected_failures.items()):
             if regex.findall(test.classname):
                 test.expected_failures.extend(failures)
@@ -1103,10 +1107,11 @@ class TestsManager(Loggable):
         """
         @generators: A list of, or one single #TestsGenerator to be used to generate tests
         """
-        if isinstance(generators, list):
+        if not isinstance(generators, list):
-            self._generators.extend(generators)
+            generators = [generators]
-        else:
+        self._generators.extend(generators)
-            self._generators.append(generators)
+        for generator in generators:
+            generator.testsuite = self.loading_testsuite
 
         self._generators = list(set(self._generators))
 
@@ -1122,7 +1127,10 @@ class TestsManager(Loggable):
                 self.blacklisted_tests_patterns.append(re.compile(pattern))
 
     def set_default_blacklist(self, default_blacklist):
-        self.blacklisted_tests += default_blacklist
+        for test_regex, reason in default_blacklist:
+            if not test_regex.startswith(self.loading_testsuite + '.'):
+                test_regex = self.loading_testsuite + '.' + test_regex
+            self.blacklisted_tests.append((test_regex, reason))
 
     def add_options(self, parser):
         """ Add more arguments. """
@@ -1330,6 +1338,7 @@ class TestsGenerator(Loggable):
         Loggable.__init__(self)
         self.name = name
         self.test_manager = test_manager
+        self.testsuite = None
         self._tests = {}
         for test in tests:
             self._tests[test.classname] = test
@@ -1341,6 +1350,8 @@ class TestsGenerator(Loggable):
         return list(self._tests.values())
 
     def add_test(self, test):
+        test.generator = self
+        test.classname = self.testsuite + '.' + test.classname
         self._tests[test.classname] = test
 
 
@@ -1438,8 +1449,13 @@ class _TestsLauncher(Loggable):
 
             module = loaded_module[0]
             if not loaded_module[0]:
-                printc("Could not load testsuite: %s, reasons: %s" % (
+                if "." in testsuite:
-                    testsuite, loaded_module[1]), Colors.FAIL)
+                    self.options.testsuites.append(testsuite.split('.')[0])
+                    self.info("%s looks like a test name, trying that" % testsuite)
+                    self.options.wanted_tests.append(testsuite)
+                else:
+                    printc("Could not load testsuite: %s, reasons: %s" % (
+                        testsuite, loaded_module[1]), Colors.FAIL)
                 continue
 
             testsuites.append(module)
@@ -1465,10 +1481,14 @@ class _TestsLauncher(Loggable):
                     continue
 
                 if self.options.user_paths:
+                    TestsManager.loading_testsuite = tester.name
                     tester.register_defaults()
                     loaded = True
-                elif testsuite.setup_tests(tester, self.options):
+                else:
-                    loaded = True
+                    TestsManager.loading_testsuite = testsuite.__name__
+                    if testsuite.setup_tests(tester, self.options):
+                        loaded = True
+                    TestsManager.loading_testsuite = None
 
             if not loaded:
                 printc("Could not load testsuite: %s"

validate/launcher/main.py

@@ -353,7 +353,7 @@ def main(libsdir):
         prog='gst-validate-launcher', description=_help_message)
 
     parser.add_argument('testsuites', metavar='N', nargs='*',
-                        help="""Lets you specify a file where the testsuite to execute is defined.
+                        help="""Lets you specify a test to run, a testsuite name or a file where the testsuite to execute is defined.
 
 In the module if you want to work with a specific test manager(s) (for example,
 'ges' or 'validate'), you should define the TEST_MANAGER variable in the