validate: Add a cleaner API to setup tests in testsuite files

With the testsuite format you will get a setup_tests(tests_manager,
options) function called for each TestManager.

The function will have the exact same role as with old config
file but with a clean API and not magic global variables.

This implies that we need default blacklist to be directly set
on the TestManager and not on options.blacklisted_test

validate/launcher/apps/gstvalidate.py

@@ -458,6 +458,7 @@ class GstValidateTestManager(GstValidateBaseTestManager):
         self._uris = []
         self._run_defaults = True
         self._is_populated = False
+        self._default_generators_registered = False
 
     def init(self):
         if which(GST_VALIDATE_COMMAND) and which(GST_VALIDATE_TRANSCODING_COMMAND):
@@ -475,7 +476,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
         if self._is_populated is True:
             return
 
-        if not self.options.config:
+        if not self.options.config and not self.options.testsuites:
             if self._run_defaults:
                 self.register_defaults()
             else:
@@ -687,9 +688,6 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
              "https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
             ("validate.hls.*seek_forward.*",
              "https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
-            ("validate.hls.*",
-             "FIXME! The HLS tests are not stable enough "
-             "(at least on the server), try again later."),
 
             # Matroska/WEBM known issues:
             ("validate.*.reverse_playback.*webm$",
@@ -747,6 +745,10 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
         """
         Registers default test generators
         """
+        if self._default_generators_registered:
+            return
+
         self.add_generators([GstValidatePlaybinTestsGenerator(self),
                              GstValidateMediaCheckTestsGenerator(self),
                              GstValidateTranscodingTestsGenerator(self)])
+        self._default_generators_registered = True

validate/launcher/baseclasses.py

@@ -21,6 +21,7 @@
 
 import os
 import sys
+import imp
 import re
 import time
 import utils
@@ -658,13 +659,23 @@ class TestsManager(Loggable):
         else:
             self._generators.append(generators)
 
+        self._generators = list(set(self._generators))
+
     def get_generators(self):
         return self._generators
 
+    def _add_blacklist(self, blacklisted_tests):
+        if not isinstance(blacklisted_tests, list):
+            blacklisted_tests = [blacklisted_tests]
+
+        for patterns in blacklisted_tests:
+            for pattern in patterns.split(","):
+                self.blacklisted_tests_patterns.append(re.compile(pattern))
+
     def set_default_blacklist(self, default_blacklist):
         msg = "\nCurrently 'hardcoded' %s blacklisted tests:\n\n" % self.name
         for name, bug in default_blacklist:
-            self.options.blacklisted_tests.append(name)
+            self._add_blacklist(name)
             msg += "  + %s \n   --> bug: %s\n" % (name, bug)
 
         printc(msg, Colors.FAIL, True)
@@ -687,8 +698,7 @@ class TestsManager(Loggable):
 
         if options.blacklisted_tests:
             for patterns in options.blacklisted_tests:
-                for pattern in patterns.split(","):
-                    self.blacklisted_tests_patterns.append(re.compile(pattern))
+                self._add_blacklist(patterns)
 
     def _check_blacklisted(self, test):
         for pattern in self.blacklisted_tests_patterns:
@@ -824,6 +834,65 @@ class _TestsLauncher(Loggable):
         for tester in self.testers:
             tester.add_options(parser)
 
+    def _load_testsuites(self):
+        testsuites = []
+        for testsuite in self.options.testsuites:
+            if not os.path.isabs(testsuite):
+                testsuite = os.path.join(self.options.testsuites_dir, testsuite + ".py")
+
+            try:
+                sys.path.insert(0, os.path.dirname(testsuite))
+                module = __import__(os.path.basename(testsuite).replace(".py", ""))
+            except Exception as e:
+                printc("Could not load testsuite: %s, reason: %s"
+                       % (testsuite, e), Colors.FAIL)
+                continue
+            finally:
+                sys.path.remove(os.path.dirname(testsuite))
+
+            testsuites.append(module)
+            if not hasattr(module, "TEST_MANAGER"):
+                module.TEST_MANAGER = [tester.name for tester in self.testers]
+            elif not isinstance(module.TEST_MANAGER, list):
+                module.TEST_MANAGER = [module.TEST_MANAGER]
+
+        self.options.testsuites = testsuites
+
+    def _setup_testsuites(self):
+        for testsuite in self.options.testsuites:
+            loaded = False
+            wanted_test_manager = None
+            if hasattr(testsuite, "TEST_MANAGER"):
+                wanted_test_manager = testsuite.TEST_MANAGER
+                if not isinstance(wanted_test_manager, list):
+                    wanted_test_manager = [wanted_test_manager]
+
+            for tester in self.testers:
+                if wanted_test_manager is not None and \
+                        tester.name not in wanted_test_manager:
+                    continue
+
+                if testsuite.setup_tests(tester, self.options):
+                    loaded = True
+
+            if not loaded:
+                printc("Could not load testsuite: %s"
+                       " maybe because of missing TestManager"
+                       % (testsuite), Colors.FAIL)
+
+    def _load_config(self):
+        printc("Loading config files is DEPRECATED"
+               " you should use the new testsuite format now",)
+
+        for tester in self.testers:
+            tester.options = options
+            globals()[tester.name] = tester
+        globals()["options"] = options
+        c__file__ = __file__
+        globals()["__file__"] = self.options.config
+        execfile(self.options.config, globals())
+        globals()["__file__"] = c__file__
+
     def set_settings(self, options, args):
         self.reporter = reporters.XunitReporter(options)
         if not options.logsdir in[sys.stderr, sys.stdout]:
@@ -834,6 +903,7 @@ class _TestsLauncher(Loggable):
         for tester in self.testers:
             if tester.name in args:
                 wanted_testers = tester.name
+
         if wanted_testers:
             testers = self.testers
             self.testers = []
@@ -843,18 +913,16 @@ class _TestsLauncher(Loggable):
                     args.remove(tester.name)
 
         if options.config:
-            for tester in self.testers:
-                tester.options = options
-                globals()[tester.name] = tester
-            globals()["options"] = options
-            c__file__ = __file__
-            globals()["__file__"] = self.options.config
-            execfile(self.options.config, globals())
-            globals()["__file__"] = c__file__
+            self._load_config()
+
+        self._load_testsuites()
 
         for tester in self.testers:
             tester.set_settings(options, args, self.reporter)
 
+        if not options.config and options.testsuites:
+            self._setup_testsuites()
+
     def list_tests(self):
         for tester in self.testers:
             self.tests.extend(tester.list_tests())
@@ -1069,6 +1137,8 @@ class GstValidateBaseTestManager(TestsManager):
         else:
             self._scenarios.append(scenarios)
 
+        self._scenarios = list(set(self._scenarios))
+
     def get_scenarios(self):
         return self._scenarios
 
@@ -1085,6 +1155,8 @@ class GstValidateBaseTestManager(TestsManager):
         else:
             self._encoding_formats.append(encoding_formats)
 
+        self._encoding_formats = list(set(self._encoding_formats))
+
     def get_encoding_formats(self):
         return self._encoding_formats
 

validate/launcher/main.py

@@ -133,10 +133,7 @@ MEDIAS_FOLDER = "medias"
 DEFAULT_GST_QA_ASSETS_REPO = "git://people.freedesktop.org/~tsaunier/gst-qa-assets/"
 DEFAULT_SYNC_ASSET_COMMAND = "git fetch origin && git checkout origin/master && git annex get medias/default/"
 DEFAULT_SYNC_ALL_ASSET_COMMAND = "git fetch origin && git checkout origin/master && git annex get ."
-DEFAULT_VALIDATE_TESTSUITE = os.path.join(DEFAULT_MAIN_DIR,
-                                          QA_ASSETS,
-                                          "testsuites",
-                                          "gst-validate-default.testsuite")
+DEFAULT_TESTSUITES_DIR = os.path.join(DEFAULT_MAIN_DIR, QA_ASSETS, "testsuites")
 
 
 def update_assets(options):
@@ -194,6 +191,37 @@ def main(libsdir):
     parser = argparse.ArgumentParser(
         formatter_class=argparse.RawTextHelpFormatter,
         prog='gst-validate-launcher', description=HELP)
+    parser.add_argument('testsuites', metavar='N', nargs='*',
+                        help="""Lets you specify a file where the testsuite to execute is defined.
+
+In the module if you want to work with a specific test manager(s) (for example,
+'ges' or 'validate'), you should define the TEST_MANAGER variable in the
+testsuite file (it can be a list of test manager names)
+
+In this file you should implement a setup_tests function. That function takes
+a TestManager and the GstValidateLauncher option as parameters and return True
+if it succeeded loading the tests, False otherwise.
+You will be able to configure the TestManager with its various methods. This
+function will be called with each TestManager usable, for example you will be
+passed the 'validate' TestManager in case the GstValidateManager launcher is
+avalaible. You should configure it using:
+
+   * test_manager.add_scenarios: which allows you to register a list of scenario names to be run
+   * test_manager.set_default_blacklist: Lets you set a list of tuple of the form:
+         (@regex_defining_blacklister_test_names, @reason_for_the_blacklisting)
+   * test_manager.add_generators: which allows you to register a list of #GstValidateTestsGenerator
+     to be used to generate tests
+   * test_manager.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests
+
+You can also set default values with:
+    * test_manager.register_defaults: Sets default values for all parametters
+    * test_manager.register_default_test_generators: Sets default values for the TestsGenerators to be used
+    * test_manager.register_default_scenarios: Sets default values for the scenarios to be executed
+    * test_manager.register_default_encoding_formats: Sets default values for the encoding formats to be tested
+
+Note that all testsuite should be inside python modules, so the directory should contain a __init__.py file
+""",
+                        default=["validate", "ges"])
     parser.add_argument("-d", "--debug", dest="debug",
                         action="store_true",
                         default=False,
@@ -243,27 +271,9 @@ def main(libsdir):
                              " note that 0 will enable all tests",
                         type=int),
     parser.add_argument("-c", "--config", dest="config",
-                        default=DEFAULT_VALIDATE_TESTSUITE,
-                        help="""Lets you specify a file where the testsuite to execute is defined.
-In this file you will have acces to the TestManager objects that you can configure with
-its various methods. For example you can find the 'validate' variable, in case the GstValidateManager
-launcher is avalaible. You should configure it using:
-   * validate.add_scenarios: which allows you to register a list of scenario names to be run
-   * validate.set_default_blacklist: Lets you set a list of tuple of the form:
-         (@regex_defining_blacklister_test_names, @reason_for_the_blacklisting)
-   * validate.add_generators: which allows you to register a list of #GstValidateTestsGenerator
-     to be used to generate tests
-   * validate.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests
-
-You can also set default values with:
-    * validate.register_defaults: Sets default values for all parameters
-    * validate.register_default_test_generators: Sets default values for the TestsGenerators to be used
-    * gst_validate_register_default_scenarios: Sets default values for the scenarios to be executed
-    * gst_validate_register_default_encoding_formats: Sets default values for the encoding formats to be tested
-
-Note: In the config file, you have access to the options variable resulting from the parsing of the command line
-user argument, you can thus override command line options using that.
-""")
+                        default=None,
+                        help="This is DEPRECATED, prefer using the testsuite format"
+                        " to configure testsuites")
     dir_group = parser.add_argument_group(
         "Directories and files to be used by the launcher")
     parser.add_argument('--xunit-file', action='store',
@@ -274,6 +284,11 @@ user argument, you can thus override command line options using that.
     dir_group.add_argument("-M", "--main-dir", dest="main_dir",
                            default=DEFAULT_MAIN_DIR,
                            help="Main directory where to put files. Default is %s" % DEFAULT_MAIN_DIR)
+    dir_group.add_argument("--testsuites-dir", dest="testsuites_dir",
+                           default=DEFAULT_TESTSUITES_DIR,
+                           help="Directory where to look for testsuites. Default is %s"
+                           " Note that GstValidate expect testsuite file to have .testsuite"
+                           " as an extension in this folder." % DEFAULT_TESTSUITES_DIR)
     dir_group.add_argument("-o", "--output-dir", dest="output_dir",
                            default=None,
                            help="Directory where to store logs and rendered files. Default is MAIN_DIR")