mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-12-27 10:40:34 +00:00
validate: Add a cleaner API to setup tests in testsuite files
With the testsuite format you will get a setup_tests(tests_manager, options) function called for each TestManager. The function will have the exact same role as with old config file but with a clean API and not magic global variables. This implies that we need default blacklist to be directly set on the TestManager and not on options.blacklisted_test
This commit is contained in:
Thibault Saunier
parent
b852fbcbfb
commit
d38307a073
3 changed files with 129 additions and 40 deletions
|
|
@ -458,6 +458,7 @@ class GstValidateTestManager(GstValidateBaseTestManager):
|
|||
self._uris = []
|
||||
self._run_defaults = True
|
||||
self._is_populated = False
|
||||
self._default_generators_registered = False
|
||||
|
||||
def init(self):
|
||||
if which(GST_VALIDATE_COMMAND) and which(GST_VALIDATE_TRANSCODING_COMMAND):
|
||||
|
|
@ -475,7 +476,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
|
|||
if self._is_populated is True:
|
||||
return
|
||||
|
||||
if not self.options.config:
|
||||
if not self.options.config and not self.options.testsuites:
|
||||
if self._run_defaults:
|
||||
self.register_defaults()
|
||||
else:
|
||||
|
|
@ -687,9 +688,6 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
|
|||
"https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
|
||||
("validate.hls.*seek_forward.*",
|
||||
"https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
|
||||
("validate.hls.*",
|
||||
"FIXME! The HLS tests are not stable enough "
|
||||
"(at least on the server), try again later."),
|
||||
|
||||
# Matroska/WEBM known issues:
|
||||
("validate.*.reverse_playback.*webm$",
|
||||
|
|
@ -747,6 +745,10 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
|
|||
"""
|
||||
Registers default test generators
|
||||
"""
|
||||
if self._default_generators_registered:
|
||||
return
|
||||
|
||||
self.add_generators([GstValidatePlaybinTestsGenerator(self),
|
||||
GstValidateMediaCheckTestsGenerator(self),
|
||||
GstValidateTranscodingTestsGenerator(self)])
|
||||
self._default_generators_registered = True
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import imp
|
||||
import re
|
||||
import time
|
||||
import utils
|
||||
|
|
@ -658,13 +659,23 @@ class TestsManager(Loggable):
|
|||
else:
|
||||
self._generators.append(generators)
|
||||
|
||||
self._generators = list(set(self._generators))
|
||||
|
||||
def get_generators(self):
|
||||
return self._generators
|
||||
|
||||
def _add_blacklist(self, blacklisted_tests):
|
||||
if not isinstance(blacklisted_tests, list):
|
||||
blacklisted_tests = [blacklisted_tests]
|
||||
|
||||
for patterns in blacklisted_tests:
|
||||
for pattern in patterns.split(","):
|
||||
self.blacklisted_tests_patterns.append(re.compile(pattern))
|
||||
|
||||
def set_default_blacklist(self, default_blacklist):
|
||||
msg = "\nCurrently 'hardcoded' %s blacklisted tests:\n\n" % self.name
|
||||
for name, bug in default_blacklist:
|
||||
self.options.blacklisted_tests.append(name)
|
||||
self._add_blacklist(name)
|
||||
msg += " + %s \n --> bug: %s\n" % (name, bug)
|
||||
|
||||
printc(msg, Colors.FAIL, True)
|
||||
|
|
@ -687,8 +698,7 @@ class TestsManager(Loggable):
|
|||
|
||||
if options.blacklisted_tests:
|
||||
for patterns in options.blacklisted_tests:
|
||||
for pattern in patterns.split(","):
|
||||
self.blacklisted_tests_patterns.append(re.compile(pattern))
|
||||
self._add_blacklist(patterns)
|
||||
|
||||
def _check_blacklisted(self, test):
|
||||
for pattern in self.blacklisted_tests_patterns:
|
||||
|
|
@ -824,6 +834,65 @@ class _TestsLauncher(Loggable):
|
|||
for tester in self.testers:
|
||||
tester.add_options(parser)
|
||||
|
||||
def _load_testsuites(self):
|
||||
testsuites = []
|
||||
for testsuite in self.options.testsuites:
|
||||
if not os.path.isabs(testsuite):
|
||||
testsuite = os.path.join(self.options.testsuites_dir, testsuite + ".py")
|
||||
|
||||
try:
|
||||
sys.path.insert(0, os.path.dirname(testsuite))
|
||||
module = __import__(os.path.basename(testsuite).replace(".py", ""))
|
||||
except Exception as e:
|
||||
printc("Could not load testsuite: %s, reason: %s"
|
||||
% (testsuite, e), Colors.FAIL)
|
||||
continue
|
||||
finally:
|
||||
sys.path.remove(os.path.dirname(testsuite))
|
||||
|
||||
testsuites.append(module)
|
||||
if not hasattr(module, "TEST_MANAGER"):
|
||||
module.TEST_MANAGER = [tester.name for tester in self.testers]
|
||||
elif not isinstance(module.TEST_MANAGER, list):
|
||||
module.TEST_MANAGER = [module.TEST_MANAGER]
|
||||
|
||||
self.options.testsuites = testsuites
|
||||
|
||||
def _setup_testsuites(self):
|
||||
for testsuite in self.options.testsuites:
|
||||
loaded = False
|
||||
wanted_test_manager = None
|
||||
if hasattr(testsuite, "TEST_MANAGER"):
|
||||
wanted_test_manager = testsuite.TEST_MANAGER
|
||||
if not isinstance(wanted_test_manager, list):
|
||||
wanted_test_manager = [wanted_test_manager]
|
||||
|
||||
for tester in self.testers:
|
||||
if wanted_test_manager is not None and \
|
||||
tester.name not in wanted_test_manager:
|
||||
continue
|
||||
|
||||
if testsuite.setup_tests(tester, self.options):
|
||||
loaded = True
|
||||
|
||||
if not loaded:
|
||||
printc("Could not load testsuite: %s"
|
||||
" maybe because of missing TestManager"
|
||||
% (testsuite), Colors.FAIL)
|
||||
|
||||
def _load_config(self):
|
||||
printc("Loading config files is DEPRECATED"
|
||||
" you should use the new testsuite format now",)
|
||||
|
||||
for tester in self.testers:
|
||||
tester.options = options
|
||||
globals()[tester.name] = tester
|
||||
globals()["options"] = options
|
||||
c__file__ = __file__
|
||||
globals()["__file__"] = self.options.config
|
||||
execfile(self.options.config, globals())
|
||||
globals()["__file__"] = c__file__
|
||||
|
||||
def set_settings(self, options, args):
|
||||
self.reporter = reporters.XunitReporter(options)
|
||||
if not options.logsdir in[sys.stderr, sys.stdout]:
|
||||
|
|
@ -834,6 +903,7 @@ class _TestsLauncher(Loggable):
|
|||
for tester in self.testers:
|
||||
if tester.name in args:
|
||||
wanted_testers = tester.name
|
||||
|
||||
if wanted_testers:
|
||||
testers = self.testers
|
||||
self.testers = []
|
||||
|
|
@ -843,18 +913,16 @@ class _TestsLauncher(Loggable):
|
|||
args.remove(tester.name)
|
||||
|
||||
if options.config:
|
||||
for tester in self.testers:
|
||||
tester.options = options
|
||||
globals()[tester.name] = tester
|
||||
globals()["options"] = options
|
||||
c__file__ = __file__
|
||||
globals()["__file__"] = self.options.config
|
||||
execfile(self.options.config, globals())
|
||||
globals()["__file__"] = c__file__
|
||||
self._load_config()
|
||||
|
||||
self._load_testsuites()
|
||||
|
||||
for tester in self.testers:
|
||||
tester.set_settings(options, args, self.reporter)
|
||||
|
||||
if not options.config and options.testsuites:
|
||||
self._setup_testsuites()
|
||||
|
||||
def list_tests(self):
|
||||
for tester in self.testers:
|
||||
self.tests.extend(tester.list_tests())
|
||||
|
|
@ -1069,6 +1137,8 @@ class GstValidateBaseTestManager(TestsManager):
|
|||
else:
|
||||
self._scenarios.append(scenarios)
|
||||
|
||||
self._scenarios = list(set(self._scenarios))
|
||||
|
||||
def get_scenarios(self):
|
||||
return self._scenarios
|
||||
|
||||
|
|
@ -1085,6 +1155,8 @@ class GstValidateBaseTestManager(TestsManager):
|
|||
else:
|
||||
self._encoding_formats.append(encoding_formats)
|
||||
|
||||
self._encoding_formats = list(set(self._encoding_formats))
|
||||
|
||||
def get_encoding_formats(self):
|
||||
return self._encoding_formats
|
||||
|
||||
|
|
|
|||
|
|
@ -133,10 +133,7 @@ MEDIAS_FOLDER = "medias"
|
|||
DEFAULT_GST_QA_ASSETS_REPO = "git://people.freedesktop.org/~tsaunier/gst-qa-assets/"
|
||||
DEFAULT_SYNC_ASSET_COMMAND = "git fetch origin && git checkout origin/master && git annex get medias/default/"
|
||||
DEFAULT_SYNC_ALL_ASSET_COMMAND = "git fetch origin && git checkout origin/master && git annex get ."
|
||||
DEFAULT_VALIDATE_TESTSUITE = os.path.join(DEFAULT_MAIN_DIR,
|
||||
QA_ASSETS,
|
||||
"testsuites",
|
||||
"gst-validate-default.testsuite")
|
||||
DEFAULT_TESTSUITES_DIR = os.path.join(DEFAULT_MAIN_DIR, QA_ASSETS, "testsuites")
|
||||
|
||||
|
||||
def update_assets(options):
|
||||
|
|
@ -194,6 +191,37 @@ def main(libsdir):
|
|||
parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
prog='gst-validate-launcher', description=HELP)
|
||||
parser.add_argument('testsuites', metavar='N', nargs='*',
|
||||
help="""Lets you specify a file where the testsuite to execute is defined.
|
||||
|
||||
In the module if you want to work with a specific test manager(s) (for example,
|
||||
'ges' or 'validate'), you should define the TEST_MANAGER variable in the
|
||||
testsuite file (it can be a list of test manager names)
|
||||
|
||||
In this file you should implement a setup_tests function. That function takes
|
||||
a TestManager and the GstValidateLauncher option as parameters and return True
|
||||
if it succeeded loading the tests, False otherwise.
|
||||
You will be able to configure the TestManager with its various methods. This
|
||||
function will be called with each TestManager usable, for example you will be
|
||||
passed the 'validate' TestManager in case the GstValidateManager launcher is
|
||||
avalaible. You should configure it using:
|
||||
|
||||
* test_manager.add_scenarios: which allows you to register a list of scenario names to be run
|
||||
* test_manager.set_default_blacklist: Lets you set a list of tuple of the form:
|
||||
(@regex_defining_blacklister_test_names, @reason_for_the_blacklisting)
|
||||
* test_manager.add_generators: which allows you to register a list of #GstValidateTestsGenerator
|
||||
to be used to generate tests
|
||||
* test_manager.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests
|
||||
|
||||
You can also set default values with:
|
||||
* test_manager.register_defaults: Sets default values for all parametters
|
||||
* test_manager.register_default_test_generators: Sets default values for the TestsGenerators to be used
|
||||
* test_manager.register_default_scenarios: Sets default values for the scenarios to be executed
|
||||
* test_manager.register_default_encoding_formats: Sets default values for the encoding formats to be tested
|
||||
|
||||
Note that all testsuite should be inside python modules, so the directory should contain a __init__.py file
|
||||
""",
|
||||
default=["validate", "ges"])
|
||||
parser.add_argument("-d", "--debug", dest="debug",
|
||||
action="store_true",
|
||||
default=False,
|
||||
|
|
@ -243,27 +271,9 @@ def main(libsdir):
|
|||
" note that 0 will enable all tests",
|
||||
type=int),
|
||||
parser.add_argument("-c", "--config", dest="config",
|
||||
default=DEFAULT_VALIDATE_TESTSUITE,
|
||||
help="""Lets you specify a file where the testsuite to execute is defined.
|
||||
In this file you will have acces to the TestManager objects that you can configure with
|
||||
its various methods. For example you can find the 'validate' variable, in case the GstValidateManager
|
||||
launcher is avalaible. You should configure it using:
|
||||
* validate.add_scenarios: which allows you to register a list of scenario names to be run
|
||||
* validate.set_default_blacklist: Lets you set a list of tuple of the form:
|
||||
(@regex_defining_blacklister_test_names, @reason_for_the_blacklisting)
|
||||
* validate.add_generators: which allows you to register a list of #GstValidateTestsGenerator
|
||||
to be used to generate tests
|
||||
* validate.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests
|
||||
|
||||
You can also set default values with:
|
||||
* validate.register_defaults: Sets default values for all parameters
|
||||
* validate.register_default_test_generators: Sets default values for the TestsGenerators to be used
|
||||
* gst_validate_register_default_scenarios: Sets default values for the scenarios to be executed
|
||||
* gst_validate_register_default_encoding_formats: Sets default values for the encoding formats to be tested
|
||||
|
||||
Note: In the config file, you have access to the options variable resulting from the parsing of the command line
|
||||
user argument, you can thus override command line options using that.
|
||||
""")
|
||||
default=None,
|
||||
help="This is DEPRECATED, prefer using the testsuite format"
|
||||
" to configure testsuites")
|
||||
dir_group = parser.add_argument_group(
|
||||
"Directories and files to be used by the launcher")
|
||||
parser.add_argument('--xunit-file', action='store',
|
||||
|
|
@ -274,6 +284,11 @@ user argument, you can thus override command line options using that.
|
|||
dir_group.add_argument("-M", "--main-dir", dest="main_dir",
|
||||
default=DEFAULT_MAIN_DIR,
|
||||
help="Main directory where to put files. Default is %s" % DEFAULT_MAIN_DIR)
|
||||
dir_group.add_argument("--testsuites-dir", dest="testsuites_dir",
|
||||
default=DEFAULT_TESTSUITES_DIR,
|
||||
help="Directory where to look for testsuites. Default is %s"
|
||||
" Note that GstValidate expect testsuite file to have .testsuite"
|
||||
" as an extension in this folder." % DEFAULT_TESTSUITES_DIR)
|
||||
dir_group.add_argument("-o", "--output-dir", dest="output_dir",
|
||||
default=None,
|
||||
help="Directory where to store logs and rendered files. Default is MAIN_DIR")
|
||||
|
|
|
|||
Loading…
Reference in a new issue