gst-validate-launcher: update documentation

Use the new api to create your custom testsuite.
Fix some broken links and enhance the logging system.

docs/gst-validate-launcher.md

@@ -71,59 +71,81 @@ Then you will need to write the `testsuite.py` file. You can for example
 implement the following testsuite:
 
 ``` python
+"""
+The GstValidate custom testsuite
+"""
+
 import os
+from launcher.baseclasses import MediaFormatCombination
+from launcher.apps.gstvalidate import *
+TEST_MANAGER = "validate"
 
-# Make sure gst-validate-launcher uses our media files
+KNOWN_ISSUES = {}
-options.paths = os.path.dirname(os.path.realpath(__file__))
 
-# Make sure GstValidate is able to use our scenarios
+def setup_tests(test_manager, options):
-# from the testsuite_folder/scenarios folder
+    print("Setting up the custom testsuite")
-os.environ["GST_VALIDATE_SCENARIOS_PATH"] = \
+    assets_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".", "samples_files"))
-    os.path.join(os.path.dirname(os.path.realpath(__file__)), "scenarios")
+    options.add_paths(assets_dir)
 
-# You can activate the following if you only care about critical issues in
+    # This step will register default data for the test manager:
-# the report:
+    # - scenarios such as `play_15s`, `reverse_playback` etc.
-# os.environ["GST_VALIDATE"] = "print_criticals"
+    # - encoding formats such as "mp4,h264,mp3" etc.
+    # - blacklist such as dash.media_check.*
+    # - test generators:
+    #   - GstValidatePlaybinTestsGenerator
+    #   - GstValidateMediaCheckTestsGenerator
+    #   - GstValidateTranscodingTestsGenerator
+    # This 'defaults' can be found in 'gst-devtools/validate/launcher/apps/gstvalidate.py#register_defaults'
+    # test_manager.register_defaults()
 
-# Make gst-validate use our scenarios
+    # Add scenarios
-validate.add_scenarios(["scenario", "scenario1"])
+    scenarios = []
+    scenarios.append("play_5s")
+    scenarios.append("seek_backward")
+    test_manager.set_scenarios(scenarios)
 
+    # Add encoding formats used by the transcoding generator
+    test_manager.add_encoding_formats([
+            MediaFormatCombination("mp4", "mp3", "h264"),])
 
-# Now add "Theora and Vorbis in OGG container" as a wanted transcoding format. That means
+    # Add generators
-# that conversion to this format will be tested on all the media files/streams.
+    # GstValidatePlaybinTestsGenerator needs at least one media file
-validate.add_encoding_formats([MediaFormatCombination("ogg", "vorbis", "theora")])
+    test_manager.add_generators([GstValidateMediaCheckTestsGenerator(test_manager)])
+    # GstValidatePlaybinTestsGenerator needs at least one scenario
+    test_manager.add_generators([GstValidatePlaybinTestsGenerator(test_manager)])
+    # GstValidateTranscodingTestsGenerator needs at least one MediaFormatCombination
+    test_manager.add_generators([GstValidateTranscodingTestsGenerator(test_manager)])
 
-# Use the GstValidatePlaybinTestsGenerator to generate tests that will use playbin
+    # list of combo to blacklist tests. Here it blacklists all tests with playback.seek_backward
-# and GstValidateTranscodingTestsGenerator to create media transcoding tests that
+    test_manager.set_default_blacklist([
-# will use all the media format added with validate.add_encoding_formats
+            ("custom_testsuite.file.playback.seek_backward.*",
-validate.add_generators([validate.GstValidatePlaybinTestsGenerator(validate),
+             "Not supported by this testsuite."),])
-                         GstValidateTranscodingTestsGenerator(self)])
 
-# Blacklist some tests that are known to fail because a feature is not supported
+    # you can even pass known issues to bypass an existing error in your custom testsuite
-# or due to any other reason.
+    test_manager.add_expected_issues(KNOWN_ISSUES)
-# The tuple defining those tests is of the form:
+    return True
-# ("regex defining the test name", "Reason why the test should be disabled")
-validate.set_default_blacklist([
-        ("validate.*.scenario1.*ogv$"
-         "oggdemux does not support some action executed in scenario1")]
-        )
 ```
 
 Once this is done, you've got a testsuite that will:
 
 -   Run playbin pipelines on `file.mp4`, `file1.mkv` and `file2.ogv`>
-    executing `scenario` and `scenario1` scenarios
+    executing `play_5s` and `seek_backward` scenarios
 
--   Transcode `file.mp4,` `file1.mkv` and `file2.ogv` to Theora and
+-   Transcode `file.mp4,` `file1.mkv` and `file2.ogv` to h264 and
-    Vorbis in a OGG container
+    mp3 in a MP4 container
 
 The only thing to do to run the testsuite is:
 
 
-    gst-validate-launcher --config /path/to/testsuite_folder/testsuite.py
+    gst-validate-launcher --testsuites-dir=/path/to/testsuite_folder/ testsuite
+
 
 # Invocation
 
 You can find detailed information about the launcher by launching it:
 
     gst-validate-launcher --help
+
+You can list all the tests with:
+
+    gst-validate-launcher --testsuites-dir=/path/to/testsuite_folder/ testsuite -L

validate/launcher/apps/gstvalidate.py

@@ -844,7 +844,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
         group.add_argument("--validate-gdb-server", dest="validate_gdb_server",
                            help="Run the server in GDB.")
         group.add_argument("--validate-disable-rtsp", dest="disable_rtsp",
-                           help="Disable RTSP tests.")
+                           help="Disable RTSP tests.", default=False, action='store_true')
         group.add_argument("--validate-enable-iqa-tests", dest="validate_enable_iqa_tests",
                            help="Enable Image Quality Assessment validation tests.",
                            default=False, action='store_true')

validate/launcher/main.py

@@ -59,7 +59,7 @@ been compiled against GstValidate.
 2. Default test suite
 ---------------------
 
-A default suite of tests is provided and is available at: http://cgit.freedesktop.org/gstreamer/gst-integration-testsuites/
+A default suite of tests is provided and is available at: http://gitlab.freedesktop.org/gstreamer/gst-integration-testsuites/
 You can run it pretty simply doing:
 
 .    $gst-validate-launcher --sync
@@ -125,8 +125,11 @@ same way as if they were local files.
 ----------------------------------------
 
 You can activate debug logs setting the environment variable GST_VALIDATE_LAUNCHER_DEBUG.
+
+.   $GST_VALIDATE_LAUNCHER_DEBUG=6 gst-validate-launcher
+
 It uses the same syntax as PITIVI_DEBUG (more information at:
-http://wiki.pitivi.org/wiki/Bug_reporting#Debug_logs).
+https://developer.pitivi.org/Bug_reporting.html#debug-logs).
 ''' % ("\n  * ".join([reporter.name for reporter in
                       utils.get_subclasses(reporters.Reporter, reporters.__dict__)]
                      ),
@@ -145,6 +148,8 @@ DEFAULT_GST_QA_ASSETS_REPO = "https://gitlab.freedesktop.org/gstreamer/gst-integ
 
 def download_assets(options):
     try:
+        printc("About to download assets from %s to %s" % options.remote_assets_url,
+               options.clone_dir)
         launch_command("%s %s %s" % (options.get_assets_command,
                                      options.remote_assets_url,
                                      options.clone_dir),