validate: Add a gst-validate-launcher documentation

validate/tools/launcher/apps/ges-launch.py

@@ -185,7 +185,18 @@ class GESTestsManager(TestsManager):
         except OSError as e:
             self.warning("Can not use ges-launch: %s" % e)
 
-    def add_options(self, group):
+    def add_options(self, parser):
+        group = parser.add_argument_group("GStreamer Editing Services specific option"
+                            " and behaviours",
+                            description="""
+The GStreamer Editing Services launcher will be usable only if GES has been compiled against GstValidate
+You can simply run scenarios specifying project as args. For example the following will run all available
+and activated scenarios on project.xges:
+
+    $gst-validate-launcher ges /some/ges/project.xges
+
+
+Available options:""")
         group.add_argument("-P", "--projects-paths", dest="projects_paths",
                          default=os.path.join(utils.DEFAULT_GST_QA_ASSETS,
                                               "ges-projects"),

validate/tools/launcher/apps/gst-validate.py

@@ -364,9 +364,16 @@ class GstValidateManager(TestsManager, Loggable):
     def init(self):
         if which(GST_VALIDATE_COMMAND) and which(GST_VALIDATE_TRANSCODING_COMMAND):
             return True
-
         return False
 
+    def add_options(self, parser):
+        group = parser.add_argument_group("GstValidate tools specific options"
+                            " and behaviours",
+                            description="""
+When using --wanted-tests, all the scenarios can be used, even those which have
+not been tested and explicitely activated, in order to only use those, you should
+use --wanted-tests defaults_only""")
+
     def list_tests(self):
         if self.tests:
             return self.tests

validate/tools/launcher/baseclasses.py

@@ -548,24 +548,13 @@ class _TestsLauncher(Loggable):
         self.wanted_tests_patterns = []
 
     def _list_testers(self):
-        def get_subclasses(c, env):
-            subclasses = []
-            for symb in env.iteritems():
-                try:
-                    if issubclass(symb[1], c) and not symb[1] is c:
-                        subclasses.append(symb[1])
-                except TypeError:
-                    pass
-
-            return subclasses
-
         env = globals().copy()
         d = os.path.dirname(__file__)
         for f in os.listdir(os.path.join(d, "apps")):
             if f.endswith(".py"):
                 execfile(os.path.join(d, "apps", f), env)
 
-        testers = [i() for i in get_subclasses(TestsManager, env)]
+        testers = [i() for i in utils.get_subclasses(TestsManager, env)]
         for tester in testers:
             if tester.init() is True:
                 self.testers.append(tester)
@@ -575,10 +564,7 @@ class _TestsLauncher(Loggable):
 
     def add_options(self, parser):
         for tester in self.testers:
-            group = parser.add_argument_group("%s options" % tester.name,
-                                "Options specific to the %s test manager"
-                                % tester.name)
-            tester.add_options(group)
+            tester.add_options(parser)
 
     def set_settings(self, options, args):
         self.reporter = reporters.XunitReporter(options)

validate/tools/launcher/main.py

@@ -21,19 +21,133 @@ import sys
 import utils
 import urlparse
 import loggable
-from argparse import ArgumentParser
+import argparse
+import textwrap
+import reporters
+
 
 from httpserver import HTTPServer
 from baseclasses import _TestsLauncher, ScenarioManager
-from utils import printc, path2url, DEFAULT_MAIN_DIR, DEFAULT_GST_QA_ASSETS, launch_command, Colors
+from utils import printc, path2url, DEFAULT_MAIN_DIR, DEFAULT_GST_QA_ASSETS, launch_command, Colors, Protocols
 
 
+HELP = '''
+
+===============================================================================
+                       gst-validate-launcher
+===============================================================================
+
+1. Introduction
+----------------
+
+gst-validate-launcher is a test launcher tool, it has been designed to
+launch the various tools included in GstValidate running tests on real
+media files. This means that with gst-validate-launcher, you can launch
+many tests automatically in one simple command. It then permits to
+aggregate results and print them in a human readable way on stdout
+and serializing them in the following implemented formats:
+
+ * %s
+
+We support all the tools provided in GstValidate in the launcher, but
+we also support ges-launch when the GStreamer Editing Services have
+been compiled against GstValidate.
+
+2. Default test suite
+---------------------
+
+A default suite of tests is provided and you can run it pretty simply doing:
+
+.    $gst-validate-launch --sync
+
+That will download Gstreamer upstream default assets into the
+default folder (%s) and run all currently
+activated tests. Note that we use git-annex https://git-annex.branchable.com/ so
+you will need that tool to get started.
+
+3. Implement your own tests
+---------------------------
+
+To implement new tests, you will just need to set the media path using the
+--medias-paths argument. If you want to run all avalaible scenarios on all the
+file present in that folder, you should run the first time:
+
+.    $gst-validate-launch --medias-paths /path/to/media/files --generate-media-info
+
+That will generate the .media_info files that contain informations about the media
+files present in that folder. Those media_info file are simple XML file describing
+the topology of the media files. You should not reuse the --generate-media-info
+next times. The generated media files will be used as a reference for following
+runs. You might want to check that they contain the right informations yourself
+the first time.
+
+Those .media_info are the files that are used by gst-validate-launcher to know
+what media files can be used for the different scenarios. For example if a
+file is not seekable, seeking scenarios will not be run on it etc...
+
+3.1 Scenarios specific to a media file/stream:
+----------------------------------------------
+
+It is possible that some scenarios are very specific to one media file, in that case,
+the .scenario file should be present in the same folder as the .media_info file and
+be called similarly. For example for a file called /some/media/file.mp4, the media_info
+file will be called /some/media/file.mp4 and a scenario that will seek to a position that
+is known to fail would be called: /some/media/file.mp4.seek_to_failing_pos.scenario and
+gst-validate-launcher will run that scenario only on that media file.
+
+3.2 Test media accessible through other protocols:
+--------------------------------------------------
+
+Currently gst-validate-launcher supports the following protocols:
+
+  * %s
+
+It does not mean you can not test other protocols but it means that it has not been
+properly tested.
+
+To test medias that use those protocols, you should simply make sure that there
+is a media descriptor file with .stream_info as an extension in your --media-paths.
+You can generate such a file doing:
+
+.   $gst-validate-media-check-1.0 http://devimages.apple.com/iphone/samples/bipbop/bipbopall.m3u8 --output-file /somewhere/in/you/media/path/bipbop.stream_info
+
+Once this is done, gst-validate-launcher will run the scenarios on those media files the
+same way as if they were local files.
+
+
+4. Debug gst-validate-launcher execution
+----------------------------------------
+
+You can activate debug logs setting the environment variable GST_VALIDATE_LAUNCHER_DEBUG.
+It uses the same synthax as PITIVI_DEBUG (more information at:
+http://wiki.pitivi.org/wiki/Bug_reporting#Debug_logs).
+''' % ("\n  * ".join([reporter.name for reporter in
+                      utils.get_subclasses(reporters.Reporter, reporters.__dict__)]
+                     ),
+       DEFAULT_MAIN_DIR,
+       "\n  * ".join([getattr(Protocols, att) for att in
+                     dir(Protocols) if not att.startswith("_")]))
+
 QA_ASSETS = "gst-qa-assets"
 MEDIAS_FOLDER = "medias"
 DEFAULT_GST_QA_ASSETS_REPO = "git://people.freedesktop.org/~tsaunier/gst-qa-assets/"
 
+class Formatter(argparse.RawDescriptionHelpFormatter):
+    def _format_usage(self, usage, actions, groups, prefix):
+        pass
+
+class PrintUsage(argparse.Action):
+    def __init__(self, option_strings, dest=argparse.SUPPRESS, default=argparse.SUPPRESS, help=None):
+        super(PrintUsage, self).__init__( option_strings=option_strings, dest=dest,
+                                          default=default, nargs=0, help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        print(HELP)
+        parser.exit()
+
 def main():
-    parser = ArgumentParser()
+    parser = argparse.ArgumentParser(formatter_class=Formatter, prog='gst-validate-launcher',
+                                     description=HELP)
     parser.add_argument("-d", "--debug", dest="debug",
                       action="store_true",
                       default=False,
@@ -116,6 +230,8 @@ def main():
                             help="Url to the remote assets (default:%s)" % DEFAULT_GST_QA_ASSETS_REPO)
     assets_group.add_argument("-S", "--sync", dest="sync", action="store_true",
                             default=False, help="Synchronize asset repository")
+    assets_group.add_argument("--usage", dest="sync", action=PrintUsage,
+                            help="Print usage documentation")
 
     loggable.init("GST_VALIDATE_LAUNCHER_DEBUG", True, False)
 

validate/tools/launcher/utils.py

@@ -153,6 +153,17 @@ def touch(fname, times=None):
     with open(fname, 'a'):
         os.utime(fname, times)
 
+def get_subclasses(klass, env):
+    subclasses = []
+    for symb in env.iteritems():
+        try:
+            if issubclass(symb[1], klass) and not symb[1] is klass:
+                subclasses.append(symb[1])
+        except TypeError:
+            pass
+
+    return subclasses
+
 ##############################
 #    Encoding related utils  #
 ##############################