validate: Implement RTSP support

validate/data/scenarios/switch_audio_track_while_paused.scenario

@@ -1,4 +1,4 @@
-description, summary="Change audio track while pipeline is paused", min-audio-track=2, duration=6.0, need-clock-sync=true
+description, summary="Change audio track while pipeline is paused", min-audio-track=2, duration=6.0, need-clock-sync=true, needs_preroll=true
 pause, playback-time=1.0;
 
 # Wait so that humans can see the pipeline is paused

validate/data/scenarios/switch_subtitle_track_while_paused.scenario

@@ -1,4 +1,4 @@
-description, summary="Change subtitle track while pipeline is PAUSED", min-subtitle-track=2, duration=5.0, handles-states=true, need-clock-sync=true
+description, summary="Change subtitle track while pipeline is PAUSED", min-subtitle-track=2, duration=5.0, handles-states=true, need-clock-sync=true, needs_preroll=true
 pause;
 wait, duration=0.5
 switch-track, type=text, index=(string)+1

validate/docs/validate-design.txt

@@ -1,4 +1,3 @@
-
 == Main components
 
 Gst-validate is composed of 4 parts: the issues, the reports, the runner and
@@ -58,4 +57,3 @@ The file checker is another reporter that is used to make sure a file has a
 few expected properties. It inspects the file and compares the results with
 expected values set by the user. Values such as file duration, file size, if
 it can be played back and also if its encoding and container types.
-

validate/gst/validate/media-descriptor-writer.c

@@ -137,9 +137,10 @@ serialize_filenode (GstValidateMediaDescriptorWriter * writer)
     caps_str = g_strdup ("");
 
   res = g_string_new (tmpstr);
-  g_string_append_printf (res, "  <streams caps=\"%s\">\n", caps_str);
-  g_free (caps_str);
   g_free (tmpstr);
+  tmpstr = g_markup_printf_escaped ("  <streams caps=\"%s\">\n", caps_str);
+  g_string_append (res, tmpstr);
+  g_free (caps_str);
   for (tmp = filenode->streams; tmp; tmp = tmp->next) {
     GList *tmp3;
     GstValidateMediaStreamNode
@@ -658,6 +659,7 @@ gst_validate_media_descriptor_writer_new_discover (GstValidateRunner * runner,
   media_descriptor = (GstValidateMediaDescriptor *) writer;
   if (streams == NULL && media_descriptor->filenode->caps)
     writer->priv->raw_caps = gst_caps_copy (media_descriptor->filenode->caps);
+
   gst_discoverer_stream_info_list_free (streams);
 
 

validate/launcher/apps/gstvalidate.py

@@ -18,9 +18,12 @@
 # Boston, MA 02110-1301, USA.
 import argparse
 import os
+import copy
+import sys
 import time
 import urllib.parse
 import shlex
+import socket
 import subprocess
 import configparser
 from launcher.loggable import Loggable
@@ -31,7 +34,8 @@ from launcher.baseclasses import GstValidateTest, Test, \
     GstValidateBaseTestManager, MediaDescriptor, MediaFormatCombination
 
 from launcher.utils import path2url, url2path, DEFAULT_TIMEOUT, which, \
-    GST_SECOND, Result, Protocols, mkdir, printc, Colors, get_data_file
+    GST_SECOND, Result, Protocols, mkdir, printc, Colors, get_data_file, \
+    kill_subprocess
 
 #
 # Private global variables     #
@@ -40,14 +44,17 @@ from launcher.utils import path2url, url2path, DEFAULT_TIMEOUT, which, \
 # definitions of commands to use
 parser = argparse.ArgumentParser(add_help=False)
 parser.add_argument("--validate-tools-path", dest="validate_tools_path",
-                default="",
+                    default="",
-                help="defines the paths to look for GstValidate tools.")
+                    help="defines the paths to look for GstValidate tools.")
 options, args = parser.parse_known_args()
 
 GST_VALIDATE_COMMAND = which("gst-validate-1.0", options.validate_tools_path)
-GST_VALIDATE_TRANSCODING_COMMAND = which("gst-validate-transcoding-1.0", options.validate_tools_path)
+GST_VALIDATE_TRANSCODING_COMMAND = which(
-G_V_DISCOVERER_COMMAND = which("gst-validate-media-check-1.0", options.validate_tools_path)
+    "gst-validate-transcoding-1.0", options.validate_tools_path)
+G_V_DISCOVERER_COMMAND = which(
+    "gst-validate-media-check-1.0", options.validate_tools_path)
 ScenarioManager.GST_VALIDATE_COMMAND = GST_VALIDATE_COMMAND
+RTSP_SERVER_COMMAND = "gst-rtsp-server-example-uri-1.0"
 
 AUDIO_ONLY_FILE_TRANSCODING_RATIO = 5
 
@@ -62,6 +69,7 @@ GST_VALIDATE_CAPS_TO_PROTOCOL = [("application/x-hls", Protocols.HLS),
                                  ("application/dash+xml", Protocols.DASH)]
 GST_VALIDATE_PROTOCOL_TIMEOUTS = {Protocols.HTTP: 120,
                                   Protocols.HLS: 240,
+                                  Protocols.RTSP: 240,
                                   Protocols.DASH: 240}
 
 
@@ -99,6 +107,10 @@ class GstValidateTranscodingTestsGenerator(GstValidateTestsGenerator):
             if mediainfo.media_descriptor.is_image():
                 continue
 
+            protocol = mediainfo.media_descriptor.get_protocol()
+            if protocol == Protocols.RTSP:
+                continue
+
             for comb in self.test_manager.get_encoding_formats():
                 classname = "validate.%s.transcode.to_%s.%s" % (mediainfo.media_descriptor.get_protocol(),
                                                                 str(comb).replace(
@@ -113,6 +125,7 @@ class GstValidateTranscodingTestsGenerator(GstValidateTestsGenerator):
 
 
 class FakeMediaDescriptor(MediaDescriptor):
+
     def __init__(self, infos, pipeline_desc):
         MediaDescriptor.__init__(self)
         self._infos = infos
@@ -210,7 +223,8 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator):
 
             for scenario in extra_datas.get('scenarios', scenarios):
                 if isinstance(scenario, str):
-                    scenario = self.test_manager.scenarios_manager.get_scenario(scenario)
+                    scenario = self.test_manager.scenarios_manager.get_scenario(
+                        scenario)
 
                 mediainfo = FakeMediaDescriptor(extra_datas, pipeline)
                 if not mediainfo.is_compatible(scenario):
@@ -229,7 +243,8 @@ class GstValidatePipelineTestsGenerator(GstValidateTestsGenerator):
                 pipeline_desc = pipeline % {'videosink': videosink,
                                             'audiosink': audiosink}
 
-                fname = self.get_fname(scenario, protocol=mediainfo.get_protocol(), name=name)
+                fname = self.get_fname(
+                    scenario, protocol=mediainfo.get_protocol(), name=name)
 
                 expected_failures = extra_datas.get("expected-failures")
                 extra_env_vars = extra_datas.get("extra_env_vars")
@@ -254,11 +269,43 @@ class GstValidatePlaybinTestsGenerator(GstValidatePipelineTestsGenerator):
             GstValidatePipelineTestsGenerator.__init__(
                 self, "playback", test_manager, "playbin3")
 
+    def _set_sinks(self, minfo, pipe_str, scenario):
+        if self.test_manager.options.mute:
+            if scenario.needs_clock_sync() or \
+                    minfo.media_descriptor.need_clock_sync():
+                afakesink = "'fakesink sync=true'"
+                vfakesink = "'fakesink sync=true qos=true max-lateness=20000000'"
+            else:
+                vfakesink = afakesink = "'fakesink'"
+
+            pipe_str += " audio-sink=%s video-sink=%s" % (
+                afakesink, vfakesink)
+
+        return pipe_str
+
+    def _get_name(self, scenario, protocol, minfo):
+        return "%s.%s" % (self.get_fname(scenario,
+                                         protocol),
+                          os.path.basename(minfo.media_descriptor.get_clean_name()))
+
     def populate_tests(self, uri_minfo_special_scenarios, scenarios):
+        test_rtsp = which(RTSP_SERVER_COMMAND)
+        if not test_rtsp:
+            printc("\n\nRTSP server not available, you should make sure"
+                   " that %s is available in your $PATH." % RTSP_SERVER_COMMAND,
+                   Colors.FAIL)
+        elif self.test_manager.options.disable_rtsp:
+            printc("\n\nRTSP tests are disabled")
+            test_rtsp = False
+
         for uri, minfo, special_scenarios in uri_minfo_special_scenarios:
             pipe = self._pipeline_template
             protocol = minfo.media_descriptor.get_protocol()
 
+            if protocol == Protocols.RTSP:
+                self.debug("SKIPPING %s as it is a RTSP stream" % uri)
+                continue
+
             pipe += " uri=%s" % uri
 
             for scenario in special_scenarios + scenarios:
@@ -266,20 +313,9 @@ class GstValidatePlaybinTestsGenerator(GstValidatePipelineTestsGenerator):
                 if not minfo.media_descriptor.is_compatible(scenario):
                     continue
 
-                if self.test_manager.options.mute:
+                cpipe = self._set_sinks(minfo, cpipe, scenario)
-                    if scenario.needs_clock_sync() or \
+                fname = self._get_name(scenario, protocol, minfo)
-                            minfo.media_descriptor.need_clock_sync():
-                        afakesink = "'fakesink sync=true'"
-                        vfakesink = "'fakesink sync=true qos=true max-lateness=20000000'"
-                    else:
-                        vfakesink = afakesink = "'fakesink'"
 
-                    cpipe += " audio-sink=%s video-sink=%s" % (
-                        afakesink, vfakesink)
-
-                fname = "%s.%s" % (self.get_fname(scenario,
-                                   protocol),
-                                   os.path.basename(minfo.media_descriptor.get_clean_name()))
                 self.debug("Adding: %s", fname)
 
                 if scenario.does_reverse_playback() and protocol == Protocols.HTTP:
@@ -294,6 +330,21 @@ class GstValidatePlaybinTestsGenerator(GstValidatePipelineTestsGenerator):
                                                     media_descriptor=minfo.media_descriptor)
                               )
 
+                if test_rtsp and protocol == Protocols.FILE and not minfo.media_descriptor.is_image():
+                    rtspminfo = copy.deepcopy(minfo)
+                    rtspminfo.media_descriptor = GstValidateRTSPMediaDesciptor(minfo.media_descriptor.get_path())
+                    if not rtspminfo.media_descriptor.is_compatible(scenario):
+                        continue
+
+                    cpipe = self._set_sinks(rtspminfo, "%s uri=rtsp://127.0.0.1:<RTSPPORTNUMBER>/test"
+                                            % self._pipeline_template, scenario)
+                    fname = self._get_name(scenario, Protocols.RTSP, rtspminfo)
+
+                    self.add_test(GstValidateRTSPTest(
+                        fname, self.test_manager.options, self.test_manager.reporter,
+                        cpipe, uri, scenario=scenario,
+                        media_descriptor=rtspminfo.media_descriptor))
+
 
 class GstValidateMixerTestsGenerator(GstValidatePipelineTestsGenerator):
 
@@ -443,7 +494,8 @@ class GstValidateMediaCheckTest(GstValidateTest):
 
     def build_arguments(self):
         Test.build_arguments(self)
-        self.add_arguments(self._uri, "--expected-results", self._media_info_path)
+        self.add_arguments(self._uri, "--expected-results",
+                           self._media_info_path)
 
 
 class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterface):
@@ -541,6 +593,112 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa
         self.set_result(res, msg)
 
 
+class GstValidateBaseRTSPTest:
+    """ Interface for RTSP tests, requires implementing Test"""
+    __used_ports = set()
+
+    def __init__(self, local_uri):
+        self._local_uri = local_uri
+        self.rtsp_server = None
+        self._unsetport_pipeline_desc = None
+        self.optional = True
+
+    @classmethod
+    def __get_open_port(cls):
+        while True:
+            # hackish trick from
+            # http://stackoverflow.com/questions/2838244/get-open-tcp-port-in-python?answertab=votes#tab-top
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.bind(("", 0))
+            port = s.getsockname()[1]
+            if port not in cls.__used_ports:
+                cls.__used_ports.add(port)
+                s.close()
+                return port
+
+            s.close()
+
+    def launch_server(self):
+        if self.options.redirect_logs == 'stdout':
+            self.rtspserver_logs = sys.stdout
+        elif self.options.redirect_logs == 'stderr':
+            self.rtspserver_logs = sys.stderr
+
+        self.server_port = self.__get_open_port()
+        command = [RTSP_SERVER_COMMAND, self._local_uri, '--port', str(self.server_port)]
+
+        if self.options.validate_gdb_server:
+            command = self.use_gdb(command)
+            self.rtspserver_logs = sys.stdout
+        elif self.options.redirect_logs:
+            self.rtspserver_logs = sys.stdout
+        else:
+            self.rtspserver_logs = open(self.logfile + '_rtspserver.log', 'w+')
+            self.extra_logfiles.append(self.rtspserver_logs.name)
+
+        server_env = os.environ.copy()
+        server_env['GST_TRACERS'] = 'validate'
+
+        self.rtsp_server = subprocess.Popen(command,
+                                            stderr=self.rtspserver_logs,
+                                            stdout=self.rtspserver_logs,
+                                            env=server_env)
+        while True:
+            s = socket.socket()
+            try:
+                s.connect((("127.0.0.1", self.server_port)))
+                break
+            except ConnectionRefusedError:
+                time.sleep(0.5)
+                continue
+            finally:
+                s.close()
+
+        if not self._unsetport_pipeline_desc:
+            self._unsetport_pipeline_desc = self.pipeline_desc
+
+        self.pipeline_desc = self._unsetport_pipeline_desc.replace(
+            "<RTSPPORTNUMBER>", str(self.server_port))
+
+        return 'GST_TRACERS=validate ' + ' '.join(command)
+
+    def close_logfile(self):
+        super().close_logfile()
+        if not self.options.redirect_logs:
+            self.rtspserver_logs.close()
+
+    def process_update(self):
+        res = super().process_update()
+        if res:
+            kill_subprocess(self, self.rtsp_server, DEFAULT_TIMEOUT)
+            self.__used_ports.remove(self.server_port)
+
+        return res
+
+
+class GstValidateRTSPTest(GstValidateBaseRTSPTest, GstValidateLaunchTest):
+
+    def __init__(self, classname, options, reporter, pipeline_desc,
+                 local_uri, timeout=DEFAULT_TIMEOUT, scenario=None,
+                 media_descriptor=None):
+        GstValidateLaunchTest.__init__(self, classname, options, reporter,
+                                       pipeline_desc, timeout, scenario,
+                                       media_descriptor)
+        GstValidateBaseRTSPTest.__init__(self, local_uri)
+
+
+class GstValidateRTSPMediaDesciptor(GstValidateMediaDescriptor):
+
+    def __init__(self, xml_path):
+        GstValidateMediaDescriptor.__init__(self, xml_path)
+
+    def get_uri(self):
+        return "rtsp://127.0.0.1:8554/test"
+
+    def get_protocol(self):
+        return Protocols.RTSP
+
+
 class GstValidateTestManager(GstValidateBaseTestManager):
 
     name = "validate"
@@ -582,6 +740,10 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
                            action="append", help="defines the uris to run default tests on")
         group.add_argument("--validate-tools-path", dest="validate_tools_path",
                            action="append", help="defines the paths to look for GstValidate tools.")
+        group.add_argument("--validate-gdb-server", dest="validate_gdb_server",
+                           help="Run the server in GDB.")
+        group.add_argument("--validate-disable-rtsp", dest="disable_rtsp",
+                           help="Disable RTSP tests.")
 
     def print_valgrind_bugs(self):
         # Look for all the 'pending' bugs in our supp file
@@ -629,7 +791,8 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
                 self.add_test(test)
 
         if not self.tests and not uris:
-            printc("No valid uris present in the path. Check if media files and info files exist", Colors.FAIL)
+            printc(
+                "No valid uris present in the path. Check if media files and info files exist", Colors.FAIL)
 
         return self.tests
 
@@ -681,10 +844,12 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
                 self._add_media(fpath)
                 return True
             elif not self.options.generate_info and not self.options.update_media_info and not self.options.validate_uris:
-                self.info("%s not present. Use --generate-media-info", media_info)
+                self.info(
+                    "%s not present. Use --generate-media-info", media_info)
                 return True
             elif self.options.update_media_info and not os.path.isfile(media_info):
-                self.info("%s not present. Use --generate-media-info", media_info)
+                self.info(
+                    "%s not present. Use --generate-media-info", media_info)
                 return True
 
             include_frames = 0
@@ -750,7 +915,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
 
                 if protocol in [Protocols.HTTP, Protocols.HLS, Protocols.DASH] and \
                         ("127.0.0.1:%s" % (self.options.http_server_port) in uri or
-                        "127.0.0.1:8079" in uri):
+                         "127.0.0.1:8079" in uri):
                     return True
         return False
 
@@ -862,9 +1027,9 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
              "https://bugzilla.gnome.org/show_bug.cgi?id=702595"),
 
             # Fragmented MP4 disabled tests:
-            ('validate.file.playback..*seek.*.fragmented_nonseekable_sink_mp4',
+            ('validate.*.playback..*seek.*.fragmented_nonseekable_sink_mp4',
              "Seeking on fragmented files without indexes isn't implemented"),
-            ('validate.file.playback.reverse_playback.fragmented_nonseekable_sink_mp4',
+            ('validate.*.playback.reverse_playback.fragmented_nonseekable_sink_mp4',
              "Seeking on fragmented files without indexes isn't implemented"),
 
             # HTTP known issues:
@@ -889,7 +1054,12 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
 
             # ogg known issues
             ("validate.http.playback.seek.*vorbis_theora_1_ogg",
-             "https://bugzilla.gnome.org/show_bug.cgi?id=769545")
+             "https://bugzilla.gnome.org/show_bug.cgi?id=769545"),
+            # RTSP known issues
+            ('validate.rtsp.playback.reverse.*',
+             'https://bugzilla.gnome.org/show_bug.cgi?id=626811'),
+            ('validate.rtsp.playback.fast_*',
+             'https://bugzilla.gnome.org/show_bug.cgi?id=754575'),
         ])
 
     def register_default_test_generators(self):

validate/launcher/baseclasses.py

@@ -82,6 +82,7 @@ class Test(Loggable):
         self.options = options
         self.application = application_name
         self.command = []
+        self.server_command = None
         self.reporter = reporter
         self.process = None
         self.proc_env = None
@@ -98,6 +99,7 @@ class Test(Loggable):
 
         extra_env_variables = extra_env_variables or {}
         self.extra_env_variables = extra_env_variables
+        self.optional = False
 
         self.clean()
 
@@ -333,26 +335,7 @@ class Test(Loggable):
         return os.environ.copy()
 
     def kill_subprocess(self):
-        if self.process is None:
+        utils.kill_subprocess(self, self.process, DEFAULT_TIMEOUT)
-            return
-
-        stime = time.time()
-        res = self.process.poll()
-        while res is None:
-            try:
-                self.debug("Subprocess is still alive, sending KILL signal")
-                if utils.is_windows():
-                    subprocess.call(['taskkill', '/F', '/T', '/PID', str(self.process.pid)])
-                else:
-                    self.process.send_signal(signal.SIGKILL)
-                time.sleep(1)
-            except OSError:
-                pass
-            if time.time() - stime > DEFAULT_TIMEOUT:
-                raise RuntimeError("Could not kill subprocess after %s second"
-                                   " Something is really wrong, => EXITING"
-                                   % DEFAULT_TIMEOUT)
-            res = self.process.poll()
 
     def thread_wrapper(self):
         self.process = subprocess.Popen(self.command,
@@ -373,14 +356,15 @@ class Test(Loggable):
     def get_valgrind_suppressions(self):
         return [self.get_valgrind_suppression_file('data', 'gstvalidate.supp')]
 
-    def use_gdb(self):
+    def use_gdb(self, command):
         if self.hard_timeout is not None:
             self.hard_timeout *= GDB_TIMEOUT_FACTOR
         self.timeout *= GDB_TIMEOUT_FACTOR
-        self.command = ["gdb", "-ex", "run", "-ex", "quit",
+        return ["gdb", "-ex", "run", "-ex", "backtrace", "-ex", "quit", "--args"] + command
-                        "--args"] + self.command
 
-    def use_valgrind(self):
+    def use_valgrind(self, command, subenv):
+        vglogsfile = self.logfile + '.valgrind'
+        self.extra_logfiles.append(vglogsfile)
 
         vg_args = []
 
@@ -404,14 +388,11 @@ class Test(Loggable):
         for supp in self.get_valgrind_suppressions():
             vg_args.append("--suppressions=%s" % supp)
 
-        self.command = ["valgrind"] + vg_args + self.command
+        command = ["valgrind"] + vg_args + command
 
         # Tune GLib's memory allocator to be more valgrind friendly
-        self.proc_env['G_DEBUG'] = 'gc-friendly'
+        subenv['G_DEBUG'] = 'gc-friendly'
-        self.add_env_variable('G_DEBUG', 'gc-friendly')
+        subenv['G_SLICE'] = 'always-malloc'
-
-        self.proc_env['G_SLICE'] = 'always-malloc'
-        self.add_env_variable('G_SLICE', 'always-malloc')
 
         if self.hard_timeout is not None:
             self.hard_timeout *= VALGRIND_TIMEOUT_FACTOR
@@ -421,15 +402,24 @@ class Test(Loggable):
         vg_config = get_data_file('data', 'valgrind.config')
 
         if self.proc_env.get('GST_VALIDATE_CONFIG'):
-            self.proc_env['GST_VALIDATE_CONFIG'] = '%s%s%s' % (self.proc_env['GST_VALIDATE_CONFIG'], os.pathsep, vg_config)
+            subenv['GST_VALIDATE_CONFIG'] = '%s%s%s' % (self.proc_env['GST_VALIDATE_CONFIG'], os.pathsep, vg_config)
         else:
-            self.proc_env['GST_VALIDATE_CONFIG'] = vg_config
+            subenv['GST_VALIDATE_CONFIG'] = vg_config
 
-        self.add_env_variable('GST_VALIDATE_CONFIG', self.proc_env['GST_VALIDATE_CONFIG'])
+        if subenv == self.proc_env:
+            self.add_env_variable('G_DEBUG', 'gc-friendly')
+            self.add_env_variable('G_SLICE', 'always-malloc')
+            self.add_env_variable('GST_VALIDATE_CONFIG', self.proc_env['GST_VALIDATE_CONFIG'])
+
+        return command
+
+    def launch_server(self):
+        return None
 
     def test_start(self, queue):
         self.open_logfile()
 
+        server_command = self.launch_server()
         self.queue = queue
         self.command = [self.application]
         self._starting_time = time.time()
@@ -442,14 +432,17 @@ class Test(Loggable):
             self.add_env_variable(var, self.proc_env[var])
 
         if self.options.gdb:
-            self.use_gdb()
+            self.command = self.use_gdb(self.command)
-
         if self.options.valgrind:
-            self.use_valgrind()
+            self.command = self.use_valgrind(self.command, self.proc_env)
 
         message = "Launching: %s%s\n" \
-                  "    Command: '%s %s'\n" % (Colors.ENDC, self.classname,
+                  "    Command: '%s & %s %s'\n" % (
-                                              self._env_variable, ' '.join(self.command))
+                      Colors.ENDC, self.classname, server_command,
+                      self._env_variable, ' '.join(self.command))
+        if server_command:
+            message += "    Server command: %s\n" % server_command
+
         if not self.options.redirect_logs:
             message += "    Logs:\n" \
                        "         - %s" % (self.logfile)
@@ -1541,16 +1534,18 @@ class _TestsLauncher(Loggable):
                     testlist_file = open(os.path.splitext(testsuite.__file__)[0] + ".testslist",
                                          'w')
                 except IOError:
-                    return
+                    continue
 
                 for test in know_tests:
-                    if test and test not in tests_names:
+                    if test and test.strip('~') not in tests_names:
-                        testlist_changed = True
+                        if not test.startswith('~'):
-                        printc("Test %s Not in testsuite %s anymore"
+                            testlist_changed = True
-                               % (test, testsuite.__file__), Colors.FAIL)
+                            printc("Test %s Not in testsuite %s anymore"
+                                % (test, testsuite.__file__), Colors.FAIL)
 
-                for test in tests_names:
+                for test in tests:
-                    testlist_file.write("%s\n" % test)
+                    testlist_file.write("%s%s\n" % ('~' if test.optional else '',
+                                                    test.classname))
                     if test and test not in know_tests:
                         printc("Test %s is NEW in testsuite %s"
                                % (test, testsuite.__file__), Colors.OKGREEN)
@@ -1719,6 +1714,7 @@ class Scenario(object):
     def __repr__(self):
         return "<Scenario %s>" % self.name
 
+
 class ScenarioManager(Loggable):
     _instance = None
     all_scenarios = []
@@ -1844,7 +1840,7 @@ class GstValidateBaseTestManager(TestsManager):
         """
         self._scenarios = []
         self.add_scenarios(scenarios)
-        
+
     def get_scenarios(self):
         return self._scenarios
 

validate/launcher/utils.py

@@ -27,6 +27,7 @@ import os
 import platform
 import re
 import shutil
+import signal
 import subprocess
 import sys
 import tempfile
@@ -43,7 +44,8 @@ from xml.etree import ElementTree
 GST_SECOND = int(1000000000)
 DEFAULT_TIMEOUT = 30
 DEFAULT_MAIN_DIR = os.path.join(os.path.expanduser("~"), "gst-validate")
-DEFAULT_GST_QA_ASSETS = os.path.join(DEFAULT_MAIN_DIR, "gst-integration-testsuites")
+DEFAULT_GST_QA_ASSETS = os.path.join(
+    DEFAULT_MAIN_DIR, "gst-integration-testsuites")
 DISCOVERER_COMMAND = "gst-discoverer-1.0"
 # Use to set the duration from which a test is considered as being 'long'
 LONG_TEST = 40
@@ -63,6 +65,7 @@ class Protocols(object):
     FILE = "file"
     HLS = "hls"
     DASH = "dash"
+    RTSP = "rtsp"
 
     @staticmethod
     def needs_clock_sync(protocol):
@@ -208,7 +211,8 @@ def TIME_ARGS(time):
 
 
 def look_for_file_in_source_dir(subdir, name):
-    root_dir = os.path.abspath(os.path.dirname(os.path.join(os.path.dirname(os.path.abspath(__file__)))))
+    root_dir = os.path.abspath(os.path.dirname(
+        os.path.join(os.path.dirname(os.path.abspath(__file__)))))
     p = os.path.join(root_dir, subdir, name)
     if os.path.exists(p):
         return p
@@ -253,7 +257,8 @@ def get_duration(media_file):
     duration = 0
     res = ''
     try:
-        res = subprocess.check_output([DISCOVERER_COMMAND, media_file]).decode()
+        res = subprocess.check_output(
+            [DISCOVERER_COMMAND, media_file]).decode()
     except subprocess.CalledProcessError:
         # gst-media-check returns !0 if seeking is not possible, we do not care
         # in that case.
@@ -308,7 +313,7 @@ class BackTraceGenerator(Loggable):
                 "installed."
 
         gdb = ['gdb', '-ex', 't a a bt', '-batch',
-            '-p', str(test.process.pid)]
+               '-p', str(test.process.pid)]
 
         try:
             return subprocess.check_output(
@@ -320,7 +325,8 @@ class BackTraceGenerator(Loggable):
     def get_trace_from_systemd(self, test):
         for ntry in range(10):
             if ntry != 0:
-                # Loopping, it means we conceder the logs might not be ready yet.
+                # Loopping, it means we conceder the logs might not be ready
+                # yet.
                 time.sleep(1)
 
             try:
@@ -334,7 +340,8 @@ class BackTraceGenerator(Loggable):
 
             info = info.decode()
             try:
-                executable = BackTraceGenerator._executable_regex.findall(info)[0]
+                executable = BackTraceGenerator._executable_regex.findall(info)[
+                    0]
             except IndexError:
                 self.debug("Backtrace could not be found yet, trying harder.")
                 # The trace might not be ready yet
@@ -357,11 +364,11 @@ class BackTraceGenerator(Loggable):
                 try:
                     tf = tempfile.NamedTemporaryFile()
                     subprocess.check_output(['coredumpctl', 'dump',
-                                            str(test.process.pid), '--output=' +
+                                             str(test.process.pid), '--output=' +
-                                            tf.name], stderr=subprocess.STDOUT)
+                                             tf.name], stderr=subprocess.STDOUT)
 
                     gdb = ['gdb', '-ex', 't a a bt', '-ex', 'quit',
-                        test.application, tf.name]
+                           test.application, tf.name]
                     bt_all = subprocess.check_output(
                         gdb, stderr=subprocess.STDOUT).decode()
 
@@ -387,13 +394,14 @@ def check_bugs_resolution(bugs_definitions):
 
             if "bugzilla" not in url.netloc:
                 printc("  + %s \n   --> bug: %s\n   --> Status: Not a bugzilla report\n" % (regex, bug),
-                    Colors.WARNING)
+                       Colors.WARNING)
                 continue
 
             query = urllib.parse.parse_qs(url.query)
             _id = query.get('id')
             if not _id:
-                printc("  + '%s' -- Can't check bug '%s'\n" % (regex, bug), Colors.WARNING)
+                printc("  + '%s' -- Can't check bug '%s'\n" %
+                       (regex, bug), Colors.WARNING)
                 continue
 
             if isinstance(_id, list):
@@ -447,3 +455,29 @@ def check_bugs_resolution(bugs_definitions):
                    regex, bugid, desc, status), Colors.OKGREEN)
 
     return res
+
+
+def kill_subprocess(owner, process, timeout):
+    if process is None:
+        return
+
+    stime = time.time()
+    res = process.poll()
+    while res is None:
+        try:
+            owner.debug("Subprocess is still alive, sending KILL signal")
+            if is_windows():
+                subprocess.call(
+                    ['taskkill', '/F', '/T', '/PID', str(process.pid)])
+            else:
+                process.send_signal(signal.SIGKILL)
+            time.sleep(1)
+        except OSError:
+            pass
+        if time.time() - stime > DEFAULT_TIMEOUT:
+            raise RuntimeError("Could not kill subprocess after %s second"
+                               " Something is really wrong, => EXITING"
+                               % DEFAULT_TIMEOUT)
+        res = process.poll()
+
+    return res