From b65b2bc2fe80bd257c9c20fc21b785f8fd8583d5 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 3 May 2020 01:20:19 -0400 Subject: [PATCH] validate: Add a mechanism to mark tests as skipped And use it when a plugin is missing and the user didn't ask for failure when it happens And use the TAP[0] synthax to report it [0]: https://testanything.org Part-of: --- docs/gst-validate-config.md | 4 +++ .../validate/gst-validate-override-registry.c | 2 +- .../validate/gst-validate-pipeline-monitor.c | 12 +++++-- validate/gst/validate/gst-validate-report.c | 33 ++++++++++++++++++ validate/gst/validate/gst-validate-report.h | 2 ++ validate/gst/validate/gst-validate-utils.c | 18 ++++++++++ validate/gst/validate/gst-validate-utils.h | 2 ++ validate/launcher/baseclasses.py | 34 ++++++++----------- validate/launcher/reporters.py | 2 ++ validate/tools/gst-validate-rtsp-server.c | 1 + validate/tools/gst-validate.c | 4 +++ 11 files changed, 92 insertions(+), 22 deletions(-) diff --git a/docs/gst-validate-config.md b/docs/gst-validate-config.md index c2f5e0784b..a4a9c5ab89 100644 --- a/docs/gst-validate-config.md +++ b/docs/gst-validate-config.md @@ -60,6 +60,10 @@ Default: `GST_CLOCK_TIME_NONE` - disabled The maximum number of dropped buffers, a `config::too-many-buffers-dropped` issue will be reported if that limit is reached. +### `fail-on-missing-plugin` + +Default: `false` meaning that tests are marked as skipped when a GStreamer plugin is missing. + ## Variables You can use variables in the configs the same way you can set them in diff --git a/validate/gst/validate/gst-validate-override-registry.c b/validate/gst/validate/gst-validate-override-registry.c index 32073e39ff..e7aa09c8cc 100644 --- a/validate/gst/validate/gst-validate-override-registry.c +++ b/validate/gst/validate/gst-validate-override-registry.c @@ -182,7 +182,7 @@ static void for (iter = registry->name_overrides.head; iter; iter = g_list_next (iter)) { entry = iter->data; if (g_regex_match_simple (entry->name, name, 0, 0)) { - GST_INFO_OBJECT (registry, "Adding override %s to %s", entry->name, name); + GST_INFO ("%p Adding override %s to %s", registry, entry->name, name); gst_validate_monitor_attach_override (monitor, entry->override); } diff --git a/validate/gst/validate/gst-validate-pipeline-monitor.c b/validate/gst/validate/gst-validate-pipeline-monitor.c index e0676d242f..1b434cb318 100644 --- a/validate/gst/validate/gst-validate-pipeline-monitor.c +++ b/validate/gst/validate/gst-validate-pipeline-monitor.c @@ -28,6 +28,9 @@ #include "gst-validate-pipeline-monitor.h" #include "gst-validate-pad-monitor.h" #include "gst-validate-monitor-factory.h" +#include "gst-validate-report.h" +#include "gst-validate-utils.h" +#include "validate.h" #define PRINT_POSITION_TIMEOUT 250 @@ -579,8 +582,13 @@ _bus_handler (GstBus * bus, GstMessage * message, gst_message_parse_error_details (message, &details); if (g_error_matches (err, GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN)) { - GST_VALIDATE_REPORT (monitor, MISSING_PLUGIN, - "Error: %s -- Debug message: %s", err->message, debug); + if (!gst_validate_fail_on_missing_plugin ()) { + gst_validate_skip_test ("missing plugin: %s -- Debug message: %s\n", + err->message, debug); + } else { + GST_VALIDATE_REPORT (monitor, MISSING_PLUGIN, + "Error: %s -- Debug message: %s", err->message, debug); + } } else if ((g_error_matches (err, GST_STREAM_ERROR, GST_STREAM_ERROR_FAILED) && details && gst_structure_get_int (details, "flow-return", &error_flow) diff --git a/validate/gst/validate/gst-validate-report.c b/validate/gst/validate/gst-validate-report.c index 558c640078..cd30dc13f9 100644 --- a/validate/gst/validate/gst-validate-report.c +++ b/validate/gst/validate/gst-validate-report.c @@ -26,6 +26,7 @@ #endif +#include /* exit */ #include /* fprintf */ #include #include @@ -1277,6 +1278,38 @@ gst_validate_print_position (GstClockTime position, GstClockTime duration, g_free (extra_info); } +void +gst_validate_skip_test (const gchar * format, ...) +{ + JsonBuilder *jbuilder; + va_list va_args; + gchar *tmp; + + va_start (va_args, format); + tmp = gst_info_strdup_vprintf (format, va_args); + va_end (va_args); + + if (!server_ostream) { + gchar *f = g_strconcat ("ok 1 # SKIP ", tmp, NULL); + + g_free (tmp); + gst_validate_printf (NULL, "%s", f); + return; + } + + jbuilder = json_builder_new (); + json_builder_begin_object (jbuilder); + json_builder_set_member_name (jbuilder, "type"); + json_builder_add_string_value (jbuilder, "skip-test"); + json_builder_set_member_name (jbuilder, "details"); + json_builder_add_string_value (jbuilder, tmp); + json_builder_end_object (jbuilder); + g_free (tmp); + + gst_validate_send (json_builder_get_root (jbuilder)); + g_object_unref (jbuilder); +} + static void print_issue (gpointer key, GstValidateIssue * issue, gpointer user_data) { diff --git a/validate/gst/validate/gst-validate-report.h b/validate/gst/validate/gst-validate-report.h index 6029f1a8c2..1643b1b911 100644 --- a/validate/gst/validate/gst-validate-report.h +++ b/validate/gst/validate/gst-validate-report.h @@ -318,6 +318,8 @@ GST_VALIDATE_API void gst_validate_error_structure (gpointer action, const gchar* format, ...) G_GNUC_PRINTF (2, 3); GST_VALIDATE_API void gst_validate_abort (const gchar * format, ...) G_GNUC_PRINTF (1, 2); +GST_VALIDATE_API +void gst_validate_skip_test (const gchar* format, ...); G_END_DECLS #endif /* __GST_VALIDATE_REPORT_H__ */ diff --git a/validate/gst/validate/gst-validate-utils.c b/validate/gst/validate/gst-validate-utils.c index 0700f555c4..7d5dc57016 100644 --- a/validate/gst/validate/gst-validate-utils.c +++ b/validate/gst/validate/gst-validate-utils.c @@ -40,6 +40,7 @@ #include "gst-validate-utils.h" #include "gst-validate-internal.h" +#include "validate.h" #include #define PARSER_BOOLEAN_EQUALITY_THRESHOLD (1e-10) @@ -1323,3 +1324,20 @@ gst_validate_set_test_file_globals (GstStructure * meta, const gchar * testfile, "videosink", G_TYPE_STRING, videosink, "audiosink", G_TYPE_STRING, audiosink, NULL); } + +gboolean +gst_validate_fail_on_missing_plugin (void) +{ + + GList *config; + + for (config = gst_validate_plugin_get_config (NULL); config; + config = config->next) { + gboolean fail_on_missing_plugin; + + if (gst_structure_get_boolean (config->data, + "fail-on-missing-plugin", &fail_on_missing_plugin)) + return fail_on_missing_plugin; + } + return FALSE; +} diff --git a/validate/gst/validate/gst-validate-utils.h b/validate/gst/validate/gst-validate-utils.h index e6d89fd353..f1f158aa6b 100644 --- a/validate/gst/validate/gst-validate-utils.h +++ b/validate/gst/validate/gst-validate-utils.h @@ -79,5 +79,7 @@ GST_VALIDATE_API void gst_validate_structure_resolve_variables (GstStructure *structure, GstStructure *local_variables); void gst_validate_structure_set_variables_from_struct_file(GstStructure* vars, const gchar* struct_file); void gst_validate_set_globals(GstStructure* structure); +GST_VALIDATE_API +gboolean gst_validate_fail_on_missing_plugin(void); #endif diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index c356a5aef2..1bc50c5b17 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -187,7 +187,7 @@ class Test(Loggable): def add_env_variable(self, variable, value=None): """ - Only usefull so that the gst-validate-launcher can print the exact + Only useful so that the gst-validate-launcher can print the exact right command line to reproduce the tests """ if value is None: @@ -356,7 +356,7 @@ class Test(Loggable): self.message = message self.error_str = error - if result not in [Result.PASSED, Result.NOT_RUN]: + if result not in [Result.PASSED, Result.NOT_RUN, Result.SKIPPED]: self.add_known_issue_information() def check_results(self): @@ -753,6 +753,8 @@ class GstValidateListener(socketserver.BaseRequestHandler, Loggable): test.actions_infos[-1]['execution-duration'] = obj['execution-duration'] elif obj_type == 'report': test.add_report(obj) + elif obj_type == 'skip-test': + test.set_result(Result.SKIPPED) class GstValidateTest(Test): @@ -1004,15 +1006,9 @@ class GstValidateTest(Test): return result, msg def check_results(self): - if self.result in [Result.FAILED, self.result is Result.PASSED]: + if self.result in [Result.FAILED, Result.PASSED, Result.SKIPPED]: return - for report in self.reports: - if report.get('issue-id') == 'runtime::missing-plugin': - self.set_result(Result.SKIPPED, "%s\n%s" % (report['summary'], - report['details'])) - return - self.debug("%s returncode: %s", self, self.process.returncode) expected_issues = copy.deepcopy(self.expected_issues) @@ -1090,9 +1086,9 @@ class GstValidateTest(Test): msg += " (Expected errors not found: %s) " % mandatory_failures result = Result.FAILED elif self.expected_issues: - msg += ' %s(Expected errors occured: %s)%s' % (Colors.OKBLUE, - self.expected_issues, - Colors.ENDC) + msg += ' %s(Expected errors occurred: %s)%s' % (Colors.OKBLUE, + self.expected_issues, + Colors.ENDC) result = Result.KNOWN_ERROR self.set_result(result, msg.strip()) @@ -1321,7 +1317,7 @@ class GstValidateEncodingTestInterface(object): 'summary': 'Expected stream caps during transcoding do not match expectations', 'level': 'critical', 'detected-on': 'pipeline', - 'details': "Field: %s (from %s) not in caps of the outputed file %s" % ( + 'details': "Field: %s (from %s) not in caps of the outputted file %s" % ( wanted_caps, c, ccaps) } ) @@ -1374,7 +1370,7 @@ class TestsManager(Loggable): if regex.findall(test.classname): if failure_def.get('allow_flakiness'): test.allow_flakiness = True - self.debug("%s allow flakyness" % (test.classname)) + self.debug("%s allow flakiness" % (test.classname)) else: for issue in failure_def['issues']: issue['bug'] = bugid @@ -1395,7 +1391,7 @@ class TestsManager(Loggable): if regex.findall(test.classname): if failure_def.get('allow_flakiness'): test.allow_flakiness = True - self.debug("%s allow flakyness" % (test.classname)) + self.debug("%s allow flakiness" % (test.classname)) else: for issue in failure_def['issues']: issue['bug'] = bugid @@ -2155,7 +2151,7 @@ class Scenario(object): return False def compatible_with_live_content(self): - # if a live content is required it's implicitely compatible with + # if a live content is required it's implicitly compatible with # live content if self.needs_live_content(): return True @@ -2342,7 +2338,7 @@ class GstValidateBaseTestManager(TestsManager): @scenarios A list or a unic scenario name(s) to be run on the tests. They are just the default scenarios, and then depending on the TestsGenerator to be used you can have more fine grained - control on what to be run on each serie of tests. + control on what to be run on each series of tests. """ if isinstance(scenarios, list): self._scenarios.extend(scenarios) @@ -2367,7 +2363,7 @@ class GstValidateBaseTestManager(TestsManager): formats for transcoding test. They are just the default encoding formats, and then depending on the TestsGenerator to be used you can have more fine grained - control on what to be run on each serie of tests. + control on what to be run on each series of tests. """ if isinstance(encoding_formats, list): self._encoding_formats.extend(encoding_formats) @@ -2563,7 +2559,7 @@ class GstValidateMediaDescriptor(MediaDescriptor): for ext in [self.MEDIA_INFO_EXT, self.PUSH_MEDIA_INFO_EXT, self.STREAM_INFO_EXT, self.SKIPPED_MEDIA_INFO_EXT, ]: if self._xml_path.endswith(ext): - return self._xml_path[:len(self._xml_path) - (len(ext) + 1)] + return self._xml_path[:len(self._xml_path) - (len(ext) + 1)] assert "Not reached" is None diff --git a/validate/launcher/reporters.py b/validate/launcher/reporters.py index 0bec1ae499..3f100bd695 100644 --- a/validate/launcher/reporters.py +++ b/validate/launcher/reporters.py @@ -110,6 +110,8 @@ class Reporter(Loggable): Colors.OKBLUE) printc("%sPassed: %d" % (lenstat * " ", self.stats["passed"]), Colors.OKGREEN) + printc("%sSkipped: %d" % + (lenstat * " ", self.stats["skipped"]), Colors.WARNING) printc("%sFailed: %d" % (lenstat * " ", self.stats["failures"]), Colors.FAIL) printc("%sKnown error: %d" % diff --git a/validate/tools/gst-validate-rtsp-server.c b/validate/tools/gst-validate-rtsp-server.c index 00abbd5d55..017b2ad1b1 100644 --- a/validate/tools/gst-validate-rtsp-server.c +++ b/validate/tools/gst-validate-rtsp-server.c @@ -20,6 +20,7 @@ /* Cc'd from the test-uri example */ #include +#include #include #include diff --git a/validate/tools/gst-validate.c b/validate/tools/gst-validate.c index 77682d75ff..f91035360d 100644 --- a/validate/tools/gst-validate.c +++ b/validate/tools/gst-validate.c @@ -503,6 +503,10 @@ main (int argc, gchar ** argv) g_free (argvn); exit (1); } else if (err) { + if (g_error_matches (err, GST_PARSE_ERROR, GST_PARSE_ERROR_NO_SUCH_ELEMENT)) { + if (!gst_validate_fail_on_missing_plugin ()) + gst_validate_skip_test ("missing plugin: %s", err->message); + } g_printerr ("Erroneous pipeline: %s\n", err->message ? err->message : "unknown reason"); g_clear_error (&err);