diff --git a/validate/launcher/apps/gstvalidate.py b/validate/launcher/apps/gstvalidate.py index 6f7a319c2e..3274b29c9a 100644 --- a/validate/launcher/apps/gstvalidate.py +++ b/validate/launcher/apps/gstvalidate.py @@ -626,7 +626,7 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa self.set_result(Result.PASSED, """Got no EOS 30 seconds after sending EOS, in HLS known and tolerated issue: - https://bugzilla.gnome.org/show_bug.cgi?id=723868""") + https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/132""") return Result.KNOWN_ERROR self.set_result( @@ -1081,10 +1081,6 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") def register_default_blacklist(self): self.set_default_blacklist([ - # hls known issues - # ("hls.playback.seek_with_stop.*", - # "https://bugzilla.gnome.org/show_bug.cgi?id=753689"), - # testbin known issues ("testbin.media_check.*", "Not supported by GstDiscoverer."), @@ -1095,9 +1091,9 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") # Matroska/WEBM known issues: ("*.reverse_playback.*webm$", - "https://bugzilla.gnome.org/show_bug.cgi?id=679250"), + "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/65"), ("*.reverse_playback.*mkv$", - "https://bugzilla.gnome.org/show_bug.cgi?id=679250"), + "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/65"), ("http.playback.seek_with_stop.*webm", "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"), ("http.playback.seek_with_stop.*mkv", @@ -1105,7 +1101,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") # MPEG TS known issues: ('(?i)*playback.reverse_playback.*(?:_|.)(?:|m)ts$', - "https://bugzilla.gnome.org/show_bug.cgi?id=702595"), + "https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/97"), # Fragmented MP4 disabled tests: ('*.playback..*seek.*.fragmented_nonseekable_sink_mp4', @@ -1133,14 +1129,14 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") # ogg known issues ("http.playback.seek.*vorbis_theora_1_ogg", - "https://bugzilla.gnome.org/show_bug.cgi?id=769545"), + "https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/281"), # RTSP known issues ('rtsp.*playback.reverse.*', - 'https://bugzilla.gnome.org/show_bug.cgi?id=626811'), + 'https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/32'), ('rtsp.*playback.seek_with_stop.*', - 'https://bugzilla.gnome.org/show_bug.cgi?id=784298'), + 'https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/386'), ('rtsp.*playback.fast_*', - 'https://bugzilla.gnome.org/show_bug.cgi?id=754575'), + 'https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/14'), ]) def register_default_test_generators(self): diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index 35502449e8..835f85f2d1 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -45,6 +45,7 @@ from . import reporters from . import loggable from .loggable import Loggable +from collections import defaultdict try: from lxml import etree as ET except ImportError: @@ -1320,16 +1321,10 @@ class TestsManager(Loggable): if not self.expected_failures or not self.options.check_bugs_status: return True - if self.expected_failures: - printc("\nCurrently known failures in the %s testsuite:" - % self.name, Colors.WARNING, title_char='-') - - bugs_definitions = {} + bugs_definitions = defaultdict(list) for regex, failures in list(self.expected_failures.items()): for failure in failures: bugs = failure.get('bug') - if not bugs: - bugs = failure.get('bugs') if not bugs: printc('+ %s:\n --> no bug reported associated with %s\n' % ( regex.pattern, failure), Colors.WARNING) @@ -1339,7 +1334,7 @@ class TestsManager(Loggable): bugs = [bugs] cbugs = bugs_definitions.get(regex.pattern, []) bugs.extend([b for b in bugs if b not in cbugs]) - bugs_definitions[regex.pattern] = bugs + bugs_definitions[regex.pattern].extend(bugs) return check_bugs_resolution(bugs_definitions.items()) @@ -1630,6 +1625,9 @@ class _TestsLauncher(Loggable): if self._setup_testsuites() is False: return False + if self.options.check_bugs_status: + printc("-> Checking bugs resolution... ", end='') + for tester in self.testers: if not tester.set_blacklists(): return False @@ -1637,6 +1635,9 @@ class _TestsLauncher(Loggable): if not tester.check_expected_failures(): return False + if self.options.check_bugs_status: + printc("OK", Colors.OKGREEN) + if self.needs_http_server() or options.httponly is True: self.httpsrv = HTTPServer(options) self.httpsrv.start() diff --git a/validate/launcher/main.py b/validate/launcher/main.py index da0270dfff..2739260140 100644 --- a/validate/launcher/main.py +++ b/validate/launcher/main.py @@ -417,8 +417,7 @@ class LauncherConfig(Loggable): parser.add_argument("--check-bugs", dest="check_bugs_status", action="store_true", help="Check if the bug linked to blacklisted tests has" - " been marked as resolved. (only work with bugzilla " - "for the time being).") + " been marked as resolved. (works with gitlab and bugzilla)") parser.add_argument("-L", "--list-tests", dest="list_tests", action="store_true", diff --git a/validate/launcher/utils.py b/validate/launcher/utils.py index a8925f8d25..164da69efc 100644 --- a/validate/launcher/utils.py +++ b/validate/launcher/utils.py @@ -23,6 +23,7 @@ try: except ImportError: from . import config +import json import os import platform import re @@ -40,6 +41,7 @@ import urllib.parse from .loggable import Loggable from operator import itemgetter from xml.etree import ElementTree +from collections import defaultdict GST_SECOND = int(1000000000) @@ -439,8 +441,13 @@ class BackTraceGenerator(Loggable): return None +ALL_GITLAB_ISSUES = defaultdict(list) + + def check_bugs_resolution(bugs_definitions): bugz = {} + gitlab_issues = defaultdict(list) + regexes = {} for regex, bugs in bugs_definitions: if isinstance(bugs, str): @@ -449,15 +456,29 @@ def check_bugs_resolution(bugs_definitions): for bug in bugs: url = urllib.parse.urlparse(bug) + if "gitlab" in url.netloc: + components = [c for c in url.path.split('/') if c] + if len(components) != 4: + printc("\n + %s \n --> bug: %s\n --> Status: Not a proper gitlab report" % (regex, bug), + Colors.WARNING) + continue + project_id = components[0] + '%2F' + components[1] + issue_id = components[3] + + gitlab_url: str = "https://%s/api/v4/projects/%s/issues/%s" % (url.hostname, project_id, issue_id) + if gitlab_url in ALL_GITLAB_ISSUES: + continue + gitlab_issues[gitlab_url].append(regex) + ALL_GITLAB_ISSUES[gitlab_url].append(regex) + continue + if "bugzilla" not in url.netloc: - printc(" + %s \n --> bug: %s\n --> Status: Not a bugzilla report\n" % (regex, bug), - Colors.WARNING) continue query = urllib.parse.parse_qs(url.query) _id = query.get('id') if not _id: - printc(" + '%s' -- Can't check bug '%s'\n" % + printc("\n + '%s' -- Can't check bug '%s'" % (regex, bug), Colors.WARNING) continue @@ -471,6 +492,20 @@ def check_bugs_resolution(bugs_definitions): bugz[url_parts] = ids res = True + for gitlab_url, regexe in gitlab_issues.items(): + try: + issue = json.load(urllib.request.urlopen(gitlab_url)) + except Exception as e: + printc("\n + Could not properly check bugs status for: %s (%s)" + % (gitlab_url, e), Colors.FAIL) + continue + + if issue['state'] in ['closed']: + printc("\n + %s \n --> %s: '%s'\n ==> Bug CLOSED already (status: %s)" % ( + regexe, issue['web_url'], issue['title'], issue['state']), Colors.FAIL) + + res = False + for url_parts, ids in bugz.items(): url_parts = list(url_parts) query = {'id': ','.join(ids)} @@ -479,7 +514,7 @@ def check_bugs_resolution(bugs_definitions): try: res = urllib.request.urlopen(urllib.parse.urlunparse(url_parts)) except Exception as e: - printc(" + Could not properly check bugs status for: %s (%s)\n" + printc("\n + Could not properly check bugs status for: %s (%s)" % (urllib.parse.urlunparse(url_parts), e), Colors.FAIL) continue @@ -487,7 +522,7 @@ def check_bugs_resolution(bugs_definitions): bugs = root.findall('./bug') if len(bugs) != len(ids): - printc(" + Could not properly check bugs status on server %s\n" % + printc("\n + Could not properly check bugs status on server %s" % urllib.parse.urlunparse(url_parts), Colors.FAIL) continue @@ -498,19 +533,22 @@ def check_bugs_resolution(bugs_definitions): desc = bugelem.findtext('./short_desc') if not status: - printc(" + %s \n --> bug: %s\n --> Status: UNKNOWN\n" % (regex, bug), + printc("\n + %s \n --> bug: %s\n --> Status: UNKNOWN" % (regex, bug), Colors.WARNING) continue if not status.lower() in ['new', 'verified']: - printc(" + %s \n --> bug: #%s: '%s'\n ==> Bug CLOSED already (status: %s)\n" % ( + printc("\n + %s \n --> bug: #%s: '%s'\n ==> Bug CLOSED already (status: %s)" % ( regex, bugid, desc, status), Colors.WARNING) res = False - printc(" + %s \n --> bug: #%s: '%s'\n --> Status: %s\n" % ( + printc("\n + %s \n --> bug: #%s: '%s'\n --> Status: %s" % ( regex, bugid, desc, status), Colors.OKGREEN) + if not res: + printc("\n==> Some bugs marked as known issues have been closed!", Colors.FAIL) + return res