mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-12-19 23:06:49 +00:00
tools: New tool to view and compare xunit results
This commit is contained in:
Edward Hervey
Edward Hervey
parent
c4091f82a9
commit
98e31f52ef
1 changed files with 154 additions and 0 deletions
154
validate/tools/gst-validate-analyze
Executable file
154
validate/tools/gst-validate-analyze
Executable file
|
|
@ -0,0 +1,154 @@
|
|||
#!/usr/bin/env python2
|
||||
#
|
||||
# Copyright (c) 2015, Edward Hervey <edward@centricular.com>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this program; if not, write to the
|
||||
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
||||
# Boston, MA 02110-1301, USA.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import xml.etree.cElementTree
|
||||
|
||||
def extract_info(xmlfile):
|
||||
e = xml.etree.cElementTree.parse(xmlfile).getroot()
|
||||
r = {}
|
||||
for i in e:
|
||||
r[(i.get("classname"), i.get("name"))] = i
|
||||
return r
|
||||
|
||||
if "__main__" == __name__:
|
||||
if len(sys.argv) < 2:
|
||||
print "Usage : %s [<old run xml>] <new run xml>" % sys.argv[0]
|
||||
if len(sys.argv) == 3:
|
||||
oldfile = extract_info(sys.argv[1])
|
||||
newfile = extract_info(sys.argv[2])
|
||||
else:
|
||||
oldfile = []
|
||||
newfile = extract_info(sys.argv[1])
|
||||
|
||||
# new failures (pass in old run, fail in new run)
|
||||
newfail = []
|
||||
# new fixes (fail in old run, pass in new run)
|
||||
newfix = []
|
||||
# tests that are still failing
|
||||
stillfail = []
|
||||
# tests that are still failling but for a different reason
|
||||
failchange = []
|
||||
|
||||
# failed tests sorted by reason
|
||||
reasons = {}
|
||||
|
||||
# all files
|
||||
allfiles = []
|
||||
|
||||
# failed tests sorted by file
|
||||
failedfiles = {}
|
||||
|
||||
if oldfile:
|
||||
# tests that weren't present in old run
|
||||
newtests = [x for x in newfile.keys() if not oldfile.has_key(x)]
|
||||
# tests that are no longer present in new run
|
||||
gonetests = [x for x in oldfile.keys() if not newfile.has_key(x)]
|
||||
|
||||
# go over new tests
|
||||
for k,v in newfile.iteritems():
|
||||
tn, fn = k
|
||||
if not fn in allfiles:
|
||||
allfiles.append(fn)
|
||||
|
||||
newf = v.findall("failure")
|
||||
if newf:
|
||||
# extract the failure reason
|
||||
r = newf[0].get("message")
|
||||
if "Application returned 18" in r or "Application returned -5" in r:
|
||||
rs = r.split('[')[1].split(']')[0].split(',')
|
||||
for la in rs:
|
||||
la = la.strip()
|
||||
if not reasons.has_key(la):
|
||||
reasons[la] = []
|
||||
reasons[la].append(k)
|
||||
if not failedfiles.has_key(fn):
|
||||
failedfiles[fn] = []
|
||||
failedfiles[fn].append((tn, r))
|
||||
|
||||
if k in oldfile:
|
||||
oldone = oldfile.get(k)
|
||||
|
||||
# compare failures
|
||||
oldf = oldone.findall("failure")
|
||||
if newf and not oldf:
|
||||
newfail.append(k)
|
||||
if oldf and not newf:
|
||||
newfix.append(k)
|
||||
if oldf and newf:
|
||||
stillfail.append(k)
|
||||
a = oldf[0]
|
||||
b = newf[0]
|
||||
# check if the failure reasons are the same
|
||||
if a.get("type") != b.get("type"):
|
||||
failchange.append(k)
|
||||
elif a.get("message") != b.get("message"):
|
||||
failchange.append(k)
|
||||
|
||||
|
||||
if newfail:
|
||||
print "New failures", len(newfail)
|
||||
newfail.sort()
|
||||
for i in newfail:
|
||||
print " %s : %s" % (i[0], i[1])
|
||||
f = newfile[i].find("failure")
|
||||
print " ", f.get("type"), f.get("message")
|
||||
print
|
||||
|
||||
if newfix:
|
||||
print "New fixes", len(newfix)
|
||||
newfix.sort()
|
||||
for i in newfix:
|
||||
print " %s : %s" % (i[0], i[1])
|
||||
print
|
||||
|
||||
if failchange:
|
||||
print "Failure changes", len(failchange)
|
||||
failchange.sort()
|
||||
for i in failchange:
|
||||
print " %s : %s" % (i[0], i[1])
|
||||
oldt = oldfile[i].find("failure").get("type")
|
||||
newt = newfile[i].find("failure").get("type")
|
||||
if oldt != newt:
|
||||
print " Went from '%s' to '%s'" % (oldt, newt)
|
||||
print " Previous message :", oldfile[i].find("failure").get("message")
|
||||
print " New message :", newfile[i].find("failure").get("message")
|
||||
print
|
||||
|
||||
for k,v in reasons.iteritems():
|
||||
print "Failure type : ", k, len(v)
|
||||
v.sort()
|
||||
for i in v:
|
||||
print " %s : %s" % (i[0], i[1])
|
||||
print
|
||||
|
||||
nofailfiles = [fn for fn in allfiles if not failedfiles.has_key(fn)]
|
||||
nofailfiles.sort()
|
||||
if nofailfiles:
|
||||
print "Files without failures", len(nofailfiles)
|
||||
for f in nofailfiles:
|
||||
print " ", f
|
||||
print
|
||||
|
||||
for k,v in failedfiles.iteritems():
|
||||
print "Failed File :", k
|
||||
for i in v:
|
||||
print " %s : %s" % (i[0], i[1])
|
||||
|
||||
Loading…
Reference in a new issue