gstreamer/validate/tools/gst-validate-analyze

#!/usr/bin/env python2
#
# Copyright (c) 2015, Edward Hervey <edward@centricular.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
# Boston, MA 02110-1301, USA.

import os
import sys
import xml.etree.cElementTree

def extract_info(xmlfile):
    e = xml.etree.cElementTree.parse(xmlfile).getroot()
    r = {}
    for i in e:
        r[(i.get("classname"), i.get("name"))] = i
    return r

if "__main__" == __name__:
    if len(sys.argv) < 2:
        print "Usage : %s [<old run xml>] <new run xml>" % sys.argv[0]
    if len(sys.argv) == 3:
        oldfile = extract_info(sys.argv[1])
        newfile = extract_info(sys.argv[2])
    else:
        oldfile = []
        newfile = extract_info(sys.argv[1])

    # new failures (pass in old run, fail in new run)
    newfail = []
    # new fixes (fail in old run, pass in new run)
    newfix = []
    # tests that are still failing
    stillfail = []
    # tests that are still failling but for a different reason
    failchange = []

    # failed tests sorted by reason
    reasons = {}

    # all files
    allfiles = []

    # failed tests sorted by file
    failedfiles = {}

    if oldfile:
        # tests that weren't present in old run
        newtests = [x for x in newfile.keys() if not oldfile.has_key(x)]
        # tests that are no longer present in new run
        gonetests = [x for x in oldfile.keys() if not newfile.has_key(x)]

    # go over new tests
    for k,v in newfile.iteritems():
        tn, fn = k
        if not fn in allfiles:
            allfiles.append(fn)

        newf = v.findall("failure")
        if newf:
            # extract the failure reason
            r = newf[0].get("message")
            if "Application returned 18" in r or "Application returned -5" in r:
                rs = r.split('[')[1].split(']')[0].split(',')
                for la in rs:
                    la = la.strip()
                    if not reasons.has_key(la):
                        reasons[la] = []
                    reasons[la].append(k)
            if not failedfiles.has_key(fn):
                failedfiles[fn] = []
            failedfiles[fn].append((tn, r))

        if k in oldfile:
            oldone = oldfile.get(k)

            # compare failures
            oldf = oldone.findall("failure")
            if newf and not oldf:
                newfail.append(k)
            if oldf and not newf:
                newfix.append(k)
            if oldf and newf:
                stillfail.append(k)
                a = oldf[0]
                b = newf[0]
                # check if the failure reasons are the same
                if a.get("type") != b.get("type"):
                    failchange.append(k)
                elif a.get("message") != b.get("message"):
                    failchange.append(k)


    if newfail:
        print "New failures", len(newfail)
        newfail.sort()
        for i in newfail:
            print "   %s : %s" % (i[0], i[1])
            f = newfile[i].find("failure")
            print "     ", f.get("type"), f.get("message")
        print

    if newfix:
        print "New fixes", len(newfix)
        newfix.sort()
        for i in newfix:
            print "   %s : %s" % (i[0], i[1])
        print

    if failchange:
        print "Failure changes", len(failchange)
        failchange.sort()
        for i in failchange:
            print "   %s : %s" % (i[0], i[1])
            oldt = oldfile[i].find("failure").get("type")
            newt = newfile[i].find("failure").get("type")
            if oldt != newt:
                print "       Went from '%s' to '%s'" % (oldt, newt)
            print "       Previous message :", oldfile[i].find("failure").get("message")
            print "       New message      :", newfile[i].find("failure").get("message")
        print

    for k,v in reasons.iteritems():
        print "Failure type : ", k, len(v)
        v.sort()
        for i in v:
            print "   %s : %s" % (i[0], i[1])
        print

    nofailfiles = [fn for fn in allfiles if not failedfiles.has_key(fn)]
    nofailfiles.sort()
    if nofailfiles:
        print "Files without failures", len(nofailfiles)
        for f in nofailfiles:
            print "    ", f
        print

    for k,v in failedfiles.iteritems():
        print "Failed File :", k
        for i in v:
            print "    %s : %s" % (i[0], i[1])