mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-01-11 18:05:37 +00:00
Code Issues 1 Projects Releases Wiki Activity

validate: tools: Cleanup test launcher tool

Browse source 
Previous commit was not meant to be pushed and those two should have
been fixed up together, sorry for the mistake
This commit is contained in:
Thibault Saunier 2013-12-31 11:45:07 +01:00
parent 8c39dbfaa8
commit e9b2967504
5 changed files with 166 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
validate/tools/apps/ges-projects-tests.py
View file

@ -21,8 +21,8 @@ import os
import time import time
from urllib import unquote from urllib import unquote
from urlparse import urlsplit from urlparse import urlsplit
from utils import launch_command
from gi.repository import GES, Gst, GLib from gi.repository import GES, Gst, GLib
from testdefinitions import Test, DEFAULT_QA_SAMPLE_PATH, TestsManager
DURATION_TOLERANCE = Gst.SECOND / 2 DURATION_TOLERANCE = Gst.SECOND / 2
DEFAULT_GES_LAUNCH = "ges-launch-1.0" DEFAULT_GES_LAUNCH = "ges-launch-1.0"
@ -110,8 +110,8 @@ def quote_uri(uri):
class GESTest(Test): class GESTest(Test):
def __init__(self, classname, options, reporter, project_uri, scenario, def __init__(self, classname, options, reporter, project_uri, scenario,
combination=None): combination=None):
super(GESTest, self).__init__(DEFAULT_GES_LAUNCH, classname, options, reporter) super(GESTest, self).__init__(DEFAULT_GES_LAUNCH, classname, options, reporter,
self.scenario = scenario scenario)
self.project_uri = project_uri self.project_uri = project_uri
self.combination = combination self.combination = combination
proj = GES.Project.new(project_uri) proj = GES.Project.new(project_uri)
@ -160,9 +160,7 @@ class GESTest(Test):
self.add_arguments("--sample-paths", "file://" + path) self.add_arguments("--sample-paths", "file://" + path)
def build_arguments(self): def build_arguments(self):
print "\OOO %s" % self.combination Test.build_arguments(self)
if self.scenario is not None:
self.add_arguments("--set-scenario", self.scenario)
if self.combination is not None: if self.combination is not None:
self.set_rendering_info() self.set_rendering_info()
@ -205,19 +203,7 @@ class GESTest(Test):
self.set_result(Result.TIMEOUT, "The rendered file add right duration, MISSING EOS?\n", self.set_result(Result.TIMEOUT, "The rendered file add right duration, MISSING EOS?\n",
"failure", e) "failure", e)
else: else:
if self.result == Result.TIMEOUT: Test.check_results(self)
self.set_result(Result.TIMEOUT, "Application timed out", "timeout")
else:
if self.process.returncode == 139:
self.get_backtrace("SEGFAULT")
self.set_result("Application segfaulted")
else:
self.set_result(Result.FAILED,
"Application returned %d (issues: %s)" % (
self.process.returncode,
self.get_validate_criticals_errors()),
"error")
def wait_process(self): def wait_process(self):
last_val = 0 last_val = 0
@ -263,41 +249,29 @@ class GESTest(Test):
class GESTestsManager(TestsManager): class GESTestsManager(TestsManager):
name = "ges"
def __init__(self): def __init__(self):
super(GESTestsManager, self).__init__() super(GESTestsManager, self).__init__()
Gst.init(None) Gst.init(None)
GES.init() GES.init()
default_opath = GLib.get_user_special_dir( def add_options(self, group):
GLib.UserDirectory.DIRECTORY_VIDEOS) group.add_option("-o", "--output-path", dest="dest",
if default_opath: default=None,
self.default_path = os.path.join(default_opath, "ges-projects") help="Set the path to which projects should be"
else: " renderd")
self.default_path = os.path.join(os.path.expanduser('~'), "Video", group.add_option("-P", "--projects-paths", dest="projects_paths",
"ges-projects") default=os.path.join(DEFAULT_QA_SAMPLE_PATH, "ges-projects"),
help="Paths in which to look for moved medias")
def add_options(self, parser): group.add_option("-r", "--recurse-paths", dest="recurse_paths",
parser.add_option("-o", "--output-path", dest="dest", default=False, action="store_true",
default=os.path.join(self.default_path, "rendered"), help="Whether to recurse into paths to find medias")
help="Set the path to which projects should be"
" renderd")
parser.add_option("-P", "--sample-path", dest="paths",
default=[],
help="Paths in which to look for moved assets")
parser.add_option("-r", "--recurse-paths", dest="recurse_paths",
default=False, action="store_true",
help="Whether to recurse into paths to find assets")
parser.add_option("-m", "--mute", dest="mute",
action="store_true", default=False,
help="Mute playback output, which mean that we use "
"a fakesink")
def set_settings(self, options, args, reporter): def set_settings(self, options, args, reporter):
TestsManager.set_settings(self, options, args, reporter) TestsManager.set_settings(self, options, args, reporter)
if not args and not os.path.exists(self.default_path):
launch_command("git clone %s" % DEFAULT_ASSET_REPO, if options.dest is None:
"Getting assets") options.dest = os.path.join(options.logsdir, "rendered")
if not Gst.uri_is_valid(options.dest): if not Gst.uri_is_valid(options.dest):
options.dest = GLib.filename_to_uri(options.dest, None) options.dest = GLib.filename_to_uri(options.dest, None)
@ -311,8 +285,8 @@ class GESTestsManager(TestsManager):
def list_tests(self): def list_tests(self):
projects = list() projects = list()
if not self.args: if not self.args:
self.options.paths = [os.path.join(self.default_path, "assets")] path = self.options.projects_paths
path = os.path.join(self.default_path, "projects") print path
for root, dirs, files in os.walk(path): for root, dirs, files in os.walk(path):
for f in files: for f in files:
if not f.endswith(".xges"): if not f.endswith(".xges"):

60
validate/tools/gst-validate-launcher.py
View file

@ -1,20 +1,42 @@
#!/usr//bin/python #!/usr//bin/python
#
# Copyright (c) 2013,Thibault Saunier <thibault.saunier@collabora.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
# Boston, MA 02110-1301, USA.
import os import os
from testdefinitions import _TestsLauncher import logging
from testdefinitions import _TestsLauncher, DEFAULT_QA_SAMPLE_PATH
from utils import printc
from optparse import OptionParser from optparse import OptionParser
def main(): def main():
parser = OptionParser() parser = OptionParser()
parser.add_option("-g", "--gdb", dest="gdb", # FIXME:
action="store_true", #parser.add_option("-g", "--gdb", dest="gdb",
default=False, #action="store_true",
help="Run applications into gdb") #default=False,
parser.add_option("-f", "--forever", dest="forever", #help="Run applications into gdb")
action="store_true", default=False, #parser.add_option("-f", "--forever", dest="forever",
help="Keep running tests until one fails") #action="store_true", default=False,
parser.add_option("-F", "--fatal-error", dest="fatal_error", #help="Keep running tests until one fails")
action="store_true", default=False, #parser.add_option("-F", "--fatal-error", dest="fatal_error",
help="Stop on first fail") #action="store_true", default=False,
#help="Stop on first fail")
parser.add_option('--xunit-file', action='store', parser.add_option('--xunit-file', action='store',
dest='xunit_file', metavar="FILE", dest='xunit_file', metavar="FILE",
default=None, default=None,
@ -31,6 +53,20 @@ def main():
parser.add_option("-l", "--logs-dir", dest="logsdir", parser.add_option("-l", "--logs-dir", dest="logsdir",
action="store_true", default=os.path.expanduser("~/gst-validate/logs/"), action="store_true", default=os.path.expanduser("~/gst-validate/logs/"),
help="Directory where to store logs") help="Directory where to store logs")
parser.add_option("-p", "--medias-paths", dest="paths",
default=[os.path.join(DEFAULT_QA_SAMPLE_PATH, "medias")],
help="Paths in which to look for media files")
parser.add_option("-m", "--mute", dest="mute",
action="store_true", default=False,
help="Mute playback output, which mean that we use "
"a fakesink")
try:
level = getattr(logging,
os.environ["GST_VALIDATE_LAUNCHER_DEBUG"].upper(),
None)
logging.basicConfig(level=level)
except:
pass
tests_launcher = _TestsLauncher() tests_launcher = _TestsLauncher()
tests_launcher.add_options(parser) tests_launcher.add_options(parser)
@ -41,7 +77,7 @@ def main():
tests_launcher.list_tests() tests_launcher.list_tests()
if options.list_tests: if options.list_tests:
for test in tests_launcher.tests: for test in tests_launcher.tests:
print test printc(test)
return 0 return 0
tests_launcher.run_tests() tests_launcher.run_tests()
tests_launcher.final_report() tests_launcher.final_report()

18
validate/tools/reporters.py
View file

@ -22,9 +22,9 @@
import os import os
import re import re
import codecs import codecs
import testdefinitions import logging
from xml.sax import saxutils from xml.sax import saxutils
from utils import mkdir, Result from utils import mkdir, Result, printc
UNICODE_STRINGS = (type(unicode()) == type(str())) UNICODE_STRINGS = (type(unicode()) == type(str()))
@ -75,6 +75,7 @@ class Reporter(object):
self.stats["passed"] += 1 self.stats["passed"] += 1
def add_results(self, test): def add_results(self, test):
logging.debug("%s", test)
if test.result == Result.PASSED: if test.result == Result.PASSED:
self.set_passed(test) self.set_passed(test)
elif test.result == Result.FAILED or \ elif test.result == Result.FAILED or \
@ -84,14 +85,15 @@ class Reporter(object):
raise UnknownResult("%s" % test.result) raise UnknownResult("%s" % test.result)
def after_test(self): def after_test(self):
self.out.close()
self.out = None
self.results.append(self._current_test) self.results.append(self._current_test)
self.add_results(self._current_test) self.add_results(self._current_test)
self.out.close()
self.out = None
self._current_test = None self._current_test = None
def final_report(self): def final_report(self):
pass for test in self.results:
printc(test)
class XunitReporter(Reporter): class XunitReporter(Reporter):
@ -106,6 +108,7 @@ class XunitReporter(Reporter):
def final_report(self): def final_report(self):
self.report() self.report()
super(XunitReporter, self).final_report()
def _get_captured(self): def _get_captured(self):
if self.out: if self.out:
@ -129,13 +132,12 @@ class XunitReporter(Reporter):
The file includes a report of test errors and failures. The file includes a report of test errors and failures.
""" """
print "Writing XML file to: %s" % self.options.xunit_file logging.debug("Writing XML file to: %s", self.options.xunit_file)
self.xml_file = codecs.open(self.options.xunit_file, 'w', self.xml_file = codecs.open(self.options.xunit_file, 'w',
self.encoding, 'replace') self.encoding, 'replace')
self.stats['encoding'] = self.encoding self.stats['encoding'] = self.encoding
self.stats['total'] = (self.stats['timeout'] + self.stats['failures'] self.stats['total'] = (self.stats['timeout'] + self.stats['failures']
+ self.stats['passes'] + self.stats['skipped']) + self.stats['passes'] + self.stats['skipped'])
print self.stats
self.xml_file.write( u'<?xml version="1.0" encoding="%(encoding)s"?>' self.xml_file.write( u'<?xml version="1.0" encoding="%(encoding)s"?>'
u'<testsuite name="gesprojectslauncher" tests="%(total)d" ' u'<testsuite name="gesprojectslauncher" tests="%(total)d" '
u'errors="%(timeout)d" failures="%(failures)d" ' u'errors="%(timeout)d" failures="%(failures)d" '
@ -149,7 +151,6 @@ class XunitReporter(Reporter):
"""Add failure output to Xunit report. """Add failure output to Xunit report.
""" """
self.stats['failures'] += 1 self.stats['failures'] += 1
self.results.insert(0, test)
self.errorlist.append( self.errorlist.append(
'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">' '<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
'<failure type=%(errtype)s message=%(message)s>' '<failure type=%(errtype)s message=%(message)s>'
@ -166,7 +167,6 @@ class XunitReporter(Reporter):
"""Add success output to Xunit report. """Add success output to Xunit report.
""" """
self.stats['passes'] += 1 self.stats['passes'] += 1
self.results.append(test)
self.errorlist.append( self.errorlist.append(
'<testcase classname=%(cls)s name=%(name)s ' '<testcase classname=%(cls)s name=%(name)s '
'time="%(taken).3f">%(systemout)s</testcase>' % 'time="%(taken).3f">%(systemout)s</testcase>' %

91
validate/tools/testdefinitions.py
View file

@ -24,37 +24,46 @@ import re
import time import time
import subprocess import subprocess
import reporters import reporters
import logging
from optparse import OptionGroup
from utils import mkdir, Result from utils import mkdir, Result, Colors, printc
DEFAULT_TIMEOUT = 10 DEFAULT_TIMEOUT = 10
DEFAULT_QA_SAMPLE_PATH = os.path.join(os.path.expanduser('~'), "Videos",
"gst-qa-samples")
class Test(object): class Test(object):
""" A class representing a particular test. """ """ A class representing a particular test. """
def __init__(self, application_name, classname, options, reporter): def __init__(self, application_name, classname, options, reporter, scenario=None, timeout=DEFAULT_TIMEOUT):
self.timeout = DEFAULT_TIMEOUT self.timeout = timeout
self.classname = classname self.classname = classname
self.options = options self.options = options
self.application = application_name self.application = application_name
self.command = "" self.command = ""
self.reporter = reporter self.reporter = reporter
self.process = None self.process = None
if scenario.lower() == "none":
self.scenario = None
else:
self.scenario = scenario
self.message = None self.message = ""
self.error = None self.error = ""
self.time_taken = None self.time_taken = 0.0
self._starting_time = None self._starting_time = None
self.result = Result.NOT_RUN self.result = Result.NOT_RUN
def __str__(self): def __str__(self):
string = self.classname string = self.classname
if self.result: if self.result != Result.NOT_RUN:
string += ": " + self.result string += ": " + self.result
if "FAILED" in self.result: if self.result == Result.FAILED:
string += "\n You can reproduce with: " + self.command string += "'%s'\n You can reproduce with: %s" \
% (self.message, self.command)
return string return string
@ -63,19 +72,32 @@ class Test(object):
self.command += " " + arg self.command += " " + arg
def build_arguments(self): def build_arguments(self):
pass if self.scenario is not None:
self.add_arguments("--set-scenario", self.scenario)
def set_result(self, result, message=None, error=None): def set_result(self, result, message="", error=""):
print "SETTING TER"
self.result = result self.result = result
self.message = message self.message = message
self.error = error self.error = error
def check_results(self): def check_results(self):
if self.process.returncode == 0: logging.debug("%s returncode: %d", self, self.process.returncode)
if self.result == Result.TIMEOUT:
self.set_result(Result.TIMEOUT, "Application timed out", "timeout")
elif self.process.returncode == 0:
self.result = Result.PASSED self.result = Result.PASSED
else:
self.result = Result.FAILED if self.process.returncode == 139:
self.get_backtrace("SEGFAULT")
self.set_result(Result.FAILED,
"Application segfaulted",
"segfault")
else:
self.set_result(Result.FAILED,
"Application returned %d (issues: %s)" % (
self.process.returncode,
self.get_validate_criticals_errors()),
"error")
def wait_process(self): def wait_process(self):
last_change_ts = time.time() last_change_ts = time.time()
@ -96,11 +118,16 @@ class Test(object):
def get_validate_criticals_errors(self): def get_validate_criticals_errors(self):
self.reporter.out.seek(0) self.reporter.out.seek(0)
ret = "[" ret = "["
errors = []
for l in self.reporter.out.readlines(): for l in self.reporter.out.readlines():
if "critical : " in l: if "critical : " in l:
if ret != "[": if ret != "[":
ret += ", " ret += ", "
ret += l.split("critical : ")[1].replace("\n", '') error = l.split("critical : ")[1].replace("\n", '')
print "%s -- %s" %(error, errors)
if error not in errors:
ret += error
errors.append(error)
if ret == "[": if ret == "[":
return "No critical" return "No critical"
@ -111,7 +138,8 @@ class Test(object):
self.command = "%s " % (self.application) self.command = "%s " % (self.application)
self._starting_time = time.time() self._starting_time = time.time()
self.build_arguments() self.build_arguments()
print "Launching %s" % self.command printc("Launching:%s '%s' -- logs are in %s" % (Colors.ENDC, self.classname,
self.reporter.out.name), Colors.OKBLUE)
try: try:
self.process = subprocess.Popen(self.command, self.process = subprocess.Popen(self.command,
stderr=self.reporter.out, stderr=self.reporter.out,
@ -134,6 +162,8 @@ class TestsManager(object):
""" A class responsible for managing tests. """ """ A class responsible for managing tests. """
name = ""
def __init__(self): def __init__(self):
self.tests = [] self.tests = []
self.options = None self.options = None
@ -163,6 +193,9 @@ class TestsManager(object):
def _is_test_wanted(self, test): def _is_test_wanted(self, test):
if not self.wanted_tests_patterns:
return True
for pattern in self.wanted_tests_patterns: for pattern in self.wanted_tests_patterns:
if pattern.findall(test.classname): if pattern.findall(test.classname):
return True return True
@ -190,7 +223,7 @@ class _TestsLauncher(object):
subclasses = [] subclasses = []
for symb in env.iteritems(): for symb in env.iteritems():
try: try:
if issubclass(symb[1], c): if issubclass(symb[1], c) and not symb[1] is c:
subclasses.append(symb[1]) subclasses.append(symb[1])
except TypeError: except TypeError:
pass pass
@ -201,16 +234,34 @@ class _TestsLauncher(object):
d = os.path.dirname(__file__) d = os.path.dirname(__file__)
for f in os.listdir(os.path.join(d, "apps")): for f in os.listdir(os.path.join(d, "apps")):
execfile(os.path.join(d, "apps", f), env) execfile(os.path.join(d, "apps", f), env)
self.testers = [i() for i in get_subclasses(TestsManager, env)] self.testers = [i() for i in get_subclasses(TestsManager, env)]
print self.testers
def add_options(self, parser): def add_options(self, parser):
for tester in self.testers: for tester in self.testers:
tester.add_options(parser) group = OptionGroup(parser, "%s Options" % tester.name,
"Options specific to the %s test manager"
% tester.name)
tester.add_options(group)
parser.add_option_group(group)
def set_settings(self, options, args): def set_settings(self, options, args):
self.reporter = reporters.XunitReporter(options) self.reporter = reporters.XunitReporter(options)
mkdir(options.logsdir) mkdir(options.logsdir)
wanted_testers = None
for tester in self.testers:
if tester.name in args:
wanted_testers = tester.name
if wanted_testers:
testers = self.testers
self.testers = []
for tester in testers:
if tester.name in args:
self.testers.append(tester)
args.remove(tester.name)
for tester in self.testers: for tester in self.testers:
tester.set_settings(options, args, self.reporter) tester.set_settings(options, args, self.reporter)

23
validate/tools/utils.py
View file

@ -53,11 +53,20 @@ def mkdir(directory):
pass pass
def launch_command(command, name="", color=None): def printc (message, color="", title=False):
if name != "": if title:
if color is not None: message = len(message) * '=' + message + len(message) * '='
print "%s%s" % (color, len(name) * "=") if hasattr(message, "result") and color == '':
print name if message.result == Result.FAILED:
if color is not None: color = Colors.FAIL
print "%s%s" % (len(name) * "=", Colors.ENDC) elif message.result == Result.PASSED:
color = Colors.OKGREEN
else:
color = Colors.OKBLUE
print color + str(message) + Colors.ENDC
def launch_command(command, color=None):
printc(command, Colors.OKGREEN, True)
os.system(command) os.system(command)