diff --git a/validate/tools/apps/ges-projects-tests.py b/validate/tools/apps/ges-projects-tests.py index 54fded71a2..6a16bfff25 100644 --- a/validate/tools/apps/ges-projects-tests.py +++ b/validate/tools/apps/ges-projects-tests.py @@ -21,8 +21,8 @@ import os import time from urllib import unquote from urlparse import urlsplit -from utils import launch_command from gi.repository import GES, Gst, GLib +from testdefinitions import Test, DEFAULT_QA_SAMPLE_PATH, TestsManager DURATION_TOLERANCE = Gst.SECOND / 2 DEFAULT_GES_LAUNCH = "ges-launch-1.0" @@ -110,8 +110,8 @@ def quote_uri(uri): class GESTest(Test): def __init__(self, classname, options, reporter, project_uri, scenario, combination=None): - super(GESTest, self).__init__(DEFAULT_GES_LAUNCH, classname, options, reporter) - self.scenario = scenario + super(GESTest, self).__init__(DEFAULT_GES_LAUNCH, classname, options, reporter, + scenario) self.project_uri = project_uri self.combination = combination proj = GES.Project.new(project_uri) @@ -160,9 +160,7 @@ class GESTest(Test): self.add_arguments("--sample-paths", "file://" + path) def build_arguments(self): - print "\OOO %s" % self.combination - if self.scenario is not None: - self.add_arguments("--set-scenario", self.scenario) + Test.build_arguments(self) if self.combination is not None: self.set_rendering_info() @@ -205,19 +203,7 @@ class GESTest(Test): self.set_result(Result.TIMEOUT, "The rendered file add right duration, MISSING EOS?\n", "failure", e) else: - if self.result == Result.TIMEOUT: - self.set_result(Result.TIMEOUT, "Application timed out", "timeout") - else: - if self.process.returncode == 139: - self.get_backtrace("SEGFAULT") - self.set_result("Application segfaulted") - else: - self.set_result(Result.FAILED, - "Application returned %d (issues: %s)" % ( - self.process.returncode, - self.get_validate_criticals_errors()), - "error") - + Test.check_results(self) def wait_process(self): last_val = 0 @@ -263,41 +249,29 @@ class GESTest(Test): class GESTestsManager(TestsManager): + name = "ges" def __init__(self): super(GESTestsManager, self).__init__() Gst.init(None) GES.init() - default_opath = GLib.get_user_special_dir( - GLib.UserDirectory.DIRECTORY_VIDEOS) - if default_opath: - self.default_path = os.path.join(default_opath, "ges-projects") - else: - self.default_path = os.path.join(os.path.expanduser('~'), "Video", - "ges-projects") - - def add_options(self, parser): - parser.add_option("-o", "--output-path", dest="dest", - default=os.path.join(self.default_path, "rendered"), - help="Set the path to which projects should be" - " renderd") - parser.add_option("-P", "--sample-path", dest="paths", - default=[], - help="Paths in which to look for moved assets") - parser.add_option("-r", "--recurse-paths", dest="recurse_paths", - default=False, action="store_true", - help="Whether to recurse into paths to find assets") - parser.add_option("-m", "--mute", dest="mute", - action="store_true", default=False, - help="Mute playback output, which mean that we use " - "a fakesink") - + def add_options(self, group): + group.add_option("-o", "--output-path", dest="dest", + default=None, + help="Set the path to which projects should be" + " renderd") + group.add_option("-P", "--projects-paths", dest="projects_paths", + default=os.path.join(DEFAULT_QA_SAMPLE_PATH, "ges-projects"), + help="Paths in which to look for moved medias") + group.add_option("-r", "--recurse-paths", dest="recurse_paths", + default=False, action="store_true", + help="Whether to recurse into paths to find medias") def set_settings(self, options, args, reporter): TestsManager.set_settings(self, options, args, reporter) - if not args and not os.path.exists(self.default_path): - launch_command("git clone %s" % DEFAULT_ASSET_REPO, - "Getting assets") + + if options.dest is None: + options.dest = os.path.join(options.logsdir, "rendered") if not Gst.uri_is_valid(options.dest): options.dest = GLib.filename_to_uri(options.dest, None) @@ -311,8 +285,8 @@ class GESTestsManager(TestsManager): def list_tests(self): projects = list() if not self.args: - self.options.paths = [os.path.join(self.default_path, "assets")] - path = os.path.join(self.default_path, "projects") + path = self.options.projects_paths + print path for root, dirs, files in os.walk(path): for f in files: if not f.endswith(".xges"): diff --git a/validate/tools/gst-validate-launcher.py b/validate/tools/gst-validate-launcher.py index 23cf52412a..5ee63025e8 100644 --- a/validate/tools/gst-validate-launcher.py +++ b/validate/tools/gst-validate-launcher.py @@ -1,20 +1,42 @@ #!/usr//bin/python +# +# Copyright (c) 2013,Thibault Saunier +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301, USA. + import os -from testdefinitions import _TestsLauncher +import logging +from testdefinitions import _TestsLauncher, DEFAULT_QA_SAMPLE_PATH +from utils import printc from optparse import OptionParser + def main(): parser = OptionParser() - parser.add_option("-g", "--gdb", dest="gdb", - action="store_true", - default=False, - help="Run applications into gdb") - parser.add_option("-f", "--forever", dest="forever", - action="store_true", default=False, - help="Keep running tests until one fails") - parser.add_option("-F", "--fatal-error", dest="fatal_error", - action="store_true", default=False, - help="Stop on first fail") + # FIXME: + #parser.add_option("-g", "--gdb", dest="gdb", + #action="store_true", + #default=False, + #help="Run applications into gdb") + #parser.add_option("-f", "--forever", dest="forever", + #action="store_true", default=False, + #help="Keep running tests until one fails") + #parser.add_option("-F", "--fatal-error", dest="fatal_error", + #action="store_true", default=False, + #help="Stop on first fail") parser.add_option('--xunit-file', action='store', dest='xunit_file', metavar="FILE", default=None, @@ -31,6 +53,20 @@ def main(): parser.add_option("-l", "--logs-dir", dest="logsdir", action="store_true", default=os.path.expanduser("~/gst-validate/logs/"), help="Directory where to store logs") + parser.add_option("-p", "--medias-paths", dest="paths", + default=[os.path.join(DEFAULT_QA_SAMPLE_PATH, "medias")], + help="Paths in which to look for media files") + parser.add_option("-m", "--mute", dest="mute", + action="store_true", default=False, + help="Mute playback output, which mean that we use " + "a fakesink") + try: + level = getattr(logging, + os.environ["GST_VALIDATE_LAUNCHER_DEBUG"].upper(), + None) + logging.basicConfig(level=level) + except: + pass tests_launcher = _TestsLauncher() tests_launcher.add_options(parser) @@ -41,7 +77,7 @@ def main(): tests_launcher.list_tests() if options.list_tests: for test in tests_launcher.tests: - print test + printc(test) return 0 tests_launcher.run_tests() tests_launcher.final_report() diff --git a/validate/tools/reporters.py b/validate/tools/reporters.py index 1917856815..6348747b3c 100644 --- a/validate/tools/reporters.py +++ b/validate/tools/reporters.py @@ -22,9 +22,9 @@ import os import re import codecs -import testdefinitions +import logging from xml.sax import saxutils -from utils import mkdir, Result +from utils import mkdir, Result, printc UNICODE_STRINGS = (type(unicode()) == type(str())) @@ -75,6 +75,7 @@ class Reporter(object): self.stats["passed"] += 1 def add_results(self, test): + logging.debug("%s", test) if test.result == Result.PASSED: self.set_passed(test) elif test.result == Result.FAILED or \ @@ -84,14 +85,15 @@ class Reporter(object): raise UnknownResult("%s" % test.result) def after_test(self): - self.out.close() - self.out = None self.results.append(self._current_test) self.add_results(self._current_test) + self.out.close() + self.out = None self._current_test = None def final_report(self): - pass + for test in self.results: + printc(test) class XunitReporter(Reporter): @@ -106,6 +108,7 @@ class XunitReporter(Reporter): def final_report(self): self.report() + super(XunitReporter, self).final_report() def _get_captured(self): if self.out: @@ -129,13 +132,12 @@ class XunitReporter(Reporter): The file includes a report of test errors and failures. """ - print "Writing XML file to: %s" % self.options.xunit_file + logging.debug("Writing XML file to: %s", self.options.xunit_file) self.xml_file = codecs.open(self.options.xunit_file, 'w', self.encoding, 'replace') self.stats['encoding'] = self.encoding self.stats['total'] = (self.stats['timeout'] + self.stats['failures'] + self.stats['passes'] + self.stats['skipped']) - print self.stats self.xml_file.write( u'' u'' '' @@ -166,7 +167,6 @@ class XunitReporter(Reporter): """Add success output to Xunit report. """ self.stats['passes'] += 1 - self.results.append(test) self.errorlist.append( '%(systemout)s' % diff --git a/validate/tools/testdefinitions.py b/validate/tools/testdefinitions.py index 7771624c1e..964dc6a6bc 100644 --- a/validate/tools/testdefinitions.py +++ b/validate/tools/testdefinitions.py @@ -24,37 +24,46 @@ import re import time import subprocess import reporters +import logging +from optparse import OptionGroup -from utils import mkdir, Result +from utils import mkdir, Result, Colors, printc DEFAULT_TIMEOUT = 10 +DEFAULT_QA_SAMPLE_PATH = os.path.join(os.path.expanduser('~'), "Videos", + "gst-qa-samples") class Test(object): """ A class representing a particular test. """ - def __init__(self, application_name, classname, options, reporter): - self.timeout = DEFAULT_TIMEOUT + def __init__(self, application_name, classname, options, reporter, scenario=None, timeout=DEFAULT_TIMEOUT): + self.timeout = timeout self.classname = classname self.options = options self.application = application_name self.command = "" self.reporter = reporter self.process = None + if scenario.lower() == "none": + self.scenario = None + else: + self.scenario = scenario - self.message = None - self.error = None - self.time_taken = None + self.message = "" + self.error = "" + self.time_taken = 0.0 self._starting_time = None self.result = Result.NOT_RUN def __str__(self): string = self.classname - if self.result: + if self.result != Result.NOT_RUN: string += ": " + self.result - if "FAILED" in self.result: - string += "\n You can reproduce with: " + self.command + if self.result == Result.FAILED: + string += "'%s'\n You can reproduce with: %s" \ + % (self.message, self.command) return string @@ -63,19 +72,32 @@ class Test(object): self.command += " " + arg def build_arguments(self): - pass + if self.scenario is not None: + self.add_arguments("--set-scenario", self.scenario) - def set_result(self, result, message=None, error=None): - print "SETTING TER" + def set_result(self, result, message="", error=""): self.result = result self.message = message self.error = error def check_results(self): - if self.process.returncode == 0: + logging.debug("%s returncode: %d", self, self.process.returncode) + if self.result == Result.TIMEOUT: + self.set_result(Result.TIMEOUT, "Application timed out", "timeout") + elif self.process.returncode == 0: self.result = Result.PASSED - - self.result = Result.FAILED + else: + if self.process.returncode == 139: + self.get_backtrace("SEGFAULT") + self.set_result(Result.FAILED, + "Application segfaulted", + "segfault") + else: + self.set_result(Result.FAILED, + "Application returned %d (issues: %s)" % ( + self.process.returncode, + self.get_validate_criticals_errors()), + "error") def wait_process(self): last_change_ts = time.time() @@ -96,11 +118,16 @@ class Test(object): def get_validate_criticals_errors(self): self.reporter.out.seek(0) ret = "[" + errors = [] for l in self.reporter.out.readlines(): if "critical : " in l: if ret != "[": ret += ", " - ret += l.split("critical : ")[1].replace("\n", '') + error = l.split("critical : ")[1].replace("\n", '') + print "%s -- %s" %(error, errors) + if error not in errors: + ret += error + errors.append(error) if ret == "[": return "No critical" @@ -111,7 +138,8 @@ class Test(object): self.command = "%s " % (self.application) self._starting_time = time.time() self.build_arguments() - print "Launching %s" % self.command + printc("Launching:%s '%s' -- logs are in %s" % (Colors.ENDC, self.classname, + self.reporter.out.name), Colors.OKBLUE) try: self.process = subprocess.Popen(self.command, stderr=self.reporter.out, @@ -134,6 +162,8 @@ class TestsManager(object): """ A class responsible for managing tests. """ + name = "" + def __init__(self): self.tests = [] self.options = None @@ -163,6 +193,9 @@ class TestsManager(object): def _is_test_wanted(self, test): + if not self.wanted_tests_patterns: + return True + for pattern in self.wanted_tests_patterns: if pattern.findall(test.classname): return True @@ -190,7 +223,7 @@ class _TestsLauncher(object): subclasses = [] for symb in env.iteritems(): try: - if issubclass(symb[1], c): + if issubclass(symb[1], c) and not symb[1] is c: subclasses.append(symb[1]) except TypeError: pass @@ -201,16 +234,34 @@ class _TestsLauncher(object): d = os.path.dirname(__file__) for f in os.listdir(os.path.join(d, "apps")): execfile(os.path.join(d, "apps", f), env) + self.testers = [i() for i in get_subclasses(TestsManager, env)] - print self.testers + def add_options(self, parser): for tester in self.testers: - tester.add_options(parser) + group = OptionGroup(parser, "%s Options" % tester.name, + "Options specific to the %s test manager" + % tester.name) + tester.add_options(group) + parser.add_option_group(group) def set_settings(self, options, args): self.reporter = reporters.XunitReporter(options) mkdir(options.logsdir) + + wanted_testers = None + for tester in self.testers: + if tester.name in args: + wanted_testers = tester.name + if wanted_testers: + testers = self.testers + self.testers = [] + for tester in testers: + if tester.name in args: + self.testers.append(tester) + args.remove(tester.name) + for tester in self.testers: tester.set_settings(options, args, self.reporter) diff --git a/validate/tools/utils.py b/validate/tools/utils.py index a7eee76fd6..d32237e3a5 100644 --- a/validate/tools/utils.py +++ b/validate/tools/utils.py @@ -53,11 +53,20 @@ def mkdir(directory): pass -def launch_command(command, name="", color=None): - if name != "": - if color is not None: - print "%s%s" % (color, len(name) * "=") - print name - if color is not None: - print "%s%s" % (len(name) * "=", Colors.ENDC) +def printc (message, color="", title=False): + if title: + message = len(message) * '=' + message + len(message) * '=' + if hasattr(message, "result") and color == '': + if message.result == Result.FAILED: + color = Colors.FAIL + elif message.result == Result.PASSED: + color = Colors.OKGREEN + else: + color = Colors.OKBLUE + + print color + str(message) + Colors.ENDC + + +def launch_command(command, color=None): + printc(command, Colors.OKGREEN, True) os.system(command)