diff --git a/validate/configure.ac b/validate/configure.ac index abd6819899..f229cb5dc8 100644 --- a/validate/configure.ac +++ b/validate/configure.ac @@ -342,6 +342,7 @@ po/Makefile.in tools/Makefile launcher/Makefile launcher/apps/Makefile +launcher/testsuites/Makefile docs/Makefile docs/version.entities docs/validate/Makefile diff --git a/validate/launcher/Makefile.am b/validate/launcher/Makefile.am index b4fd617658..6aaa5f8a20 100644 --- a/validate/launcher/Makefile.am +++ b/validate/launcher/Makefile.am @@ -1,7 +1,8 @@ launcherdir = $(libdir)/gst-validate-launcher/python/launcher/ SUBDIRS = \ - apps + apps \ + testsuites launcher_PYTHON = \ baseclasses.py \ diff --git a/validate/launcher/apps/gstcheck.py b/validate/launcher/apps/gstcheck.py new file mode 100644 index 0000000000..5655d6cb1e --- /dev/null +++ b/validate/launcher/apps/gstcheck.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2016,Thibault Saunier +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301, USA. +import argparse +import config +import os +import pickle +import platform +import shutil +import threading +import concurrent.futures as conc + +from launcher.utils import printc, Colors + + +class MesonTest(Test): + + def __init__(self, name, options, reporter, test, child_env=None): + if child_env is None: + child_env = dict() + if not isinstance(test.env, dict): + test.env = test.env.get_env(child_env) + child_env.update(test.env) + if len(test.extra_paths) > 0: + child_env['PATH'] = child_env['PATH'] + \ + ';'.join([''] + test.extra_paths) + self.child_env = child_env + + timeout = int(child_env.pop('CK_DEFAULT_TIMEOUT', test.timeout)) + + Test.__init__(self, test.fname[0], name, options, + reporter, timeout=timeout, hard_timeout=timeout) + + self.mesontest = test + + def build_arguments(self): + self.add_arguments(*self.mesontest.fname[1:]) + self.add_arguments(*self.mesontest.cmd_args) + + def get_subproc_env(self): + env = os.environ.copy() + env.update(self.child_env) + # No reason to fork since we are launching + # each test individually + env['CK_FORK'] = 'no' + for var, val in self.child_env.items(): + self.add_env_variable(var, val) + + return env + + +class MesonTestsManager(TestsManager): + name = "mesontest" + arggroup = None + + def __init__(self): + super().__init__() + self.rebuilt = None + + def add_options(self, parser): + if self.arggroup: + return + + MesonTestsManager.arggroup = parser.add_argument_group( + "meson tests specific options and behaviours") + parser.add_argument("--meson-build-dir", + action="append", + dest='meson_build_dirs', + default=[config.BUILDDIR], + help="defines the paths to look for GstValidate tools.") + parser.add_argument("--meson-no-rebuild", + action="store_true", + default=False, + help="Whether to avoid to rebuild tests before running them.") + + def get_meson_tests(self): + mesontests = [] + for i, bdir in enumerate(self.options.meson_build_dirs): + bdir = os.path.abspath(bdir) + datafile = os.path.join( + bdir, 'meson-private/meson_test_setup.dat') + + if not os.path.isfile(datafile): + self.error("%s does not exists, can't use meson test launcher", + datafile) + continue + + with open(datafile, 'rb') as f: + tests = pickle.load(f) + mesontests.extend(tests) + + return mesontests + + def rebuild(self, all=False): + if self.options.meson_no_rebuild: + return True + + if self.rebuilt is not None: + return self.rebuilt + + for bdir in self.options.meson_build_dirs: + if not os.path.isfile(os.path.join(bdir, 'build.ninja')): + printc("Only ninja backend is supported to rebuilt tests before running them.\n", + Colors.OKBLUE) + self.rebuilt = True + return True + + ninja = shutil.which('ninja') + if not ninja: + ninja = shutil.which('ninja-build') + if not ninja: + printc("Can't find ninja, can't rebuild test.\n", Colors.FAIL) + self.rebuilt = False + return False + + print("-> Rebuilding %s.\n" % bdir) + try: + subprocess.check_call([ninja, '-C', bdir]) + except subprocess.CalledProcessError: + self.rebuilt = False + return False + + self.rebuilt = True + return True + + def run_tests(self, starting_test_num, total_num_tests): + if not self.rebuild(): + self.error("Rebuilding FAILED!") + return Result.FAILED + + return TestsManager.run_tests(self, starting_test_num, total_num_tests) + + def get_test_name(self, test): + name = test.name.replace('/', '.') + if test.suite: + name = '.'.join(test.suite) + '.' + name + + return self.name + '.' + name + + def list_tests(self): + if self.tests: + return self.tests + + mesontests = self.get_meson_tests() + for test in mesontests: + self.add_test(MesonTest(self.get_test_name(test), + self.options, self.reporter, test)) + + return self.tests + + +class GstCheckTestsManager(MesonTestsManager): + name = "check" + + def __init__(self): + MesonTestsManager.__init__(self) + self.tests_info = {} + + def init(self): + return True + + def check_binary_ts(self, binary): + try: + last_touched = os.stat(binary).st_mtime + test_info = self.tests_info.get(binary) + if not test_info: + return last_touched, [] + elif test_info[0] == 0: + return True + elif test_info[0] == last_touched: + return True + except FileNotFoundError: + return None + + return last_touched, [] + + def _list_gst_check_tests(self, test, recurse=False): + binary = test.fname[0] + + self.tests_info[binary] = self.check_binary_ts(binary) + + tmpenv = os.environ.copy() + tmpenv['GST_DEBUG'] = "0" + pe = subprocess.Popen([binary, '--list-tests'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=tmpenv) + + output = pe.communicate()[0].decode() + if pe.returncode != 0: + self.debug("%s not able to list tests" % binary) + return + for t in output.split("\n"): + test_name = re.findall(r'(?<=^Test: )\w+$', t) + if len(test_name) == 1: + self.tests_info[binary][1].append(test_name[0]) + + def load_tests_info(self): + dumpfile = os.path.join(self.options.privatedir, self.name + '.dat') + try: + with open(dumpfile, 'rb') as f: + self.tests_info = pickle.load(f) + except FileNotFoundError: + self.tests_info = {} + + def save_tests_info(self): + dumpfile = os.path.join(self.options.privatedir, self.name + '.dat') + with open(dumpfile, 'wb') as f: + pickle.dump(self.tests_info, f) + + def list_tests(self): + if self.tests: + return self.tests + + self.rebuild(all=True) + self.load_tests_info() + mesontests = self.get_meson_tests() + to_inspect = [] + for test in mesontests: + binary = test.fname[0] + test_info = self.check_binary_ts(binary) + if test_info is True: + continue + elif test_info is None: + test_info = self.check_binary_ts(binary) + if test_info is None: + raise RuntimeError("Test binary %s does not exist" + " even after a full rebuild" % binary) + + with open(binary, 'rb') as f: + if b"gstcheck" not in f.read(): + self.tests_info[binary] = [0, []] + continue + to_inspect.append(test) + + if to_inspect: + executor = conc.ThreadPoolExecutor( + max_workers=self.options.num_jobs) + tmp = [] + for test in to_inspect: + tmp.append(executor.submit(self._list_gst_check_tests, test)) + + for e in tmp: + e.result() + + for test in mesontests: + gst_tests = self.tests_info[test.fname[0]][1] + if not gst_tests: + self.add_test(MesonTest(self.get_test_name(test), + self.options, self.reporter, test)) + else: + for ltest in gst_tests: + name = self.get_test_name(test) + '.' + ltest + self.add_test(MesonTest(name, self.options, self.reporter, test, + {'GST_CHECKS': ltest})) + self.save_tests_info() + return self.tests diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index a46c8b706d..3d890b2ed4 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -155,6 +155,9 @@ class Test(Loggable): return res def open_logfile(self): + if self.out: + return + path = os.path.join(self.options.logsdir, self.classname.replace(".", os.sep)) mkdir(os.path.dirname(path)) @@ -356,6 +359,13 @@ class Test(Loggable): if self.result is not Result.TIMEOUT: self.queue.put(None) + def get_valgrind_suppression_file(self, subdir, name): + p = get_data_file(subdir, name) + if p: + return p + + self.error("Could not find any %s file" % name) + def get_valgrind_suppressions(self): return [self.get_valgrind_suppression_file('data', 'gstvalidate.supp')] @@ -853,13 +863,6 @@ class GstValidateTest(Test): self.set_result(result, msg.strip()) - def get_valgrind_suppression_file(self, subdir, name): - p = get_data_file(subdir, name) - if p: - return p - - self.error("Could not find any %s file" % name) - def get_valgrind_suppressions(self): result = super(GstValidateTest, self).get_valgrind_suppressions() return result + [self.get_valgrind_suppression_file('common', 'gst.supp')] diff --git a/validate/launcher/config.py.in b/validate/launcher/config.py.in index 3c6e0cd7f2..e501c65db0 100644 --- a/validate/launcher/config.py.in +++ b/validate/launcher/config.py.in @@ -19,4 +19,5 @@ LIBDIR = '@LIBDIR@' DATADIR = '@DATADIR@' +BUILDDIR = '@BUILDDIR@' GST_VALIDATE_TESTSUITE_VERSION = '@GST_VALIDATE_TESTSUITE_VERSION@' diff --git a/validate/launcher/main.py b/validate/launcher/main.py index d04c6434fe..08e115cb2e 100644 --- a/validate/launcher/main.py +++ b/validate/launcher/main.py @@ -200,6 +200,7 @@ class LauncherConfig(Loggable): self.main_dir = utils.DEFAULT_MAIN_DIR self.output_dir = None self.logsdir = None + self.privatedir = None self.redirect_logs = False self.num_jobs = multiprocessing.cpu_count() self.dest = None @@ -259,11 +260,14 @@ class LauncherConfig(Loggable): self.logsdir = os.path.join(self.output_dir, "logs") if self.dest is None: self.dest = os.path.join(self.output_dir, "rendered") + self.privatedir = os.path.join(self.output_dir, "launcher-private") if not os.path.exists(self.dest): os.makedirs(self.dest) if not os.path.exists(self.logsdir): os.makedirs(self.logsdir) + if not os.path.exists(self.privatedir): + os.makedirs(self.privatedir) if self.redirect_logs not in ['stdout', 'stderr', False]: printc("Log redirection (%s) must be either 'stdout' or 'stderr'." diff --git a/validate/launcher/meson.build b/validate/launcher/meson.build index f2d39f744c..262b3ec2a6 100644 --- a/validate/launcher/meson.build +++ b/validate/launcher/meson.build @@ -2,6 +2,7 @@ _launcherdir = get_option('libdir') + '/gst-validate-launcher/python/launcher/' launcher_configure = configuration_data() launcher_configure.set('GST_VALIDATE_TESTSUITE_VERSION', '"@0@"'.format(TESTUITE_VERSION)) +launcher_configure.set('BUILDDIR', meson.build_root()) configure_file(input : 'config.py.in', output : 'config.py', install_dir: _launcherdir, @@ -21,3 +22,4 @@ install_data(sources: _sources, install_dir: _launcherdir) subdir('apps') +subdir('testsuites') diff --git a/validate/launcher/testsuites/Makefile.am b/validate/launcher/testsuites/Makefile.am new file mode 100644 index 0000000000..75b0182154 --- /dev/null +++ b/validate/launcher/testsuites/Makefile.am @@ -0,0 +1,6 @@ +appsdir = $(libdir)/gst-validate-launcher/python/launcher/testsuites/ + +SUBDIRS = + +apps_PYTHON = \ + check.py diff --git a/validate/launcher/testsuites/check.py b/validate/launcher/testsuites/check.py new file mode 100644 index 0000000000..e989116dee --- /dev/null +++ b/validate/launcher/testsuites/check.py @@ -0,0 +1,28 @@ +# -*- Mode: Python -*- vi:si:et:sw=4:sts=4:ts=4:syntax=python +# +# Copyright (c) 2016,Thibault Saunier +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301, USA. + +""" +GStreamer unit tests +""" + +TEST_MANAGER = "check" + + +def setup_tests(test_manager, options): + return True diff --git a/validate/launcher/testsuites/meson.build b/validate/launcher/testsuites/meson.build new file mode 100644 index 0000000000..dc016989d5 --- /dev/null +++ b/validate/launcher/testsuites/meson.build @@ -0,0 +1,3 @@ +install_data(sources: ['check.py'], + install_dir: _launcherdir + '/testsuites') +