From 76043141fcfbfed1b80ac4a3874c75aac2798faa Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 20 Apr 2022 09:45:12 -0400 Subject: [PATCH] meson: Cleanup our meson integration Writing a proper "depfile" to follow depending files, based on depfiles generated by rustc. This is based on work done while working on gobject-examples-rs --- cargo_wrapper.py | 197 +++++++++++++++++++++++++++++------------------ meson.build | 25 ++---- 2 files changed, 128 insertions(+), 94 deletions(-) diff --git a/cargo_wrapper.py b/cargo_wrapper.py index 3ab35ab7..b81d6f03 100644 --- a/cargo_wrapper.py +++ b/cargo_wrapper.py @@ -1,89 +1,138 @@ #!/usr/bin/env python3 +import hashlib +import re import glob import os -import os.path import shutil import subprocess import sys +from argparse import ArgumentParser +from pathlib import Path as P -PLUGIN_DIRS = ['audio', 'generic', 'net', 'text', 'utils', 'video'] - -command, meson_build_dir, meson_current_source_dir, meson_build_root, target, include, extra_env, prefix, libdir = sys.argv[ - 1:10] -include = include.split(',') - -cargo_target_dir = os.path.join(meson_build_dir, 'target') - -env = os.environ.copy() -env['CARGO_TARGET_DIR'] = cargo_target_dir - -pkg_config_path = env.get('PKG_CONFIG_PATH', '').split(':') -pkg_config_path.append(os.path.join(meson_build_root, 'meson-uninstalled')) -env['PKG_CONFIG_PATH'] = ':'.join(pkg_config_path) - -if len(extra_env) > 0: - for e in extra_env.split(','): - k, v = e.split(':') - env[k] = v - -if command == 'build': - # cargo build - ext = sys.argv[10] - # when using --default-library=both 2 extensions are passed - try: - ext2 = sys.argv[11] - except IndexError: - ext2 = None - - # Build with the 'static' feature enforcing the minimal gst version required for static builds - cargo_cmd = ['cargo', 'cbuild', '--features', 'static'] - if target == 'release': - cargo_cmd.append('--release') -elif command == 'test': - # cargo test - cargo_cmd = ['cargo', 'ctest', '--no-fail-fast', '--color=always'] -else: - print("Unknown command:", command) - sys.exit(1) - -cargo_cmd.extend( - ['--manifest-path', os.path.join(meson_current_source_dir, 'Cargo.toml')]) - -cargo_cmd.extend(['--prefix', prefix, '--libdir', - os.path.join(prefix, libdir)]) +PARSER = ArgumentParser() +PARSER.add_argument('command', choices=['build', 'test']) +PARSER.add_argument('build_dir', type=P) +PARSER.add_argument('src_dir', type=P) +PARSER.add_argument('root_dir', type=P) +PARSER.add_argument('target', choices=['release', 'debug']) +PARSER.add_argument('include') +PARSER.add_argument('extra_env') +PARSER.add_argument('prefix', type=P) +PARSER.add_argument('libdir', type=P) +PARSER.add_argument('--version', default=None) +PARSER.add_argument('--exts', nargs="+", default=[]) +PARSER.add_argument('--depfile') -def run(cargo_cmd, env): - try: - subprocess.run(cargo_cmd, env=env, check=True) - except subprocess.SubprocessError: +def generate_depfile_for(libfile): + file_stem = libfile.parent / libfile.stem + depfile_content = "" + with open(f"{file_stem}.d", 'r') as depfile: + for l in depfile.readlines(): + if l.startswith(str(file_stem)): + output, srcs = l.split(":", maxsplit=2) + output = re.sub(f"^{file_stem}", + f"{opts.build_dir / libfile.stem}", f) + + all_deps = [] + for src in srcs.split(" "): + all_deps.append(src) + src = P(src) + if src.name == 'lib.rs': + # `rustc` doesn't take `Cargo.toml` into account + # but we need to + cargo_toml = src.parent.parent / 'Cargo.toml' + if cargo_toml.exists(): + all_deps.append(str(cargo_toml)) + + depfile_content += f"{output}: {' '.join(all_deps)}\n" + + return depfile_content + + +if __name__ == "__main__": + opts = PARSER.parse_args() + + logfile = open(opts.root_dir / 'meson-logs' / + f'{opts.src_dir.name}-cargo-wrapper.log', 'w') + + print(opts, file=logfile) + cargo_target_dir = opts.build_dir / 'target' + + env = os.environ.copy() + env['CARGO_TARGET_DIR'] = str(cargo_target_dir) + + pkg_config_path = env.get('PKG_CONFIG_PATH', '').split(':') + pkg_config_path.append(str(opts.root_dir / 'meson-uninstalled')) + env['PKG_CONFIG_PATH'] = ':'.join(pkg_config_path) + + if opts.extra_env: + for e in opts.extra_env.split(','): + k, v = e.split(':') + env[k] = v + + if opts.command == 'build': + cargo_cmd = ['cargo', 'cbuild'] + if opts.target == 'release': + cargo_cmd.append('--release') + elif opts.command == 'test': + # cargo test + cargo_cmd = ['cargo', 'ctest', '--no-fail-fast', '--color=always'] + else: + print("Unknown command:", opts.command, file=logfile) sys.exit(1) + cargo_cmd.extend(['--manifest-path', opts.src_dir / 'Cargo.toml']) + cargo_cmd.extend(['--prefix', opts.prefix, '--libdir', + opts.prefix / opts.libdir]) -for p in include: - cargo_cmd.extend(['-p', p]) -run(cargo_cmd, env) + def run(cargo_cmd, env): + try: + subprocess.run(cargo_cmd, env=env, check=True) + except subprocess.SubprocessError: + sys.exit(1) -if command == 'build': - target_dir = os.path.join(cargo_target_dir, '**', target) - # Copy so files to build dir - for f in glob.glob(os.path.join(target_dir, '*.' + ext), recursive=True): - shutil.copy(f, meson_build_dir) - if ext2: - for f in glob.glob(os.path.join(target_dir, '*.' + ext2), recursive=True): - shutil.copy(f, meson_build_dir) - # Copy generated pkg-config files - for f in glob.glob(os.path.join(target_dir, '*.pc'), recursive=True): - shutil.copy(f, meson_build_dir) + for p in opts.include.split(','): + cargo_cmd.extend(['-p', p]) + run(cargo_cmd, env) - # Move -uninstalled.pc to meson-uninstalled - uninstalled = os.path.join(meson_build_dir, 'meson-uninstalled') - if not os.path.exists(uninstalled): - os.mkdir(uninstalled) - for f in glob.glob(os.path.join(meson_build_dir, '*-uninstalled.pc')): - # move() does not allow us to update the file so remove it if it already exists - dest = os.path.join(uninstalled, os.path.basename(f)) - if os.path.exists(dest): - os.unlink(dest) - shutil.move(f, uninstalled) + if opts.command == 'build': + target_dir = cargo_target_dir / '**' / opts.target + + # Copy so files to build dir + depfile_content = "" + for ext in opts.exts: + for f in glob.glob(str(target_dir / f'*.{ext}'), recursive=True): + libfile = P(f) + + depfile_content += generate_depfile_for(libfile) + + copied_file = (opts.build_dir / libfile.name) + try: + if copied_file.stat().st_mtime == libfile.stat().st_mtime: + print(f"{copied_file} has not changed.", file=logfile) + continue + except FileNotFoundError: + pass + + print(f"Copying {copied_file}", file=logfile) + shutil.copy2(f, opts.build_dir) + + with open(opts.depfile, 'w') as depfile: + depfile.write(depfile_content) + + # Copy generated pkg-config files + for f in glob.glob(str(target_dir / '*.pc'), recursive=True): + shutil.copy(f, opts.build_dir) + + # Move -uninstalled.pc to meson-uninstalled + uninstalled = opts.build_dir / 'meson-uninstalled' + os.makedirs(uninstalled, exist_ok=True) + + for f in opts.build_dir.glob('*-uninstalled.pc'): + # move() does not allow us to update the file so remove it if it already exists + dest = uninstalled / P(f).name + if dest.exists(): + dest.unlink() + shutil.move(f, uninstalled) diff --git a/meson.build b/meson.build index dbe3bad8..a5a540a9 100644 --- a/meson.build +++ b/meson.build @@ -176,32 +176,16 @@ extra_env_str = ','.join(extra_env_list) plugins_install_dir = get_option('libdir') / 'gstreamer-1.0' pkgconfig_install_dir = get_option('libdir') / 'pkgconfig' -# Use all files checked in git to figure out when to rebuild -git = find_program('git', required: false) -files = [] -if git.found() - tmp = run_command([git, 'ls-files', '.']) - if tmp.returncode() == 0 - files = tmp.stdout().split() - endif -endif - -if files.length() > 0 - build_always_stale = false -else - warning('Could not retrieve list of files, always building the target') - build_always_stale = true -endif +python = import('python').find_installation() rs_plugins = custom_target('gst-plugins-rs', build_by_default: true, output: output, console: true, install: true, install_dir: plugins_install_dir, - input: files, - build_always_stale: build_always_stale, depends: depends, + depfile: 'gst-plugins-rs.dep', command: [cargo_wrapper, 'build', meson.current_build_dir(), @@ -212,14 +196,15 @@ rs_plugins = custom_target('gst-plugins-rs', extra_env_str, get_option('prefix'), get_option('libdir'), - extensions]) + '--depfile', '@DEPFILE@', + '--exts', extensions, + ]) plugins = rs_plugins.to_list() # We don't need to pass a command as we depends on the target above # but it is currently mandatory ( https://github.com/mesonbuild/meson/issues/8059 ) # so use python as it's guaranteed to be present on any setup -python = import('python').find_installation() custom_target('gst-plugins-rs-pc-files', build_by_default: true, output: pc_files,