diff --git a/scripts/git-hooks/pre-commit-python.hook b/scripts/git-hooks/pre-commit-python.hook index 16a808a7e4..e741368b32 100755 --- a/scripts/git-hooks/pre-commit-python.hook +++ b/scripts/git-hooks/pre-commit-python.hook @@ -3,6 +3,10 @@ import os import subprocess import sys import tempfile +import json +import glob +from pathlib import Path +from typing import Dict, Optional, Set, Tuple NOT_PYCODESTYLE_COMPLIANT_MESSAGE_PRE = \ "Your code is not fully pycodestyle compliant and contains"\ @@ -45,10 +49,155 @@ def copy_files_to_tmp_dir(files): return tempdir +def find_builddir() -> Optional[Path]: + # Explicitly-defined builddir takes precedence + if 'GST_DOC_BUILDDIR' in os.environ: + return Path(os.environ['GST_DOC_BUILDDIR']) + + # Now try the usual suspects + for name in ('build', '_build', 'builddir', 'b'): + if Path(name, 'build.ninja').exists(): + return Path(name) + + # Out of luck, look for the most recent folder with a `build.ninja` file + for d in sorted([p for p in Path('.').iterdir() if p.is_dir()], key=lambda p: p.stat().st_mtime): + if Path(d, 'build.ninja').exists(): + print ('Found', d) + return d + + return None + +def hotdoc_conf_needs_rebuild(conf_path: Path, conf_data: Dict, modified_fpaths): + if not isinstance(conf_data, dict): + return False + + for (key, value) in conf_data.items(): + if key.endswith('c_sources'): + if any(['*' in f for f in value]): + continue + conf_dir = conf_path.parent + for f in value: + fpath = Path(f) + if not fpath.is_absolute(): + fpath = Path(conf_dir, fpath) + + fpath = fpath.resolve() + if fpath in modified_fpaths: + return True + + return False + + +def get_hotdoc_confs_to_rebuild(builddir, modified_files) -> Tuple[Set, Set]: + srcdir = Path(os.getcwd()) + modified_fpaths = set() + for f in modified_files: + modified_fpaths.add(Path(srcdir, f)) + + confs_need_rebuild = set() + caches_need_rebuild = set() + for path in glob.glob('**/docs/*.json', root_dir=builddir, recursive=True): + conf_path = Path(srcdir, builddir, path) + with open(conf_path) as f: + conf_data = json.load(f) + + if hotdoc_conf_needs_rebuild(conf_path, conf_data, modified_fpaths): + confs_need_rebuild.add(conf_path) + caches_need_rebuild.add(conf_data.get('gst_plugin_library')) + + return (confs_need_rebuild, caches_need_rebuild) + +def build(builddir): + subprocess.run(['ninja', '-C', builddir], check=True) + subprocess.run(['ninja', '-C', builddir, 'subprojects/gstreamer/docs/hotdoc-configs.json'], check=True) + +def build_cache(builddir, subproject, targets): + if not targets: + return + + print (f'Rebuilding {subproject} cache with changes from {targets}') + + cmd = [ + os.path.join(builddir, f'subprojects/{subproject}/docs/gst-plugins-doc-cache-generator'), + os.path.join(os.getcwd(), f'subprojects/{subproject}/docs/plugins/gst_plugins_cache.json'), + os.path.join(builddir, f'subprojects/{subproject}/docs/gst_plugins_cache.json'), + ] + targets + + subprocess.run(cmd) + +class StashManager: + def __enter__(self): + print ('Stashing changes') + # First, save the difference with the current index to a patch file + tree = subprocess.run(['git', 'write-tree'], capture_output=True, check=True).stdout.strip() + result = subprocess.run(['git', 'diff-index', '--ignore-submodules', '--binary', '--no-color', '--no-ext-diff', tree], check=True, capture_output=True) + # Don't delete the temporary file, we want to make sure to prevent data loss + with tempfile.NamedTemporaryFile(delete_on_close=False, delete=False) as f: + f.write(result.stdout) + self.patch_file_name = f.name + + # Print the path to the diff file, useful is something goes wrong + print ("unstaged diff saved to ", self.patch_file_name) + + # Now stash the changes, we do not use git stash --keep-index because it causes spurious rebuilds + subprocess.run(['git', '-c', 'submodule.recurse=0', 'checkout', '--', '.'], check=True) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Now re-apply the non-staged changes + subprocess.run(['git', 'apply', '--allow-empty', self.patch_file_name], check=True) + print ('Unstashed changes') + +def run_doc_checks(modified_files): + builddir = find_builddir() + + if builddir is None: + raise Exception('cannot run doc pre-commit hook without a build directory') + + builddir = builddir.absolute() + + build(builddir) + + # Each subproject holds its own cache file. For each we keep track of the + # dynamic library associated with the hotdoc configuration files that need + # rebuilding, and only update the caches using those libraries. + # This is done in order to minimize spurious diffs as much as possible. + caches = { + 'gstreamer': [] + } + + (confs_need_rebuild, caches_need_rebuild) = get_hotdoc_confs_to_rebuild(builddir, modified_files) + + for libpath in caches_need_rebuild: + cache_project = Path(libpath).relative_to(builddir).parts[1] + caches[cache_project].append(libpath) + + for (subproject, libpaths) in caches.items(): + build_cache(builddir, subproject, libpaths) + + try: + subprocess.run(['git', 'diff', '--ignore-submodules', '--exit-code'], check=True) + except subprocess.CalledProcessError as e: + print ('You have a diff in the plugin cache, please commit it') + raise e + + print ('No pending diff in plugin caches, checking since tags') + + for conf_path in confs_need_rebuild: + subprocess.run(['hotdoc', 'run', '--fatal-warnings', '--disable-warnings', '--enabled-warnings', 'missing-since-marker', '--conf-file', conf_path, '--previous-symbol-index', 'subprojects/gst-docs/symbols/symbol_index.json'], check=True) def main(): modified_files = system('git', 'diff-index', '--cached', '--name-only', 'HEAD', '--diff-filter=ACMR').split("\n")[:-1] + + if os.environ.get('GST_ENABLE_DOC_PRE_COMMIT_HOOK', '0') != '0': + with StashManager(): + try: + run_doc_checks(modified_files) + except Exception as e: + print (e) + sys.exit(1) + non_compliant_files = [] output_message = None diff --git a/subprojects/gst-docs/markdown/contribute/index.md b/subprojects/gst-docs/markdown/contribute/index.md index a682da265d..904a5c533a 100644 --- a/subprojects/gst-docs/markdown/contribute/index.md +++ b/subprojects/gst-docs/markdown/contribute/index.md @@ -722,6 +722,24 @@ If you have a concern it might be the case you can look at the relevant hotdoc.json file for your subproject to see exactly what sources are included / excluded. +You can enable checks for up-to-date plugin caches and presence of the necessary +since tags at commit time by setting the `GST_ENABLE_DOC_PRE_COMMIT_HOOK` +environment variable to any value other than "0": + +``` shell +GST_ENABLE_DOC_PRE_COMMIT_HOOK=1 git commit +``` + +The pre-commit hook will: + +* Stash unstaged changes (the path to the patch file is printed out) +* Locate the build directory (the location can be specified through the `GST_DOC_BUILDDIR` environment variable) +* Build the version of the code that is to be committed +* Build the relevant plugins caches and error out if there is a diff +* Build the relevant doc subprojects using `hotdoc` and error out in case of since tag errors + +In any case, the stashed changes are then re-applied + ## Backporting to a stable branch Before backporting any changes to a stable branch, they should first be