mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2025-04-13 19:44:10 +00:00
docs: implement pre-commit hook to check cache updates and since tags
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8231>
This commit is contained in:
parent
c276f5daca
commit
a15c786db5
2 changed files with 167 additions and 0 deletions
|
@ -3,6 +3,10 @@ import os
|
|||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import json
|
||||
import glob
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, Set, Tuple
|
||||
|
||||
NOT_PYCODESTYLE_COMPLIANT_MESSAGE_PRE = \
|
||||
"Your code is not fully pycodestyle compliant and contains"\
|
||||
|
@ -45,10 +49,155 @@ def copy_files_to_tmp_dir(files):
|
|||
|
||||
return tempdir
|
||||
|
||||
def find_builddir() -> Optional[Path]:
|
||||
# Explicitly-defined builddir takes precedence
|
||||
if 'GST_DOC_BUILDDIR' in os.environ:
|
||||
return Path(os.environ['GST_DOC_BUILDDIR'])
|
||||
|
||||
# Now try the usual suspects
|
||||
for name in ('build', '_build', 'builddir', 'b'):
|
||||
if Path(name, 'build.ninja').exists():
|
||||
return Path(name)
|
||||
|
||||
# Out of luck, look for the most recent folder with a `build.ninja` file
|
||||
for d in sorted([p for p in Path('.').iterdir() if p.is_dir()], key=lambda p: p.stat().st_mtime):
|
||||
if Path(d, 'build.ninja').exists():
|
||||
print ('Found', d)
|
||||
return d
|
||||
|
||||
return None
|
||||
|
||||
def hotdoc_conf_needs_rebuild(conf_path: Path, conf_data: Dict, modified_fpaths):
|
||||
if not isinstance(conf_data, dict):
|
||||
return False
|
||||
|
||||
for (key, value) in conf_data.items():
|
||||
if key.endswith('c_sources'):
|
||||
if any(['*' in f for f in value]):
|
||||
continue
|
||||
conf_dir = conf_path.parent
|
||||
for f in value:
|
||||
fpath = Path(f)
|
||||
if not fpath.is_absolute():
|
||||
fpath = Path(conf_dir, fpath)
|
||||
|
||||
fpath = fpath.resolve()
|
||||
if fpath in modified_fpaths:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_hotdoc_confs_to_rebuild(builddir, modified_files) -> Tuple[Set, Set]:
|
||||
srcdir = Path(os.getcwd())
|
||||
modified_fpaths = set()
|
||||
for f in modified_files:
|
||||
modified_fpaths.add(Path(srcdir, f))
|
||||
|
||||
confs_need_rebuild = set()
|
||||
caches_need_rebuild = set()
|
||||
for path in glob.glob('**/docs/*.json', root_dir=builddir, recursive=True):
|
||||
conf_path = Path(srcdir, builddir, path)
|
||||
with open(conf_path) as f:
|
||||
conf_data = json.load(f)
|
||||
|
||||
if hotdoc_conf_needs_rebuild(conf_path, conf_data, modified_fpaths):
|
||||
confs_need_rebuild.add(conf_path)
|
||||
caches_need_rebuild.add(conf_data.get('gst_plugin_library'))
|
||||
|
||||
return (confs_need_rebuild, caches_need_rebuild)
|
||||
|
||||
def build(builddir):
|
||||
subprocess.run(['ninja', '-C', builddir], check=True)
|
||||
subprocess.run(['ninja', '-C', builddir, 'subprojects/gstreamer/docs/hotdoc-configs.json'], check=True)
|
||||
|
||||
def build_cache(builddir, subproject, targets):
|
||||
if not targets:
|
||||
return
|
||||
|
||||
print (f'Rebuilding {subproject} cache with changes from {targets}')
|
||||
|
||||
cmd = [
|
||||
os.path.join(builddir, f'subprojects/{subproject}/docs/gst-plugins-doc-cache-generator'),
|
||||
os.path.join(os.getcwd(), f'subprojects/{subproject}/docs/plugins/gst_plugins_cache.json'),
|
||||
os.path.join(builddir, f'subprojects/{subproject}/docs/gst_plugins_cache.json'),
|
||||
] + targets
|
||||
|
||||
subprocess.run(cmd)
|
||||
|
||||
class StashManager:
|
||||
def __enter__(self):
|
||||
print ('Stashing changes')
|
||||
# First, save the difference with the current index to a patch file
|
||||
tree = subprocess.run(['git', 'write-tree'], capture_output=True, check=True).stdout.strip()
|
||||
result = subprocess.run(['git', 'diff-index', '--ignore-submodules', '--binary', '--no-color', '--no-ext-diff', tree], check=True, capture_output=True)
|
||||
# Don't delete the temporary file, we want to make sure to prevent data loss
|
||||
with tempfile.NamedTemporaryFile(delete_on_close=False, delete=False) as f:
|
||||
f.write(result.stdout)
|
||||
self.patch_file_name = f.name
|
||||
|
||||
# Print the path to the diff file, useful is something goes wrong
|
||||
print ("unstaged diff saved to ", self.patch_file_name)
|
||||
|
||||
# Now stash the changes, we do not use git stash --keep-index because it causes spurious rebuilds
|
||||
subprocess.run(['git', '-c', 'submodule.recurse=0', 'checkout', '--', '.'], check=True)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
# Now re-apply the non-staged changes
|
||||
subprocess.run(['git', 'apply', '--allow-empty', self.patch_file_name], check=True)
|
||||
print ('Unstashed changes')
|
||||
|
||||
def run_doc_checks(modified_files):
|
||||
builddir = find_builddir()
|
||||
|
||||
if builddir is None:
|
||||
raise Exception('cannot run doc pre-commit hook without a build directory')
|
||||
|
||||
builddir = builddir.absolute()
|
||||
|
||||
build(builddir)
|
||||
|
||||
# Each subproject holds its own cache file. For each we keep track of the
|
||||
# dynamic library associated with the hotdoc configuration files that need
|
||||
# rebuilding, and only update the caches using those libraries.
|
||||
# This is done in order to minimize spurious diffs as much as possible.
|
||||
caches = {
|
||||
'gstreamer': []
|
||||
}
|
||||
|
||||
(confs_need_rebuild, caches_need_rebuild) = get_hotdoc_confs_to_rebuild(builddir, modified_files)
|
||||
|
||||
for libpath in caches_need_rebuild:
|
||||
cache_project = Path(libpath).relative_to(builddir).parts[1]
|
||||
caches[cache_project].append(libpath)
|
||||
|
||||
for (subproject, libpaths) in caches.items():
|
||||
build_cache(builddir, subproject, libpaths)
|
||||
|
||||
try:
|
||||
subprocess.run(['git', 'diff', '--ignore-submodules', '--exit-code'], check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print ('You have a diff in the plugin cache, please commit it')
|
||||
raise e
|
||||
|
||||
print ('No pending diff in plugin caches, checking since tags')
|
||||
|
||||
for conf_path in confs_need_rebuild:
|
||||
subprocess.run(['hotdoc', 'run', '--fatal-warnings', '--disable-warnings', '--enabled-warnings', 'missing-since-marker', '--conf-file', conf_path, '--previous-symbol-index', 'subprojects/gst-docs/symbols/symbol_index.json'], check=True)
|
||||
|
||||
def main():
|
||||
modified_files = system('git', 'diff-index', '--cached',
|
||||
'--name-only', 'HEAD', '--diff-filter=ACMR').split("\n")[:-1]
|
||||
|
||||
if os.environ.get('GST_ENABLE_DOC_PRE_COMMIT_HOOK', '0') != '0':
|
||||
with StashManager():
|
||||
try:
|
||||
run_doc_checks(modified_files)
|
||||
except Exception as e:
|
||||
print (e)
|
||||
sys.exit(1)
|
||||
|
||||
non_compliant_files = []
|
||||
output_message = None
|
||||
|
||||
|
|
|
@ -722,6 +722,24 @@ If you have a concern it might be the case you can look at the relevant
|
|||
hotdoc.json file for your subproject to see exactly what sources are
|
||||
included / excluded.
|
||||
|
||||
You can enable checks for up-to-date plugin caches and presence of the necessary
|
||||
since tags at commit time by setting the `GST_ENABLE_DOC_PRE_COMMIT_HOOK`
|
||||
environment variable to any value other than "0":
|
||||
|
||||
``` shell
|
||||
GST_ENABLE_DOC_PRE_COMMIT_HOOK=1 git commit
|
||||
```
|
||||
|
||||
The pre-commit hook will:
|
||||
|
||||
* Stash unstaged changes (the path to the patch file is printed out)
|
||||
* Locate the build directory (the location can be specified through the `GST_DOC_BUILDDIR` environment variable)
|
||||
* Build the version of the code that is to be committed
|
||||
* Build the relevant plugins caches and error out if there is a diff
|
||||
* Build the relevant doc subprojects using `hotdoc` and error out in case of since tag errors
|
||||
|
||||
In any case, the stashed changes are then re-applied
|
||||
|
||||
## Backporting to a stable branch
|
||||
|
||||
Before backporting any changes to a stable branch, they should first be
|
||||
|
|
Loading…
Reference in a new issue