gstreamer/scripts/git-hooks/pre-commit-python.hook

#!/usr/bin/env python3
import os
import subprocess
import sys
import tempfile
import json
import glob
from pathlib import Path
from typing import Dict, Optional, Set, Tuple

NOT_PYCODESTYLE_COMPLIANT_MESSAGE_PRE = \
    "Your code is not fully pycodestyle compliant and contains"\
    " the following coding style issues:\n\n"

NOT_PYCODESTYLE_COMPLIANT_MESSAGE_POST = \
    "Please fix these errors and commit again, you can do so "\
    "from the root directory automatically like this, assuming the whole "\
    "file is to be committed:"

NO_PYCODESTYLE_MESSAGE = \
    "You should install the pycodestyle style checker to be able"\
    " to commit in this repo.\nIt allows us to guarantee that "\
    "anything that is committed respects the pycodestyle coding style "\
    "standard.\nYou can install it:\n"\
    "  * on ubuntu, debian: $sudo apt-get install pycodestyle \n"\
    "  * on fedora: #yum install python3-pycodestyle \n"\
    "  * on arch: #pacman -S python-pycodestyle \n"\
    "  * or `pip install --user pycodestyle`"


def system(*args, **kwargs):
    kwargs.setdefault('stdout', subprocess.PIPE)
    proc = subprocess.Popen(args, **kwargs)
    out, err = proc.communicate()
    if isinstance(out, bytes):
        out = out.decode()
    return out


def copy_files_to_tmp_dir(files):
    tempdir = tempfile.mkdtemp()
    for name in files:
        filename = os.path.join(tempdir, name)
        filepath = os.path.dirname(filename)
        if not os.path.exists(filepath):
            os.makedirs(filepath)
        with open(filename, 'w', encoding="utf-8") as f:
            system('git', 'show', ':' + name, stdout=f)

    return tempdir

def find_builddir() -> Optional[Path]:
    # Explicitly-defined builddir takes precedence
    if 'GST_DOC_BUILDDIR' in os.environ:
        return Path(os.environ['GST_DOC_BUILDDIR'])

    # Now try the usual suspects
    for name in ('build', '_build', 'builddir', 'b'):
        if Path(name, 'build.ninja').exists():
            return Path(name)

    # Out of luck, look for the most recent folder with a `build.ninja` file
    for d in sorted([p for p in Path('.').iterdir() if p.is_dir()], key=lambda p: p.stat().st_mtime):
        if Path(d, 'build.ninja').exists():
            print ('Found', d)
            return d

    return None

def hotdoc_conf_needs_rebuild(conf_path: Path, conf_data: Dict, modified_fpaths):
    if not isinstance(conf_data, dict):
        return False

    for (key, value) in conf_data.items():
        if key.endswith('c_sources'):
            if any(['*' in f for f in value]):
                continue
            conf_dir = conf_path.parent
            for f in value:
                fpath = Path(f)
                if not fpath.is_absolute():
                    fpath = Path(conf_dir, fpath)

                fpath = fpath.resolve()
                if fpath in modified_fpaths:
                    return True

    return False


def get_hotdoc_confs_to_rebuild(builddir, modified_files) -> Tuple[Set, Set]:
    srcdir = Path(os.getcwd())
    modified_fpaths = set()
    for f in modified_files:
        modified_fpaths.add(Path(srcdir, f))

    confs_need_rebuild = set()
    caches_need_rebuild = set()
    for path in glob.glob('**/docs/*.json', root_dir=builddir, recursive=True):
        conf_path = Path(srcdir, builddir, path)
        with open(conf_path) as f:
            conf_data = json.load(f)

        if hotdoc_conf_needs_rebuild(conf_path, conf_data, modified_fpaths):
            confs_need_rebuild.add(conf_path)
            caches_need_rebuild.add(conf_data.get('gst_plugin_library'))

    return (confs_need_rebuild, caches_need_rebuild)

def build(builddir):
    subprocess.run(['ninja', '-C', builddir], check=True)
    for subproject in ('gstreamer', 'gst-plugins-base', 'gst-plugins-good', 'gst-plugins-bad', 'gst-plugins-ugly', 'gst-rtsp-server', 'gst-libav', 'gst-editing-services'):
        subprocess.run(['ninja', '-C', builddir, f'subprojects/{subproject}/docs/hotdoc-configs.json'], check=True)

def build_cache(builddir, subproject, targets, subdir):
    if not targets:
        return

    print (f'Rebuilding {subproject} cache with changes from {targets}')

    cmd = [
        os.path.join(builddir, f'subprojects/gstreamer/docs/gst-plugins-doc-cache-generator'),
        os.path.join(os.getcwd(), f'subprojects/{subproject}/docs/{subdir}gst_plugins_cache.json'),
        os.path.join(builddir, f'subprojects/{subproject}/docs/gst_plugins_cache.json'),
    ] + targets

    subprocess.run(cmd)

class StashManager:
    def __enter__(self):
        print ('Stashing changes')
        # First, save the difference with the current index to a patch file
        tree = subprocess.run(['git', 'write-tree'], capture_output=True, check=True).stdout.strip()
        result = subprocess.run(['git', 'diff-index', '--ignore-submodules', '--binary', '--no-color', '--no-ext-diff', tree], check=True, capture_output=True)
        # Don't delete the temporary file, we want to make sure to prevent data loss
        with tempfile.NamedTemporaryFile(delete_on_close=False, delete=False) as f:
            f.write(result.stdout)
            self.patch_file_name = f.name

        # Print the path to the diff file, useful is something goes wrong
        print ("unstaged diff saved to ", self.patch_file_name)

        # Now stash the changes, we do not use git stash --keep-index because it causes spurious rebuilds
        subprocess.run(['git', '-c', 'submodule.recurse=0', 'checkout', '--', '.'], check=True)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        # Now re-apply the non-staged changes
        subprocess.run(['git', 'apply', '--allow-empty', self.patch_file_name], check=True)
        print ('Unstashed changes')

def run_doc_checks(modified_files):
    builddir = find_builddir()

    if builddir is None:
        raise Exception('cannot run doc pre-commit hook without a build directory')

    builddir = builddir.absolute()

    build(builddir)

    # Each subproject holds its own cache file. For each we keep track of the
    # dynamic library associated with the hotdoc configuration files that need
    # rebuilding, and only update the caches using those libraries.
    # This is done in order to minimize spurious diffs as much as possible.
    caches = {}

    (confs_need_rebuild, caches_need_rebuild) = get_hotdoc_confs_to_rebuild(builddir, modified_files)

    for libpath in caches_need_rebuild:
        cache_project = Path(libpath).relative_to(builddir).parts[1]
        if cache_project not in caches:
            caches[cache_project] = [libpath]
        else:
            caches[cache_project].append(libpath)

    for (subproject, libpaths) in caches.items():
        cache_subdir = 'plugins/' if subproject in ['gst-plugins-bad', 'gst-plugins-base', 'gst-rtsp-server', 'gstreamer', 'gst-plugins-rs'] else ''
        build_cache(builddir, subproject, libpaths, cache_subdir)

    try:
        subprocess.run(['git', 'diff', '--ignore-submodules', '--exit-code'], check=True)
    except subprocess.CalledProcessError as e:
        print ('You have a diff in the plugin cache, please commit it')
        raise e

    print ('No pending diff in plugin caches, checking since tags')

    for conf_path in confs_need_rebuild:
        subprocess.run(['hotdoc', 'run', '--fatal-warnings', '--disable-warnings', '--enabled-warnings', 'missing-since-marker', '--conf-file', conf_path, '--previous-symbol-index', 'subprojects/gst-docs/symbols/symbol_index.json'], check=True)

def main():
    modified_files = system('git', 'diff-index', '--cached',
                            '--name-only', 'HEAD', '--diff-filter=ACMR').split("\n")[:-1]

    if os.environ.get('GST_ENABLE_DOC_PRE_COMMIT_HOOK', '0') != '0':
        with StashManager():
            try:
                run_doc_checks(modified_files)
            except Exception as e:
                print (e)
                sys.exit(1)

    non_compliant_files = []
    output_message = None

    for modified_file in modified_files:
        try:
            if not modified_file.endswith(".py"):
                continue
            pycodestyle_errors = system('pycodestyle', '--repeat', '--ignore', 'E402,E501,E128,W605,W503', modified_file)
            if pycodestyle_errors:
                if output_message is None:
                    output_message = NOT_PYCODESTYLE_COMPLIANT_MESSAGE_PRE
                output_message += pycodestyle_errors
                non_compliant_files.append(modified_file)
        except OSError as e:
            output_message = NO_PYCODESTYLE_MESSAGE
            break

    if output_message:
        print(output_message)
        if non_compliant_files:
            print(NOT_PYCODESTYLE_COMPLIANT_MESSAGE_POST)
            for non_compliant_file in non_compliant_files:
                print("autopep8 -i --max-line-length 120", non_compliant_file, "; git add ",
                      non_compliant_file)
            print("git commit")
        sys.exit(1)


if __name__ == '__main__':
    main()