gstreamer/git-update

#!/usr/bin/env python3
import argparse
import os
import subprocess
import xml.etree.ElementTree as ET
import sys

from scripts.common import git
from scripts.common import Colors
from scripts.common import accept_command
from scripts.common import get_meson


SCRIPTDIR = os.path.normpath(os.path.dirname(__file__))
# Force a checkout to happen and throw away local changes
FORCE_CHECKOUT = False
CI = os.environ.get('CI', False)


def manifest_get_commits(manifest):
    res = {}
    tree = ET.parse(manifest)
    root = tree.getroot()
    remotes = {}
    for child in root:
        if child.tag == 'remote':
            remotes[child.attrib['name']] = child.attrib['fetch']
        if child.tag == 'project':
            name = child.attrib['name']
            path = child.attrib.get('path', name)

            remote = child.attrib.get('remote')
            if remote:
                res[path] = [child.attrib["revision"], [os.path.join(remotes[remote], name), child.attrib.get('refname', child.attrib["revision"])]]
            else:
                res[path] = [child.attrib["revision"], []]

    return res


def get_branch_name(repo_dir):
    return git('-C', repo_dir, 'rev-parse', '--symbolic-full-name', 'HEAD').strip()


def ensure_revision_if_necessary(repo_dir, revision):
    """
    Makes sure that @revision is set if the current repo is detached.
    """
    if not revision:
        if get_branch_name(repo_dir) == 'HEAD':
            revision = git('-C', repo_dir, 'rev-parse', 'HEAD').strip()

    return revision


def update_subprojects(manifest, no_interaction=False, status=False):
    subprojects_dir = os.path.join(SCRIPTDIR, "subprojects")
    for repo_name in os.listdir(subprojects_dir):
        repo_dir = os.path.normpath(os.path.join(SCRIPTDIR, subprojects_dir, repo_name))
        if not os.path.exists(os.path.join(repo_dir, '.git')):
            continue

        revision, args = repos_commits.get(repo_name, [None, []])
        if not update_repo(repo_name, repo_dir, revision, no_interaction, fetch_args=args, status=status):
          return False

    return True

def repo_status(commit_message):
    status = "clean"
    for message in commit_message:
      if message.startswith('??'):
        status = "%sclean but untracked files%s" % (Colors.WARNING,Colors.ENDC)
      elif message.startswith(' M'):
        status = "%shas local modifications%s" % (Colors.WARNING,Colors.ENDC)
        break;
    return status

def check_repo_status(repo_name, worktree_dir):
    branch_message = git("status", repository_path=worktree_dir).split("\n")
    commit_message = git("status", "--porcelain", repository_path=worktree_dir).split("\n")

    print(u"%s%s%s - %s - %s" % (Colors.HEADER, repo_name, Colors.ENDC,
                                    branch_message[0].strip(), repo_status(commit_message)))
    return True

def fatal_git_fetches(repo_dir):
    '''
    When running on the CI, we usually have a cache, in which case we don't
    want the git update to be fatal since we don't want our CI to fail when
    there's downtime on external repos.
    '''
    if not CI:
        return True
    remote = git("remote", "get-url", "origin", repository_path=repo_dir)
    if 'gitlab.freedesktop.org' in remote:
        return True
    return False

def update_repo(repo_name, repo_dir, revision, no_interaction, fetch_args=None, recurse_i=0, status=False):
    if status:
      return check_repo_status(repo_name, repo_dir)
    revision = ensure_revision_if_necessary(repo_dir, revision)
    git("config", "rebase.autoStash", "true", repository_path=repo_dir)

    fetch_args = fetch_args if fetch_args is not None else []
    fetch_args.append('--tags')
    fatal = fatal_git_fetches(repo_dir)
    try:
        if revision:
            print("Checking out %s in %s" % (revision, repo_name))
            git("fetch", *fetch_args, repository_path=repo_dir, fatal=fatal)
            checkout_args = ["--force"] if FORCE_CHECKOUT else []
            checkout_args += ["--detach", revision]
            git("checkout", *checkout_args, repository_path=repo_dir)
        else:
            print("Updating branch %s in %s" % (get_branch_name(repo_dir), repo_name))
            git("pull", "--rebase", repository_path=repo_dir, fatal=fatal)
        git("submodule", "update", repository_path=repo_dir)
    except Exception as e:
        out = getattr(e, "output", b"").decode()
        if not no_interaction:
            print("====================================="
                  "\n%s\nEntering a shell in %s to fix that"
                  " just `exit 0` once done, or `exit 255`"
                  " to skip update for that repository"
                  "\n=====================================" % (
                        out, repo_dir))
            try:
                if os.name == 'nt':
                    shell = os.environ.get("COMSPEC", r"C:\WINDOWS\system32\cmd.exe")
                else:
                    shell = os.environ.get("SHELL", os.path.realpath("/bin/sh"))
                subprocess.check_call(shell, cwd=repo_dir)
            except subprocess.CalledProcessError as e:
                if e.returncode == 255:
                    print("Skipping '%s' update" % repo_name)
                    return True
            except:
                # Result of subshell does not really matter
                pass

            if recurse_i < 3:
                return update_repo(repo_name, repo_dir, revision, no_interaction,
                                    recurse_i=recurse_i + 1)
            return False
        else:
            print("\nCould not rebase %s, please fix and try again."
                    " Error:\n\n%s %s" % (repo_dir, out, e))

            return False


    commit_message = git("show", "--shortstat", repository_path=repo_dir).split("\n")
    print(u"  -> %s%s%s - %s" % (Colors.HEADER, commit_message[0][7:14], Colors.ENDC,
                                    commit_message[4].strip()))

    return True


# Update gst-plugins-rs dependencies
def update_cargo(build_dir):
    cargo_toml = os.path.join('subprojects', 'gst-plugins-rs', 'Cargo.toml')
    if not os.path.exists(cargo_toml):
        return True

    cmd = ['cargo', 'update', '--manifest-path', cargo_toml]

    try:
        ret = subprocess.run(cmd)
    except FileNotFoundError:
        # silenty ignore if cargo isn't installed
        return False

    return ret == 0


if __name__ == "__main__":
    parser = argparse.ArgumentParser(prog="git-update")

    parser.add_argument("--no-color",
                        default=False,
                        action='store_true',
                        help="Do not output ansi colors.")
    parser.add_argument("--builddir",
                        default=None,
                        help="Specifies the build directory where to"
                        " invoke ninja after updating.")
    parser.add_argument("--no-interaction",
                        default=False,
                        action='store_true',
                        help="Do not allow interaction with the user.")
    parser.add_argument("--status",
                        default=False,
                        action='store_true',
                        help="Check repositories status only.")
    parser.add_argument("--manifest",
                        default=None,
                        help="Use a android repo manifest to sync repositories"
                        " Note that it will let all repositories in detached state")
    options = parser.parse_args()
    if options.no_color or not Colors.can_enable():
        Colors.disable()

    if options.no_interaction:
        sys.stdin.close()

    if options.manifest:
        meson = get_meson()
        try:
            targets_s = subprocess.check_output(meson + ['subprojects', 'download'], shell=False, universal_newlines=True)
        except subprocess.CalledProcessError as err:
            print(f"Subproject download Failed")
            print(f"Output: {err.output}")
            print(f"Exit code: {err.returncode}")
            exit(err.returncode)

        repos_commits = manifest_get_commits(options.manifest)
        FORCE_CHECKOUT = True
    else:
        repos_commits = {}

    revision, args = repos_commits.get('gst-build', [None, []])
    if not update_repo('gst-build', SCRIPTDIR, revision, options.no_interaction, fetch_args=args, status=options.status):
        exit(1)
    if not update_subprojects(options.manifest, options.no_interaction, status=options.status):
        exit(1)
    if not options.status:
      update_cargo(options.builddir)

    if options.builddir:
        ninja = accept_command(["ninja", "ninja-build"])
        if not ninja:
            print("Can't find ninja, other backends are not supported for rebuilding")
            exit(1)

        if not os.path.exists(os.path.join (options.builddir, 'build.ninja')):
            print("Can't rebuild in %s as no build.ninja file found." % options.builddir)

        print("Rebuilding all GStreamer modules.")
        exit(subprocess.call([ninja, '-C', options.builddir]))