gstreamer/scripts/move_mrs_to_monorepo.py

#!/usr/bin/env python3

from urllib.parse import urlparse
from contextlib import contextmanager
import os
import re
import sys
try:
    import gitlab
except ModuleNotFoundError:
    print("========================================================================", file=sys.stderr)
    print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab python-dateutil`", file=sys.stderr)
    print("========================================================================", file=sys.stderr)
    sys.exit(1)

try:
    from dateutil import parser as dateparse
except ModuleNotFoundError:
    print("========================================================================", file=sys.stderr)
    print("ERROR: Install dateutil with `python3 -m pip install dateutil`", file=sys.stderr)
    print("========================================================================", file=sys.stderr)
    sys.exit(1)
import argparse
import requests

import subprocess

ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))

URL = "https://gitlab.freedesktop.org/"
SIGN_IN_URL = URL + 'sign_in'
LOGIN_URL = URL + 'users/sign_in'
LOGIN_URL_LDAP = URL + '/users/auth/ldapmain/callback'

MONOREPO_REMOTE_NAME = 'origin'
NAMESPACE = "gstreamer"
MONOREPO_NAME = 'gstreamer'
MONOREPO_REMOTE = URL + f'{NAMESPACE}/{MONOREPO_NAME}'
MONOREPO_BRANCH = 'main'
PING_SIGN = '@'
MOVING_NAMESPACE = NAMESPACE

PARSER = argparse.ArgumentParser(
    description="Move merge request from old GStreamer module to the new"
                "GStreamer 'monorepo'.\n"
                " All your pending merge requests from all GStreamer modules will"
                " be moved the the mono repository."
)
PARSER.add_argument("--skip-branch", action="store", nargs="*",
                    help="Ignore MRs for branches which match those names.", dest="skipped_branches")
PARSER.add_argument("--skip-on-failure", action="store_true", default=False)
PARSER.add_argument("--dry-run", "-n", action="store_true", default=False)
PARSER.add_argument("--use-branch-if-exists",
                    action="store_true", default=False)
PARSER.add_argument("--list-mrs-only", action="store_true", default=False)
PARSER.add_argument(
    "-c",
    "--config-file",
    action="append",
    dest='config_files',
    help="Configuration file to use. Can be used multiple times.",
    required=False,
)
PARSER.add_argument(
    "-g",
    "--gitlab",
    help=(
        "Which configuration section should "
        "be used. If not defined, the default selection "
        "will be used."
    ),
    required=False,
)
PARSER.add_argument(
    "-m",
    "--module",
    help="GStreamer module to move MRs for. All if none specified. Can be used multiple times.",
    dest='modules',
    action="append",
    required=False,
)
PARSER.add_argument(
    "-mr",
    "--mr-url",
    default=None,
    type=str,
    help=(
        "URL of the MR to work on."
    ),
    required=False,
)

GST_PROJECTS = [
    'gstreamer',
    'gst-plugins-base',
    'gst-plugins-good',
    'gst-plugins-bad',
    'gst-plugins-ugly',
    'gst-libav',
    'gst-rtsp-server',
    'gstreamer-vaapi',
    'gstreamer-sharp',
    'gst-python',
    'gst-omx',
    'gst-editing-services',
    'gst-devtools',
    'gst-docs',
    'gst-examples',
    'gst-build',
    'gst-ci',
]

GST_PROJECTS_ID = {
    'gstreamer': 1357,
    'gst-rtsp-server': 1362,
    'gstreamer-vaapi': 1359,
    'gstreamer-sharp': 1358,
    'gst-python': 1355,
    'gst-plugins-ugly': 1354,
    'gst-plugins-good': 1353,
    'gst-plugins-base': 1352,
    'gst-plugins-bad': 1351,
    'gst-omx': 1350,
    'gst-libav': 1349,
    'gst-integration-testsuites': 1348,
    'gst-examples': 1347,
    'gst-editing-services': 1346,
    'gst-docs': 1345,
    'gst-devtools': 1344,
    'gst-ci': 1343,
    'gst-build': 1342,
}

# We do not want to deal with LFS
os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"


log_depth = []               # type: T.List[str]


@contextmanager
def nested(name=''):
    global log_depth
    log_depth.append(name)
    try:
        yield
    finally:
        log_depth.pop()


def bold(text: str):
    return f"\033[1m{text}\033[0m"


def green(text: str):
    return f"\033[1;32m{text}\033[0m"


def red(text: str):
    return f"\033[1;31m{text}\033[0m"


def yellow(text: str):
    return f"\033[1;33m{text}\033[0m"


def fprint(msg, nested=True):
    if log_depth:
        prepend = log_depth[-1] + ' | ' if nested else ''
    else:
        prepend = ''

    print(prepend + msg, end="")
    sys.stdout.flush()


class GstMRMover:
    def __init__(self):

        self.modules = []
        self.gitlab = None
        self.config_files = []
        self.gl = None
        self.mr = None
        self.mr_url = None
        self.all_projects = []
        self.skipped_branches = []
        self.git_rename_limit = None
        self.skip_on_failure = None
        self.dry_run = False

    def connect(self):
        fprint("Logging into gitlab...")

        if self.gitlab:
            gl = gitlab.Gitlab.from_config(self.gitlab, self.config_files)
            fprint(f"{green(' OK')}\n", nested=False)
            return gl

        gitlab_api_token = os.environ.get('GITLAB_API_TOKEN')
        if gitlab_api_token:
            gl = gitlab.Gitlab(URL, private_token=gitlab_api_token)
            fprint(f"{green(' OK')}\n", nested=False)
            return gl

        session = requests.Session()
        sign_in_page = session.get(SIGN_IN_URL).content.decode()
        for line in sign_in_page.split('\n'):
            m = re.search('name="authenticity_token" value="([^"]+)"', line)
            if m:
                break

        token = None
        if m:
            token = m.group(1)

        if not token:
            fprint(f"{red('Unable to find the authenticity token')}\n")
            sys.exit(1)

        for data, url in [
            ({'user[login]': 'login_or_email',
              'user[password]': 'SECRET',
              'authenticity_token': token}, LOGIN_URL),
            ({'username': 'login_or_email',
              'password': 'SECRET',
              'authenticity_token': token}, LOGIN_URL_LDAP)]:

            r = session.post(url, data=data)
            if r.status_code != 200:
                continue

            try:
                gl = gitlab.Gitlab(URL, api_version=4, session=session)
                gl.auth()
            except gitlab.exceptions.GitlabAuthenticationError as e:
                continue
            return gl

        sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n"
                      '   https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n'
                      f'and generate a token {bold("with read/write access to all but the registry")},'
                      ' then set it in the "GITLAB_API_TOKEN" environment variable:"'
                      f'\n\n  $ GITLAB_API_TOKEN=<your token> {" ".join(sys.argv)}\n'))

    def git(self, *args, can_fail=False, interaction_message=None, call=False, revert_operation=None):
        cwd = ROOT_DIR
        retry = True
        while retry:
            retry = False
            try:
                if not call:
                    try:
                        return subprocess.check_output(["git"] + list(args), cwd=cwd,
                                                       stdin=subprocess.DEVNULL,
                                                       stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError:
                        if not can_fail:
                            fprint(
                                f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False)
                        raise
                else:
                    subprocess.call(["git"] + list(args), cwd=cwd)
                    return "All good"
            except Exception as e:
                if interaction_message:
                    if self.skip_on_failure:
                        return "SKIP"
                    output = getattr(e, "output", b"")
                    if output is not None:
                        out = output.decode()
                    else:
                        out = "????"
                    fprint(f"\n```"
                           f"\n{out}\n"
                           f"Entering a shell in {cwd} to fix:\n\n"
                           f" {bold(interaction_message)}\n\n"
                           f"You should then exit with the following codes:\n\n"
                           f"  - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n"
                           f"  - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n"
                           f"  - {bold('`exit 2`')}: to skip that merge request\n"
                           f"  - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n"
                           "\n```\n", nested=False)
                    try:
                        if os.name == 'nt':
                            shell = os.environ.get(
                                "COMSPEC", r"C:\WINDOWS\system32\cmd.exe")
                        else:
                            shell = os.environ.get(
                                "SHELL", os.path.realpath("/bin/sh"))
                        subprocess.check_call(shell, cwd=cwd)
                    except subprocess.CalledProcessError as e:
                        if e.returncode == 1:
                            retry = True
                            continue
                        elif e.returncode == 2:
                            if revert_operation:
                                self.git(*revert_operation, can_fail=True)
                            return "SKIP"
                        elif e.returncode == 3:
                            if revert_operation:
                                self.git(*revert_operation, can_fail=True)
                            sys.exit(3)
                    except Exception:
                        # Result of subshell does not really matter
                        pass

                    return "User fixed it"

                if can_fail:
                    return "Failed but we do not care"

                raise e

    def cleanup_args(self):
        if self.mr_url:
            self.modules.append(GST_PROJECTS[0])
            (namespace, module, _, _, mr) = os.path.normpath(urlparse(self.mr_url).path).split('/')[1:]
            self.modules.append(module)
            self.mr = int(mr)
        elif not self.modules:
            if self.mr:
                sys.exit(f"{red(f'Merge request #{self.mr} specified without module')}\n\n"
                         f"{bold(' -> Use `--module` to specify which module the MR is from.')}")

            self.modules = GST_PROJECTS
        else:
            VALID_PROJECTS = GST_PROJECTS[1:]
            for m in self.modules:
                if m not in VALID_PROJECTS:
                    projects = '\n- '.join(VALID_PROJECTS)
                    sys.exit(
                        f"{red(f'Unknown module {m}')}\nModules are:\n- {projects}")
            if self.mr and len(self.modules) > 1:
                sys.exit(f"{red(f'Merge request #{self.mr} specified but several modules where specified')}\n\n"
                         f"{bold(' -> Use `--module` only once to specify an merge request.')}")
            self.modules.append(GST_PROJECTS[0])

    def run(self):
        self.cleanup_args()
        self.gl = self.connect()
        self.gl.auth()

        # Skip pre-commit hooks when migrating. Some users may have a
        # different version of gnu indent and that can lead to cherry-pick
        # failing.
        os.environ["GST_DISABLE_PRE_COMMIT_HOOKS"] = "1"

        try:
            prevbranch = self.git(
                "rev-parse", "--abbrev-ref", "HEAD", can_fail=True).strip()
        except Exception:
            fprint(bold(yellow("Not on a branch?\n")), indent=False)
            prevbranch = None

        try:
            self.setup_repo()

            from_projects, to_project = self.fetch_projects()

            with nested('  '):
                self.move_mrs(from_projects, to_project)
        finally:
            if self.git_rename_limit is not None:
                self.git("config", "merge.renameLimit",
                         str(self.git_rename_limit))
            if prevbranch:
                fprint(f'Back to {prevbranch}\n')
                self.git("checkout", prevbranch)

    def fetch_projects(self):
        fprint("Fetching projects... ")
        self.all_projects = [proj for proj in self.gl.projects.list(
            membership=1, all=True) if proj.name in self.modules]

        try:
            self.user_project, = [p for p in self.all_projects
                                  if p.namespace['path'] == self.gl.user.username
                                  and p.name == MONOREPO_NAME]
        except ValueError:
            fprint(
                f"{red(f'ERROR')}\n\nCould not find repository {self.gl.user.name}/{MONOREPO_NAME}")
            fprint(f"{red(f'Got to https://gitlab.freedesktop.org/gstreamer/gstreamer/ and create a fork so we can move your Merge requests.')}")
            sys.exit(1)
        fprint(f"{green(' OK')}\n", nested=False)

        from_projects = []
        user_projects_name = [proj.name for proj in self.all_projects if proj.namespace['path']
                              == self.gl.user.username and proj.name in GST_PROJECTS]
        for project, id in GST_PROJECTS_ID.items():
            if project not in user_projects_name or project == 'gstreamer':
                continue

            projects = [p for p in self.all_projects if p.id == id]
            if not projects:
                upstream_project = self.gl.projects.get(id)
            else:
                upstream_project, = projects
            assert project

            from_projects.append(upstream_project)

        fprint(f"\nMoving MRs from:\n")
        fprint(f"----------------\n")
        for p in from_projects:
            fprint(f"  - {bold(p.path_with_namespace)}\n")

        to_project = self.gl.projects.get(GST_PROJECTS_ID['gstreamer'])
        fprint(f"To: {bold(to_project.path_with_namespace)}\n\n")

        return from_projects, to_project

    def recreate_mr(self, project, to_project, mr):
        branch = f"{project.name}-{mr.source_branch}"
        if not self.create_branch_for_mr(branch, project, mr):
            return None

        description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}"

        title = mr.title
        if ':' not in mr.title:
            title = f"{project.name}: {mr.title}"

        new_mr_dict = {
            'source_branch': branch,
            'allow_collaboration': True,
            'remove_source_branch': True,
            'target_project_id': to_project.id,
            'target_branch': MONOREPO_BRANCH,
            'title': title,
            'labels': mr.labels,
            'description': description,
        }

        try:
            fprint(f"-> Recreating MR '{bold(mr.title)}'...")
            if self.dry_run:
                fprint(f"\nDry info:\n{new_mr_dict}\n")
            else:
                new_mr = self.user_project.mergerequests.create(new_mr_dict)
                fprint(f"{green(' OK')}\n", nested=False)
        except gitlab.exceptions.GitlabCreateError as e:
            fprint(f"{yellow('SKIPPED')} (An MR already exists)\n", nested=False)
            return None

        fprint(f"-> Adding discussings from MR '{mr.title}'...")
        if self.dry_run:
            fprint(f"{green(' OK')}\n", nested=False)
            return None

        new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
        for issue in mr.closes_issues():
            obj = {'body': f'Fixing MR moved to: {new_mr_url}'}
            issue.discussions.create(obj)

        mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
        for discussion in mr.discussions.list():
            # FIXME notes = [n for n in discussion.attributes['notes'] if n['type'] is not None]
            notes = [n for n in discussion.attributes['notes']]
            if not notes:
                continue

            new_discussion = None
            for note in notes:
                note = discussion.notes.get(note['id'])

                note_url = f"{mr_url}#note_{note.id}"
                when = dateparse.parse(
                    note.created_at).strftime('on %d, %b %Y')
                body = f"**{note.author['name']} - {PING_SIGN}{note.author['username']} wrote [here]({note_url})** {when}:\n\n"
                body += '\n'.join([line for line in note.body.split('\n')])

                obj = {
                    'body': body,
                    'type': note.type,
                    'resolvable': note.resolvable,
                }

                if new_discussion:
                    new_discussion.notes.create(obj)
                else:
                    new_discussion = new_mr.discussions.create(obj)

                if not note.resolvable or note.resolved:
                    new_discussion.resolved = True
                    new_discussion.save()

        fprint(f"{green(' OK')}\n", nested=False)

        print(f"New MR available at: {bold(new_mr_url)}\n")

        return new_mr

    def push_branch(self, branch):
        fprint(
            f"-> Pushing branch {branch} to remote {self.gl.user.username}...")
        if self.git("push", "--no-verify", self.gl.user.username, branch,
                    interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n  "
                    f" `$git push {self.gl.user.username} {branch}`") == "SKIP":
            fprint(yellow("'SKIPPED' (couldn't push)"), nested=False)

            return False

        fprint(f"{green(' OK')}\n", nested=False)

        return True

    def create_branch_for_mr(self, branch, project, mr):
        remote_name = project.name + '-' + self.gl.user.username
        remote_branch = f"{MONOREPO_REMOTE_NAME}/{MONOREPO_BRANCH}"
        if self.use_branch_if_exists:
            try:
                self.git("checkout", branch)
                self.git("show", remote_branch + "..", call=True)
                if self.dry_run:
                    fprint("Dry run... not creating MR")
                    return True
                cont = input('\n     Create MR [y/n]? ')
                if cont.strip().lower() != 'y':
                    fprint("Cancelled")
                    return False
                return self.push_branch(branch)
            except subprocess.CalledProcessError as e:
                pass

        self.git("remote", "add", remote_name,
                 f"{URL}{self.gl.user.username}/{project.name}.git", can_fail=True)
        self.git("fetch", remote_name)

        if self.git("checkout", remote_branch, "-b", branch,
                    interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP":
            fprint(
                bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False)
            return False

        # unset upstream to avoid to push to main (ie push.default = tracking)
        self.git("branch", branch, "--unset-upstream")

        for commit in reversed([c for c in mr.commits()]):
            if self.git("cherry-pick", commit.id,
                        interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n  "
                        f" `$ git cherry-pick {commit.id}`",
                        revert_operation=["cherry-pick", "--abort"]) == "SKIP":
                fprint(
                    f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False)
                return False

        self.git("show", remote_branch + "..", call=True)
        if self.dry_run:
            fprint("Dry run... not creating MR\n")
            return True
        cont = input('\n     Create MR [y/n]? ')
        if cont.strip().lower() != 'y':
            fprint(f"{red('Cancelled')}\n", nested=False)
            return False

        return self.push_branch(branch)

    def move_mrs(self, from_projects, to_project):
        failed_mrs = []
        found_mr = None
        for from_project in from_projects:
            with nested(f'{bold(from_project.path_with_namespace)}'):
                fprint(f'Fetching mrs')
                mrs = [mr for mr in from_project.mergerequests.list(
                    all=True, author_id=self.gl.user.id) if mr.author['username'] == self.gl.user.username and mr.state == "opened"]
                if not mrs:
                    fprint(f"{yellow(' None')}\n", nested=False)
                    continue

                fprint(f"{green(' DONE')}\n", nested=False)

                for mr in mrs:
                    if self.mr:
                        if self.mr != mr.iid:
                            continue
                        found_mr = True
                    fprint(
                        f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ')
                    if mr.source_branch in self.skipped_branches:
                        print(f"{yellow('SKIPPED')} (blacklisted branch)")
                        failed_mrs.append(
                            f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
                        continue
                    if self.list_mrs_only:
                        fprint("\n"f"List only: {yellow('SKIPPED')}\n")
                        continue

                    with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'):
                        new_mr = self.recreate_mr(from_project, to_project, mr)
                        if not new_mr:
                            if not self.dry_run:
                                failed_mrs.append(
                                    f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
                        else:
                            fprint(f"{green(' OK')}\n", nested=False)

                        self.close_mr(from_project, to_project, mr, new_mr)

            fprint(
                f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False)

        if self.mr and not found_mr:
            sys.exit(
                bold(red(f"\n==> Couldn't find MR {self.mr} in {self.modules[0]}\n")))

        for mr in failed_mrs:
            fprint(f"Didn't move MR: {mr}\n")

    def close_mr(self, project, to_project, mr, new_mr):
        if new_mr:
            new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
        else:
            new_mr_url = None
        mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
        cont = input(f'\n  Close old MR {mr_url} "{bold(mr.title)}" ? [y/n]')
        if cont.strip().lower() != 'y':
            fprint(f"{yellow('Not closing old MR')}\n")
        else:
            obj = None
            if new_mr_url:
                obj = {'body': f"Moved to: {new_mr_url}"}
            else:
                ret = input(
                    f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip()
                if ret:
                    obj = {'body': ret}

            if self.dry_run:
                fprint(f"{bold('Dry run, not closing')}\n", nested=False)
            else:
                if obj:
                    mr.discussions.create(obj)
                mr.state_event = 'close'
                mr.save()
                fprint(
                    f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n')

    def setup_repo(self):
        fprint(f"Setting up '{bold(ROOT_DIR)}'...")

        try:
            out = self.git("status", "--porcelain")
            if out:
                fprint("\n" + red('Git repository is not clean:')
                       + "\n```\n" + out + "\n```\n")
                sys.exit(1)

        except Exception as e:
            exit(
                f"Git repository{ROOT_DIR} is not clean. Clean it up before running {sys.argv[0]}\n ({e})")

        self.git('remote', 'add', MONOREPO_REMOTE_NAME,
                 MONOREPO_REMOTE, can_fail=True)
        self.git('fetch', MONOREPO_REMOTE_NAME)

        self.git('remote', 'add', self.gl.user.username,
                 f"git@gitlab.freedesktop.org:{self.gl.user.username}/gstreamer.git", can_fail=True)
        self.git('fetch', self.gl.user.username,
                 interaction_message=f"Setup your fork of {URL}gstreamer/gstreamer as remote called {self.gl.user.username}")
        fprint(f"{green(' OK')}\n", nested=False)

        try:
            git_rename_limit = int(self.git("config", "merge.renameLimit"))
        except subprocess.CalledProcessError:
            git_rename_limit = 0
        if int(git_rename_limit) < 999999:
            self.git_rename_limit = git_rename_limit
            fprint(
                "-> Setting git rename limit to 999999 so we can properly cherry-pick between repos\n")
            self.git("config", "merge.renameLimit", "999999")


def main():
    mover = GstMRMover()
    PARSER.parse_args(namespace=mover)
    mover.run()


if __name__ == '__main__':
    main()