From 085a4a9ec4ab9034cfe36dc62c89a834e9dca7de Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 23 Feb 2021 13:20:33 -0300 Subject: [PATCH] Add a scripts to ease moving pending MRs to the monorepo Part-of: --- scripts/move_mrs_to_monorepo.py | 490 ++++++++++++++++++++++++++++++++ 1 file changed, 490 insertions(+) create mode 100755 scripts/move_mrs_to_monorepo.py diff --git a/scripts/move_mrs_to_monorepo.py b/scripts/move_mrs_to_monorepo.py new file mode 100755 index 0000000000..8a71c88e7a --- /dev/null +++ b/scripts/move_mrs_to_monorepo.py @@ -0,0 +1,490 @@ +#!/usr/bin/env python3 + +from pathlib import Path +from urllib.parse import urlparse +from contextlib import contextmanager +import os +import re +import sys +try: + import gitlab +except ModuleNotFoundError: + print("========================================================================", file=sys.stderr) + print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab`", file=sys.stderr) + print("========================================================================", file=sys.stderr) + sys.exit(1) +import argparse +import requests + +import subprocess + +ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")) + +URL = "https://gitlab.freedesktop.org/" +SIGN_IN_URL = URL + 'sign_in' +LOGIN_URL = URL + 'users/sign_in' +LOGIN_URL_LDAP = URL + '/users/auth/ldapmain/callback' + +MONOREPO_REMOTE_NAME = 'origin' +NAMESPACE = "gstreamer" +MONOREPO_NAME = 'gstreamer' +MONOREPO_REMOTE = URL + f'{NAMESPACE}/{MONOREPO_NAME}' +MONOREPO_BRANCH = 'main' +PING_SIGN = '@' +MOVING_NAMESPACE = NAMESPACE + +PARSER = argparse.ArgumentParser( + description="Move merge request from old GStreamer module to the new" + "GStreamer 'monorepo'.\n" + " All your pending merge requests from all GStreamer modules will" + " be moved the the mono repository." +) +PARSER.add_argument("--skip-branch", action="store", nargs="*", + help="Ignore MRs for branches which match those names.", dest="skipped_branches") +PARSER.add_argument("--skip-on-failure", action="store_true", default=False) +PARSER.add_argument("--dry-run", "-n", action="store_true", default=False) +PARSER.add_argument("--use-branch-if-exists", action="store_true", default=False) + +GST_PROJECTS = [ + 'gstreamer', + 'gst-plugins-base', + 'gst-plugins-good', + 'gst-plugins-bad', + 'gst-plugins-ugly', + 'gst-libav', + 'gst-rtsp-server', + 'gstreamer-vaapi', + 'gstreamer-sharp', + 'gst-python', + 'gst-omx', + 'gst-editing-services', + 'gst-devtools', + 'gst-integration-testsuites', + 'gst-docs', + 'gst-examples', + 'gst-build', + 'gst-ci', +] + +# We do not want to deal with LFS +os.environ["GIT_LFS_SKIP_SMUDGE"] = "1" + + +log_depth = [] # type: T.List[str] + +@contextmanager +def nested(name=''): + global log_depth + log_depth.append(name) + try: + yield + finally: + log_depth.pop() + +def bold(text: str): + return f"\033[1m{text}\033[0m" + +def green(text: str): + return f"\033[1;32m{text}\033[0m" + +def red(text: str): + return f"\033[1;31m{text}\033[0m" + +def yellow(text: str): + return f"\033[1;33m{text}\033[0m" + +def fprint(msg, nested=True): + if log_depth: + prepend = log_depth[-1] + ' | ' if nested else '' + else: + prepend = '' + + print(prepend + msg, end="") + sys.stdout.flush() + + +class GstMRMover: + def __init__(self): + + self.gl = self.connect() + self.gl.auth() + self.all_projects = [] + self.skipped_branches = [] + self.git_rename_limit = None + self.skip_on_failure = None + self.dry_run = False + + def connect(self): + fprint("Logging into gitlab...") + gitlab_api_token = os.environ.get('GITLAB_API_TOKEN') + + if gitlab_api_token: + gl = gitlab.Gitlab(URL, private_token=gitlab_api_token) + fprint(f"{green(' OK')}\n", nested=False) + return gl + + session = requests.Session() + sign_in_page = session.get(SIGN_IN_URL).content.decode() + for l in sign_in_page.split('\n'): + m = re.search('name="authenticity_token" value="([^"]+)"', l) + if m: + break + + token = None + if m: + token = m.group(1) + + if not token: + fprint(f"{red('Unable to find the authenticity token')}\n") + sys.exit(1) + + + for data, url in [ + ({'user[login]': 'login_or_email', + 'user[password]': 'SECRET', + 'authenticity_token': token}, LOGIN_URL), + ({'username': 'login_or_email', + 'password': 'SECRET', + 'authenticity_token': token}, LOGIN_URL_LDAP)]: + + r = session.post(url, data=data) + if r.status_code != 200: + continue + + try: + gl = gitlab.Gitlab(URL, api_version=4, session=session) + gl.auth() + except gitlab.exceptions.GitlabAuthenticationError as e: + continue + return gl + + sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n" + ' https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n' + f'and generate a token {bold("with read/write access to all but the registry")},' + ' then set it in the "GITLAB_API_TOKEN" environment variable:"' + f'\n\n $ GITLAB_API_TOKEN= {" ".join(sys.argv)}\n')) + + def git(self, *args, can_fail=False, interaction_message=None, call=False): + cwd = ROOT_DIR + retry = True + while retry: + retry = False + try: + if not call: + try: + return subprocess.check_output(["git"] + list(args), cwd=cwd, + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT).decode() + except: + if not can_fail: + fprint(f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False) + raise + else: + subprocess.call(["git"] + list(args), cwd=cwd) + return "All good" + except Exception as e: + if interaction_message: + if self.skip_on_failure: + return "SKIP" + output = getattr(e, "output", b"") + if output is not None: + out = output.decode() + else: + out = "????" + fprint(f"\n```" + f"\n{out}\n" + f"Entering a shell in {cwd} to fix:\n\n" + f" {bold(interaction_message)}\n\n" + f"You should then exit with the following codes:\n\n" + f" - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n" + f" - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n" + f" - {bold('`exit 2`')}: to skip that merge request\n" + f" - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n" + "\n```\n", nested=False) + try: + if os.name == 'nt': + shell = os.environ.get( + "COMSPEC", r"C:\WINDOWS\system32\cmd.exe") + else: + shell = os.environ.get( + "SHELL", os.path.realpath("/bin/sh")) + subprocess.check_call(shell, cwd=cwd) + except subprocess.CalledProcessError as e: + if e.returncode == 1: + retry = True + continue + elif e.returncode == 2: + return "SKIP" + elif e.returncode == 3: + sys.exit(3) + except: + # Result of subshell does not really matter + pass + + return "User fixed it" + + if can_fail: + return "Failed but we do not care" + + raise e + + def run(self): + try: + self.setup_repo() + + from_projects, to_project = self.fetch_projects() + + with nested(' '): + self.move_mrs(from_projects, to_project) + finally: + if self.git_rename_limit is not None: + self.git("config", "merge.renameLimit", str(self.git_rename_limit)) + + def fetch_projects(self): + fprint("Fetching projects... ") + self.all_projects = [proj for proj in self.gl.projects.list( + membership=1, all=True) if proj.name in GST_PROJECTS] + self.user_project, = [p for p in self.all_projects if p.namespace['path'] == self.gl.user.username and p.name == MONOREPO_NAME] + fprint(f"{green(' OK')}\n", nested=False) + + from_projects = [proj for proj in self.all_projects if proj.namespace['path'] + == NAMESPACE and proj.name != "gstreamer"] + fprint(f"\nMoving MRs from:\n") + fprint(f"----------------\n") + for p in from_projects: + fprint(f" - {bold(p.path_with_namespace)}\n") + + to_project, = [p for p in self.all_projects if p.path_with_namespace == + MOVING_NAMESPACE + "/gstreamer"] + + fprint(f"To: {bold(to_project.path_with_namespace)}\n\n") + + return from_projects, to_project + + def recreate_mr(self, project, to_project, mr): + branch = f"{project.name}-{mr.source_branch}" + if not self.create_branch_for_mr(branch, project, mr): + return None + + description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}" + + new_mr_dict = { + 'source_branch': branch, + 'allow_collaboration': True, + 'remove_source_branch': True, + 'target_project_id': to_project.id, + 'target_branch': MONOREPO_BRANCH, + 'title': mr.title, + 'labels': mr.labels, + 'description': description, + } + + try: + fprint(f"-> Recreating MR '{bold(mr.title)}'...") + if self.dry_run: + fprint(f"\nDry info:\n{new_mr_dict}\n") + else: + new_mr = self.user_project.mergerequests.create(new_mr_dict) + fprint(f"{green(' OK')}\n", nested=False) + except gitlab.exceptions.GitlabCreateError as e: + fprint(f"{yellow('SKIPPED')} (An MR already exists)\n", nested=False) + return None + + fprint(f"-> Adding discussings from MR '{mr.title}'...") + if self.dry_run: + fprint(f"{green(' OK')}\n", nested=False) + return None + + new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}" + for issue in mr.closes_issues(): + obj = {'body': f'Fixing MR moved to: {new_mr_url}'} + issue.discussions.create(obj) + + mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}" + for discussion in mr.discussions.list(): + # FIXME notes = [n for n in discussion.attributes['notes'] if n['type'] is not None] + notes = [n for n in discussion.attributes['notes']] + if not notes: + continue + + new_discussion = None + for note in notes: + note_url = f"{mr_url}#note_{note['id']}" + body = f"**{note['author']['name']} - {PING_SIGN}{note['author']['username']} wrote [here]({note_url})**:\n\n" + body += '\n'.join([l for l in note['body'].split('\n')]) + + obj = {'body': body, 'type': note['type']} + if new_discussion: + new_discussion.notes.create(obj) + else: + new_discussion = new_mr.discussions.create(obj) + fprint(f"{green(' OK')}\n", nested=False) + + print(f"New MR available at: {bold(new_mr_url)}\n") + + return new_mr + + def push_branch(self, branch): + fprint(f"-> Pushing branch {branch} to remote {self.gl.user.username}...") + if self.git("push", "--no-verify", self.gl.user.username, branch, + interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n " + f" `$git push {self.gl.user.username} {branch}`") == "SKIP": + fprint(yellow("'SKIPPED' (couldn't push)"), nested=False) + + return False + + fprint(f"{green(' OK')}\n", nested=False) + + return True + + def create_branch_for_mr(self, branch, project, mr): + remote_name = project.name + '-' + self.gl.user.username + remote_branch = f"{MONOREPO_REMOTE_NAME}/{MONOREPO_BRANCH}" + if self.use_branch_if_exists: + try: + self.git("checkout", branch) + self.git("show", remote_branch + "..", call=True) + if self.dry_run: + fprint("Dry run... not creating MR") + return True + cont = input('\n Create MR [y/n]? ') + if cont.strip().lower() != 'y': + fprint("Cancelled") + return False + return self.push_branch(branch) + except subprocess.CalledProcessError as e: + pass + + self.git("remote", "add", remote_name, + f"{URL}{self.gl.user.username}/{project.name}.git", can_fail=True) + self.git("fetch", remote_name) + + if self.git("checkout", remote_branch, "-b", branch, + interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP": + fprint(bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False) + return False + + for commit in reversed([c for c in mr.commits()]): + if self.git("cherry-pick", commit.id, + interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n " + f" `$ git cherry-pick {commit.id}`") == "SKIP": + fprint(f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False) + self.git("cherry-pick", "--abort", can_fail=True) + return False + + self.git("show", remote_branch + "..", call=True) + if self.dry_run: + fprint("Dry run... not creating MR\n") + return True + cont = input('\n Create MR [y/n]? ') + if cont.strip().lower() != 'y': + fprint(f"{red('Cancelled')}\n", nested=False) + return False + + return self.push_branch(branch) + + def move_mrs(self, from_projects, to_project): + failed_mrs = [] + for from_project in from_projects: + with nested(f'{bold(from_project.path_with_namespace)}'): + fprint(f'Fetching mrs') + mrs = [mr for mr in from_project.mergerequests.list( + all=True, author_id=self.gl.user.id) if mr.author['username'] == self.gl.user.username and mr.state == "opened"] + if not mrs: + fprint(f"{yellow(' None')}\n", nested=False) + continue + + fprint(f"{green(' DONE')}\n", nested=False) + + for mr in mrs: + fprint(f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ') + if mr.source_branch in self.skipped_branches: + print(f"{yellow('SKIPPED')} (blacklisted branch)") + failed_mrs.append( + f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}") + continue + + with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'): + new_mr = self.recreate_mr(from_project, to_project, mr) + if not new_mr: + if not self.dry_run: + failed_mrs.append( + f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}") + else: + fprint(f"{green(' OK')}\n", nested=False) + + self.close_mr(from_project, to_project, mr, new_mr) + + fprint(f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False) + + for mr in failed_mrs: + print(f"Didn't move MR: {mr}") + + def close_mr(self, project, to_project, mr, new_mr): + if new_mr: + new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}" + else: + new_mr_url = None + mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}" + cont = input(f'\n Close old MR {mr_url} "{bold(mr.title)}" ? [y/n]') + if cont.strip().lower() != 'y': + fprint(f"{yellow('Not closing old MR')}\n") + else: + obj = None + if new_mr_url: + obj = {'body': f"Moved to: {new_mr_url}"} + else: + ret = input(f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip() + if ret: + obj = {'body': ret} + + if self.dry_run: + fprint(f"{bold('Dry run, not closing')}\n", nested=False) + else: + if obj: + mr.discussions.create(obj) + mr.state_event = 'close' + mr.save() + fprint(f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n') + + def setup_repo(self): + fprint(f"Setting up '{bold(ROOT_DIR)}'...") + + try: + out = self.git("status", "--porcelain") + if out: + fprint("\n" + red('Git repository is not clean:') + "\n```\n" + out + "\n```\n") + sys.exit(1) + + except Exception as e: + exit( + f"Git repository{ROOT_DIR} is not clean. Clean it up before running {sys.argv[0]}\n ({e})") + + self.git('remote', 'add', MONOREPO_REMOTE_NAME, + MONOREPO_REMOTE, can_fail=True) + self.git('fetch', MONOREPO_REMOTE_NAME) + + self.git('remote', 'add', self.gl.user.username, + f"git@gitlab.freedesktop.org:{self.gl.user.username}/gstreamer.git", can_fail=True) + self.git('fetch', self.gl.user.username, + interaction_message=f"Setup your fork of {URL}gstreamer/gstreamer as remote called {self.gl.user.username}") + fprint(f"{green(' OK')}\n", nested=False) + + try: + git_rename_limit = int(self.git("config", "merge.renameLimit")) + except subprocess.CalledProcessError: + git_rename_limit = 0 + if int(git_rename_limit) < 999999: + self.git_rename_limit = git_rename_limit + fprint("-> Setting git rename limit to 999999 so we can properly cherry-pick between repos") + self.git("config", "merge.renameLimit", "999999") + + +def main(): + mover = GstMRMover() + PARSER.parse_args(namespace=mover) + mover.run() + + +if __name__ == '__main__': + main()