gstreamer/gitlab/build_manifest.py
Jordan Petridis 2cde4bef1e gitlab/build_manifest: allow for upstream branch to be specified
When the pipeline is based on top of a stable branch, we want
to track that branch isntead of the primary development branch.

This patch makes it so the upstream branch can be specified with
an env var.

part of #11
2019-06-05 22:51:04 +00:00

231 lines
7.7 KiB
Python
Executable file

#!/usr/bin/env python3
import argparse
import os
import sys
import subprocess
import urllib.error
import urllib.parse
import urllib.request
import json
from typing import Dict, Tuple, List
# from pprint import pprint
GSTREAMER_MODULES: List[str] = [
'orc',
'cerbero',
'gst-build',
'gstreamer',
'gst-plugins-base',
'gst-plugins-good',
'gst-plugins-bad',
'gst-plugins-ugly',
'gst-libav',
'gst-devtools',
'gst-docs',
'gst-editing-services',
'gst-omx',
'gst-python',
'gst-rtsp-server',
'gstreamer-sharp',
'gstreamer-vaapi',
'gst-integration-testsuites',
'gst-examples',
]
MANIFEST_TEMPLATE: str = """<?xml version="1.0" encoding="UTF-8"?>
<manifest>
<remote fetch="{}" name="user"/>
<remote fetch="https://gitlab.freedesktop.org/gstreamer/" name="origin"/>
{}
</manifest>"""
# Disallow git prompting for a username/password
os.environ['GIT_TERMINAL_PROMPT'] = '0'
def git(*args, repository_path='.'):
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
def get_cerbero_last_build_info (namespace : str, branch : str):
base_url = f"https://gitlab.freedesktop.org/{namespace}/cerbero/-/jobs"
url = f"{base_url}/artifacts/{branch}/raw/cerbero-build/cerbero-deps.log"
deps = [{'commit': None}]
try:
# The logs are only available if all jobs have passed so it does not
# matter which distro/arch is picked.
values = { 'job': "cerbero deps fedora x86_64" }
data = urllib.parse.urlencode(values)
req = urllib.request.Request(f"{url}?{data}")
resp = urllib.request.urlopen(req);
deps = json.loads(resp.read())
except urllib.error.URLError:
return None
return deps[0]['commit']
def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]:
try:
res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch)
except subprocess.CalledProcessError:
return None, None
if not res:
return None, None
# Special case cerbero to avoid cache misses
if module == 'cerbero':
sha = get_cerbero_last_build_info(namespace, branch)
if sha is not None:
return sha, sha
lines = res.split('\n')
for line in lines:
if line.endswith('/' + branch):
try:
sha, refname = line.split('\t')
except ValueError:
continue
return refname.strip(), sha
return None, None
def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]:
namespace: str = os.environ["CI_PROJECT_NAMESPACE"]
ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master')
if module == "orc":
ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master')
if module == os.environ['CI_PROJECT_NAME']:
return 'user', branchname, os.environ['CI_COMMIT_SHA']
if branchname != ups_branch:
remote_refname, sha = get_branch_info(module, namespace, branchname)
if sha is not None:
return 'user', remote_refname, sha
# Check upstream project for a branch
remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch)
if sha is not None:
return 'origin', remote_refname, sha
# This should never occur given the upstream fallback above
print(f"Could not find anything for {module}:{branchname}")
print("If something reaches that point, please file a bug")
print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues")
assert False
# --- Unit tests --- #
# Basically, pytest will happily let a test mutate a variable, and then run
# the next tests one the same environment without reset the vars.
def preserve_ci_vars(func):
"""Preserve the original CI Variable values"""
def wrapper():
try:
url = os.environ["CI_PROJECT_URL"]
user = os.environ["CI_PROJECT_NAMESPACE"]
except KeyError:
url = "invalid"
user = ""
private = os.getenv("READ_PROJECTS_TOKEN", default=None)
if not private:
os.environ["READ_PROJECTS_TOKEN"] = "FOO"
func()
os.environ["CI_PROJECT_URL"] = url
os.environ["CI_PROJECT_NAMESPACE"] = user
if private:
os.environ["READ_PROJECTS_TOKEN"] = private
# if it was set after
elif os.getenv("READ_PROJECTS_TOKEN", default=None):
del os.environ["READ_PROJECTS_TOKEN"]
return wrapper
@preserve_ci_vars
def test_find_repository_sha():
os.environ["CI_PROJECT_NAME"] = "some-random-project"
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
os.environ["CI_PROJECT_NAMESPACE"] = "alatiera"
os.environ["GST_UPSTREAM_BRANCH"] = "master"
del os.environ["READ_PROJECTS_TOKEN"]
# This should find the repository in the user namespace
remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2")
assert remote == "user"
assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325"
assert refname == "refs/heads/1.2"
# This should fallback to upstream master branch since no matching branch was found
remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name")
assert remote == "origin"
assert refname == "refs/heads/master"
os.environ["CI_PROJECT_NAME"] = "the_project"
os.environ["CI_COMMIT_SHA"] = "MySha"
remote, refname, git_ref = find_repository_sha("the_project", "whatever")
assert remote == "user"
assert git_ref == "MySha"
assert refname == "whatever"
@preserve_ci_vars
def test_get_project_branch():
os.environ["CI_PROJECT_NAME"] = "some-random-project"
os.environ["CI_COMMIT_SHA"] = "dwbuiw"
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_"
del os.environ["READ_PROJECTS_TOKEN"]
os.environ['GST_UPSTREAM_BRANCH'] = '1.12'
remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12')
assert twelve is not None
assert remote == 'origin'
assert refname == "refs/heads/1.12"
os.environ['GST_UPSTREAM_BRANCH'] = '1.14'
remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14')
assert fourteen is not None
assert remote == 'origin'
assert refname == "refs/heads/1.14"
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--self-update", action="store_true", default=False)
parser.add_argument(dest="output", default='manifest.xml', nargs='?')
options = parser.parse_args()
current_branch: str = os.environ['CI_COMMIT_REF_NAME']
user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL'])
if not user_remote_url.endswith('/'):
user_remote_url += '/'
if options.self_update:
remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch)
if remote == 'user':
remote = user_remote_url + 'gst-ci'
else:
remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci"
git('fetch', remote, remote_refname)
git('checkout', '--detach', sha)
sys.exit(0)
projects: str = ''
for module in GSTREAMER_MODULES:
print(f"Checking {module}:", end=' ')
remote, refname, revision = find_repository_sha(module, current_branch)
print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'")
projects += f" <project path=\"{module}\" name=\"{module}.git\" remote=\"{remote}\" revision=\"{revision}\" refname=\"{refname}\" />\n"
with open(options.output, mode='w') as manifest:
print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest)