diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab1a9e87bd..18ddda1aa7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,6 @@ variables: # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'main' - ORC_UPSTREAM_BRANCH: 'master' ### # IMPORTANT diff --git a/ci/gitlab/build_manifest.py b/ci/gitlab/build_manifest.py deleted file mode 100755 index 306fe1075e..0000000000 --- a/ci/gitlab/build_manifest.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import sys -import subprocess -import urllib.error -import urllib.parse -import urllib.request -import json - -from typing import Dict, Tuple, List -# from pprint import pprint - -if sys.version_info < (3, 6): - raise SystemExit('Need Python 3.6 or newer') - -GSTREAMER_MODULES: List[str] = [ - 'orc', - 'cerbero', - 'gst-build', - 'gstreamer', - 'gst-plugins-base', - 'gst-plugins-good', - 'gst-plugins-bad', - 'gst-plugins-ugly', - 'gst-libav', - 'gst-devtools', - 'gst-docs', - 'gst-editing-services', - 'gst-omx', - 'gst-python', - 'gst-rtsp-server', - 'gstreamer-sharp', - 'gstreamer-vaapi', - 'gst-integration-testsuites', - 'gst-examples', -] - -MANIFEST_TEMPLATE: str = """ - - - -{} -""" - - -CERBERO_DEPS_LOGS_TARGETS = ( - ('cross-ios', 'universal'), - ('cross-windows-mingw', 'x86'), - ('cross-windows-mingw', 'x86_64'), - ('cross-android', 'universal'), - ('fedora', 'x86_64'), - ('macos', 'x86_64'), - ('windows-msvc', 'x86_64'), -) - -# Disallow git prompting for a username/password -os.environ['GIT_TERMINAL_PROMPT'] = '0' -def git(*args, repository_path='.'): - return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() - -def get_cerbero_last_build_info (branch : str): - # Fetch the deps log for all (distro, arch) targets - all_commits = {} - for distro, arch in CERBERO_DEPS_LOGS_TARGETS: - url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/{distro}/{arch}/cerbero-deps.log' - print(f'Fetching {url}') - try: - req = urllib.request.Request(url) - resp = urllib.request.urlopen(req); - deps = json.loads(resp.read()) - except urllib.error.URLError as e: - print(f'WARNING: Failed to GET {url}: {e!s}') - continue - - for dep in deps: - commit = dep['commit'] - if commit not in all_commits: - all_commits[commit] = [] - all_commits[commit].append((distro, arch)) - - # Fetch the cerbero commit that has the most number of caches - best_commit = None - newest_commit = None - max_caches = 0 - total_caches = len(CERBERO_DEPS_LOGS_TARGETS) - for commit, targets in all_commits.items(): - if newest_commit is None: - newest_commit = commit - have_caches = len(targets) - # If this commit has caches for all targets, just use it - if have_caches == total_caches: - best_commit = commit - break - # Else, try to find the commit with the most caches - if have_caches > max_caches: - max_caches = have_caches - best_commit = commit - if newest_commit is None: - print('WARNING: No deps logs were found, will build from scratch') - if best_commit != newest_commit: - print(f'WARNING: Cache is not up-to-date for commit {newest_commit}, using commit {best_commit} instead') - return best_commit - - -def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]: - try: - res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch) - except subprocess.CalledProcessError: - return None, None - - if not res: - return None, None - - # Special case cerbero to avoid cache misses - if module == 'cerbero': - sha = get_cerbero_last_build_info(branch) - if sha is not None: - return sha, sha - - lines = res.split('\n') - for line in lines: - if line.endswith('/' + branch): - try: - sha, refname = line.split('\t') - except ValueError: - continue - return refname.strip(), sha - - return None, None - - -def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]: - namespace: str = os.environ["CI_PROJECT_NAMESPACE"] - ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master') - - if module == "orc": - ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master') - - if module == os.environ['CI_PROJECT_NAME']: - return 'user', branchname, os.environ['CI_COMMIT_SHA'] - - if branchname != ups_branch: - remote_refname, sha = get_branch_info(module, namespace, branchname) - if sha is not None: - return 'user', remote_refname, sha - - # Check upstream project for a branch - remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch) - if sha is not None: - return 'origin', remote_refname, sha - - # This should never occur given the upstream fallback above - print(f"Could not find anything for {module}:{branchname}") - print("If something reaches that point, please file a bug") - print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues") - assert False - - -# --- Unit tests --- # -# Basically, pytest will happily let a test mutate a variable, and then run -# the next tests one the same environment without reset the vars. -def preserve_ci_vars(func): - """Preserve the original CI Variable values""" - def wrapper(): - try: - url = os.environ["CI_PROJECT_URL"] - user = os.environ["CI_PROJECT_NAMESPACE"] - except KeyError: - url = "invalid" - user = "" - - private = os.getenv("READ_PROJECTS_TOKEN", default=None) - if not private: - os.environ["READ_PROJECTS_TOKEN"] = "FOO" - - func() - - os.environ["CI_PROJECT_URL"] = url - os.environ["CI_PROJECT_NAMESPACE"] = user - - if private: - os.environ["READ_PROJECTS_TOKEN"] = private - # if it was set after - elif os.getenv("READ_PROJECTS_TOKEN", default=None): - del os.environ["READ_PROJECTS_TOKEN"] - - return wrapper - -@preserve_ci_vars -def test_find_repository_sha(): - os.environ["CI_PROJECT_NAME"] = "some-random-project" - os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" - os.environ["CI_PROJECT_NAMESPACE"] = "alatiera" - os.environ["GST_UPSTREAM_BRANCH"] = "master" - del os.environ["READ_PROJECTS_TOKEN"] - - # This should find the repository in the user namespace - remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2") - assert remote == "user" - assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325" - assert refname == "refs/heads/1.2" - - # This should fallback to upstream master branch since no matching branch was found - remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name") - assert remote == "origin" - assert refname == "refs/heads/master" - - os.environ["CI_PROJECT_NAME"] = "the_project" - os.environ["CI_COMMIT_SHA"] = "MySha" - - remote, refname, git_ref = find_repository_sha("the_project", "whatever") - assert remote == "user" - assert git_ref == "MySha" - assert refname == "whatever" - - -@preserve_ci_vars -def test_get_project_branch(): - os.environ["CI_PROJECT_NAME"] = "some-random-project" - os.environ["CI_COMMIT_SHA"] = "dwbuiw" - os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" - os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_" - del os.environ["READ_PROJECTS_TOKEN"] - - os.environ['GST_UPSTREAM_BRANCH'] = '1.12' - remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12') - assert twelve is not None - assert remote == 'origin' - assert refname == "refs/heads/1.12" - - os.environ['GST_UPSTREAM_BRANCH'] = '1.14' - remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14') - assert fourteen is not None - assert remote == 'origin' - assert refname == "refs/heads/1.14" - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--self-update", action="store_true", default=False) - parser.add_argument(dest="output", default='manifest.xml', nargs='?') - options = parser.parse_args() - - current_branch: str = os.environ['CI_COMMIT_REF_NAME'] - user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL']) - if not user_remote_url.endswith('/'): - user_remote_url += '/' - - if options.self_update: - remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch) - if remote == 'user': - remote = user_remote_url + 'gst-ci' - else: - remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci" - - git('fetch', remote, remote_refname) - git('checkout', '--detach', sha) - sys.exit(0) - - projects: str = '' - for module in GSTREAMER_MODULES: - print(f"Checking {module}:", end=' ') - remote, refname, revision = find_repository_sha(module, current_branch) - print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'") - projects += f" \n" - - with open(options.output, mode='w') as manifest: - print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest) diff --git a/ci/gitlab/clone_manifest_ref.py b/ci/gitlab/clone_manifest_ref.py deleted file mode 100755 index b26043b510..0000000000 --- a/ci/gitlab/clone_manifest_ref.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import subprocess - -from collections import namedtuple -import xml.etree.ElementTree as ET - -# Disallow git prompting for a username/password -os.environ['GIT_TERMINAL_PROMPT'] = '0' -def git(*args, repository_path='.'): - return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() - -class Manifest(object): - ''' - Parse and store the content of a manifest file - ''' - - remotes = {} - projects = {} - default_remote = 'origin' - default_revision = 'refs/heads/master' - - def __init__(self, manifest_path): - self.manifest_path = manifest_path - - def parse(self): - try: - tree = ET.parse(self.manifest_path) - except Exception as ex: - raise Exception("Error loading manifest %s in file %s" % (self.manifest_path, ex)) - - root = tree.getroot() - - for child in root: - if child.tag == 'remote': - self.remotes[child.attrib['name']] = child.attrib['fetch'] - if child.tag == 'default': - self.default_remote = child.attrib['remote'] or self.default_remote - self.default_revision = child.attrib['revision'] or self.default_revision - if child.tag == 'project': - project = namedtuple('Project', ['name', 'remote', - 'revision', 'fetch_uri']) - - project.name = child.attrib['name'] - if project.name.endswith('.git'): - project.name = project.name[:-4] - project.remote = child.attrib.get('remote') or self.default_remote - project.revision = child.attrib.get('revision') or self.default_revision - project.fetch_uri = self.remotes[project.remote] + project.name + '.git' - - self.projects[project.name] = project - - def find_project(self, name): - try: - return self.projects[name] - except KeyError as ex: - raise Exception("Could not find project %s in manifest %s" % (name, self.manifest_path)) - - def get_fetch_uri(self, project, remote): - fetch = self.remotes[remote] - return fetch + project.name + '.git' - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--project", action="store", type=str) - parser.add_argument("--destination", action="store", type=str, default='.') - parser.add_argument("--manifest", action="store", type=str) - parser.add_argument("--fetch", action="store_true", default=False) - options = parser.parse_args() - - if not options.project: - raise ValueError("--project argument not provided") - if not options.manifest: - raise ValueError("--manifest argument not provided") - - manifest = Manifest(options.manifest) - manifest.parse() - project = manifest.find_project(options.project) - - dest = options.destination - if dest == '.': - dest = os.path.join (os.getcwd(), project.name) - - if options.fetch: - assert os.path.exists(dest) == True - git('fetch', project.fetch_uri, project.revision, repository_path=dest) - else: - git('clone', project.fetch_uri, dest) - - git('checkout', '--detach', project.revision, repository_path=dest)