mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-11-22 09:41:07 +00:00
ci: remove unused files
These are leftovers of the gst-ci repo, before the mono repo merge and can be removed now. Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2403>
This commit is contained in:
Jordan Petridis
GStreamer Marge Bot
parent
a5cd1adc97
commit
fff82f5650
3 changed files with 0 additions and 363 deletions
|
|
@ -17,7 +17,6 @@ variables:
|
|||
|
||||
# Branch to track for modules that have no ref specified in the manifest
|
||||
GST_UPSTREAM_BRANCH: 'main'
|
||||
ORC_UPSTREAM_BRANCH: 'master'
|
||||
|
||||
###
|
||||
# IMPORTANT
|
||||
|
|
|
|||
|
|
@ -1,270 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import json
|
||||
|
||||
from typing import Dict, Tuple, List
|
||||
# from pprint import pprint
|
||||
|
||||
if sys.version_info < (3, 6):
|
||||
raise SystemExit('Need Python 3.6 or newer')
|
||||
|
||||
GSTREAMER_MODULES: List[str] = [
|
||||
'orc',
|
||||
'cerbero',
|
||||
'gst-build',
|
||||
'gstreamer',
|
||||
'gst-plugins-base',
|
||||
'gst-plugins-good',
|
||||
'gst-plugins-bad',
|
||||
'gst-plugins-ugly',
|
||||
'gst-libav',
|
||||
'gst-devtools',
|
||||
'gst-docs',
|
||||
'gst-editing-services',
|
||||
'gst-omx',
|
||||
'gst-python',
|
||||
'gst-rtsp-server',
|
||||
'gstreamer-sharp',
|
||||
'gstreamer-vaapi',
|
||||
'gst-integration-testsuites',
|
||||
'gst-examples',
|
||||
]
|
||||
|
||||
MANIFEST_TEMPLATE: str = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<manifest>
|
||||
<remote fetch="{}" name="user"/>
|
||||
<remote fetch="https://gitlab.freedesktop.org/gstreamer/" name="origin"/>
|
||||
{}
|
||||
</manifest>"""
|
||||
|
||||
|
||||
CERBERO_DEPS_LOGS_TARGETS = (
|
||||
('cross-ios', 'universal'),
|
||||
('cross-windows-mingw', 'x86'),
|
||||
('cross-windows-mingw', 'x86_64'),
|
||||
('cross-android', 'universal'),
|
||||
('fedora', 'x86_64'),
|
||||
('macos', 'x86_64'),
|
||||
('windows-msvc', 'x86_64'),
|
||||
)
|
||||
|
||||
# Disallow git prompting for a username/password
|
||||
os.environ['GIT_TERMINAL_PROMPT'] = '0'
|
||||
def git(*args, repository_path='.'):
|
||||
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
|
||||
|
||||
def get_cerbero_last_build_info (branch : str):
|
||||
# Fetch the deps log for all (distro, arch) targets
|
||||
all_commits = {}
|
||||
for distro, arch in CERBERO_DEPS_LOGS_TARGETS:
|
||||
url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/{distro}/{arch}/cerbero-deps.log'
|
||||
print(f'Fetching {url}')
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
resp = urllib.request.urlopen(req);
|
||||
deps = json.loads(resp.read())
|
||||
except urllib.error.URLError as e:
|
||||
print(f'WARNING: Failed to GET {url}: {e!s}')
|
||||
continue
|
||||
|
||||
for dep in deps:
|
||||
commit = dep['commit']
|
||||
if commit not in all_commits:
|
||||
all_commits[commit] = []
|
||||
all_commits[commit].append((distro, arch))
|
||||
|
||||
# Fetch the cerbero commit that has the most number of caches
|
||||
best_commit = None
|
||||
newest_commit = None
|
||||
max_caches = 0
|
||||
total_caches = len(CERBERO_DEPS_LOGS_TARGETS)
|
||||
for commit, targets in all_commits.items():
|
||||
if newest_commit is None:
|
||||
newest_commit = commit
|
||||
have_caches = len(targets)
|
||||
# If this commit has caches for all targets, just use it
|
||||
if have_caches == total_caches:
|
||||
best_commit = commit
|
||||
break
|
||||
# Else, try to find the commit with the most caches
|
||||
if have_caches > max_caches:
|
||||
max_caches = have_caches
|
||||
best_commit = commit
|
||||
if newest_commit is None:
|
||||
print('WARNING: No deps logs were found, will build from scratch')
|
||||
if best_commit != newest_commit:
|
||||
print(f'WARNING: Cache is not up-to-date for commit {newest_commit}, using commit {best_commit} instead')
|
||||
return best_commit
|
||||
|
||||
|
||||
def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]:
|
||||
try:
|
||||
res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch)
|
||||
except subprocess.CalledProcessError:
|
||||
return None, None
|
||||
|
||||
if not res:
|
||||
return None, None
|
||||
|
||||
# Special case cerbero to avoid cache misses
|
||||
if module == 'cerbero':
|
||||
sha = get_cerbero_last_build_info(branch)
|
||||
if sha is not None:
|
||||
return sha, sha
|
||||
|
||||
lines = res.split('\n')
|
||||
for line in lines:
|
||||
if line.endswith('/' + branch):
|
||||
try:
|
||||
sha, refname = line.split('\t')
|
||||
except ValueError:
|
||||
continue
|
||||
return refname.strip(), sha
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]:
|
||||
namespace: str = os.environ["CI_PROJECT_NAMESPACE"]
|
||||
ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master')
|
||||
|
||||
if module == "orc":
|
||||
ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master')
|
||||
|
||||
if module == os.environ['CI_PROJECT_NAME']:
|
||||
return 'user', branchname, os.environ['CI_COMMIT_SHA']
|
||||
|
||||
if branchname != ups_branch:
|
||||
remote_refname, sha = get_branch_info(module, namespace, branchname)
|
||||
if sha is not None:
|
||||
return 'user', remote_refname, sha
|
||||
|
||||
# Check upstream project for a branch
|
||||
remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch)
|
||||
if sha is not None:
|
||||
return 'origin', remote_refname, sha
|
||||
|
||||
# This should never occur given the upstream fallback above
|
||||
print(f"Could not find anything for {module}:{branchname}")
|
||||
print("If something reaches that point, please file a bug")
|
||||
print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues")
|
||||
assert False
|
||||
|
||||
|
||||
# --- Unit tests --- #
|
||||
# Basically, pytest will happily let a test mutate a variable, and then run
|
||||
# the next tests one the same environment without reset the vars.
|
||||
def preserve_ci_vars(func):
|
||||
"""Preserve the original CI Variable values"""
|
||||
def wrapper():
|
||||
try:
|
||||
url = os.environ["CI_PROJECT_URL"]
|
||||
user = os.environ["CI_PROJECT_NAMESPACE"]
|
||||
except KeyError:
|
||||
url = "invalid"
|
||||
user = ""
|
||||
|
||||
private = os.getenv("READ_PROJECTS_TOKEN", default=None)
|
||||
if not private:
|
||||
os.environ["READ_PROJECTS_TOKEN"] = "FOO"
|
||||
|
||||
func()
|
||||
|
||||
os.environ["CI_PROJECT_URL"] = url
|
||||
os.environ["CI_PROJECT_NAMESPACE"] = user
|
||||
|
||||
if private:
|
||||
os.environ["READ_PROJECTS_TOKEN"] = private
|
||||
# if it was set after
|
||||
elif os.getenv("READ_PROJECTS_TOKEN", default=None):
|
||||
del os.environ["READ_PROJECTS_TOKEN"]
|
||||
|
||||
return wrapper
|
||||
|
||||
@preserve_ci_vars
|
||||
def test_find_repository_sha():
|
||||
os.environ["CI_PROJECT_NAME"] = "some-random-project"
|
||||
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
|
||||
os.environ["CI_PROJECT_NAMESPACE"] = "alatiera"
|
||||
os.environ["GST_UPSTREAM_BRANCH"] = "master"
|
||||
del os.environ["READ_PROJECTS_TOKEN"]
|
||||
|
||||
# This should find the repository in the user namespace
|
||||
remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2")
|
||||
assert remote == "user"
|
||||
assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325"
|
||||
assert refname == "refs/heads/1.2"
|
||||
|
||||
# This should fallback to upstream master branch since no matching branch was found
|
||||
remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name")
|
||||
assert remote == "origin"
|
||||
assert refname == "refs/heads/master"
|
||||
|
||||
os.environ["CI_PROJECT_NAME"] = "the_project"
|
||||
os.environ["CI_COMMIT_SHA"] = "MySha"
|
||||
|
||||
remote, refname, git_ref = find_repository_sha("the_project", "whatever")
|
||||
assert remote == "user"
|
||||
assert git_ref == "MySha"
|
||||
assert refname == "whatever"
|
||||
|
||||
|
||||
@preserve_ci_vars
|
||||
def test_get_project_branch():
|
||||
os.environ["CI_PROJECT_NAME"] = "some-random-project"
|
||||
os.environ["CI_COMMIT_SHA"] = "dwbuiw"
|
||||
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
|
||||
os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_"
|
||||
del os.environ["READ_PROJECTS_TOKEN"]
|
||||
|
||||
os.environ['GST_UPSTREAM_BRANCH'] = '1.12'
|
||||
remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12')
|
||||
assert twelve is not None
|
||||
assert remote == 'origin'
|
||||
assert refname == "refs/heads/1.12"
|
||||
|
||||
os.environ['GST_UPSTREAM_BRANCH'] = '1.14'
|
||||
remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14')
|
||||
assert fourteen is not None
|
||||
assert remote == 'origin'
|
||||
assert refname == "refs/heads/1.14"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--self-update", action="store_true", default=False)
|
||||
parser.add_argument(dest="output", default='manifest.xml', nargs='?')
|
||||
options = parser.parse_args()
|
||||
|
||||
current_branch: str = os.environ['CI_COMMIT_REF_NAME']
|
||||
user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL'])
|
||||
if not user_remote_url.endswith('/'):
|
||||
user_remote_url += '/'
|
||||
|
||||
if options.self_update:
|
||||
remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch)
|
||||
if remote == 'user':
|
||||
remote = user_remote_url + 'gst-ci'
|
||||
else:
|
||||
remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci"
|
||||
|
||||
git('fetch', remote, remote_refname)
|
||||
git('checkout', '--detach', sha)
|
||||
sys.exit(0)
|
||||
|
||||
projects: str = ''
|
||||
for module in GSTREAMER_MODULES:
|
||||
print(f"Checking {module}:", end=' ')
|
||||
remote, refname, revision = find_repository_sha(module, current_branch)
|
||||
print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'")
|
||||
projects += f" <project path=\"{module}\" name=\"{module}.git\" remote=\"{remote}\" revision=\"{revision}\" refname=\"{refname}\" />\n"
|
||||
|
||||
with open(options.output, mode='w') as manifest:
|
||||
print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest)
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from collections import namedtuple
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
# Disallow git prompting for a username/password
|
||||
os.environ['GIT_TERMINAL_PROMPT'] = '0'
|
||||
def git(*args, repository_path='.'):
|
||||
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
|
||||
|
||||
class Manifest(object):
|
||||
'''
|
||||
Parse and store the content of a manifest file
|
||||
'''
|
||||
|
||||
remotes = {}
|
||||
projects = {}
|
||||
default_remote = 'origin'
|
||||
default_revision = 'refs/heads/master'
|
||||
|
||||
def __init__(self, manifest_path):
|
||||
self.manifest_path = manifest_path
|
||||
|
||||
def parse(self):
|
||||
try:
|
||||
tree = ET.parse(self.manifest_path)
|
||||
except Exception as ex:
|
||||
raise Exception("Error loading manifest %s in file %s" % (self.manifest_path, ex))
|
||||
|
||||
root = tree.getroot()
|
||||
|
||||
for child in root:
|
||||
if child.tag == 'remote':
|
||||
self.remotes[child.attrib['name']] = child.attrib['fetch']
|
||||
if child.tag == 'default':
|
||||
self.default_remote = child.attrib['remote'] or self.default_remote
|
||||
self.default_revision = child.attrib['revision'] or self.default_revision
|
||||
if child.tag == 'project':
|
||||
project = namedtuple('Project', ['name', 'remote',
|
||||
'revision', 'fetch_uri'])
|
||||
|
||||
project.name = child.attrib['name']
|
||||
if project.name.endswith('.git'):
|
||||
project.name = project.name[:-4]
|
||||
project.remote = child.attrib.get('remote') or self.default_remote
|
||||
project.revision = child.attrib.get('revision') or self.default_revision
|
||||
project.fetch_uri = self.remotes[project.remote] + project.name + '.git'
|
||||
|
||||
self.projects[project.name] = project
|
||||
|
||||
def find_project(self, name):
|
||||
try:
|
||||
return self.projects[name]
|
||||
except KeyError as ex:
|
||||
raise Exception("Could not find project %s in manifest %s" % (name, self.manifest_path))
|
||||
|
||||
def get_fetch_uri(self, project, remote):
|
||||
fetch = self.remotes[remote]
|
||||
return fetch + project.name + '.git'
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--project", action="store", type=str)
|
||||
parser.add_argument("--destination", action="store", type=str, default='.')
|
||||
parser.add_argument("--manifest", action="store", type=str)
|
||||
parser.add_argument("--fetch", action="store_true", default=False)
|
||||
options = parser.parse_args()
|
||||
|
||||
if not options.project:
|
||||
raise ValueError("--project argument not provided")
|
||||
if not options.manifest:
|
||||
raise ValueError("--manifest argument not provided")
|
||||
|
||||
manifest = Manifest(options.manifest)
|
||||
manifest.parse()
|
||||
project = manifest.find_project(options.project)
|
||||
|
||||
dest = options.destination
|
||||
if dest == '.':
|
||||
dest = os.path.join (os.getcwd(), project.name)
|
||||
|
||||
if options.fetch:
|
||||
assert os.path.exists(dest) == True
|
||||
git('fetch', project.fetch_uri, project.revision, repository_path=dest)
|
||||
else:
|
||||
git('clone', project.fetch_uri, dest)
|
||||
|
||||
git('checkout', '--detach', project.revision, repository_path=dest)
|
||||
Loading…
Reference in a new issue