gitlab/build_manifest: allow for upstream branch to be specified

When the pipeline is based on top of a stable branch, we want
to track that branch isntead of the primary development branch.

This patch makes it so the upstream branch can be specified with
an env var.

part of #11

gitlab/build_manifest.py

@@ -65,9 +65,9 @@ def get_cerbero_last_build_info (namespace : str, branch : str):
 
     return deps[0]['commit']
 
-def get_branches_info(module: str, namespace: str, branches: List[str]) -> Tuple[str, str]:
+def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]:
     try:
-        res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', *branches)
+        res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch)
     except subprocess.CalledProcessError:
         return None, None
 
@@ -76,37 +76,39 @@ def get_branches_info(module: str, namespace: str, branches: List[str]) -> Tuple
 
     # Special case cerbero to avoid cache misses
     if module == 'cerbero':
-        for branch in branches:
-            sha = get_cerbero_last_build_info(namespace, branch)
-            if sha is not None:
-                return sha, sha
+        sha = get_cerbero_last_build_info(namespace, branch)
+        if sha is not None:
+            return sha, sha
 
     lines = res.split('\n')
-    for branch in branches:
-        for line in lines:
-            if line.endswith('/' + branch):
-                try:
-                    sha, refname = line.split('\t')
-                except ValueError:
-                    continue
-                return refname.strip(), sha
+    for line in lines:
+        if line.endswith('/' + branch):
+            try:
+                sha, refname = line.split('\t')
+            except ValueError:
+                continue
+            return refname.strip(), sha
 
     return None, None
 
 
 def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]:
     namespace: str = os.environ["CI_PROJECT_NAMESPACE"]
+    ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master')
+
+    if module == "orc":
+        ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master')
 
     if module == os.environ['CI_PROJECT_NAME']:
         return 'user', branchname, os.environ['CI_COMMIT_SHA']
 
-    if branchname != "master":
-        remote_refname, sha = get_branches_info(module, namespace, [branchname])
+    if branchname != ups_branch:
+        remote_refname, sha = get_branch_info(module, namespace, branchname)
         if sha is not None:
             return 'user', remote_refname, sha
 
     # Check upstream project for a branch
-    remote_refname, sha = get_branches_info(module, 'gstreamer', [branchname, 'master'])
+    remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch)
     if sha is not None:
         return 'origin', remote_refname, sha
 
@@ -152,6 +154,7 @@ def test_find_repository_sha():
     os.environ["CI_PROJECT_NAME"] = "some-random-project"
     os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
     os.environ["CI_PROJECT_NAMESPACE"] = "alatiera"
+    os.environ["GST_UPSTREAM_BRANCH"] = "master"
     del os.environ["READ_PROJECTS_TOKEN"]
 
     # This should find the repository in the user namespace
@@ -182,11 +185,13 @@ def test_get_project_branch():
     os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_"
     del os.environ["READ_PROJECTS_TOKEN"]
 
+    os.environ['GST_UPSTREAM_BRANCH'] = '1.12'
     remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12')
     assert twelve is not None
     assert remote == 'origin'
     assert refname == "refs/heads/1.12"
 
+    os.environ['GST_UPSTREAM_BRANCH'] = '1.14'
     remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14')
     assert fourteen is not None
     assert remote == 'origin'

gitlab/ci_template.yml

@@ -18,6 +18,10 @@ variables:
   INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb'
   MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e'
 
+  # Branch to track for modules that have no ref specified in the manifest
+  GST_UPSTREAM_BRANCH: 'master'
+  ORC_UPSTREAM_BRANCH: 'master'
+
   GIT_STRATEGY: none
   MESON_BUILDTYPE_ARGS: --default-library=both
   DEFAULT_MESON_ARGS: >
@@ -57,7 +61,7 @@ gst indent:
   script:
     # man indent. grep RETURN VALUE, grab a beer on my behalf...
     - indent --version || true
-    - curl -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent
+    - curl -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/${GST_UPSTREAM_BRANCH}/tools/gst-indent
     - chmod +x gst-indent
     - find . -name '*.c' -exec ./gst-indent {} +
     - |
@@ -523,7 +527,7 @@ build cerbero cross win64:
     GSTREAMER_ROOT_ANDROID: ${CI_PROJECT_DIR}/examples/cerbero-android-universal
   script:
     - mkdir -p ${EXAMPLES_HOME}/outputs
-    - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/master/gitlab/clone_manifest_ref.py
+    - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py
     - chmod +x clone_manifest_ref.py
     - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples
     - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs