ci: Update to use new external storage for cerbero deps

The goal of this change is to reduce egress on the FDO servers.

gitlab/build_manifest.py

@@ -47,17 +47,14 @@ os.environ['GIT_TERMINAL_PROMPT'] = '0'
 def git(*args, repository_path='.'):
     return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
 
-def get_cerbero_last_build_info (namespace : str, branch : str):
+def get_cerbero_last_build_info (branch : str):
-    base_url = f"https://gitlab.freedesktop.org/{namespace}/cerbero/-/jobs"
+    # Take the log from slowest build to reduce cache misses, the logs are
-    url = f"{base_url}/artifacts/{branch}/raw/cerbero-build/cerbero-deps.log"
+    # uploaded as soon as they are ready.
+    url = "https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/cross-ios/universal/cerbero-deps.log"
     deps = [{'commit': None}]
 
     try:
-        # The logs are only available if all jobs have passed so it does not
+        req = urllib.request.Request(url)
-        # matter which distro/arch is picked.
-        values = { 'job': "cerbero deps fedora x86_64" }
-        data = urllib.parse.urlencode(values)
-        req = urllib.request.Request(f"{url}?{data}")
         resp = urllib.request.urlopen(req);
         deps = json.loads(resp.read())
     except urllib.error.URLError:
@@ -76,7 +73,7 @@ def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]
 
     # Special case cerbero to avoid cache misses
     if module == 'cerbero':
-        sha = get_cerbero_last_build_info(namespace, branch)
+        sha = get_cerbero_last_build_info(branch)
         if sha is not None:
             return sha, sha
 

gitlab/ci_template.yml

@@ -371,6 +371,7 @@ valgrind ges:
     CERBERO_OVERRIDDEN_DIST_DIR: ""
     # location where cerbero is cached on the host
     CERBERO_HOST_DIR: "/cerbero/"
+    CERBERO_DEPS: "${CERBERO_HOME}/cerbero-deps.tar.xz"
   before_script:
     # FIXME Wrong namespace
     # Workaround build-tools having hardcoded internal path
@@ -381,13 +382,10 @@ valgrind ges:
     - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero
     - rsync -aH ${CERBERO_HOST_DIR} .
     - |
-      if test -f cerbero-deps.tar.xz ; then
+      if test -f ${CERBERO_DEPS} ; then
           echo "Extracting xz deps tarball"
           export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible
-          time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf cerbero-deps.tar.xz
+          time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf ${CERBERO_DEPS}
-      elif test -f cerbero-deps.tar.gz; then
-          echo "Extracting gz deps tarball"
-          time tar -C ${CERBERO_HOME} -xf cerbero-deps.tar.gz
       fi
     - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc
     - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc
@@ -397,7 +395,7 @@ valgrind ges:
     - $CERBERO $CERBERO_ARGS show-config
     - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
     - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
-    - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH}
+    - test -f ${CERBERO_DEPS} || $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH}
     - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x"
           && test -d ${CERBERO_HOME}/dist/${ARCH}
           && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR}
@@ -559,15 +557,12 @@ build msys2 :
           gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0
           gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0
           gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice
-    - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} --skip-fetch --job-id=${CI_JOB_ID}
     - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x"
           && mkdir -p ${CERBERO_HOME}/dist/${ARCH}
           && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH}
-    - export CERBERO_DEPS="cerbero-deps.tar.xz"
+    - $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH}
-    - export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible
+    - test "x${CERBERO_PRIVATE_SSH_KEY}" = "x"
-    - time tar -C ${CERBERO_HOME} --exclude=var/tmp --use-compress-program=xz -cf $CERBERO_DEPS
+          || $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH}
-              build-tools build-tools.cache
-              dist/${ARCH} ${ARCH}.cache
   artifacts:
     name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}"
     expire_in: '10 days'
@@ -576,7 +571,7 @@ build msys2 :
       - "manifest.xml"
       - "${CERBERO_HOME}/logs"
       - "${CERBERO_HOME}/cerbero-deps.log"
-      - "cerbero-deps.tar.*"
+      - "${CERBERO_DEPS}"
 
 #
 # Cerbero Linux X86_64 build