From 06f1dfc72474d906a94c77532ff9036f796ecc49 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 2 Dec 2021 17:57:09 -0500 Subject: [PATCH] ci: fedora-docker: Cache media files As of now all integration tests endup fetching 600MB of media every run. This will increase the docker side, but this is to the traceof that the docker image will quickly be cached on all runners. Fixes #841 Part-of: --- .gitlab-ci.yml | 21 ++++++++------------- ci/docker/fedora/prepare.sh | 9 ++++++--- ci/scripts/handle-subprojects-cache.py | 24 ++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc5791db3e..917cf6e527 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2021-10-05.0' + FEDORA_TAG: '2021-12-03.1' INDENT_TAG: '2021-10-04.0' WINDOWS_TAG: "2021-10-12.0" @@ -136,7 +136,7 @@ trigger: FDO_DISTRIBUTION_VERSION: '31' FDO_REPO_SUFFIX: "$FEDORA_AMD64_SUFFIX" FDO_DISTRIBUTION_TAG: "$FEDORA_TAG-$GST_UPSTREAM_BRANCH" - FDO_DISTRIBUTION_EXEC: 'DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH bash ci/docker/fedora/prepare.sh' + FDO_DISTRIBUTION_EXEC: 'GIT_BRANCH=$CI_COMMIT_REF_NAME GIT_URL=$CI_REPOSITORY_URL bash ci/docker/fedora/prepare.sh' fedora amd64 docker: extends: @@ -267,8 +267,10 @@ gst indent: # build setup templates # .build_template: &build + - ci/scripts/handle-subprojects-cache.py subprojects/ + # Update subprojects to respect `.wrap` content + - meson subprojects update --reset - echo $MESON_ARGS - - meson build/ $MESON_ARGS - ninja -C build/ - ccache --show-stats @@ -323,14 +325,6 @@ gst indent: timeout: '45min' variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} ${MESON_GST_WERROR}" - before_script: - - export RUSTUP_HOME="/usr/local/rustup" - - export CARGO_HOME="/usr/local/cargo" - - export PATH="/usr/local/cargo/bin:$PATH" - - - ci/scripts/handle-subprojects-cache.py subprojects/ - # Update subprojects to respect `.wrap` content - - meson subprojects update --reset script: *build after_script: @@ -399,9 +393,8 @@ build clang fedora x86_64: when: manual - changes: *modules_changes - before_script: - - ci/scripts/handle-subprojects-cache.py subprojects/ script: + - ci/scripts/handle-subprojects-cache.py subprojects/ # For some reason, options are separated by newline instead of space, so we # have to replace them first. - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") @@ -685,6 +678,8 @@ valgrind ges: - pip3 install --upgrade meson - *build - ./gst-env.py ninja -C build/ plugins_doc_caches + # Ignore modifications to wrap files made by meson + - git checkout subprojects/*.wrap - ./ci/scripts/check-documentation-diff.py - ./gst-env.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings - mv build/subprojects/gst-docs/GStreamer-doc/html documentation/ diff --git a/ci/docker/fedora/prepare.sh b/ci/docker/fedora/prepare.sh index 9ab1d986fc..5e0563face 100644 --- a/ci/docker/fedora/prepare.sh +++ b/ci/docker/fedora/prepare.sh @@ -238,17 +238,20 @@ cargo --version rustc --version # get gstreamer and make all subprojects available -# FIXME: we don't really need to clone it now as we have the repo checkout out already -git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gstreamer.git /gstreamer +git clone -b ${GIT_BRANCH} ${GIT_URL} /gstreamer +git -C /gstreamer submodule update --init --depth=1 meson subprojects download --sourcedir /gstreamer /gstreamer/ci/scripts/handle-subprojects-cache.py --build /gstreamer/subprojects/ # Run git gc to prune unwanted refs and reduce the size of the image -for i in $(find subprojects/ -mindepth 1 -maxdepth 1 -type d); +for i in $(find /subprojects/ -mindepth 1 -maxdepth 1 -type d); do git -C $i gc --aggressive || true; done +# Now remove the gstreamer clone +rm -rf /gstreamer + echo "Removing DNF cache" dnf clean all diff --git a/ci/scripts/handle-subprojects-cache.py b/ci/scripts/handle-subprojects-cache.py index 6281e094d6..9e3722111d 100755 --- a/ci/scripts/handle-subprojects-cache.py +++ b/ci/scripts/handle-subprojects-cache.py @@ -24,9 +24,23 @@ def create_cache_in_image(options): if project_name != "packagecache" and not os.path.exists(os.path.join(project_path, '.git')): continue + if os.path.exists(os.path.join(DEST, project_name)): + continue + print("Copying %s" % project_name) shutil.copytree(project_path, os.path.join(DEST, project_name)) + media_path = os.path.join(options.subprojects_dir, '..', '.git', + 'modules', 'subprojects', 'gst-integration-testsuites', 'medias') + if os.path.exists(os.path.join(DEST, 'medias.git')): + return + + if os.path.exists(media_path): + print("Creating media cache") + shutil.copytree(media_path, os.path.join(DEST, 'medias.git')) + else: + print("Did not find medias in %s" % media_path) + def copy_cache(options): # FIXME Remove when not needed anymore. @@ -37,8 +51,14 @@ def copy_cache(options): for project_name in os.listdir(path): project_path = os.path.join(options.subprojects_dir, project_name) - cache_dir = os.path.join(path, project_name) + + if project_name == 'medias.git': + project_path = os.path.join(options.subprojects_dir, '..', '.git', 'modules', + 'subprojects', 'gst-integration-testsuites') + os.makedirs(project_path, exist_ok=True) + project_path = os.path.join(project_path, 'medias') + if os.path.exists(project_path): print("- Ignoring %s" % cache_dir) continue @@ -47,7 +67,7 @@ def copy_cache(options): print("- Ignoring %s" % cache_dir) continue - print("Copying from %s" % cache_dir) + print("Copying from %s -> %s" % (cache_dir, project_path)) shutil.copytree(cache_dir, project_path)