From b36d813f5b2c4588886641afbf6a7c7f9774fffe Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 21 Apr 2020 11:54:04 +0300 Subject: [PATCH] ci_template: don't use prebuilt binaries for test jobs Previously we were optimizing for cpu time, so we where building gst-build once and then exporting that to be used by the test jobs. However this meant that we where uploading 200mb (previously 600mb) zipped of artifacts and then re-downloading them for each test job. This caused big costs in terms of cloud egress since the runners aren't hosted on the same cloud as the storage/artifacts instance. Instead we are going to be rebuilding gst-build for each test job from now, it also doesn't take more time than the network i/o would of downloading the artifacts, so the impact of rebuilding shouldn't be noticebly. We are also using pinned git refs the modules we rebuild from the manifest, so the binaries should be reproducible for the most part (minus things like .pyc files). Close #68 Part-of: --- .gitlab-ci.yml | 2 - gitlab/ci_template.yml | 137 +++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 82 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbd2171e98..bfad624a0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -173,8 +173,6 @@ build cerbero cross win64 local: image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest' extends: '.test' <<: *local_rules - dependencies: - - 'build fedora x86_64 local' check fedora local: extends: '.test fedora x86_64 local' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 910999ae80..58c5264d67 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -101,92 +101,75 @@ gst indent: exit 1 fi +.gst_build_template: &gst_build + - echo $MESON_ARGS + + - curl -L -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" + - python3 clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build + - cd gst-build/ + # only copy immediate directories. We don't want to modify .wrap files + - find /gst-build/subprojects/ -mindepth 1 -maxdepth 1 -type d -exec cp -r \{\} subprojects/ \; + - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" + - meson build/ $MESON_ARGS + - ninja -C build/ + - ccache --show-stats + +.gst_build_ccache_vars: + variables: + CCACHE_COMPILERCHECK: 'content' + CCACHE_COMPRESS: 'true' + CCACHE_BASEDIR: '/cache/gstreamer/gst-build' + CCACHE_DIR: '/cache/gstreamer/gst-build/ccache/' + # shared across everything really + CCACHE_MAXSIZE: '10G' + CARGO_HOME: '/cache/gstreamer/cargo' + +.simple_fedora_build: &simple_build >- + ${DEFAULT_MESON_ARGS} + -Dsharp=enabled + -Domx=enabled + -Dgst-omx:target=generic + -Ddoc=enabled + -Drs=disabled + ${MESON_BUILDTYPE_ARGS} + --werror + .build: stage: 'build' extends: - '.global_ci_policy' + - '.gst_build_ccache_vars' needs: - "manifest" # Taking into account the slowest shared runner + time needed to upload the binaries to artifacts # Also need to take into account I/O of pulling docker images and uploading artifacts timeout: '45min' variables: - CCACHE_COMPILERCHECK: "content" - CCACHE_COMPRESS: "true" - CCACHE_BASEDIR: "/cache/gstreamer/gst-build" - CCACHE_DIR: "/cache/gstreamer/gst-build/ccache/" - # shared across everything really - CCACHE_MAXSIZE: "10G" - CARGO_HOME: "/cache/gstreamer/cargo" - MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} --werror" rules: - if: '$CI_PROJECT_NAME != "cerbero"' - script: - - ccache --show-stats - - curl -L -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py - - chmod +x clone_manifest_ref.py - - ./clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build - - cd gst-build/ - # only copy immediate directories. We don't want to modify .wrap files - - find /gst-build/subprojects/ -mindepth 1 -maxdepth 1 -type d -exec cp -r \{\} subprojects/ \; - - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" - - meson build/ $MESON_ARGS - - ninja -C build/ - - ccache --show-stats + *gst_build after_script: - - cd gst-build/ - # Clean the artifacts packages to avoid copying "useless" build products. - - test -d build && find build -name '*.[ao]' -delete - # Clean the .git repos since we won't need them anymore - - rm -rf subprojects/*/.git/ - - rm -rf build/subprojects/*/.git/ - # Clean the big files that are not useful to us to reduce the artifact size - - rm -rf subprojects/FFmpeg/tests - - rm -rf subprojects/FFmpeg/doc - - rm -rf subprojects/openh264/res - - rm -rf subprojects/libxml2-2.9.7/result - - rm -rf subprojects/libxml2-2.9.7/doc - - rm -rf subprojects/libxml2-2.9.7/test - - rm -rf subprojects/libpsl/fuzz - - rm -rf subprojects/glib/gio/tests - - rm -rf subprojects/glib/glib/tests - - rm -rf subprojects/glib/gobject/tests - - rm -rf subprojects/glib/docs/tests - - rm -rf subprojects/glib/tests - - rm -rf subprojects/glib/po - - rm -rf subprojects/glib/docs - - # Clear gst-intergrations-testsuites report as it contains raw media - # files which is only needed for validate jobs. When needed we copy the - # cached repo in the docker image and fetch. - - rm -rf subprojects/gst-integration-testsuites + - mv gst-build/build/meson-logs/ meson-logs + # Cleanup everything else to reduce the size + # of the docker volume we leave behind + - rm -rf gst-build artifacts: expire_in: "7 days" when: "always" paths: - - 'gst-build/build/meson-logs/' - # Needed by the integration tests + - 'meson-logs/' - 'manifest.xml' .build fedora x86_64: extends: '.build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: *simple_build rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-omx|gst-plugins-rs)$/' -build fedora x86_64: - extends: '.build fedora x86_64' - artifacts: - expire_in: '5 days' - when: always - paths: - - "manifest.xml" - - "gst-build/" - build nodebug fedora x86_64: extends: '.build' image: $FEDORA_IMAGE @@ -221,7 +204,12 @@ build clang fedora x86_64: stage: 'test' extends: - '.global_ci_policy' + - '.gst_build_ccache_vars' + needs: + - "manifest" variables: + MESON_ARGS: *simple_build + # Disable colored output to avoid weird rendering issues GST_DEBUG_NO_COLOR: "true" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" @@ -231,7 +219,8 @@ build clang fedora x86_64: rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-examples|gst-docs)$/' script: - - cd gst-build/ + - *gst_build + - echo "-> Running ${TEST_SUITE}" - > ./gst-uninstalled.py @@ -252,6 +241,7 @@ build clang fedora x86_64: paths: - 'gst-build/build/meson-logs/' - 'validate-logs' + - 'manifest.xml' reports: junit: - "validate-logs/*.xml" @@ -260,8 +250,6 @@ build clang fedora x86_64: image: $FEDORA_IMAGE extends: '.test' tags: ['gstreamer'] - needs: - - 'build fedora x86_64' check fedora: extends: '.test fedora x86_64' @@ -278,18 +266,6 @@ integration testsuites fedora: TEST_SUITE: "validate ges" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-omx|gst-plugins-rs)$/' - before_script: - # Avoid cloning the repo since it contains media files, - # instead copy it from the cache and do a git fetch - # We remove this copy first since it had its mediafiles stripped - - cp -r /gst-build/subprojects/gst-integration-testsuites/ "${CI_PROJECT_DIR}/gst-build/subprojects/gst-integration-testsuites/" - - curl -L -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" - - >- - python3 clone_manifest_ref.py - --fetch - --manifest manifest.xml - --project gst-integration-testsuites - --destination gst-build/subprojects/gst-integration-testsuites gstreamer-full: extends: 'build static fedora x86_64' @@ -505,8 +481,6 @@ valgrind ges: - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && meson build $env:MESON_ARGS && ninja -C build" - # FIXME: extract builddir for tests - #after_script: # XXX: Re-enable when uploading stops timing out #artifacts: # expire_in: "7 days" @@ -869,15 +843,18 @@ documentation: image: $FEDORA_IMAGE extends: - '.global_ci_policy' + - '.gst_build_ccache_vars' rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-sharp|gst-integration-testsuites|gst-omx|gst-plugins-rs|gst-python)$/' needs: - - "build fedora x86_64" + - "manifest" stage: integrate - + variables: + MESON_ARGS: *simple_build script: - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git - - cd gst-build/ + - *gst_build + - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings - cd - - mv gst-build/build/subprojects/gst-docs/GStreamer-doc/html documentation/ @@ -944,7 +921,7 @@ cerbero cross-ios universal examples: - "cerbero cross-ios universal" build gst-omx zynq fedora x86_64: - extends: 'build fedora x86_64' + extends: '.build fedora x86_64' variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" rules: @@ -953,7 +930,7 @@ build gst-omx zynq fedora x86_64: - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2019.2 ${CI_PROJECT_DIR}/vcu-omx-il build gst-omx tizonia fedora x86_64: - extends: 'build fedora x86_64' + extends: '.build fedora x86_64' variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}:${CI_PROJECT_DIR}/tizonia-install/lib64/pkgconfig/