gstreamer/.gitlab-ci.yml
Jordan Petridis b36d813f5b ci_template: don't use prebuilt binaries for test jobs
Previously we were optimizing for cpu time, so we where building
gst-build once and then exporting that to be used by the test jobs.
However this meant that we where uploading 200mb (previously 600mb)
zipped of artifacts and then re-downloading them for each test job.
This caused big costs in terms of cloud egress since the runners
aren't hosted on the same cloud as the storage/artifacts instance.

Instead we are going to be rebuilding gst-build for each test
job from now, it also doesn't take more time than the network
i/o would of downloading the artifacts, so the impact of rebuilding
shouldn't be noticebly.

We are also using pinned git refs the modules we rebuild from
the manifest, so the binaries should be reproducible for the most
part (minus things like .pyc files).

Close #68

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-ci/-/merge_requests/280>
2020-04-22 09:23:18 +03:00

196 lines
5.2 KiB
YAML

include: "gitlab/ci_template.yml"
stages:
- "build docker"
- "preparation"
- "pre-build"
- "build"
- "test"
# use the binaries in some way
- "integrate"
test manifest:
variables:
GIT_STRATEGY: fetch
rules:
- when: 'always'
image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:2019-10-23-793476'
stage: "preparation"
script:
- pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py
coverage: '/TOTAL.*\s+(\d+%)$/'
artifacts:
reports:
junit:
- "junit.xml"
.base:
image: 'registry.fedoraproject.org/fedora:31'
extends:
- '.global_ci_policy'
tags: ['gstreamer']
rules:
- when: 'manual'
allow_failure: true
variables:
STORAGE_DRIVER: 'vfs'
BUILDAH_FORMAT: 'docker'
GIT_STRATEGY: fetch
script:
- export DATE=$(date +"%Y-%m-%d")
- export IMAGE="${CI_REGISTRY_IMAGE}/${ARCH}/${TAG}"
- export _UID="${IMAGE}:${DATE}-${CI_JOB_ID}"
- export LATEST="${IMAGE}:latest"
- dnf install -y buildah runc
# Newer versions of podman/buildah try to set overlayfs mount options when
# using the vfs driver, and this causes errors.
- sed -i '/^mountopt =.*/d' /etc/containers/storage.conf
- buildah bud --build-arg DEFAULT_BRANCH=${GST_UPSTREAM_BRANCH} -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR}
# If we are in a fork, push the image to the reigstry regardless the branch
- |
if [ "$CI_PROJECT_NAMESPACE" != "gstreamer" ]; then
buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
buildah push ${LATEST}
fi
# Push the images to the upstream registry
- |
if [ "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then
# Tag the image with the git ref
buildah image tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME}
# Push the tags
buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
buildah push ${_UID}-${CI_COMMIT_REF_NAME}
buildah push ${LATEST}
fi
alpine amd64 manifest builder docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "build-manifest"
CONTEXT_DIR: "docker/build_manifest/"
DOCKERFILE: "docker/build_manifest/Dockerfile"
extends: .base
test manifest amd64 docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "test-manifest"
CONTEXT_DIR: "docker/test_manifest/"
DOCKERFILE: "docker/test_manifest/Dockerfile"
extends: .base
gst-indent amd64 docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "gst-indent"
CONTEXT_DIR: "docker/indent/"
DOCKERFILE: "docker/indent/Dockerfile"
extends: .base
fedora amd64 docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "fedora"
CONTEXT_DIR: "docker/fedora/"
DOCKERFILE: "docker/fedora/Dockerfile"
extends: .base
cerbero fedora amd64 docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "cerbero-fedora"
CONTEXT_DIR: "docker/cerbero/"
DOCKERFILE: "docker/cerbero/Dockerfile-fedora"
extends: .base
android docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "android"
CONTEXT_DIR: "docker/android/"
DOCKERFILE: "docker/android/Dockerfile"
extends: .base
.local_rules: &local_rules
rules:
- if: '$CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_PATH == "gstreamer/gst-ci"'
when: 'never'
- when: 'manual'
allow_failure: true
.local_template: &local_template
<<: *local_rules
needs:
- 'manifest'
# Test the build job against the latest build image tag and the local manifest
build fedora x86_64 local:
extends: '.build'
stage: 'build'
image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest"
artifacts:
expire_in: '5 days'
when: always
paths:
- "manifest.xml"
- "gst-build/"
<<: *local_template
build cerbero fedora x86_64 local:
extends: '.cerbero fedora x86_64'
image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest"
<<: *local_template
build cerbero cross-android universal local:
extends: '.cerbero cross-android universal'
image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest"
<<: *local_template
build cerbero cross win32 local:
extends: '.cerbero cross win32'
image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest"
<<: *local_template
build cerbero cross win64 local:
extends: '.cerbero cross win64'
image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest"
<<: *local_template
# Note: dependencies: will be deprecated in the future, but current manual
# jobs with needs: (even if they are allowed to fail) will leave the pipeline
# pending.
.test fedora x86_64 local:
image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest'
extends: '.test'
<<: *local_rules
check fedora local:
extends: '.test fedora x86_64 local'
variables:
TEST_SUITE: 'check.gst*'
cross-android universal examples local:
extends: '.cross-android universal examples'
image: "${CI_REGISTRY_IMAGE}/amd64/android:latest"
<<: *local_rules
dependencies:
- 'build cerbero cross-android universal local'
integration testsuites fedora local:
extends: '.test fedora x86_64 local'
before_script:
- rm -f gst-build/build/subprojects/gstreamer-vaapi/gst/vaapi/libgstvaapi.so
variables:
EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures"
TEST_SUITE: "validate ges"