.gitlab-ci.yml: adapt the docker build jobs to be more like citemplates

This is the inital step towards migrating our docker images setup
to something closer and eventually freedesktop/citemplates [1]

The idea is that jobs always run from the registry in your fork. If the
image sha/id matches the one from the upstream registry, its copied
over else a new one is build, pushed and tested.

Only change the fedora job for now while testing.

[1]: https://gitlab.freedesktop.org/freedesktop/ci-templates

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-ci/-/merge_requests/308>
This commit is contained in:
Jordan Petridis 2020-06-18 19:18:27 +03:00 committed by Jordan Petridis
parent c731fab0ca
commit fe959679b8
2 changed files with 89 additions and 22 deletions

View file

@ -24,6 +24,59 @@ test manifest:
junit: junit:
- "junit.xml" - "junit.xml"
# have a special definition for the test if the image already exists
.check_image: &check_image_exists |
# if-not-exists steps
set -x
if [[ -z "$GST_FORCE_REBUILD" ]]
then
# disable exit on failure
set +e
# check if our image is already in the current registry
# we store the sha of the digest and the layers
skopeo inspect docker://$LOCAL_IMAGE | jq '[.Digest, .Layers]' > local_sha
# check if our image is already in the upstream registry
if [[ -z "$GST_UPSTREAM_REPO" ]]
then
echo "WARNING! Variable \$GST_UPSTREAM_REPO is undefined, cannot check for images"
else
skopeo inspect docker://$GST_UPSTREAM_IMAGE | jq '[.Digest, .Layers]' > upstream_sha
fi
# reenable exit on failure
set -e
# if the upstream repo has an image, ensure we use the same
if [ -s upstream_sha ]
then
# ensure we use the same image from upstream
diff upstream_sha local_sha && exit 0 || true
# copy the original image into the current project registry namespace
# we do 2 attempts with skopeo copy at most
skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \
docker://$GST_UPSTREAM_IMAGE \
docker://$LOCAL_IMAGE || \
skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \
docker://$GST_UPSTREAM_IMAGE \
docker://$LOCAL_IMAGE
exit 0
fi
# if we have a local image but none in the upstream repo, use our
if [ -s local_sha ]
then
exit 0
fi
fi
set +x
.base: .base:
image: 'registry.fedoraproject.org/fedora:32' image: 'registry.fedoraproject.org/fedora:32'
extends: extends:
@ -36,19 +89,32 @@ test manifest:
STORAGE_DRIVER: 'vfs' STORAGE_DRIVER: 'vfs'
BUILDAH_FORMAT: 'docker' BUILDAH_FORMAT: 'docker'
BUILDAH_ISOLATION: 'chroot' BUILDAH_ISOLATION: 'chroot'
# FIXME: Probably needs to be changed to none and clone gst-ci repo each time, sigh
# as these jobs *could* run potentially in a fork of smth like -bad
# though given our current setup, its only possible to tweak the CI from a gst-ci
# fork so it might be fine
GIT_STRATEGY: fetch GIT_STRATEGY: fetch
script: script:
- export DATE=$(date +"%Y-%m-%d") - echo $TAG
- export IMAGE="$CI_REGISTRY_IMAGE/${ARCH:=amd64}/$TAG" - echo $REPO_SUFFIX
- export _UID="$IMAGE:$DATE-$GST_UPSTREAM_BRANCH"
- export LATEST="$IMAGE:latest" - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$REPO_SUFFIX:$TAG"
- export GST_UPSTREAM_IMAGE="$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG"
- echo $LOCAL_IMAGE
- echo $GST_UPSTREAM_IMAGE
# FIXME: make an image since there will be multiple jobs running every pipeline
- dnf install -y buildah skopeo jq runc
- dnf install -y buildah runc
# Newer versions of podman/buildah try to set overlayfs mount options when # Newer versions of podman/buildah try to set overlayfs mount options when
# using the vfs driver, and this causes errors. # using the vfs driver, and this causes errors.
- sed -i '/^mountopt =.*/d' /etc/containers/storage.conf - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf
- echo "Building image $_UID" - *check_image_exists
- echo "Building image $LOCAL_IMAGE"
- > - >
buildah bud buildah bud
@ -60,29 +126,27 @@ test manifest:
--label git.sha=$CI_COMMIT_SHA --label git.sha=$CI_COMMIT_SHA
--label gitlab.project_path=$CI_PROJECT_PATH --label gitlab.project_path=$CI_PROJECT_PATH
--label gitlab.project_url=$CI_PROJECT_URL --label gitlab.project_url=$CI_PROJECT_URL
--label fdo.upstream-repo=$GST_UPSTREAM_REPO
--label fdo.expires-after="3w"
-f $DOCKERFILE -f $DOCKERFILE
-t $IMAGE -t $LOCAL_IMAGE
$CONTEXT_DIR $CONTEXT_DIR
- buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
# If we are in a fork, push the image to the reigstry regardless the branch
# Sanity check
- | - |
if [ "$CI_PROJECT_NAMESPACE" != "gstreamer" ]; then if [ "$CI_PROJECT_PATH" = "gstreamer/gst-ci" ]; then
# Tag the image with the git ref if [ "$LOCAL_IMAGE" != "$GST_UPSTREAM_IMAGE" ]; then
buildah tag $_UID $LATEST echo "ERROR!!! AAAAA! THE IMAGE URI DOES NOT MATCH THE EXPECTED UPSTREAM ONE"
# Only push the :latest so with each push the previous echo $LOCAL_IMAGE
# image in the forked registry gets overrwritten echo $GST_UPSTRAM_IMAGE
# to avoid bloating the registry and making exit 1
# pruning easier for the gitlab admins fi
buildah push $LATEST
fi fi
# Push the images to the upstream registry - buildah push $LOCAL_IMAGE
- |
if [ "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then
buildah push $_UID
fi
alpine amd64 manifest builder docker: alpine amd64 manifest builder docker:
stage: "build docker" stage: "build docker"
@ -111,7 +175,9 @@ gst-indent amd64 docker:
fedora amd64 docker: fedora amd64 docker:
stage: "build docker" stage: "build docker"
variables: variables:
TAG: "fedora" REPO_SUFFIX: "amd64/fedora"
TAG: "2020-06-18.0-$GST_UPSTREAM_BRANCH"
CONTEXT_DIR: "docker/fedora/" CONTEXT_DIR: "docker/fedora/"
DOCKERFILE: "docker/fedora/Dockerfile" DOCKERFILE: "docker/fedora/Dockerfile"
extends: .base extends: .base

View file

@ -17,6 +17,7 @@ variables:
# Branch to track for modules that have no ref specified in the manifest # Branch to track for modules that have no ref specified in the manifest
GST_UPSTREAM_BRANCH: 'master' GST_UPSTREAM_BRANCH: 'master'
ORC_UPSTREAM_BRANCH: 'master' ORC_UPSTREAM_BRANCH: 'master'
GST_UPSTRAM_REPO: 'gstreamer/gst-ci'
GIT_STRATEGY: none GIT_STRATEGY: none
MESON_BUILDTYPE_ARGS: --default-library=both MESON_BUILDTYPE_ARGS: --default-library=both