rafaelcaricio/gstreamer-rs
1
0
Fork 0
forked from mirrors/gstreamer-rs
Code Pull requests Activity
gstreamer-rs/.gitlab-ci.yml
Jordan Petridis e87de384e1 ci: avoid always running build/test jobs 
Introduce a trigger job with rules that will allow it to always
run when the MR is assigned to Marge and the final tests will be
run before being merge.

This means that Post-merge pipelines and MR pipelines that are
not assigned to Marge, will require a manual run of the trigger
job before the build/test will execute.

This should help easy the load on the gitlab runners.

Related to gstreamer/gstreamer-rs#417

Close gstreamer/gstreamer-rs#401

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-rs/-/merge_requests/1155>
2022-11-23 18:44:12 +02:00

727 lines
21 KiB
YAML
Raw Blame History

# We use https://gitlab.freedesktop.org/freedesktop/ci-templates
# to build the images used by the ci.
#
# Here is how to properly update those images:
# - new Rust stable version: update GST_RS_IMG_TAG and update Rust version
# - add dependencies: update FDO_DISTRIBUTION_PACKAGES and update GST_RS_IMG_TAG
# - update GStreamer version: update the tag in ci/install-gst.sh and update GST_RS_IMG_TAG
#
# GST_RS_IMG_TAG is defined in ci/images_template.yml and should be updated
# either by:
# - setting it to the current date and the version suffix to 0
# - incrementing the version suffix
#
# After each update commit your changes and push to your personal repo.
# After review and ci approval merge the branch as usual.
#
# Updating the nightly image should be done by simply running a scheduled ci
# pipeline on the upstream repo with the $UPDATE_NIGHTLY variable defined.
.templates_sha: &templates_sha 567700e483aabed992d0a4fea84994a0472deff6
include:
- project: 'freedesktop/ci-templates'
ref: *templates_sha
file: '/templates/debian.yml'
- local: "ci/images_template.yml"
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_MERGE_REQUEST_IID
# don't create a pipeline if its a commit pipeline, on a branch and that branch has
# open merge requests (bc we will get a MR build instead)
- if: $CI_OPEN_MERGE_REQUESTS
when: never
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
default:
interruptible: true
variables:
FDO_UPSTREAM_REPO: gstreamer/gstreamer-rs
# DIY CI-templates like setup for windows
WINDOWS_RUST_MINIMUM_IMAGE: "$CI_REGISTRY_IMAGE/windows:$GST_RS_IMG_TAG-main-$GST_RS_MSRV"
WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE: "$CI_REGISTRY/$FDO_UPSTREAM_REPO/windows:$GST_RS_IMG_TAG-main-$GST_RS_MSRV"
WINDOWS_RUST_STABLE_IMAGE: "$CI_REGISTRY_IMAGE/windows:$GST_RS_IMG_TAG-main-$GST_RS_STABLE"
WINDOWS_RUST_STABLE_UPSTREAM_IMAGE: "$CI_REGISTRY/$FDO_UPSTREAM_REPO/windows:$GST_RS_IMG_TAG-main-$GST_RS_STABLE"
RUST_DOCS_FLAGS: "--extern-html-root-url=muldiv=https://docs.rs/muldiv/1.0.0/muldiv/ -Z unstable-options"
NAMESPACE: gstreamer
# format is <branch>=<name>
# the name is used in the URL
# latest release must be at the top
# (only relevant on main branch)
RELEASES:
0.19=0.19
0.18=0.18
stages:
- "trigger"
- "container-base"
- "container-final"
- "lint"
- "test"
- "extras"
- "deploy"
# This is an empty job that is used to trigger the pipeline.
trigger:
image: alpine:latest
stage: 'trigger'
variables:
GIT_STRATEGY: none
script:
- echo "Trigger job done, now running the pipeline."
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
# If the MR is assigned to the Merge bot, trigger the pipeline automatically
- if: '$CI_MERGE_REQUEST_ASSIGNEES == "gstreamer-merge-bot"'
# Require explicit action to trigger tests post merge
- if: '$CI_PROJECT_NAMESPACE == "gstreamer" && $CI_COMMIT_BRANCH == "main"'
when: 'manual'
# When the assignee isn't the merge bot, require an explicit action to trigger the pipeline
# to avoid wasting CI resources
- if: '$CI_MERGE_REQUEST_ASSIGNEES != "gstreamer-merge-bot"'
when: 'manual'
allow_failure: false
.debian:11:
needs: []
variables:
FDO_DISTRIBUTION_VERSION: 'bullseye-slim'
before_script:
- source ./ci/env.sh
- mkdir .cargo && echo -e "[net]\ngit-fetch-with-cli = true" > .cargo/config
# If cargo exists assume we probably will want to update
# the lockfile
- |
if command -v cargo; then
cargo generate-lockfile --color=always
cargo update --color=always
fi
.debian:11-base:
extends: .debian:11
variables:
FDO_DISTRIBUTION_TAG: 'base-$GST_RS_IMG_TAG'
.debian:11-stable:
extends: .debian:11
variables:
RUST_IMAGE_FULL: "1"
FDO_DISTRIBUTION_TAG: '$GST_RS_STABLE-$GST_RS_IMG_TAG'
FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh $GST_RS_STABLE $RUST_IMAGE_FULL'
.debian:11-msrv:
extends: .debian:11
variables:
FDO_DISTRIBUTION_TAG: '$GST_RS_MSRV-$GST_RS_IMG_TAG'
FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh $GST_RS_MSRV $RUST_IMAGE_FULL'
.debian:11-nightly:
extends: .debian:11
variables:
FDO_DISTRIBUTION_TAG: 'nightly-$GST_RS_IMG_TAG'
FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh nightly $RUST_IMAGE_FULL'
.build-base-image:
extends:
- .fdo.container-build@debian
stage: container-base
variables:
FDO_DISTRIBUTION_PACKAGES: "build-essential curl python3-setuptools liborc-0.4-dev libglib2.0-dev libxml2-dev libgtk-3-dev libegl1-mesa libgles2-mesa libgl1-mesa-dri libgl1-mesa-glx libwayland-egl1-mesa xz-utils libssl-dev git wget ca-certificates ninja-build python3-pip flex bison libglib2.0-dev libx11-dev libx11-xcb-dev libsoup2.4-dev libvorbis-dev libogg-dev libtheora-dev libmatroska-dev libvpx-dev libopus-dev libgraphene-1.0-dev libjpeg-dev libwayland-dev python3-gi"
FDO_DISTRIBUTION_EXEC: >-
bash ci/install-gst.sh &&
bash ci/install-gtk4.sh &&
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates
.build-final-image:
extends:
- .fdo.container-build@debian
stage: container-final
variables:
FDO_BASE_IMAGE: '$CI_REGISTRY_IMAGE/debian/bullseye-slim:base-$GST_RS_IMG_TAG'
build-base:
extends:
- .build-base-image
- .debian:11-base
build-stable:
extends:
- .build-final-image
- .debian:11-stable
build-msrv:
extends:
- .build-final-image
- .debian:11-msrv
build-nightly:
extends:
- .build-final-image
- .debian:11-nightly
update-nightly:
extends: build-nightly
rules:
- if: $UPDATE_NIGHTLY == "1"
variables:
FDO_FORCE_REBUILD: 1
.dist-debian-container:
extends:
- .fdo.distribution-image@debian
after_script:
- rm -rf target
.img-stable:
extends:
- .debian:11-stable
- .dist-debian-container
.img-msrv:
extends:
- .debian:11-msrv
- .dist-debian-container
.img-nightly:
extends:
- .debian:11-nightly
- .dist-debian-container
# GST_PLUGINS_RS_TOKEN is a variable of type 'Var' defined in gstreamer-rs CI
# settings and containing a gst-plugins-rs pipeline trigger token
.plugins-update:
stage: deploy
script:
- |
# FDO_DISTRIBUTION_IMAGE still has indirections
- echo $FDO_DISTRIBUTION_IMAGE
- DISTRO_IMAGE=$(eval echo ${FDO_DISTRIBUTION_IMAGE})
- echo $DISTRO_IMAGE
# retrieve the infos from the registry
- JSON_IMAGE=$(skopeo inspect docker://$DISTRO_IMAGE)
- IMAGE_PIPELINE_ID=$(echo $JSON_IMAGE | jq -r '.Labels["fdo.pipeline_id"]')
- echo $IMAGE_PIPELINE_ID
- echo $CI_PIPELINE_ID
- |
if [[ x"$IMAGE_PIPELINE_ID" == x"$CI_PIPELINE_ID" ]]; then
echo "Image has been updated, notify gst-plugins-rs"
curl -X POST -F "token=$GST_PLUGINS_RS_TOKEN" -F "ref=main" -F "variables[UPDATE_IMG]=$UPDATE_IMG" https://gitlab.freedesktop.org/api/v4/projects/1400/trigger/pipeline
else
echo "Image has not been updated, ignore"
fi
rules:
- if: '$CI_COMMIT_REF_NAME == "main" && $CI_PROJECT_PATH == "gstreamer/gstreamer-rs"'
# Those jobs need to use another image as ours doesn't have 'skopeo'
# and it's not easily installable in Debian stable for now.
plugins-update-stable:
extends:
- .plugins-update
- .img-stable
image: quay.io/freedesktop.org/ci-templates:container-build-base-2021-07-29.0
variables:
UPDATE_IMG: "stable"
plugins-update-msrv:
extends:
- .plugins-update
- .img-msrv
image: quay.io/freedesktop.org/ci-templates:container-build-base-2021-07-29.0
variables:
UPDATE_IMG: "msrv"
plugins-update-nightly:
extends:
- .plugins-update
- .img-nightly
image: quay.io/freedesktop.org/ci-templates:container-build-base-2021-07-29.0
variables:
UPDATE_IMG: "nightly"
.cargo_test_var: &cargo_test
- rustc --version
# First build and test all the crates with their relevant features
# Keep features in sync with below
- |
get_features() {
crate=$1
case "$crate" in
gstreamer-audio|gstreamer-editing-services|gstreamer-gl|gstreamer-pbutils|gstreamer-rtp|gstreamer-rtsp|gstreamer-video|gstreamer)
echo "--features=serde,v1_22"
;;
gstreamer-validate)
echo ""
;;
*)
echo "--features=v1_22"
;;
esac
}
for crate in gstreamer* gstreamer-gl/{egl,wayland,x11}; do
if [ -e $crate/Cargo.toml ]; then
if [ -n "$ALL_FEATURES" ]; then
FEATURES="$(get_features $crate)"
else
FEATURES=""
fi
echo "Building and testing $crate with $FEATURES"
cargo build --locked --color=always --manifest-path $crate/Cargo.toml $FEATURES
G_DEBUG=fatal_warnings cargo test --color=always --manifest-path $crate/Cargo.toml $FEATURES
fi
done
- |
if [ -n "$EXAMPLES_TUTORIALS" ]; then
cargo build --locked --color=always --manifest-path examples/Cargo.toml --bins --examples --all-features
cargo build --locked --color=always --manifest-path tutorials/Cargo.toml --bins --examples --all-features
fi
.cargo test:
stage: "test"
script:
- *cargo_test
test msrv:
extends:
- '.cargo test'
- .img-msrv
needs:
- job: 'trigger'
artifacts: false
- job: 'build-msrv'
artifacts: false
test stable:
extends:
- '.cargo test'
- .img-stable
needs:
- job: 'trigger'
artifacts: false
- job: 'build-stable'
artifacts: false
test stable all-features:
variables:
ALL_FEATURES: 'yes'
EXAMPLES_TUTORIALS: 'yes'
extends:
- '.cargo test'
- .img-stable
needs:
- job: 'trigger'
artifacts: false
- job: 'build-stable'
artifacts: false
test nightly:
allow_failure: true
extends:
- '.cargo test'
- .img-nightly
needs:
- job: 'trigger'
artifacts: false
- job: 'build-nightly'
artifacts: false
test nightly all-features:
allow_failure: true
variables:
ALL_FEATURES: 'yes'
EXAMPLES_TUTORIALS: 'yes'
extends:
- '.cargo test'
- .img-nightly
needs:
- job: 'trigger'
artifacts: false
- job: 'build-nightly'
artifacts: false
.cargo test sys:
stage: "test"
script:
- rustc --version
- |
get_features() {
module=${1%%/sys}
case "$module" in
gstreamer-validate)
echo ""
;;
*)
echo "--features=v1_22"
;;
esac
}
# First build and test all the crates with their relevant features
# Keep features in sync with below
for crate in gstreamer*/sys gstreamer-gl/*/sys; do
if [ -e $crate/Cargo.toml ]; then
echo "Building $crate with $(get_features $crate)"
cargo build --locked --color=always --manifest-path $crate/Cargo.toml $(get_features $crate)
fi
done
# Run tests for crates we can currently run.
# Other tests are broken currently.
for crate in gstreamer/sys \
gstreamer-app/sys \
gstreamer-audio/sys \
gstreamer-base/sys \
gstreamer-check/sys \
gstreamer-controller/sys \
gstreamer-gl/sys \
gstreamer-gl/egl/sys \
gstreamer-gl/wayland/sys \
gstreamer-gl/x11/sys \
gstreamer-mpegts/sys \
gstreamer-net/sys \
gstreamer-pbutils/sys \
gstreamer-player/sys \
gstreamer-rtsp-server/sys \
gstreamer-rtsp/sys \
gstreamer-sdp/sys \
gstreamer-tag/sys \
gstreamer-video/sys \
gstreamer-webrtc/sys; do
echo "Testing $crate with $(get_features $crate)"
cargo test --locked --color=always --manifest-path $crate/Cargo.toml $(get_features $crate)
done
test stable sys:
extends:
- '.cargo test sys'
- .img-stable
needs:
- job: 'trigger'
artifacts: false
- job: 'build-stable'
artifacts: false
test msrv sys:
extends:
- '.cargo test sys'
- .img-msrv
needs:
- job: 'trigger'
artifacts: false
- job: 'build-msrv'
artifacts: false
test nightly sys:
extends:
- '.cargo test sys'
- .img-nightly
needs:
- job: 'trigger'
artifacts: false
- job: 'build-nightly'
artifacts: false
rustfmt:
extends: .img-stable
stage: "lint"
script:
- cargo fmt --version
- cargo fmt -- --color=always --check
needs:
- job: 'build-stable'
artifacts: false
check commits:
extends: .img-stable
stage: "lint"
script:
- ci-fairy check-commits --textwidth 0 --no-signed-off-by
needs:
- job: 'build-stable'
artifacts: false
clippy:
extends: .img-stable
stage: 'extras'
variables:
CLIPPY_LINTS: -D warnings -W unknown-lints
needs:
- job: 'trigger'
artifacts: false
- job: 'build-stable'
artifacts: false
script:
- cargo clippy --version
# Keep features in sync with above
- |
get_features() {
crate=$1
case "$crate" in
gstreamer-audio|gstreamer-editing-services|gstreamer-gl|gstreamer-pbutils|gstreamer-rtp|gstreamer-rtsp|gstreamer-video|gstreamer)
echo "--features=serde,v1_22"
;;
gstreamer-validate)
echo ""
;;
*)
echo "--features=v1_22"
;;
esac
}
for crate in gstreamer* gstreamer-gl/{egl,wayland,x11}; do
if [ -e $crate/Cargo.toml ]; then
FEATURES=$(get_features $crate)
echo "Running clippy on $crate with $FEATURES"
cargo clippy --locked --color=always --manifest-path $crate/Cargo.toml $FEATURES --all-targets -- $CLIPPY_LINTS
fi
done
# And also run over all the examples/tutorials
- |
cargo clippy --locked --color=always --manifest-path examples/Cargo.toml --all-targets --all-features -- $CLIPPY_LINTS
cargo clippy --locked --color=always --manifest-path tutorials/Cargo.toml --all-targets --all-features -- $CLIPPY_LINTS
deny:
extends: .img-stable
stage: 'extras'
needs:
- job: 'build-stable'
artifacts: false
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
script:
- cargo deny --color=always check
gir-checks:
variables:
GIT_SUBMODULE_STRATEGY: recursive
extends: .img-stable
stage: 'extras'
needs:
- job: 'build-stable'
artifacts: false
script:
- git submodule update --checkout
- python3 ci/gir-checks.py
outdated:
extends: .img-stable
stage: 'extras'
needs:
- job: 'build-stable'
artifacts: false
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
script:
- cargo outdated --color=always --root-deps-only --exit-code 1 -v
coverage:
allow_failure: true
extends:
- '.cargo test'
- .img-stable
stage: 'extras'
needs:
- job: 'trigger'
artifacts: false
- job: 'build-stable'
artifacts: false
variables:
ALL_FEATURES: 'yes'
RUSTFLAGS: "-Cinstrument-coverage"
LLVM_PROFILE_FILE: "gstreamer-rs-%p-%m.profraw"
script:
- *cargo_test
# generate html report
- grcov . --binary-path ./target/debug/ -s . -t html --branch --ignore-not-existing --ignore "*target*" --ignore "*/sys/*" --ignore "examples/*" --ignore "tutorials/*" --ignore "*/build.rs" -o ./coverage/
# generate cobertura report for gitlab integration
- grcov . --binary-path ./target/debug/ -s . -t cobertura --branch --ignore-not-existing --ignore "*target*" --ignore "*/sys/*" --ignore "examples/*" --ignore "tutorials/*" --ignore "*/build.rs" -o coverage.xml
# output coverage summary for gitlab parsing.
# TODO: use grcov once https://github.com/mozilla/grcov/issues/556 is fixed
- grep % coverage/index.html | head -1 ; true
artifacts:
paths:
- 'coverage'
reports:
coverage_report:
coverage_format: cobertura
path: coverage.xml
doc-stripping:
variables:
GIT_SUBMODULE_STRATEGY: recursive
extends: .img-nightly
stage: 'extras'
needs:
- job: 'build-nightly'
artifacts: false
script:
- git submodule update --checkout
- PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --embed-docs
- PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --strip-docs
- git diff --quiet || (echo 'Files changed after running `rustdoc-stripper -s`, make sure all documentation is protected with `// rustdoc-stripper-ignore-next`!'; git diff; false)
regen-check:
variables:
GIT_SUBMODULE_STRATEGY: recursive
extends: .img-nightly
stage: 'extras'
needs:
- job: 'build-nightly'
artifacts: false
script:
- git submodule update --checkout
- PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --yes
- git diff --quiet || (echo 'Files changed after running `generator.py`, make sure all submodules and generated files are in the correct version!'; git diff; false)
docs:
variables:
GIT_SUBMODULE_STRATEGY: recursive
extends: .img-nightly
stage: 'extras'
needs:
- job: 'build-nightly'
artifacts: false
script:
- git submodule update --checkout
- curl --proto '=https' --tlsv1.2 -sSf -o gir-rustdoc.py
https://gitlab.gnome.org/World/Rust/gir-rustdoc/-/raw/main/gir-rustdoc.py
- chmod +x gir-rustdoc.py
- PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --embed-docs --no-fmt
- |
RUSTDOCFLAGS="$RUST_DOCS_FLAGS"
eval $(./gir-rustdoc.py pre-docs)
cargo +nightly doc --workspace --exclude examples --exclude tutorials --color=always --features=dox --no-deps
- mv target/doc docs
artifacts:
paths:
- 'docs'
# https://docs.gitlab.com/ee/user/project/pages/#how-it-works
# GitLab automatically deploys the `public/` folder from an
# artifact generated by the job named `pages`. This step
# re-uses the docs from the build-test `docs` step above.
pages:
extends: .img-nightly
stage: 'deploy'
needs: [ 'docs' ]
interruptible: false
script:
- curl --proto '=https' --tlsv1.2 -sSf -o gir-rustdoc.py
https://gitlab.gnome.org/World/Rust/gir-rustdoc/-/raw/main/gir-rustdoc.py
- chmod +x gir-rustdoc.py
- ./gir-rustdoc.py html-index
# development docs
- mkdir public/git
- mv docs public/git/docs
# stable docs
- ./gir-rustdoc.py docs-from-artifacts
- ls public/
artifacts:
paths:
- 'public'
rules:
- if: ($CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH) && ($CI_PROJECT_NAMESPACE == $NAMESPACE)
when: 'manual'
.windows rust docker build:
stage: 'container-final'
needs: []
variables:
# Unlike the buildah/linux jobs, this file
# needs to be relative to windows-docker/ subdir
# as it makes life easier in the powershell script
#
# We also don't need a CONTEXT_DIR var as its also
# hardcoded to be windows-docker/
DOCKERFILE: 'ci/windows-docker/Dockerfile'
GST_UPSTREAM_BRANCH: 'main'
tags:
- 'windows'
- 'shell'
- '2022'
script:
# We need to pass an array and to resolve the env vars, so we can't use a variable:
- $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "RUST_VERSION=$RUST_VERSION")
- "& ci/windows-docker/container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $RUST_IMAGE $RUST_UPSTREAM_IMAGE $DOCKERFILE"
- |
if (!($?)) {
echo "Failed to build the image"
Exit 1
}
windows rust docker stable:
extends: '.windows rust docker build'
variables:
RUST_IMAGE: !reference [variables, "WINDOWS_RUST_STABLE_IMAGE"]
RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_STABLE_UPSTREAM_IMAGE"]
RUST_VERSION: !reference [variables, "GST_RS_STABLE"]
windows rust docker msrv:
extends: '.windows rust docker build'
when: 'manual'
variables:
RUST_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_IMAGE"]
RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE"]
RUST_VERSION: !reference [variables, "GST_RS_MSRV"]
.msvc2019 build:
stage: 'test'
tags:
- 'docker'
- 'windows'
- '2022'
parallel:
matrix:
- FEATURES:
- "--features=v1_18,"
- "--features=v1_20,"
- "--features=v1_22,"
- "--no-default-features"
- ""
script:
- echo $env:FEATURES
# Skip -sys tests as they don't work
# https://github.com/gtk-rs/gtk3-rs/issues/54
#
# We need to build each crate separately to avoid crates like -egl,-wayland etc on windows
- cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 &&
powershell ./ci/run_windows_tests.ps1"
- |
if (!$?) {
Write-Host "Tests Failed!"
Exit 1
}
test windows msrv:
image: $WINDOWS_RUST_MINIMUM_IMAGE
needs:
- job: 'trigger'
artifacts: false
- job: 'windows rust docker msrv'
artifacts: false
extends: '.msvc2019 build'
test windows stable:
needs:
- job: 'trigger'
artifacts: false
- job: 'windows rust docker stable'
artifacts: false
image: "$WINDOWS_RUST_STABLE_IMAGE"
extends: '.msvc2019 build'
View git blame Copy permalink