gstreamer-rs/.gitlab-ci.yml

# We use https://gitlab.freedesktop.org/freedesktop/ci-templates
# to build the images used by the ci.
#
# Here is how to properly update those images:
# - new Rust stable version: update GST_RS_IMG_TAG and update Rust version
# - add dependencies: update FDO_DISTRIBUTION_PACKAGES and update GST_RS_IMG_TAG
# - update GStreamer version: update the tag in ci/install-gst.sh and update GST_RS_IMG_TAG
#
# GST_RS_IMG_TAG is defined in ci/images_template.yml and should be updated
# either by:
# - setting it to the current date and the version suffix to 0
# - incrementing the version suffix
#
# Same for GST_RS_IMG_WINDOWS_TAG. There's a separate tag for it to cater for
# image-only updates that only affect Windows or only Linux.
#
# After each update commit your changes and push to your personal repo.
# After review and ci approval merge the branch as usual.
#
# Updating the nightly image should be done by simply running a scheduled ci
# pipeline on the upstream repo with the $UPDATE_NIGHTLY variable defined.

.templates_sha: &templates_sha 6a40df92957c8ce9ee741aaccc5daaaf70545b1e

include:
  - project: 'freedesktop/ci-templates'
    ref: *templates_sha
    file: '/templates/debian.yml'

  - local: "ci/images_template.yml"

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_MERGE_REQUEST_IID
    # don't create a pipeline if its a commit pipeline, on a branch and that branch has
    # open merge requests (bc we will get a MR build instead)
    - if: $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH

default:
  interruptible: true
  # Auto-retry jobs in case of infra failures
  retry:
    max: 1
    when:
      - 'runner_system_failure'
      - 'stuck_or_timeout_failure'
      - 'scheduler_failure'
      - 'api_failure'

variables:
  FDO_UPSTREAM_REPO: gstreamer/gstreamer-rs

  # DIY CI-templates like setup for windows
  WINDOWS_RUST_MINIMUM_IMAGE: "$CI_REGISTRY_IMAGE/windows:$GST_RS_IMG_WINDOWS_TAG-main-$GST_RS_MSRV"
  WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE: "$CI_REGISTRY/$FDO_UPSTREAM_REPO/windows:$GST_RS_IMG_WINDOWS_TAG-main-$GST_RS_MSRV"
  WINDOWS_RUST_STABLE_IMAGE: "$CI_REGISTRY_IMAGE/windows:$GST_RS_IMG_WINDOWS_TAG-main-$GST_RS_STABLE"
  WINDOWS_RUST_STABLE_UPSTREAM_IMAGE: "$CI_REGISTRY/$FDO_UPSTREAM_REPO/windows:$GST_RS_IMG_WINDOWS_TAG-main-$GST_RS_STABLE"

  RUST_DOCS_FLAGS: "--cfg docsrs --extern-html-root-url=muldiv=https://docs.rs/muldiv/1.0.0/muldiv/ -Z unstable-options --generate-link-to-definition"
  NAMESPACE: gstreamer
  # format is <branch>=<name>
  # the name is used in the URL
  # latest release must be at the top
  # (only relevant on main branch)
  RELEASES:
    0.22=0.22

stages:
  - "trigger"
  - "container-base"
  - "container-final"
  - "lint"
  - "test"
  - "extras"
  - "deploy"

# This is an empty job that is used to trigger the pipeline.
trigger:
  image: alpine:latest
  stage: 'trigger'
  variables:
    GIT_STRATEGY: none
    GIT_SUBMODULE_STRATEGY: "none"
  tags: [ 'placeholder-job' ]
  script:
    - echo "Trigger job done, now running the pipeline."
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    # If the MR is assigned to the Merge bot, trigger the pipeline automatically
    - if: '$CI_MERGE_REQUEST_ASSIGNEES == "gstreamer-merge-bot"'
    # Require explicit action to trigger tests post merge
    - if: '$CI_PROJECT_NAMESPACE == "gstreamer" && $CI_COMMIT_BRANCH == "main"'
      when: 'manual'
    # When the assignee isn't the merge bot, require an explicit action to trigger the pipeline
    # to avoid wasting CI resources
    - if: '$CI_MERGE_REQUEST_ASSIGNEES != "gstreamer-merge-bot"'
      when: 'manual'
      allow_failure: false

.debian:12:
  needs: []
  variables:
    FDO_DISTRIBUTION_VERSION: 'bookworm-slim'
  before_script:
    - source ./ci/env.sh
    - mkdir .cargo && echo -e "[net]\ngit-fetch-with-cli = true" > .cargo/config.toml

.debian:12-base:
  extends: .debian:12
  variables:
    FDO_DISTRIBUTION_TAG: 'base-$GST_RS_IMG_TAG'

.debian:12-stable:
  extends: .debian:12
  variables:
    RUST_IMAGE_FULL: "1"
    FDO_DISTRIBUTION_TAG: '$GST_RS_STABLE-$GST_RS_IMG_TAG'
    FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh $GST_RS_STABLE $RUST_IMAGE_FULL'

.debian:12-msrv:
  extends: .debian:12
  variables:
    FDO_DISTRIBUTION_TAG: '$GST_RS_MSRV-$GST_RS_IMG_TAG'
    FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh $GST_RS_MSRV $RUST_IMAGE_FULL'

.debian:12-nightly:
  extends: .debian:12
  variables:
    FDO_DISTRIBUTION_TAG: 'nightly-$GST_RS_IMG_TAG'
    FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh nightly $RUST_IMAGE_FULL'

.build-base-image:
  extends:
  - .fdo.container-build@debian
  stage: container-base
  variables:
    FDO_DISTRIBUTION_PACKAGES: >-
      build-essential curl python3-setuptools libglib2.0-dev libxml2-dev
      libdrm-dev libegl1-mesa-dev libgl1-mesa-dev libgbm-dev libgles2-mesa-dev
      libgl1-mesa-dri libegl-dev libgl1-mesa-glx libwayland-egl1-mesa xz-utils
      libssl-dev git wget ca-certificates ninja-build python3-pip flex bison
      libglib2.0-dev libx11-dev libx11-xcb-dev libsoup2.4-dev libvorbis-dev
      libogg-dev libtheora-dev libmatroska-dev libvpx-dev libopus-dev
      libgraphene-1.0-dev libjpeg-dev libwayland-dev wayland-protocols
      python3-gi libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev
      libswscale-dev yasm libx264-dev libfontconfig-dev libfreetype-dev
      libxkbcommon-dev libxi-dev libxcb-render0-dev libxcb-shm0-dev
      libxcb1-dev libxext-dev libxrender-dev libxrandr-dev libxcursor-dev
      libxdamage-dev libxfixes-dev libxinerama-dev libgudev-1.0-dev
      libpango1.0-dev libcairo2-dev libjson-glib-dev libgdk-pixbuf-2.0-dev
      libtiff-dev libpng-dev libjpeg-dev libepoxy-dev libsass-dev sassc
      libcsound64-dev llvm clang nasm libsodium-dev libwebp-dev
      libflac-dev libmysofa-dev libgtk-4-dev
    FDO_DISTRIBUTION_EXEC: >-
      bash ci/install-gst.sh &&
      bash ci/install-dav1d.sh &&
      pip3 install --break-system-packages git+http://gitlab.freedesktop.org/freedesktop/ci-templates

.build-final-image:
  extends:
  - .fdo.container-build@debian
  stage: container-final
  variables:
    FDO_BASE_IMAGE: '$CI_REGISTRY_IMAGE/debian/bookworm-slim:base-$GST_RS_IMG_TAG'

build-base:
  extends:
    - .build-base-image
    - .debian:12-base

build-stable:
  needs: ["build-base"]
  extends:
    - .build-final-image
    - .debian:12-stable

build-msrv:
  needs: ["build-base"]
  extends:
    - .build-final-image
    - .debian:12-msrv

build-nightly:
  needs: ["build-base"]
  extends:
    - .build-final-image
    - .debian:12-nightly

update-nightly:
  extends: build-nightly
  rules:
    - if: $UPDATE_NIGHTLY == "1"
  variables:
      FDO_FORCE_REBUILD: 1

.dist-debian-container:
  extends:
  - .fdo.distribution-image@debian
  after_script:
    - rm -rf target

.img-stable:
  extends:
    - .debian:12-stable
    - .dist-debian-container

.img-msrv:
  extends:
    - .debian:12-msrv
    - .dist-debian-container

.img-nightly:
  extends:
    - .debian:12-nightly
    - .dist-debian-container

.cargo_test_var: &cargo_test
    - ./ci/run-cargo-test.sh

.cargo test:
  stage: "test"
  script:
    - *cargo_test

test msrv:
  extends:
    - '.cargo test'
    - .img-msrv
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-msrv'
      artifacts: false

test stable:
  extends:
    - '.cargo test'
    - .img-stable
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-stable'
      artifacts: false

test stable all-features:
  variables:
    ALL_FEATURES: 'yes'
    EXAMPLES_TUTORIALS: 'yes'
  extends:
    - '.cargo test'
    - .img-stable
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-stable'
      artifacts: false


test nightly:
  allow_failure: true
  extends:
    - '.cargo test'
    - .img-nightly
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-nightly'
      artifacts: false


test nightly all-features:
  allow_failure: true
  variables:
    ALL_FEATURES: 'yes'
    EXAMPLES_TUTORIALS: 'yes'
  extends:
    - '.cargo test'
    - .img-nightly
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-nightly'
      artifacts: false

.cargo test sys:
  stage: "test"
  script:
    - ./ci/run-sys-cargo-test.sh

test stable sys:
  extends:
    - '.cargo test sys'
    - .img-stable
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-stable'
      artifacts: false

test msrv sys:
  extends:
    - '.cargo test sys'
    - .img-msrv
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-msrv'
      artifacts: false

test nightly sys:
  extends:
    - '.cargo test sys'
    - .img-nightly
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-nightly'
      artifacts: false

rustfmt:
  extends: .img-stable
  stage: "lint"
  tags: [ 'placeholder-job' ]
  variables:
    GIT_SUBMODULE_STRATEGY: "none"
  script:
    - cargo fmt --version
    - cargo fmt -- --color=always --check
  needs:
    - job: 'build-stable'
      artifacts: false

check commits:
  extends: .img-stable
  stage: "lint"
  tags: [ 'placeholder-job' ]
  variables:
    GIT_SUBMODULE_STRATEGY: "none"
  script:
    - ci-fairy check-commits --textwidth 0 --no-signed-off-by
  needs:
    - job: 'build-stable'
      artifacts: false

typos:
  extends: .img-stable
  stage: "lint"
  tags: [ 'placeholder-job' ]
  variables:
    GIT_SUBMODULE_STRATEGY: "none"
  script:
    - typos
  needs:
    - job: 'build-stable'
      artifacts: false

clippy:
  extends: .img-stable
  stage: 'extras'
  variables:
    CLIPPY_LINTS: -D warnings -W unknown-lints
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-stable'
      artifacts: false
  script:
    - ./ci/run-clippy.sh

deny:
  extends: .img-stable
  stage: 'extras'
  needs:
    - job: 'build-stable'
      artifacts: false
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
  script:
    - cargo update --color=always
    - cargo deny --color=always --workspace --all-features check all

gir-checks:
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
  extends: .img-stable
  stage: 'extras'
  tags:
    - "gstreamer"
  needs:
    - job: 'build-stable'
      artifacts: false
  script:
    - git submodule update --checkout
    - python3 ci/gir-checks.py

outdated:
  extends: .img-stable
  stage: 'extras'
  variables:
    GIT_SUBMODULE_STRATEGY: "none"
  needs:
    - job: 'build-stable'
      artifacts: false
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
  script:
    - cargo update --color=always
    - cargo outdated --color=always --root-deps-only --exit-code 1 -v

coverage:
  allow_failure: true
  extends:
    - '.cargo test'
    - .img-stable
  stage: 'extras'
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'build-stable'
      artifacts: false
  variables:
    ALL_FEATURES: 'yes'
    RUSTFLAGS: "-Cinstrument-coverage"
    LLVM_PROFILE_FILE: "gstreamer-rs-%p-%m.profraw"
  script:
    - *cargo_test
    # generate html and cobertura report for gitlab integration
    - mkdir -p coverage
    - grcov . --binary-path ./target/debug/ -s . -t html,cobertura --branch --ignore-not-existing --ignore "*target*" --ignore "*/sys/*" --ignore "examples/*" --ignore "tutorials/*" --ignore "*/build.rs" -o ./coverage/
    # output coverage summary for gitlab parsing.
    # TODO: use grcov once https://github.com/mozilla/grcov/issues/556 is fixed
    - grep % coverage/html/index.html | head -1 ; true
  artifacts:
    paths:
      - 'coverage'
    reports:
      coverage_report:
        coverage_format: cobertura
        path: "coverage/cobertura.xml"

doc-stripping:
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
  extends: .img-nightly
  tags:
    - "gstreamer"
  stage: 'extras'
  needs:
    - job: 'build-nightly'
      artifacts: false
  script:
    - git submodule update --checkout
    - PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --embed-docs
    - PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --strip-docs
    - git diff --quiet || (echo 'Files changed after running `rustdoc-stripper -s`, make sure all documentation is protected with `// rustdoc-stripper-ignore-next`!'; git diff; false)

regen-check:
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
  extends: .img-nightly
  tags:
    - "gstreamer"
  stage: 'extras'
  needs:
    - job: 'build-nightly'
      artifacts: false
  script:
    - git submodule update --checkout
    - PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --yes
    - git diff --quiet || (echo 'Files changed after running `generator.py`, make sure all submodules and generated files are in the correct version!'; git diff; false)

docs:
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
  extends: .img-nightly
  stage: 'extras'
  tags:
    - "gstreamer"
  needs:
    - job: 'build-nightly'
      artifacts: false
  script:
    - git submodule update --checkout
    - curl --proto '=https' --tlsv1.2 -sSf -o gir-rustdoc.py
        https://gitlab.gnome.org/World/Rust/gir-rustdoc/-/raw/main/gir-rustdoc.py
    - chmod +x gir-rustdoc.py
    - PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --embed-docs --no-fmt
    - |
      RUSTDOCFLAGS="$RUST_DOCS_FLAGS"
      RUSTFLAGS="--cfg docsrs"
      eval $(./gir-rustdoc.py pre-docs)
      cargo +nightly doc --workspace --exclude examples --exclude tutorials --all-features --color=always --no-deps
    - mv target/doc docs
  artifacts:
    paths:
      - 'docs'

# https://docs.gitlab.com/ee/user/project/pages/#how-it-works
# GitLab automatically deploys the `public/` folder from an
# artifact generated by the job named `pages`.  This step
# re-uses the docs from the build-test `docs` step above.
pages:
  extends: .img-nightly
  stage: 'deploy'
  needs: [ 'docs' ]
  interruptible: false
  script:
    - curl --proto '=https' --tlsv1.2 -sSf -o gir-rustdoc.py
        https://gitlab.gnome.org/World/Rust/gir-rustdoc/-/raw/main/gir-rustdoc.py
    - chmod +x gir-rustdoc.py
    - ./gir-rustdoc.py html-index
    # development docs
    - mkdir public/git
    - mv docs public/git/docs
    # stable docs
    - ./gir-rustdoc.py docs-from-artifacts
    - ls public/
  artifacts:
    paths:
      - 'public'
  rules:
    - if: ($CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH) && ($CI_PROJECT_NAMESPACE == $NAMESPACE)
      when: 'manual'


.windows rust docker build:
  stage: 'container-final'
  timeout: '2h'
  needs: []
  variables:
    # Unlike the buildah/linux jobs, this file
    # needs to be relative to windows-docker/ subdir
    # as it makes life easier in the powershell script
    #
    # We also don't need a CONTEXT_DIR var as its also
    # hardcoded to be windows-docker/
    DOCKERFILE: 'ci/windows-docker/Dockerfile'
  tags:
    - 'windows'
    - 'shell'
    - '2022'
    - "gstreamer-windows"
  script:
    # We need to pass an array and to resolve the env vars, so we can't use a variable:
    - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "RUST_VERSION=$RUST_VERSION")

    - "& ci/windows-docker/container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $RUST_IMAGE $RUST_UPSTREAM_IMAGE $DOCKERFILE"
    - |
      if (!($?)) {
        echo "Failed to build the image"
        Exit 1
      }

windows rust docker stable:
  extends: '.windows rust docker build'
  variables:
    RUST_IMAGE: !reference [variables, "WINDOWS_RUST_STABLE_IMAGE"]
    RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_STABLE_UPSTREAM_IMAGE"]
    RUST_VERSION: !reference [variables, "GST_RS_STABLE"]

windows rust docker msrv:
  extends: '.windows rust docker build'
  variables:
    RUST_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_IMAGE"]
    RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE"]
    RUST_VERSION: !reference [variables, "GST_RS_MSRV"]

.msvc2019 build:
  stage: 'test'
  tags:
    - 'docker'
    - 'windows'
    - '2022'
    - "gstreamer-windows"
  script:
    # Skip -sys tests as they don't work
    # https://github.com/gtk-rs/gtk3-rs/issues/54
    #
    # We need to build each crate separately to avoid crates like -egl,-wayland etc on windows
    - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 &&
      powershell ./ci/run_windows_tests.ps1"

    - |
      if (!$?) {
        Write-Host "Tests Failed!"
        Exit 1
      }

test windows msrv:
  image: $WINDOWS_RUST_MINIMUM_IMAGE
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'windows rust docker msrv'
      artifacts: false
  extends: '.msvc2019 build'

test windows stable:
  needs:
    - job: 'trigger'
      artifacts: false
    - job: 'windows rust docker stable'
      artifacts: false
  image: "$WINDOWS_RUST_STABLE_IMAGE"
  extends: '.msvc2019 build'