diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..b25c15b81f --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..c911c1af4d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,24 @@ +include: "gitlab/ci_template.yml" + +test manifest: + variables: + GIT_STRATEGY: fetch + image: "$TEST_MANIFEST_IMAGE" + stage: "preparation" + script: + - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py + coverage: '/TOTAL.*\s+(\d+%)$/' + artifacts: + reports: + junit: + - "junit.xml" + +test manifest amd64 docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$TEST_MANIFEST_AMD64_SUFFIX" + TAG: "$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" + + CONTEXT_DIR: "docker/test_manifest/" + DOCKERFILE: "docker/test_manifest/Dockerfile" + extends: .base diff --git a/README.txt b/README.txt new file mode 100644 index 0000000000..1a3174d163 --- /dev/null +++ b/README.txt @@ -0,0 +1,11 @@ +GStreamer Continuous Integration +================================ + +This repository contains all material relevant to the GStreamer +Continuous Integration system. + +* Jenkins scripts + +* Docker images + +* Build scripts and code diff --git a/docker/README b/docker/README new file mode 100644 index 0000000000..1e99ca0a6f --- /dev/null +++ b/docker/README @@ -0,0 +1,2 @@ +GStreamer Docker images + diff --git a/docker/android/Dockerfile b/docker/android/Dockerfile new file mode 100644 index 0000000000..f874ff01d2 --- /dev/null +++ b/docker/android/Dockerfile @@ -0,0 +1,11 @@ +FROM registry.fedoraproject.org/fedora:31 + +ENV ANDROID_HOME=/android/sdk +ENV ANDROID_NDK_HOME=/android/ndk + +ARG DEFAULT_BRANCH="master" + +COPY prepare.sh cleanup.sh /root/ + +RUN /usr/bin/sh /root/prepare.sh && \ + /usr/bin/sh /root/cleanup.sh diff --git a/docker/android/cleanup.sh b/docker/android/cleanup.sh new file mode 100644 index 0000000000..a60a9fcf4b --- /dev/null +++ b/docker/android/cleanup.sh @@ -0,0 +1,7 @@ +set -eux + +echo "Removing DNF cache" +dnf clean all + +rm -R /root/* +rm -rf /var/cache/dnf /var/log/dnf* diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh new file mode 100644 index 0000000000..953bc5ec5a --- /dev/null +++ b/docker/android/prepare.sh @@ -0,0 +1,56 @@ +set -eux + +dnf install -y \ + file \ + git-core \ + java-1.8.0-openjdk-devel \ + lbzip2 \ + make \ + pkg-config \ + unzip \ + which \ + xz + +mkdir -p /android/sources + +curl -o /android/sources/android-ndk.zip https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip +unzip /android/sources/android-ndk.zip -d ${ANDROID_NDK_HOME}/ +# remove the intermediate versioned directory +mv ${ANDROID_NDK_HOME}/*/* ${ANDROID_NDK_HOME}/ + +curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-6200805_latest.zip +unzip /android/sources/android-sdk-tools.zip -d ${ANDROID_HOME}/ +mkdir -p ${ANDROID_HOME}/licenses + +# Accept licenses. Values taken from: +# $ANDROID_HOME/tools/bin/sdkmanager --sdk_root=$ANDROID_HOME --licenses +# cd $ANDROID_HOME +# for f in licenses/*; do echo "echo \"$(cat $f | tr -d '\n')\" > \${ANDROID_HOME}/$f"; done +echo "601085b94cd77f0b54ff86406957099ebe79c4d6" > ${ANDROID_HOME}/licenses/android-googletv-license +echo "859f317696f67ef3d7f30a50a5560e7834b43903" > ${ANDROID_HOME}/licenses/android-sdk-arm-dbt-license +echo "24333f8a63b6825ea9c5514f83c2829b004d1fee" > ${ANDROID_HOME}/licenses/android-sdk-license +echo "84831b9409646a918e30573bab4c9c91346d8abd" > ${ANDROID_HOME}/licenses/android-sdk-preview-license +echo "33b6a2b64607f11b759f320ef9dff4ae5c47d97a" > ${ANDROID_HOME}/licenses/google-gdk-license +echo "e9acab5b5fbb560a72cfaecce8946896ff6aab9d" > ${ANDROID_HOME}/licenses/mips-android-sysimage-license + +# pre-cache deps +export GSTREAMER_ROOT_ANDROID=/android/sources/gstreamer-android +curl -o /android/sources/gstreamer-android.tar.xz https://gstreamer.freedesktop.org/data/pkg/android/1.16.2/gstreamer-1.0-android-universal-1.16.2.tar.xz +mkdir $GSTREAMER_ROOT_ANDROID +tar -xvf /android/sources/gstreamer-android.tar.xz -C $GSTREAMER_ROOT_ANDROID +ls $GSTREAMER_ROOT_ANDROID + +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-examples.git /android/sources/gst-examples +chmod +x /android/sources/gst-examples/playback/player/android/gradlew +/android/sources/gst-examples/playback/player/android/gradlew --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies + +chmod +x /android/sources/gst-examples/vulkan/android/gradlew +/android/sources/gst-examples/vulkan/android/gradlew --no-daemon --project-dir /android/sources/gst-examples/vulkan/android dependencies --refresh-dependencies + +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs +chmod +x /android/sources/gst-docs/examples/tutorials/android/gradlew +/android/sources/gst-docs/examples/tutorials/android/gradlew --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies + +unset GSTREAMER_ROOT_ANDROID + +rm -rf /android/sources diff --git a/docker/build_manifest/Dockerfile b/docker/build_manifest/Dockerfile new file mode 100644 index 0000000000..e3c4921e6f --- /dev/null +++ b/docker/build_manifest/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.7.1-alpine + +ARG DEFAULT_BRANCH="master" + +RUN pip install requests +RUN apk add git +RUN cd / && git clone -b ${DEFAULT_BRANCH:=master} https://gitlab.freedesktop.org/gstreamer/gst-ci.git diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora new file mode 100644 index 0000000000..c903b9562b --- /dev/null +++ b/docker/cerbero/Dockerfile-fedora @@ -0,0 +1,35 @@ +FROM registry.fedoraproject.org/fedora:31 + +ENV TERM="dumb" + +ARG DEFAULT_BRANCH="master" + +# System setup +RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ + dnf upgrade -y && \ + dnf install -y sudo git lbzip2 rsync wine which + +# Configure git for various usage +RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ + git config --global user.name "Cerbero Build System" + +RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ + mkdir $HOME/.cerbero && \ + echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \ + echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \ + cd cerbero && \ + echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \ + echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \ + ./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap --jobs=4 && \ + ./cerbero-uninstalled -t -c localconf.cbc fetch-package --jobs=4 gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap --jobs=4 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package --jobs=4 gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap --jobs=4 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package --jobs=4 gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap --jobs=4 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package --jobs=4 gstreamer-1.0 && \ + cd .. && \ + rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \ + rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \ + dnf clean all diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile new file mode 100644 index 0000000000..624cfcdcb8 --- /dev/null +++ b/docker/fedora/Dockerfile @@ -0,0 +1,12 @@ +FROM registry.fedoraproject.org/fedora:31 + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH + +ARG DEFAULT_BRANCH="master" + +COPY prepare.sh cleanup.sh /root/ + +RUN /usr/bin/sh /root/prepare.sh && \ + /usr/bin/sh /root/cleanup.sh diff --git a/docker/fedora/cleanup.sh b/docker/fedora/cleanup.sh new file mode 100644 index 0000000000..a60a9fcf4b --- /dev/null +++ b/docker/fedora/cleanup.sh @@ -0,0 +1,7 @@ +set -eux + +echo "Removing DNF cache" +dnf clean all + +rm -R /root/* +rm -rf /var/cache/dnf /var/log/dnf* diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh new file mode 100644 index 0000000000..a85f3e495d --- /dev/null +++ b/docker/fedora/prepare.sh @@ -0,0 +1,244 @@ +set -eux + +# Fedora base image disable installing documentation files. See https://pagure.io/atomic-wg/issue/308 +# We need them to cleanly build our doc. +sed -i "s/tsflags=nodocs//g" /etc/dnf/dnf.conf + +dnf install -y git-core ninja-build dnf-plugins-core python3-pip + +# Configure git for various usage +git config --global user.email "gst-build@gstreamer.net" +git config --global user.name "Gstbuild Runner" + +# Add rpm fusion repositories in order to access all of the gst plugins +sudo dnf install -y \ + "https://mirrors.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm" \ + "https://mirrors.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm" + +dnf upgrade -y + +# install rest of the extra deps +dnf install -y \ + aalib-devel \ + aom \ + bat \ + intel-mediasdk-devel \ + libaom \ + libaom-devel \ + libcaca-devel \ + libdav1d \ + libdav1d-devel \ + ccache \ + cmake \ + clang-devel \ + elfutils \ + elfutils-libs \ + elfutils-devel \ + gcc \ + gcc-c++ \ + gdb \ + git-lfs \ + glslc \ + gtk3 \ + gtk3-devel \ + gtest \ + gtest-devel \ + graphene \ + graphene-devel \ + gsl \ + gsl-devel \ + gupnp \ + gupnp-devel \ + gupnp-igd \ + gupnp-igd-devel \ + gssdp \ + gssdp-devel \ + faac-devel \ + ffmpeg \ + ffmpeg-libs \ + ffmpeg-devel \ + flex \ + flite \ + flite-devel \ + mono-devel \ + procps-ng \ + patch \ + qt5-devel \ + redhat-rpm-config \ + json-glib \ + json-glib-devel \ + libnice \ + libnice-devel \ + libsodium-devel \ + libunwind \ + libunwind-devel \ + libyaml-devel \ + libxml2-devel \ + libxslt-devel \ + llvm-devel \ + log4c-devel \ + make \ + nasm \ + neon \ + neon-devel \ + nunit \ + npm \ + opencv \ + opencv-devel \ + openjpeg2 \ + openjpeg2-devel \ + SDL2 \ + SDL2-devel \ + sbc \ + sbc-devel \ + x264 \ + x264-libs \ + x264-devel \ + python3 \ + python3-devel \ + python3-libs \ + python3-gobject \ + python3-cairo \ + python3-cairo-devel \ + valgrind \ + vulkan \ + vulkan-devel \ + mesa-omx-drivers \ + mesa-libGL \ + mesa-libGL-devel \ + mesa-libGLU \ + mesa-libGLU-devel \ + mesa-libGLES \ + mesa-libGLES-devel \ + mesa-libOpenCL \ + mesa-libOpenCL-devel \ + mesa-libgbm \ + mesa-libgbm-devel \ + mesa-libd3d \ + mesa-libd3d-devel \ + mesa-libOSMesa \ + mesa-libOSMesa-devel \ + mesa-vulkan-drivers \ + wpewebkit \ + wpewebkit-devel \ + xorg-x11-server-utils \ + xorg-x11-server-Xvfb + +# Install common debug symbols +dnf debuginfo-install -y gtk3 \ + glib2 \ + glibc \ + gupnp \ + gupnp-igd \ + gssdp \ + freetype \ + openjpeg \ + gobject-introspection \ + python3 \ + python3-libs \ + python3-gobject \ + libappstream-glib-devel \ + libjpeg-turbo \ + glib-networking \ + libcurl \ + libsoup \ + nss \ + nss-softokn \ + nss-softokn-freebl \ + nss-sysinit \ + nss-util \ + openssl \ + openssl-libs \ + openssl-pkcs11 \ + brotli \ + bzip2-libs \ + gpm-libs \ + harfbuzz \ + harfbuzz-icu \ + json-c \ + json-glib \ + libbabeltrace \ + libffi \ + libsrtp \ + libunwind \ + mpg123-libs \ + neon \ + orc-compiler \ + orc \ + pixman \ + pulseaudio-libs \ + pulseaudio-libs-glib2 \ + wavpack \ + webrtc-audio-processing \ + ffmpeg \ + ffmpeg-libs \ + faad2-libs \ + libavdevice \ + libmpeg2 \ + faac \ + fdk-aac \ + x264 \ + x264-libs \ + x265 \ + x265-libs \ + xz \ + xz-libs \ + zip \ + zlib + +# Install the dependencies of gstreamer +dnf builddep -y gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-plugins-good \ + gstreamer1-plugins-good-extras \ + gstreamer1-plugins-ugly \ + gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-free \ + gstreamer1-plugins-bad-free-extras \ + gstreamer1-plugins-bad-freeworld \ + gstreamer1-libav \ + gstreamer1-rtsp-server \ + gstreamer1-vaapi \ + python3-gstreamer1 + +dnf remove -y meson +pip3 install meson==0.58.0 hotdoc + +# Remove gst-devel packages installed by builddep above +dnf remove -y "gstreamer1*devel" + +# FIXME: Why does installing directly with dnf doesn't actually install +# the documentation files? +dnf download glib2-doc gdk-pixbuf2-devel*x86_64* gtk3-devel-docs +rpm -i --reinstall *.rpm +rm -f *.rpm + +# Install Rust +RUSTUP_VERSION=1.24.2 +RUST_VERSION=1.52.1 +RUST_ARCH="x86_64-unknown-linux-gnu" + +dnf install -y wget +RUSTUP_URL=https://static.rust-lang.org/rustup/archive/$RUSTUP_VERSION/$RUST_ARCH/rustup-init +wget $RUSTUP_URL +dnf remove -y wget + +chmod +x rustup-init; +./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; +rm rustup-init; +chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +rustup --version +cargo --version +rustc --version + +# get gst-build and make all subprojects available +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-build.git /gst-build/ +cd /gst-build +meson subprojects download + +# Run git gc to prune unwanted refs and reduce the size of the image +for i in $(find subprojects/ -mindepth 1 -maxdepth 1 -type d); +do + git -C $i gc --aggressive || true; +done \ No newline at end of file diff --git a/docker/indent/Dockerfile b/docker/indent/Dockerfile new file mode 100644 index 0000000000..63581e3597 --- /dev/null +++ b/docker/indent/Dockerfile @@ -0,0 +1,7 @@ +FROM debian:stretch-slim + +RUN apt update -yqq \ + && apt install -y curl indent git findutils \ + && rm -R /var/lib/apt/ /var/log/apt/ \ + && curl -L -o /usr/local/bin/gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent \ + && chmod a+x /usr/local/bin/gst-indent diff --git a/docker/test_manifest/Dockerfile b/docker/test_manifest/Dockerfile new file mode 100644 index 0000000000..cdc369c822 --- /dev/null +++ b/docker/test_manifest/Dockerfile @@ -0,0 +1,8 @@ +FROM registry.fedoraproject.org/fedora:31 + +RUN dnf install -y \ + python3-pytest \ + python3-requests \ + python3-pytest-cov \ + git-core \ + && rm -rf /var/cache/dnf /var/log/dnf* diff --git a/docker/windows/.dockerignore b/docker/windows/.dockerignore new file mode 100644 index 0000000000..76bab46af2 --- /dev/null +++ b/docker/windows/.dockerignore @@ -0,0 +1 @@ +build_image.ps1 diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile new file mode 100644 index 0000000000..e36e4fa01c --- /dev/null +++ b/docker/windows/Dockerfile @@ -0,0 +1,23 @@ +# escape=` + +FROM 'mcr.microsoft.com/windows/servercore:ltsc2019' + +# Make sure any failure in PowerShell scripts is fatal +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] +ENV ErrorActionPreference='Stop' + +COPY install_choco.ps1 C:\ +RUN C:\install_choco.ps1 + +RUN 'git config --global user.email "cirunner@gstreamer.freedesktop.org"; git config --global user.name "GStreamer CI system"' + +COPY install_toolchain.ps1 C:\ +RUN C:\install_toolchain.ps1 + +ARG DEFAULT_BRANCH="master" + +COPY prepare_gst_env.ps1 C:\ +RUN C:\prepare_gst_env.ps1 + +COPY prepare_cerbero_env.sh C:\ +RUN C:\MinGW\msys\1.0\bin\bash.exe --login -c "C:/prepare_cerbero_env.sh" diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 new file mode 100644 index 0000000000..a4be56c1fa --- /dev/null +++ b/docker/windows/build_image.ps1 @@ -0,0 +1,27 @@ +$env:ErrorActionPreference='Stop' + +$env:DEFAULT_BRANCH='master' +$env:VERSION='test' +$env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" +$env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH" + +Set-Location './docker/windows/' + +Get-Date +Write-Output "Building $env:tag" +docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . +if (!$?) { + Write-Host "Failed to build docker image $env:tag" + Exit 1 +} + +Get-Date +Write-Output "Building $env:rust_tag" +docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f rust.Dockerfile -t $env:rust_tag . +if (!$?) { + Write-Host "Failed to build docker image $env:rust_tag" + Exit 1 +} + +Get-Date +Write-Output "Build Finished" \ No newline at end of file diff --git a/docker/windows/container.ps1 b/docker/windows/container.ps1 new file mode 100644 index 0000000000..e5dc2cd835 --- /dev/null +++ b/docker/windows/container.ps1 @@ -0,0 +1,60 @@ +# Copied from mesa, big kudos +# +# https://gitlab.freedesktop.org/mesa/mesa/-/blob/master/.gitlab-ci/windows/mesa_container.ps1 +# https://gitlab.freedesktop.org/mesa/mesa/-/blob/34e3e164936d1d3cef267da7780e87f062fedf39/.gitlab-ci/windows/mesa_container.ps1 + +# Implements the equivalent of ci-templates container-ifnot-exists, using +# Docker directly as we don't have buildah/podman/skopeo available under +# Windows, nor can we execute Docker-in-Docker +$registry_uri = $args[0] +$registry_username = $args[1] +$registry_password = $args[2] +$registry_user_image = $args[3] +$registry_central_image = $args[4] +$dockerfile = $args[5] + +docker --config "windows-docker.conf" login -u "$registry_username" -p "$registry_password" "$registry_uri" +if (!$?) { + Write-Host "docker login failed to $registry_uri" + Exit 1 +} + +# if the image already exists, don't rebuild it +docker --config "windows-docker.conf" pull "$registry_user_image" +if ($?) { + Write-Host "User image $registry_user_image already exists; not rebuilding" + docker --config "windows-docker.conf" logout "$registry_uri" + Exit 0 +} + +# if the image already exists upstream, copy it +docker --config "windows-docker.conf" pull "$registry_central_image" +if ($?) { + Write-Host "Copying central image $registry_central_image to user image $registry_user_image" + docker --config "windows-docker.conf" tag "$registry_central_image" "$registry_user_image" + docker --config "windows-docker.conf" push "$registry_user_image" + $pushstatus = $? + docker --config "windows-docker.conf" logout "$registry_uri" + if (!$pushstatus) { + Write-Host "Pushing image to $registry_user_image failed" + Exit 1 + } + Exit 0 +} + +Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding" +docker --config "windows-docker.conf" build $DOCKER_BUILD_ARGS --no-cache -t "$registry_user_image" -f "$dockerfile" "./docker/windows" +if (!$?) { + Write-Host "Container build failed" + docker --config "windows-docker.conf" logout "$registry_uri" + Exit 1 +} +Get-Date + +docker --config "windows-docker.conf" push "$registry_user_image" +$pushstatus = $? +docker --config "windows-docker.conf" logout "$registry_uri" +if (!$pushstatus) { + Write-Host "Pushing image to $registry_user_image failed" + Exit 1 +} diff --git a/docker/windows/install_choco.ps1 b/docker/windows/install_choco.ps1 new file mode 100644 index 0000000000..b19eb7b0a3 --- /dev/null +++ b/docker/windows/install_choco.ps1 @@ -0,0 +1,23 @@ +Get-Date +Write-Host "Installing Chocolatey" +Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) +Import-Module "$env:ProgramData\chocolatey\helpers\chocolateyProfile.psm1" +Update-SessionEnvironment + +choco install -y vcredist140 +$vc140_install = $? + +Write-Host "Installing Chocolatey packages" +choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' +$cmake_install = $? + +choco install -y git --params "/NoAutoCrlf /NoCredentialManager /NoShellHereIntegration /NoGuiHereIntegration /NoShellIntegration" +$git_install = $? + +choco install -y python3 git-lfs 7zip +$rest_installs = $? + +if (!($vc140_install -and $cmake_install -and $git_install -and $rest_installs)) { + Write-Host "Failed to install some dependencies from choco" + Exit 1 +} diff --git a/docker/windows/install_gst.ps1 b/docker/windows/install_gst.ps1 new file mode 100644 index 0000000000..6105f3623b --- /dev/null +++ b/docker/windows/install_gst.ps1 @@ -0,0 +1,71 @@ +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +# Download gst-build and all its subprojects +# git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +# FIXME: need 1.19+ for cairo subproject :/ +# Should use a stable branch instead +git clone -b master --depth 1 https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +if (!$?) { + Write-Host "Failed to clone gst-build" + Exit 1 +} + +Set-Location C:\gst-build + +# Copy the cache we already have in the image to avoid massive redownloads +Move-Item C:/subprojects/* C:\gst-build\subprojects + +if (!$?) { + Write-Host "Failed to copy subprojects cache" + Exit 1 +} + +# Update the subprojects cache +Write-Output "Running meson subproject reset" +meson subprojects update --reset + +if (!$?) { + Write-Host "Failed to reset subprojects state" + Exit 1 +} + +Write-Output "Running git update" +python git-update --no-interaction + +if (!$?) { + Write-Host "Failed to run git-update" + Exit 1 +} + +$env:MESON_ARGS = "-Dglib:installed_tests=false " + + "-Dlibnice:tests=disabled " + + "-Dlibnice:examples=disabled " + + "-Dffmpeg:tests=disabled " + + "-Dopenh264:tests=disabled " + + "-Dpygobject:tests=false " + + "-Dugly=enabled " + + "-Dbad=enabled " + + "-Dges=enabled " + + "-Drtsp_server=enabled " + + "-Ddevtools=enabled " + + "-Dsharp=disabled " + + "-Dpython=disabled " + + "-Dlibav=disabled " + + "-Dvaapi=disabled " + + "-Dgst-plugins-base:pango=enabled " + + "-Dgst-plugins-good:cairo=enabled " + +Write-Output "Building gst" +cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && meson _build $env:MESON_ARGS && meson compile -C _build && ninja -C _build install" + +if (!$?) { + Write-Host "Failed to build and install gst" + Exit 1 +} + +git clean -fdxx + +if (!$?) { + Write-Host "Failed to git clean" + Exit 1 +} \ No newline at end of file diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 new file mode 100644 index 0000000000..fd61b42d1c --- /dev/null +++ b/docker/windows/install_toolchain.ps1 @@ -0,0 +1,69 @@ +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +$msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' +$msys2_url = 'https://github.com/msys2/msys2-installer/releases/download/2021-02-15/msys2-base-x86_64-20210215.tar.xz' +$msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' + +Get-Date +Write-Host "Downloading Visual Studio 2017 build tools" +Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe + +Get-Date +Write-Host "Installing Visual Studio 2017" +Start-Process -NoNewWindow -Wait C:\vs_buildtools.exe -ArgumentList '--wait --quiet --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' +if (!$?) { + Write-Host "Failed to install Visual Studio tools" + Exit 1 +} +Remove-Item C:\vs_buildtools.exe -Force + +Get-Date +Write-Host "Downloading and extracting mingw-get for MSYS" +Invoke-WebRequest -Uri $msys_mingw_get_url -OutFile C:\mingw-get.tar.xz +7z e C:\mingw-get.tar.xz -o"C:\\" +$res1 = $? +7z x C:\mingw-get.tar -o"C:\\MinGW" +$res2 = $? + +if (!($res1 -and $res2)) { + Write-Host "Failed to extract mingw-get" + Exit 1 +} + +Remove-Item C:\mingw-get.tar.xz -Force +Remove-Item C:\mingw-get.tar -Force + +Get-Date +Write-Host "Installing MSYS for Cerbero into C:/MinGW using mingw-get" +Start-Process -Wait C:\MinGW\bin\mingw-get.exe -ArgumentList 'install msys-base mingw32-base mingw-developer-toolkit' +if (!$?) { + Write-Host "Failed to install Msys for cerbero using MinGW" + Exit 1 +} + +Get-Date +Write-Host "Installing MSYS2 into C:/msys64" +Invoke-WebRequest -Uri $msys2_url -OutFile C:\msys2-x86_64.tar.xz + +7z e C:\msys2-x86_64.tar.xz -o"C:\\" +$res1 = $? +7z x C:\msys2-x86_64.tar -o"C:\\" +$res2 = $? + +if (!($res1 -and $res2)) { + Write-Host "Failed to extract msys2" + Exit 1 +} + +Remove-Item C:\msys2-x86_64.tar.xz -Force +Remove-Item C:\msys2-x86_64.tar -Force + +Get-Date +Write-Host "Installing Meson" +pip3 install meson +if (!$?) { + Write-Host "Failed to install meson from pip" + Exit 1 +} + +Write-Host "Toolchain Install Complete" diff --git a/docker/windows/prepare_cerbero_env.sh b/docker/windows/prepare_cerbero_env.sh new file mode 100644 index 0000000000..da94acabb3 --- /dev/null +++ b/docker/windows/prepare_cerbero_env.sh @@ -0,0 +1,27 @@ +#! /bin/bash + +set -eux + +cd C:/ +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git +cd cerbero + +echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc +echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc +echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc +echo 'vs_install_version = "vs15"' >> localconf.cbc + +# Fetch all bootstrap requirements +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-bootstrap --jobs=4 +# Fetch all package requirements for a mingw gstreamer build +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-package --jobs=4 gstreamer-1.0 +# Fetch all package requirements for a visualstudio gstreamer build +./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package --jobs=4 gstreamer-1.0 + +# Fixup the MSYS installation +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline + +# Wipe visualstudio package dist, sources, logs, and the build tools recipes +./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc wipe --force --build-tools +# clean the localconf +rm /c/cerbero/localconf.cbc diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1 new file mode 100644 index 0000000000..652fe2b4f7 --- /dev/null +++ b/docker/windows/prepare_gst_env.ps1 @@ -0,0 +1,24 @@ +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +# Download gst-build and all its subprojects +git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +if (!$?) { + Write-Host "Failed to clone gst-build" + Exit 1 +} + +# download the subprojects to try and cache them +meson subprojects download --sourcedir C:\gst-build +if (!$?) { + Write-Host "Failed to download the subprojects" + Exit 1 +} + +# Remove files that will conflict with a fresh clone on the runner side +Remove-Item -Force 'C:/gst-build/subprojects/*.wrap' +Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-nasm' +Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-flex-bison-binaries' +Remove-Item -Recurse -Force 'C:/gst-build/subprojects/macos-bison-binary' + +Move-Item C:\gst-build\subprojects C:\subprojects +Remove-Item -Recurse -Force C:\gst-build diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile new file mode 100644 index 0000000000..5e7425fef9 --- /dev/null +++ b/docker/windows/rust.Dockerfile @@ -0,0 +1,21 @@ +# escape=` + +# Expect this to be set when calling docker build with +# --build-arg BASE_IMAGE="" and make it fail if not set. +ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid" +FROM $BASE_IMAGE + +ARG DEFAULT_BRANCH="master" +ARG RUST_VERSION="invalid" + +COPY install_gst.ps1 C:\ +RUN C:\install_gst.ps1 +RUN choco install -y pkgconfiglite +ENV PKG_CONFIG_PATH="C:/lib/pkgconfig" + +ADD https://win.rustup.rs/x86_64 C:\rustup-init.exe +RUN C:\rustup-init.exe -y --profile minimal --default-toolchain $env:RUST_VERSION + +# Uncomment for easy testing +# RUN git clone --depth 1 https://gitlab.freedesktop.org/gstreamer/gstreamer-rs.git +# RUN cd gstreamer-rs; cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64; cargo build --all; cargo test --all" \ No newline at end of file diff --git a/fuzzing/README.txt b/fuzzing/README.txt new file mode 100644 index 0000000000..c9621a0455 --- /dev/null +++ b/fuzzing/README.txt @@ -0,0 +1,80 @@ +Fuzzing GStreamer +================= + + This directory contains the various fuzzing targets and helper + scripts. + +* Fuzzing targets + + Fuzzing targets as small applications where we can test a specific + element or API. The goal is to have them be as small/targetted as + possible. + + ex: appsrc ! ! fakesink num-buffers= + + Not all components can be tested directly and therefore will be + indirectly tested via other targets (ex: libgstaudio will be tested + by targets/elements requiring it) + + Anything that can process externally-provided data should be + covered, but there are cases where it might not make sense to use a + fuzzer (such as most elements processing raw audio/video). + +* build-oss-fuzz.sh + + This is the script executed by the oss-fuzz project. + + It builds glib, GStreamer, plugins and the fuzzing targets. + +* *.c + + The fuzzing targets where the data to test will be provided to a + function whose signature follows the LibFuzzer signature: + https://llvm.org/docs/LibFuzzer.html + +* TODO + + * Add a standalone build script + + We need to be able to build and test the fuzzing targets outside + of the oss-fuzz infrastructure, and do that in our continous + integration system. + + We need: + + * A dummy fuzzing engine (given a directory, it opens all files and + calls the fuzzing targets with the content of those files. + * A script to be able to build those targets with that dummy engine + * A corpus of files to test those targets with. + + * Build targets with dummy engine and run with existing tests. + + * Create pull-based variants + + Currently the existing targets are push-based only. Where + applicable we should make pull-based variants to test the other + code paths. + + * Add more targets + + core: + gst_parse fuzzer ? + base: + ext/ + ogg + opus + pango + theora + vorbis + gst/ + subparse + typefind : already covered in typefind target + gst-libs/gst/ + sdp + other ones easily testable directly ? + + + + + + diff --git a/fuzzing/build-oss-fuzz.sh b/fuzzing/build-oss-fuzz.sh new file mode 100755 index 0000000000..297c322173 --- /dev/null +++ b/fuzzing/build-oss-fuzz.sh @@ -0,0 +1,185 @@ +#!/bin/bash -eu + +# build-oss-fuzz.sh +# +# Build script which is executed by oss-fuzz build.sh +# +# $SRC: location of code checkouts +# $OUT: location to put fuzzing targets and corpus +# $WORK: writable directory where all compilation should be executed +# +# /!\ Do not override any CC, CXX, CFLAGS, ... variables +# + +# This script is divided in two parts +# +# 1) Build all the dependencies statically +# +# 2) Build the fuzzing targets + +# Prefix where we will temporarily install everything +PREFIX=$WORK/prefix +mkdir -p $PREFIX +# always try getting the arguments for static compilation/linking +# Fixes GModule not being picked when gstreamer-1.0.pc is looked up by meson +# more or less https://github.com/mesonbuild/meson/pull/6629 +export PKG_CONFIG="`which pkg-config` --static" +export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig +export PATH=$PREFIX/bin:$PATH + +# Minimize gst-debug level/code +export CFLAGS="$CFLAGS -DGST_LEVEL_MAX=2" + +# +echo "CFLAGS : " $CFLAGS +echo "CXXFLAGS : " $CXXFLAGS +PLUGIN_DIR=$PREFIX/lib/gstreamer-1.0 + +rm -rf $WORK/* + +# Switch to work directory +cd $WORK + +# 1) BUILD GLIB AND GSTREAMER +# Note: we build glib ourselves so that we get proper malloc/free backtraces +tar xvJf $SRC/glib-2.64.2.tar.xz +cd glib-2.64.2 +# options taken from glib's oss-fuzz build definition +meson \ + --prefix=$PREFIX \ + --libdir=lib \ + --default-library=static \ + -Db_lundef=false \ + -Doss_fuzz=enabled \ + -Dlibmount=disabled \ + -Dinternal_pcre=true \ + _builddir +ninja -C _builddir +ninja -C _builddir install +cd .. + +# Note: We don't use/build orc since it still seems to be problematic +# with clang and the various sanitizers. + +# For now we only build core and base. Add other modules when/if needed +for i in gstreamer gst-plugins-base; +do + mkdir -p $i + cd $i + meson \ + --prefix=$PREFIX \ + --libdir=lib \ + --default-library=static \ + -Db_lundef=false \ + -Ddoc=disabled \ + -Dexamples=disabled \ + -Dintrospection=disabled \ + -Dtracer_hooks=false \ + -Dregistry=false _builddir $SRC/$i + ninja -C _builddir + ninja -C _builddir install + cd .. +done + + + +# 2) Build the target fuzzers + +# All targets will be linked in with $LIB_FUZZING_ENGINE which contains the +# actual fuzzing runner. Anything fuzzing engine can be used provided it calls +# the same function as libfuzzer. + +# Note: The fuzzer .o needs to be first compiled with CC and then linked with CXX + +# We want to statically link everything, except for shared libraries +# that are present on the base image. Those need to be specified +# beforehand and explicitely linked dynamically If any of the static +# dependencies require a pre-installed shared library, you need to add +# that library to the following list +PREDEPS_LDFLAGS="-Wl,-Bdynamic -ldl -lm -pthread -lrt -lpthread" + +# These are the basic .pc dependencies required to build any of the fuzzing targets +# That is : glib, gstreamer core and gst-app +# The extra target-specific dependencies are to be specified later +COMMON_DEPS="glib-2.0 gio-2.0 gstreamer-1.0 gstreamer-app-1.0" + +# For each target, defined the following: +# TARGET_DEPS : Extra .pc dependencies for the target (in addition to $COMMON_DEPS) +# All dependencies (including sub-dependencies) must be speecified +# PLUGINS : .a of the plugins to link +# They must match the static plugins declared/registered in the target + +# +# TARGET : push-based ogg/theora/vorbis discoverer +# +# FIXME : Rename to discoverer_push_oggtheoravorbis + +TARGET_DEPS=" gstreamer-pbutils-1.0 \ + gstreamer-video-1.0 \ + gstreamer-audio-1.0 \ + gstreamer-riff-1.0 \ + gstreamer-tag-1.0 \ + zlib ogg vorbis vorbisenc \ + theoraenc theoradec theora" + +PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \ + $PLUGIN_DIR/libgsttypefindfunctions.a \ + $PLUGIN_DIR/libgstplayback.a \ + $PLUGIN_DIR/libgstapp.a \ + $PLUGIN_DIR/libgstvorbis.a \ + $PLUGIN_DIR/libgsttheora.a \ + $PLUGIN_DIR/libgstogg.a" + +echo +echo ">>>> BUILDING gst-discoverer" +echo +BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`" +BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`" + +$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/gst-discoverer.c -o $SRC/gst-ci/fuzzing/gst-discoverer.o +$CXX $CXXFLAGS \ + -o $OUT/gst-discoverer \ + $PREDEPS_LDFLAGS \ + $SRC/gst-ci/fuzzing/gst-discoverer.o \ + $PLUGINS \ + $BUILD_LDFLAGS \ + $LIB_FUZZING_ENGINE \ + -Wl,-Bdynamic + +# +# TARGET : push-based typefind +# + +# typefindfunction depends on pbutils which depends on gst{audio|video|tag} +TARGET_DEPS=" gstreamer-pbutils-1.0 \ + gstreamer-video-1.0 \ + gstreamer-audio-1.0 \ + gstreamer-tag-1.0" + +PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \ + $PLUGIN_DIR/libgsttypefindfunctions.a \ + $PLUGIN_DIR/libgstapp.a" + +echo +echo ">>>> BUILDING typefind" +echo +BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`" +BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`" + +$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/typefind.c -o $SRC/gst-ci/fuzzing/typefind.o +$CXX $CXXFLAGS \ + -o $OUT/typefind \ + $PREDEPS_LDFLAGS \ + $SRC/gst-ci/fuzzing/typefind.o \ + $PLUGINS \ + $BUILD_LDFLAGS \ + $LIB_FUZZING_ENGINE \ + -Wl,-Bdynamic + +echo +echo ">>>> Installing seed corpus" +echo +# FIXME : Sadly we apparently need to have the corpus downloaded in the +# Dockerfile and not here. + +cp $SRC/*_seed_corpus.zip $OUT diff --git a/fuzzing/gst-discoverer.c b/fuzzing/gst-discoverer.c new file mode 100644 index 0000000000..0440606a4a --- /dev/null +++ b/fuzzing/gst-discoverer.c @@ -0,0 +1,137 @@ +/* + * Copyright 2016 Google Inc. + * author: Edward Hervey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#include +#include +#include + +#ifndef LOCAL_FUZZ_BUILD +GST_PLUGIN_STATIC_DECLARE (coreelements); +GST_PLUGIN_STATIC_DECLARE (playback); +GST_PLUGIN_STATIC_DECLARE (typefindfunctions); +GST_PLUGIN_STATIC_DECLARE (app); +GST_PLUGIN_STATIC_DECLARE (ogg); +GST_PLUGIN_STATIC_DECLARE (theora); +GST_PLUGIN_STATIC_DECLARE (vorbis); +#endif + +/* push-based discoverer fuzzing target + * + * This application can be compiled with libFuzzer to simulate + * a push-based discoverer execution. + * + * To reproduce the failing behaviour, use: + * $ gst-discoverer-1.0 pushfile:///... + * + * The goal is to cover basic usage of demuxers, parsers and + * base decoder elements. + * + * When compiling, only link the required demuxer/parser/decoder + * plugins and keep it to a limited range (ex: ogg/theora/vorbis) + * + **/ + +const guint8 *fuzztesting_data; +size_t fuzztesting_size; + +static void +appsrc_configuration (GstDiscoverer * dc, GstElement * source, gpointer data) +{ + GstBuffer *buf; + GstFlowReturn ret; + + /* Create buffer from fuzztesting_data which shouldn't be freed */ + buf = + gst_buffer_new_wrapped_full (0, (gpointer) fuzztesting_data, + fuzztesting_size, 0, fuzztesting_size, NULL, NULL); + g_object_set (G_OBJECT (source), "size", fuzztesting_size, NULL); + g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &ret); + gst_buffer_unref (buf); +} + +static void +custom_logger (const gchar * log_domain, + GLogLevelFlags log_level, const gchar * message, gpointer unused_data) +{ + if (log_level & G_LOG_LEVEL_CRITICAL) { + g_printerr ("CRITICAL ERROR : %s\n", message); + abort (); + } else if (log_level & G_LOG_LEVEL_WARNING) { + g_printerr ("WARNING : %s\n", message); + } +} + +int +LLVMFuzzerTestOneInput (const guint8 * data, size_t size) +{ + GError *err = NULL; + GstDiscoverer *dc; + gint timeout = 10; + GstDiscovererInfo *info; + static gboolean initialized = FALSE; + + if (!initialized) { + /* We want critical warnings to assert so we can fix them */ + g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL); + g_log_set_default_handler (custom_logger, NULL); + + /* Only initialize and register plugins once */ + gst_init (NULL, NULL); + +#ifndef LOCAL_FUZZ_BUILD + GST_PLUGIN_STATIC_REGISTER (coreelements); + GST_PLUGIN_STATIC_REGISTER (playback); + GST_PLUGIN_STATIC_REGISTER (typefindfunctions); + GST_PLUGIN_STATIC_REGISTER (app); + GST_PLUGIN_STATIC_REGISTER (ogg); + GST_PLUGIN_STATIC_REGISTER (theora); + GST_PLUGIN_STATIC_REGISTER (vorbis); +#endif + + initialized = TRUE; + } + + dc = gst_discoverer_new (timeout * GST_SECOND, &err); + if (G_UNLIKELY (dc == NULL)) { + g_print ("Error initializing: %s\n", err->message); + g_clear_error (&err); + exit (1); + } + + fuzztesting_data = data; + fuzztesting_size = size; + + /* Connect to source-setup signal to give the data */ + g_signal_connect (dc, "source-setup", (GCallback) appsrc_configuration, NULL); + + info = gst_discoverer_discover_uri (dc, "appsrc://", &err); + g_clear_error (&err); + if (info) + gst_discoverer_info_unref (info); + + g_object_unref (dc); + + return 0; +} diff --git a/fuzzing/localfuzzer.c b/fuzzing/localfuzzer.c new file mode 100644 index 0000000000..188d2461a8 --- /dev/null +++ b/fuzzing/localfuzzer.c @@ -0,0 +1,78 @@ +/* GStreamer + * Copyright (C) 2017 Edward Hervey + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/* Local fuzzer runner */ +#include + +extern int LLVMFuzzerTestOneInput (const guint8 * data, size_t size); + +static void +test_file (gchar * filename) +{ + GDir *dir; + gchar *path; + gchar *contents; + gsize length; + + /* if filename is a directory, process the contents */ + if ((dir = g_dir_open (filename, 0, NULL))) { + const gchar *entry; + + while ((entry = g_dir_read_name (dir))) { + gchar *spath; + + spath = g_strconcat (filename, G_DIR_SEPARATOR_S, entry, NULL); + test_file (spath); + g_free (spath); + } + + g_dir_close (dir); + return; + } + + /* Make sure path is absolute */ + if (!g_path_is_absolute (filename)) { + gchar *curdir; + + curdir = g_get_current_dir (); + path = g_build_filename (curdir, filename, NULL); + g_free (curdir); + } else + path = g_strdup (filename); + + /* Check if path exists */ + if (g_file_get_contents (path, &contents, &length, NULL)) { + g_print (">>> %s (%" G_GSIZE_FORMAT " bytes)\n", path, length); + LLVMFuzzerTestOneInput ((const guint8 *) contents, length); + g_free (contents); + } + + g_free (path); +} + +int +main (int argc, gchar ** argv) +{ + gint i; + + for (i = 1; i < argc; i++) + test_file (argv[i]); + + return 0; +} diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c new file mode 100644 index 0000000000..999ba337fc --- /dev/null +++ b/fuzzing/typefind.c @@ -0,0 +1,115 @@ +/* + * Copyright 2016 Google Inc. + * author: Edward Hervey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#include +#include + +#ifndef LOCAL_FUZZ_BUILD +GST_PLUGIN_STATIC_DECLARE (coreelements); +GST_PLUGIN_STATIC_DECLARE (typefindfunctions); +GST_PLUGIN_STATIC_DECLARE (app); +#endif + +/* push-based typefind fuzzing target + * + * This application can be compiled with libFuzzer to simulate + * a push-based typefind execution. + * + * To reproduce the failing behaviour, use: + * $ gst-launch-1.0 pushfile:///.. ! typefind ! fakesink + * + * The goal is to cover typefind code and implementation. + * + **/ +static void +custom_logger (const gchar * log_domain, + GLogLevelFlags log_level, const gchar * message, gpointer unused_data) +{ + if (log_level & G_LOG_LEVEL_CRITICAL) { + g_printerr ("CRITICAL ERROR : %s\n", message); + abort (); + } else if (log_level & G_LOG_LEVEL_WARNING) { + g_printerr ("WARNING : %s\n", message); + } +} + +int +LLVMFuzzerTestOneInput (const guint8 * data, size_t size) +{ + GError *err = NULL; + static gboolean initialized = FALSE; + GstElement *pipeline, *source, *typefind, *fakesink; + GstBuffer *buf; + GstFlowReturn flowret; + GstState state; + + if (!initialized) { + /* We want critical warnings to assert so we can fix them */ + g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL); + g_log_set_default_handler (custom_logger, NULL); + + /* Only initialize and register plugins once */ + gst_init (NULL, NULL); + +#ifndef LOCAL_FUZZ_BUILD + GST_PLUGIN_STATIC_REGISTER (coreelements); + GST_PLUGIN_STATIC_REGISTER (typefindfunctions); + GST_PLUGIN_STATIC_REGISTER (app); +#endif + + initialized = TRUE; + } + + /* Create the pipeline */ + pipeline = gst_pipeline_new ("pipeline"); + source = gst_element_factory_make ("appsrc", "source"); + typefind = gst_element_factory_make ("typefind", "typefind"); + fakesink = gst_element_factory_make ("fakesink", "fakesink"); + + gst_bin_add_many (GST_BIN (pipeline), source, typefind, fakesink, NULL); + gst_element_link_many (source, typefind, fakesink, NULL); + + /* Set pipeline to READY so we can provide data to appsrc */ + gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_READY); + buf = gst_buffer_new_wrapped_full (0, (gpointer) data, size, + 0, size, NULL, NULL); + g_object_set (G_OBJECT (source), "size", size, NULL); + g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &flowret); + gst_buffer_unref (buf); + + /* Set pipeline to PAUSED and wait (typefind will either fail or succeed) */ + gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_PAUSED); + + /* wait until state change either completes or fails */ + gst_element_get_state (GST_ELEMENT (pipeline), &state, NULL, -1); + + /* Go back to NULL */ + gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_NULL); + + /* And release the pipeline */ + gst_object_unref (pipeline); + + return 0; +} diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py new file mode 100755 index 0000000000..306fe1075e --- /dev/null +++ b/gitlab/build_manifest.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys +import subprocess +import urllib.error +import urllib.parse +import urllib.request +import json + +from typing import Dict, Tuple, List +# from pprint import pprint + +if sys.version_info < (3, 6): + raise SystemExit('Need Python 3.6 or newer') + +GSTREAMER_MODULES: List[str] = [ + 'orc', + 'cerbero', + 'gst-build', + 'gstreamer', + 'gst-plugins-base', + 'gst-plugins-good', + 'gst-plugins-bad', + 'gst-plugins-ugly', + 'gst-libav', + 'gst-devtools', + 'gst-docs', + 'gst-editing-services', + 'gst-omx', + 'gst-python', + 'gst-rtsp-server', + 'gstreamer-sharp', + 'gstreamer-vaapi', + 'gst-integration-testsuites', + 'gst-examples', +] + +MANIFEST_TEMPLATE: str = """ + + + +{} +""" + + +CERBERO_DEPS_LOGS_TARGETS = ( + ('cross-ios', 'universal'), + ('cross-windows-mingw', 'x86'), + ('cross-windows-mingw', 'x86_64'), + ('cross-android', 'universal'), + ('fedora', 'x86_64'), + ('macos', 'x86_64'), + ('windows-msvc', 'x86_64'), +) + +# Disallow git prompting for a username/password +os.environ['GIT_TERMINAL_PROMPT'] = '0' +def git(*args, repository_path='.'): + return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() + +def get_cerbero_last_build_info (branch : str): + # Fetch the deps log for all (distro, arch) targets + all_commits = {} + for distro, arch in CERBERO_DEPS_LOGS_TARGETS: + url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/{distro}/{arch}/cerbero-deps.log' + print(f'Fetching {url}') + try: + req = urllib.request.Request(url) + resp = urllib.request.urlopen(req); + deps = json.loads(resp.read()) + except urllib.error.URLError as e: + print(f'WARNING: Failed to GET {url}: {e!s}') + continue + + for dep in deps: + commit = dep['commit'] + if commit not in all_commits: + all_commits[commit] = [] + all_commits[commit].append((distro, arch)) + + # Fetch the cerbero commit that has the most number of caches + best_commit = None + newest_commit = None + max_caches = 0 + total_caches = len(CERBERO_DEPS_LOGS_TARGETS) + for commit, targets in all_commits.items(): + if newest_commit is None: + newest_commit = commit + have_caches = len(targets) + # If this commit has caches for all targets, just use it + if have_caches == total_caches: + best_commit = commit + break + # Else, try to find the commit with the most caches + if have_caches > max_caches: + max_caches = have_caches + best_commit = commit + if newest_commit is None: + print('WARNING: No deps logs were found, will build from scratch') + if best_commit != newest_commit: + print(f'WARNING: Cache is not up-to-date for commit {newest_commit}, using commit {best_commit} instead') + return best_commit + + +def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]: + try: + res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch) + except subprocess.CalledProcessError: + return None, None + + if not res: + return None, None + + # Special case cerbero to avoid cache misses + if module == 'cerbero': + sha = get_cerbero_last_build_info(branch) + if sha is not None: + return sha, sha + + lines = res.split('\n') + for line in lines: + if line.endswith('/' + branch): + try: + sha, refname = line.split('\t') + except ValueError: + continue + return refname.strip(), sha + + return None, None + + +def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]: + namespace: str = os.environ["CI_PROJECT_NAMESPACE"] + ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master') + + if module == "orc": + ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master') + + if module == os.environ['CI_PROJECT_NAME']: + return 'user', branchname, os.environ['CI_COMMIT_SHA'] + + if branchname != ups_branch: + remote_refname, sha = get_branch_info(module, namespace, branchname) + if sha is not None: + return 'user', remote_refname, sha + + # Check upstream project for a branch + remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch) + if sha is not None: + return 'origin', remote_refname, sha + + # This should never occur given the upstream fallback above + print(f"Could not find anything for {module}:{branchname}") + print("If something reaches that point, please file a bug") + print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues") + assert False + + +# --- Unit tests --- # +# Basically, pytest will happily let a test mutate a variable, and then run +# the next tests one the same environment without reset the vars. +def preserve_ci_vars(func): + """Preserve the original CI Variable values""" + def wrapper(): + try: + url = os.environ["CI_PROJECT_URL"] + user = os.environ["CI_PROJECT_NAMESPACE"] + except KeyError: + url = "invalid" + user = "" + + private = os.getenv("READ_PROJECTS_TOKEN", default=None) + if not private: + os.environ["READ_PROJECTS_TOKEN"] = "FOO" + + func() + + os.environ["CI_PROJECT_URL"] = url + os.environ["CI_PROJECT_NAMESPACE"] = user + + if private: + os.environ["READ_PROJECTS_TOKEN"] = private + # if it was set after + elif os.getenv("READ_PROJECTS_TOKEN", default=None): + del os.environ["READ_PROJECTS_TOKEN"] + + return wrapper + +@preserve_ci_vars +def test_find_repository_sha(): + os.environ["CI_PROJECT_NAME"] = "some-random-project" + os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" + os.environ["CI_PROJECT_NAMESPACE"] = "alatiera" + os.environ["GST_UPSTREAM_BRANCH"] = "master" + del os.environ["READ_PROJECTS_TOKEN"] + + # This should find the repository in the user namespace + remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2") + assert remote == "user" + assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325" + assert refname == "refs/heads/1.2" + + # This should fallback to upstream master branch since no matching branch was found + remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name") + assert remote == "origin" + assert refname == "refs/heads/master" + + os.environ["CI_PROJECT_NAME"] = "the_project" + os.environ["CI_COMMIT_SHA"] = "MySha" + + remote, refname, git_ref = find_repository_sha("the_project", "whatever") + assert remote == "user" + assert git_ref == "MySha" + assert refname == "whatever" + + +@preserve_ci_vars +def test_get_project_branch(): + os.environ["CI_PROJECT_NAME"] = "some-random-project" + os.environ["CI_COMMIT_SHA"] = "dwbuiw" + os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" + os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_" + del os.environ["READ_PROJECTS_TOKEN"] + + os.environ['GST_UPSTREAM_BRANCH'] = '1.12' + remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12') + assert twelve is not None + assert remote == 'origin' + assert refname == "refs/heads/1.12" + + os.environ['GST_UPSTREAM_BRANCH'] = '1.14' + remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14') + assert fourteen is not None + assert remote == 'origin' + assert refname == "refs/heads/1.14" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--self-update", action="store_true", default=False) + parser.add_argument(dest="output", default='manifest.xml', nargs='?') + options = parser.parse_args() + + current_branch: str = os.environ['CI_COMMIT_REF_NAME'] + user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL']) + if not user_remote_url.endswith('/'): + user_remote_url += '/' + + if options.self_update: + remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch) + if remote == 'user': + remote = user_remote_url + 'gst-ci' + else: + remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci" + + git('fetch', remote, remote_refname) + git('checkout', '--detach', sha) + sys.exit(0) + + projects: str = '' + for module in GSTREAMER_MODULES: + print(f"Checking {module}:", end=' ') + remote, refname, revision = find_repository_sha(module, current_branch) + print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'") + projects += f" \n" + + with open(options.output, mode='w') as manifest: + print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh new file mode 100644 index 0000000000..96fc7390df --- /dev/null +++ b/gitlab/cerbero_setup.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +set -ex + +show_ccache_sum() { + if [[ -n ${HAVE_CCACHE} ]]; then + ccache -s + fi +} + +# XXX: This is copied and modified from the cerbero-uninstalled script +# Use `mount` to get a list of MSYS mount points that the MSYS shell uses. +# That's our reference point for translating from MSYS paths to Win32 paths. +# We assume that the MSYS mount point directories are only in the filesystem +# root. This will break if people add their own custom mount points beyond what +# MSYS automatically creates, which is highly unlikely. +# +# /d -> d:/ +# /c -> c:/ +# /d/projects/cerbero -> d:/projects/cerbero/ +# /home/USERNAME/cerbero -> C:\\MinGW\\msys\\1.0/home/USERNAME/ +# /mingw -> C:\\MinGW/ +# /mingw/bin/foobar -> C:\\MinGW\\bin/foobar/ +# /tmp/baz -> C:\\Users\\USERNAME\\AppData\\Local\\Temp/baz/ +msys_dir_to_win32() { + set -e + local msys_path stripped_path mount_point path mounted_path + # If the path is already a native path, just return that + if [[ $1 == ?:/* ]] || [[ $1 == ?:\\* ]]; then + echo $1 + return + fi + # Convert /c or /mingw etc to /c/ or /mingw/ etc; gives us a necessary + # anchor to split the path into components + msys_path="$1/" + # Strip leading slash + stripped_path="${msys_path#/}" + # Get the first path component, which may be a mount point + mount_point="/${stripped_path%%/*}" + # Get the path inside the mountp oint + path="/${stripped_path#*/}" + mounted_path="$(mount | sed -n "s|\(.*\) on $mount_point type.*|\1|p")" + # If it's not a mounted path (like /c or /tmp or /mingw), then it's in the + # general MSYS root mount + if [[ -z $mounted_path ]]; then + mounted_path="$(mount | sed -n "s|\(.*\) on / type.*|\1|p")" + path="$1" + fi + echo ${mounted_path}${path%/} +} + +# Print the working directory in the native OS path format, but with forward +# slashes +pwd_native() { + if [[ -n "$MSYSTEM" ]]; then + msys_dir_to_win32 "$(pwd)" + else + pwd + fi +} + +fix_build_tools() { + if [[ $(uname) == Darwin ]]; then + # Bison needs these env vars set for the build-tools prefix to be + # relocatable, and we only build it on macOS. On Linux we install it + # using the package manager, and on Windows we use the MSYS Bison. + export M4="$(pwd)/${CERBERO_HOME}/build-tools/bin/m4" + export BISON_PKGDATADIR="$(pwd)/${CERBERO_HOME}/build-tools/share/bison" + fi +} + +# Produces runtime and devel tarball packages for linux/android or .pkg for macos +cerbero_package_and_check() { + $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0 + + # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. + if [[ $CONFIG != *universal* ]]; then + $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version + $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX + fi + + show_ccache_sum +} + +cerbero_before_script() { + pwd + ls -lha + + # Copy cerbero git repo stored on the image + cp -a "${CERBERO_HOST_DIR}/.git" . + git checkout . + git status + + # If there's no cerbero-sources directory in the runner cache, copy it from + # the image cache + if ! [[ -d ${CERBERO_SOURCES} ]]; then + time cp -a "${CERBERO_HOST_DIR}/${CERBERO_SOURCES}" . + fi + du -sch "${CERBERO_SOURCES}" || true + + echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc + echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc + if [[ $CONFIG == win??.cbc ]]; then + # Visual Studio 2017 build tools install path + echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc + echo 'vs_install_version = "vs15"' >> localconf.cbc + fi + cat localconf.cbc + + time ./cerbero-uninstalled --self-update manifest.xml + + # GitLab runner does not always wipe the image after each job, so do that + # to ensure we don't have any leftover data from a previous job such as + # a dirty builddir, or tarballs/pkg files, leftover files from an old + # cerbero commit, etc. Skip the things we actually need to keep. + time git clean -xdff -e cerbero_setup.sh -e manifest.xml -e localconf.cbc -e "${CERBERO_SOURCES}" +} + +cerbero_script() { + show_ccache_sum + + $CERBERO $CERBERO_ARGS show-config + $CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4 + $CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0 + du -sch "${CERBERO_SOURCES}" || true + + $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}" + + if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} && -d "${CERBERO_HOME}/dist/${ARCH}" ]]; then + mkdir -p "${CERBERO_OVERRIDDEN_DIST_DIR}" + time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}" + fi + + $CERBERO $CERBERO_ARGS bootstrap --offline --system=no + fix_build_tools + + cerbero_package_and_check +} + +cerbero_deps_script() { + show_ccache_sum + + $CERBERO $CERBERO_ARGS show-config + $CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4 + $CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0 + $CERBERO $CERBERO_ARGS bootstrap --offline --system=no + $CERBERO $CERBERO_ARGS build-deps --offline \ + gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \ + gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \ + gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice + + if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then + mkdir -p "${CERBERO_HOME}/dist/${ARCH}" + time rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}" + fi + + # Check that the env var is set. Don't expand this protected variable by + # doing something silly like [[ -n ${CERBERO_...} ]] because it will get + # printed in the CI logs due to set -x + if env | grep -q -e CERBERO_PRIVATE_SSH_KEY; then + time $CERBERO $CERBERO_ARGS gen-cache --branch "${GST_UPSTREAM_BRANCH}" + time $CERBERO $CERBERO_ARGS upload-cache --branch "${GST_UPSTREAM_BRANCH}" + fi + + cerbero_package_and_check +} + +# Run whichever function is asked of us +eval "$1" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml new file mode 100644 index 0000000000..cb8ae521d3 --- /dev/null +++ b/gitlab/ci_template.yml @@ -0,0 +1,1427 @@ +include: + - remote: "https://gitlab.freedesktop.org/freedesktop/ci-templates/-/raw/7ea696055e322cc7aa4bcbe5422b56a198c4bdff/templates/ci-fairy.yml" + +stages: + - 'build docker' + - 'preparation' + - 'pre-build' + - 'build' + - 'test' + # Use the resulting binaries + - 'integrate' + +variables: + # Branch to track for modules that have no ref specified in the manifest + GST_UPSTREAM_BRANCH: 'master' + ORC_UPSTREAM_BRANCH: 'master' + + ### + # IMPORTANT + # These are the version tags for the docker images the CI runs against. + # If you are hacking on them or need a them to rebuild, its enough + # to change any part of the string of the image you want. + ### + FEDORA_TAG: '2021-06-30.0' + CERBERO_TAG: '2020-10-22.0' + ANDROID_TAG: '2020-10-22.0' + MANIFEST_TAG: '2020-10-22.0' + TEST_MANIFEST_TAG: '2020-10-22.0' + INDENT_TAG: '2020-10-22.0' + WINDOWS_TAG: "2021-09-09.0" + + GST_UPSTREAM_REPO: 'gstreamer/gst-ci' + + FEDORA_AMD64_SUFFIX: 'amd64/fedora' + CERBERO_AMD64_SUFFIX: 'amd64/cerbero-fedora' + ANDROID_AMD64_SUFFIX: 'amd64/android-fedora' + MANIFEST_AMD64_SUFFIX: 'amd64/build-manifest' + TEST_MANIFEST_AMD64_SUFFIX: 'amd64/test-manifest' + INDENT_AMD64_SUFFIX: 'amd64/gst-indent' + WINDOWS_AMD64_SUFFIX: 'amd64/windows' + WINDOWS_RUST_AMD64_SUFFIX: 'amd64/windows-rust' + + FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" + FEDORA_DOCS_IMAGE: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-07-03.0-master" + CERBERO_IMAGE: "$CI_REGISTRY_IMAGE/$CERBERO_AMD64_SUFFIX:$CERBERO_TAG-$GST_UPSTREAM_BRANCH" + ANDROID_IMAGE: "$CI_REGISTRY_IMAGE/$ANDROID_AMD64_SUFFIX:$ANDROID_TAG-$GST_UPSTREAM_BRANCH" + MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$MANIFEST_AMD64_SUFFIX:$MANIFEST_TAG-$GST_UPSTREAM_BRANCH" + TEST_MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$TEST_MANIFEST_AMD64_SUFFIX:$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" + INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" + WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + + RUST_MINIMUM_VERSION: '1.54.0' + RUST_LATEST_VERSION: '1.55.0' + + WINDOWS_RUST_MINIMUM_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" + WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" + + WINDOWS_RUST_LATEST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_LATEST_VERSION" + WINDOWS_RUST_LATEST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_LATEST_VERSION" + + # Current windows image doesn't seem to be able to spawn procceses for reason, + # use a different image until this is debuged + # Running command 'c:/builds/alatiera/gst-ci/cerbero-build/build-tools/bin/meson --prefix=c:/builds/alatiera/gst-ci/cerbero-build/dist/msvc_x86_64 --libdir=lib -Ddebug=true --default-library=both -Doptimization=2 --backend=ninja --wrap-mode=nodownload -Db_vscrt=md --native-file c:/builds/alatiera/gst-ci/cerbero-build/sources/msvc_x86_64/gstreamer-1.0-1.19.0.1/_builddir/meson-native-file.txt -Dlibunwind=disabled -Ddbghelp=enabled -Dintrospection=disabled -Dexamples=disabled' + # failed to create process + CERBERO_WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v13-master' + + # Can't use $CI_* variables since we use this template in all repos, not just gst-ci + CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" + WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" + + GIT_STRATEGY: none + MESON_BUILDTYPE_ARGS: --default-library=both + DEFAULT_MESON_ARGS: > + -Dlibnice:tests=disabled + -Dlibnice:examples=disabled + -Dopenh264:tests=disabled + -Dpygobject:tests=false + -Dpython=enabled + -Dlibav=enabled + -Dugly=enabled + -Dbad=enabled + -Ddevtools=enabled + -Dges=enabled + -Drtsp_server=enabled + -Dvaapi=enabled + -Dsharp=disabled + + MESON_GST_WERROR: > + -Dgstreamer:werror=true + -Dgst-plugins-base:werror=true + -Dgst-plugins-good:werror=true + -Dgst-plugins-ugly:werror=true + -Dgst-plugins-bad:werror=true + -Dgst-rtsp-server:werror=true + -Dgst-libav:werror=true + -Dgst-examples:werror=true + -Dgst-editing-services:werror=true + -Dgst-docs:werror=true + -Dgst-omx:werror=true + -Dgst-devtools:werror=true + -Dgst-python:werror=true + -Dgstreamer-vaapi:werror=true + -Dgstreamer-sharp:werror=true + + DEFAULT_CERBERO_ARGS: > + --variants werror + --timestamps + +workflow: + rules: + - if: $CI_MERGE_REQUEST_IID + # don't create a pipeline if its a commit pipeline, on a branch and that branch has + # open merge requests (bc we will get a MR build instead) + - if: $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH + +# +# Global CI policy +# +# This can be used to configure global behaviour our our jobs. +# +default: + retry: + max: 2 + when: + - 'runner_system_failure' + - 'stuck_or_timeout_failure' + - 'scheduler_failure' + - 'api_failure' + interruptible: true + + +# Script to check if a docker image exists in the upstream registry +# and if so copy it to the forked registry so we can use it +# +# This is copied/adapted from citemplates/templates/fedora.yml +# https://gitlab.freedesktop.org/freedesktop/ci-templates/-/blob/96b621fe9f57ec2464f8d1a0940446afbf6c8f59/templates/fedora.yml +.check_image: &check_image_exists | + # if-not-exists steps + set -x + + if [[ -z "$GST_FORCE_REBUILD" ]] + then + # disable exit on failure + set +e + + # check if our image is already in the current registry + # we store the sha of the digest and the layers + skopeo inspect docker://$LOCAL_IMAGE | jq '[.Digest, .Layers]' > local_sha + + # check if our image is already in our gst-ci registry fork + skopeo inspect docker://$LOCAL_GST_CI_IMAGE | jq '[.Digest, .Layers]' > local_gst_ci_sha + + # check if our image is already in the upstream registry + if [[ -z "$GST_UPSTREAM_REPO" ]] + then + echo "WARNING! Variable \$GST_UPSTREAM_REPO is undefined, cannot check for images" + else + skopeo inspect docker://$GST_UPSTREAM_IMAGE | jq '[.Digest, .Layers]' > upstream_sha + fi + + # reenable exit on failure + set -e + + # if the upstream repo has an image, ensure we use the same + if [ -s upstream_sha ] + then + + echo "Checking if $LOCAL_IMAGE is up to date" + # ensure we use the same image from upstream + diff upstream_sha local_sha && exit 0 || true + + echo "Copying image from gstreamer/gst-ci to local registry" + # copy the original image into the current project registry namespace + # we do 2 attempts with skopeo copy at most + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$GST_UPSTREAM_IMAGE \ + docker://$LOCAL_IMAGE || \ + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$GST_UPSTREAM_IMAGE \ + docker://$LOCAL_IMAGE + + exit 0 + fi + + set +x + + # if the local ci fork repo has an image, ensure we use the same + if [ -s local_gst_ci_sha ] + then + echo "Checking if $LOCAL_GST_CI_IMAGE is up to date" + # ensure we use the same image from upstream + diff local_gst_ci_sha local_sha && exit 0 || true + + echo "Copying image from $CI_PROJECT_NAMESPACE/gst-ci to local registry" + # copy the original image into the current project registry namespace + # we do 2 attempts with skopeo copy at most + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$LOCAL_GST_CI_IMAGE \ + docker://$LOCAL_IMAGE || \ + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$LOCAL_GST_CI_IMAGE \ + docker://$LOCAL_IMAGE + + exit 0 + fi + + # if we have a local image but none in the upstream repo, use our + if [ -s local_sha ] + then + echo "Using $LOCAL_IMAGE" + exit 0 + fi + fi + + +# Build docker images from Dockerfiles +# +# This is copied/adapted from citemplates/templates/fedora.yml +# https://gitlab.freedesktop.org/freedesktop/ci-templates/-/blob/96b621fe9f57ec2464f8d1a0940446afbf6c8f59/templates/fedora.yml +# +# CITemplates builds uses buildah run/commit workflow to build the images which doesn't +# fit us atm and our images need further adjustment to make use of it. Porting to +# ci-templates is the next step though. +# +# All the documentation from citemplates should be applicable here, so please refer there +.base: + image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-07-20.1" + variables: + STORAGE_DRIVER: 'vfs' + BUILDAH_FORMAT: 'docker' + BUILDAH_ISOLATION: 'chroot' + + GIT_STRATEGY: fetch + script: + - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$REPO_SUFFIX:$TAG" + - export LOCAL_GST_CI_IMAGE="$CI_REGISTRY/$CI_PROJECT_NAMESPACE/gst-ci/$REPO_SUFFIX:$TAG" + - export GST_UPSTREAM_IMAGE="$CI_REGISTRY/$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG" + + # check if the docker registry is enabled, else the variable will be missing + - | + if [[ -z "$CI_REGISTRY_IMAGE" ]] + then + echo "ERROR! Looks like your repository/fork has disabled Docker Registries." + echo "Pleae enable them by following the documentation from here:" + echo "https://docs.gitlab.com/ee/user/packages/container_registry/#enable-the-container-registry-for-your-project" + exit 1 + fi + + # Newer versions of podman/buildah try to set overlayfs mount options when + # using the vfs driver, and this causes errors. + - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf + + - *check_image_exists + + # This part of the job should only ever be possible to be reach in a gst-ci fork + - | + if [[ "$CI_PROJECT_NAME" != 'gst-ci' ]] + then + echo "ERROR! Something is wrong!" + echo "This part of the job is supposed to be executed only in gst-ci forks, and its probably going to fail else" + exit 1 + fi + + - echo "Building image $LOCAL_IMAGE" + + - > + buildah bud + --build-arg DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH + --label ci.job_id=$CI_JOB_ID + --label pipeline.url=$CI_PIPELINE_URL + --label git.ref_name=$CI_COMMIT_REF_NAME + --label git.sha=$CI_COMMIT_SHA + --label gitlab.project_path=$CI_PROJECT_PATH + --label gitlab.project_url=$CI_PROJECT_URL + --label fdo.upstream-repo=$GST_UPSTREAM_REPO + -f $DOCKERFILE + -t $LOCAL_IMAGE + $CONTEXT_DIR + + - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + + # Sanity check + - | + if [ "$CI_PROJECT_PATH" = "gstreamer/gst-ci" ]; then + if [ "$LOCAL_IMAGE" != "$GST_UPSTREAM_IMAGE" ]; then + echo "ERROR!!! AAAAA! THE IMAGE URI DOES NOT MATCH THE EXPECTED UPSTREAM ONE" + echo $LOCAL_IMAGE + echo $GST_UPSTREAM_IMAGE + exit 1 + fi + fi + + - buildah push $LOCAL_IMAGE + +fedora amd64 docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$FEDORA_AMD64_SUFFIX" + TAG: "$FEDORA_TAG-$GST_UPSTREAM_BRANCH" + + CONTEXT_DIR: "./docker/fedora/" + DOCKERFILE: "docker/fedora/Dockerfile" + extends: .base + +cerbero fedora amd64 docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$CERBERO_AMD64_SUFFIX" + TAG: "$CERBERO_TAG-$GST_UPSTREAM_BRANCH" + + CONTEXT_DIR: "docker/cerbero/" + DOCKERFILE: "docker/cerbero/Dockerfile-fedora" + extends: .base + +android docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$ANDROID_AMD64_SUFFIX" + TAG: "$ANDROID_TAG-$GST_UPSTREAM_BRANCH" + + CONTEXT_DIR: "docker/android/" + DOCKERFILE: "docker/android/Dockerfile" + extends: .base + +alpine amd64 manifest builder docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$MANIFEST_AMD64_SUFFIX" + TAG: '$MANIFEST_TAG-$GST_UPSTREAM_BRANCH' + + CONTEXT_DIR: "docker/build_manifest/" + DOCKERFILE: "docker/build_manifest/Dockerfile" + extends: .base + +gst-indent amd64 docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$INDENT_AMD64_SUFFIX" + TAG: "$INDENT_TAG-$GST_UPSTREAM_BRANCH" + + CONTEXT_DIR: "docker/indent/" + DOCKERFILE: "docker/indent/Dockerfile" + extends: .base + +windows amd64 docker: + stage: "build docker" + variables: + GIT_STRATEGY: fetch + + # Unlike the buildah/linux jobs, this file + # needs to be relative to docker/windows/ subdir + # as it makes life easier in the powershell script + # + # We also don't need a CONTEXT_DIR var as its also + # hardcoded to be docker/windows/ + DOCKERFILE: "docker/windows/Dockerfile" + tags: + - windows + - shell + - "1809" + script: + # We need to pass an array and to resolve the env vars, so we can't use a variable: + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH") + + - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" + - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" + - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH + + - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_IMAGE $WINDOWS_UPSTREAM_IMAGE $DOCKERFILE" + - | + if (!($?)) { + echo "Failed to build the image" + Exit 1 + } + +.windows rust docker build: + stage: 'build docker' + needs: + - job: 'windows amd64 docker' + artifacts: false + rules: + - if: '$CI_PROJECT_NAME == "gst-ci"' + variables: + GIT_STRATEGY: 'fetch' + + # Unlike the buildah/linux jobs, this file + # needs to be relative to docker/windows/ subdir + # as it makes life easier in the powershell script + # + # We also don't need a CONTEXT_DIR var as its also + # hardcoded to be docker/windows/ + DOCKERFILE: 'docker/windows/rust.Dockerfile' + tags: + - 'windows' + - 'shell' + - '1809' + script: + # We need to pass an array and to resolve the env vars, so we can't use a variable: + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE", "--build-arg", "RUST_VERSION=$RUST_VERSION") + + - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" + - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" + - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH + + - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $RUST_IMAGE $RUST_UPSTREAM_IMAGE $DOCKERFILE" + - | + if (!($?)) { + echo "Failed to build the image" + Exit 1 + } + +windows rust amd64 docker latest stable: + extends: '.windows rust docker build' + variables: + RUST_IMAGE: !reference [variables, "WINDOWS_RUST_LATEST_IMAGE"] + RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_LATEST_UPSTREAM_IMAGE"] + RUST_VERSION: !reference [variables, "RUST_LATEST_VERSION"] + +windows rust amd64 docker minimum supported version: + extends: '.windows rust docker build' + variables: + RUST_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_IMAGE"] + RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE"] + RUST_VERSION: !reference [variables, "RUST_MINIMUM_VERSION"] + +# +# Job to create the manifest.xml to used for our builds +# +manifest: + image: $MANIFEST_IMAGE + rules: + # Always have automatic pipeline for branchs in cerbero and docs + # cause they need to update artifacts, like the docs site or cerbero deps + - if: '$CI_PROJECT_PATH == "gstreamer/cerbero"' + - if: '$CI_PROJECT_PATH == "gstreamer/gst-docs"' + + # If the MR is assigned to the Merge bot, trigger the pipeline automatically + - if: '$CI_MERGE_REQUEST_ASSIGNEES == "gstreamer-merge-bot"' + # When the assignee isn't the merge bot, require an explicit action to trigger the pipeline + # to avoid wasting CI resources + - if: '$CI_MERGE_REQUEST_ASSIGNEES != "gstreamer-merge-bot"' + when: 'manual' + allow_failure: false + # If this matches, it means the pipeline is running against either the main + # or a stable branch, so make it manual + - if: '$CI_COMMIT_BRANCH == $GST_UPSTREAM_BRANCH' + when: 'manual' + allow_failure: false + + stage: 'preparation' + script: + - cd /gst-ci + - gitlab/build_manifest.py --self-update + - gitlab/build_manifest.py ${CI_PROJECT_DIR}/manifest.xml + - cat ${CI_PROJECT_DIR}/manifest.xml + artifacts: + expire_in: "7 days" + paths: + - "manifest.xml" + +# +# gst-indent!! +# +gst indent: + image: $INDENT_IMAGE + stage: 'preparation' + variables: + GIT_STRATEGY: 'fetch' + rules: + - if: '$CI_PROJECT_NAME !~ /^(gstreamer-sharp|gst-integration-testsuites|cerbero|gst-docs)$/' + script: + # man indent. grep RETURN VALUE, grab a beer on my behalf... + - indent --version || true + - | + filter_cmd=("cat") + if test -f ".indentignore"; then + filter_args=() + while read -r line; do + if test -n "$line"; then + filter_args+=("-e" "$line") + fi + done < ".indentignore" + if [[ ${#filter_args[@]} -gt 0 ]]; then + filter_cmd=("grep" "-v" "${filter_args[@]}") + fi + fi + find . -name '*.c' | "${filter_cmd[@]}" | xargs -d '\n' gst-indent + - | + if git diff --quiet; then + echo "Code is properly formatted" + else + git diff --color=always + echo 'style diverges, please run gst-indent first' + exit 1 + fi + +# +# Check "allow-edit" checkbox on merge requests with ci-fairy +# +check allow-edit: + extends: '.fdo.ci-fairy' + needs: [] + stage: 'preparation' + variables: + GIT_STRATEGY: 'none' + script: + - ci-fairy check-merge-request --require-allow-collaboration + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + +# +# gst-build setup templates +# +.gst_build_template: &gst_build + - echo $MESON_ARGS + # Sometimes, gitlab-runner want to reuse + # existing docker volumes without cleaning them up... + # Make sure the docker volume is clean + - rm -rf gst-build || true + - rm -rf meson-logs || true + - rm -rf validate-logs || true + + - curl -L -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" + - python3 clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build + - cd gst-build/ + # only copy immediate directories. We don't want to modify .wrap files + - find /gst-build/subprojects/ -mindepth 1 -maxdepth 1 -type d -exec cp -r \{\} subprojects/ \; + + # Reset the state of the subprojects to what gst-build expects + - meson subprojects update --reset + - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" + + - meson build/ $MESON_ARGS + - ninja -C build/ + - ccache --show-stats + +.gst_build_ccache_vars: + variables: + CCACHE_COMPILERCHECK: 'content' + CCACHE_COMPRESS: 'true' + CCACHE_BASEDIR: '/cache/gstreamer/gst-build' + CCACHE_DIR: '/cache/gstreamer/gst-build/ccache/' + # shared across everything really + CCACHE_MAXSIZE: '10G' + CARGO_HOME: '/cache/gstreamer/cargo' + +.simple_fedora_build: &simple_build >- + ${DEFAULT_MESON_ARGS} + -Dsharp=enabled + -Domx=enabled + -Dgst-omx:target=generic + -Ddoc=disabled + -Drs=disabled + ${MESON_BUILDTYPE_ARGS} + ${MESON_GST_WERROR} + +.build: + stage: 'build' + extends: + - '.gst_build_ccache_vars' + needs: + - "manifest" + # Taking into account the slowest shared runner + time needed to upload the binaries to artifacts + # Also need to take into account I/O of pulling docker images and uploading artifacts + timeout: '45min' + variables: + MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} ${MESON_GST_WERROR}" + rules: + - if: '$CI_PROJECT_NAME != "cerbero"' + script: + *gst_build + after_script: + - mv gst-build/build/meson-logs/ meson-logs + # Cleanup everything else to reduce the size + # of the docker volume we leave behind + - rm -rf gst-build + artifacts: + expire_in: "7 days" + when: "always" + paths: + - 'meson-logs/' + - 'manifest.xml' + +.build fedora x86_64: + extends: '.build' + image: $FEDORA_IMAGE + variables: + MESON_ARGS: *simple_build + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-omx|gst-plugins-rs)$/' + +build nodebug fedora x86_64: + extends: '.build' + image: $FEDORA_IMAGE + variables: + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS} ${MESON_GST_WERROR}" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + +build static fedora x86_64: + extends: '.build fedora x86_64' + variables: + MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + +build static nodebug fedora x86_64: + extends: 'build nodebug fedora x86_64' + variables: + MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + +build clang fedora x86_64: + extends: '.build fedora x86_64' + variables: + CC: 'ccache clang' + CXX: 'ccache clang++' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + +.test: + stage: 'test' + extends: + - '.gst_build_ccache_vars' + needs: + - "manifest" + variables: + MESON_ARGS: *simple_build + + # Disable colored output to avoid weird rendering issues + GST_DEBUG_NO_COLOR: "true" + CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" + GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" + TIMEOUT_FACTOR: "2" + CARGO_HOME: "/cache/gstreamer/cargo" + # Enable the fault handler so we get backtraces on segfaults. + # any non-empty string will do + PYTHONFAULTHANDLER: "enabled" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-examples|gst-docs)$/' + script: + - *gst_build + + - echo "-> Running ${TEST_SUITE}" + - > + ./gst-uninstalled.py + gst-validate-launcher ${TEST_SUITE} + --dump-on-failure + --mute + --shuffle + --no-display + --meson-no-rebuild + --timeout-factor "${TIMEOUT_FACTOR}" + --fail-on-testlist-change + -l "${CI_PROJECT_DIR}/validate-logs/" + --xunit-file "${CI_PROJECT_DIR}/validate-logs/xunit.xml" + ${EXTRA_VALIDATE_ARGS} + after_script: + - mv gst-build/build/meson-logs/ meson-logs + # Cleanup everything else to reduce the size + # of the docker volume we leave behind + - rm -rf gst-build + artifacts: + expire_in: '14 days' + when: always + paths: + - 'meson-logs/' + - 'validate-logs' + - 'manifest.xml' + reports: + junit: + - "validate-logs/*.xml" + +.test fedora x86_64: + image: $FEDORA_IMAGE + extends: '.test' + tags: ['gstreamer'] + +check fedora: + extends: '.test fedora x86_64' + variables: + TEST_SUITE: "check.gst*" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + +integration testsuites fedora: + extends: '.test fedora x86_64' + parallel: 4 + variables: + EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs --parts=${CI_NODE_TOTAL} --part-index=${CI_NODE_INDEX}" + TEST_SUITE: "validate ges" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-omx|gstreamer-sharp|gst-plugins-rs)$/' + +# gstreamer-full: +# introspection has been disabled as the static build does not support it. +# See https://gitlab.freedesktop.org/gstreamer/gst-build/-/issues/162 +gstreamer-full: + extends: 'build static fedora x86_64' + stage: integrate + variables: + MESON_ARGS: > + --default-library=static + -Dintrospection=disabled + $MESON_GST_WERROR + rules: + - if: '$CI_PROJECT_NAME =~ /^(gst-build|gst-ci)$/' + + script: + - *gst_build + - meson test -C build -v test-gst-full + artifacts: + expire_in: "7 days" + when: "always" + paths: + - 'meson-logs/' + +gstreamer-full-minimal: + extends: 'build static fedora x86_64' + stage: integrate + variables: + MESON_ARGS: > + --default-library=static + -Dauto_features=disabled + -Dgstreamer:check=enabled + -Dtests=enabled + -Dgst-plugins-base:alsa=enabled + -Dgst-plugins-base:typefind=enabled + -Dgst-plugins-base:pbtypes=enabled + -Dgst-full-elements=coreelements:filesrc,fakesink,identity,input-selector + -Dgst-full-typefind-functions=typefindfunctions:wav,flv + -Dgst-full-device-providers=alsa:alsadeviceprovider + -Dgst-full-dynamic-types=pbtypes:video_multiview_flagset + $MESON_GST_WERROR + rules: + - if: '$CI_PROJECT_NAME =~ /^(gst-build|gst-ci)$/' + + script: + - *gst_build + - meson test -C build -v test-gst-full + - meson test -C build test-gst-full-features --test-args "-e filesrc,identity,fakesink -E filesink,capsfilter -t audio/x-wav -T video/vivo -d alsadeviceprovider -D v4l2deviceprovider -l GstVideoMultiviewFlagsSet" + - strip build/libgstreamer-full-1.0.so + - ls -l build/libgstreamer-full-1.0.so + artifacts: + expire_in: "7 days" + when: "always" + paths: + - 'meson-logs/' + +# Valgrind +.valgrind fedora x86_64: + extends: '.test fedora x86_64' + stage: 'test' + variables: + EXTRA_VALIDATE_ARGS: "--valgrind" + +valgrind core: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gstreamer\\..*" + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-ci|gst-devtools)$/' + +valgrind base: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-plugins-base\\..*" + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-ci|gst-devtools)$/' + +valgrind good: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-plugins-good\\..*" + # take longer time due to splitmux unit test + TIMEOUT_FACTOR: "4" + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci|gst-devtools)$/' + +valgrind ugly: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-plugins-ugly\\..*" + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci|gst-devtools)$/' + +valgrind bad: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-plugins-bad\\..*" + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci|gst-devtools)$/' + +valgrind ges: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-editing-services\\..*" + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci|gst-devtools)$/' + +# Template for Cerbero GStreamer Build +# +# Parameters: +# CONFIG: The name of the configuration file to use +# ARCH: The cerbero _ (used in cache key) +# +.cerbero: + stage: "build" + image: $CERBERO_IMAGE + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' + needs: + - "manifest" + # Ensure that the runners it will be executed on + # will have plenty of space for the cache + tags: ['gstreamer'] + timeout: '3h' + variables: + CCACHE_COMPILERCHECK: "content" + CCACHE_COMPRESS: "true" + CCACHE_BASEDIR: "/cache/gstreamer/cerbero/" + CCACHE_DIR: "/cache/gstreamer/cerbero/ccache/" + # shared across everything really + CCACHE_MAXSIZE: "50G" + + CERBERO_HOME: "cerbero-build" + CERBERO_SOURCES: "cerbero-sources" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS}" + CERBERO: "./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml" + CERBERO_PACKAGE_ARGS: "-t" + CERBERO_RUN_WRAPPER: "" # 'wine' on cross-winXX + CERBERO_RUN_SUFFIX: "" # '.exe' on cross-winXX + HAVE_CCACHE: "yes" + # used by macos packages as we only ever install to a fixed directory + CERBERO_OVERRIDDEN_DIST_DIR: "" + # location where the cerbero git repo is stored on the image + CERBERO_HOST_DIR: "/cerbero/" + before_script: + - echo "Fetching ${CERBERO_SCRIPT_URL}" + - curl -L -o cerbero_setup.sh "${CERBERO_SCRIPT_URL}" + - chmod +x cerbero_setup.sh + - ./cerbero_setup.sh cerbero_before_script + script: + - ./cerbero_setup.sh cerbero_script + cache: + key: "${CI_JOB_NAME}" + paths: + - "${CERBERO_SOURCES}" + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "*.tar.*" + +.build windows: + image: $WINDOWS_IMAGE + stage: 'build' + needs: + - 'manifest' + tags: + - 'docker' + - 'windows' + - '1809' + timeout: '45min' + variables: + MESON_ARGS: > + ${DEFAULT_MESON_ARGS} + -Dpython=disabled + -Dlibav=disabled + -Dvaapi=disabled + -Dgst-plugins-base:pango=enabled + -Dgst-plugins-good:cairo=enabled + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' + script: + # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or + # from a previous job due to some gitlab bug or implicit behavior? + # So let's always check and clear it out if its there + # https://gitlab.freedesktop.org/tpm/gstreamer-sharp/-/jobs/1672137 + - if (Test-Path $env:CI_PROJECT_DIR/gst-build) { Remove-Item -Recurse -Force $env:CI_PROJECT_DIR/gst-build } + + # Make sure powershell exists on errors + # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 + - $ErrorActionPreference = "Stop" + + - Invoke-WebRequest -Uri "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/$env:GST_UPSTREAM_BRANCH/gitlab/clone_manifest_ref.py" -OutFile $env:CI_PROJECT_DIR/clone_manifest_ref.py + - python clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination $env:CI_PROJECT_DIR/gst-build + - cd $env:CI_PROJECT_DIR/gst-build + + # Update meson + # FIXME: remove this once the windows image is rebuilt/updated + - pip install -U meson==0.56.2 + + - cp -r C:/subprojects/* subprojects/ + - cd $env:CI_PROJECT_DIR/gst-build + # Reset the subprojects to invalidate the cache if needed + - meson subprojects update --reset + # Run the git-update script and feed it the manifest to setup the environment + - python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml + + # For some reason, options are separated by newline instead of space, so we + # have to replace them first. + - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") + # Gitlab executes PowerShell in docker, but VsDevCmd.bat is a batch script. + # Environment variables substitutions is done by PowerShell before calling + # cmd.exe, that's why we use $env:FOO instead of %FOO% + - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && + meson build $env:MESON_ARGS && + ninja -C build" + artifacts: + expire_in: "7 days" + when: "always" + paths: + - 'gst-build/build/meson-logs/' + +build vs2017 amd64: + extends: '.build windows' + variables: + ARCH: 'amd64' + +build vs2017 x86: + extends: '.build windows' + variables: + ARCH: 'x86' + +build msys2 : + extends: '.build windows' + timeout: '60min' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' + allow_failure: true + when: 'manual' + script: + # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or + # from a previous job due to some gitlab bug or implicit behavior? + # So let's always check and clear it out if its there + # https://gitlab.freedesktop.org/tpm/gstreamer-sharp/-/jobs/1672137 + - if (Test-Path $env:CI_PROJECT_DIR/gst-build) { Remove-Item -Recurse -Force $env:CI_PROJECT_DIR/gst-build } + + # Make sure powershell exists on errors + # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 + - $ErrorActionPreference = "Stop" + + # For some reason docker build hangs if this is included in the image, needs more troubleshooting + - $env:PATH += ';C:\msys64\usr\bin;C:\msys64\mingw64\bin;C:\msys64\mingw32\bin' + # Copied from https://github.com/msys2/setup-msys2/blob/master/main.js#L98 + - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true" + - C:\msys64\usr\bin\bash -c "sed -i 's/^CheckSpace/#CheckSpace/g' /etc/pacman.conf" + - echo "Updating MSYS2" + - C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm || echo Update failed, ignoring" + - echo "Killing all MSYS2 processes" + - taskkill /F /FI "MODULES eq msys-2.0.dll" + - echo "Completing MSYS2 update" + - C:\msys64\usr\bin\bash -c "pacman -Suu --noconfirm" + - echo "Installing needed MSYS2 packages" + - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" + + - Invoke-WebRequest -Uri "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/$env:GST_UPSTREAM_BRANCH/gitlab/clone_manifest_ref.py" -OutFile $env:CI_PROJECT_DIR/clone_manifest_ref.py + - python clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination $env:CI_PROJECT_DIR/gst-build + - cd $env:CI_PROJECT_DIR/gst-build + + - cp -r C:\subprojects\* subprojects\ + + # Run the git-update script and feed it the manifest to setup the environment + - cd $env:CI_PROJECT_DIR/gst-build; python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml + # For some reason, options are separated by newline instead of space, so we + # have to replace them first. + - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") + # Replace forward slashes with backwards so bash doesn't complain + - $env:_PROJECT_DIR = $env:CI_PROJECT_DIR.replace('\','/') + - C:\msys64\usr\bin\bash -c "cd $env:_PROJECT_DIR/gst-build && + meson build $env:MESON_ARGS && + ninja -C build" + + +# Template for Cerbero GStreamer Deps +# +# This template is used by cerbero/ project to pre-built the GStreamer +# dependencies. When available, the .cerbero jobs will download this artifact +# in order to speed up the build. +# +# Parameters: +# CONFIG: The name of the configuration file to use +# ARCH: The cerbero _ (used in cache key) +# +# Produce an artifact with the dist/ and .cache along +# with the associated build-tools. +.cerbero deps: + extends: .cerbero + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' + script: + - ./cerbero_setup.sh cerbero_deps_script + +# +# Cerbero Linux X86_64 build +# +cerbero deps fedora x86_64: + extends: '.cerbero deps' + variables: + CONFIG: "linux.config" + ARCH: "linux_x86_64" + +build cerbero fedora x86_64: + extends: '.cerbero' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + variables: + CONFIG: "linux.config" + +# +# Cerbero Android Universal build +# +.cerbero cross-android universal: + variables: + CONFIG: "cross-android-universal.cbc" + ARCH: "android_universal" + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "*[0-9].tar.*" + +cerbero deps cross-android universal: + extends: + - '.cerbero deps' + - '.cerbero cross-android universal' + +build cerbero cross-android universal: + extends: + - '.cerbero' + - '.cerbero cross-android universal' + rules: + - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + +# +# Cerbero Cross Windows builds +# +cerbero deps cross-windows x86: + extends: '.cerbero deps' + variables: + CONFIG: "cross-win32.cbc" + ARCH: "mingw_x86" + +build cerbero cross win32: + extends: '.cerbero' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + variables: + CONFIG: "cross-win32.cbc" + CERBERO_RUN_WRAPPER: "wine" + CERBERO_RUN_SUFFIX: ".exe" + + +cerbero deps cross-windows x86_64: + extends: '.cerbero deps' + variables: + CONFIG: "cross-win64.cbc" + ARCH: "mingw_x86_64" + +build cerbero cross win64: + extends: '.cerbero' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + variables: + CONFIG: "cross-win64.cbc" + CERBERO_RUN_WRAPPER: "wine" + CERBERO_RUN_SUFFIX: ".exe" + +# +# Build an Android App using the android binaries +# +.cross-android universal examples: + image: $ANDROID_IMAGE + stage: 'integrate' + variables: + EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples + GSTREAMER_ROOT_ANDROID: ${CI_PROJECT_DIR}/examples/cerbero-android-universal + script: + - mkdir -p ${EXAMPLES_HOME}/outputs + - curl -L -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py + - chmod +x clone_manifest_ref.py + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs + - rm clone_manifest_ref.py + + # extract our binaries + - rm -f gstreamer-1.0-android-universal-*-runtime.tar.* + - mkdir ${GSTREAMER_ROOT_ANDROID} + - time tar -C ${GSTREAMER_ROOT_ANDROID} -xf gstreamer-1.0-android-universal-*.tar.* + + # gst-examples - player + - chmod +x ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew + - ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/playback/player/android assembleDebug + - cp ${EXAMPLES_HOME}/gst-examples/playback/player/android/app/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ + + # gst-examples - vulkan + - chmod +x ${EXAMPLES_HOME}/gst-examples/vulkan/android/gradlew + - ${EXAMPLES_HOME}/gst-examples/vulkan/android/gradlew --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/vulkan/android assembleDebug + - cp ${EXAMPLES_HOME}/gst-examples/vulkan/android/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ + + # gst-docs android tutorials + - chmod +x ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew + - ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew --no-daemon --project-dir ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android assembleDebug + - cp ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/android-tutorial-*/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ + after_script: + - rm -rf ${GSTREAMER_ROOT_ANDROID} + - rm -rf ${EXAMPLES_HOME}/gst-examples ${EXAMPLES_HOME}/gst-docs + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${EXAMPLES_HOME}/outputs" + +cross-android universal examples: + extends: ".cross-android universal examples" + rules: + - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + needs: + - "build cerbero cross-android universal" + +cerbero cross-android universal examples: + extends: ".cross-android universal examples" + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' + needs: + - "cerbero deps cross-android universal" + +# +# Cerbero macOS X86_64 build +# +.cerbero macos x86_64: + variables: + ARCH: "darwin_x86_64" + CONFIG: "osx-x86-64.cbc" + CERBERO_PACKAGE_ARGS: "" + HAVE_CCACHE: "" + CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" + CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" + tags: + - gst-macos-11.1 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "gstreamer-1.0-1.*.pkg" + - "gstreamer-1.0-devel-1.*.pkg" + +cerbero deps macos x86_64: + extends: + - '.cerbero deps' + - '.cerbero macos x86_64' + +build cerbero macos x86_64: + extends: + - '.cerbero' + - '.cerbero macos x86_64' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + +# +# Cerbero macOS Universal build +# +.cerbero cross-macos universal: + variables: + ARCH: "darwin_universal" + CONFIG: "cross-macos-universal.cbc" + CERBERO_PACKAGE_ARGS: "" + HAVE_CCACHE: "" + CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" + tags: + - gst-macos-11.1 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "gstreamer-1.0-1.*.pkg" + - "gstreamer-1.0-devel-1.*.pkg" + +cerbero deps cross-macos universal: + extends: + - '.cerbero deps' + - '.cerbero cross-macos universal' + +build cerbero cross-macos universal: + extends: + - '.cerbero' + - '.cerbero cross-macos universal' + script: + # We have build-tools that have fixed paths pointing to the upstream + # repository. We need to point + - pwd + - mkdir -p "../../gstreamer" + - ln -sf "$(pwd)" "../../gstreamer/cerbero" + - mkdir -p "../../${CI_PROJECT_NAMESPACE}" + - ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + + - ./cerbero_setup.sh cerbero_script + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + +# +# Cerbero iOS build +# +.cerbero cross-ios universal: + variables: + ARCH: "ios_universal" + CONFIG: "cross-ios-universal.cbc" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" + CERBERO_PACKAGE_ARGS: "" + HAVE_CCACHE: "" + CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" + tags: + - gst-ios-14.3 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "gstreamer-1.0-*-ios-universal.pkg" + +cerbero deps cross-ios universal: + extends: + - '.cerbero deps' + - '.cerbero cross-ios universal' + +build cerbero cross-ios universal: + extends: + - '.cerbero' + - '.cerbero cross-ios universal' + rules: + - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + +documentation: + image: $FEDORA_DOCS_IMAGE + extends: + - '.gst_build_ccache_vars' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-integration-testsuites|gst-plugins-rs|gst-python)$/' + needs: + - "manifest" + stage: integrate + variables: + MESON_ARGS: *simple_build + MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" + CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" + script: + - pip3 install --upgrade hotdoc + + # FIXME: remove this once the docs image is updated + - pip3 install -U meson==0.56.2 + + - *gst_build + - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches + - | + python3 -c "import os, subprocess, sys + subprojects = [os.environ['CI_PROJECT_NAME']] + if subprojects[0] in ['gst-ci', 'gst-build']: + subprojects = ['gstreamer', 'gst-plugins-ugly', 'gst-omx', 'gst-plugins-good', 'gst-plugins-base', 'gstreamer-vaapi', 'gst-libav', 'gst-editing-services', 'gst-rtsp-server', 'gst-plugins-bad',] + + diffsdir = '../plugins-cache-diffs' + os.makedirs(diffsdir, exist_ok=True) + res = 0 + for subproject in subprojects: + cwd = os.path.join('subprojects', subproject) + try: + subprocess.check_call(['git', 'diff', '--quiet'], cwd=cwd) + except subprocess.CalledProcessError: + diffname = os.path.join(diffsdir, '%s_cache.diff' % subproject) + res += 1 + with open(diffname, 'w') as diff: + subprocess.check_call(['git', 'diff'], stdout=diff, cwd=cwd) + print('\033[91mYou have a diff in the %s documentation cache. Please update with:\033[0m' % subproject) + print(' $ curl %s/%s | git apply -' % (os.environ['CI_ARTIFACTS_URL'], diffname.replace('../', ''))) + if res != 0: + print('(note that it might take a few minutes for artefacts to be available on the server)\n') + sys.exit(res)" + + - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings + - cd - + - mv gst-build/build/subprojects/gst-docs/GStreamer-doc/html documentation/ + + artifacts: + when: always + paths: + - documentation/ + - plugins-cache-diffs/ + +# +# Build an iOS App using the iOS binaries +# +.cross-ios universal examples: + stage: 'integrate' + variables: + EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples + # disable codesigning so we don't need developer identities on the CI + # machines + XCODE_BUILD_ARGS: > + CODE_SIGNING_REQUIRED="NO" + CODE_SIGN_IDENTITY="" + CODE_SIGNING_ALLOWED="NO" + CODE_SIGN_ENTITLEMENTS="" + script: + # install the binaries + - installer -pkg gstreamer-1.0-devel-*-ios-universal.pkg -target CurrentUserHomeDirectory -verbose + + - curl -L -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py + - chmod +x clone_manifest_ref.py + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs + - rm clone_manifest_ref.py + + # dump some useful information + - xcodebuild -version + - xcodebuild -showsdks + + # gst-docs ios tutorials + - xcodebuild -showBuildSettings -alltargets -project ${EXAMPLES_HOME}/gst-docs/examples/tutorials/xcode\ iOS/GStreamer\ iOS\ Tutorials.xcodeproj + - xcodebuild -alltargets -destination generic/platform=iOS -project ${EXAMPLES_HOME}/gst-docs/examples/tutorials/xcode\ iOS/GStreamer\ iOS\ Tutorials.xcodeproj ${XCODE_BUILD_ARGS} + + # gst-examples + - xcodebuild -showBuildSettings -alltargets -project ${EXAMPLES_HOME}/gst-examples/playback/player/ios/GstPlay.xcodeproj + - xcodebuild -alltargets -destination generic/platform=iOS -project ${EXAMPLES_HOME}/gst-examples/playback/player/ios/GstPlay.xcodeproj ${XCODE_BUILD_ARGS} + after_script: + - rm -rf ${EXAMPLES_HOME}/gst-examples ${EXAMPLES_HOME}/gst-docs + tags: + - gst-ios-14.3 + +cross-ios universal examples: + extends: ".cross-ios universal examples" + rules: + - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + needs: + - "build cerbero cross-ios universal" + +cerbero cross-ios universal examples: + extends: ".cross-ios universal examples" + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' + needs: + - "cerbero deps cross-ios universal" + +build gst-omx zynq fedora x86_64: + extends: '.build fedora x86_64' + variables: + MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} $MESON_GST_WERROR" + rules: + - if: '$CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/' + before_script: + - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2020.1 ${CI_PROJECT_DIR}/vcu-omx-il + +build gst-omx tizonia fedora x86_64: + extends: '.build fedora x86_64' + variables: + MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} $MESON_GST_WERROR" + PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}:${CI_PROJECT_DIR}/tizonia-install/lib64/pkgconfig/ + rules: + - if: '$CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/' + before_script: + - git clone https://github.com/tizonia/tizonia-openmax-il --branch v0.20.2 + - cd tizonia-openmax-il + - meson build -Dclients=false -Dplugins='' -Dplayer=false -Dprefix=${CI_PROJECT_DIR}/tizonia-install + - ninja -C build + - ninja -C build install + - cd .. + +# +# Cerbero Native Windows builds +# +.cerbero windows native: + needs: ['manifest'] + image: $CERBERO_WINDOWS_IMAGE + tags: + - 'docker' + - 'windows' + - 'gstreamer-windows' + - '1809' + variables: + CONFIG: 'win64.cbc' + ARCH: 'msvc_x86_64' + CERBERO_HOST_DIR: "C:/cerbero" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v visualstudio -v nowerror" + CERBERO_RUN_SUFFIX: ".exe" + HAVE_CCACHE: "" + before_script: + - $env:CI_PROJECT_DIR = $env:CI_PROJECT_DIR.replace('\', '/') + - $env:CERBERO_SCRIPTS_PATH = "$env:CI_PROJECT_DIR/cerbero_setup.sh" + - echo "Fetching $env:CERBERO_SCRIPT_URL" + - Invoke-WebRequest -Uri $env:CERBERO_SCRIPT_URL -OutFile $env:CERBERO_SCRIPTS_PATH + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_before_script" + +cerbero deps msvc x86_64: + extends: ['.cerbero deps', '.cerbero windows native'] + script: + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_deps_script" + +build cerbero msvc x86_64: + extends: ['.cerbero', '.cerbero windows native'] + script: + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_script" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' diff --git a/gitlab/clone_manifest_ref.py b/gitlab/clone_manifest_ref.py new file mode 100755 index 0000000000..b26043b510 --- /dev/null +++ b/gitlab/clone_manifest_ref.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess + +from collections import namedtuple +import xml.etree.ElementTree as ET + +# Disallow git prompting for a username/password +os.environ['GIT_TERMINAL_PROMPT'] = '0' +def git(*args, repository_path='.'): + return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() + +class Manifest(object): + ''' + Parse and store the content of a manifest file + ''' + + remotes = {} + projects = {} + default_remote = 'origin' + default_revision = 'refs/heads/master' + + def __init__(self, manifest_path): + self.manifest_path = manifest_path + + def parse(self): + try: + tree = ET.parse(self.manifest_path) + except Exception as ex: + raise Exception("Error loading manifest %s in file %s" % (self.manifest_path, ex)) + + root = tree.getroot() + + for child in root: + if child.tag == 'remote': + self.remotes[child.attrib['name']] = child.attrib['fetch'] + if child.tag == 'default': + self.default_remote = child.attrib['remote'] or self.default_remote + self.default_revision = child.attrib['revision'] or self.default_revision + if child.tag == 'project': + project = namedtuple('Project', ['name', 'remote', + 'revision', 'fetch_uri']) + + project.name = child.attrib['name'] + if project.name.endswith('.git'): + project.name = project.name[:-4] + project.remote = child.attrib.get('remote') or self.default_remote + project.revision = child.attrib.get('revision') or self.default_revision + project.fetch_uri = self.remotes[project.remote] + project.name + '.git' + + self.projects[project.name] = project + + def find_project(self, name): + try: + return self.projects[name] + except KeyError as ex: + raise Exception("Could not find project %s in manifest %s" % (name, self.manifest_path)) + + def get_fetch_uri(self, project, remote): + fetch = self.remotes[remote] + return fetch + project.name + '.git' + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--project", action="store", type=str) + parser.add_argument("--destination", action="store", type=str, default='.') + parser.add_argument("--manifest", action="store", type=str) + parser.add_argument("--fetch", action="store_true", default=False) + options = parser.parse_args() + + if not options.project: + raise ValueError("--project argument not provided") + if not options.manifest: + raise ValueError("--manifest argument not provided") + + manifest = Manifest(options.manifest) + manifest.parse() + project = manifest.find_project(options.project) + + dest = options.destination + if dest == '.': + dest = os.path.join (os.getcwd(), project.name) + + if options.fetch: + assert os.path.exists(dest) == True + git('fetch', project.fetch_uri, project.revision, repository_path=dest) + else: + git('clone', project.fetch_uri, dest) + + git('checkout', '--detach', project.revision, repository_path=dest) diff --git a/gitlab/freedesktop_doc_importer.sh b/gitlab/freedesktop_doc_importer.sh new file mode 100755 index 0000000000..be68222d0f --- /dev/null +++ b/gitlab/freedesktop_doc_importer.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +set -e + +BRANCH=master +NAMESPACE=gstreamer +JOB=documentation + +WORK_DIR=`mktemp -d -p "$DIR"` + +# deletes the temp directory +function cleanup { + rm -rf "$WORK_DIR" + echo "Deleted temp working directory $WORK_DIR" +} + +# register the cleanup function to be called on the EXIT signal +trap cleanup EXIT + +echo "" +echo "============================================================================================================================" +echo "Updating documentation from: https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=$JOB" + +date + +cd $WORK_DIR +wget https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=$JOB -O gstdocs.zip + +unzip gstdocs.zip + +DOC_BASE="/srv/gstreamer.freedesktop.org/public_html/documentation" + +rsync -rvaz --links --delete documentation/ $DOC_BASE || /bin/true +chmod -R g+w $DOC_BASE; chgrp -R gstreamer $DOC_BASE + +echo "Done updating documentation" +echo "" \ No newline at end of file diff --git a/jenkins/Jenkinsfile-cerbero-debian b/jenkins/Jenkinsfile-cerbero-debian new file mode 100644 index 0000000000..14c5a68f42 --- /dev/null +++ b/jenkins/Jenkinsfile-cerbero-debian @@ -0,0 +1,51 @@ +node("docker") { + docker.image('gstreamer/build-base-ubuntu:latest').inside { + + env.OUTPREFIX="${env.WORKSPACE}/../output/${params.build_tag}/linux_x86_64/" + + stage('Checkout') { + // FIXME: Only checkout the manifest and not all dependencies ? + checkout([$class: 'RepoScm', + manifestRepositoryUrl:'git+ssh://git.arracacha.collabora.co.uk/git/gst-manifest.git', + manifestBranch:"refs/tags/${params.build_tag}", + jobs:4, + currentBranch:true, + quiet:true, + depth:0]) + } + + stage('Setup') { + sh "find ../output -maxdepth 1 -ctime +1 | xargs rm -Rf" + sh "cd .repo/manifests/; git checkout ${params.build_tag}; cd ../.." + sh "rm -Rf ./workdir/sources/linux_x86_64/" + sh "rm -f *.rpm" + sh "rm -Rf ./workdir/temp; mkdir -p ./workdir/temp" + sh "rm -Rf ./workdir/tmp*" + + // Create custom configuration file + sh "./gst-ci-scripts/manifest2cerbero.py .repo/manifests/default.xml ./cerbero/config/linux.config --output localconf.cbc" + sh '''echo "home_dir = \\"$WORKSPACE/workdir\\"" >> localconf.cbc''' + sh '''echo "logs = \\"$OUTPREFIX/logs\\"" >> localconf.cbc''' + sh './cerbero/cerbero-uninstalled -c localconf.cbc show-config' + } + + stage('bootstrap') { + sh './cerbero/cerbero-uninstalled -c localconf.cbc bootstrap' + sh 'rm -Rf ./workdir/sources/build-tools/' + } + + stage('fetch') { + sh './cerbero/cerbero-uninstalled -c localconf.cbc fetch-package --reset-rdeps --full-reset gstreamer-1.0' + } + + stage('package') { + sh './cerbero/cerbero-uninstalled -c localconf.cbc package gstreamer-1.0' + } + + stage('Cleanup') { + sh 'rm -f *.rpm' + sh 'find ../output -maxdepth 1 -ctime +1 | xargs rm -Rf' + } + // FIXME: IRC Notification + } +} diff --git a/jenkins/Jenkinsfile-master-fedora b/jenkins/Jenkinsfile-master-fedora new file mode 100644 index 0000000000..000416a983 --- /dev/null +++ b/jenkins/Jenkinsfile-master-fedora @@ -0,0 +1,66 @@ +node('docker') { + docker.image('gstreamer/build-base-fedora:latest').inside { + + env.CCACHE_DIR = "${env.WORKSPACE}/.ccache" + env.CCACHE_MAXSIZE = "2G" + env.CC = "ccache gcc" + env.CXX = "ccache g++" + env.MAKEFLAGS = "-j6" + env.PATH = "${env.WORKSPACE}:${env.PATH}" + env.GST_UNINSTALLED_ROOT="${env.WORKSPACE}" + env.HOME="${env.WORKSPACE}" + env.DISPLAY=":0" + + stage('Checkout') { + if (params.wipe) { + sh 'rm -Rf *' + } + checkout([$class: 'RepoScm', + manifestRepositoryUrl:'https://git.arracacha.collabora.co.uk/git/gst-manifest.git', + manifestBranch:"refs/tags/${params.build_tag}", + jobs:4, + currentBranch:true, + quiet:true, + depth:0, + mirrorDir:'/repositories']) + } + + stage('Cleanup') { + sh 'rm -f **/tests/check/*/*.xml' + } + + stage ('Build') { + sh "uname -a" + sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh fast-build-only" + } + + withEnv(['DISPLAY=:0']) { + stage ('Check') { + env.GST_CHECKS_IGNORE="test_allocate_udp_ports_multicast,test_allocate_udp_ports_client_settings,test_reorder_buffer,test_redirect_yes" + env.GST_CHECK_XML=1 + sh 'Xvfb :0 -screen 0 1024x768x24 -fbdir /tmp &' + sh 'env' + sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh check" + + step([$class: 'XUnitBuilder', + testTimeMargin: '3000', thresholdMode: 1, + thresholds: [[$class: 'FailedThreshold', + failureNewThreshold: '', + failureThreshold: '400', + unstableNewThreshold: '', + unstableThreshold: '1'], + [$class: 'SkippedThreshold', + failureNewThreshold: '', + failureThreshold: '', + unstableNewThreshold: '', + unstableThreshold: '']], + tools: [[$class: 'CheckType', + deleteOutputFiles: true, + failIfNotNew: true, + pattern: '**/tests/check/*/*.xml', + skipNoTestFiles: true, + stopProcessingIfError: true]]]) + } + } + } +} diff --git a/jenkins/Jenkinsfile-master-ubuntu b/jenkins/Jenkinsfile-master-ubuntu new file mode 100644 index 0000000000..01522b9794 --- /dev/null +++ b/jenkins/Jenkinsfile-master-ubuntu @@ -0,0 +1,66 @@ +node('docker') { + docker.image('gstreamer/build-base-ubuntu:latest').inside { + + env.CCACHE_DIR = "${env.WORKSPACE}/.ccache" + env.CCACHE_MAXSIZE = "2G" + env.CC = "ccache gcc" + env.CXX = "ccache g++" + env.MAKEFLAGS = "-j6" + env.PATH = "${env.WORKSPACE}:${env.PATH}" + env.GST_UNINSTALLED_ROOT="${env.WORKSPACE}" + env.HOME="${env.WORKSPACE}" + env.DISPLAY=":0" + + stage('Checkout') { + if (params.wipe) { + sh 'rm -Rf *' + } + checkout([$class: 'RepoScm', + manifestRepositoryUrl:'https://git.arracacha.collabora.co.uk/git/gst-manifest.git', + manifestBranch:"refs/tags/${params.build_tag}", + jobs:4, + currentBranch:true, + quiet:true, + depth:0, + mirrorDir:'/repositories']) + } + + stage('Cleanup') { + sh 'rm -f **/tests/check/*/*.xml' + } + + stage ('Build') { + sh "uname -a" + sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh fast-build-only" + } + + withEnv(['DISPLAY=:0']) { + stage ('Check') { + env.GST_CHECKS_IGNORE="test_allocate_udp_ports_multicast,test_allocate_udp_ports_client_settings,test_reorder_buffer,test_redirect_yes" + env.GST_CHECK_XML=1 + sh 'Xvfb :0 -screen 0 1024x768x24 -fbdir /tmp &' + sh 'env' + sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh check" + + step([$class: 'XUnitBuilder', + testTimeMargin: '3000', thresholdMode: 1, + thresholds: [[$class: 'FailedThreshold', + failureNewThreshold: '', + failureThreshold: '400', + unstableNewThreshold: '', + unstableThreshold: '1'], + [$class: 'SkippedThreshold', + failureNewThreshold: '', + failureThreshold: '', + unstableNewThreshold: '', + unstableThreshold: '']], + tools: [[$class: 'CheckType', + deleteOutputFiles: true, + failIfNotNew: true, + pattern: '**/tests/check/*/*.xml', + skipNoTestFiles: true, + stopProcessingIfError: true]]]) + } + } + } +} diff --git a/jenkins/Jenkinsfile-meson-fedora b/jenkins/Jenkinsfile-meson-fedora new file mode 100644 index 0000000000..bc3800d911 --- /dev/null +++ b/jenkins/Jenkinsfile-meson-fedora @@ -0,0 +1,71 @@ +node('docker') { + docker.image('gstreamer/build-meson-fedora:latest').inside { + + env.CCACHE_DIR = "${env.WORKSPACE}/.ccache" + env.CCACHE_MAXSIZE = "2G" + env.CC = "ccache gcc" + env.CXX = "ccache g++" + env.MAKEFLAGS = "-j6" + env.PATH = "${env.WORKSPACE}:${env.PATH}" + env.HOME="${env.WORKSPACE}" + env.DISPLAY=":0" + + stage('Checkout') { + if (params.wipe) { + sh 'rm -Rf build/' + } + checkout([$class: 'GitSCM', branches: [[name: '*/master']], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'CloneOption', + depth: 0, + noTags: false, + reference: '/gstbuild/gst-build/', + shallow: false]], + submoduleCfg: [], + userRemoteConfigs: [[url: 'git://anongit.freedesktop.org/gstreamer/gst-build']]] + ) + sh 'git checkout master && git reset --hard origin/master' + sh 'curl "https://git.arracacha.collabora.co.uk/cgit/gst-manifest.git/plain/default.xml?id=$BUILD_TAG" -k -o manifest.xml' + } + + stage('Setup') { + sh './git-update --no-color --manifest=manifest.xml --no-interaction' + sh './setup.py -Ddisable_gstreamer_vaapi=true' + } + + stage ('Build') { + sh "ninja -C build" + } + + stage ('Check') { + sh "./gst-uninstalled.py gst-validate-launcher --check-bugs --no-display --mute -n check --xunit-file $WORKSPACE/xunit.xml -M $WORKSPACE/validate-output --ignore-numfailures" + + step([$class: 'XUnitBuilder', + testTimeMargin: '3000', thresholdMode: 1, + thresholds: [[$class: 'FailedThreshold', + failureNewThreshold: '', + failureThreshold: '5', + unstableNewThreshold: '', + unstableThreshold: '1'], + [$class: 'SkippedThreshold', + failureNewThreshold: '', + failureThreshold: '', + unstableNewThreshold: '', + unstableThreshold: '']], + tools: [[$class: 'JUnitType', + deleteOutputFiles: true, + failIfNotNew: true, + pattern: 'xunit.xml', + skipNoTestFiles: true, + stopProcessingIfError: true]]]) + } + + stage('install') { + sh 'mkdir -p dest' + sh 'DESTDIR=$PWD/dest ninja -C build install' + } + stage('package') { + sh 'cd dest && tar caJf gstreamer-$BUILD_TAG.tar.xz usr' + } + } +}