Merging gst-ci

This commit is contained in:
Thibault Saunier 2021-09-24 16:15:12 -03:00
commit c7253318b3
38 changed files with 3622 additions and 0 deletions

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
*~

24
.gitlab-ci.yml Normal file
View file

@ -0,0 +1,24 @@
include: "gitlab/ci_template.yml"
test manifest:
variables:
GIT_STRATEGY: fetch
image: "$TEST_MANIFEST_IMAGE"
stage: "preparation"
script:
- pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py
coverage: '/TOTAL.*\s+(\d+%)$/'
artifacts:
reports:
junit:
- "junit.xml"
test manifest amd64 docker:
stage: "build docker"
variables:
REPO_SUFFIX: "$TEST_MANIFEST_AMD64_SUFFIX"
TAG: "$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH"
CONTEXT_DIR: "docker/test_manifest/"
DOCKERFILE: "docker/test_manifest/Dockerfile"
extends: .base

11
README.txt Normal file
View file

@ -0,0 +1,11 @@
GStreamer Continuous Integration
================================
This repository contains all material relevant to the GStreamer
Continuous Integration system.
* Jenkins scripts
* Docker images
* Build scripts and code

2
docker/README Normal file
View file

@ -0,0 +1,2 @@
GStreamer Docker images

11
docker/android/Dockerfile Normal file
View file

@ -0,0 +1,11 @@
FROM registry.fedoraproject.org/fedora:31
ENV ANDROID_HOME=/android/sdk
ENV ANDROID_NDK_HOME=/android/ndk
ARG DEFAULT_BRANCH="master"
COPY prepare.sh cleanup.sh /root/
RUN /usr/bin/sh /root/prepare.sh && \
/usr/bin/sh /root/cleanup.sh

View file

@ -0,0 +1,7 @@
set -eux
echo "Removing DNF cache"
dnf clean all
rm -R /root/*
rm -rf /var/cache/dnf /var/log/dnf*

56
docker/android/prepare.sh Normal file
View file

@ -0,0 +1,56 @@
set -eux
dnf install -y \
file \
git-core \
java-1.8.0-openjdk-devel \
lbzip2 \
make \
pkg-config \
unzip \
which \
xz
mkdir -p /android/sources
curl -o /android/sources/android-ndk.zip https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip
unzip /android/sources/android-ndk.zip -d ${ANDROID_NDK_HOME}/
# remove the intermediate versioned directory
mv ${ANDROID_NDK_HOME}/*/* ${ANDROID_NDK_HOME}/
curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-6200805_latest.zip
unzip /android/sources/android-sdk-tools.zip -d ${ANDROID_HOME}/
mkdir -p ${ANDROID_HOME}/licenses
# Accept licenses. Values taken from:
# $ANDROID_HOME/tools/bin/sdkmanager --sdk_root=$ANDROID_HOME --licenses
# cd $ANDROID_HOME
# for f in licenses/*; do echo "echo \"$(cat $f | tr -d '\n')\" > \${ANDROID_HOME}/$f"; done
echo "601085b94cd77f0b54ff86406957099ebe79c4d6" > ${ANDROID_HOME}/licenses/android-googletv-license
echo "859f317696f67ef3d7f30a50a5560e7834b43903" > ${ANDROID_HOME}/licenses/android-sdk-arm-dbt-license
echo "24333f8a63b6825ea9c5514f83c2829b004d1fee" > ${ANDROID_HOME}/licenses/android-sdk-license
echo "84831b9409646a918e30573bab4c9c91346d8abd" > ${ANDROID_HOME}/licenses/android-sdk-preview-license
echo "33b6a2b64607f11b759f320ef9dff4ae5c47d97a" > ${ANDROID_HOME}/licenses/google-gdk-license
echo "e9acab5b5fbb560a72cfaecce8946896ff6aab9d" > ${ANDROID_HOME}/licenses/mips-android-sysimage-license
# pre-cache deps
export GSTREAMER_ROOT_ANDROID=/android/sources/gstreamer-android
curl -o /android/sources/gstreamer-android.tar.xz https://gstreamer.freedesktop.org/data/pkg/android/1.16.2/gstreamer-1.0-android-universal-1.16.2.tar.xz
mkdir $GSTREAMER_ROOT_ANDROID
tar -xvf /android/sources/gstreamer-android.tar.xz -C $GSTREAMER_ROOT_ANDROID
ls $GSTREAMER_ROOT_ANDROID
git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-examples.git /android/sources/gst-examples
chmod +x /android/sources/gst-examples/playback/player/android/gradlew
/android/sources/gst-examples/playback/player/android/gradlew --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies
chmod +x /android/sources/gst-examples/vulkan/android/gradlew
/android/sources/gst-examples/vulkan/android/gradlew --no-daemon --project-dir /android/sources/gst-examples/vulkan/android dependencies --refresh-dependencies
git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs
chmod +x /android/sources/gst-docs/examples/tutorials/android/gradlew
/android/sources/gst-docs/examples/tutorials/android/gradlew --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies
unset GSTREAMER_ROOT_ANDROID
rm -rf /android/sources

View file

@ -0,0 +1,7 @@
FROM python:3.7.1-alpine
ARG DEFAULT_BRANCH="master"
RUN pip install requests
RUN apk add git
RUN cd / && git clone -b ${DEFAULT_BRANCH:=master} https://gitlab.freedesktop.org/gstreamer/gst-ci.git

View file

@ -0,0 +1,35 @@
FROM registry.fedoraproject.org/fedora:31
ENV TERM="dumb"
ARG DEFAULT_BRANCH="master"
# System setup
RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \
dnf upgrade -y && \
dnf install -y sudo git lbzip2 rsync wine which
# Configure git for various usage
RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \
git config --global user.name "Cerbero Build System"
RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git && \
mkdir $HOME/.cerbero && \
echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \
echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \
cd cerbero && \
echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \
echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \
./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap --jobs=4 && \
./cerbero-uninstalled -t -c localconf.cbc fetch-package --jobs=4 gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap --jobs=4 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package --jobs=4 gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap --jobs=4 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package --jobs=4 gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap --jobs=4 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package --jobs=4 gstreamer-1.0 && \
cd .. && \
rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \
rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \
dnf clean all

12
docker/fedora/Dockerfile Normal file
View file

@ -0,0 +1,12 @@
FROM registry.fedoraproject.org/fedora:31
ENV RUSTUP_HOME=/usr/local/rustup \
CARGO_HOME=/usr/local/cargo \
PATH=/usr/local/cargo/bin:$PATH
ARG DEFAULT_BRANCH="master"
COPY prepare.sh cleanup.sh /root/
RUN /usr/bin/sh /root/prepare.sh && \
/usr/bin/sh /root/cleanup.sh

7
docker/fedora/cleanup.sh Normal file
View file

@ -0,0 +1,7 @@
set -eux
echo "Removing DNF cache"
dnf clean all
rm -R /root/*
rm -rf /var/cache/dnf /var/log/dnf*

244
docker/fedora/prepare.sh Normal file
View file

@ -0,0 +1,244 @@
set -eux
# Fedora base image disable installing documentation files. See https://pagure.io/atomic-wg/issue/308
# We need them to cleanly build our doc.
sed -i "s/tsflags=nodocs//g" /etc/dnf/dnf.conf
dnf install -y git-core ninja-build dnf-plugins-core python3-pip
# Configure git for various usage
git config --global user.email "gst-build@gstreamer.net"
git config --global user.name "Gstbuild Runner"
# Add rpm fusion repositories in order to access all of the gst plugins
sudo dnf install -y \
"https://mirrors.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm" \
"https://mirrors.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm"
dnf upgrade -y
# install rest of the extra deps
dnf install -y \
aalib-devel \
aom \
bat \
intel-mediasdk-devel \
libaom \
libaom-devel \
libcaca-devel \
libdav1d \
libdav1d-devel \
ccache \
cmake \
clang-devel \
elfutils \
elfutils-libs \
elfutils-devel \
gcc \
gcc-c++ \
gdb \
git-lfs \
glslc \
gtk3 \
gtk3-devel \
gtest \
gtest-devel \
graphene \
graphene-devel \
gsl \
gsl-devel \
gupnp \
gupnp-devel \
gupnp-igd \
gupnp-igd-devel \
gssdp \
gssdp-devel \
faac-devel \
ffmpeg \
ffmpeg-libs \
ffmpeg-devel \
flex \
flite \
flite-devel \
mono-devel \
procps-ng \
patch \
qt5-devel \
redhat-rpm-config \
json-glib \
json-glib-devel \
libnice \
libnice-devel \
libsodium-devel \
libunwind \
libunwind-devel \
libyaml-devel \
libxml2-devel \
libxslt-devel \
llvm-devel \
log4c-devel \
make \
nasm \
neon \
neon-devel \
nunit \
npm \
opencv \
opencv-devel \
openjpeg2 \
openjpeg2-devel \
SDL2 \
SDL2-devel \
sbc \
sbc-devel \
x264 \
x264-libs \
x264-devel \
python3 \
python3-devel \
python3-libs \
python3-gobject \
python3-cairo \
python3-cairo-devel \
valgrind \
vulkan \
vulkan-devel \
mesa-omx-drivers \
mesa-libGL \
mesa-libGL-devel \
mesa-libGLU \
mesa-libGLU-devel \
mesa-libGLES \
mesa-libGLES-devel \
mesa-libOpenCL \
mesa-libOpenCL-devel \
mesa-libgbm \
mesa-libgbm-devel \
mesa-libd3d \
mesa-libd3d-devel \
mesa-libOSMesa \
mesa-libOSMesa-devel \
mesa-vulkan-drivers \
wpewebkit \
wpewebkit-devel \
xorg-x11-server-utils \
xorg-x11-server-Xvfb
# Install common debug symbols
dnf debuginfo-install -y gtk3 \
glib2 \
glibc \
gupnp \
gupnp-igd \
gssdp \
freetype \
openjpeg \
gobject-introspection \
python3 \
python3-libs \
python3-gobject \
libappstream-glib-devel \
libjpeg-turbo \
glib-networking \
libcurl \
libsoup \
nss \
nss-softokn \
nss-softokn-freebl \
nss-sysinit \
nss-util \
openssl \
openssl-libs \
openssl-pkcs11 \
brotli \
bzip2-libs \
gpm-libs \
harfbuzz \
harfbuzz-icu \
json-c \
json-glib \
libbabeltrace \
libffi \
libsrtp \
libunwind \
mpg123-libs \
neon \
orc-compiler \
orc \
pixman \
pulseaudio-libs \
pulseaudio-libs-glib2 \
wavpack \
webrtc-audio-processing \
ffmpeg \
ffmpeg-libs \
faad2-libs \
libavdevice \
libmpeg2 \
faac \
fdk-aac \
x264 \
x264-libs \
x265 \
x265-libs \
xz \
xz-libs \
zip \
zlib
# Install the dependencies of gstreamer
dnf builddep -y gstreamer1 \
gstreamer1-plugins-base \
gstreamer1-plugins-good \
gstreamer1-plugins-good-extras \
gstreamer1-plugins-ugly \
gstreamer1-plugins-ugly-free \
gstreamer1-plugins-bad-free \
gstreamer1-plugins-bad-free-extras \
gstreamer1-plugins-bad-freeworld \
gstreamer1-libav \
gstreamer1-rtsp-server \
gstreamer1-vaapi \
python3-gstreamer1
dnf remove -y meson
pip3 install meson==0.58.0 hotdoc
# Remove gst-devel packages installed by builddep above
dnf remove -y "gstreamer1*devel"
# FIXME: Why does installing directly with dnf doesn't actually install
# the documentation files?
dnf download glib2-doc gdk-pixbuf2-devel*x86_64* gtk3-devel-docs
rpm -i --reinstall *.rpm
rm -f *.rpm
# Install Rust
RUSTUP_VERSION=1.24.2
RUST_VERSION=1.52.1
RUST_ARCH="x86_64-unknown-linux-gnu"
dnf install -y wget
RUSTUP_URL=https://static.rust-lang.org/rustup/archive/$RUSTUP_VERSION/$RUST_ARCH/rustup-init
wget $RUSTUP_URL
dnf remove -y wget
chmod +x rustup-init;
./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION;
rm rustup-init;
chmod -R a+w $RUSTUP_HOME $CARGO_HOME
rustup --version
cargo --version
rustc --version
# get gst-build and make all subprojects available
git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-build.git /gst-build/
cd /gst-build
meson subprojects download
# Run git gc to prune unwanted refs and reduce the size of the image
for i in $(find subprojects/ -mindepth 1 -maxdepth 1 -type d);
do
git -C $i gc --aggressive || true;
done

7
docker/indent/Dockerfile Normal file
View file

@ -0,0 +1,7 @@
FROM debian:stretch-slim
RUN apt update -yqq \
&& apt install -y curl indent git findutils \
&& rm -R /var/lib/apt/ /var/log/apt/ \
&& curl -L -o /usr/local/bin/gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent \
&& chmod a+x /usr/local/bin/gst-indent

View file

@ -0,0 +1,8 @@
FROM registry.fedoraproject.org/fedora:31
RUN dnf install -y \
python3-pytest \
python3-requests \
python3-pytest-cov \
git-core \
&& rm -rf /var/cache/dnf /var/log/dnf*

View file

@ -0,0 +1 @@
build_image.ps1

23
docker/windows/Dockerfile Normal file
View file

@ -0,0 +1,23 @@
# escape=`
FROM 'mcr.microsoft.com/windows/servercore:ltsc2019'
# Make sure any failure in PowerShell scripts is fatal
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"]
ENV ErrorActionPreference='Stop'
COPY install_choco.ps1 C:\
RUN C:\install_choco.ps1
RUN 'git config --global user.email "cirunner@gstreamer.freedesktop.org"; git config --global user.name "GStreamer CI system"'
COPY install_toolchain.ps1 C:\
RUN C:\install_toolchain.ps1
ARG DEFAULT_BRANCH="master"
COPY prepare_gst_env.ps1 C:\
RUN C:\prepare_gst_env.ps1
COPY prepare_cerbero_env.sh C:\
RUN C:\MinGW\msys\1.0\bin\bash.exe --login -c "C:/prepare_cerbero_env.sh"

View file

@ -0,0 +1,27 @@
$env:ErrorActionPreference='Stop'
$env:DEFAULT_BRANCH='master'
$env:VERSION='test'
$env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH"
$env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH"
Set-Location './docker/windows/'
Get-Date
Write-Output "Building $env:tag"
docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag .
if (!$?) {
Write-Host "Failed to build docker image $env:tag"
Exit 1
}
Get-Date
Write-Output "Building $env:rust_tag"
docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f rust.Dockerfile -t $env:rust_tag .
if (!$?) {
Write-Host "Failed to build docker image $env:rust_tag"
Exit 1
}
Get-Date
Write-Output "Build Finished"

View file

@ -0,0 +1,60 @@
# Copied from mesa, big kudos
#
# https://gitlab.freedesktop.org/mesa/mesa/-/blob/master/.gitlab-ci/windows/mesa_container.ps1
# https://gitlab.freedesktop.org/mesa/mesa/-/blob/34e3e164936d1d3cef267da7780e87f062fedf39/.gitlab-ci/windows/mesa_container.ps1
# Implements the equivalent of ci-templates container-ifnot-exists, using
# Docker directly as we don't have buildah/podman/skopeo available under
# Windows, nor can we execute Docker-in-Docker
$registry_uri = $args[0]
$registry_username = $args[1]
$registry_password = $args[2]
$registry_user_image = $args[3]
$registry_central_image = $args[4]
$dockerfile = $args[5]
docker --config "windows-docker.conf" login -u "$registry_username" -p "$registry_password" "$registry_uri"
if (!$?) {
Write-Host "docker login failed to $registry_uri"
Exit 1
}
# if the image already exists, don't rebuild it
docker --config "windows-docker.conf" pull "$registry_user_image"
if ($?) {
Write-Host "User image $registry_user_image already exists; not rebuilding"
docker --config "windows-docker.conf" logout "$registry_uri"
Exit 0
}
# if the image already exists upstream, copy it
docker --config "windows-docker.conf" pull "$registry_central_image"
if ($?) {
Write-Host "Copying central image $registry_central_image to user image $registry_user_image"
docker --config "windows-docker.conf" tag "$registry_central_image" "$registry_user_image"
docker --config "windows-docker.conf" push "$registry_user_image"
$pushstatus = $?
docker --config "windows-docker.conf" logout "$registry_uri"
if (!$pushstatus) {
Write-Host "Pushing image to $registry_user_image failed"
Exit 1
}
Exit 0
}
Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding"
docker --config "windows-docker.conf" build $DOCKER_BUILD_ARGS --no-cache -t "$registry_user_image" -f "$dockerfile" "./docker/windows"
if (!$?) {
Write-Host "Container build failed"
docker --config "windows-docker.conf" logout "$registry_uri"
Exit 1
}
Get-Date
docker --config "windows-docker.conf" push "$registry_user_image"
$pushstatus = $?
docker --config "windows-docker.conf" logout "$registry_uri"
if (!$pushstatus) {
Write-Host "Pushing image to $registry_user_image failed"
Exit 1
}

View file

@ -0,0 +1,23 @@
Get-Date
Write-Host "Installing Chocolatey"
Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
Import-Module "$env:ProgramData\chocolatey\helpers\chocolateyProfile.psm1"
Update-SessionEnvironment
choco install -y vcredist140
$vc140_install = $?
Write-Host "Installing Chocolatey packages"
choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
$cmake_install = $?
choco install -y git --params "/NoAutoCrlf /NoCredentialManager /NoShellHereIntegration /NoGuiHereIntegration /NoShellIntegration"
$git_install = $?
choco install -y python3 git-lfs 7zip
$rest_installs = $?
if (!($vc140_install -and $cmake_install -and $git_install -and $rest_installs)) {
Write-Host "Failed to install some dependencies from choco"
Exit 1
}

View file

@ -0,0 +1,71 @@
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
# Download gst-build and all its subprojects
# git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
# FIXME: need 1.19+ for cairo subproject :/
# Should use a stable branch instead
git clone -b master --depth 1 https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
if (!$?) {
Write-Host "Failed to clone gst-build"
Exit 1
}
Set-Location C:\gst-build
# Copy the cache we already have in the image to avoid massive redownloads
Move-Item C:/subprojects/* C:\gst-build\subprojects
if (!$?) {
Write-Host "Failed to copy subprojects cache"
Exit 1
}
# Update the subprojects cache
Write-Output "Running meson subproject reset"
meson subprojects update --reset
if (!$?) {
Write-Host "Failed to reset subprojects state"
Exit 1
}
Write-Output "Running git update"
python git-update --no-interaction
if (!$?) {
Write-Host "Failed to run git-update"
Exit 1
}
$env:MESON_ARGS = "-Dglib:installed_tests=false " +
"-Dlibnice:tests=disabled " +
"-Dlibnice:examples=disabled " +
"-Dffmpeg:tests=disabled " +
"-Dopenh264:tests=disabled " +
"-Dpygobject:tests=false " +
"-Dugly=enabled " +
"-Dbad=enabled " +
"-Dges=enabled " +
"-Drtsp_server=enabled " +
"-Ddevtools=enabled " +
"-Dsharp=disabled " +
"-Dpython=disabled " +
"-Dlibav=disabled " +
"-Dvaapi=disabled " +
"-Dgst-plugins-base:pango=enabled " +
"-Dgst-plugins-good:cairo=enabled "
Write-Output "Building gst"
cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && meson _build $env:MESON_ARGS && meson compile -C _build && ninja -C _build install"
if (!$?) {
Write-Host "Failed to build and install gst"
Exit 1
}
git clean -fdxx
if (!$?) {
Write-Host "Failed to git clean"
Exit 1
}

View file

@ -0,0 +1,69 @@
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
$msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe'
$msys2_url = 'https://github.com/msys2/msys2-installer/releases/download/2021-02-15/msys2-base-x86_64-20210215.tar.xz'
$msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz'
Get-Date
Write-Host "Downloading Visual Studio 2017 build tools"
Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe
Get-Date
Write-Host "Installing Visual Studio 2017"
Start-Process -NoNewWindow -Wait C:\vs_buildtools.exe -ArgumentList '--wait --quiet --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended'
if (!$?) {
Write-Host "Failed to install Visual Studio tools"
Exit 1
}
Remove-Item C:\vs_buildtools.exe -Force
Get-Date
Write-Host "Downloading and extracting mingw-get for MSYS"
Invoke-WebRequest -Uri $msys_mingw_get_url -OutFile C:\mingw-get.tar.xz
7z e C:\mingw-get.tar.xz -o"C:\\"
$res1 = $?
7z x C:\mingw-get.tar -o"C:\\MinGW"
$res2 = $?
if (!($res1 -and $res2)) {
Write-Host "Failed to extract mingw-get"
Exit 1
}
Remove-Item C:\mingw-get.tar.xz -Force
Remove-Item C:\mingw-get.tar -Force
Get-Date
Write-Host "Installing MSYS for Cerbero into C:/MinGW using mingw-get"
Start-Process -Wait C:\MinGW\bin\mingw-get.exe -ArgumentList 'install msys-base mingw32-base mingw-developer-toolkit'
if (!$?) {
Write-Host "Failed to install Msys for cerbero using MinGW"
Exit 1
}
Get-Date
Write-Host "Installing MSYS2 into C:/msys64"
Invoke-WebRequest -Uri $msys2_url -OutFile C:\msys2-x86_64.tar.xz
7z e C:\msys2-x86_64.tar.xz -o"C:\\"
$res1 = $?
7z x C:\msys2-x86_64.tar -o"C:\\"
$res2 = $?
if (!($res1 -and $res2)) {
Write-Host "Failed to extract msys2"
Exit 1
}
Remove-Item C:\msys2-x86_64.tar.xz -Force
Remove-Item C:\msys2-x86_64.tar -Force
Get-Date
Write-Host "Installing Meson"
pip3 install meson
if (!$?) {
Write-Host "Failed to install meson from pip"
Exit 1
}
Write-Host "Toolchain Install Complete"

View file

@ -0,0 +1,27 @@
#! /bin/bash
set -eux
cd C:/
git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git
cd cerbero
echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc
echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc
echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc
echo 'vs_install_version = "vs15"' >> localconf.cbc
# Fetch all bootstrap requirements
./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-bootstrap --jobs=4
# Fetch all package requirements for a mingw gstreamer build
./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-package --jobs=4 gstreamer-1.0
# Fetch all package requirements for a visualstudio gstreamer build
./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package --jobs=4 gstreamer-1.0
# Fixup the MSYS installation
./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline
# Wipe visualstudio package dist, sources, logs, and the build tools recipes
./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc wipe --force --build-tools
# clean the localconf
rm /c/cerbero/localconf.cbc

View file

@ -0,0 +1,24 @@
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
# Download gst-build and all its subprojects
git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
if (!$?) {
Write-Host "Failed to clone gst-build"
Exit 1
}
# download the subprojects to try and cache them
meson subprojects download --sourcedir C:\gst-build
if (!$?) {
Write-Host "Failed to download the subprojects"
Exit 1
}
# Remove files that will conflict with a fresh clone on the runner side
Remove-Item -Force 'C:/gst-build/subprojects/*.wrap'
Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-nasm'
Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-flex-bison-binaries'
Remove-Item -Recurse -Force 'C:/gst-build/subprojects/macos-bison-binary'
Move-Item C:\gst-build\subprojects C:\subprojects
Remove-Item -Recurse -Force C:\gst-build

View file

@ -0,0 +1,21 @@
# escape=`
# Expect this to be set when calling docker build with
# --build-arg BASE_IMAGE="" and make it fail if not set.
ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid"
FROM $BASE_IMAGE
ARG DEFAULT_BRANCH="master"
ARG RUST_VERSION="invalid"
COPY install_gst.ps1 C:\
RUN C:\install_gst.ps1
RUN choco install -y pkgconfiglite
ENV PKG_CONFIG_PATH="C:/lib/pkgconfig"
ADD https://win.rustup.rs/x86_64 C:\rustup-init.exe
RUN C:\rustup-init.exe -y --profile minimal --default-toolchain $env:RUST_VERSION
# Uncomment for easy testing
# RUN git clone --depth 1 https://gitlab.freedesktop.org/gstreamer/gstreamer-rs.git
# RUN cd gstreamer-rs; cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64; cargo build --all; cargo test --all"

80
fuzzing/README.txt Normal file
View file

@ -0,0 +1,80 @@
Fuzzing GStreamer
=================
This directory contains the various fuzzing targets and helper
scripts.
* Fuzzing targets
Fuzzing targets as small applications where we can test a specific
element or API. The goal is to have them be as small/targetted as
possible.
ex: appsrc ! <some_element> ! fakesink num-buffers=<small>
Not all components can be tested directly and therefore will be
indirectly tested via other targets (ex: libgstaudio will be tested
by targets/elements requiring it)
Anything that can process externally-provided data should be
covered, but there are cases where it might not make sense to use a
fuzzer (such as most elements processing raw audio/video).
* build-oss-fuzz.sh
This is the script executed by the oss-fuzz project.
It builds glib, GStreamer, plugins and the fuzzing targets.
* *.c
The fuzzing targets where the data to test will be provided to a
function whose signature follows the LibFuzzer signature:
https://llvm.org/docs/LibFuzzer.html
* TODO
* Add a standalone build script
We need to be able to build and test the fuzzing targets outside
of the oss-fuzz infrastructure, and do that in our continous
integration system.
We need:
* A dummy fuzzing engine (given a directory, it opens all files and
calls the fuzzing targets with the content of those files.
* A script to be able to build those targets with that dummy engine
* A corpus of files to test those targets with.
* Build targets with dummy engine and run with existing tests.
* Create pull-based variants
Currently the existing targets are push-based only. Where
applicable we should make pull-based variants to test the other
code paths.
* Add more targets
core:
gst_parse fuzzer ?
base:
ext/
ogg
opus
pango
theora
vorbis
gst/
subparse
typefind : already covered in typefind target
gst-libs/gst/
sdp
other ones easily testable directly ?

185
fuzzing/build-oss-fuzz.sh Executable file
View file

@ -0,0 +1,185 @@
#!/bin/bash -eu
# build-oss-fuzz.sh
#
# Build script which is executed by oss-fuzz build.sh
#
# $SRC: location of code checkouts
# $OUT: location to put fuzzing targets and corpus
# $WORK: writable directory where all compilation should be executed
#
# /!\ Do not override any CC, CXX, CFLAGS, ... variables
#
# This script is divided in two parts
#
# 1) Build all the dependencies statically
#
# 2) Build the fuzzing targets
# Prefix where we will temporarily install everything
PREFIX=$WORK/prefix
mkdir -p $PREFIX
# always try getting the arguments for static compilation/linking
# Fixes GModule not being picked when gstreamer-1.0.pc is looked up by meson
# more or less https://github.com/mesonbuild/meson/pull/6629
export PKG_CONFIG="`which pkg-config` --static"
export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig
export PATH=$PREFIX/bin:$PATH
# Minimize gst-debug level/code
export CFLAGS="$CFLAGS -DGST_LEVEL_MAX=2"
#
echo "CFLAGS : " $CFLAGS
echo "CXXFLAGS : " $CXXFLAGS
PLUGIN_DIR=$PREFIX/lib/gstreamer-1.0
rm -rf $WORK/*
# Switch to work directory
cd $WORK
# 1) BUILD GLIB AND GSTREAMER
# Note: we build glib ourselves so that we get proper malloc/free backtraces
tar xvJf $SRC/glib-2.64.2.tar.xz
cd glib-2.64.2
# options taken from glib's oss-fuzz build definition
meson \
--prefix=$PREFIX \
--libdir=lib \
--default-library=static \
-Db_lundef=false \
-Doss_fuzz=enabled \
-Dlibmount=disabled \
-Dinternal_pcre=true \
_builddir
ninja -C _builddir
ninja -C _builddir install
cd ..
# Note: We don't use/build orc since it still seems to be problematic
# with clang and the various sanitizers.
# For now we only build core and base. Add other modules when/if needed
for i in gstreamer gst-plugins-base;
do
mkdir -p $i
cd $i
meson \
--prefix=$PREFIX \
--libdir=lib \
--default-library=static \
-Db_lundef=false \
-Ddoc=disabled \
-Dexamples=disabled \
-Dintrospection=disabled \
-Dtracer_hooks=false \
-Dregistry=false _builddir $SRC/$i
ninja -C _builddir
ninja -C _builddir install
cd ..
done
# 2) Build the target fuzzers
# All targets will be linked in with $LIB_FUZZING_ENGINE which contains the
# actual fuzzing runner. Anything fuzzing engine can be used provided it calls
# the same function as libfuzzer.
# Note: The fuzzer .o needs to be first compiled with CC and then linked with CXX
# We want to statically link everything, except for shared libraries
# that are present on the base image. Those need to be specified
# beforehand and explicitely linked dynamically If any of the static
# dependencies require a pre-installed shared library, you need to add
# that library to the following list
PREDEPS_LDFLAGS="-Wl,-Bdynamic -ldl -lm -pthread -lrt -lpthread"
# These are the basic .pc dependencies required to build any of the fuzzing targets
# That is : glib, gstreamer core and gst-app
# The extra target-specific dependencies are to be specified later
COMMON_DEPS="glib-2.0 gio-2.0 gstreamer-1.0 gstreamer-app-1.0"
# For each target, defined the following:
# TARGET_DEPS : Extra .pc dependencies for the target (in addition to $COMMON_DEPS)
# All dependencies (including sub-dependencies) must be speecified
# PLUGINS : .a of the plugins to link
# They must match the static plugins declared/registered in the target
#
# TARGET : push-based ogg/theora/vorbis discoverer
#
# FIXME : Rename to discoverer_push_oggtheoravorbis
TARGET_DEPS=" gstreamer-pbutils-1.0 \
gstreamer-video-1.0 \
gstreamer-audio-1.0 \
gstreamer-riff-1.0 \
gstreamer-tag-1.0 \
zlib ogg vorbis vorbisenc \
theoraenc theoradec theora"
PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \
$PLUGIN_DIR/libgsttypefindfunctions.a \
$PLUGIN_DIR/libgstplayback.a \
$PLUGIN_DIR/libgstapp.a \
$PLUGIN_DIR/libgstvorbis.a \
$PLUGIN_DIR/libgsttheora.a \
$PLUGIN_DIR/libgstogg.a"
echo
echo ">>>> BUILDING gst-discoverer"
echo
BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`"
BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`"
$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/gst-discoverer.c -o $SRC/gst-ci/fuzzing/gst-discoverer.o
$CXX $CXXFLAGS \
-o $OUT/gst-discoverer \
$PREDEPS_LDFLAGS \
$SRC/gst-ci/fuzzing/gst-discoverer.o \
$PLUGINS \
$BUILD_LDFLAGS \
$LIB_FUZZING_ENGINE \
-Wl,-Bdynamic
#
# TARGET : push-based typefind
#
# typefindfunction depends on pbutils which depends on gst{audio|video|tag}
TARGET_DEPS=" gstreamer-pbutils-1.0 \
gstreamer-video-1.0 \
gstreamer-audio-1.0 \
gstreamer-tag-1.0"
PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \
$PLUGIN_DIR/libgsttypefindfunctions.a \
$PLUGIN_DIR/libgstapp.a"
echo
echo ">>>> BUILDING typefind"
echo
BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`"
BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`"
$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/typefind.c -o $SRC/gst-ci/fuzzing/typefind.o
$CXX $CXXFLAGS \
-o $OUT/typefind \
$PREDEPS_LDFLAGS \
$SRC/gst-ci/fuzzing/typefind.o \
$PLUGINS \
$BUILD_LDFLAGS \
$LIB_FUZZING_ENGINE \
-Wl,-Bdynamic
echo
echo ">>>> Installing seed corpus"
echo
# FIXME : Sadly we apparently need to have the corpus downloaded in the
# Dockerfile and not here.
cp $SRC/*_seed_corpus.zip $OUT

137
fuzzing/gst-discoverer.c Normal file
View file

@ -0,0 +1,137 @@
/*
* Copyright 2016 Google Inc.
* author: Edward Hervey <bilboed@bilboed.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <locale.h>
#include <stdlib.h>
#include <glib.h>
#include <gst/gst.h>
#include <gst/pbutils/pbutils.h>
#ifndef LOCAL_FUZZ_BUILD
GST_PLUGIN_STATIC_DECLARE (coreelements);
GST_PLUGIN_STATIC_DECLARE (playback);
GST_PLUGIN_STATIC_DECLARE (typefindfunctions);
GST_PLUGIN_STATIC_DECLARE (app);
GST_PLUGIN_STATIC_DECLARE (ogg);
GST_PLUGIN_STATIC_DECLARE (theora);
GST_PLUGIN_STATIC_DECLARE (vorbis);
#endif
/* push-based discoverer fuzzing target
*
* This application can be compiled with libFuzzer to simulate
* a push-based discoverer execution.
*
* To reproduce the failing behaviour, use:
* $ gst-discoverer-1.0 pushfile:///...
*
* The goal is to cover basic usage of demuxers, parsers and
* base decoder elements.
*
* When compiling, only link the required demuxer/parser/decoder
* plugins and keep it to a limited range (ex: ogg/theora/vorbis)
*
**/
const guint8 *fuzztesting_data;
size_t fuzztesting_size;
static void
appsrc_configuration (GstDiscoverer * dc, GstElement * source, gpointer data)
{
GstBuffer *buf;
GstFlowReturn ret;
/* Create buffer from fuzztesting_data which shouldn't be freed */
buf =
gst_buffer_new_wrapped_full (0, (gpointer) fuzztesting_data,
fuzztesting_size, 0, fuzztesting_size, NULL, NULL);
g_object_set (G_OBJECT (source), "size", fuzztesting_size, NULL);
g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &ret);
gst_buffer_unref (buf);
}
static void
custom_logger (const gchar * log_domain,
GLogLevelFlags log_level, const gchar * message, gpointer unused_data)
{
if (log_level & G_LOG_LEVEL_CRITICAL) {
g_printerr ("CRITICAL ERROR : %s\n", message);
abort ();
} else if (log_level & G_LOG_LEVEL_WARNING) {
g_printerr ("WARNING : %s\n", message);
}
}
int
LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
{
GError *err = NULL;
GstDiscoverer *dc;
gint timeout = 10;
GstDiscovererInfo *info;
static gboolean initialized = FALSE;
if (!initialized) {
/* We want critical warnings to assert so we can fix them */
g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL);
g_log_set_default_handler (custom_logger, NULL);
/* Only initialize and register plugins once */
gst_init (NULL, NULL);
#ifndef LOCAL_FUZZ_BUILD
GST_PLUGIN_STATIC_REGISTER (coreelements);
GST_PLUGIN_STATIC_REGISTER (playback);
GST_PLUGIN_STATIC_REGISTER (typefindfunctions);
GST_PLUGIN_STATIC_REGISTER (app);
GST_PLUGIN_STATIC_REGISTER (ogg);
GST_PLUGIN_STATIC_REGISTER (theora);
GST_PLUGIN_STATIC_REGISTER (vorbis);
#endif
initialized = TRUE;
}
dc = gst_discoverer_new (timeout * GST_SECOND, &err);
if (G_UNLIKELY (dc == NULL)) {
g_print ("Error initializing: %s\n", err->message);
g_clear_error (&err);
exit (1);
}
fuzztesting_data = data;
fuzztesting_size = size;
/* Connect to source-setup signal to give the data */
g_signal_connect (dc, "source-setup", (GCallback) appsrc_configuration, NULL);
info = gst_discoverer_discover_uri (dc, "appsrc://", &err);
g_clear_error (&err);
if (info)
gst_discoverer_info_unref (info);
g_object_unref (dc);
return 0;
}

78
fuzzing/localfuzzer.c Normal file
View file

@ -0,0 +1,78 @@
/* GStreamer
* Copyright (C) 2017 Edward Hervey <bilboed@bilboed.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/* Local fuzzer runner */
#include <glib.h>
extern int LLVMFuzzerTestOneInput (const guint8 * data, size_t size);
static void
test_file (gchar * filename)
{
GDir *dir;
gchar *path;
gchar *contents;
gsize length;
/* if filename is a directory, process the contents */
if ((dir = g_dir_open (filename, 0, NULL))) {
const gchar *entry;
while ((entry = g_dir_read_name (dir))) {
gchar *spath;
spath = g_strconcat (filename, G_DIR_SEPARATOR_S, entry, NULL);
test_file (spath);
g_free (spath);
}
g_dir_close (dir);
return;
}
/* Make sure path is absolute */
if (!g_path_is_absolute (filename)) {
gchar *curdir;
curdir = g_get_current_dir ();
path = g_build_filename (curdir, filename, NULL);
g_free (curdir);
} else
path = g_strdup (filename);
/* Check if path exists */
if (g_file_get_contents (path, &contents, &length, NULL)) {
g_print (">>> %s (%" G_GSIZE_FORMAT " bytes)\n", path, length);
LLVMFuzzerTestOneInput ((const guint8 *) contents, length);
g_free (contents);
}
g_free (path);
}
int
main (int argc, gchar ** argv)
{
gint i;
for (i = 1; i < argc; i++)
test_file (argv[i]);
return 0;
}

115
fuzzing/typefind.c Normal file
View file

@ -0,0 +1,115 @@
/*
* Copyright 2016 Google Inc.
* author: Edward Hervey <bilboed@bilboed.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <locale.h>
#include <stdlib.h>
#include <glib.h>
#include <gst/gst.h>
#ifndef LOCAL_FUZZ_BUILD
GST_PLUGIN_STATIC_DECLARE (coreelements);
GST_PLUGIN_STATIC_DECLARE (typefindfunctions);
GST_PLUGIN_STATIC_DECLARE (app);
#endif
/* push-based typefind fuzzing target
*
* This application can be compiled with libFuzzer to simulate
* a push-based typefind execution.
*
* To reproduce the failing behaviour, use:
* $ gst-launch-1.0 pushfile:///.. ! typefind ! fakesink
*
* The goal is to cover typefind code and implementation.
*
**/
static void
custom_logger (const gchar * log_domain,
GLogLevelFlags log_level, const gchar * message, gpointer unused_data)
{
if (log_level & G_LOG_LEVEL_CRITICAL) {
g_printerr ("CRITICAL ERROR : %s\n", message);
abort ();
} else if (log_level & G_LOG_LEVEL_WARNING) {
g_printerr ("WARNING : %s\n", message);
}
}
int
LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
{
GError *err = NULL;
static gboolean initialized = FALSE;
GstElement *pipeline, *source, *typefind, *fakesink;
GstBuffer *buf;
GstFlowReturn flowret;
GstState state;
if (!initialized) {
/* We want critical warnings to assert so we can fix them */
g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL);
g_log_set_default_handler (custom_logger, NULL);
/* Only initialize and register plugins once */
gst_init (NULL, NULL);
#ifndef LOCAL_FUZZ_BUILD
GST_PLUGIN_STATIC_REGISTER (coreelements);
GST_PLUGIN_STATIC_REGISTER (typefindfunctions);
GST_PLUGIN_STATIC_REGISTER (app);
#endif
initialized = TRUE;
}
/* Create the pipeline */
pipeline = gst_pipeline_new ("pipeline");
source = gst_element_factory_make ("appsrc", "source");
typefind = gst_element_factory_make ("typefind", "typefind");
fakesink = gst_element_factory_make ("fakesink", "fakesink");
gst_bin_add_many (GST_BIN (pipeline), source, typefind, fakesink, NULL);
gst_element_link_many (source, typefind, fakesink, NULL);
/* Set pipeline to READY so we can provide data to appsrc */
gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_READY);
buf = gst_buffer_new_wrapped_full (0, (gpointer) data, size,
0, size, NULL, NULL);
g_object_set (G_OBJECT (source), "size", size, NULL);
g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &flowret);
gst_buffer_unref (buf);
/* Set pipeline to PAUSED and wait (typefind will either fail or succeed) */
gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_PAUSED);
/* wait until state change either completes or fails */
gst_element_get_state (GST_ELEMENT (pipeline), &state, NULL, -1);
/* Go back to NULL */
gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_NULL);
/* And release the pipeline */
gst_object_unref (pipeline);
return 0;
}

270
gitlab/build_manifest.py Executable file
View file

@ -0,0 +1,270 @@
#!/usr/bin/env python3
import argparse
import os
import sys
import subprocess
import urllib.error
import urllib.parse
import urllib.request
import json
from typing import Dict, Tuple, List
# from pprint import pprint
if sys.version_info < (3, 6):
raise SystemExit('Need Python 3.6 or newer')
GSTREAMER_MODULES: List[str] = [
'orc',
'cerbero',
'gst-build',
'gstreamer',
'gst-plugins-base',
'gst-plugins-good',
'gst-plugins-bad',
'gst-plugins-ugly',
'gst-libav',
'gst-devtools',
'gst-docs',
'gst-editing-services',
'gst-omx',
'gst-python',
'gst-rtsp-server',
'gstreamer-sharp',
'gstreamer-vaapi',
'gst-integration-testsuites',
'gst-examples',
]
MANIFEST_TEMPLATE: str = """<?xml version="1.0" encoding="UTF-8"?>
<manifest>
<remote fetch="{}" name="user"/>
<remote fetch="https://gitlab.freedesktop.org/gstreamer/" name="origin"/>
{}
</manifest>"""
CERBERO_DEPS_LOGS_TARGETS = (
('cross-ios', 'universal'),
('cross-windows-mingw', 'x86'),
('cross-windows-mingw', 'x86_64'),
('cross-android', 'universal'),
('fedora', 'x86_64'),
('macos', 'x86_64'),
('windows-msvc', 'x86_64'),
)
# Disallow git prompting for a username/password
os.environ['GIT_TERMINAL_PROMPT'] = '0'
def git(*args, repository_path='.'):
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
def get_cerbero_last_build_info (branch : str):
# Fetch the deps log for all (distro, arch) targets
all_commits = {}
for distro, arch in CERBERO_DEPS_LOGS_TARGETS:
url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/{distro}/{arch}/cerbero-deps.log'
print(f'Fetching {url}')
try:
req = urllib.request.Request(url)
resp = urllib.request.urlopen(req);
deps = json.loads(resp.read())
except urllib.error.URLError as e:
print(f'WARNING: Failed to GET {url}: {e!s}')
continue
for dep in deps:
commit = dep['commit']
if commit not in all_commits:
all_commits[commit] = []
all_commits[commit].append((distro, arch))
# Fetch the cerbero commit that has the most number of caches
best_commit = None
newest_commit = None
max_caches = 0
total_caches = len(CERBERO_DEPS_LOGS_TARGETS)
for commit, targets in all_commits.items():
if newest_commit is None:
newest_commit = commit
have_caches = len(targets)
# If this commit has caches for all targets, just use it
if have_caches == total_caches:
best_commit = commit
break
# Else, try to find the commit with the most caches
if have_caches > max_caches:
max_caches = have_caches
best_commit = commit
if newest_commit is None:
print('WARNING: No deps logs were found, will build from scratch')
if best_commit != newest_commit:
print(f'WARNING: Cache is not up-to-date for commit {newest_commit}, using commit {best_commit} instead')
return best_commit
def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]:
try:
res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch)
except subprocess.CalledProcessError:
return None, None
if not res:
return None, None
# Special case cerbero to avoid cache misses
if module == 'cerbero':
sha = get_cerbero_last_build_info(branch)
if sha is not None:
return sha, sha
lines = res.split('\n')
for line in lines:
if line.endswith('/' + branch):
try:
sha, refname = line.split('\t')
except ValueError:
continue
return refname.strip(), sha
return None, None
def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]:
namespace: str = os.environ["CI_PROJECT_NAMESPACE"]
ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master')
if module == "orc":
ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master')
if module == os.environ['CI_PROJECT_NAME']:
return 'user', branchname, os.environ['CI_COMMIT_SHA']
if branchname != ups_branch:
remote_refname, sha = get_branch_info(module, namespace, branchname)
if sha is not None:
return 'user', remote_refname, sha
# Check upstream project for a branch
remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch)
if sha is not None:
return 'origin', remote_refname, sha
# This should never occur given the upstream fallback above
print(f"Could not find anything for {module}:{branchname}")
print("If something reaches that point, please file a bug")
print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues")
assert False
# --- Unit tests --- #
# Basically, pytest will happily let a test mutate a variable, and then run
# the next tests one the same environment without reset the vars.
def preserve_ci_vars(func):
"""Preserve the original CI Variable values"""
def wrapper():
try:
url = os.environ["CI_PROJECT_URL"]
user = os.environ["CI_PROJECT_NAMESPACE"]
except KeyError:
url = "invalid"
user = ""
private = os.getenv("READ_PROJECTS_TOKEN", default=None)
if not private:
os.environ["READ_PROJECTS_TOKEN"] = "FOO"
func()
os.environ["CI_PROJECT_URL"] = url
os.environ["CI_PROJECT_NAMESPACE"] = user
if private:
os.environ["READ_PROJECTS_TOKEN"] = private
# if it was set after
elif os.getenv("READ_PROJECTS_TOKEN", default=None):
del os.environ["READ_PROJECTS_TOKEN"]
return wrapper
@preserve_ci_vars
def test_find_repository_sha():
os.environ["CI_PROJECT_NAME"] = "some-random-project"
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
os.environ["CI_PROJECT_NAMESPACE"] = "alatiera"
os.environ["GST_UPSTREAM_BRANCH"] = "master"
del os.environ["READ_PROJECTS_TOKEN"]
# This should find the repository in the user namespace
remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2")
assert remote == "user"
assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325"
assert refname == "refs/heads/1.2"
# This should fallback to upstream master branch since no matching branch was found
remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name")
assert remote == "origin"
assert refname == "refs/heads/master"
os.environ["CI_PROJECT_NAME"] = "the_project"
os.environ["CI_COMMIT_SHA"] = "MySha"
remote, refname, git_ref = find_repository_sha("the_project", "whatever")
assert remote == "user"
assert git_ref == "MySha"
assert refname == "whatever"
@preserve_ci_vars
def test_get_project_branch():
os.environ["CI_PROJECT_NAME"] = "some-random-project"
os.environ["CI_COMMIT_SHA"] = "dwbuiw"
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_"
del os.environ["READ_PROJECTS_TOKEN"]
os.environ['GST_UPSTREAM_BRANCH'] = '1.12'
remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12')
assert twelve is not None
assert remote == 'origin'
assert refname == "refs/heads/1.12"
os.environ['GST_UPSTREAM_BRANCH'] = '1.14'
remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14')
assert fourteen is not None
assert remote == 'origin'
assert refname == "refs/heads/1.14"
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--self-update", action="store_true", default=False)
parser.add_argument(dest="output", default='manifest.xml', nargs='?')
options = parser.parse_args()
current_branch: str = os.environ['CI_COMMIT_REF_NAME']
user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL'])
if not user_remote_url.endswith('/'):
user_remote_url += '/'
if options.self_update:
remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch)
if remote == 'user':
remote = user_remote_url + 'gst-ci'
else:
remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci"
git('fetch', remote, remote_refname)
git('checkout', '--detach', sha)
sys.exit(0)
projects: str = ''
for module in GSTREAMER_MODULES:
print(f"Checking {module}:", end=' ')
remote, refname, revision = find_repository_sha(module, current_branch)
print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'")
projects += f" <project path=\"{module}\" name=\"{module}.git\" remote=\"{remote}\" revision=\"{revision}\" refname=\"{refname}\" />\n"
with open(options.output, mode='w') as manifest:
print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest)

169
gitlab/cerbero_setup.sh Normal file
View file

@ -0,0 +1,169 @@
#!/bin/bash
set -ex
show_ccache_sum() {
if [[ -n ${HAVE_CCACHE} ]]; then
ccache -s
fi
}
# XXX: This is copied and modified from the cerbero-uninstalled script
# Use `mount` to get a list of MSYS mount points that the MSYS shell uses.
# That's our reference point for translating from MSYS paths to Win32 paths.
# We assume that the MSYS mount point directories are only in the filesystem
# root. This will break if people add their own custom mount points beyond what
# MSYS automatically creates, which is highly unlikely.
#
# /d -> d:/
# /c -> c:/
# /d/projects/cerbero -> d:/projects/cerbero/
# /home/USERNAME/cerbero -> C:\\MinGW\\msys\\1.0/home/USERNAME/
# /mingw -> C:\\MinGW/
# /mingw/bin/foobar -> C:\\MinGW\\bin/foobar/
# /tmp/baz -> C:\\Users\\USERNAME\\AppData\\Local\\Temp/baz/
msys_dir_to_win32() {
set -e
local msys_path stripped_path mount_point path mounted_path
# If the path is already a native path, just return that
if [[ $1 == ?:/* ]] || [[ $1 == ?:\\* ]]; then
echo $1
return
fi
# Convert /c or /mingw etc to /c/ or /mingw/ etc; gives us a necessary
# anchor to split the path into components
msys_path="$1/"
# Strip leading slash
stripped_path="${msys_path#/}"
# Get the first path component, which may be a mount point
mount_point="/${stripped_path%%/*}"
# Get the path inside the mountp oint
path="/${stripped_path#*/}"
mounted_path="$(mount | sed -n "s|\(.*\) on $mount_point type.*|\1|p")"
# If it's not a mounted path (like /c or /tmp or /mingw), then it's in the
# general MSYS root mount
if [[ -z $mounted_path ]]; then
mounted_path="$(mount | sed -n "s|\(.*\) on / type.*|\1|p")"
path="$1"
fi
echo ${mounted_path}${path%/}
}
# Print the working directory in the native OS path format, but with forward
# slashes
pwd_native() {
if [[ -n "$MSYSTEM" ]]; then
msys_dir_to_win32 "$(pwd)"
else
pwd
fi
}
fix_build_tools() {
if [[ $(uname) == Darwin ]]; then
# Bison needs these env vars set for the build-tools prefix to be
# relocatable, and we only build it on macOS. On Linux we install it
# using the package manager, and on Windows we use the MSYS Bison.
export M4="$(pwd)/${CERBERO_HOME}/build-tools/bin/m4"
export BISON_PKGDATADIR="$(pwd)/${CERBERO_HOME}/build-tools/share/bison"
fi
}
# Produces runtime and devel tarball packages for linux/android or .pkg for macos
cerbero_package_and_check() {
$CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0
# Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course.
if [[ $CONFIG != *universal* ]]; then
$CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version
$CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX
fi
show_ccache_sum
}
cerbero_before_script() {
pwd
ls -lha
# Copy cerbero git repo stored on the image
cp -a "${CERBERO_HOST_DIR}/.git" .
git checkout .
git status
# If there's no cerbero-sources directory in the runner cache, copy it from
# the image cache
if ! [[ -d ${CERBERO_SOURCES} ]]; then
time cp -a "${CERBERO_HOST_DIR}/${CERBERO_SOURCES}" .
fi
du -sch "${CERBERO_SOURCES}" || true
echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc
echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc
if [[ $CONFIG == win??.cbc ]]; then
# Visual Studio 2017 build tools install path
echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc
echo 'vs_install_version = "vs15"' >> localconf.cbc
fi
cat localconf.cbc
time ./cerbero-uninstalled --self-update manifest.xml
# GitLab runner does not always wipe the image after each job, so do that
# to ensure we don't have any leftover data from a previous job such as
# a dirty builddir, or tarballs/pkg files, leftover files from an old
# cerbero commit, etc. Skip the things we actually need to keep.
time git clean -xdff -e cerbero_setup.sh -e manifest.xml -e localconf.cbc -e "${CERBERO_SOURCES}"
}
cerbero_script() {
show_ccache_sum
$CERBERO $CERBERO_ARGS show-config
$CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4
$CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0
du -sch "${CERBERO_SOURCES}" || true
$CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}"
if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} && -d "${CERBERO_HOME}/dist/${ARCH}" ]]; then
mkdir -p "${CERBERO_OVERRIDDEN_DIST_DIR}"
time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}"
fi
$CERBERO $CERBERO_ARGS bootstrap --offline --system=no
fix_build_tools
cerbero_package_and_check
}
cerbero_deps_script() {
show_ccache_sum
$CERBERO $CERBERO_ARGS show-config
$CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4
$CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0
$CERBERO $CERBERO_ARGS bootstrap --offline --system=no
$CERBERO $CERBERO_ARGS build-deps --offline \
gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \
gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \
gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice
if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then
mkdir -p "${CERBERO_HOME}/dist/${ARCH}"
time rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}"
fi
# Check that the env var is set. Don't expand this protected variable by
# doing something silly like [[ -n ${CERBERO_...} ]] because it will get
# printed in the CI logs due to set -x
if env | grep -q -e CERBERO_PRIVATE_SSH_KEY; then
time $CERBERO $CERBERO_ARGS gen-cache --branch "${GST_UPSTREAM_BRANCH}"
time $CERBERO $CERBERO_ARGS upload-cache --branch "${GST_UPSTREAM_BRANCH}"
fi
cerbero_package_and_check
}
# Run whichever function is asked of us
eval "$1"

1427
gitlab/ci_template.yml Normal file

File diff suppressed because it is too large Load diff

92
gitlab/clone_manifest_ref.py Executable file
View file

@ -0,0 +1,92 @@
#!/usr/bin/env python3
import argparse
import os
import subprocess
from collections import namedtuple
import xml.etree.ElementTree as ET
# Disallow git prompting for a username/password
os.environ['GIT_TERMINAL_PROMPT'] = '0'
def git(*args, repository_path='.'):
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
class Manifest(object):
'''
Parse and store the content of a manifest file
'''
remotes = {}
projects = {}
default_remote = 'origin'
default_revision = 'refs/heads/master'
def __init__(self, manifest_path):
self.manifest_path = manifest_path
def parse(self):
try:
tree = ET.parse(self.manifest_path)
except Exception as ex:
raise Exception("Error loading manifest %s in file %s" % (self.manifest_path, ex))
root = tree.getroot()
for child in root:
if child.tag == 'remote':
self.remotes[child.attrib['name']] = child.attrib['fetch']
if child.tag == 'default':
self.default_remote = child.attrib['remote'] or self.default_remote
self.default_revision = child.attrib['revision'] or self.default_revision
if child.tag == 'project':
project = namedtuple('Project', ['name', 'remote',
'revision', 'fetch_uri'])
project.name = child.attrib['name']
if project.name.endswith('.git'):
project.name = project.name[:-4]
project.remote = child.attrib.get('remote') or self.default_remote
project.revision = child.attrib.get('revision') or self.default_revision
project.fetch_uri = self.remotes[project.remote] + project.name + '.git'
self.projects[project.name] = project
def find_project(self, name):
try:
return self.projects[name]
except KeyError as ex:
raise Exception("Could not find project %s in manifest %s" % (name, self.manifest_path))
def get_fetch_uri(self, project, remote):
fetch = self.remotes[remote]
return fetch + project.name + '.git'
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--project", action="store", type=str)
parser.add_argument("--destination", action="store", type=str, default='.')
parser.add_argument("--manifest", action="store", type=str)
parser.add_argument("--fetch", action="store_true", default=False)
options = parser.parse_args()
if not options.project:
raise ValueError("--project argument not provided")
if not options.manifest:
raise ValueError("--manifest argument not provided")
manifest = Manifest(options.manifest)
manifest.parse()
project = manifest.find_project(options.project)
dest = options.destination
if dest == '.':
dest = os.path.join (os.getcwd(), project.name)
if options.fetch:
assert os.path.exists(dest) == True
git('fetch', project.fetch_uri, project.revision, repository_path=dest)
else:
git('clone', project.fetch_uri, dest)
git('checkout', '--detach', project.revision, repository_path=dest)

View file

@ -0,0 +1,37 @@
#!/bin/sh
set -e
BRANCH=master
NAMESPACE=gstreamer
JOB=documentation
WORK_DIR=`mktemp -d -p "$DIR"`
# deletes the temp directory
function cleanup {
rm -rf "$WORK_DIR"
echo "Deleted temp working directory $WORK_DIR"
}
# register the cleanup function to be called on the EXIT signal
trap cleanup EXIT
echo ""
echo "============================================================================================================================"
echo "Updating documentation from: https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=$JOB"
date
cd $WORK_DIR
wget https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=$JOB -O gstdocs.zip
unzip gstdocs.zip
DOC_BASE="/srv/gstreamer.freedesktop.org/public_html/documentation"
rsync -rvaz --links --delete documentation/ $DOC_BASE || /bin/true
chmod -R g+w $DOC_BASE; chgrp -R gstreamer $DOC_BASE
echo "Done updating documentation"
echo ""

View file

@ -0,0 +1,51 @@
node("docker") {
docker.image('gstreamer/build-base-ubuntu:latest').inside {
env.OUTPREFIX="${env.WORKSPACE}/../output/${params.build_tag}/linux_x86_64/"
stage('Checkout') {
// FIXME: Only checkout the manifest and not all dependencies ?
checkout([$class: 'RepoScm',
manifestRepositoryUrl:'git+ssh://git.arracacha.collabora.co.uk/git/gst-manifest.git',
manifestBranch:"refs/tags/${params.build_tag}",
jobs:4,
currentBranch:true,
quiet:true,
depth:0])
}
stage('Setup') {
sh "find ../output -maxdepth 1 -ctime +1 | xargs rm -Rf"
sh "cd .repo/manifests/; git checkout ${params.build_tag}; cd ../.."
sh "rm -Rf ./workdir/sources/linux_x86_64/"
sh "rm -f *.rpm"
sh "rm -Rf ./workdir/temp; mkdir -p ./workdir/temp"
sh "rm -Rf ./workdir/tmp*"
// Create custom configuration file
sh "./gst-ci-scripts/manifest2cerbero.py .repo/manifests/default.xml ./cerbero/config/linux.config --output localconf.cbc"
sh '''echo "home_dir = \\"$WORKSPACE/workdir\\"" >> localconf.cbc'''
sh '''echo "logs = \\"$OUTPREFIX/logs\\"" >> localconf.cbc'''
sh './cerbero/cerbero-uninstalled -c localconf.cbc show-config'
}
stage('bootstrap') {
sh './cerbero/cerbero-uninstalled -c localconf.cbc bootstrap'
sh 'rm -Rf ./workdir/sources/build-tools/'
}
stage('fetch') {
sh './cerbero/cerbero-uninstalled -c localconf.cbc fetch-package --reset-rdeps --full-reset gstreamer-1.0'
}
stage('package') {
sh './cerbero/cerbero-uninstalled -c localconf.cbc package gstreamer-1.0'
}
stage('Cleanup') {
sh 'rm -f *.rpm'
sh 'find ../output -maxdepth 1 -ctime +1 | xargs rm -Rf'
}
// FIXME: IRC Notification
}
}

View file

@ -0,0 +1,66 @@
node('docker') {
docker.image('gstreamer/build-base-fedora:latest').inside {
env.CCACHE_DIR = "${env.WORKSPACE}/.ccache"
env.CCACHE_MAXSIZE = "2G"
env.CC = "ccache gcc"
env.CXX = "ccache g++"
env.MAKEFLAGS = "-j6"
env.PATH = "${env.WORKSPACE}:${env.PATH}"
env.GST_UNINSTALLED_ROOT="${env.WORKSPACE}"
env.HOME="${env.WORKSPACE}"
env.DISPLAY=":0"
stage('Checkout') {
if (params.wipe) {
sh 'rm -Rf *'
}
checkout([$class: 'RepoScm',
manifestRepositoryUrl:'https://git.arracacha.collabora.co.uk/git/gst-manifest.git',
manifestBranch:"refs/tags/${params.build_tag}",
jobs:4,
currentBranch:true,
quiet:true,
depth:0,
mirrorDir:'/repositories'])
}
stage('Cleanup') {
sh 'rm -f **/tests/check/*/*.xml'
}
stage ('Build') {
sh "uname -a"
sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh fast-build-only"
}
withEnv(['DISPLAY=:0']) {
stage ('Check') {
env.GST_CHECKS_IGNORE="test_allocate_udp_ports_multicast,test_allocate_udp_ports_client_settings,test_reorder_buffer,test_redirect_yes"
env.GST_CHECK_XML=1
sh 'Xvfb :0 -screen 0 1024x768x24 -fbdir /tmp &'
sh 'env'
sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh check"
step([$class: 'XUnitBuilder',
testTimeMargin: '3000', thresholdMode: 1,
thresholds: [[$class: 'FailedThreshold',
failureNewThreshold: '',
failureThreshold: '400',
unstableNewThreshold: '',
unstableThreshold: '1'],
[$class: 'SkippedThreshold',
failureNewThreshold: '',
failureThreshold: '',
unstableNewThreshold: '',
unstableThreshold: '']],
tools: [[$class: 'CheckType',
deleteOutputFiles: true,
failIfNotNew: true,
pattern: '**/tests/check/*/*.xml',
skipNoTestFiles: true,
stopProcessingIfError: true]]])
}
}
}
}

View file

@ -0,0 +1,66 @@
node('docker') {
docker.image('gstreamer/build-base-ubuntu:latest').inside {
env.CCACHE_DIR = "${env.WORKSPACE}/.ccache"
env.CCACHE_MAXSIZE = "2G"
env.CC = "ccache gcc"
env.CXX = "ccache g++"
env.MAKEFLAGS = "-j6"
env.PATH = "${env.WORKSPACE}:${env.PATH}"
env.GST_UNINSTALLED_ROOT="${env.WORKSPACE}"
env.HOME="${env.WORKSPACE}"
env.DISPLAY=":0"
stage('Checkout') {
if (params.wipe) {
sh 'rm -Rf *'
}
checkout([$class: 'RepoScm',
manifestRepositoryUrl:'https://git.arracacha.collabora.co.uk/git/gst-manifest.git',
manifestBranch:"refs/tags/${params.build_tag}",
jobs:4,
currentBranch:true,
quiet:true,
depth:0,
mirrorDir:'/repositories'])
}
stage('Cleanup') {
sh 'rm -f **/tests/check/*/*.xml'
}
stage ('Build') {
sh "uname -a"
sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh fast-build-only"
}
withEnv(['DISPLAY=:0']) {
stage ('Check') {
env.GST_CHECKS_IGNORE="test_allocate_udp_ports_multicast,test_allocate_udp_ports_client_settings,test_reorder_buffer,test_redirect_yes"
env.GST_CHECK_XML=1
sh 'Xvfb :0 -screen 0 1024x768x24 -fbdir /tmp &'
sh 'env'
sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh check"
step([$class: 'XUnitBuilder',
testTimeMargin: '3000', thresholdMode: 1,
thresholds: [[$class: 'FailedThreshold',
failureNewThreshold: '',
failureThreshold: '400',
unstableNewThreshold: '',
unstableThreshold: '1'],
[$class: 'SkippedThreshold',
failureNewThreshold: '',
failureThreshold: '',
unstableNewThreshold: '',
unstableThreshold: '']],
tools: [[$class: 'CheckType',
deleteOutputFiles: true,
failIfNotNew: true,
pattern: '**/tests/check/*/*.xml',
skipNoTestFiles: true,
stopProcessingIfError: true]]])
}
}
}
}

View file

@ -0,0 +1,71 @@
node('docker') {
docker.image('gstreamer/build-meson-fedora:latest').inside {
env.CCACHE_DIR = "${env.WORKSPACE}/.ccache"
env.CCACHE_MAXSIZE = "2G"
env.CC = "ccache gcc"
env.CXX = "ccache g++"
env.MAKEFLAGS = "-j6"
env.PATH = "${env.WORKSPACE}:${env.PATH}"
env.HOME="${env.WORKSPACE}"
env.DISPLAY=":0"
stage('Checkout') {
if (params.wipe) {
sh 'rm -Rf build/'
}
checkout([$class: 'GitSCM', branches: [[name: '*/master']],
doGenerateSubmoduleConfigurations: false,
extensions: [[$class: 'CloneOption',
depth: 0,
noTags: false,
reference: '/gstbuild/gst-build/',
shallow: false]],
submoduleCfg: [],
userRemoteConfigs: [[url: 'git://anongit.freedesktop.org/gstreamer/gst-build']]]
)
sh 'git checkout master && git reset --hard origin/master'
sh 'curl "https://git.arracacha.collabora.co.uk/cgit/gst-manifest.git/plain/default.xml?id=$BUILD_TAG" -k -o manifest.xml'
}
stage('Setup') {
sh './git-update --no-color --manifest=manifest.xml --no-interaction'
sh './setup.py -Ddisable_gstreamer_vaapi=true'
}
stage ('Build') {
sh "ninja -C build"
}
stage ('Check') {
sh "./gst-uninstalled.py gst-validate-launcher --check-bugs --no-display --mute -n check --xunit-file $WORKSPACE/xunit.xml -M $WORKSPACE/validate-output --ignore-numfailures"
step([$class: 'XUnitBuilder',
testTimeMargin: '3000', thresholdMode: 1,
thresholds: [[$class: 'FailedThreshold',
failureNewThreshold: '',
failureThreshold: '5',
unstableNewThreshold: '',
unstableThreshold: '1'],
[$class: 'SkippedThreshold',
failureNewThreshold: '',
failureThreshold: '',
unstableNewThreshold: '',
unstableThreshold: '']],
tools: [[$class: 'JUnitType',
deleteOutputFiles: true,
failIfNotNew: true,
pattern: 'xunit.xml',
skipNoTestFiles: true,
stopProcessingIfError: true]]])
}
stage('install') {
sh 'mkdir -p dest'
sh 'DESTDIR=$PWD/dest ninja -C build install'
}
stage('package') {
sh 'cd dest && tar caJf gstreamer-$BUILD_TAG.tar.xz usr'
}
}
}