mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-12-19 14:56:36 +00:00
Merging gst-ci
This commit is contained in:
commit
c7253318b3
38 changed files with 3622 additions and 0 deletions
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
*~
|
24
.gitlab-ci.yml
Normal file
24
.gitlab-ci.yml
Normal file
|
@ -0,0 +1,24 @@
|
|||
include: "gitlab/ci_template.yml"
|
||||
|
||||
test manifest:
|
||||
variables:
|
||||
GIT_STRATEGY: fetch
|
||||
image: "$TEST_MANIFEST_IMAGE"
|
||||
stage: "preparation"
|
||||
script:
|
||||
- pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py
|
||||
coverage: '/TOTAL.*\s+(\d+%)$/'
|
||||
artifacts:
|
||||
reports:
|
||||
junit:
|
||||
- "junit.xml"
|
||||
|
||||
test manifest amd64 docker:
|
||||
stage: "build docker"
|
||||
variables:
|
||||
REPO_SUFFIX: "$TEST_MANIFEST_AMD64_SUFFIX"
|
||||
TAG: "$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH"
|
||||
|
||||
CONTEXT_DIR: "docker/test_manifest/"
|
||||
DOCKERFILE: "docker/test_manifest/Dockerfile"
|
||||
extends: .base
|
11
README.txt
Normal file
11
README.txt
Normal file
|
@ -0,0 +1,11 @@
|
|||
GStreamer Continuous Integration
|
||||
================================
|
||||
|
||||
This repository contains all material relevant to the GStreamer
|
||||
Continuous Integration system.
|
||||
|
||||
* Jenkins scripts
|
||||
|
||||
* Docker images
|
||||
|
||||
* Build scripts and code
|
2
docker/README
Normal file
2
docker/README
Normal file
|
@ -0,0 +1,2 @@
|
|||
GStreamer Docker images
|
||||
|
11
docker/android/Dockerfile
Normal file
11
docker/android/Dockerfile
Normal file
|
@ -0,0 +1,11 @@
|
|||
FROM registry.fedoraproject.org/fedora:31
|
||||
|
||||
ENV ANDROID_HOME=/android/sdk
|
||||
ENV ANDROID_NDK_HOME=/android/ndk
|
||||
|
||||
ARG DEFAULT_BRANCH="master"
|
||||
|
||||
COPY prepare.sh cleanup.sh /root/
|
||||
|
||||
RUN /usr/bin/sh /root/prepare.sh && \
|
||||
/usr/bin/sh /root/cleanup.sh
|
7
docker/android/cleanup.sh
Normal file
7
docker/android/cleanup.sh
Normal file
|
@ -0,0 +1,7 @@
|
|||
set -eux
|
||||
|
||||
echo "Removing DNF cache"
|
||||
dnf clean all
|
||||
|
||||
rm -R /root/*
|
||||
rm -rf /var/cache/dnf /var/log/dnf*
|
56
docker/android/prepare.sh
Normal file
56
docker/android/prepare.sh
Normal file
|
@ -0,0 +1,56 @@
|
|||
set -eux
|
||||
|
||||
dnf install -y \
|
||||
file \
|
||||
git-core \
|
||||
java-1.8.0-openjdk-devel \
|
||||
lbzip2 \
|
||||
make \
|
||||
pkg-config \
|
||||
unzip \
|
||||
which \
|
||||
xz
|
||||
|
||||
mkdir -p /android/sources
|
||||
|
||||
curl -o /android/sources/android-ndk.zip https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip
|
||||
unzip /android/sources/android-ndk.zip -d ${ANDROID_NDK_HOME}/
|
||||
# remove the intermediate versioned directory
|
||||
mv ${ANDROID_NDK_HOME}/*/* ${ANDROID_NDK_HOME}/
|
||||
|
||||
curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-6200805_latest.zip
|
||||
unzip /android/sources/android-sdk-tools.zip -d ${ANDROID_HOME}/
|
||||
mkdir -p ${ANDROID_HOME}/licenses
|
||||
|
||||
# Accept licenses. Values taken from:
|
||||
# $ANDROID_HOME/tools/bin/sdkmanager --sdk_root=$ANDROID_HOME --licenses
|
||||
# cd $ANDROID_HOME
|
||||
# for f in licenses/*; do echo "echo \"$(cat $f | tr -d '\n')\" > \${ANDROID_HOME}/$f"; done
|
||||
echo "601085b94cd77f0b54ff86406957099ebe79c4d6" > ${ANDROID_HOME}/licenses/android-googletv-license
|
||||
echo "859f317696f67ef3d7f30a50a5560e7834b43903" > ${ANDROID_HOME}/licenses/android-sdk-arm-dbt-license
|
||||
echo "24333f8a63b6825ea9c5514f83c2829b004d1fee" > ${ANDROID_HOME}/licenses/android-sdk-license
|
||||
echo "84831b9409646a918e30573bab4c9c91346d8abd" > ${ANDROID_HOME}/licenses/android-sdk-preview-license
|
||||
echo "33b6a2b64607f11b759f320ef9dff4ae5c47d97a" > ${ANDROID_HOME}/licenses/google-gdk-license
|
||||
echo "e9acab5b5fbb560a72cfaecce8946896ff6aab9d" > ${ANDROID_HOME}/licenses/mips-android-sysimage-license
|
||||
|
||||
# pre-cache deps
|
||||
export GSTREAMER_ROOT_ANDROID=/android/sources/gstreamer-android
|
||||
curl -o /android/sources/gstreamer-android.tar.xz https://gstreamer.freedesktop.org/data/pkg/android/1.16.2/gstreamer-1.0-android-universal-1.16.2.tar.xz
|
||||
mkdir $GSTREAMER_ROOT_ANDROID
|
||||
tar -xvf /android/sources/gstreamer-android.tar.xz -C $GSTREAMER_ROOT_ANDROID
|
||||
ls $GSTREAMER_ROOT_ANDROID
|
||||
|
||||
git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-examples.git /android/sources/gst-examples
|
||||
chmod +x /android/sources/gst-examples/playback/player/android/gradlew
|
||||
/android/sources/gst-examples/playback/player/android/gradlew --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies
|
||||
|
||||
chmod +x /android/sources/gst-examples/vulkan/android/gradlew
|
||||
/android/sources/gst-examples/vulkan/android/gradlew --no-daemon --project-dir /android/sources/gst-examples/vulkan/android dependencies --refresh-dependencies
|
||||
|
||||
git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs
|
||||
chmod +x /android/sources/gst-docs/examples/tutorials/android/gradlew
|
||||
/android/sources/gst-docs/examples/tutorials/android/gradlew --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies
|
||||
|
||||
unset GSTREAMER_ROOT_ANDROID
|
||||
|
||||
rm -rf /android/sources
|
7
docker/build_manifest/Dockerfile
Normal file
7
docker/build_manifest/Dockerfile
Normal file
|
@ -0,0 +1,7 @@
|
|||
FROM python:3.7.1-alpine
|
||||
|
||||
ARG DEFAULT_BRANCH="master"
|
||||
|
||||
RUN pip install requests
|
||||
RUN apk add git
|
||||
RUN cd / && git clone -b ${DEFAULT_BRANCH:=master} https://gitlab.freedesktop.org/gstreamer/gst-ci.git
|
35
docker/cerbero/Dockerfile-fedora
Normal file
35
docker/cerbero/Dockerfile-fedora
Normal file
|
@ -0,0 +1,35 @@
|
|||
FROM registry.fedoraproject.org/fedora:31
|
||||
|
||||
ENV TERM="dumb"
|
||||
|
||||
ARG DEFAULT_BRANCH="master"
|
||||
|
||||
# System setup
|
||||
RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \
|
||||
dnf upgrade -y && \
|
||||
dnf install -y sudo git lbzip2 rsync wine which
|
||||
|
||||
# Configure git for various usage
|
||||
RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \
|
||||
git config --global user.name "Cerbero Build System"
|
||||
|
||||
RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git && \
|
||||
mkdir $HOME/.cerbero && \
|
||||
echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \
|
||||
echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \
|
||||
cd cerbero && \
|
||||
echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \
|
||||
echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap --jobs=4 && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc fetch-package --jobs=4 gstreamer-1.0 && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap --jobs=4 && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package --jobs=4 gstreamer-1.0 && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap --jobs=4 && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package --jobs=4 gstreamer-1.0 && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap --jobs=4 && \
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package --jobs=4 gstreamer-1.0 && \
|
||||
cd .. && \
|
||||
rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \
|
||||
rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \
|
||||
dnf clean all
|
12
docker/fedora/Dockerfile
Normal file
12
docker/fedora/Dockerfile
Normal file
|
@ -0,0 +1,12 @@
|
|||
FROM registry.fedoraproject.org/fedora:31
|
||||
|
||||
ENV RUSTUP_HOME=/usr/local/rustup \
|
||||
CARGO_HOME=/usr/local/cargo \
|
||||
PATH=/usr/local/cargo/bin:$PATH
|
||||
|
||||
ARG DEFAULT_BRANCH="master"
|
||||
|
||||
COPY prepare.sh cleanup.sh /root/
|
||||
|
||||
RUN /usr/bin/sh /root/prepare.sh && \
|
||||
/usr/bin/sh /root/cleanup.sh
|
7
docker/fedora/cleanup.sh
Normal file
7
docker/fedora/cleanup.sh
Normal file
|
@ -0,0 +1,7 @@
|
|||
set -eux
|
||||
|
||||
echo "Removing DNF cache"
|
||||
dnf clean all
|
||||
|
||||
rm -R /root/*
|
||||
rm -rf /var/cache/dnf /var/log/dnf*
|
244
docker/fedora/prepare.sh
Normal file
244
docker/fedora/prepare.sh
Normal file
|
@ -0,0 +1,244 @@
|
|||
set -eux
|
||||
|
||||
# Fedora base image disable installing documentation files. See https://pagure.io/atomic-wg/issue/308
|
||||
# We need them to cleanly build our doc.
|
||||
sed -i "s/tsflags=nodocs//g" /etc/dnf/dnf.conf
|
||||
|
||||
dnf install -y git-core ninja-build dnf-plugins-core python3-pip
|
||||
|
||||
# Configure git for various usage
|
||||
git config --global user.email "gst-build@gstreamer.net"
|
||||
git config --global user.name "Gstbuild Runner"
|
||||
|
||||
# Add rpm fusion repositories in order to access all of the gst plugins
|
||||
sudo dnf install -y \
|
||||
"https://mirrors.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm" \
|
||||
"https://mirrors.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm"
|
||||
|
||||
dnf upgrade -y
|
||||
|
||||
# install rest of the extra deps
|
||||
dnf install -y \
|
||||
aalib-devel \
|
||||
aom \
|
||||
bat \
|
||||
intel-mediasdk-devel \
|
||||
libaom \
|
||||
libaom-devel \
|
||||
libcaca-devel \
|
||||
libdav1d \
|
||||
libdav1d-devel \
|
||||
ccache \
|
||||
cmake \
|
||||
clang-devel \
|
||||
elfutils \
|
||||
elfutils-libs \
|
||||
elfutils-devel \
|
||||
gcc \
|
||||
gcc-c++ \
|
||||
gdb \
|
||||
git-lfs \
|
||||
glslc \
|
||||
gtk3 \
|
||||
gtk3-devel \
|
||||
gtest \
|
||||
gtest-devel \
|
||||
graphene \
|
||||
graphene-devel \
|
||||
gsl \
|
||||
gsl-devel \
|
||||
gupnp \
|
||||
gupnp-devel \
|
||||
gupnp-igd \
|
||||
gupnp-igd-devel \
|
||||
gssdp \
|
||||
gssdp-devel \
|
||||
faac-devel \
|
||||
ffmpeg \
|
||||
ffmpeg-libs \
|
||||
ffmpeg-devel \
|
||||
flex \
|
||||
flite \
|
||||
flite-devel \
|
||||
mono-devel \
|
||||
procps-ng \
|
||||
patch \
|
||||
qt5-devel \
|
||||
redhat-rpm-config \
|
||||
json-glib \
|
||||
json-glib-devel \
|
||||
libnice \
|
||||
libnice-devel \
|
||||
libsodium-devel \
|
||||
libunwind \
|
||||
libunwind-devel \
|
||||
libyaml-devel \
|
||||
libxml2-devel \
|
||||
libxslt-devel \
|
||||
llvm-devel \
|
||||
log4c-devel \
|
||||
make \
|
||||
nasm \
|
||||
neon \
|
||||
neon-devel \
|
||||
nunit \
|
||||
npm \
|
||||
opencv \
|
||||
opencv-devel \
|
||||
openjpeg2 \
|
||||
openjpeg2-devel \
|
||||
SDL2 \
|
||||
SDL2-devel \
|
||||
sbc \
|
||||
sbc-devel \
|
||||
x264 \
|
||||
x264-libs \
|
||||
x264-devel \
|
||||
python3 \
|
||||
python3-devel \
|
||||
python3-libs \
|
||||
python3-gobject \
|
||||
python3-cairo \
|
||||
python3-cairo-devel \
|
||||
valgrind \
|
||||
vulkan \
|
||||
vulkan-devel \
|
||||
mesa-omx-drivers \
|
||||
mesa-libGL \
|
||||
mesa-libGL-devel \
|
||||
mesa-libGLU \
|
||||
mesa-libGLU-devel \
|
||||
mesa-libGLES \
|
||||
mesa-libGLES-devel \
|
||||
mesa-libOpenCL \
|
||||
mesa-libOpenCL-devel \
|
||||
mesa-libgbm \
|
||||
mesa-libgbm-devel \
|
||||
mesa-libd3d \
|
||||
mesa-libd3d-devel \
|
||||
mesa-libOSMesa \
|
||||
mesa-libOSMesa-devel \
|
||||
mesa-vulkan-drivers \
|
||||
wpewebkit \
|
||||
wpewebkit-devel \
|
||||
xorg-x11-server-utils \
|
||||
xorg-x11-server-Xvfb
|
||||
|
||||
# Install common debug symbols
|
||||
dnf debuginfo-install -y gtk3 \
|
||||
glib2 \
|
||||
glibc \
|
||||
gupnp \
|
||||
gupnp-igd \
|
||||
gssdp \
|
||||
freetype \
|
||||
openjpeg \
|
||||
gobject-introspection \
|
||||
python3 \
|
||||
python3-libs \
|
||||
python3-gobject \
|
||||
libappstream-glib-devel \
|
||||
libjpeg-turbo \
|
||||
glib-networking \
|
||||
libcurl \
|
||||
libsoup \
|
||||
nss \
|
||||
nss-softokn \
|
||||
nss-softokn-freebl \
|
||||
nss-sysinit \
|
||||
nss-util \
|
||||
openssl \
|
||||
openssl-libs \
|
||||
openssl-pkcs11 \
|
||||
brotli \
|
||||
bzip2-libs \
|
||||
gpm-libs \
|
||||
harfbuzz \
|
||||
harfbuzz-icu \
|
||||
json-c \
|
||||
json-glib \
|
||||
libbabeltrace \
|
||||
libffi \
|
||||
libsrtp \
|
||||
libunwind \
|
||||
mpg123-libs \
|
||||
neon \
|
||||
orc-compiler \
|
||||
orc \
|
||||
pixman \
|
||||
pulseaudio-libs \
|
||||
pulseaudio-libs-glib2 \
|
||||
wavpack \
|
||||
webrtc-audio-processing \
|
||||
ffmpeg \
|
||||
ffmpeg-libs \
|
||||
faad2-libs \
|
||||
libavdevice \
|
||||
libmpeg2 \
|
||||
faac \
|
||||
fdk-aac \
|
||||
x264 \
|
||||
x264-libs \
|
||||
x265 \
|
||||
x265-libs \
|
||||
xz \
|
||||
xz-libs \
|
||||
zip \
|
||||
zlib
|
||||
|
||||
# Install the dependencies of gstreamer
|
||||
dnf builddep -y gstreamer1 \
|
||||
gstreamer1-plugins-base \
|
||||
gstreamer1-plugins-good \
|
||||
gstreamer1-plugins-good-extras \
|
||||
gstreamer1-plugins-ugly \
|
||||
gstreamer1-plugins-ugly-free \
|
||||
gstreamer1-plugins-bad-free \
|
||||
gstreamer1-plugins-bad-free-extras \
|
||||
gstreamer1-plugins-bad-freeworld \
|
||||
gstreamer1-libav \
|
||||
gstreamer1-rtsp-server \
|
||||
gstreamer1-vaapi \
|
||||
python3-gstreamer1
|
||||
|
||||
dnf remove -y meson
|
||||
pip3 install meson==0.58.0 hotdoc
|
||||
|
||||
# Remove gst-devel packages installed by builddep above
|
||||
dnf remove -y "gstreamer1*devel"
|
||||
|
||||
# FIXME: Why does installing directly with dnf doesn't actually install
|
||||
# the documentation files?
|
||||
dnf download glib2-doc gdk-pixbuf2-devel*x86_64* gtk3-devel-docs
|
||||
rpm -i --reinstall *.rpm
|
||||
rm -f *.rpm
|
||||
|
||||
# Install Rust
|
||||
RUSTUP_VERSION=1.24.2
|
||||
RUST_VERSION=1.52.1
|
||||
RUST_ARCH="x86_64-unknown-linux-gnu"
|
||||
|
||||
dnf install -y wget
|
||||
RUSTUP_URL=https://static.rust-lang.org/rustup/archive/$RUSTUP_VERSION/$RUST_ARCH/rustup-init
|
||||
wget $RUSTUP_URL
|
||||
dnf remove -y wget
|
||||
|
||||
chmod +x rustup-init;
|
||||
./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION;
|
||||
rm rustup-init;
|
||||
chmod -R a+w $RUSTUP_HOME $CARGO_HOME
|
||||
|
||||
rustup --version
|
||||
cargo --version
|
||||
rustc --version
|
||||
|
||||
# get gst-build and make all subprojects available
|
||||
git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-build.git /gst-build/
|
||||
cd /gst-build
|
||||
meson subprojects download
|
||||
|
||||
# Run git gc to prune unwanted refs and reduce the size of the image
|
||||
for i in $(find subprojects/ -mindepth 1 -maxdepth 1 -type d);
|
||||
do
|
||||
git -C $i gc --aggressive || true;
|
||||
done
|
7
docker/indent/Dockerfile
Normal file
7
docker/indent/Dockerfile
Normal file
|
@ -0,0 +1,7 @@
|
|||
FROM debian:stretch-slim
|
||||
|
||||
RUN apt update -yqq \
|
||||
&& apt install -y curl indent git findutils \
|
||||
&& rm -R /var/lib/apt/ /var/log/apt/ \
|
||||
&& curl -L -o /usr/local/bin/gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent \
|
||||
&& chmod a+x /usr/local/bin/gst-indent
|
8
docker/test_manifest/Dockerfile
Normal file
8
docker/test_manifest/Dockerfile
Normal file
|
@ -0,0 +1,8 @@
|
|||
FROM registry.fedoraproject.org/fedora:31
|
||||
|
||||
RUN dnf install -y \
|
||||
python3-pytest \
|
||||
python3-requests \
|
||||
python3-pytest-cov \
|
||||
git-core \
|
||||
&& rm -rf /var/cache/dnf /var/log/dnf*
|
1
docker/windows/.dockerignore
Normal file
1
docker/windows/.dockerignore
Normal file
|
@ -0,0 +1 @@
|
|||
build_image.ps1
|
23
docker/windows/Dockerfile
Normal file
23
docker/windows/Dockerfile
Normal file
|
@ -0,0 +1,23 @@
|
|||
# escape=`
|
||||
|
||||
FROM 'mcr.microsoft.com/windows/servercore:ltsc2019'
|
||||
|
||||
# Make sure any failure in PowerShell scripts is fatal
|
||||
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"]
|
||||
ENV ErrorActionPreference='Stop'
|
||||
|
||||
COPY install_choco.ps1 C:\
|
||||
RUN C:\install_choco.ps1
|
||||
|
||||
RUN 'git config --global user.email "cirunner@gstreamer.freedesktop.org"; git config --global user.name "GStreamer CI system"'
|
||||
|
||||
COPY install_toolchain.ps1 C:\
|
||||
RUN C:\install_toolchain.ps1
|
||||
|
||||
ARG DEFAULT_BRANCH="master"
|
||||
|
||||
COPY prepare_gst_env.ps1 C:\
|
||||
RUN C:\prepare_gst_env.ps1
|
||||
|
||||
COPY prepare_cerbero_env.sh C:\
|
||||
RUN C:\MinGW\msys\1.0\bin\bash.exe --login -c "C:/prepare_cerbero_env.sh"
|
27
docker/windows/build_image.ps1
Normal file
27
docker/windows/build_image.ps1
Normal file
|
@ -0,0 +1,27 @@
|
|||
$env:ErrorActionPreference='Stop'
|
||||
|
||||
$env:DEFAULT_BRANCH='master'
|
||||
$env:VERSION='test'
|
||||
$env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH"
|
||||
$env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH"
|
||||
|
||||
Set-Location './docker/windows/'
|
||||
|
||||
Get-Date
|
||||
Write-Output "Building $env:tag"
|
||||
docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag .
|
||||
if (!$?) {
|
||||
Write-Host "Failed to build docker image $env:tag"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
Get-Date
|
||||
Write-Output "Building $env:rust_tag"
|
||||
docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f rust.Dockerfile -t $env:rust_tag .
|
||||
if (!$?) {
|
||||
Write-Host "Failed to build docker image $env:rust_tag"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
Get-Date
|
||||
Write-Output "Build Finished"
|
60
docker/windows/container.ps1
Normal file
60
docker/windows/container.ps1
Normal file
|
@ -0,0 +1,60 @@
|
|||
# Copied from mesa, big kudos
|
||||
#
|
||||
# https://gitlab.freedesktop.org/mesa/mesa/-/blob/master/.gitlab-ci/windows/mesa_container.ps1
|
||||
# https://gitlab.freedesktop.org/mesa/mesa/-/blob/34e3e164936d1d3cef267da7780e87f062fedf39/.gitlab-ci/windows/mesa_container.ps1
|
||||
|
||||
# Implements the equivalent of ci-templates container-ifnot-exists, using
|
||||
# Docker directly as we don't have buildah/podman/skopeo available under
|
||||
# Windows, nor can we execute Docker-in-Docker
|
||||
$registry_uri = $args[0]
|
||||
$registry_username = $args[1]
|
||||
$registry_password = $args[2]
|
||||
$registry_user_image = $args[3]
|
||||
$registry_central_image = $args[4]
|
||||
$dockerfile = $args[5]
|
||||
|
||||
docker --config "windows-docker.conf" login -u "$registry_username" -p "$registry_password" "$registry_uri"
|
||||
if (!$?) {
|
||||
Write-Host "docker login failed to $registry_uri"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
# if the image already exists, don't rebuild it
|
||||
docker --config "windows-docker.conf" pull "$registry_user_image"
|
||||
if ($?) {
|
||||
Write-Host "User image $registry_user_image already exists; not rebuilding"
|
||||
docker --config "windows-docker.conf" logout "$registry_uri"
|
||||
Exit 0
|
||||
}
|
||||
|
||||
# if the image already exists upstream, copy it
|
||||
docker --config "windows-docker.conf" pull "$registry_central_image"
|
||||
if ($?) {
|
||||
Write-Host "Copying central image $registry_central_image to user image $registry_user_image"
|
||||
docker --config "windows-docker.conf" tag "$registry_central_image" "$registry_user_image"
|
||||
docker --config "windows-docker.conf" push "$registry_user_image"
|
||||
$pushstatus = $?
|
||||
docker --config "windows-docker.conf" logout "$registry_uri"
|
||||
if (!$pushstatus) {
|
||||
Write-Host "Pushing image to $registry_user_image failed"
|
||||
Exit 1
|
||||
}
|
||||
Exit 0
|
||||
}
|
||||
|
||||
Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding"
|
||||
docker --config "windows-docker.conf" build $DOCKER_BUILD_ARGS --no-cache -t "$registry_user_image" -f "$dockerfile" "./docker/windows"
|
||||
if (!$?) {
|
||||
Write-Host "Container build failed"
|
||||
docker --config "windows-docker.conf" logout "$registry_uri"
|
||||
Exit 1
|
||||
}
|
||||
Get-Date
|
||||
|
||||
docker --config "windows-docker.conf" push "$registry_user_image"
|
||||
$pushstatus = $?
|
||||
docker --config "windows-docker.conf" logout "$registry_uri"
|
||||
if (!$pushstatus) {
|
||||
Write-Host "Pushing image to $registry_user_image failed"
|
||||
Exit 1
|
||||
}
|
23
docker/windows/install_choco.ps1
Normal file
23
docker/windows/install_choco.ps1
Normal file
|
@ -0,0 +1,23 @@
|
|||
Get-Date
|
||||
Write-Host "Installing Chocolatey"
|
||||
Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
|
||||
Import-Module "$env:ProgramData\chocolatey\helpers\chocolateyProfile.psm1"
|
||||
Update-SessionEnvironment
|
||||
|
||||
choco install -y vcredist140
|
||||
$vc140_install = $?
|
||||
|
||||
Write-Host "Installing Chocolatey packages"
|
||||
choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
|
||||
$cmake_install = $?
|
||||
|
||||
choco install -y git --params "/NoAutoCrlf /NoCredentialManager /NoShellHereIntegration /NoGuiHereIntegration /NoShellIntegration"
|
||||
$git_install = $?
|
||||
|
||||
choco install -y python3 git-lfs 7zip
|
||||
$rest_installs = $?
|
||||
|
||||
if (!($vc140_install -and $cmake_install -and $git_install -and $rest_installs)) {
|
||||
Write-Host "Failed to install some dependencies from choco"
|
||||
Exit 1
|
||||
}
|
71
docker/windows/install_gst.ps1
Normal file
71
docker/windows/install_gst.ps1
Normal file
|
@ -0,0 +1,71 @@
|
|||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
|
||||
|
||||
# Download gst-build and all its subprojects
|
||||
# git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
|
||||
# FIXME: need 1.19+ for cairo subproject :/
|
||||
# Should use a stable branch instead
|
||||
git clone -b master --depth 1 https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
|
||||
if (!$?) {
|
||||
Write-Host "Failed to clone gst-build"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
Set-Location C:\gst-build
|
||||
|
||||
# Copy the cache we already have in the image to avoid massive redownloads
|
||||
Move-Item C:/subprojects/* C:\gst-build\subprojects
|
||||
|
||||
if (!$?) {
|
||||
Write-Host "Failed to copy subprojects cache"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
# Update the subprojects cache
|
||||
Write-Output "Running meson subproject reset"
|
||||
meson subprojects update --reset
|
||||
|
||||
if (!$?) {
|
||||
Write-Host "Failed to reset subprojects state"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
Write-Output "Running git update"
|
||||
python git-update --no-interaction
|
||||
|
||||
if (!$?) {
|
||||
Write-Host "Failed to run git-update"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
$env:MESON_ARGS = "-Dglib:installed_tests=false " +
|
||||
"-Dlibnice:tests=disabled " +
|
||||
"-Dlibnice:examples=disabled " +
|
||||
"-Dffmpeg:tests=disabled " +
|
||||
"-Dopenh264:tests=disabled " +
|
||||
"-Dpygobject:tests=false " +
|
||||
"-Dugly=enabled " +
|
||||
"-Dbad=enabled " +
|
||||
"-Dges=enabled " +
|
||||
"-Drtsp_server=enabled " +
|
||||
"-Ddevtools=enabled " +
|
||||
"-Dsharp=disabled " +
|
||||
"-Dpython=disabled " +
|
||||
"-Dlibav=disabled " +
|
||||
"-Dvaapi=disabled " +
|
||||
"-Dgst-plugins-base:pango=enabled " +
|
||||
"-Dgst-plugins-good:cairo=enabled "
|
||||
|
||||
Write-Output "Building gst"
|
||||
cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && meson _build $env:MESON_ARGS && meson compile -C _build && ninja -C _build install"
|
||||
|
||||
if (!$?) {
|
||||
Write-Host "Failed to build and install gst"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
git clean -fdxx
|
||||
|
||||
if (!$?) {
|
||||
Write-Host "Failed to git clean"
|
||||
Exit 1
|
||||
}
|
69
docker/windows/install_toolchain.ps1
Normal file
69
docker/windows/install_toolchain.ps1
Normal file
|
@ -0,0 +1,69 @@
|
|||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
|
||||
|
||||
$msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe'
|
||||
$msys2_url = 'https://github.com/msys2/msys2-installer/releases/download/2021-02-15/msys2-base-x86_64-20210215.tar.xz'
|
||||
$msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz'
|
||||
|
||||
Get-Date
|
||||
Write-Host "Downloading Visual Studio 2017 build tools"
|
||||
Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe
|
||||
|
||||
Get-Date
|
||||
Write-Host "Installing Visual Studio 2017"
|
||||
Start-Process -NoNewWindow -Wait C:\vs_buildtools.exe -ArgumentList '--wait --quiet --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended'
|
||||
if (!$?) {
|
||||
Write-Host "Failed to install Visual Studio tools"
|
||||
Exit 1
|
||||
}
|
||||
Remove-Item C:\vs_buildtools.exe -Force
|
||||
|
||||
Get-Date
|
||||
Write-Host "Downloading and extracting mingw-get for MSYS"
|
||||
Invoke-WebRequest -Uri $msys_mingw_get_url -OutFile C:\mingw-get.tar.xz
|
||||
7z e C:\mingw-get.tar.xz -o"C:\\"
|
||||
$res1 = $?
|
||||
7z x C:\mingw-get.tar -o"C:\\MinGW"
|
||||
$res2 = $?
|
||||
|
||||
if (!($res1 -and $res2)) {
|
||||
Write-Host "Failed to extract mingw-get"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
Remove-Item C:\mingw-get.tar.xz -Force
|
||||
Remove-Item C:\mingw-get.tar -Force
|
||||
|
||||
Get-Date
|
||||
Write-Host "Installing MSYS for Cerbero into C:/MinGW using mingw-get"
|
||||
Start-Process -Wait C:\MinGW\bin\mingw-get.exe -ArgumentList 'install msys-base mingw32-base mingw-developer-toolkit'
|
||||
if (!$?) {
|
||||
Write-Host "Failed to install Msys for cerbero using MinGW"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
Get-Date
|
||||
Write-Host "Installing MSYS2 into C:/msys64"
|
||||
Invoke-WebRequest -Uri $msys2_url -OutFile C:\msys2-x86_64.tar.xz
|
||||
|
||||
7z e C:\msys2-x86_64.tar.xz -o"C:\\"
|
||||
$res1 = $?
|
||||
7z x C:\msys2-x86_64.tar -o"C:\\"
|
||||
$res2 = $?
|
||||
|
||||
if (!($res1 -and $res2)) {
|
||||
Write-Host "Failed to extract msys2"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
Remove-Item C:\msys2-x86_64.tar.xz -Force
|
||||
Remove-Item C:\msys2-x86_64.tar -Force
|
||||
|
||||
Get-Date
|
||||
Write-Host "Installing Meson"
|
||||
pip3 install meson
|
||||
if (!$?) {
|
||||
Write-Host "Failed to install meson from pip"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
Write-Host "Toolchain Install Complete"
|
27
docker/windows/prepare_cerbero_env.sh
Normal file
27
docker/windows/prepare_cerbero_env.sh
Normal file
|
@ -0,0 +1,27 @@
|
|||
#! /bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
cd C:/
|
||||
git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git
|
||||
cd cerbero
|
||||
|
||||
echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc
|
||||
echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc
|
||||
echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc
|
||||
echo 'vs_install_version = "vs15"' >> localconf.cbc
|
||||
|
||||
# Fetch all bootstrap requirements
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-bootstrap --jobs=4
|
||||
# Fetch all package requirements for a mingw gstreamer build
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-package --jobs=4 gstreamer-1.0
|
||||
# Fetch all package requirements for a visualstudio gstreamer build
|
||||
./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package --jobs=4 gstreamer-1.0
|
||||
|
||||
# Fixup the MSYS installation
|
||||
./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline
|
||||
|
||||
# Wipe visualstudio package dist, sources, logs, and the build tools recipes
|
||||
./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc wipe --force --build-tools
|
||||
# clean the localconf
|
||||
rm /c/cerbero/localconf.cbc
|
24
docker/windows/prepare_gst_env.ps1
Normal file
24
docker/windows/prepare_gst_env.ps1
Normal file
|
@ -0,0 +1,24 @@
|
|||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
|
||||
|
||||
# Download gst-build and all its subprojects
|
||||
git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
|
||||
if (!$?) {
|
||||
Write-Host "Failed to clone gst-build"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
# download the subprojects to try and cache them
|
||||
meson subprojects download --sourcedir C:\gst-build
|
||||
if (!$?) {
|
||||
Write-Host "Failed to download the subprojects"
|
||||
Exit 1
|
||||
}
|
||||
|
||||
# Remove files that will conflict with a fresh clone on the runner side
|
||||
Remove-Item -Force 'C:/gst-build/subprojects/*.wrap'
|
||||
Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-nasm'
|
||||
Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-flex-bison-binaries'
|
||||
Remove-Item -Recurse -Force 'C:/gst-build/subprojects/macos-bison-binary'
|
||||
|
||||
Move-Item C:\gst-build\subprojects C:\subprojects
|
||||
Remove-Item -Recurse -Force C:\gst-build
|
21
docker/windows/rust.Dockerfile
Normal file
21
docker/windows/rust.Dockerfile
Normal file
|
@ -0,0 +1,21 @@
|
|||
# escape=`
|
||||
|
||||
# Expect this to be set when calling docker build with
|
||||
# --build-arg BASE_IMAGE="" and make it fail if not set.
|
||||
ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid"
|
||||
FROM $BASE_IMAGE
|
||||
|
||||
ARG DEFAULT_BRANCH="master"
|
||||
ARG RUST_VERSION="invalid"
|
||||
|
||||
COPY install_gst.ps1 C:\
|
||||
RUN C:\install_gst.ps1
|
||||
RUN choco install -y pkgconfiglite
|
||||
ENV PKG_CONFIG_PATH="C:/lib/pkgconfig"
|
||||
|
||||
ADD https://win.rustup.rs/x86_64 C:\rustup-init.exe
|
||||
RUN C:\rustup-init.exe -y --profile minimal --default-toolchain $env:RUST_VERSION
|
||||
|
||||
# Uncomment for easy testing
|
||||
# RUN git clone --depth 1 https://gitlab.freedesktop.org/gstreamer/gstreamer-rs.git
|
||||
# RUN cd gstreamer-rs; cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64; cargo build --all; cargo test --all"
|
80
fuzzing/README.txt
Normal file
80
fuzzing/README.txt
Normal file
|
@ -0,0 +1,80 @@
|
|||
Fuzzing GStreamer
|
||||
=================
|
||||
|
||||
This directory contains the various fuzzing targets and helper
|
||||
scripts.
|
||||
|
||||
* Fuzzing targets
|
||||
|
||||
Fuzzing targets as small applications where we can test a specific
|
||||
element or API. The goal is to have them be as small/targetted as
|
||||
possible.
|
||||
|
||||
ex: appsrc ! <some_element> ! fakesink num-buffers=<small>
|
||||
|
||||
Not all components can be tested directly and therefore will be
|
||||
indirectly tested via other targets (ex: libgstaudio will be tested
|
||||
by targets/elements requiring it)
|
||||
|
||||
Anything that can process externally-provided data should be
|
||||
covered, but there are cases where it might not make sense to use a
|
||||
fuzzer (such as most elements processing raw audio/video).
|
||||
|
||||
* build-oss-fuzz.sh
|
||||
|
||||
This is the script executed by the oss-fuzz project.
|
||||
|
||||
It builds glib, GStreamer, plugins and the fuzzing targets.
|
||||
|
||||
* *.c
|
||||
|
||||
The fuzzing targets where the data to test will be provided to a
|
||||
function whose signature follows the LibFuzzer signature:
|
||||
https://llvm.org/docs/LibFuzzer.html
|
||||
|
||||
* TODO
|
||||
|
||||
* Add a standalone build script
|
||||
|
||||
We need to be able to build and test the fuzzing targets outside
|
||||
of the oss-fuzz infrastructure, and do that in our continous
|
||||
integration system.
|
||||
|
||||
We need:
|
||||
|
||||
* A dummy fuzzing engine (given a directory, it opens all files and
|
||||
calls the fuzzing targets with the content of those files.
|
||||
* A script to be able to build those targets with that dummy engine
|
||||
* A corpus of files to test those targets with.
|
||||
|
||||
* Build targets with dummy engine and run with existing tests.
|
||||
|
||||
* Create pull-based variants
|
||||
|
||||
Currently the existing targets are push-based only. Where
|
||||
applicable we should make pull-based variants to test the other
|
||||
code paths.
|
||||
|
||||
* Add more targets
|
||||
|
||||
core:
|
||||
gst_parse fuzzer ?
|
||||
base:
|
||||
ext/
|
||||
ogg
|
||||
opus
|
||||
pango
|
||||
theora
|
||||
vorbis
|
||||
gst/
|
||||
subparse
|
||||
typefind : already covered in typefind target
|
||||
gst-libs/gst/
|
||||
sdp
|
||||
other ones easily testable directly ?
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
185
fuzzing/build-oss-fuzz.sh
Executable file
185
fuzzing/build-oss-fuzz.sh
Executable file
|
@ -0,0 +1,185 @@
|
|||
#!/bin/bash -eu
|
||||
|
||||
# build-oss-fuzz.sh
|
||||
#
|
||||
# Build script which is executed by oss-fuzz build.sh
|
||||
#
|
||||
# $SRC: location of code checkouts
|
||||
# $OUT: location to put fuzzing targets and corpus
|
||||
# $WORK: writable directory where all compilation should be executed
|
||||
#
|
||||
# /!\ Do not override any CC, CXX, CFLAGS, ... variables
|
||||
#
|
||||
|
||||
# This script is divided in two parts
|
||||
#
|
||||
# 1) Build all the dependencies statically
|
||||
#
|
||||
# 2) Build the fuzzing targets
|
||||
|
||||
# Prefix where we will temporarily install everything
|
||||
PREFIX=$WORK/prefix
|
||||
mkdir -p $PREFIX
|
||||
# always try getting the arguments for static compilation/linking
|
||||
# Fixes GModule not being picked when gstreamer-1.0.pc is looked up by meson
|
||||
# more or less https://github.com/mesonbuild/meson/pull/6629
|
||||
export PKG_CONFIG="`which pkg-config` --static"
|
||||
export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig
|
||||
export PATH=$PREFIX/bin:$PATH
|
||||
|
||||
# Minimize gst-debug level/code
|
||||
export CFLAGS="$CFLAGS -DGST_LEVEL_MAX=2"
|
||||
|
||||
#
|
||||
echo "CFLAGS : " $CFLAGS
|
||||
echo "CXXFLAGS : " $CXXFLAGS
|
||||
PLUGIN_DIR=$PREFIX/lib/gstreamer-1.0
|
||||
|
||||
rm -rf $WORK/*
|
||||
|
||||
# Switch to work directory
|
||||
cd $WORK
|
||||
|
||||
# 1) BUILD GLIB AND GSTREAMER
|
||||
# Note: we build glib ourselves so that we get proper malloc/free backtraces
|
||||
tar xvJf $SRC/glib-2.64.2.tar.xz
|
||||
cd glib-2.64.2
|
||||
# options taken from glib's oss-fuzz build definition
|
||||
meson \
|
||||
--prefix=$PREFIX \
|
||||
--libdir=lib \
|
||||
--default-library=static \
|
||||
-Db_lundef=false \
|
||||
-Doss_fuzz=enabled \
|
||||
-Dlibmount=disabled \
|
||||
-Dinternal_pcre=true \
|
||||
_builddir
|
||||
ninja -C _builddir
|
||||
ninja -C _builddir install
|
||||
cd ..
|
||||
|
||||
# Note: We don't use/build orc since it still seems to be problematic
|
||||
# with clang and the various sanitizers.
|
||||
|
||||
# For now we only build core and base. Add other modules when/if needed
|
||||
for i in gstreamer gst-plugins-base;
|
||||
do
|
||||
mkdir -p $i
|
||||
cd $i
|
||||
meson \
|
||||
--prefix=$PREFIX \
|
||||
--libdir=lib \
|
||||
--default-library=static \
|
||||
-Db_lundef=false \
|
||||
-Ddoc=disabled \
|
||||
-Dexamples=disabled \
|
||||
-Dintrospection=disabled \
|
||||
-Dtracer_hooks=false \
|
||||
-Dregistry=false _builddir $SRC/$i
|
||||
ninja -C _builddir
|
||||
ninja -C _builddir install
|
||||
cd ..
|
||||
done
|
||||
|
||||
|
||||
|
||||
# 2) Build the target fuzzers
|
||||
|
||||
# All targets will be linked in with $LIB_FUZZING_ENGINE which contains the
|
||||
# actual fuzzing runner. Anything fuzzing engine can be used provided it calls
|
||||
# the same function as libfuzzer.
|
||||
|
||||
# Note: The fuzzer .o needs to be first compiled with CC and then linked with CXX
|
||||
|
||||
# We want to statically link everything, except for shared libraries
|
||||
# that are present on the base image. Those need to be specified
|
||||
# beforehand and explicitely linked dynamically If any of the static
|
||||
# dependencies require a pre-installed shared library, you need to add
|
||||
# that library to the following list
|
||||
PREDEPS_LDFLAGS="-Wl,-Bdynamic -ldl -lm -pthread -lrt -lpthread"
|
||||
|
||||
# These are the basic .pc dependencies required to build any of the fuzzing targets
|
||||
# That is : glib, gstreamer core and gst-app
|
||||
# The extra target-specific dependencies are to be specified later
|
||||
COMMON_DEPS="glib-2.0 gio-2.0 gstreamer-1.0 gstreamer-app-1.0"
|
||||
|
||||
# For each target, defined the following:
|
||||
# TARGET_DEPS : Extra .pc dependencies for the target (in addition to $COMMON_DEPS)
|
||||
# All dependencies (including sub-dependencies) must be speecified
|
||||
# PLUGINS : .a of the plugins to link
|
||||
# They must match the static plugins declared/registered in the target
|
||||
|
||||
#
|
||||
# TARGET : push-based ogg/theora/vorbis discoverer
|
||||
#
|
||||
# FIXME : Rename to discoverer_push_oggtheoravorbis
|
||||
|
||||
TARGET_DEPS=" gstreamer-pbutils-1.0 \
|
||||
gstreamer-video-1.0 \
|
||||
gstreamer-audio-1.0 \
|
||||
gstreamer-riff-1.0 \
|
||||
gstreamer-tag-1.0 \
|
||||
zlib ogg vorbis vorbisenc \
|
||||
theoraenc theoradec theora"
|
||||
|
||||
PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \
|
||||
$PLUGIN_DIR/libgsttypefindfunctions.a \
|
||||
$PLUGIN_DIR/libgstplayback.a \
|
||||
$PLUGIN_DIR/libgstapp.a \
|
||||
$PLUGIN_DIR/libgstvorbis.a \
|
||||
$PLUGIN_DIR/libgsttheora.a \
|
||||
$PLUGIN_DIR/libgstogg.a"
|
||||
|
||||
echo
|
||||
echo ">>>> BUILDING gst-discoverer"
|
||||
echo
|
||||
BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`"
|
||||
BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`"
|
||||
|
||||
$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/gst-discoverer.c -o $SRC/gst-ci/fuzzing/gst-discoverer.o
|
||||
$CXX $CXXFLAGS \
|
||||
-o $OUT/gst-discoverer \
|
||||
$PREDEPS_LDFLAGS \
|
||||
$SRC/gst-ci/fuzzing/gst-discoverer.o \
|
||||
$PLUGINS \
|
||||
$BUILD_LDFLAGS \
|
||||
$LIB_FUZZING_ENGINE \
|
||||
-Wl,-Bdynamic
|
||||
|
||||
#
|
||||
# TARGET : push-based typefind
|
||||
#
|
||||
|
||||
# typefindfunction depends on pbutils which depends on gst{audio|video|tag}
|
||||
TARGET_DEPS=" gstreamer-pbutils-1.0 \
|
||||
gstreamer-video-1.0 \
|
||||
gstreamer-audio-1.0 \
|
||||
gstreamer-tag-1.0"
|
||||
|
||||
PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \
|
||||
$PLUGIN_DIR/libgsttypefindfunctions.a \
|
||||
$PLUGIN_DIR/libgstapp.a"
|
||||
|
||||
echo
|
||||
echo ">>>> BUILDING typefind"
|
||||
echo
|
||||
BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`"
|
||||
BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`"
|
||||
|
||||
$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/typefind.c -o $SRC/gst-ci/fuzzing/typefind.o
|
||||
$CXX $CXXFLAGS \
|
||||
-o $OUT/typefind \
|
||||
$PREDEPS_LDFLAGS \
|
||||
$SRC/gst-ci/fuzzing/typefind.o \
|
||||
$PLUGINS \
|
||||
$BUILD_LDFLAGS \
|
||||
$LIB_FUZZING_ENGINE \
|
||||
-Wl,-Bdynamic
|
||||
|
||||
echo
|
||||
echo ">>>> Installing seed corpus"
|
||||
echo
|
||||
# FIXME : Sadly we apparently need to have the corpus downloaded in the
|
||||
# Dockerfile and not here.
|
||||
|
||||
cp $SRC/*_seed_corpus.zip $OUT
|
137
fuzzing/gst-discoverer.c
Normal file
137
fuzzing/gst-discoverer.c
Normal file
|
@ -0,0 +1,137 @@
|
|||
/*
|
||||
* Copyright 2016 Google Inc.
|
||||
* author: Edward Hervey <bilboed@bilboed.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
|
||||
#include <locale.h>
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <glib.h>
|
||||
#include <gst/gst.h>
|
||||
#include <gst/pbutils/pbutils.h>
|
||||
|
||||
#ifndef LOCAL_FUZZ_BUILD
|
||||
GST_PLUGIN_STATIC_DECLARE (coreelements);
|
||||
GST_PLUGIN_STATIC_DECLARE (playback);
|
||||
GST_PLUGIN_STATIC_DECLARE (typefindfunctions);
|
||||
GST_PLUGIN_STATIC_DECLARE (app);
|
||||
GST_PLUGIN_STATIC_DECLARE (ogg);
|
||||
GST_PLUGIN_STATIC_DECLARE (theora);
|
||||
GST_PLUGIN_STATIC_DECLARE (vorbis);
|
||||
#endif
|
||||
|
||||
/* push-based discoverer fuzzing target
|
||||
*
|
||||
* This application can be compiled with libFuzzer to simulate
|
||||
* a push-based discoverer execution.
|
||||
*
|
||||
* To reproduce the failing behaviour, use:
|
||||
* $ gst-discoverer-1.0 pushfile:///...
|
||||
*
|
||||
* The goal is to cover basic usage of demuxers, parsers and
|
||||
* base decoder elements.
|
||||
*
|
||||
* When compiling, only link the required demuxer/parser/decoder
|
||||
* plugins and keep it to a limited range (ex: ogg/theora/vorbis)
|
||||
*
|
||||
**/
|
||||
|
||||
const guint8 *fuzztesting_data;
|
||||
size_t fuzztesting_size;
|
||||
|
||||
static void
|
||||
appsrc_configuration (GstDiscoverer * dc, GstElement * source, gpointer data)
|
||||
{
|
||||
GstBuffer *buf;
|
||||
GstFlowReturn ret;
|
||||
|
||||
/* Create buffer from fuzztesting_data which shouldn't be freed */
|
||||
buf =
|
||||
gst_buffer_new_wrapped_full (0, (gpointer) fuzztesting_data,
|
||||
fuzztesting_size, 0, fuzztesting_size, NULL, NULL);
|
||||
g_object_set (G_OBJECT (source), "size", fuzztesting_size, NULL);
|
||||
g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &ret);
|
||||
gst_buffer_unref (buf);
|
||||
}
|
||||
|
||||
static void
|
||||
custom_logger (const gchar * log_domain,
|
||||
GLogLevelFlags log_level, const gchar * message, gpointer unused_data)
|
||||
{
|
||||
if (log_level & G_LOG_LEVEL_CRITICAL) {
|
||||
g_printerr ("CRITICAL ERROR : %s\n", message);
|
||||
abort ();
|
||||
} else if (log_level & G_LOG_LEVEL_WARNING) {
|
||||
g_printerr ("WARNING : %s\n", message);
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
|
||||
{
|
||||
GError *err = NULL;
|
||||
GstDiscoverer *dc;
|
||||
gint timeout = 10;
|
||||
GstDiscovererInfo *info;
|
||||
static gboolean initialized = FALSE;
|
||||
|
||||
if (!initialized) {
|
||||
/* We want critical warnings to assert so we can fix them */
|
||||
g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL);
|
||||
g_log_set_default_handler (custom_logger, NULL);
|
||||
|
||||
/* Only initialize and register plugins once */
|
||||
gst_init (NULL, NULL);
|
||||
|
||||
#ifndef LOCAL_FUZZ_BUILD
|
||||
GST_PLUGIN_STATIC_REGISTER (coreelements);
|
||||
GST_PLUGIN_STATIC_REGISTER (playback);
|
||||
GST_PLUGIN_STATIC_REGISTER (typefindfunctions);
|
||||
GST_PLUGIN_STATIC_REGISTER (app);
|
||||
GST_PLUGIN_STATIC_REGISTER (ogg);
|
||||
GST_PLUGIN_STATIC_REGISTER (theora);
|
||||
GST_PLUGIN_STATIC_REGISTER (vorbis);
|
||||
#endif
|
||||
|
||||
initialized = TRUE;
|
||||
}
|
||||
|
||||
dc = gst_discoverer_new (timeout * GST_SECOND, &err);
|
||||
if (G_UNLIKELY (dc == NULL)) {
|
||||
g_print ("Error initializing: %s\n", err->message);
|
||||
g_clear_error (&err);
|
||||
exit (1);
|
||||
}
|
||||
|
||||
fuzztesting_data = data;
|
||||
fuzztesting_size = size;
|
||||
|
||||
/* Connect to source-setup signal to give the data */
|
||||
g_signal_connect (dc, "source-setup", (GCallback) appsrc_configuration, NULL);
|
||||
|
||||
info = gst_discoverer_discover_uri (dc, "appsrc://", &err);
|
||||
g_clear_error (&err);
|
||||
if (info)
|
||||
gst_discoverer_info_unref (info);
|
||||
|
||||
g_object_unref (dc);
|
||||
|
||||
return 0;
|
||||
}
|
78
fuzzing/localfuzzer.c
Normal file
78
fuzzing/localfuzzer.c
Normal file
|
@ -0,0 +1,78 @@
|
|||
/* GStreamer
|
||||
* Copyright (C) 2017 Edward Hervey <bilboed@bilboed.com>
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Library General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Library General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Library General Public
|
||||
* License along with this library; if not, write to the
|
||||
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
||||
* Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/* Local fuzzer runner */
|
||||
#include <glib.h>
|
||||
|
||||
extern int LLVMFuzzerTestOneInput (const guint8 * data, size_t size);
|
||||
|
||||
static void
|
||||
test_file (gchar * filename)
|
||||
{
|
||||
GDir *dir;
|
||||
gchar *path;
|
||||
gchar *contents;
|
||||
gsize length;
|
||||
|
||||
/* if filename is a directory, process the contents */
|
||||
if ((dir = g_dir_open (filename, 0, NULL))) {
|
||||
const gchar *entry;
|
||||
|
||||
while ((entry = g_dir_read_name (dir))) {
|
||||
gchar *spath;
|
||||
|
||||
spath = g_strconcat (filename, G_DIR_SEPARATOR_S, entry, NULL);
|
||||
test_file (spath);
|
||||
g_free (spath);
|
||||
}
|
||||
|
||||
g_dir_close (dir);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Make sure path is absolute */
|
||||
if (!g_path_is_absolute (filename)) {
|
||||
gchar *curdir;
|
||||
|
||||
curdir = g_get_current_dir ();
|
||||
path = g_build_filename (curdir, filename, NULL);
|
||||
g_free (curdir);
|
||||
} else
|
||||
path = g_strdup (filename);
|
||||
|
||||
/* Check if path exists */
|
||||
if (g_file_get_contents (path, &contents, &length, NULL)) {
|
||||
g_print (">>> %s (%" G_GSIZE_FORMAT " bytes)\n", path, length);
|
||||
LLVMFuzzerTestOneInput ((const guint8 *) contents, length);
|
||||
g_free (contents);
|
||||
}
|
||||
|
||||
g_free (path);
|
||||
}
|
||||
|
||||
int
|
||||
main (int argc, gchar ** argv)
|
||||
{
|
||||
gint i;
|
||||
|
||||
for (i = 1; i < argc; i++)
|
||||
test_file (argv[i]);
|
||||
|
||||
return 0;
|
||||
}
|
115
fuzzing/typefind.c
Normal file
115
fuzzing/typefind.c
Normal file
|
@ -0,0 +1,115 @@
|
|||
/*
|
||||
* Copyright 2016 Google Inc.
|
||||
* author: Edward Hervey <bilboed@bilboed.com>
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
|
||||
#include <locale.h>
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <glib.h>
|
||||
#include <gst/gst.h>
|
||||
|
||||
#ifndef LOCAL_FUZZ_BUILD
|
||||
GST_PLUGIN_STATIC_DECLARE (coreelements);
|
||||
GST_PLUGIN_STATIC_DECLARE (typefindfunctions);
|
||||
GST_PLUGIN_STATIC_DECLARE (app);
|
||||
#endif
|
||||
|
||||
/* push-based typefind fuzzing target
|
||||
*
|
||||
* This application can be compiled with libFuzzer to simulate
|
||||
* a push-based typefind execution.
|
||||
*
|
||||
* To reproduce the failing behaviour, use:
|
||||
* $ gst-launch-1.0 pushfile:///.. ! typefind ! fakesink
|
||||
*
|
||||
* The goal is to cover typefind code and implementation.
|
||||
*
|
||||
**/
|
||||
static void
|
||||
custom_logger (const gchar * log_domain,
|
||||
GLogLevelFlags log_level, const gchar * message, gpointer unused_data)
|
||||
{
|
||||
if (log_level & G_LOG_LEVEL_CRITICAL) {
|
||||
g_printerr ("CRITICAL ERROR : %s\n", message);
|
||||
abort ();
|
||||
} else if (log_level & G_LOG_LEVEL_WARNING) {
|
||||
g_printerr ("WARNING : %s\n", message);
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
|
||||
{
|
||||
GError *err = NULL;
|
||||
static gboolean initialized = FALSE;
|
||||
GstElement *pipeline, *source, *typefind, *fakesink;
|
||||
GstBuffer *buf;
|
||||
GstFlowReturn flowret;
|
||||
GstState state;
|
||||
|
||||
if (!initialized) {
|
||||
/* We want critical warnings to assert so we can fix them */
|
||||
g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL);
|
||||
g_log_set_default_handler (custom_logger, NULL);
|
||||
|
||||
/* Only initialize and register plugins once */
|
||||
gst_init (NULL, NULL);
|
||||
|
||||
#ifndef LOCAL_FUZZ_BUILD
|
||||
GST_PLUGIN_STATIC_REGISTER (coreelements);
|
||||
GST_PLUGIN_STATIC_REGISTER (typefindfunctions);
|
||||
GST_PLUGIN_STATIC_REGISTER (app);
|
||||
#endif
|
||||
|
||||
initialized = TRUE;
|
||||
}
|
||||
|
||||
/* Create the pipeline */
|
||||
pipeline = gst_pipeline_new ("pipeline");
|
||||
source = gst_element_factory_make ("appsrc", "source");
|
||||
typefind = gst_element_factory_make ("typefind", "typefind");
|
||||
fakesink = gst_element_factory_make ("fakesink", "fakesink");
|
||||
|
||||
gst_bin_add_many (GST_BIN (pipeline), source, typefind, fakesink, NULL);
|
||||
gst_element_link_many (source, typefind, fakesink, NULL);
|
||||
|
||||
/* Set pipeline to READY so we can provide data to appsrc */
|
||||
gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_READY);
|
||||
buf = gst_buffer_new_wrapped_full (0, (gpointer) data, size,
|
||||
0, size, NULL, NULL);
|
||||
g_object_set (G_OBJECT (source), "size", size, NULL);
|
||||
g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &flowret);
|
||||
gst_buffer_unref (buf);
|
||||
|
||||
/* Set pipeline to PAUSED and wait (typefind will either fail or succeed) */
|
||||
gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_PAUSED);
|
||||
|
||||
/* wait until state change either completes or fails */
|
||||
gst_element_get_state (GST_ELEMENT (pipeline), &state, NULL, -1);
|
||||
|
||||
/* Go back to NULL */
|
||||
gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_NULL);
|
||||
|
||||
/* And release the pipeline */
|
||||
gst_object_unref (pipeline);
|
||||
|
||||
return 0;
|
||||
}
|
270
gitlab/build_manifest.py
Executable file
270
gitlab/build_manifest.py
Executable file
|
@ -0,0 +1,270 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import json
|
||||
|
||||
from typing import Dict, Tuple, List
|
||||
# from pprint import pprint
|
||||
|
||||
if sys.version_info < (3, 6):
|
||||
raise SystemExit('Need Python 3.6 or newer')
|
||||
|
||||
GSTREAMER_MODULES: List[str] = [
|
||||
'orc',
|
||||
'cerbero',
|
||||
'gst-build',
|
||||
'gstreamer',
|
||||
'gst-plugins-base',
|
||||
'gst-plugins-good',
|
||||
'gst-plugins-bad',
|
||||
'gst-plugins-ugly',
|
||||
'gst-libav',
|
||||
'gst-devtools',
|
||||
'gst-docs',
|
||||
'gst-editing-services',
|
||||
'gst-omx',
|
||||
'gst-python',
|
||||
'gst-rtsp-server',
|
||||
'gstreamer-sharp',
|
||||
'gstreamer-vaapi',
|
||||
'gst-integration-testsuites',
|
||||
'gst-examples',
|
||||
]
|
||||
|
||||
MANIFEST_TEMPLATE: str = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<manifest>
|
||||
<remote fetch="{}" name="user"/>
|
||||
<remote fetch="https://gitlab.freedesktop.org/gstreamer/" name="origin"/>
|
||||
{}
|
||||
</manifest>"""
|
||||
|
||||
|
||||
CERBERO_DEPS_LOGS_TARGETS = (
|
||||
('cross-ios', 'universal'),
|
||||
('cross-windows-mingw', 'x86'),
|
||||
('cross-windows-mingw', 'x86_64'),
|
||||
('cross-android', 'universal'),
|
||||
('fedora', 'x86_64'),
|
||||
('macos', 'x86_64'),
|
||||
('windows-msvc', 'x86_64'),
|
||||
)
|
||||
|
||||
# Disallow git prompting for a username/password
|
||||
os.environ['GIT_TERMINAL_PROMPT'] = '0'
|
||||
def git(*args, repository_path='.'):
|
||||
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
|
||||
|
||||
def get_cerbero_last_build_info (branch : str):
|
||||
# Fetch the deps log for all (distro, arch) targets
|
||||
all_commits = {}
|
||||
for distro, arch in CERBERO_DEPS_LOGS_TARGETS:
|
||||
url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/{distro}/{arch}/cerbero-deps.log'
|
||||
print(f'Fetching {url}')
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
resp = urllib.request.urlopen(req);
|
||||
deps = json.loads(resp.read())
|
||||
except urllib.error.URLError as e:
|
||||
print(f'WARNING: Failed to GET {url}: {e!s}')
|
||||
continue
|
||||
|
||||
for dep in deps:
|
||||
commit = dep['commit']
|
||||
if commit not in all_commits:
|
||||
all_commits[commit] = []
|
||||
all_commits[commit].append((distro, arch))
|
||||
|
||||
# Fetch the cerbero commit that has the most number of caches
|
||||
best_commit = None
|
||||
newest_commit = None
|
||||
max_caches = 0
|
||||
total_caches = len(CERBERO_DEPS_LOGS_TARGETS)
|
||||
for commit, targets in all_commits.items():
|
||||
if newest_commit is None:
|
||||
newest_commit = commit
|
||||
have_caches = len(targets)
|
||||
# If this commit has caches for all targets, just use it
|
||||
if have_caches == total_caches:
|
||||
best_commit = commit
|
||||
break
|
||||
# Else, try to find the commit with the most caches
|
||||
if have_caches > max_caches:
|
||||
max_caches = have_caches
|
||||
best_commit = commit
|
||||
if newest_commit is None:
|
||||
print('WARNING: No deps logs were found, will build from scratch')
|
||||
if best_commit != newest_commit:
|
||||
print(f'WARNING: Cache is not up-to-date for commit {newest_commit}, using commit {best_commit} instead')
|
||||
return best_commit
|
||||
|
||||
|
||||
def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]:
|
||||
try:
|
||||
res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch)
|
||||
except subprocess.CalledProcessError:
|
||||
return None, None
|
||||
|
||||
if not res:
|
||||
return None, None
|
||||
|
||||
# Special case cerbero to avoid cache misses
|
||||
if module == 'cerbero':
|
||||
sha = get_cerbero_last_build_info(branch)
|
||||
if sha is not None:
|
||||
return sha, sha
|
||||
|
||||
lines = res.split('\n')
|
||||
for line in lines:
|
||||
if line.endswith('/' + branch):
|
||||
try:
|
||||
sha, refname = line.split('\t')
|
||||
except ValueError:
|
||||
continue
|
||||
return refname.strip(), sha
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]:
|
||||
namespace: str = os.environ["CI_PROJECT_NAMESPACE"]
|
||||
ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master')
|
||||
|
||||
if module == "orc":
|
||||
ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master')
|
||||
|
||||
if module == os.environ['CI_PROJECT_NAME']:
|
||||
return 'user', branchname, os.environ['CI_COMMIT_SHA']
|
||||
|
||||
if branchname != ups_branch:
|
||||
remote_refname, sha = get_branch_info(module, namespace, branchname)
|
||||
if sha is not None:
|
||||
return 'user', remote_refname, sha
|
||||
|
||||
# Check upstream project for a branch
|
||||
remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch)
|
||||
if sha is not None:
|
||||
return 'origin', remote_refname, sha
|
||||
|
||||
# This should never occur given the upstream fallback above
|
||||
print(f"Could not find anything for {module}:{branchname}")
|
||||
print("If something reaches that point, please file a bug")
|
||||
print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues")
|
||||
assert False
|
||||
|
||||
|
||||
# --- Unit tests --- #
|
||||
# Basically, pytest will happily let a test mutate a variable, and then run
|
||||
# the next tests one the same environment without reset the vars.
|
||||
def preserve_ci_vars(func):
|
||||
"""Preserve the original CI Variable values"""
|
||||
def wrapper():
|
||||
try:
|
||||
url = os.environ["CI_PROJECT_URL"]
|
||||
user = os.environ["CI_PROJECT_NAMESPACE"]
|
||||
except KeyError:
|
||||
url = "invalid"
|
||||
user = ""
|
||||
|
||||
private = os.getenv("READ_PROJECTS_TOKEN", default=None)
|
||||
if not private:
|
||||
os.environ["READ_PROJECTS_TOKEN"] = "FOO"
|
||||
|
||||
func()
|
||||
|
||||
os.environ["CI_PROJECT_URL"] = url
|
||||
os.environ["CI_PROJECT_NAMESPACE"] = user
|
||||
|
||||
if private:
|
||||
os.environ["READ_PROJECTS_TOKEN"] = private
|
||||
# if it was set after
|
||||
elif os.getenv("READ_PROJECTS_TOKEN", default=None):
|
||||
del os.environ["READ_PROJECTS_TOKEN"]
|
||||
|
||||
return wrapper
|
||||
|
||||
@preserve_ci_vars
|
||||
def test_find_repository_sha():
|
||||
os.environ["CI_PROJECT_NAME"] = "some-random-project"
|
||||
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
|
||||
os.environ["CI_PROJECT_NAMESPACE"] = "alatiera"
|
||||
os.environ["GST_UPSTREAM_BRANCH"] = "master"
|
||||
del os.environ["READ_PROJECTS_TOKEN"]
|
||||
|
||||
# This should find the repository in the user namespace
|
||||
remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2")
|
||||
assert remote == "user"
|
||||
assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325"
|
||||
assert refname == "refs/heads/1.2"
|
||||
|
||||
# This should fallback to upstream master branch since no matching branch was found
|
||||
remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name")
|
||||
assert remote == "origin"
|
||||
assert refname == "refs/heads/master"
|
||||
|
||||
os.environ["CI_PROJECT_NAME"] = "the_project"
|
||||
os.environ["CI_COMMIT_SHA"] = "MySha"
|
||||
|
||||
remote, refname, git_ref = find_repository_sha("the_project", "whatever")
|
||||
assert remote == "user"
|
||||
assert git_ref == "MySha"
|
||||
assert refname == "whatever"
|
||||
|
||||
|
||||
@preserve_ci_vars
|
||||
def test_get_project_branch():
|
||||
os.environ["CI_PROJECT_NAME"] = "some-random-project"
|
||||
os.environ["CI_COMMIT_SHA"] = "dwbuiw"
|
||||
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
|
||||
os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_"
|
||||
del os.environ["READ_PROJECTS_TOKEN"]
|
||||
|
||||
os.environ['GST_UPSTREAM_BRANCH'] = '1.12'
|
||||
remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12')
|
||||
assert twelve is not None
|
||||
assert remote == 'origin'
|
||||
assert refname == "refs/heads/1.12"
|
||||
|
||||
os.environ['GST_UPSTREAM_BRANCH'] = '1.14'
|
||||
remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14')
|
||||
assert fourteen is not None
|
||||
assert remote == 'origin'
|
||||
assert refname == "refs/heads/1.14"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--self-update", action="store_true", default=False)
|
||||
parser.add_argument(dest="output", default='manifest.xml', nargs='?')
|
||||
options = parser.parse_args()
|
||||
|
||||
current_branch: str = os.environ['CI_COMMIT_REF_NAME']
|
||||
user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL'])
|
||||
if not user_remote_url.endswith('/'):
|
||||
user_remote_url += '/'
|
||||
|
||||
if options.self_update:
|
||||
remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch)
|
||||
if remote == 'user':
|
||||
remote = user_remote_url + 'gst-ci'
|
||||
else:
|
||||
remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci"
|
||||
|
||||
git('fetch', remote, remote_refname)
|
||||
git('checkout', '--detach', sha)
|
||||
sys.exit(0)
|
||||
|
||||
projects: str = ''
|
||||
for module in GSTREAMER_MODULES:
|
||||
print(f"Checking {module}:", end=' ')
|
||||
remote, refname, revision = find_repository_sha(module, current_branch)
|
||||
print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'")
|
||||
projects += f" <project path=\"{module}\" name=\"{module}.git\" remote=\"{remote}\" revision=\"{revision}\" refname=\"{refname}\" />\n"
|
||||
|
||||
with open(options.output, mode='w') as manifest:
|
||||
print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest)
|
169
gitlab/cerbero_setup.sh
Normal file
169
gitlab/cerbero_setup.sh
Normal file
|
@ -0,0 +1,169 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
show_ccache_sum() {
|
||||
if [[ -n ${HAVE_CCACHE} ]]; then
|
||||
ccache -s
|
||||
fi
|
||||
}
|
||||
|
||||
# XXX: This is copied and modified from the cerbero-uninstalled script
|
||||
# Use `mount` to get a list of MSYS mount points that the MSYS shell uses.
|
||||
# That's our reference point for translating from MSYS paths to Win32 paths.
|
||||
# We assume that the MSYS mount point directories are only in the filesystem
|
||||
# root. This will break if people add their own custom mount points beyond what
|
||||
# MSYS automatically creates, which is highly unlikely.
|
||||
#
|
||||
# /d -> d:/
|
||||
# /c -> c:/
|
||||
# /d/projects/cerbero -> d:/projects/cerbero/
|
||||
# /home/USERNAME/cerbero -> C:\\MinGW\\msys\\1.0/home/USERNAME/
|
||||
# /mingw -> C:\\MinGW/
|
||||
# /mingw/bin/foobar -> C:\\MinGW\\bin/foobar/
|
||||
# /tmp/baz -> C:\\Users\\USERNAME\\AppData\\Local\\Temp/baz/
|
||||
msys_dir_to_win32() {
|
||||
set -e
|
||||
local msys_path stripped_path mount_point path mounted_path
|
||||
# If the path is already a native path, just return that
|
||||
if [[ $1 == ?:/* ]] || [[ $1 == ?:\\* ]]; then
|
||||
echo $1
|
||||
return
|
||||
fi
|
||||
# Convert /c or /mingw etc to /c/ or /mingw/ etc; gives us a necessary
|
||||
# anchor to split the path into components
|
||||
msys_path="$1/"
|
||||
# Strip leading slash
|
||||
stripped_path="${msys_path#/}"
|
||||
# Get the first path component, which may be a mount point
|
||||
mount_point="/${stripped_path%%/*}"
|
||||
# Get the path inside the mountp oint
|
||||
path="/${stripped_path#*/}"
|
||||
mounted_path="$(mount | sed -n "s|\(.*\) on $mount_point type.*|\1|p")"
|
||||
# If it's not a mounted path (like /c or /tmp or /mingw), then it's in the
|
||||
# general MSYS root mount
|
||||
if [[ -z $mounted_path ]]; then
|
||||
mounted_path="$(mount | sed -n "s|\(.*\) on / type.*|\1|p")"
|
||||
path="$1"
|
||||
fi
|
||||
echo ${mounted_path}${path%/}
|
||||
}
|
||||
|
||||
# Print the working directory in the native OS path format, but with forward
|
||||
# slashes
|
||||
pwd_native() {
|
||||
if [[ -n "$MSYSTEM" ]]; then
|
||||
msys_dir_to_win32 "$(pwd)"
|
||||
else
|
||||
pwd
|
||||
fi
|
||||
}
|
||||
|
||||
fix_build_tools() {
|
||||
if [[ $(uname) == Darwin ]]; then
|
||||
# Bison needs these env vars set for the build-tools prefix to be
|
||||
# relocatable, and we only build it on macOS. On Linux we install it
|
||||
# using the package manager, and on Windows we use the MSYS Bison.
|
||||
export M4="$(pwd)/${CERBERO_HOME}/build-tools/bin/m4"
|
||||
export BISON_PKGDATADIR="$(pwd)/${CERBERO_HOME}/build-tools/share/bison"
|
||||
fi
|
||||
}
|
||||
|
||||
# Produces runtime and devel tarball packages for linux/android or .pkg for macos
|
||||
cerbero_package_and_check() {
|
||||
$CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0
|
||||
|
||||
# Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course.
|
||||
if [[ $CONFIG != *universal* ]]; then
|
||||
$CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version
|
||||
$CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX
|
||||
fi
|
||||
|
||||
show_ccache_sum
|
||||
}
|
||||
|
||||
cerbero_before_script() {
|
||||
pwd
|
||||
ls -lha
|
||||
|
||||
# Copy cerbero git repo stored on the image
|
||||
cp -a "${CERBERO_HOST_DIR}/.git" .
|
||||
git checkout .
|
||||
git status
|
||||
|
||||
# If there's no cerbero-sources directory in the runner cache, copy it from
|
||||
# the image cache
|
||||
if ! [[ -d ${CERBERO_SOURCES} ]]; then
|
||||
time cp -a "${CERBERO_HOST_DIR}/${CERBERO_SOURCES}" .
|
||||
fi
|
||||
du -sch "${CERBERO_SOURCES}" || true
|
||||
|
||||
echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc
|
||||
echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc
|
||||
if [[ $CONFIG == win??.cbc ]]; then
|
||||
# Visual Studio 2017 build tools install path
|
||||
echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc
|
||||
echo 'vs_install_version = "vs15"' >> localconf.cbc
|
||||
fi
|
||||
cat localconf.cbc
|
||||
|
||||
time ./cerbero-uninstalled --self-update manifest.xml
|
||||
|
||||
# GitLab runner does not always wipe the image after each job, so do that
|
||||
# to ensure we don't have any leftover data from a previous job such as
|
||||
# a dirty builddir, or tarballs/pkg files, leftover files from an old
|
||||
# cerbero commit, etc. Skip the things we actually need to keep.
|
||||
time git clean -xdff -e cerbero_setup.sh -e manifest.xml -e localconf.cbc -e "${CERBERO_SOURCES}"
|
||||
}
|
||||
|
||||
cerbero_script() {
|
||||
show_ccache_sum
|
||||
|
||||
$CERBERO $CERBERO_ARGS show-config
|
||||
$CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4
|
||||
$CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0
|
||||
du -sch "${CERBERO_SOURCES}" || true
|
||||
|
||||
$CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}"
|
||||
|
||||
if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} && -d "${CERBERO_HOME}/dist/${ARCH}" ]]; then
|
||||
mkdir -p "${CERBERO_OVERRIDDEN_DIST_DIR}"
|
||||
time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}"
|
||||
fi
|
||||
|
||||
$CERBERO $CERBERO_ARGS bootstrap --offline --system=no
|
||||
fix_build_tools
|
||||
|
||||
cerbero_package_and_check
|
||||
}
|
||||
|
||||
cerbero_deps_script() {
|
||||
show_ccache_sum
|
||||
|
||||
$CERBERO $CERBERO_ARGS show-config
|
||||
$CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4
|
||||
$CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0
|
||||
$CERBERO $CERBERO_ARGS bootstrap --offline --system=no
|
||||
$CERBERO $CERBERO_ARGS build-deps --offline \
|
||||
gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \
|
||||
gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \
|
||||
gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice
|
||||
|
||||
if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then
|
||||
mkdir -p "${CERBERO_HOME}/dist/${ARCH}"
|
||||
time rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}"
|
||||
fi
|
||||
|
||||
# Check that the env var is set. Don't expand this protected variable by
|
||||
# doing something silly like [[ -n ${CERBERO_...} ]] because it will get
|
||||
# printed in the CI logs due to set -x
|
||||
if env | grep -q -e CERBERO_PRIVATE_SSH_KEY; then
|
||||
time $CERBERO $CERBERO_ARGS gen-cache --branch "${GST_UPSTREAM_BRANCH}"
|
||||
time $CERBERO $CERBERO_ARGS upload-cache --branch "${GST_UPSTREAM_BRANCH}"
|
||||
fi
|
||||
|
||||
cerbero_package_and_check
|
||||
}
|
||||
|
||||
# Run whichever function is asked of us
|
||||
eval "$1"
|
1427
gitlab/ci_template.yml
Normal file
1427
gitlab/ci_template.yml
Normal file
File diff suppressed because it is too large
Load diff
92
gitlab/clone_manifest_ref.py
Executable file
92
gitlab/clone_manifest_ref.py
Executable file
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from collections import namedtuple
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
# Disallow git prompting for a username/password
|
||||
os.environ['GIT_TERMINAL_PROMPT'] = '0'
|
||||
def git(*args, repository_path='.'):
|
||||
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
|
||||
|
||||
class Manifest(object):
|
||||
'''
|
||||
Parse and store the content of a manifest file
|
||||
'''
|
||||
|
||||
remotes = {}
|
||||
projects = {}
|
||||
default_remote = 'origin'
|
||||
default_revision = 'refs/heads/master'
|
||||
|
||||
def __init__(self, manifest_path):
|
||||
self.manifest_path = manifest_path
|
||||
|
||||
def parse(self):
|
||||
try:
|
||||
tree = ET.parse(self.manifest_path)
|
||||
except Exception as ex:
|
||||
raise Exception("Error loading manifest %s in file %s" % (self.manifest_path, ex))
|
||||
|
||||
root = tree.getroot()
|
||||
|
||||
for child in root:
|
||||
if child.tag == 'remote':
|
||||
self.remotes[child.attrib['name']] = child.attrib['fetch']
|
||||
if child.tag == 'default':
|
||||
self.default_remote = child.attrib['remote'] or self.default_remote
|
||||
self.default_revision = child.attrib['revision'] or self.default_revision
|
||||
if child.tag == 'project':
|
||||
project = namedtuple('Project', ['name', 'remote',
|
||||
'revision', 'fetch_uri'])
|
||||
|
||||
project.name = child.attrib['name']
|
||||
if project.name.endswith('.git'):
|
||||
project.name = project.name[:-4]
|
||||
project.remote = child.attrib.get('remote') or self.default_remote
|
||||
project.revision = child.attrib.get('revision') or self.default_revision
|
||||
project.fetch_uri = self.remotes[project.remote] + project.name + '.git'
|
||||
|
||||
self.projects[project.name] = project
|
||||
|
||||
def find_project(self, name):
|
||||
try:
|
||||
return self.projects[name]
|
||||
except KeyError as ex:
|
||||
raise Exception("Could not find project %s in manifest %s" % (name, self.manifest_path))
|
||||
|
||||
def get_fetch_uri(self, project, remote):
|
||||
fetch = self.remotes[remote]
|
||||
return fetch + project.name + '.git'
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--project", action="store", type=str)
|
||||
parser.add_argument("--destination", action="store", type=str, default='.')
|
||||
parser.add_argument("--manifest", action="store", type=str)
|
||||
parser.add_argument("--fetch", action="store_true", default=False)
|
||||
options = parser.parse_args()
|
||||
|
||||
if not options.project:
|
||||
raise ValueError("--project argument not provided")
|
||||
if not options.manifest:
|
||||
raise ValueError("--manifest argument not provided")
|
||||
|
||||
manifest = Manifest(options.manifest)
|
||||
manifest.parse()
|
||||
project = manifest.find_project(options.project)
|
||||
|
||||
dest = options.destination
|
||||
if dest == '.':
|
||||
dest = os.path.join (os.getcwd(), project.name)
|
||||
|
||||
if options.fetch:
|
||||
assert os.path.exists(dest) == True
|
||||
git('fetch', project.fetch_uri, project.revision, repository_path=dest)
|
||||
else:
|
||||
git('clone', project.fetch_uri, dest)
|
||||
|
||||
git('checkout', '--detach', project.revision, repository_path=dest)
|
37
gitlab/freedesktop_doc_importer.sh
Executable file
37
gitlab/freedesktop_doc_importer.sh
Executable file
|
@ -0,0 +1,37 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
BRANCH=master
|
||||
NAMESPACE=gstreamer
|
||||
JOB=documentation
|
||||
|
||||
WORK_DIR=`mktemp -d -p "$DIR"`
|
||||
|
||||
# deletes the temp directory
|
||||
function cleanup {
|
||||
rm -rf "$WORK_DIR"
|
||||
echo "Deleted temp working directory $WORK_DIR"
|
||||
}
|
||||
|
||||
# register the cleanup function to be called on the EXIT signal
|
||||
trap cleanup EXIT
|
||||
|
||||
echo ""
|
||||
echo "============================================================================================================================"
|
||||
echo "Updating documentation from: https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=$JOB"
|
||||
|
||||
date
|
||||
|
||||
cd $WORK_DIR
|
||||
wget https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=$JOB -O gstdocs.zip
|
||||
|
||||
unzip gstdocs.zip
|
||||
|
||||
DOC_BASE="/srv/gstreamer.freedesktop.org/public_html/documentation"
|
||||
|
||||
rsync -rvaz --links --delete documentation/ $DOC_BASE || /bin/true
|
||||
chmod -R g+w $DOC_BASE; chgrp -R gstreamer $DOC_BASE
|
||||
|
||||
echo "Done updating documentation"
|
||||
echo ""
|
51
jenkins/Jenkinsfile-cerbero-debian
Normal file
51
jenkins/Jenkinsfile-cerbero-debian
Normal file
|
@ -0,0 +1,51 @@
|
|||
node("docker") {
|
||||
docker.image('gstreamer/build-base-ubuntu:latest').inside {
|
||||
|
||||
env.OUTPREFIX="${env.WORKSPACE}/../output/${params.build_tag}/linux_x86_64/"
|
||||
|
||||
stage('Checkout') {
|
||||
// FIXME: Only checkout the manifest and not all dependencies ?
|
||||
checkout([$class: 'RepoScm',
|
||||
manifestRepositoryUrl:'git+ssh://git.arracacha.collabora.co.uk/git/gst-manifest.git',
|
||||
manifestBranch:"refs/tags/${params.build_tag}",
|
||||
jobs:4,
|
||||
currentBranch:true,
|
||||
quiet:true,
|
||||
depth:0])
|
||||
}
|
||||
|
||||
stage('Setup') {
|
||||
sh "find ../output -maxdepth 1 -ctime +1 | xargs rm -Rf"
|
||||
sh "cd .repo/manifests/; git checkout ${params.build_tag}; cd ../.."
|
||||
sh "rm -Rf ./workdir/sources/linux_x86_64/"
|
||||
sh "rm -f *.rpm"
|
||||
sh "rm -Rf ./workdir/temp; mkdir -p ./workdir/temp"
|
||||
sh "rm -Rf ./workdir/tmp*"
|
||||
|
||||
// Create custom configuration file
|
||||
sh "./gst-ci-scripts/manifest2cerbero.py .repo/manifests/default.xml ./cerbero/config/linux.config --output localconf.cbc"
|
||||
sh '''echo "home_dir = \\"$WORKSPACE/workdir\\"" >> localconf.cbc'''
|
||||
sh '''echo "logs = \\"$OUTPREFIX/logs\\"" >> localconf.cbc'''
|
||||
sh './cerbero/cerbero-uninstalled -c localconf.cbc show-config'
|
||||
}
|
||||
|
||||
stage('bootstrap') {
|
||||
sh './cerbero/cerbero-uninstalled -c localconf.cbc bootstrap'
|
||||
sh 'rm -Rf ./workdir/sources/build-tools/'
|
||||
}
|
||||
|
||||
stage('fetch') {
|
||||
sh './cerbero/cerbero-uninstalled -c localconf.cbc fetch-package --reset-rdeps --full-reset gstreamer-1.0'
|
||||
}
|
||||
|
||||
stage('package') {
|
||||
sh './cerbero/cerbero-uninstalled -c localconf.cbc package gstreamer-1.0'
|
||||
}
|
||||
|
||||
stage('Cleanup') {
|
||||
sh 'rm -f *.rpm'
|
||||
sh 'find ../output -maxdepth 1 -ctime +1 | xargs rm -Rf'
|
||||
}
|
||||
// FIXME: IRC Notification
|
||||
}
|
||||
}
|
66
jenkins/Jenkinsfile-master-fedora
Normal file
66
jenkins/Jenkinsfile-master-fedora
Normal file
|
@ -0,0 +1,66 @@
|
|||
node('docker') {
|
||||
docker.image('gstreamer/build-base-fedora:latest').inside {
|
||||
|
||||
env.CCACHE_DIR = "${env.WORKSPACE}/.ccache"
|
||||
env.CCACHE_MAXSIZE = "2G"
|
||||
env.CC = "ccache gcc"
|
||||
env.CXX = "ccache g++"
|
||||
env.MAKEFLAGS = "-j6"
|
||||
env.PATH = "${env.WORKSPACE}:${env.PATH}"
|
||||
env.GST_UNINSTALLED_ROOT="${env.WORKSPACE}"
|
||||
env.HOME="${env.WORKSPACE}"
|
||||
env.DISPLAY=":0"
|
||||
|
||||
stage('Checkout') {
|
||||
if (params.wipe) {
|
||||
sh 'rm -Rf *'
|
||||
}
|
||||
checkout([$class: 'RepoScm',
|
||||
manifestRepositoryUrl:'https://git.arracacha.collabora.co.uk/git/gst-manifest.git',
|
||||
manifestBranch:"refs/tags/${params.build_tag}",
|
||||
jobs:4,
|
||||
currentBranch:true,
|
||||
quiet:true,
|
||||
depth:0,
|
||||
mirrorDir:'/repositories'])
|
||||
}
|
||||
|
||||
stage('Cleanup') {
|
||||
sh 'rm -f **/tests/check/*/*.xml'
|
||||
}
|
||||
|
||||
stage ('Build') {
|
||||
sh "uname -a"
|
||||
sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh fast-build-only"
|
||||
}
|
||||
|
||||
withEnv(['DISPLAY=:0']) {
|
||||
stage ('Check') {
|
||||
env.GST_CHECKS_IGNORE="test_allocate_udp_ports_multicast,test_allocate_udp_ports_client_settings,test_reorder_buffer,test_redirect_yes"
|
||||
env.GST_CHECK_XML=1
|
||||
sh 'Xvfb :0 -screen 0 1024x768x24 -fbdir /tmp &'
|
||||
sh 'env'
|
||||
sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh check"
|
||||
|
||||
step([$class: 'XUnitBuilder',
|
||||
testTimeMargin: '3000', thresholdMode: 1,
|
||||
thresholds: [[$class: 'FailedThreshold',
|
||||
failureNewThreshold: '',
|
||||
failureThreshold: '400',
|
||||
unstableNewThreshold: '',
|
||||
unstableThreshold: '1'],
|
||||
[$class: 'SkippedThreshold',
|
||||
failureNewThreshold: '',
|
||||
failureThreshold: '',
|
||||
unstableNewThreshold: '',
|
||||
unstableThreshold: '']],
|
||||
tools: [[$class: 'CheckType',
|
||||
deleteOutputFiles: true,
|
||||
failIfNotNew: true,
|
||||
pattern: '**/tests/check/*/*.xml',
|
||||
skipNoTestFiles: true,
|
||||
stopProcessingIfError: true]]])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
66
jenkins/Jenkinsfile-master-ubuntu
Normal file
66
jenkins/Jenkinsfile-master-ubuntu
Normal file
|
@ -0,0 +1,66 @@
|
|||
node('docker') {
|
||||
docker.image('gstreamer/build-base-ubuntu:latest').inside {
|
||||
|
||||
env.CCACHE_DIR = "${env.WORKSPACE}/.ccache"
|
||||
env.CCACHE_MAXSIZE = "2G"
|
||||
env.CC = "ccache gcc"
|
||||
env.CXX = "ccache g++"
|
||||
env.MAKEFLAGS = "-j6"
|
||||
env.PATH = "${env.WORKSPACE}:${env.PATH}"
|
||||
env.GST_UNINSTALLED_ROOT="${env.WORKSPACE}"
|
||||
env.HOME="${env.WORKSPACE}"
|
||||
env.DISPLAY=":0"
|
||||
|
||||
stage('Checkout') {
|
||||
if (params.wipe) {
|
||||
sh 'rm -Rf *'
|
||||
}
|
||||
checkout([$class: 'RepoScm',
|
||||
manifestRepositoryUrl:'https://git.arracacha.collabora.co.uk/git/gst-manifest.git',
|
||||
manifestBranch:"refs/tags/${params.build_tag}",
|
||||
jobs:4,
|
||||
currentBranch:true,
|
||||
quiet:true,
|
||||
depth:0,
|
||||
mirrorDir:'/repositories'])
|
||||
}
|
||||
|
||||
stage('Cleanup') {
|
||||
sh 'rm -f **/tests/check/*/*.xml'
|
||||
}
|
||||
|
||||
stage ('Build') {
|
||||
sh "uname -a"
|
||||
sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh fast-build-only"
|
||||
}
|
||||
|
||||
withEnv(['DISPLAY=:0']) {
|
||||
stage ('Check') {
|
||||
env.GST_CHECKS_IGNORE="test_allocate_udp_ports_multicast,test_allocate_udp_ports_client_settings,test_reorder_buffer,test_redirect_yes"
|
||||
env.GST_CHECK_XML=1
|
||||
sh 'Xvfb :0 -screen 0 1024x768x24 -fbdir /tmp &'
|
||||
sh 'env'
|
||||
sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh check"
|
||||
|
||||
step([$class: 'XUnitBuilder',
|
||||
testTimeMargin: '3000', thresholdMode: 1,
|
||||
thresholds: [[$class: 'FailedThreshold',
|
||||
failureNewThreshold: '',
|
||||
failureThreshold: '400',
|
||||
unstableNewThreshold: '',
|
||||
unstableThreshold: '1'],
|
||||
[$class: 'SkippedThreshold',
|
||||
failureNewThreshold: '',
|
||||
failureThreshold: '',
|
||||
unstableNewThreshold: '',
|
||||
unstableThreshold: '']],
|
||||
tools: [[$class: 'CheckType',
|
||||
deleteOutputFiles: true,
|
||||
failIfNotNew: true,
|
||||
pattern: '**/tests/check/*/*.xml',
|
||||
skipNoTestFiles: true,
|
||||
stopProcessingIfError: true]]])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
71
jenkins/Jenkinsfile-meson-fedora
Normal file
71
jenkins/Jenkinsfile-meson-fedora
Normal file
|
@ -0,0 +1,71 @@
|
|||
node('docker') {
|
||||
docker.image('gstreamer/build-meson-fedora:latest').inside {
|
||||
|
||||
env.CCACHE_DIR = "${env.WORKSPACE}/.ccache"
|
||||
env.CCACHE_MAXSIZE = "2G"
|
||||
env.CC = "ccache gcc"
|
||||
env.CXX = "ccache g++"
|
||||
env.MAKEFLAGS = "-j6"
|
||||
env.PATH = "${env.WORKSPACE}:${env.PATH}"
|
||||
env.HOME="${env.WORKSPACE}"
|
||||
env.DISPLAY=":0"
|
||||
|
||||
stage('Checkout') {
|
||||
if (params.wipe) {
|
||||
sh 'rm -Rf build/'
|
||||
}
|
||||
checkout([$class: 'GitSCM', branches: [[name: '*/master']],
|
||||
doGenerateSubmoduleConfigurations: false,
|
||||
extensions: [[$class: 'CloneOption',
|
||||
depth: 0,
|
||||
noTags: false,
|
||||
reference: '/gstbuild/gst-build/',
|
||||
shallow: false]],
|
||||
submoduleCfg: [],
|
||||
userRemoteConfigs: [[url: 'git://anongit.freedesktop.org/gstreamer/gst-build']]]
|
||||
)
|
||||
sh 'git checkout master && git reset --hard origin/master'
|
||||
sh 'curl "https://git.arracacha.collabora.co.uk/cgit/gst-manifest.git/plain/default.xml?id=$BUILD_TAG" -k -o manifest.xml'
|
||||
}
|
||||
|
||||
stage('Setup') {
|
||||
sh './git-update --no-color --manifest=manifest.xml --no-interaction'
|
||||
sh './setup.py -Ddisable_gstreamer_vaapi=true'
|
||||
}
|
||||
|
||||
stage ('Build') {
|
||||
sh "ninja -C build"
|
||||
}
|
||||
|
||||
stage ('Check') {
|
||||
sh "./gst-uninstalled.py gst-validate-launcher --check-bugs --no-display --mute -n check --xunit-file $WORKSPACE/xunit.xml -M $WORKSPACE/validate-output --ignore-numfailures"
|
||||
|
||||
step([$class: 'XUnitBuilder',
|
||||
testTimeMargin: '3000', thresholdMode: 1,
|
||||
thresholds: [[$class: 'FailedThreshold',
|
||||
failureNewThreshold: '',
|
||||
failureThreshold: '5',
|
||||
unstableNewThreshold: '',
|
||||
unstableThreshold: '1'],
|
||||
[$class: 'SkippedThreshold',
|
||||
failureNewThreshold: '',
|
||||
failureThreshold: '',
|
||||
unstableNewThreshold: '',
|
||||
unstableThreshold: '']],
|
||||
tools: [[$class: 'JUnitType',
|
||||
deleteOutputFiles: true,
|
||||
failIfNotNew: true,
|
||||
pattern: 'xunit.xml',
|
||||
skipNoTestFiles: true,
|
||||
stopProcessingIfError: true]]])
|
||||
}
|
||||
|
||||
stage('install') {
|
||||
sh 'mkdir -p dest'
|
||||
sh 'DESTDIR=$PWD/dest ninja -C build install'
|
||||
}
|
||||
stage('package') {
|
||||
sh 'cd dest && tar caJf gstreamer-$BUILD_TAG.tar.xz usr'
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue