From 74f2d5735c6af31a301974a4c50c1ab749bb1dd7 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 21 Oct 2017 12:07:02 +0200 Subject: [PATCH 001/605] Initial commit --- README.txt | 11 +++++ jenkins/Jenkinsfile-master-fedora | 66 +++++++++++++++++++++++++++++ jenkins/Jenkinsfile-meson-fedora | 69 +++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 README.txt create mode 100644 jenkins/Jenkinsfile-master-fedora create mode 100644 jenkins/Jenkinsfile-meson-fedora diff --git a/README.txt b/README.txt new file mode 100644 index 0000000000..1a3174d163 --- /dev/null +++ b/README.txt @@ -0,0 +1,11 @@ +GStreamer Continuous Integration +================================ + +This repository contains all material relevant to the GStreamer +Continuous Integration system. + +* Jenkins scripts + +* Docker images + +* Build scripts and code diff --git a/jenkins/Jenkinsfile-master-fedora b/jenkins/Jenkinsfile-master-fedora new file mode 100644 index 0000000000..000416a983 --- /dev/null +++ b/jenkins/Jenkinsfile-master-fedora @@ -0,0 +1,66 @@ +node('docker') { + docker.image('gstreamer/build-base-fedora:latest').inside { + + env.CCACHE_DIR = "${env.WORKSPACE}/.ccache" + env.CCACHE_MAXSIZE = "2G" + env.CC = "ccache gcc" + env.CXX = "ccache g++" + env.MAKEFLAGS = "-j6" + env.PATH = "${env.WORKSPACE}:${env.PATH}" + env.GST_UNINSTALLED_ROOT="${env.WORKSPACE}" + env.HOME="${env.WORKSPACE}" + env.DISPLAY=":0" + + stage('Checkout') { + if (params.wipe) { + sh 'rm -Rf *' + } + checkout([$class: 'RepoScm', + manifestRepositoryUrl:'https://git.arracacha.collabora.co.uk/git/gst-manifest.git', + manifestBranch:"refs/tags/${params.build_tag}", + jobs:4, + currentBranch:true, + quiet:true, + depth:0, + mirrorDir:'/repositories']) + } + + stage('Cleanup') { + sh 'rm -f **/tests/check/*/*.xml' + } + + stage ('Build') { + sh "uname -a" + sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh fast-build-only" + } + + withEnv(['DISPLAY=:0']) { + stage ('Check') { + env.GST_CHECKS_IGNORE="test_allocate_udp_ports_multicast,test_allocate_udp_ports_client_settings,test_reorder_buffer,test_redirect_yes" + env.GST_CHECK_XML=1 + sh 'Xvfb :0 -screen 0 1024x768x24 -fbdir /tmp &' + sh 'env' + sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh check" + + step([$class: 'XUnitBuilder', + testTimeMargin: '3000', thresholdMode: 1, + thresholds: [[$class: 'FailedThreshold', + failureNewThreshold: '', + failureThreshold: '400', + unstableNewThreshold: '', + unstableThreshold: '1'], + [$class: 'SkippedThreshold', + failureNewThreshold: '', + failureThreshold: '', + unstableNewThreshold: '', + unstableThreshold: '']], + tools: [[$class: 'CheckType', + deleteOutputFiles: true, + failIfNotNew: true, + pattern: '**/tests/check/*/*.xml', + skipNoTestFiles: true, + stopProcessingIfError: true]]]) + } + } + } +} diff --git a/jenkins/Jenkinsfile-meson-fedora b/jenkins/Jenkinsfile-meson-fedora new file mode 100644 index 0000000000..553a19d086 --- /dev/null +++ b/jenkins/Jenkinsfile-meson-fedora @@ -0,0 +1,69 @@ +node('docker') { + docker.image('gstreamer/build-meson-fedora:latest').inside { + + env.CCACHE_DIR = "${env.WORKSPACE}/.ccache" + env.CCACHE_MAXSIZE = "2G" + env.CC = "ccache gcc" + env.CXX = "ccache g++" + env.MAKEFLAGS = "-j6" + env.PATH = "${env.WORKSPACE}:${env.PATH}" + env.HOME="${env.WORKSPACE}" + env.DISPLAY=":0" + + stage('Checkout') { + if (params.wipe) { + sh 'rm -Rf build/' + } + checkout([$class: 'GitSCM', branches: [[name: '*/master']], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'CloneOption', + depth: 0, + noTags: false, + reference: '/gstbuild/gst-build/', + shallow: false]], + submoduleCfg: [], + userRemoteConfigs: [[url: 'git://anongit.freedesktop.org/gstreamer/gst-build']]] + ) + sh 'git checkout master && git reset --hard origin/master' + sh 'curl "https://git.arracacha.collabora.co.uk/cgit/gst-manifest.git/plain/default.xml?id=$BUILD_TAG" -k -o manifest.xml' + } + + stage('Setup') { + sh './git-update --no-color --manifest=manifest.xml --no-interaction' + sh './setup.py -Ddisable_gstreamer_vaapi=true' + } + + stage ('Build') { + sh "ninja -C build" + } + + stage ('Check') { + sh "./gst-uninstalled.py gst-validate-launcher --check-bugs --no-display --mute -n check --xunit-file $WORKSPACE/xunit.xml -M $WORKSPACE/validate-output --ignore-numfailures" + + step([$class: 'XUnitBuilder', + testTimeMargin: '3000', thresholdMode: 1, + thresholds: [[$class: 'FailedThreshold', + failureNewThreshold: '', + failureThreshold: '400', + unstableNewThreshold: '', + unstableThreshold: '1'], + [$class: 'SkippedThreshold', + failureNewThreshold: '', + failureThreshold: '', + unstableNewThreshold: '', + unstableThreshold: '']], + tools: [[$class: 'JUnitType', + deleteOutputFiles: true, + failIfNotNew: true, + pattern: 'xunit.xml', + skipNoTestFiles: true, + stopProcessingIfError: true]]]) + } + + stage('install') { + sh 'mkdir dest' + sh 'DESTDIR=$PWD/dest ninja -C build install' + } + + } +} From aa441579c4e0c9272935c131dbdc5663bbff401c Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 21 Oct 2017 12:45:38 +0200 Subject: [PATCH 002/605] docker: Add initial images --- docker/README | 2 + docker/build-base-images/Dockerfile-fedora | 76 +++++++++++++++++++ .../build-base-images/Dockerfile-meson-fedora | 12 +++ docker/build-base-images/Dockerfile-ubuntu | 62 +++++++++++++++ docker/build-base-images/build.sh | 8 ++ docker/runtime-images/Dockerfile-fedora | 47 ++++++++++++ docker/runtime-images/Dockerfile-ubuntu | 21 +++++ docker/runtime-images/build.sh | 9 +++ 8 files changed, 237 insertions(+) create mode 100644 docker/README create mode 100644 docker/build-base-images/Dockerfile-fedora create mode 100644 docker/build-base-images/Dockerfile-meson-fedora create mode 100644 docker/build-base-images/Dockerfile-ubuntu create mode 100755 docker/build-base-images/build.sh create mode 100644 docker/runtime-images/Dockerfile-fedora create mode 100644 docker/runtime-images/Dockerfile-ubuntu create mode 100755 docker/runtime-images/build.sh diff --git a/docker/README b/docker/README new file mode 100644 index 0000000000..1e99ca0a6f --- /dev/null +++ b/docker/README @@ -0,0 +1,2 @@ +GStreamer Docker images + diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora new file mode 100644 index 0000000000..eadfa9d64e --- /dev/null +++ b/docker/build-base-images/Dockerfile-fedora @@ -0,0 +1,76 @@ +FROM gstreamer/base-fedora:latest + +# Add rpmfusion repositories and install all build dependencies of GStreamer +RUN dnf builddep -y \ + gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-plugins-good \ + gstreamer1-plugins-good-extras \ + gstreamer1-plugins-ugly \ + gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-nonfree \ + gstreamer1-plugins-bad-free \ + gstreamer1-plugins-bad-free-extras \ + gstreamer1-plugins-bad-freeworld \ + gstreamer1-libav \ + gstreamer1-rtsp-server \ + && \ + rpm -e \ + gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-devel \ + gstreamer1-plugins-base-devel \ + && \ + dnf install -y \ + ccache \ + gcc-c++ \ + git \ + json-glib-devel \ + libunwind-devel \ + opencv-devel \ + openjpeg2-devel \ + patch \ + redhat-rpm-config \ + xorg-x11-server-Xvfb \ + xorg-x11-server-utils \ + wget \ + && \ + dnf clean all + +# Clone copies of all repositories to avoid extra downloads +RUN mkdir /repositories && \ + cd /repositories && \ + for i in orc \ + common \ + gstreamer \ + gst-plugins-base \ + gst-plugins-good \ + gst-plugins-bad \ + gst-plugins-ugly \ + gst-libav \ + gst-build \ + gst-devtools\ + gst-docs \ + gst-editing-services \ + gst-omx \ + gst-python \ + gst-rtsp-server; \ + do \ + git clone --mirror https://anongit.freedesktop.org/git/gstreamer/$i; \ + done + +# Install repo tool +RUN wget -nd https://storage.googleapis.com/git-repo-downloads/repo -O /usr/bin/repo && chmod a+x /usr/bin/repo + +# Add the user UID:1000, GID:1000, home at /gstbuild +RUN groupadd -r gstbuild -g 1000 && useradd -u 1000 -r -g gstbuild -m -d /gstbuild -s /sbin/nologin -c "Gstbuild user" gstbuild && \ + chmod 755 /gstbuild + +# Set the working directory to gstbuild home directory +WORKDIR /gstbuild + +# Specify the user to execute all commands below +USER gstbuild + +# Configure git for various usage +RUN git config --global user.email "gstbuild@gstreamer.net" && git config --global user.name "Gstbuild Runner" diff --git a/docker/build-base-images/Dockerfile-meson-fedora b/docker/build-base-images/Dockerfile-meson-fedora new file mode 100644 index 0000000000..f6eba9db3b --- /dev/null +++ b/docker/build-base-images/Dockerfile-meson-fedora @@ -0,0 +1,12 @@ +FROM gstreamer/build-base-fedora:latest + +USER root + +RUN dnf install -y \ + meson + +USER gstbuild + +RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build + +WORKDIR /gstbuild/gst-build diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu new file mode 100644 index 0000000000..776c96541d --- /dev/null +++ b/docker/build-base-images/Dockerfile-ubuntu @@ -0,0 +1,62 @@ +FROM gstreamer/base-ubuntu:latest + +# make source packages available in order to figure out build dependencies +RUN \ + sed -i "s/# deb-src/deb-src/g" /etc/apt/sources.list && \ + apt-get update && \ + apt-get -y build-dep \ + orc \ + gstreamer1.0 \ + gst-plugins-base1.0 \ + gst-plugins-good1.0 \ + gst-plugins-bad1.0 \ + gst-plugins-ugly1.0 \ + gst-libav1.0 \ + gst-rtsp-server1.0 \ + gst-python1.0 \ + gstreamer-vaapi \ + gstreamer-editing-services1.0 && \ + apt-get -y install \ + ccache \ + git \ + xvfb \ + wget + +# Clone copies of all repositories to avoid extra downloads +RUN mkdir /repositories && \ + cd /repositories && \ + for i in orc \ + common \ + gstreamer \ + gst-plugins-base \ + gst-plugins-good \ + gst-plugins-bad \ + gst-plugins-ugly \ + gst-libav \ + gst-build \ + gst-devtools\ + gst-docs \ + gst-editing-services \ + gst-omx \ + gst-python \ + gst-rtsp-server; \ + do \ + git clone --mirror https://anongit.freedesktop.org/git/gstreamer/$i; \ + done + +# Install repo tool +RUN wget -nd https://storage.googleapis.com/git-repo-downloads/repo -O /usr/bin/repo && chmod a+x /usr/bin/repo + +# Add the user UID:1000, GID:1000, home at /gstbuild +RUN groupadd -r gstbuild -g 1000 && useradd -u 1000 -r -g gstbuild -m -d /gstbuild -s /sbin/nologin -c "Gstbuild user" gstbuild && \ + chmod 755 /gstbuild + +# Set the working directory to gstbuild home directory +WORKDIR /gstbuild + +# Specify the user to execute all commands below +USER gstbuild + +# Configure git for various usage +RUN git config --global user.email "gstbuild@gstreamer.net" && git config --global user.name "Gstbuild Runner" + diff --git a/docker/build-base-images/build.sh b/docker/build-base-images/build.sh new file mode 100755 index 0000000000..c8214c5b5a --- /dev/null +++ b/docker/build-base-images/build.sh @@ -0,0 +1,8 @@ +#!/bin/sh +echo Building gstreamer/build-base-fedora:latest + +docker build -t gstreamer/build-base-fedora:latest . -f Dockerfile-fedora + +echo Building gstreamer/build-base-ubuntu:latest + +docker build -t gstreamer/build-base-ubuntu:latest . -f Dockerfile-ubuntu diff --git a/docker/runtime-images/Dockerfile-fedora b/docker/runtime-images/Dockerfile-fedora new file mode 100644 index 0000000000..fe5c80a14c --- /dev/null +++ b/docker/runtime-images/Dockerfile-fedora @@ -0,0 +1,47 @@ +FROM fedora:26 + +# Add rpmfusion repositories and install all build dependencies of GStreamer +# and then remove the gst packages themselves +RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ + dnf install -y http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-26.noarch.rpm \ + http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-26.noarch.rpm && \ + rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-nonfree-fedora-26 && \ + rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-26 && \ + dnf upgrade -y && \ + dnf install -y \ + dnf-plugins-core \ + json-glib \ + libunwind \ + opencv \ + openjpeg2 \ + procps-ng \ + && \ + dnf install -y \ + gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-plugins-good \ + gstreamer1-plugins-good-extras \ + gstreamer1-plugins-ugly \ + gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-nonfree \ + gstreamer1-plugins-bad-free \ + gstreamer1-plugins-bad-free-extras \ + gstreamer1-plugins-bad-freeworld \ + gstreamer1-libav \ + gstreamer1-rtsp-server \ + && \ + rpm -e \ + gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-plugins-good \ + gstreamer1-plugins-good-extras \ + gstreamer1-plugins-ugly \ + gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-nonfree \ + gstreamer1-plugins-bad-free \ + gstreamer1-plugins-bad-free-extras \ + gstreamer1-plugins-bad-freeworld \ + gstreamer1-rtsp-server \ + gstreamer1-libav \ + && \ + dnf clean all diff --git a/docker/runtime-images/Dockerfile-ubuntu b/docker/runtime-images/Dockerfile-ubuntu new file mode 100644 index 0000000000..a9e4f8e387 --- /dev/null +++ b/docker/runtime-images/Dockerfile-ubuntu @@ -0,0 +1,21 @@ +FROM ubuntu:17.04 + +# upgrade and install all runtime dependencies for gstreamer + +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install --no-install-recommends\ + gstreamer1.0-libav \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-bad-faad \ + gstreamer1.0-plugins-bad-videoparsers \ + gstreamer1.0-plugins-ugly \ + gstreamer1.0-plugins-ugly-amr \ + gstreamer1.0-rtsp \ + gstreamer1.0-alsa \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-pulseaudio \ + gstreamer1.0-x \ + libgstreamer-plugins-base1.0-0 \ + libgstreamer1.0 diff --git a/docker/runtime-images/build.sh b/docker/runtime-images/build.sh new file mode 100755 index 0000000000..7cdff2b090 --- /dev/null +++ b/docker/runtime-images/build.sh @@ -0,0 +1,9 @@ +#!/bin/sh +echo Building gstreamer/base-fedora:latest + +docker build -t gstreamer/base-fedora:latest . -f Dockerfile-fedora + +echo Building gstreamer/base-ubuntu:latest + +docker build -t gstreamer/base-ubuntu:latest . -f Dockerfile-ubuntu + From 73a08ac254c0f637b81de718527fbfe500c4afa3 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 21 Oct 2017 12:49:11 +0200 Subject: [PATCH 003/605] jenkins: Add master ubuntu jenkinsfile --- jenkins/Jenkinsfile-master-ubuntu | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 jenkins/Jenkinsfile-master-ubuntu diff --git a/jenkins/Jenkinsfile-master-ubuntu b/jenkins/Jenkinsfile-master-ubuntu new file mode 100644 index 0000000000..01522b9794 --- /dev/null +++ b/jenkins/Jenkinsfile-master-ubuntu @@ -0,0 +1,66 @@ +node('docker') { + docker.image('gstreamer/build-base-ubuntu:latest').inside { + + env.CCACHE_DIR = "${env.WORKSPACE}/.ccache" + env.CCACHE_MAXSIZE = "2G" + env.CC = "ccache gcc" + env.CXX = "ccache g++" + env.MAKEFLAGS = "-j6" + env.PATH = "${env.WORKSPACE}:${env.PATH}" + env.GST_UNINSTALLED_ROOT="${env.WORKSPACE}" + env.HOME="${env.WORKSPACE}" + env.DISPLAY=":0" + + stage('Checkout') { + if (params.wipe) { + sh 'rm -Rf *' + } + checkout([$class: 'RepoScm', + manifestRepositoryUrl:'https://git.arracacha.collabora.co.uk/git/gst-manifest.git', + manifestBranch:"refs/tags/${params.build_tag}", + jobs:4, + currentBranch:true, + quiet:true, + depth:0, + mirrorDir:'/repositories']) + } + + stage('Cleanup') { + sh 'rm -f **/tests/check/*/*.xml' + } + + stage ('Build') { + sh "uname -a" + sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh fast-build-only" + } + + withEnv(['DISPLAY=:0']) { + stage ('Check') { + env.GST_CHECKS_IGNORE="test_allocate_udp_ports_multicast,test_allocate_udp_ports_client_settings,test_reorder_buffer,test_redirect_yes" + env.GST_CHECK_XML=1 + sh 'Xvfb :0 -screen 0 1024x768x24 -fbdir /tmp &' + sh 'env' + sh "./gstreamer/scripts/gst-uninstalled ./gst-ci-scripts/ci-build.sh check" + + step([$class: 'XUnitBuilder', + testTimeMargin: '3000', thresholdMode: 1, + thresholds: [[$class: 'FailedThreshold', + failureNewThreshold: '', + failureThreshold: '400', + unstableNewThreshold: '', + unstableThreshold: '1'], + [$class: 'SkippedThreshold', + failureNewThreshold: '', + failureThreshold: '', + unstableNewThreshold: '', + unstableThreshold: '']], + tools: [[$class: 'CheckType', + deleteOutputFiles: true, + failIfNotNew: true, + pattern: '**/tests/check/*/*.xml', + skipNoTestFiles: true, + stopProcessingIfError: true]]]) + } + } + } +} From 639dc49bdb1b8bf31300d328ddebdf094dec5116 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 21 Oct 2017 13:10:26 +0200 Subject: [PATCH 004/605] docker: Add gdb to build images So we can get backtraces --- docker/build-base-images/Dockerfile-fedora | 1 + docker/build-base-images/Dockerfile-ubuntu | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index eadfa9d64e..5896af1627 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -24,6 +24,7 @@ RUN dnf builddep -y \ dnf install -y \ ccache \ gcc-c++ \ + gdb \ git \ json-glib-devel \ libunwind-devel \ diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu index 776c96541d..2010b43801 100644 --- a/docker/build-base-images/Dockerfile-ubuntu +++ b/docker/build-base-images/Dockerfile-ubuntu @@ -18,6 +18,7 @@ RUN \ gstreamer-editing-services1.0 && \ apt-get -y install \ ccache \ + gdb \ git \ xvfb \ wget From f4ce9bf4e6b9a88c582bd5cceabd1ce298a0ac98 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 21 Oct 2017 14:44:53 +0200 Subject: [PATCH 005/605] jenkins: Add cerbero debian docker script --- jenkins/Jenkinsfile-cerbero-debian | 51 ++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 jenkins/Jenkinsfile-cerbero-debian diff --git a/jenkins/Jenkinsfile-cerbero-debian b/jenkins/Jenkinsfile-cerbero-debian new file mode 100644 index 0000000000..377e0047e8 --- /dev/null +++ b/jenkins/Jenkinsfile-cerbero-debian @@ -0,0 +1,51 @@ +node("debian") { + docker.image('gstreamer/build-base-ubuntu:latest').inside { + + env.OUTPREFIX="${env.WORKSPACE}/../output/${params.build_tag}/linux_x86_64/" + + stage('Checkout') { + // FIXME: Only checkout the manifest and not all dependencies ? + checkout([$class: 'RepoScm', + manifestRepositoryUrl:'git+ssh://git.arracacha.collabora.co.uk/git/gst-manifest.git', + manifestBranch:"refs/tags/${params.build_tag}", + jobs:4, + currentBranch:true, + quiet:true, + depth:0]) + } + + stage('Setup') { + sh "find ../output -maxdepth 1 -ctime +1 | xargs rm -Rf" + sh "cd .repo/manifests/; git checkout ${params.build_tag}; cd ../.." + sh "rm -Rf ./workdir/sources/linux_x86_64/" + sh "rm -f *.rpm" + sh "rm -Rf ./workdir/temp; mkdir -p ./workdir/temp" + sh "rm -Rf ./workdir/tmp*" + + // Create custom configuration file + sh "./gst-ci-scripts/manifest2cerbero.py .repo/manifests/default.xml ./cerbero/config/linux.config --output localconf.cbc" + sh '''echo "home_dir = \\"$WORKSPACE/workdir\\"" >> localconf.cbc''' + sh '''echo "logs = \\"$OUTPREFIX/logs\\"" >> localconf.cbc''' + sh './cerbero/cerbero-uninstalled -c localconf.cbc show-config' + } + + stage('bootstrap') { + sh './cerbero/cerbero-uninstalled -c localconf.cbc bootstrap' + sh 'rm -Rf ./workdir/sources/build-tools/' + } + + stage('fetch') { + sh './cerbero/cerbero-uninstalled -c localconf.cbc fetch-package --reset-rdeps --full-reset gstreamer-1.0' + } + + stage('package') { + sh './cerbero/cerbero-uninstalled -c localconf.cbc package gstreamer-1.0' + } + + stage('Cleanup') { + sh 'rm -f *.rpm' + sh 'find ../output -maxdepth 1 -ctime +1 | xargs rm -Rf' + } + // FIXME: IRC Notification + } +} From bf72e064619877d94e37b60c6289d5bffa932362 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 21 Oct 2017 14:46:07 +0200 Subject: [PATCH 006/605] jenkins: Fix cerbero jenkins file --- jenkins/Jenkinsfile-cerbero-debian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/Jenkinsfile-cerbero-debian b/jenkins/Jenkinsfile-cerbero-debian index 377e0047e8..14c5a68f42 100644 --- a/jenkins/Jenkinsfile-cerbero-debian +++ b/jenkins/Jenkinsfile-cerbero-debian @@ -1,4 +1,4 @@ -node("debian") { +node("docker") { docker.image('gstreamer/build-base-ubuntu:latest').inside { env.OUTPREFIX="${env.WORKSPACE}/../output/${params.build_tag}/linux_x86_64/" From 4da90f0d8c63cade09d4f3f28be0d51b3aba1e0a Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 21 Oct 2017 15:09:33 +0200 Subject: [PATCH 007/605] jenkins: Update meson script --- jenkins/Jenkinsfile-meson-fedora | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/Jenkinsfile-meson-fedora b/jenkins/Jenkinsfile-meson-fedora index 553a19d086..c686798f46 100644 --- a/jenkins/Jenkinsfile-meson-fedora +++ b/jenkins/Jenkinsfile-meson-fedora @@ -44,7 +44,7 @@ node('docker') { testTimeMargin: '3000', thresholdMode: 1, thresholds: [[$class: 'FailedThreshold', failureNewThreshold: '', - failureThreshold: '400', + failureThreshold: '5', unstableNewThreshold: '', unstableThreshold: '1'], [$class: 'SkippedThreshold', @@ -61,7 +61,7 @@ node('docker') { } stage('install') { - sh 'mkdir dest' + sh 'mkdir -p dest' sh 'DESTDIR=$PWD/dest ninja -C build install' } From d023ae0eb2297132b899ea2fc78b8a4583c3d617 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 28 Oct 2017 10:59:08 +0200 Subject: [PATCH 008/605] jenkins: Update meson-fedora --- jenkins/Jenkinsfile-meson-fedora | 54 +++++++++++++++++--------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/jenkins/Jenkinsfile-meson-fedora b/jenkins/Jenkinsfile-meson-fedora index c686798f46..bc3800d911 100644 --- a/jenkins/Jenkinsfile-meson-fedora +++ b/jenkins/Jenkinsfile-meson-fedora @@ -15,14 +15,14 @@ node('docker') { sh 'rm -Rf build/' } checkout([$class: 'GitSCM', branches: [[name: '*/master']], - doGenerateSubmoduleConfigurations: false, - extensions: [[$class: 'CloneOption', - depth: 0, - noTags: false, - reference: '/gstbuild/gst-build/', - shallow: false]], - submoduleCfg: [], - userRemoteConfigs: [[url: 'git://anongit.freedesktop.org/gstreamer/gst-build']]] + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'CloneOption', + depth: 0, + noTags: false, + reference: '/gstbuild/gst-build/', + shallow: false]], + submoduleCfg: [], + userRemoteConfigs: [[url: 'git://anongit.freedesktop.org/gstreamer/gst-build']]] ) sh 'git checkout master && git reset --hard origin/master' sh 'curl "https://git.arracacha.collabora.co.uk/cgit/gst-manifest.git/plain/default.xml?id=$BUILD_TAG" -k -o manifest.xml' @@ -41,29 +41,31 @@ node('docker') { sh "./gst-uninstalled.py gst-validate-launcher --check-bugs --no-display --mute -n check --xunit-file $WORKSPACE/xunit.xml -M $WORKSPACE/validate-output --ignore-numfailures" step([$class: 'XUnitBuilder', - testTimeMargin: '3000', thresholdMode: 1, - thresholds: [[$class: 'FailedThreshold', - failureNewThreshold: '', - failureThreshold: '5', - unstableNewThreshold: '', - unstableThreshold: '1'], - [$class: 'SkippedThreshold', - failureNewThreshold: '', - failureThreshold: '', - unstableNewThreshold: '', - unstableThreshold: '']], - tools: [[$class: 'JUnitType', - deleteOutputFiles: true, - failIfNotNew: true, - pattern: 'xunit.xml', - skipNoTestFiles: true, - stopProcessingIfError: true]]]) + testTimeMargin: '3000', thresholdMode: 1, + thresholds: [[$class: 'FailedThreshold', + failureNewThreshold: '', + failureThreshold: '5', + unstableNewThreshold: '', + unstableThreshold: '1'], + [$class: 'SkippedThreshold', + failureNewThreshold: '', + failureThreshold: '', + unstableNewThreshold: '', + unstableThreshold: '']], + tools: [[$class: 'JUnitType', + deleteOutputFiles: true, + failIfNotNew: true, + pattern: 'xunit.xml', + skipNoTestFiles: true, + stopProcessingIfError: true]]]) } stage('install') { sh 'mkdir -p dest' sh 'DESTDIR=$PWD/dest ninja -C build install' } - + stage('package') { + sh 'cd dest && tar caJf gstreamer-$BUILD_TAG.tar.xz usr' + } } } From 69133548453142dfc0a001e79cc04d255423c104 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 28 Oct 2017 11:01:33 +0200 Subject: [PATCH 009/605] Add gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..b25c15b81f --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ From f51f8eea4c5e08d3e906ade511fc87b2e77e8155 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 28 Oct 2017 11:02:06 +0200 Subject: [PATCH 010/605] Add dockerignore --- docker/build-base-images/.dockerignore | 2 ++ docker/runtime-images/.dockerignore | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 docker/build-base-images/.dockerignore create mode 100644 docker/runtime-images/.dockerignore diff --git a/docker/build-base-images/.dockerignore b/docker/build-base-images/.dockerignore new file mode 100644 index 0000000000..131a0fd029 --- /dev/null +++ b/docker/build-base-images/.dockerignore @@ -0,0 +1,2 @@ +build.sh + diff --git a/docker/runtime-images/.dockerignore b/docker/runtime-images/.dockerignore new file mode 100644 index 0000000000..131a0fd029 --- /dev/null +++ b/docker/runtime-images/.dockerignore @@ -0,0 +1,2 @@ +build.sh + From 7a4aec73c5df0a28d235bb124cf30d6ee6b12ba4 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sun, 29 Oct 2017 08:15:21 +0100 Subject: [PATCH 011/605] fuzzing: Add push-based discoverer fuzzing target --- fuzzing/gst-discoverer.c | 118 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 fuzzing/gst-discoverer.c diff --git a/fuzzing/gst-discoverer.c b/fuzzing/gst-discoverer.c new file mode 100644 index 0000000000..1f862aaa95 --- /dev/null +++ b/fuzzing/gst-discoverer.c @@ -0,0 +1,118 @@ +/* + * Copyright 2016 Google Inc. + * author: Edward Hervey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#include +#include +#include + +GST_PLUGIN_STATIC_DECLARE(coreelements); +GST_PLUGIN_STATIC_DECLARE(playback); +GST_PLUGIN_STATIC_DECLARE(typefindfunctions); +GST_PLUGIN_STATIC_DECLARE(app); +GST_PLUGIN_STATIC_DECLARE(ogg); +GST_PLUGIN_STATIC_DECLARE(theora); +GST_PLUGIN_STATIC_DECLARE(vorbis); + +/* push-based discoverer fuzzing target + * + * This application can be compiled with libFuzzer to simulate + * a push-based discoverer execution. + * + * To reproduce the failing behaviour, use: + * $ gst-discoverer-1.0 pushfile:///... + * + * The goal is to cover basic usage of demuxers, parsers and + * base decoder elements. + * + * When compiling, only link the required demuxer/parser/decoder + * plugins and keep it to a limited range (ex: ogg/theora/vorbis) + * + **/ + +const guint8 *fuzztesting_data; +size_t fuzztesting_size; + +static void +appsrc_configuration (GstDiscoverer *dc, GstElement *source, gpointer data) +{ + GstBuffer *buf; + GstFlowReturn ret; + + /* Create buffer from fuzztesting_data which shouldn't be freed */ + buf = gst_buffer_new_wrapped_full (0, (gpointer) fuzztesting_data, fuzztesting_size, + 0, fuzztesting_size, NULL, NULL); + g_object_set (G_OBJECT (source), "size", fuzztesting_size, NULL); + g_signal_emit_by_name (G_OBJECT(source), "push-buffer", buf, &ret); + gst_buffer_unref (buf); +} + +int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) +{ + GError *err = NULL; + GstDiscoverer *dc; + gint timeout = 10; + GstDiscovererInfo *info; + static gboolean initialized = 0; + + if (!initialized) { + /* We want critical warnings to assert so we can fix them + * But somehow it's not causing oss-fuzz to crash ... */ + g_setenv ("G_DEBUG", "fatal-criticals", TRUE); + + /* Only initialize and register plugins once */ + gst_init (NULL, NULL); + + GST_PLUGIN_STATIC_REGISTER(coreelements); + GST_PLUGIN_STATIC_REGISTER(playback); + GST_PLUGIN_STATIC_REGISTER(typefindfunctions); + GST_PLUGIN_STATIC_REGISTER(app); + GST_PLUGIN_STATIC_REGISTER(ogg); + GST_PLUGIN_STATIC_REGISTER(theora); + GST_PLUGIN_STATIC_REGISTER(vorbis); + } + + dc = gst_discoverer_new (timeout * GST_SECOND, &err); + if (G_UNLIKELY (dc == NULL)) { + g_print ("Error initializing: %s\n", err->message); + g_clear_error (&err); + exit (1); + } + + fuzztesting_data = data; + fuzztesting_size = size; + + /* Connect to source-setup signal to give the data */ + g_signal_connect (dc, "source-setup", (GCallback) appsrc_configuration, NULL); + + info = gst_discoverer_discover_uri (dc, "appsrc://", &err); + g_clear_error (&err); + if (info) + gst_discoverer_info_unref (info); + + g_object_unref (dc); + + return 0; + } + From 87e1f69b195ef4618df2e06ca110e74737746ed8 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Tue, 31 Oct 2017 07:43:07 +0100 Subject: [PATCH 012/605] fuzzing: New typefind target Simple appsrc ! typefind ! fakesink pipeline --- fuzzing/typefind.c | 105 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 fuzzing/typefind.c diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c new file mode 100644 index 0000000000..04b3092f63 --- /dev/null +++ b/fuzzing/typefind.c @@ -0,0 +1,105 @@ +/* + * Copyright 2016 Google Inc. + * author: Edward Hervey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#include +#include + +GST_PLUGIN_STATIC_DECLARE(coreelements); +GST_PLUGIN_STATIC_DECLARE(typefindfunctions); +GST_PLUGIN_STATIC_DECLARE(app); + +/* push-based typefind fuzzing target + * + * This application can be compiled with libFuzzer to simulate + * a push-based typefind execution. + * + * To reproduce the failing behaviour, use: + * $ gst-launch-1.0 pushfile:///.. ! typefind ! fakesink + * + * The goal is to cover typefind code and implementation. + * + **/ + +static void +appsrc_configuration (GstDiscoverer *dc, GstElement *source, gpointer data) +{ + GstBuffer *buf; + GstFlowReturn ret; + + /* Create buffer from fuzztesting_data which shouldn't be freed */ +} + +int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) +{ + GError *err = NULL; + static gboolean initialized = 0; + GstElement *pipeline, *source, *typefind, *fakesink; + GstBuffer *buf; + GstFlowReturn flowret; + + if (!initialized) { + /* We want critical warnings to assert so we can fix them + * But somehow it's not causing oss-fuzz to crash ... */ + g_setenv ("G_DEBUG", "fatal-criticals", TRUE); + + /* Only initialize and register plugins once */ + gst_init (NULL, NULL); + + GST_PLUGIN_STATIC_REGISTER(coreelements); + GST_PLUGIN_STATIC_REGISTER(typefindfunctions); + GST_PLUGIN_STATIC_REGISTER(app); + } + + /* Create the pipeline */ + source = gst_element_factory_make ("appsrc", "source"); + typefind = gst_element_factory_make ("typefind", "typefind"); + fakesink = gst_element_factory_make ("fakesink", "fakesink"); + + gst_bin_add_many (GST_BIN (pipeline), source, typefind, fakesink, NULL); + gst_element_link_many (source, typefind, fakesink, NULL); + + /* Set pipeline to READY so we can provide data to appsrc */ + gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_READY); + buf = gst_buffer_new_wrapped_full (0, (gpointer) data, size, + 0, size, NULL, NULL); + g_object_set (G_OBJECT (source), "size", size, NULL); + g_signal_emit_by_name (G_OBJECT(source), "push-buffer", buf, &ret); + gst_buffer_unref (buf); + + /* Set pipeline to PAUSED and wait (typefind will either fail or succeed) */ + gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_PAUSED); + + /* wait until state change either completes or fails */ + sret = gst_element_get_state (GST_ELEMENT (pipeline), &state, NULL, -1); + + /* Go back to NULL */ + gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_NULL); + + /* And release the pipeline */ + gst_object_unref (pipeline); + + return 0; + } + From 80d8fcec0c0e2b8071560d34d229dfd6ba0da450 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Tue, 31 Oct 2017 08:03:48 +0100 Subject: [PATCH 013/605] fuzzing: Compilation fixes --- fuzzing/typefind.c | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c index 04b3092f63..c783e509f9 100644 --- a/fuzzing/typefind.c +++ b/fuzzing/typefind.c @@ -26,9 +26,9 @@ #include #include -GST_PLUGIN_STATIC_DECLARE(coreelements); -GST_PLUGIN_STATIC_DECLARE(typefindfunctions); -GST_PLUGIN_STATIC_DECLARE(app); +GST_PLUGIN_STATIC_DECLARE (coreelements); +GST_PLUGIN_STATIC_DECLARE (typefindfunctions); +GST_PLUGIN_STATIC_DECLARE (app); /* push-based typefind fuzzing target * @@ -42,22 +42,15 @@ GST_PLUGIN_STATIC_DECLARE(app); * **/ -static void -appsrc_configuration (GstDiscoverer *dc, GstElement *source, gpointer data) -{ - GstBuffer *buf; - GstFlowReturn ret; - - /* Create buffer from fuzztesting_data which shouldn't be freed */ -} - -int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) +int +LLVMFuzzerTestOneInput (const guint8 * data, size_t size) { GError *err = NULL; static gboolean initialized = 0; GstElement *pipeline, *source, *typefind, *fakesink; GstBuffer *buf; GstFlowReturn flowret; + GstState state; if (!initialized) { /* We want critical warnings to assert so we can fix them @@ -66,12 +59,12 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) /* Only initialize and register plugins once */ gst_init (NULL, NULL); - - GST_PLUGIN_STATIC_REGISTER(coreelements); - GST_PLUGIN_STATIC_REGISTER(typefindfunctions); - GST_PLUGIN_STATIC_REGISTER(app); + + GST_PLUGIN_STATIC_REGISTER (coreelements); + GST_PLUGIN_STATIC_REGISTER (typefindfunctions); + GST_PLUGIN_STATIC_REGISTER (app); } - + /* Create the pipeline */ source = gst_element_factory_make ("appsrc", "source"); typefind = gst_element_factory_make ("typefind", "typefind"); @@ -83,23 +76,22 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) /* Set pipeline to READY so we can provide data to appsrc */ gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_READY); buf = gst_buffer_new_wrapped_full (0, (gpointer) data, size, - 0, size, NULL, NULL); + 0, size, NULL, NULL); g_object_set (G_OBJECT (source), "size", size, NULL); - g_signal_emit_by_name (G_OBJECT(source), "push-buffer", buf, &ret); + g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &flowret); gst_buffer_unref (buf); - + /* Set pipeline to PAUSED and wait (typefind will either fail or succeed) */ gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_PAUSED); /* wait until state change either completes or fails */ - sret = gst_element_get_state (GST_ELEMENT (pipeline), &state, NULL, -1); + gst_element_get_state (GST_ELEMENT (pipeline), &state, NULL, -1); /* Go back to NULL */ gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_NULL); /* And release the pipeline */ gst_object_unref (pipeline); - + return 0; - } - +} From d197eaa5080455b6d8c35f5b70c9a13f3c2200be Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Tue, 31 Oct 2017 08:12:43 +0100 Subject: [PATCH 014/605] fuzzer: Might work better with a pipeline ... --- fuzzing/typefind.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c index c783e509f9..780a4a9010 100644 --- a/fuzzing/typefind.c +++ b/fuzzing/typefind.c @@ -66,6 +66,7 @@ LLVMFuzzerTestOneInput (const guint8 * data, size_t size) } /* Create the pipeline */ + pipeline = gst_pipeline_new ("pipeline"); source = gst_element_factory_make ("appsrc", "source"); typefind = gst_element_factory_make ("typefind", "typefind"); fakesink = gst_element_factory_make ("fakesink", "fakesink"); From 12ea11632b8e85eea2a4c74d34f8ffd293141309 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Wed, 1 Nov 2017 08:50:25 +0100 Subject: [PATCH 015/605] fuzzer: Make critical warnings fatal --- fuzzing/gst-discoverer.c | 5 ++--- fuzzing/typefind.c | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/fuzzing/gst-discoverer.c b/fuzzing/gst-discoverer.c index 1f862aaa95..21ed67b552 100644 --- a/fuzzing/gst-discoverer.c +++ b/fuzzing/gst-discoverer.c @@ -77,9 +77,8 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) static gboolean initialized = 0; if (!initialized) { - /* We want critical warnings to assert so we can fix them - * But somehow it's not causing oss-fuzz to crash ... */ - g_setenv ("G_DEBUG", "fatal-criticals", TRUE); + /* We want critical warnings to assert so we can fix them */ + g_log_set_always_fatal(G_LOG_LEVEL_CRITICAL); /* Only initialize and register plugins once */ gst_init (NULL, NULL); diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c index 780a4a9010..fd80148c24 100644 --- a/fuzzing/typefind.c +++ b/fuzzing/typefind.c @@ -53,9 +53,8 @@ LLVMFuzzerTestOneInput (const guint8 * data, size_t size) GstState state; if (!initialized) { - /* We want critical warnings to assert so we can fix them - * But somehow it's not causing oss-fuzz to crash ... */ - g_setenv ("G_DEBUG", "fatal-criticals", TRUE); + /* We want critical warnings to assert so we can fix them */ + g_log_set_always_fatal(G_LOG_LEVEL_CRITICAL); /* Only initialize and register plugins once */ gst_init (NULL, NULL); From bb4aadf968dfe7aa5a812de67b1fe041c50c9737 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Wed, 1 Nov 2017 09:45:36 +0100 Subject: [PATCH 016/605] fuzzer: Add custom logger that explicitely raises abort() --- fuzzing/gst-discoverer.c | 15 +++++++++++++++ fuzzing/typefind.c | 14 ++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/fuzzing/gst-discoverer.c b/fuzzing/gst-discoverer.c index 21ed67b552..9138a1c17b 100644 --- a/fuzzing/gst-discoverer.c +++ b/fuzzing/gst-discoverer.c @@ -68,6 +68,20 @@ appsrc_configuration (GstDiscoverer *dc, GstElement *source, gpointer data) gst_buffer_unref (buf); } +static void +custom_logger (const gchar *log_domain, + GLogLevelFlags log_level, + const gchar *message, + gpointer unused_data) +{ + if (log_level & G_LOG_LEVEL_CRITICAL) { + g_printerr ("CRITICAL ERROR : %s\n", message); + g_abort(); + } else if (log_level & G_LOG_LEVEL_WARNING) { + g_printerr ("WARNING : %s\n", message); + } +} + int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) { GError *err = NULL; @@ -79,6 +93,7 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) if (!initialized) { /* We want critical warnings to assert so we can fix them */ g_log_set_always_fatal(G_LOG_LEVEL_CRITICAL); + g_log_set_default_handler (custom_logger, NULL); /* Only initialize and register plugins once */ gst_init (NULL, NULL); diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c index fd80148c24..3917f97e94 100644 --- a/fuzzing/typefind.c +++ b/fuzzing/typefind.c @@ -41,6 +41,19 @@ GST_PLUGIN_STATIC_DECLARE (app); * The goal is to cover typefind code and implementation. * **/ +static void +custom_logger (const gchar *log_domain, + GLogLevelFlags log_level, + const gchar *message, + gpointer unused_data) +{ + if (log_level & G_LOG_LEVEL_CRITICAL) { + g_printerr ("CRITICAL ERROR : %s\n", message); + g_abort(); + } else if (log_level & G_LOG_LEVEL_WARNING) { + g_printerr ("WARNING : %s\n", message); + } +} int LLVMFuzzerTestOneInput (const guint8 * data, size_t size) @@ -55,6 +68,7 @@ LLVMFuzzerTestOneInput (const guint8 * data, size_t size) if (!initialized) { /* We want critical warnings to assert so we can fix them */ g_log_set_always_fatal(G_LOG_LEVEL_CRITICAL); + g_log_set_default_handler (custom_logger, NULL); /* Only initialize and register plugins once */ gst_init (NULL, NULL); From 39ce1f76a4d150b3d9660725263986ab925d6c76 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Wed, 1 Nov 2017 09:55:46 +0100 Subject: [PATCH 017/605] fuzzing: use abort instead of (private) g_abort --- fuzzing/gst-discoverer.c | 2 +- fuzzing/typefind.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzzing/gst-discoverer.c b/fuzzing/gst-discoverer.c index 9138a1c17b..17ed7d44d3 100644 --- a/fuzzing/gst-discoverer.c +++ b/fuzzing/gst-discoverer.c @@ -76,7 +76,7 @@ custom_logger (const gchar *log_domain, { if (log_level & G_LOG_LEVEL_CRITICAL) { g_printerr ("CRITICAL ERROR : %s\n", message); - g_abort(); + abort(); } else if (log_level & G_LOG_LEVEL_WARNING) { g_printerr ("WARNING : %s\n", message); } diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c index 3917f97e94..70f1a3813f 100644 --- a/fuzzing/typefind.c +++ b/fuzzing/typefind.c @@ -49,7 +49,7 @@ custom_logger (const gchar *log_domain, { if (log_level & G_LOG_LEVEL_CRITICAL) { g_printerr ("CRITICAL ERROR : %s\n", message); - g_abort(); + abort(); } else if (log_level & G_LOG_LEVEL_WARNING) { g_printerr ("WARNING : %s\n", message); } From 8f4bcce46ea7fdfcb3e2a9750f34252e34c39860 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Wed, 1 Nov 2017 14:10:05 +0100 Subject: [PATCH 018/605] fuzzing: Fix leak due to wrong initialization We were registering plugins again and again and again ... --- fuzzing/gst-discoverer.c | 4 +++- fuzzing/typefind.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fuzzing/gst-discoverer.c b/fuzzing/gst-discoverer.c index 17ed7d44d3..519537a61b 100644 --- a/fuzzing/gst-discoverer.c +++ b/fuzzing/gst-discoverer.c @@ -88,7 +88,7 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) GstDiscoverer *dc; gint timeout = 10; GstDiscovererInfo *info; - static gboolean initialized = 0; + static gboolean initialized = FALSE; if (!initialized) { /* We want critical warnings to assert so we can fix them */ @@ -105,6 +105,8 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) GST_PLUGIN_STATIC_REGISTER(ogg); GST_PLUGIN_STATIC_REGISTER(theora); GST_PLUGIN_STATIC_REGISTER(vorbis); + + initialized = TRUE; } dc = gst_discoverer_new (timeout * GST_SECOND, &err); diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c index 70f1a3813f..16615ef130 100644 --- a/fuzzing/typefind.c +++ b/fuzzing/typefind.c @@ -59,7 +59,7 @@ int LLVMFuzzerTestOneInput (const guint8 * data, size_t size) { GError *err = NULL; - static gboolean initialized = 0; + static gboolean initialized = FALSE; GstElement *pipeline, *source, *typefind, *fakesink; GstBuffer *buf; GstFlowReturn flowret; @@ -76,6 +76,8 @@ LLVMFuzzerTestOneInput (const guint8 * data, size_t size) GST_PLUGIN_STATIC_REGISTER (coreelements); GST_PLUGIN_STATIC_REGISTER (typefindfunctions); GST_PLUGIN_STATIC_REGISTER (app); + + initialized = TRUE; } /* Create the pipeline */ From feb718129f019a1f6a30e91a37ebe296f531a3f7 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sun, 5 Nov 2017 16:51:14 +0100 Subject: [PATCH 019/605] fuzzing: oss-fuzz build script Script that will be executed by oss-fuzz to build fuzzing targets and dependencies --- fuzzing/build-oss-fuzz.sh | 161 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 fuzzing/build-oss-fuzz.sh diff --git a/fuzzing/build-oss-fuzz.sh b/fuzzing/build-oss-fuzz.sh new file mode 100644 index 0000000000..b36e4b5442 --- /dev/null +++ b/fuzzing/build-oss-fuzz.sh @@ -0,0 +1,161 @@ +#!/bin/bash -eu + +# build-oss-fuzz.sh +# +# Build script which is executed by oss-fuzz build.sh +# +# $SRC: location of code checkouts +# $OUT: location to put fuzzing targets and corpus +# $WORK: writable directory where all compilation should be executed +# +# /!\ Do not override any CC, CXX, CFLAGS, ... variables +# + +# This script is divided in two parts +# +# 1) Build all the dependencies statically +# +# 2) Build the fuzzing targets + +# Prefix where we will temporarily install everything +PREFIX=$WORK/prefix +mkdir -p $PREFIX +export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig +export PATH=$PREFIX/bin:$PATH + +# Minimize gst-debug level/code +export CFLAGS="$CFLAGS -DGST_LEVEL_MAX=2" + +# +echo "CFLAGS : " $CFLAGS +echo "CXXFLAGS : " $CXXFLAGS +PLUGIN_DIR=$PREFIX/lib/gstreamer-1.0 + +# Switch to work directory +cd $WORK + +# 1) BUILD GLIB AND GSTREAMER +# Note: we build glib ourselves so that we get proper malloc/free backtraces +tar xvJf $SRC/glib-2.54.2.tar.xz +cd glib-2.54.2 +./configure --prefix=$PREFIX --enable-static --disable-shared --disable-libmount --with-pcre=internal && make -j$(nproc) && make install +cd .. + +# Note: We don't use/build orc since it still seems to be problematic +# with clang and the various sanitizers. + +# For now we only build core and base. Add other modules when/if needed +for i in gstreamer gst-plugins-base; +do + mkdir -p $i + cd $i + $SRC/$i/autogen.sh --prefix=$PREFIX --disable-shared --enable-static --disable-examples \ + --disable-gtk-doc --disable-introspection --enable-static-plugins \ + --disable-gst-tracer-hooks --disable-registry + make -j$(nproc) + make install + cd .. +done + + + +# 2) Build the target fuzzers + +# All targets will be linked in with $LIB_FUZZING_ENGINE which contains the +# actual fuzzing runner. Anything fuzzing engine can be used provided it calls +# the same function as libfuzzer. + +# Note: The fuzzer .o needs to be first compiled with CC and then linked with CXX + +# We want to statically link everything, except for shared libraries +# that are present on the base image. Those need to be specified +# beforehand and explicitely linked dynamically If any of the static +# dependencies require a pre-installed shared library, you need to add +# that library to the following list +PREDEPS_LDFLAGS="-Wl,-Bdynamic -ldl -lm -pthread -lrt -lpthread" + +# These are the basic .pc dependencies required to build any of the fuzzing targets +# That is : glib, gstreamer core and gst-app +# The extra target-specific dependencies are to be specified later +COMMON_DEPS="glib-2.0 gstreamer-1.0 gstreamer-app-1.0" + +# For each target, defined the following: +# TARGET_DEPS : Extra .pc dependencies for the target (in addition to $COMMON_DEPS) +# All dependencies (including sub-dependencies) must be speecified +# PLUGINS : .a of the plugins to link +# They must match the static plugins declared/registered in the target + +# +# TARGET : push-based ogg/theora/vorbis discoverer +# +# FIXME : Rename to discoverer_push_oggtheoravorbis + +TARGET_DEPS=" gstreamer-pbutils-1.0 \ + gstreamer-video-1.0 \ + gstreamer-audio-1.0 \ + gstreamer-riff-1.0 \ + gstreamer-tag-1.0 \ + zlib ogg vorbis vorbisenc \ + theoraenc theoradec theora" + +PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \ + $PLUGIN_DIR/libgsttypefindfunctions.a \ + $PLUGIN_DIR/libgstplayback.a \ + $PLUGIN_DIR/libgstapp.a \ + $PLUGIN_DIR/libgstvorbis.a \ + $PLUGIN_DIR/libgsttheora.a \ + $PLUGIN_DIR/libgstogg.a" + +echo +echo ">>>> BUILDING gst-discoverer.o" +echo +BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`" +BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`" + +$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/gst-discoverer.c -o $SRC/gst-ci/fuzzing/gst-discoverer.o +$CXX $CXXFLAGS \ + -o $OUT/gst-discoverer \ + $PREDEPS_LDFLAGS \ + $SRC/gst-ci/fuzzing/gst-discoverer.o \ + $PLUGINS \ + $BUILD_LDFLAGS \ + $LIB_FUZZING_ENGINE \ + -Wl,-Bdynamic + +# +# TARGET : push-based typefind +# + +# typefindfunction depends on pbutils which depends on gst{audio|video|tag} +TARGET_DEPS=" gstreamer-pbutils-1.0 \ + gstreamer-video-1.0 \ + gstreamer-audio-1.0 \ + gstreamer-tag-1.0" + +PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \ + $PLUGIN_DIR/libgsttypefindfunctions.a \ + $PLUGIN_DIR/libgstapp.a" + +echo +echo ">>>> BUILDING gst-discoverer.o" +echo +BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`" +BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`" + +$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gst-ci/fuzzing/typefind.c -o $SRC/gst-ci/fuzzing/typefind.o +$CXX $CXXFLAGS \ + -o $OUT/typefind \ + $PREDEPS_LDFLAGS \ + $SRC/gst-ci/fuzzing/typefind.o \ + $PLUGINS \ + $BUILD_LDFLAGS \ + $LIB_FUZZING_ENGINE \ + -Wl,-Bdynamic + +echo +echo ">>>> Installing seed corpus" +echo +# FIXME : Sadly we apparently need to have the corpus downloaded in the +# Dockerfile and not here. + +cp $SRC/*_seed_corpus.zip $OUT From f2025fd079d3873db044fcfe1d9b6346e5e0918d Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sun, 5 Nov 2017 16:58:15 +0100 Subject: [PATCH 020/605] fuzzing: Make the script executable --- fuzzing/build-oss-fuzz.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 fuzzing/build-oss-fuzz.sh diff --git a/fuzzing/build-oss-fuzz.sh b/fuzzing/build-oss-fuzz.sh old mode 100644 new mode 100755 From 6738712d3e02f8fa747208b687870b9b579a1be2 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sun, 5 Nov 2017 17:07:49 +0100 Subject: [PATCH 021/605] fuzzing: minor script fix --- fuzzing/build-oss-fuzz.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzzing/build-oss-fuzz.sh b/fuzzing/build-oss-fuzz.sh index b36e4b5442..a06f4facd5 100755 --- a/fuzzing/build-oss-fuzz.sh +++ b/fuzzing/build-oss-fuzz.sh @@ -107,7 +107,7 @@ PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \ $PLUGIN_DIR/libgstogg.a" echo -echo ">>>> BUILDING gst-discoverer.o" +echo ">>>> BUILDING gst-discoverer" echo BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`" BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`" @@ -137,7 +137,7 @@ PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \ $PLUGIN_DIR/libgstapp.a" echo -echo ">>>> BUILDING gst-discoverer.o" +echo ">>>> BUILDING typefind" echo BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`" BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`" From cb5d37e796258e68a04051f70821e1742fa6b31a Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Mon, 6 Nov 2017 10:00:32 +0100 Subject: [PATCH 022/605] fuzzing: Add README --- fuzzing/README.txt | 80 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 fuzzing/README.txt diff --git a/fuzzing/README.txt b/fuzzing/README.txt new file mode 100644 index 0000000000..c9621a0455 --- /dev/null +++ b/fuzzing/README.txt @@ -0,0 +1,80 @@ +Fuzzing GStreamer +================= + + This directory contains the various fuzzing targets and helper + scripts. + +* Fuzzing targets + + Fuzzing targets as small applications where we can test a specific + element or API. The goal is to have them be as small/targetted as + possible. + + ex: appsrc ! ! fakesink num-buffers= + + Not all components can be tested directly and therefore will be + indirectly tested via other targets (ex: libgstaudio will be tested + by targets/elements requiring it) + + Anything that can process externally-provided data should be + covered, but there are cases where it might not make sense to use a + fuzzer (such as most elements processing raw audio/video). + +* build-oss-fuzz.sh + + This is the script executed by the oss-fuzz project. + + It builds glib, GStreamer, plugins and the fuzzing targets. + +* *.c + + The fuzzing targets where the data to test will be provided to a + function whose signature follows the LibFuzzer signature: + https://llvm.org/docs/LibFuzzer.html + +* TODO + + * Add a standalone build script + + We need to be able to build and test the fuzzing targets outside + of the oss-fuzz infrastructure, and do that in our continous + integration system. + + We need: + + * A dummy fuzzing engine (given a directory, it opens all files and + calls the fuzzing targets with the content of those files. + * A script to be able to build those targets with that dummy engine + * A corpus of files to test those targets with. + + * Build targets with dummy engine and run with existing tests. + + * Create pull-based variants + + Currently the existing targets are push-based only. Where + applicable we should make pull-based variants to test the other + code paths. + + * Add more targets + + core: + gst_parse fuzzer ? + base: + ext/ + ogg + opus + pango + theora + vorbis + gst/ + subparse + typefind : already covered in typefind target + gst-libs/gst/ + sdp + other ones easily testable directly ? + + + + + + From f4b08721f7756d8fa361ea4e691aea08b49458b1 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Wed, 6 Dec 2017 08:41:59 +0100 Subject: [PATCH 023/605] fuzzing: Add local fuzz runner Allows testing without full oss-fuzz infrastructure --- fuzzing/gst-discoverer.c | 6 +++- fuzzing/localfuzzer.c | 78 ++++++++++++++++++++++++++++++++++++++++ fuzzing/typefind.c | 4 +++ 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 fuzzing/localfuzzer.c diff --git a/fuzzing/gst-discoverer.c b/fuzzing/gst-discoverer.c index 519537a61b..17c91590b9 100644 --- a/fuzzing/gst-discoverer.c +++ b/fuzzing/gst-discoverer.c @@ -27,6 +27,7 @@ #include #include +#ifndef LOCAL_FUZZ_BUILD GST_PLUGIN_STATIC_DECLARE(coreelements); GST_PLUGIN_STATIC_DECLARE(playback); GST_PLUGIN_STATIC_DECLARE(typefindfunctions); @@ -34,6 +35,7 @@ GST_PLUGIN_STATIC_DECLARE(app); GST_PLUGIN_STATIC_DECLARE(ogg); GST_PLUGIN_STATIC_DECLARE(theora); GST_PLUGIN_STATIC_DECLARE(vorbis); +#endif /* push-based discoverer fuzzing target * @@ -97,7 +99,8 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) /* Only initialize and register plugins once */ gst_init (NULL, NULL); - + +#ifndef LOCAL_FUZZ_BUILD GST_PLUGIN_STATIC_REGISTER(coreelements); GST_PLUGIN_STATIC_REGISTER(playback); GST_PLUGIN_STATIC_REGISTER(typefindfunctions); @@ -105,6 +108,7 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) GST_PLUGIN_STATIC_REGISTER(ogg); GST_PLUGIN_STATIC_REGISTER(theora); GST_PLUGIN_STATIC_REGISTER(vorbis); +#endif initialized = TRUE; } diff --git a/fuzzing/localfuzzer.c b/fuzzing/localfuzzer.c new file mode 100644 index 0000000000..188d2461a8 --- /dev/null +++ b/fuzzing/localfuzzer.c @@ -0,0 +1,78 @@ +/* GStreamer + * Copyright (C) 2017 Edward Hervey + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/* Local fuzzer runner */ +#include + +extern int LLVMFuzzerTestOneInput (const guint8 * data, size_t size); + +static void +test_file (gchar * filename) +{ + GDir *dir; + gchar *path; + gchar *contents; + gsize length; + + /* if filename is a directory, process the contents */ + if ((dir = g_dir_open (filename, 0, NULL))) { + const gchar *entry; + + while ((entry = g_dir_read_name (dir))) { + gchar *spath; + + spath = g_strconcat (filename, G_DIR_SEPARATOR_S, entry, NULL); + test_file (spath); + g_free (spath); + } + + g_dir_close (dir); + return; + } + + /* Make sure path is absolute */ + if (!g_path_is_absolute (filename)) { + gchar *curdir; + + curdir = g_get_current_dir (); + path = g_build_filename (curdir, filename, NULL); + g_free (curdir); + } else + path = g_strdup (filename); + + /* Check if path exists */ + if (g_file_get_contents (path, &contents, &length, NULL)) { + g_print (">>> %s (%" G_GSIZE_FORMAT " bytes)\n", path, length); + LLVMFuzzerTestOneInput ((const guint8 *) contents, length); + g_free (contents); + } + + g_free (path); +} + +int +main (int argc, gchar ** argv) +{ + gint i; + + for (i = 1; i < argc; i++) + test_file (argv[i]); + + return 0; +} diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c index 16615ef130..3d5d14cfec 100644 --- a/fuzzing/typefind.c +++ b/fuzzing/typefind.c @@ -26,9 +26,11 @@ #include #include +#ifndef LOCAL_FUZZ_BUILD GST_PLUGIN_STATIC_DECLARE (coreelements); GST_PLUGIN_STATIC_DECLARE (typefindfunctions); GST_PLUGIN_STATIC_DECLARE (app); +#endif /* push-based typefind fuzzing target * @@ -73,9 +75,11 @@ LLVMFuzzerTestOneInput (const guint8 * data, size_t size) /* Only initialize and register plugins once */ gst_init (NULL, NULL); +#ifndef LOCAL_FUZZ_BUILD GST_PLUGIN_STATIC_REGISTER (coreelements); GST_PLUGIN_STATIC_REGISTER (typefindfunctions); GST_PLUGIN_STATIC_REGISTER (app); +#endif initialized = TRUE; } From a080432a323be56c5e53a42066782f378002c59d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 7 Aug 2018 19:29:17 -0400 Subject: [PATCH 024/605] gitlab: Add a script to build manifest from a gitlab CI context --- gitlab/build_manifest.py | 100 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100755 gitlab/build_manifest.py diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py new file mode 100755 index 0000000000..0a7c7be21a --- /dev/null +++ b/gitlab/build_manifest.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +import os +import requests +import sys + + +GSTREAMER_MODULES = [ + 'gst-build', + 'gstreamer', + 'gst-plugins-base', + 'gst-plugins-good', + 'gst-plugins-bad', + 'gst-plugins-ugly', + 'gst-libav', + 'gst-devtools', + 'gst-docs', + 'gst-editing-services', + 'gst-omx', + 'gst-python', + 'gst-rtsp-server' +] + +MANIFEST_TEMPLATE = """ + + + + +%s +""" + + +def request(path): + gitlab_header = {'JOB_TOKEN': os.environ["CI_JOB_TOKEN"]} + + return requests.get('https://gitlab.gnome.org/api/v4/' + path, headers=gitlab_header).json() + + +def find_repository_sha(module, branchname): + for project in request('projects?search=' + module): + if project['name'] != module: + continue + + if 'namespace' not in project: + # print("No 'namespace' in: %s - ignoring?" % project, file=sys.stderr) + continue + + if project['namespace']['name'] in useful_namespaces: + if project['namespace']['name'] == user_namespace: + # If we have a branch with same name, use it. + for branch in request('%s/repository/branches' % project['id']): + if branch['name'] == branchname: + print("%s/%s" % (project['namespace']['name'], branchname)) + + return 'user', branch['commit']['id'] + else: + for branch in request('%s/repository/branches"' % project['id']): + if branch['name'] == branchname: + print("gstreamer/%s" % (branchname)) + return 'gstreamer', branch['commit']['id'] + + branch, = request('%s/repository/branches?search=master' % project['id']) + print('gstreamer/master') + return 'gstreamer', branch.attributes['commit']['id'] + + print('origin/master') + return 'origin', 'master' + +if __name__ == "__main__": + user_namespace = os.environ['CI_PROJECT_NAMESPACE'] + project_name = os.environ['CI_PROJECT_NAME'] + branchname = os.environ['CI_COMMIT_REF_NAME'] + + useful_namespaces = ['gstreamer'] + if branchname != 'master': + useful_namespaces.append(user_namespace) + + # Shouldn't be needed. + remote = "git://anongit.freedesktop.org/gstreamer/" + projects = '' + project_template = ' \n' + user_remote = os.path.dirname(os.environ['CI_PROJECT_URL']) + for module in GSTREAMER_MODULES: + print("Checking %s:" % module, end=' ') + + remote = "origin" + revision = None + if module == project_name: + revision = os.environ['CI_COMMIT_SHA'] + remote = "user" + print("%s/%s" % (user_namespace, branchname)) + else: + remote, revision = find_repository_sha(module, branchname) + + if not revision: + revision = 'master' + projects += project_template % (module, remote, revision) + + with open('manifest.xml', mode='w') as manifest: + print(MANIFEST_TEMPLATE % (user_remote, projects), file=manifest) From 095c93c62ce8318fa519c1b1a4d25b2cc940ce92 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 23 Oct 2018 15:41:58 +0300 Subject: [PATCH 025/605] build_manifest.py: add type annotations --- gitlab/build_manifest.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 0a7c7be21a..bf92b8a1ee 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -4,8 +4,10 @@ import os import requests import sys +from typing import Dict, Tuple, List -GSTREAMER_MODULES = [ +GSTREAMER_MODULES: List[str] = [ + # 'orc', 'gst-build', 'gstreamer', 'gst-plugins-base', @@ -21,7 +23,7 @@ GSTREAMER_MODULES = [ 'gst-rtsp-server' ] -MANIFEST_TEMPLATE = """ +MANIFEST_TEMPLATE: str = """ @@ -30,13 +32,13 @@ MANIFEST_TEMPLATE = """ """ -def request(path): - gitlab_header = {'JOB_TOKEN': os.environ["CI_JOB_TOKEN"]} +def request(path: str) -> Dict[str, str]: + gitlab_header: Dict[str, str] = {'JOB_TOKEN': os.environ["CI_JOB_TOKEN"]} return requests.get('https://gitlab.gnome.org/api/v4/' + path, headers=gitlab_header).json() -def find_repository_sha(module, branchname): +def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: for project in request('projects?search=' + module): if project['name'] != module: continue @@ -67,19 +69,19 @@ def find_repository_sha(module, branchname): return 'origin', 'master' if __name__ == "__main__": - user_namespace = os.environ['CI_PROJECT_NAMESPACE'] - project_name = os.environ['CI_PROJECT_NAME'] - branchname = os.environ['CI_COMMIT_REF_NAME'] + user_namespace: str = os.environ['CI_PROJECT_NAMESPACE'] + project_name: str = os.environ['CI_PROJECT_NAME'] + branchname: str = os.environ['CI_COMMIT_REF_NAME'] - useful_namespaces = ['gstreamer'] + useful_namespaces: List[str] = ['gstreamer'] if branchname != 'master': useful_namespaces.append(user_namespace) # Shouldn't be needed. - remote = "git://anongit.freedesktop.org/gstreamer/" - projects = '' - project_template = ' \n' - user_remote = os.path.dirname(os.environ['CI_PROJECT_URL']) + remote: str = "git://anongit.freedesktop.org/gstreamer/" + projects: str = '' + project_template: str = ' \n' + user_remote: str = os.path.dirname(os.environ['CI_PROJECT_URL']) for module in GSTREAMER_MODULES: print("Checking %s:" % module, end=' ') From c457ca0570d863c7bce8c1c19b55a1d618b4cafa Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 23 Oct 2018 16:34:19 +0300 Subject: [PATCH 026/605] build_manifest.py: Use fstrings for string formatting --- gitlab/build_manifest.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index bf92b8a1ee..71a5f57fcd 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -47,21 +47,23 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: # print("No 'namespace' in: %s - ignoring?" % project, file=sys.stderr) continue + id = project['id'] if project['namespace']['name'] in useful_namespaces: if project['namespace']['name'] == user_namespace: # If we have a branch with same name, use it. - for branch in request('%s/repository/branches' % project['id']): + for branch in request(f"{id}/repository/branches"): if branch['name'] == branchname: - print("%s/%s" % (project['namespace']['name'], branchname)) + name = project['namespace']['name'] + print(f"{name}/{branchname}") return 'user', branch['commit']['id'] else: - for branch in request('%s/repository/branches"' % project['id']): + for branch in request(f"{id}/repository/branches"): if branch['name'] == branchname: - print("gstreamer/%s" % (branchname)) + print(f"gstreamer/{branchname}") return 'gstreamer', branch['commit']['id'] - branch, = request('%s/repository/branches?search=master' % project['id']) + branch, = request(f"{id}/repository/branches?search=master") print('gstreamer/master') return 'gstreamer', branch.attributes['commit']['id'] @@ -83,14 +85,14 @@ if __name__ == "__main__": project_template: str = ' \n' user_remote: str = os.path.dirname(os.environ['CI_PROJECT_URL']) for module in GSTREAMER_MODULES: - print("Checking %s:" % module, end=' ') + print(f"Checking {module}:", end=' ') - remote = "origin" + remote = 'origin' revision = None if module == project_name: revision = os.environ['CI_COMMIT_SHA'] - remote = "user" - print("%s/%s" % (user_namespace, branchname)) + remote = 'user' + print(f"{user_namespace}/{branchname}") else: remote, revision = find_repository_sha(module, branchname) From 6e77bfb893ce580bc4c5454d356cbed7e3f0884a Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 23 Oct 2018 17:02:37 +0300 Subject: [PATCH 027/605] build_manifest.py: Use .format instead %s for templates --- gitlab/build_manifest.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 71a5f57fcd..2ee0e8850d 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -25,10 +25,10 @@ GSTREAMER_MODULES: List[str] = [ MANIFEST_TEMPLATE: str = """ - + -%s +{} """ @@ -38,6 +38,7 @@ def request(path: str) -> Dict[str, str]: return requests.get('https://gitlab.gnome.org/api/v4/' + path, headers=gitlab_header).json() + def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: for project in request('projects?search=' + module): if project['name'] != module: @@ -82,7 +83,7 @@ if __name__ == "__main__": # Shouldn't be needed. remote: str = "git://anongit.freedesktop.org/gstreamer/" projects: str = '' - project_template: str = ' \n' + project_template: str = " \n" user_remote: str = os.path.dirname(os.environ['CI_PROJECT_URL']) for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') @@ -98,7 +99,7 @@ if __name__ == "__main__": if not revision: revision = 'master' - projects += project_template % (module, remote, revision) + projects += project_template.format(module, remote, revision) with open('manifest.xml', mode='w') as manifest: - print(MANIFEST_TEMPLATE % (user_remote, projects), file=manifest) + print(MANIFEST_TEMPLATE.format(user_remote, projects), file=manifest) From 9c53cd0db0fb7521b18424bce1110cc69b8336fb Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 23 Oct 2018 20:04:43 +0300 Subject: [PATCH 028/605] build_manifest.py: abstract over the gitlab instance base_url --- gitlab/build_manifest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 2ee0e8850d..2560e73226 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -5,6 +5,7 @@ import requests import sys from typing import Dict, Tuple, List +from urllib.parse import urlparse GSTREAMER_MODULES: List[str] = [ # 'orc', @@ -34,9 +35,9 @@ MANIFEST_TEMPLATE: str = """ def request(path: str) -> Dict[str, str]: gitlab_header: Dict[str, str] = {'JOB_TOKEN': os.environ["CI_JOB_TOKEN"]} + base_url: str = urlparse(os.environ('CI_PROJECT_URL')).hostname - return requests.get('https://gitlab.gnome.org/api/v4/' + path, headers=gitlab_header).json() - + return requests.get(f"https://{base_url}/api/v4/" + path, headers=gitlab_header).json() def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: From 6772fa76aaa6920ac76b983050efed125280216e Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 23 Oct 2018 22:04:20 +0300 Subject: [PATCH 029/605] build_manifest.py: Split the hostname resolution from the requests --- gitlab/build_manifest.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 2560e73226..a94ce31694 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -35,11 +35,23 @@ MANIFEST_TEMPLATE: str = """ def request(path: str) -> Dict[str, str]: gitlab_header: Dict[str, str] = {'JOB_TOKEN': os.environ["CI_JOB_TOKEN"]} - base_url: str = urlparse(os.environ('CI_PROJECT_URL')).hostname + base_url: str = get_hostname(os.environ['CI_PROJECT_URL']) return requests.get(f"https://{base_url}/api/v4/" + path, headers=gitlab_header).json() +def get_hostname(url: str) -> str: + return urlparse(url).hostname + + +def test_get_hostname(): + gitlab = 'https://gitlab.com/example/a_project' + assert get_hostname(gitlab) == 'gitlab.com' + + fdo = 'https://gitlab.freedesktop.org/example/a_project' + assert get_hostname(fdo) == 'gitlab.freedesktop.org' + + def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: for project in request('projects?search=' + module): if project['name'] != module: From 61c5d7b6cf943a708bab3400c8947925f63fae46 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 23 Oct 2018 22:10:16 +0300 Subject: [PATCH 030/605] build_manifest.py: split the request function in two Make it so it does not depend on env variables so it will be easier to test. --- gitlab/build_manifest.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index a94ce31694..dda8266da6 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -33,13 +33,19 @@ MANIFEST_TEMPLATE: str = """ """ -def request(path: str) -> Dict[str, str]: - gitlab_header: Dict[str, str] = {'JOB_TOKEN': os.environ["CI_JOB_TOKEN"]} - base_url: str = get_hostname(os.environ['CI_PROJECT_URL']) +def request_raw(path: str, token: str, project_url: str) -> Dict[str, str]: + gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } + base_url: str = get_hostname(project_url) return requests.get(f"https://{base_url}/api/v4/" + path, headers=gitlab_header).json() +def request(path: str) -> Dict[str, str]: + token = os.environ["CI_JOB_TOKEN"] + project_url = os.environ['CI_PROJECT_URL'] + return request_raw(path, token, project_url) + + def get_hostname(url: str) -> str: return urlparse(url).hostname From 7ede52d7c72b813e0805d3a796286208805d429f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 23 Oct 2018 23:14:17 +0300 Subject: [PATCH 031/605] build_manifest.py: Use the project path not the name The project path is what's shown in the url bar, ex gitlab.example.com/john/gstreamer -> path == gstreamer The project name is defined in project settings and its just. Its more common to change the name of a fork than its path. --- gitlab/build_manifest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index dda8266da6..11de839d52 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -68,12 +68,12 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: continue id = project['id'] - if project['namespace']['name'] in useful_namespaces: - if project['namespace']['name'] == user_namespace: + if project['namespace']['path'] in useful_namespaces: + if project['namespace']['path'] == user_namespace: # If we have a branch with same name, use it. for branch in request(f"{id}/repository/branches"): if branch['name'] == branchname: - name = project['namespace']['name'] + name = project['namespace']['path'] print(f"{name}/{branchname}") return 'user', branch['commit']['id'] From 7ad193f276fe88459214815d3dcc64cd53c0f248 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 23 Oct 2018 23:23:16 +0300 Subject: [PATCH 032/605] build_manifest.py: Add a FIXME comment --- gitlab/build_manifest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 11de839d52..f1f08c993b 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -59,6 +59,9 @@ def test_get_hostname(): def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: + # FIXME: This does global search query in the whole gitlab instance. + # It has been working so far by a miracle. It should be limited only to + # the current namespace instead. for project in request('projects?search=' + module): if project['name'] != module: continue From dc275a204e9f3028d9a0412278d307d5de284252 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 13:30:12 +0300 Subject: [PATCH 033/605] build_manifest.py: Query the exact branch needed --- gitlab/build_manifest.py | 50 ++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index f1f08c993b..9be22fc6d9 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -6,6 +6,7 @@ import sys from typing import Dict, Tuple, List from urllib.parse import urlparse +# from pprint import pprint GSTREAMER_MODULES: List[str] = [ # 'orc', @@ -46,6 +47,30 @@ def request(path: str) -> Dict[str, str]: return request_raw(path, token, project_url) +def get_project_branch(project_id: int, name: str) -> Dict[str, str]: + path = f"projects/{project_id}/repository/branches?search={name}" + resp: List[Dict[str, str]] = request(path) + + if resp is None: + return None + + return resp[0] + + +def test_get_project_branch(): + id = 1353 + os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" + os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" + + twelve = get_project_branch(id, '1.12') + assert twelve is not None + assert twelve['name'] == '1.12' + + fourteen = get_project_branch(id, '1.14') + assert fourteen is not None + assert fourteen['name'] == '1.14' + + def get_hostname(url: str) -> str: return urlparse(url).hostname @@ -74,21 +99,22 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: if project['namespace']['path'] in useful_namespaces: if project['namespace']['path'] == user_namespace: # If we have a branch with same name, use it. - for branch in request(f"{id}/repository/branches"): - if branch['name'] == branchname: - name = project['namespace']['path'] - print(f"{name}/{branchname}") + branch = get_project_branch(id, branchname) + if branch is not None: + name = project['namespace']['path'] + print(f"{name}/{branchname}") - return 'user', branch['commit']['id'] + return 'user', branch['commit']['id'] else: - for branch in request(f"{id}/repository/branches"): - if branch['name'] == branchname: - print(f"gstreamer/{branchname}") - return 'gstreamer', branch['commit']['id'] + branch = get_project_branch(id, branchname) + if branch is not None: + print(f"gstreamer/{branchname}") + return 'gstreamer', branch['commit']['id'] - branch, = request(f"{id}/repository/branches?search=master") - print('gstreamer/master') - return 'gstreamer', branch.attributes['commit']['id'] + branch = get_project_branch(id, 'master') + if branch is not None: + print('gstreamer/master') + return 'gstreamer', branch.attributes['commit']['id'] print('origin/master') return 'origin', 'master' From ca9b1362b3f23adb45550b9d0dfe44d7df81861e Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 14:09:17 +0300 Subject: [PATCH 034/605] build_manifest.py: Make sure the remote always ends with a slash that's how `git-update` in `gst-build` expects it to be --- gitlab/build_manifest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 9be22fc6d9..7ace070a59 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -133,6 +133,9 @@ if __name__ == "__main__": projects: str = '' project_template: str = " \n" user_remote: str = os.path.dirname(os.environ['CI_PROJECT_URL']) + if not user_remote.endswith('/'): + user_remote += '/' + for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') From 0b9a8b5cace63e4d74b742452c1abda68582cbd4 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 16:54:35 +0300 Subject: [PATCH 035/605] build_manifest.py: Add a function to search user namespace of projects Also fix a type annotation error in the request functions. --- gitlab/build_manifest.py | 44 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 7ace070a59..455853d4f0 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -34,14 +34,14 @@ MANIFEST_TEMPLATE: str = """ """ -def request_raw(path: str, token: str, project_url: str) -> Dict[str, str]: +def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]]: gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } base_url: str = get_hostname(project_url) return requests.get(f"https://{base_url}/api/v4/" + path, headers=gitlab_header).json() -def request(path: str) -> Dict[str, str]: +def request(path: str) -> List[Dict[str, str]]: token = os.environ["CI_JOB_TOKEN"] project_url = os.environ['CI_PROJECT_URL'] return request_raw(path, token, project_url) @@ -51,9 +51,14 @@ def get_project_branch(project_id: int, name: str) -> Dict[str, str]: path = f"projects/{project_id}/repository/branches?search={name}" resp: List[Dict[str, str]] = request(path) - if resp is None: + if not resp: + return None + if not resp[0]: return None + # Not sure if there will be any edge cases where it returns more than one + # so lets see if anyone complains + assert len(resp) == 1 return resp[0] @@ -71,6 +76,39 @@ def test_get_project_branch(): assert fourteen['name'] == '1.14' +# Documentation: https://docs.gitlab.com/ce/api/projects.html#list-user-projects +def search_user_namespace(user: str, project: str) -> Dict[str, str]: + path = f"/users/{user}/projects?search={project}" + resp: List[Dict[str, str]] = request(path) + + if not resp: + return None + if not resp[0]: + return None + + # Not sure if there will be any edge cases where it returns more than one + # so lets see if anyone complains + assert len(resp) == 1 + return resp[0] + + +def test_search_user_namespace(): + os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" + os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/alatiera/gst-plugins-good" + user = "alatiera" + + gst = search_user_namespace("alatiera", "gstreamer") + assert gst is not None + assert gst['path'] == 'gstreamer' + + gst_good = search_user_namespace("alatiera", "gst-plugins-good") + assert gst_good is not None + assert gst_good['path'] == 'gst-plugins-good' + + res = search_user_namespace("alatiera", "404-project-not-found") + assert res is None + + def get_hostname(url: str) -> str: return urlparse(url).hostname From ea5459e12764d2de4f199872feef2b4bd7a3dfc5 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 17:34:58 +0300 Subject: [PATCH 036/605] build_manifest.py: search user namespaces instead of global Previously find_repository_sha would do a global search of all the projects in the gitlab instance. This ports it to use the user namespace endpoint. Additionally seems like the else: block never worked. --- gitlab/build_manifest.py | 46 +++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 455853d4f0..f98c946ed8 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -122,37 +122,29 @@ def test_get_hostname(): def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: - # FIXME: This does global search query in the whole gitlab instance. - # It has been working so far by a miracle. It should be limited only to - # the current namespace instead. - for project in request('projects?search=' + module): - if project['name'] != module: - continue - - if 'namespace' not in project: - # print("No 'namespace' in: %s - ignoring?" % project, file=sys.stderr) - continue + project = search_user_namespace(os.environ["GITLAB_USER_LOGIN"], module) + if project: id = project['id'] - if project['namespace']['path'] in useful_namespaces: - if project['namespace']['path'] == user_namespace: - # If we have a branch with same name, use it. - branch = get_project_branch(id, branchname) - if branch is not None: - name = project['namespace']['path'] - print(f"{name}/{branchname}") + # If we have a branch with same name, use it. + branch = get_project_branch(id, branchname) + if branch is not None: + name = project['namespace']['path'] + print(f"{name}/{branchname}") - return 'user', branch['commit']['id'] - else: - branch = get_project_branch(id, branchname) - if branch is not None: - print(f"gstreamer/{branchname}") - return 'gstreamer', branch['commit']['id'] + return 'user', branch['commit']['id'] + # This won't work until gstreamer migrates to gitlab + else: + id = 'FIXME: query the gstreamer group in fd.o' + branch = get_project_branch(id, branchname) + if branch is not None: + print(f"gstreamer/{branchname}") + return 'gstreamer', branch['commit']['id'] - branch = get_project_branch(id, 'master') - if branch is not None: - print('gstreamer/master') - return 'gstreamer', branch.attributes['commit']['id'] + branch = get_project_branch(id, 'master') + if branch is not None: + print('gstreamer/master') + return 'gstreamer', branch.attributes['commit']['id'] print('origin/master') return 'origin', 'master' From 17195ea3aa0f11ac031e53c9d0c1a051e57341ac Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 17:54:34 +0300 Subject: [PATCH 037/605] build_manifest.py: add a failure test case for branches --- gitlab/build_manifest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index f98c946ed8..2e366c35d9 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -75,6 +75,9 @@ def test_get_project_branch(): assert fourteen is not None assert fourteen['name'] == '1.14' + failure = get_project_branch(id, 'why-would-anyone-chose-this-branch-name') + assert failure is None + # Documentation: https://docs.gitlab.com/ce/api/projects.html#list-user-projects def search_user_namespace(user: str, project: str) -> Dict[str, str]: From 45c89ece1211c0e149136676c4c4d7008b5df92a Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 18:25:18 +0300 Subject: [PATCH 038/605] build_manifest.py: Make the needed env vars global variables --- gitlab/build_manifest.py | 46 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 2e366c35d9..38e98de358 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -34,6 +34,25 @@ MANIFEST_TEMPLATE: str = """ """ +# Predefined variables by Gitlab-CI +# Documentation: https://docs.gitlab.com/ce/ci/variables/README.html#predefined-variables-environment-variables +# +# mock: "alatiera" +GITLAB_USER_LOGIN: str = os.environ["GITLAB_USER_LOGIN"] +# mock: "xxxxxxxxxxxxxxxxxxxx" +CI_TOKEN: str = os.environ["CI_JOB_TOKEN"] +# mock: "https://gitlab.freedesktop.org/gstreamer/gstreamer" +CI_PROJECT_URL: str = os.environ['CI_PROJECT_URL'] +# mock: gstreamer +CI_PROJECT_NAME: str = os.environ['CI_PROJECT_NAME'] +# mock: gstreamer +CI_PROJECT_NAMESPACE: str = os.environ['CI_PROJECT_NAMESPACE'] +# mock: 5947ff970e3ae86b589f31772ca3c50240898710 +CI_COMMIT_SHA: str = os.environ['CI_COMMIT_SHA'] +# mock: "foobar/a-branch-name" +CURRENT_BRANCH: str = os.environ['CI_COMMIT_REF_NAME'] + + def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]]: gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } base_url: str = get_hostname(project_url) @@ -42,8 +61,8 @@ def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]] def request(path: str) -> List[Dict[str, str]]: - token = os.environ["CI_JOB_TOKEN"] - project_url = os.environ['CI_PROJECT_URL'] + token = CI_TOKEN + project_url = CI_PROJECT_URL return request_raw(path, token, project_url) @@ -125,7 +144,7 @@ def test_get_hostname(): def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: - project = search_user_namespace(os.environ["GITLAB_USER_LOGIN"], module) + project = search_user_namespace(GITLAB_USER_LOGIN, module) if project: id = project['id'] @@ -152,20 +171,11 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: print('origin/master') return 'origin', 'master' + if __name__ == "__main__": - user_namespace: str = os.environ['CI_PROJECT_NAMESPACE'] - project_name: str = os.environ['CI_PROJECT_NAME'] - branchname: str = os.environ['CI_COMMIT_REF_NAME'] - - useful_namespaces: List[str] = ['gstreamer'] - if branchname != 'master': - useful_namespaces.append(user_namespace) - - # Shouldn't be needed. - remote: str = "git://anongit.freedesktop.org/gstreamer/" projects: str = '' project_template: str = " \n" - user_remote: str = os.path.dirname(os.environ['CI_PROJECT_URL']) + user_remote: str = os.path.dirname(CI_PROJECT_URL) if not user_remote.endswith('/'): user_remote += '/' @@ -174,12 +184,12 @@ if __name__ == "__main__": remote = 'origin' revision = None - if module == project_name: - revision = os.environ['CI_COMMIT_SHA'] + if module == CI_PROJECT_NAME: + revision = CI_COMMIT_SHA remote = 'user' - print(f"{user_namespace}/{branchname}") + print(f"{CI_PROJECT_NAMESPACE}/{CURRENT_BRANCH}") else: - remote, revision = find_repository_sha(module, branchname) + remote, revision = find_repository_sha(module, CURRENT_BRANCH) if not revision: revision = 'master' From 10b1812e70b6729d5366371c30c1f44ef031dddc Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 18:26:21 +0300 Subject: [PATCH 039/605] build_manifest.py: Use more descriprive variable name --- gitlab/build_manifest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 38e98de358..9ec7573d1d 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -175,9 +175,9 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: if __name__ == "__main__": projects: str = '' project_template: str = " \n" - user_remote: str = os.path.dirname(CI_PROJECT_URL) - if not user_remote.endswith('/'): - user_remote += '/' + user_remote_url: str = os.path.dirname(CI_PROJECT_URL) + if not user_remote_url.endswith('/'): + user_remote_url += '/' for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') @@ -196,4 +196,4 @@ if __name__ == "__main__": projects += project_template.format(module, remote, revision) with open('manifest.xml', mode='w') as manifest: - print(MANIFEST_TEMPLATE.format(user_remote, projects), file=manifest) + print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest) From 412d980b62c84def2cfb34202a6cffae1bfe864b Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 19:40:30 +0300 Subject: [PATCH 040/605] build_manifest.py: Handle bad requests --- gitlab/build_manifest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 9ec7573d1d..181e20e5db 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -56,8 +56,12 @@ CURRENT_BRANCH: str = os.environ['CI_COMMIT_REF_NAME'] def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]]: gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } base_url: str = get_hostname(project_url) + resp = requests.get(f"https://{base_url}/api/v4/" + path, headers=gitlab_header) - return requests.get(f"https://{base_url}/api/v4/" + path, headers=gitlab_header).json() + if not resp.ok: + return None + + return resp.json() def request(path: str) -> List[Dict[str, str]]: @@ -97,6 +101,9 @@ def test_get_project_branch(): failure = get_project_branch(id, 'why-would-anyone-chose-this-branch-name') assert failure is None + failure2 = get_project_branch("invalid-id", '1.12') + assert failure2 is None + # Documentation: https://docs.gitlab.com/ce/api/projects.html#list-user-projects def search_user_namespace(user: str, project: str) -> Dict[str, str]: From e042a7603122b8c66f994ff296bc62df5b400858 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 19:44:19 +0300 Subject: [PATCH 041/605] build_manifest.py: Test the find_repository_sha function --- gitlab/build_manifest.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 181e20e5db..28cb0d96ff 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -179,6 +179,27 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: return 'origin', 'master' +def test_find_repository_sha(): + os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" + os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" + os.environ["GITLAB_USER_LOGIN"] = "alatiera" + + # This should find the repository in the user namespace + remote, git_ref = find_repository_sha("gst-plugins-good", "1.2") + assert remote == "user" + assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325" + + # This should fallback to upstream master branch since no matching branch was found + remote, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name") + assert remote == "origin" + assert git_ref == "master" + + # This should fallback to upstream master branch since no repository was found + remote, git_ref = find_repository_sha("totally-valid-project-name", "1.2") + assert remote == "origin" + assert git_ref == "master" + + if __name__ == "__main__": projects: str = '' project_template: str = " \n" From 64c0e6fba349fdfe9c399994a7fabab8e93b7fcf Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 19:47:04 +0300 Subject: [PATCH 042/605] build_manifest.py: Remove an early exit branch find_repository_sha will conclude to the same result so we can remove the whole if branch and extra variables that it needed. --- gitlab/build_manifest.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 28cb0d96ff..6b774c87c2 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -43,12 +43,6 @@ GITLAB_USER_LOGIN: str = os.environ["GITLAB_USER_LOGIN"] CI_TOKEN: str = os.environ["CI_JOB_TOKEN"] # mock: "https://gitlab.freedesktop.org/gstreamer/gstreamer" CI_PROJECT_URL: str = os.environ['CI_PROJECT_URL'] -# mock: gstreamer -CI_PROJECT_NAME: str = os.environ['CI_PROJECT_NAME'] -# mock: gstreamer -CI_PROJECT_NAMESPACE: str = os.environ['CI_PROJECT_NAMESPACE'] -# mock: 5947ff970e3ae86b589f31772ca3c50240898710 -CI_COMMIT_SHA: str = os.environ['CI_COMMIT_SHA'] # mock: "foobar/a-branch-name" CURRENT_BRANCH: str = os.environ['CI_COMMIT_REF_NAME'] @@ -209,15 +203,7 @@ if __name__ == "__main__": for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') - - remote = 'origin' - revision = None - if module == CI_PROJECT_NAME: - revision = CI_COMMIT_SHA - remote = 'user' - print(f"{CI_PROJECT_NAMESPACE}/{CURRENT_BRANCH}") - else: - remote, revision = find_repository_sha(module, CURRENT_BRANCH) + remote, revision = find_repository_sha(module, CURRENT_BRANCH) if not revision: revision = 'master' From 490e936dd17e13c03ce7652795ab75e18b30d00d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Oct 2018 20:42:58 +0300 Subject: [PATCH 043/605] build_manifest.py: Remove an unnecessary if branch I can't think of a scenario where this would be hit, and can't trigger make the find_reposiroty_sha function return None either. --- gitlab/build_manifest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 6b774c87c2..237a1a62f8 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -204,9 +204,6 @@ if __name__ == "__main__": for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') remote, revision = find_repository_sha(module, CURRENT_BRANCH) - - if not revision: - revision = 'master' projects += project_template.format(module, remote, revision) with open('manifest.xml', mode='w') as manifest: From 2e2864cde7192b036248dbb01ec94a3848dc08a3 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 25 Oct 2018 13:05:31 +0300 Subject: [PATCH 044/605] build_manifest.py: Properly query the group namepsace Add a function to query groups, so the fallback code will be functional once gstreamer migrates to gitlab. --- gitlab/build_manifest.py | 42 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 237a1a62f8..35acdd89ba 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -50,7 +50,8 @@ CURRENT_BRANCH: str = os.environ['CI_COMMIT_REF_NAME'] def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]]: gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } base_url: str = get_hostname(project_url) - resp = requests.get(f"https://{base_url}/api/v4/" + path, headers=gitlab_header) + url: str = f"https://{base_url}/api/v4/{path}" + resp = requests.get(url, headers=gitlab_header) if not resp.ok: return None @@ -131,6 +132,36 @@ def test_search_user_namespace(): res = search_user_namespace("alatiera", "404-project-not-found") assert res is None +# Documentation: https://docs.gitlab.com/ee/api/search.html#group-search-api +def search_group_namespace(group_id: str, project: str) -> Dict[str, str]: + path = f"groups/{group_id}/search?scope=projects&search={project}" + resp: List[Dict[str, str]] = request(path) + + if not resp: + return None + if not resp[0]: + return None + + return resp[0] + + +def test_search_group_namespace(): + import pytest + try: + os.environ["CI_TOKEN"] + except KeyError: + pytest.skip("Need to be authenticated with gitlab to succed") + + os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gstreamer" + group = "gstreamer" + + lab = search_group_namespace(group, "gstreamer") + assert lab is not None + assert lab['path'] == 'gstreamer' + + res = search_user_namespace(group, "404-project-not-found") + assert res is None + def get_hostname(url: str) -> str: return urlparse(url).hostname @@ -147,6 +178,7 @@ def test_get_hostname(): def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: project = search_user_namespace(GITLAB_USER_LOGIN, module) + # Find a fork in the User's namespace if project: id = project['id'] # If we have a branch with same name, use it. @@ -156,9 +188,13 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: print(f"{name}/{branchname}") return 'user', branch['commit']['id'] + # This won't work until gstreamer migrates to gitlab - else: - id = 'FIXME: query the gstreamer group in fd.o' + # Else check the upstream gstreamer repository + project = search_group_namespace('gstreamer', module) + if project: + id = project['id'] + # If we have a branch with same name, use it. branch = get_project_branch(id, branchname) if branch is not None: print(f"gstreamer/{branchname}") From 0fd632e8339f1d1ac33209de5cc643716bf7560b Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 25 Oct 2018 13:33:14 +0300 Subject: [PATCH 045/605] CI: run the build_manifest.py tests --- .gitlab-ci.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..1da9c73a5a --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,15 @@ +stages: + - test + +test manifest: + image: "fedora" + stage: "test" + before_script: + - dnf install -y python3-pytest python3-pytest-cov python3-requests + script: + - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py + coverage: '/TOTAL.*\s+(\d+%)$/' + artifacts: + reports: + junit: + - "junit.xml" From 07a1480fc9be91ecc7aca1fb19d1926e1c141907 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 25 Oct 2018 13:57:48 +0300 Subject: [PATCH 046/605] build_manifest.py: Avoid making the CI variables global There are half of the variables compared to what was used before, and now they are only used once so its not needed for them anymore to be global. This allows running the tests without needing to pre-define them in your local environment, since the ones needed are mocked in the tests code anyway. --- gitlab/build_manifest.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 35acdd89ba..b4a4b5d581 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -34,19 +34,6 @@ MANIFEST_TEMPLATE: str = """ """ -# Predefined variables by Gitlab-CI -# Documentation: https://docs.gitlab.com/ce/ci/variables/README.html#predefined-variables-environment-variables -# -# mock: "alatiera" -GITLAB_USER_LOGIN: str = os.environ["GITLAB_USER_LOGIN"] -# mock: "xxxxxxxxxxxxxxxxxxxx" -CI_TOKEN: str = os.environ["CI_JOB_TOKEN"] -# mock: "https://gitlab.freedesktop.org/gstreamer/gstreamer" -CI_PROJECT_URL: str = os.environ['CI_PROJECT_URL'] -# mock: "foobar/a-branch-name" -CURRENT_BRANCH: str = os.environ['CI_COMMIT_REF_NAME'] - - def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]]: gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } base_url: str = get_hostname(project_url) @@ -60,8 +47,10 @@ def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]] def request(path: str) -> List[Dict[str, str]]: - token = CI_TOKEN - project_url = CI_PROJECT_URL + # mock: "xxxxxxxxxxxxxxxxxxxx" + token: str = os.environ["CI_JOB_TOKEN"] + # mock: "https://gitlab.freedesktop.org/gstreamer/gstreamer" + project_url: str = os.environ['CI_PROJECT_URL'] return request_raw(path, token, project_url) @@ -176,7 +165,8 @@ def test_get_hostname(): def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: - project = search_user_namespace(GITLAB_USER_LOGIN, module) + user_login: str = os.environ["GITLAB_USER_LOGIN"] + project = search_user_namespace(user_login, module) # Find a fork in the User's namespace if project: @@ -233,13 +223,14 @@ def test_find_repository_sha(): if __name__ == "__main__": projects: str = '' project_template: str = " \n" - user_remote_url: str = os.path.dirname(CI_PROJECT_URL) + user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL']) if not user_remote_url.endswith('/'): user_remote_url += '/' for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') - remote, revision = find_repository_sha(module, CURRENT_BRANCH) + current_branch: str = os.environ['CI_COMMIT_REF_NAME'] + remote, revision = find_repository_sha(module, current_branch) projects += project_template.format(module, remote, revision) with open('manifest.xml', mode='w') as manifest: From f4d6e7e9c679e7719591fea3cda84439642479ca Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 25 Oct 2018 14:13:12 +0300 Subject: [PATCH 047/605] build_manifest.py: Avoid duplicating some code --- gitlab/build_manifest.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index b4a4b5d581..02da023d54 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -54,8 +54,7 @@ def request(path: str) -> List[Dict[str, str]]: return request_raw(path, token, project_url) -def get_project_branch(project_id: int, name: str) -> Dict[str, str]: - path = f"projects/{project_id}/repository/branches?search={name}" +def request_wrap(path: str) -> List[Dict[str, str]]: resp: List[Dict[str, str]] = request(path) if not resp: @@ -69,6 +68,11 @@ def get_project_branch(project_id: int, name: str) -> Dict[str, str]: return resp[0] +def get_project_branch(project_id: int, name: str) -> Dict[str, str]: + path = f"projects/{project_id}/repository/branches?search={name}" + return request_wrap(path) + + def test_get_project_branch(): id = 1353 os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" @@ -92,17 +96,7 @@ def test_get_project_branch(): # Documentation: https://docs.gitlab.com/ce/api/projects.html#list-user-projects def search_user_namespace(user: str, project: str) -> Dict[str, str]: path = f"/users/{user}/projects?search={project}" - resp: List[Dict[str, str]] = request(path) - - if not resp: - return None - if not resp[0]: - return None - - # Not sure if there will be any edge cases where it returns more than one - # so lets see if anyone complains - assert len(resp) == 1 - return resp[0] + return request_wrap(path) def test_search_user_namespace(): @@ -124,14 +118,7 @@ def test_search_user_namespace(): # Documentation: https://docs.gitlab.com/ee/api/search.html#group-search-api def search_group_namespace(group_id: str, project: str) -> Dict[str, str]: path = f"groups/{group_id}/search?scope=projects&search={project}" - resp: List[Dict[str, str]] = request(path) - - if not resp: - return None - if not resp[0]: - return None - - return resp[0] + return request_wrap(path) def test_search_group_namespace(): From 41a4ca4c8eb2eb46e69c5af66e9cc27cda097185 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 25 Oct 2018 14:36:47 +0300 Subject: [PATCH 048/605] build_manifest.py: preserve env vars when running tests pytest will happily run all the tests in the same proccess, which means if one mutates an environment variable, the change will bleed to the rest of the tests. --- gitlab/build_manifest.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 02da023d54..c85e6ff071 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -34,6 +34,29 @@ MANIFEST_TEMPLATE: str = """ """ +# Basically, pytest will happily let a test mutate a variable, and then run +# the next tests one the same environment without reset the vars. +def preserve_ci_vars(func): + """Preserve the original CI Variable values""" + def wrapper(): + try: + token = os.environ["CI_JOB_TOKEN"] + url = os.environ["CI_PROJECT_URL"] + user = os.environ["GITLAB_USER_LOGIN"] + except KeyError: + token = "invalid" + url = "invalid" + user = "" + + func() + + os.environ["CI_JOB_TOKEN"] = token + os.environ["CI_PROJECT_URL"] = url + os.environ["GITLAB_USER_LOGIN"] = user + + return wrapper + + def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]]: gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } base_url: str = get_hostname(project_url) @@ -73,6 +96,7 @@ def get_project_branch(project_id: int, name: str) -> Dict[str, str]: return request_wrap(path) +@preserve_ci_vars def test_get_project_branch(): id = 1353 os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" @@ -99,6 +123,7 @@ def search_user_namespace(user: str, project: str) -> Dict[str, str]: return request_wrap(path) +@preserve_ci_vars def test_search_user_namespace(): os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/alatiera/gst-plugins-good" @@ -121,10 +146,18 @@ def search_group_namespace(group_id: str, project: str) -> Dict[str, str]: return request_wrap(path) +@preserve_ci_vars def test_search_group_namespace(): import pytest try: - os.environ["CI_TOKEN"] + instance = get_hostname(os.environ['CI_PROJECT_URL']) + # This tests need to authenticate with the gitlab instace, + # and its hardcoded to check for the gitlab-org which exists + # on gitlab.com + # This can be changed to a gstreamer once its migrated to gitlab.fd.o + if instance != "gitlab.freedesktop.org": + # too lazy to use a different error + raise KeyError except KeyError: pytest.skip("Need to be authenticated with gitlab to succed") @@ -186,6 +219,7 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: return 'origin', 'master' +@preserve_ci_vars def test_find_repository_sha(): os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" From 02b57af288eb9c77b9887fc75a229845545f6fc0 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 25 Oct 2018 19:47:57 +0300 Subject: [PATCH 049/605] build_manifest.py: add some basic logging --- gitlab/build_manifest.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index c85e6ff071..0b678a77e5 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -61,8 +61,10 @@ def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]] gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } base_url: str = get_hostname(project_url) url: str = f"https://{base_url}/api/v4/{path}" + print(f"GET {url}") resp = requests.get(url, headers=gitlab_header) + print(f"Request returned: {resp.status_code}") if not resp.ok: return None @@ -92,6 +94,7 @@ def request_wrap(path: str) -> List[Dict[str, str]]: def get_project_branch(project_id: int, name: str) -> Dict[str, str]: + print(f"Searching for {name} branch in project {project_id}") path = f"projects/{project_id}/repository/branches?search={name}" return request_wrap(path) @@ -119,6 +122,7 @@ def test_get_project_branch(): # Documentation: https://docs.gitlab.com/ce/api/projects.html#list-user-projects def search_user_namespace(user: str, project: str) -> Dict[str, str]: + print(f"Searching for {project} project in @{user} user's namespace") path = f"/users/{user}/projects?search={project}" return request_wrap(path) @@ -142,6 +146,7 @@ def test_search_user_namespace(): # Documentation: https://docs.gitlab.com/ee/api/search.html#group-search-api def search_group_namespace(group_id: str, project: str) -> Dict[str, str]: + print(f"Searching for {project} project in @{group_id} group namespace") path = f"groups/{group_id}/search?scope=projects&search={project}" return request_wrap(path) @@ -191,31 +196,37 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: # Find a fork in the User's namespace if project: id = project['id'] + print(f"User project found, id: {id}") # If we have a branch with same name, use it. branch = get_project_branch(id, branchname) if branch is not None: - name = project['namespace']['path'] - print(f"{name}/{branchname}") + path = project['namespace']['path'] + print("Found mathcing branch in user's namespace") + print(f"{path}/{branchname}") return 'user', branch['commit']['id'] + print(f"Did not found user branch named {branchname}") # This won't work until gstreamer migrates to gitlab # Else check the upstream gstreamer repository project = search_group_namespace('gstreamer', module) if project: + print(f"Project found in Gstreamer upstream, id: {id}") id = project['id'] # If we have a branch with same name, use it. branch = get_project_branch(id, branchname) if branch is not None: + print("Found matching branch in upstream gst repo") print(f"gstreamer/{branchname}") return 'gstreamer', branch['commit']['id'] branch = get_project_branch(id, 'master') if branch is not None: + # print("Falling back to master branch in upstream repo") print('gstreamer/master') return 'gstreamer', branch.attributes['commit']['id'] - print('origin/master') + print('Falling back to origin/master') return 'origin', 'master' From b5b71baddb65282959e09278785dc2f963ad4778 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 30 Oct 2018 21:00:48 +0200 Subject: [PATCH 050/605] build_manifest.py: Prefer a custom token if set In order to be able to query the Gitlab Group API we need to be authenticated. CI_JOB_TOKEN for public jobs has a non-meaningfull value which does not actually authenticate the Runner to the intance. --- .gitlab-ci.yml | 2 +- gitlab/build_manifest.py | 44 +++++++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1da9c73a5a..3497e7d25c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,4 +12,4 @@ test manifest: artifacts: reports: junit: - - "junit.xml" + - "junit.xml" \ No newline at end of file diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 0b678a77e5..25ffafe58a 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -40,29 +40,36 @@ def preserve_ci_vars(func): """Preserve the original CI Variable values""" def wrapper(): try: - token = os.environ["CI_JOB_TOKEN"] url = os.environ["CI_PROJECT_URL"] user = os.environ["GITLAB_USER_LOGIN"] except KeyError: - token = "invalid" url = "invalid" user = "" + private = os.getenv("READ_PROJECTS_TOKEN", default=None) + if not private: + os.environ["READ_PROJECTS_TOKEN"] = "FOO" + func() - os.environ["CI_JOB_TOKEN"] = token os.environ["CI_PROJECT_URL"] = url os.environ["GITLAB_USER_LOGIN"] = user + if private: + os.environ["READ_PROJECTS_TOKEN"] = private + # if it was set after + elif os.getenv("READ_PROJECTS_TOKEN", default=None): + del os.environ["READ_PROJECTS_TOKEN"] + return wrapper -def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]]: - gitlab_header: Dict[str, str] = {'JOB_TOKEN': token } +def request_raw(path: str, headers: Dict[str, str], project_url: str) -> List[Dict[str, str]]: base_url: str = get_hostname(project_url) url: str = f"https://{base_url}/api/v4/{path}" print(f"GET {url}") - resp = requests.get(url, headers=gitlab_header) + print(f"Headers: {headers}") + resp = requests.get(url, headers=headers) print(f"Request returned: {resp.status_code}") if not resp.ok: @@ -72,11 +79,24 @@ def request_raw(path: str, token: str, project_url: str) -> List[Dict[str, str]] def request(path: str) -> List[Dict[str, str]]: - # mock: "xxxxxxxxxxxxxxxxxxxx" - token: str = os.environ["CI_JOB_TOKEN"] + # Check if there is a custom token set + # API calls to Group namespaces need to be authenticated + # regardless if the group/projects are public or not. + # CI_JOB_TOKEN has an actuall value only for private jobs + # and that's also an Gitlab EE feature. + # Which means no matter what we need to give the runner + # an actuall token if we want to query even the public + # gitlab.fd.o/gstreamer group + try: + headers: Dict[str, str] = {'Private-Token': os.environ["READ_PROJECTS_TOKEN"] } + except KeyError: + print("Custom token was not set, group api querries will fail") + # JOB_TOKEN is the default placeholder of CI_JOB_TOKEN + headers: Dict[str, str] = {'JOB_TOKEN': "xxxxxxxxxxxxxxxxxxxx" } + # mock: "https://gitlab.freedesktop.org/gstreamer/gstreamer" project_url: str = os.environ['CI_PROJECT_URL'] - return request_raw(path, token, project_url) + return request_raw(path, headers, project_url) def request_wrap(path: str) -> List[Dict[str, str]]: @@ -102,8 +122,8 @@ def get_project_branch(project_id: int, name: str) -> Dict[str, str]: @preserve_ci_vars def test_get_project_branch(): id = 1353 - os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" + del os.environ["READ_PROJECTS_TOKEN"] twelve = get_project_branch(id, '1.12') assert twelve is not None @@ -129,8 +149,8 @@ def search_user_namespace(user: str, project: str) -> Dict[str, str]: @preserve_ci_vars def test_search_user_namespace(): - os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/alatiera/gst-plugins-good" + del os.environ["READ_PROJECTS_TOKEN"] user = "alatiera" gst = search_user_namespace("alatiera", "gstreamer") @@ -232,9 +252,9 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: @preserve_ci_vars def test_find_repository_sha(): - os.environ["CI_JOB_TOKEN"] = "xxxxxxxxxxxxxxxxxxxx" os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" os.environ["GITLAB_USER_LOGIN"] = "alatiera" + del os.environ["READ_PROJECTS_TOKEN"] # This should find the repository in the user namespace remote, git_ref = find_repository_sha("gst-plugins-good", "1.2") From 76d38faf209477ebcb6257a690c5b6cdbcc6d7cb Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 30 Oct 2018 23:39:33 +0200 Subject: [PATCH 051/605] build_manifest.py: remove request assertion This does indeed brake things --- gitlab/build_manifest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 25ffafe58a..d0e7ffcf0e 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -107,9 +107,6 @@ def request_wrap(path: str) -> List[Dict[str, str]]: if not resp[0]: return None - # Not sure if there will be any edge cases where it returns more than one - # so lets see if anyone complains - assert len(resp) == 1 return resp[0] From 39e2ddf788c297e574ca0eed9317b62142020390 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 31 Oct 2018 13:39:21 +0200 Subject: [PATCH 052/605] build_manifest.py: avoid duplicate / in requests --- gitlab/build_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index d0e7ffcf0e..afb4f7e39b 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -140,7 +140,7 @@ def test_get_project_branch(): # Documentation: https://docs.gitlab.com/ce/api/projects.html#list-user-projects def search_user_namespace(user: str, project: str) -> Dict[str, str]: print(f"Searching for {project} project in @{user} user's namespace") - path = f"/users/{user}/projects?search={project}" + path = f"users/{user}/projects?search={project}" return request_wrap(path) From 7ff131879b0d818b27e8ed9fbd967ef8fec49d43 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 5 Nov 2018 13:33:51 +0200 Subject: [PATCH 053/605] build_manifest.py: Fix the api querries Previously the script assumed that the best match will be the first result, but that's not always the case apparently. --- gitlab/build_manifest.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index afb4f7e39b..676cee63b2 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -99,21 +99,21 @@ def request(path: str) -> List[Dict[str, str]]: return request_raw(path, headers, project_url) -def request_wrap(path: str) -> List[Dict[str, str]]: - resp: List[Dict[str, str]] = request(path) - - if not resp: - return None - if not resp[0]: - return None - - return resp[0] - - def get_project_branch(project_id: int, name: str) -> Dict[str, str]: print(f"Searching for {name} branch in project {project_id}") path = f"projects/{project_id}/repository/branches?search={name}" - return request_wrap(path) + results = request(path) + + if not results: + return None + + # The api returns a list of projects that match the search + # we want the exact match, which might not be the first on the list + for project_res in results: + if project_res["name"] == name: + return project_res + + return None @preserve_ci_vars @@ -141,7 +141,18 @@ def test_get_project_branch(): def search_user_namespace(user: str, project: str) -> Dict[str, str]: print(f"Searching for {project} project in @{user} user's namespace") path = f"users/{user}/projects?search={project}" - return request_wrap(path) + results = request(path) + + if not results: + return None + + # The api returns a list of projects that match the search + # we want the exact match, which might not be the first on the list + for project_res in results: + if project_res["path"] == project: + return project_res + + return None @preserve_ci_vars @@ -165,7 +176,7 @@ def test_search_user_namespace(): def search_group_namespace(group_id: str, project: str) -> Dict[str, str]: print(f"Searching for {project} project in @{group_id} group namespace") path = f"groups/{group_id}/search?scope=projects&search={project}" - return request_wrap(path) + return request(path) @preserve_ci_vars From 2320cfaffdf58d60052322b19a545e3bcc838b9c Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 5 Nov 2018 15:18:38 +0200 Subject: [PATCH 054/605] build_manifest: Drop the group search for now Since we can't authenticate with the CI_JOB_TOKEN it needs a workaroud. Currently it falls back to the upstream repo which is good enough to get us going. --- gitlab/build_manifest.py | 51 ---------------------------------------- 1 file changed, 51 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 676cee63b2..e478f16169 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -172,38 +172,6 @@ def test_search_user_namespace(): res = search_user_namespace("alatiera", "404-project-not-found") assert res is None -# Documentation: https://docs.gitlab.com/ee/api/search.html#group-search-api -def search_group_namespace(group_id: str, project: str) -> Dict[str, str]: - print(f"Searching for {project} project in @{group_id} group namespace") - path = f"groups/{group_id}/search?scope=projects&search={project}" - return request(path) - - -@preserve_ci_vars -def test_search_group_namespace(): - import pytest - try: - instance = get_hostname(os.environ['CI_PROJECT_URL']) - # This tests need to authenticate with the gitlab instace, - # and its hardcoded to check for the gitlab-org which exists - # on gitlab.com - # This can be changed to a gstreamer once its migrated to gitlab.fd.o - if instance != "gitlab.freedesktop.org": - # too lazy to use a different error - raise KeyError - except KeyError: - pytest.skip("Need to be authenticated with gitlab to succed") - - os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gstreamer" - group = "gstreamer" - - lab = search_group_namespace(group, "gstreamer") - assert lab is not None - assert lab['path'] == 'gstreamer' - - res = search_user_namespace(group, "404-project-not-found") - assert res is None - def get_hostname(url: str) -> str: return urlparse(url).hostname @@ -235,25 +203,6 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: return 'user', branch['commit']['id'] print(f"Did not found user branch named {branchname}") - # This won't work until gstreamer migrates to gitlab - # Else check the upstream gstreamer repository - project = search_group_namespace('gstreamer', module) - if project: - print(f"Project found in Gstreamer upstream, id: {id}") - id = project['id'] - # If we have a branch with same name, use it. - branch = get_project_branch(id, branchname) - if branch is not None: - print("Found matching branch in upstream gst repo") - print(f"gstreamer/{branchname}") - return 'gstreamer', branch['commit']['id'] - - branch = get_project_branch(id, 'master') - if branch is not None: - # print("Falling back to master branch in upstream repo") - print('gstreamer/master') - return 'gstreamer', branch.attributes['commit']['id'] - print('Falling back to origin/master') return 'origin', 'master' From 9d637f6694e43d8e160ec17999f052aaf27e1d0d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 6 Nov 2018 15:45:53 +0200 Subject: [PATCH 055/605] build_manifest.py: Hardcode the project ids of upstream repos. Since we can't use the group api to query their id, see previous commit, we are going to hard code the upstream project id for the time being. --- gitlab/build_manifest.py | 74 +++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index e478f16169..bb384677fd 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -8,21 +8,25 @@ from typing import Dict, Tuple, List from urllib.parse import urlparse # from pprint import pprint -GSTREAMER_MODULES: List[str] = [ - # 'orc', - 'gst-build', - 'gstreamer', - 'gst-plugins-base', - 'gst-plugins-good', - 'gst-plugins-bad', - 'gst-plugins-ugly', - 'gst-libav', - 'gst-devtools', - 'gst-docs', - 'gst-editing-services', - 'gst-omx', - 'gst-python', - 'gst-rtsp-server' +# Each item is a Tuple of (project-path, project-id) +# ex. https://gitlab.freedesktop.org/gstreamer/gst-build +# has project path 'gst-build' and project-id '1342' +# TODO: Named tuples are awesome +GSTREAMER_MODULES: List[Tuple[str, int]] = [ + # ('orc', 1360), + ('gst-build', 1342), + ('gstreamer', 1357), + ('gst-plugins-base', 1352), + ('gst-plugins-good', 1353), + ('gst-plugins-bad', 1351), + ('gst-plugins-ugly', 1354), + ('gst-libav', 1349), + ('gst-devtools', 1344), + ('gst-docs', 1345), + ('gst-editing-services', 1346), + ('gst-omx', 1350), + ('gst-python', 1355), + ('gst-rtsp-server', 1362), ] MANIFEST_TEMPLATE: str = """ @@ -185,9 +189,9 @@ def test_get_hostname(): assert get_hostname(fdo) == 'gitlab.freedesktop.org' -def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: +def find_repository_sha(module: Tuple[str, int], branchname: str) -> Tuple[str, str]: user_login: str = os.environ["GITLAB_USER_LOGIN"] - project = search_user_namespace(user_login, module) + project = search_user_namespace(user_login, module[0]) # Find a fork in the User's namespace if project: @@ -203,8 +207,24 @@ def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: return 'user', branch['commit']['id'] print(f"Did not found user branch named {branchname}") - print('Falling back to origin/master') - return 'origin', 'master' + # Check upstream project for a branch + branch = get_project_branch(module[1], branchname) + if branch is not None: + print("Found mathcing branch in upstream project") + print(f"gstreamer/{branchname}") + return 'gstreamer', branch['commit']['id'] + + # Fallback to using upstream master branch + branch = get_project_branch(module[1], 'master') + if branch is not None: + print("Falling back to master branch on upstream project") + print(f"gstreamer/master") + return 'gstreamer', branch['commit']['id'] + + # This should never occur given the upstream fallback above + print("If something reaches that point, please file a bug") + print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues") + assert False @preserve_ci_vars @@ -214,19 +234,19 @@ def test_find_repository_sha(): del os.environ["READ_PROJECTS_TOKEN"] # This should find the repository in the user namespace - remote, git_ref = find_repository_sha("gst-plugins-good", "1.2") + remote, git_ref = find_repository_sha(("gst-plugins-good", 1353), "1.2") assert remote == "user" assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325" # This should fallback to upstream master branch since no matching branch was found - remote, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name") - assert remote == "origin" - assert git_ref == "master" + remote, git_ref = find_repository_sha(("gst-plugins-good", 1353), "totally-valid-branch-name") + assert remote == "gstreamer" # This should fallback to upstream master branch since no repository was found - remote, git_ref = find_repository_sha("totally-valid-project-name", "1.2") - assert remote == "origin" - assert git_ref == "master" + remote, git_ref = find_repository_sha(("totally-valid-project-name", 42), "1.2") + assert remote == "gstreamer" + # This is now the sha of the last commit + # assert git_ref == "master" if __name__ == "__main__": @@ -240,7 +260,7 @@ if __name__ == "__main__": print(f"Checking {module}:", end=' ') current_branch: str = os.environ['CI_COMMIT_REF_NAME'] remote, revision = find_repository_sha(module, current_branch) - projects += project_template.format(module, remote, revision) + projects += project_template.format(module[0], remote, revision) with open('manifest.xml', mode='w') as manifest: print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest) From fa203572212b7b2f54fabe1d900aebe1d08f21a7 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 6 Nov 2018 15:48:55 +0200 Subject: [PATCH 056/605] build_manifest.py: remove get_hostname function Not really needed anymore now that it just calls urlparse. --- gitlab/build_manifest.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index bb384677fd..9b1e208404 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -69,7 +69,8 @@ def preserve_ci_vars(func): def request_raw(path: str, headers: Dict[str, str], project_url: str) -> List[Dict[str, str]]: - base_url: str = get_hostname(project_url) + # ex. base_url = "gitlab.freedesktop.org" + base_url: str = urlparse(project_url).hostname url: str = f"https://{base_url}/api/v4/{path}" print(f"GET {url}") print(f"Headers: {headers}") @@ -177,18 +178,6 @@ def test_search_user_namespace(): assert res is None -def get_hostname(url: str) -> str: - return urlparse(url).hostname - - -def test_get_hostname(): - gitlab = 'https://gitlab.com/example/a_project' - assert get_hostname(gitlab) == 'gitlab.com' - - fdo = 'https://gitlab.freedesktop.org/example/a_project' - assert get_hostname(fdo) == 'gitlab.freedesktop.org' - - def find_repository_sha(module: Tuple[str, int], branchname: str) -> Tuple[str, str]: user_login: str = os.environ["GITLAB_USER_LOGIN"] project = search_user_namespace(user_login, module[0]) From f1ea83a2971cf51947162ea1b4324de2dc446e42 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 7 Aug 2018 19:27:40 -0400 Subject: [PATCH 057/605] docker: fedora: Update to fedora 29 --- docker/build-base-images/Dockerfile-fedora | 34 +++------------------- docker/runtime-images/Dockerfile-fedora | 12 ++++---- 2 files changed, 9 insertions(+), 37 deletions(-) diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index 5896af1627..7440aeda8f 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -1,4 +1,4 @@ -FROM gstreamer/base-fedora:latest +FROM registry.freedesktop.org/thiblahute/gst-ci/fedora-runtime # Add rpmfusion repositories and install all build dependencies of GStreamer RUN dnf builddep -y \ @@ -14,10 +14,9 @@ RUN dnf builddep -y \ gstreamer1-plugins-bad-freeworld \ gstreamer1-libav \ gstreamer1-rtsp-server \ + python3-gstreamer1 \ && \ rpm -e \ - gstreamer1 \ - gstreamer1-plugins-base \ gstreamer1-devel \ gstreamer1-plugins-base-devel \ && \ @@ -34,35 +33,10 @@ RUN dnf builddep -y \ redhat-rpm-config \ xorg-x11-server-Xvfb \ xorg-x11-server-utils \ - wget \ - && \ + wget \ + meson && \ dnf clean all -# Clone copies of all repositories to avoid extra downloads -RUN mkdir /repositories && \ - cd /repositories && \ - for i in orc \ - common \ - gstreamer \ - gst-plugins-base \ - gst-plugins-good \ - gst-plugins-bad \ - gst-plugins-ugly \ - gst-libav \ - gst-build \ - gst-devtools\ - gst-docs \ - gst-editing-services \ - gst-omx \ - gst-python \ - gst-rtsp-server; \ - do \ - git clone --mirror https://anongit.freedesktop.org/git/gstreamer/$i; \ - done - -# Install repo tool -RUN wget -nd https://storage.googleapis.com/git-repo-downloads/repo -O /usr/bin/repo && chmod a+x /usr/bin/repo - # Add the user UID:1000, GID:1000, home at /gstbuild RUN groupadd -r gstbuild -g 1000 && useradd -u 1000 -r -g gstbuild -m -d /gstbuild -s /sbin/nologin -c "Gstbuild user" gstbuild && \ chmod 755 /gstbuild diff --git a/docker/runtime-images/Dockerfile-fedora b/docker/runtime-images/Dockerfile-fedora index fe5c80a14c..c552c743d6 100644 --- a/docker/runtime-images/Dockerfile-fedora +++ b/docker/runtime-images/Dockerfile-fedora @@ -1,12 +1,12 @@ -FROM fedora:26 +FROM fedora:29 # Add rpmfusion repositories and install all build dependencies of GStreamer # and then remove the gst packages themselves RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ - dnf install -y http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-26.noarch.rpm \ - http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-26.noarch.rpm && \ - rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-nonfree-fedora-26 && \ - rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-26 && \ + dnf install -y http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-29.noarch.rpm \ + http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-29.noarch.rpm && \ + rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-nonfree-fedora-29 && \ + rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-29 && \ dnf upgrade -y && \ dnf install -y \ dnf-plugins-core \ @@ -31,8 +31,6 @@ RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ gstreamer1-rtsp-server \ && \ rpm -e \ - gstreamer1 \ - gstreamer1-plugins-base \ gstreamer1-plugins-good \ gstreamer1-plugins-good-extras \ gstreamer1-plugins-ugly \ From fc8b90123d5d179829d07bfe0dd9e81777e371c0 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 7 Aug 2018 19:28:08 -0400 Subject: [PATCH 058/605] docker: ubuntu: Update to 18.04 --- docker/build-base-images/Dockerfile-ubuntu | 2 +- docker/runtime-images/Dockerfile-ubuntu | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu index 2010b43801..ca137c5d48 100644 --- a/docker/build-base-images/Dockerfile-ubuntu +++ b/docker/build-base-images/Dockerfile-ubuntu @@ -1,4 +1,4 @@ -FROM gstreamer/base-ubuntu:latest +FROM registry.freedesktop.org/thiblahute/gst-ci/ubuntu-runtime # make source packages available in order to figure out build dependencies RUN \ diff --git a/docker/runtime-images/Dockerfile-ubuntu b/docker/runtime-images/Dockerfile-ubuntu index a9e4f8e387..8856d7ca23 100644 --- a/docker/runtime-images/Dockerfile-ubuntu +++ b/docker/runtime-images/Dockerfile-ubuntu @@ -1,4 +1,6 @@ -FROM ubuntu:17.04 +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive # upgrade and install all runtime dependencies for gstreamer From 778359feb8c98c566a9b16a94cab88dbc4e6501f Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 4 Nov 2018 20:22:38 -0300 Subject: [PATCH 059/605] docker: Add an archlinux dockerfile --- docker/build-base-images/Dockerfile-archlinux | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docker/build-base-images/Dockerfile-archlinux diff --git a/docker/build-base-images/Dockerfile-archlinux b/docker/build-base-images/Dockerfile-archlinux new file mode 100644 index 0000000000..79e5364ed7 --- /dev/null +++ b/docker/build-base-images/Dockerfile-archlinux @@ -0,0 +1,25 @@ +FROM base/devel:latest + +ENV LANG=en_US.UTF-8 +ENV GST_BUILD_PATH="/gst-build/" +ENV GST_VALIDATE_OUTPUT="/validate-output" +ENV GST_VALIDATE_TESTSUITES="$GST_VALIDATE_OUTPUT/gst-integration-testsuites" +ENV GST_BUILD_MESON_ARGS="-Dpython=disabled -Dlibav=enabled -Dugly=enabled -Dbad=enabled -Ddevtools=enabled -Dges=enabled -Drtsp_server=enabled -Domx=disabled -Dvaapi=disabled -Dsharp=enabled" +ENV PATH="/usr/lib/ccache/bin/:${PATH}" + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen + +RUN pacman -Sy --noconfirm python3 libxml2 libxslt cmake libyaml git make nodejs ninja clang llvm flex python-gobject gstreamer gst-plugins-base gst-plugins-bad gst-plugins-ugly gst-libav gst-editing-services python-pip npm json-glib gobject-introspection wget mono unzip glib2-docs xorg-xset xorg-server-xvfb ccache +RUN pip install --upgrade git+https://github.com/hotdoc/hotdoc.git@gst git+https://github.com/hotdoc/hotdoc_c_extension.git@gst git+https://github.com/thiblahute/meson.git@hotdoc + +RUN mkdir $GST_VALIDATE_OUTPUT && \ + git clone https://anongit.freedesktop.org/git/gstreamer/gst-integration-testsuites.git $GST_VALIDATE_TESTSUITES && \ + cd $GST_VALIDATE_TESTSUITES/medias/ && \ + python get_files.py defaults/ + +# get gst-build and make all subprojects available +RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ + cd $GST_BUILD_PATH && \ + meson build/ $GST_BUILD_MESON_ARGS + +RUN rm -Rf /var/cache/pacman/ \ No newline at end of file From 8ad477eb0e29f3f39b0af003d7fe45460b76a7f4 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 4 Nov 2018 20:24:10 -0300 Subject: [PATCH 060/605] docker: Add a dockerfile to export test result with allure based on alpine --- docker/runtime-images/Dockerfile-allure | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docker/runtime-images/Dockerfile-allure diff --git a/docker/runtime-images/Dockerfile-allure b/docker/runtime-images/Dockerfile-allure new file mode 100644 index 0000000000..04818c2ee5 --- /dev/null +++ b/docker/runtime-images/Dockerfile-allure @@ -0,0 +1,9 @@ +FROM anapsix/alpine-java:latest + +# Get the allure test report framework +RUN export ALLURE_VERSION=2.7.0 && \ + wget "https://bintray.com/qameta/generic/download_file?file_path=io%2Fqameta%2Fallure%2Fallure%2F2.7.0%2Fallure-${ALLURE_VERSION}.zip" -O allure.zip + unzip allure.zip && \ + mv allure-${ALLURE_VERSION} /opt/allure && \ + chmod +x /opt/allure/bin/allure && \ + rm allure.zip \ No newline at end of file From 2c1531ab579e9a1b6a9352a777b52777acd1b42a Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 7 Aug 2018 19:34:38 -0400 Subject: [PATCH 061/605] Remove autotools support and update meson. This simplifies our docker images and we should focus on meson support for our new CI system! --- docker/build-base-images/Dockerfile-archlinux | 4 +- docker/build-base-images/Dockerfile-fedora | 25 +++--- .../build-base-images/Dockerfile-meson-fedora | 12 --- docker/build-base-images/Dockerfile-ubuntu | 86 +++++++------------ 4 files changed, 48 insertions(+), 79 deletions(-) delete mode 100644 docker/build-base-images/Dockerfile-meson-fedora diff --git a/docker/build-base-images/Dockerfile-archlinux b/docker/build-base-images/Dockerfile-archlinux index 79e5364ed7..6ffc188e27 100644 --- a/docker/build-base-images/Dockerfile-archlinux +++ b/docker/build-base-images/Dockerfile-archlinux @@ -10,7 +10,7 @@ ENV PATH="/usr/lib/ccache/bin/:${PATH}" RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen RUN pacman -Sy --noconfirm python3 libxml2 libxslt cmake libyaml git make nodejs ninja clang llvm flex python-gobject gstreamer gst-plugins-base gst-plugins-bad gst-plugins-ugly gst-libav gst-editing-services python-pip npm json-glib gobject-introspection wget mono unzip glib2-docs xorg-xset xorg-server-xvfb ccache -RUN pip install --upgrade git+https://github.com/hotdoc/hotdoc.git@gst git+https://github.com/hotdoc/hotdoc_c_extension.git@gst git+https://github.com/thiblahute/meson.git@hotdoc +RUN pip install --upgrade git+https://github.com/hotdoc/hotdoc.git@gst git+https://github.com/hotdoc/hotdoc_c_extension.git@gst meson RUN mkdir $GST_VALIDATE_OUTPUT && \ git clone https://anongit.freedesktop.org/git/gstreamer/gst-integration-testsuites.git $GST_VALIDATE_TESTSUITES && \ @@ -20,6 +20,6 @@ RUN mkdir $GST_VALIDATE_OUTPUT && \ # get gst-build and make all subprojects available RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ cd $GST_BUILD_PATH && \ - meson build/ $GST_BUILD_MESON_ARGS + meson build/ $GST_BUILD_MESON_ARGS && rm -Rf build/ RUN rm -Rf /var/cache/pacman/ \ No newline at end of file diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index 7440aeda8f..14faa1cf67 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -33,19 +33,22 @@ RUN dnf builddep -y \ redhat-rpm-config \ xorg-x11-server-Xvfb \ xorg-x11-server-utils \ - wget \ - meson && \ + ninja-build \ + wget && \ dnf clean all -# Add the user UID:1000, GID:1000, home at /gstbuild -RUN groupadd -r gstbuild -g 1000 && useradd -u 1000 -r -g gstbuild -m -d /gstbuild -s /sbin/nologin -c "Gstbuild user" gstbuild && \ - chmod 755 /gstbuild +RUN pip3 install meson -# Set the working directory to gstbuild home directory -WORKDIR /gstbuild - -# Specify the user to execute all commands below -USER gstbuild +ENV GST_BUILD_PATH="/gst-build/" +ENV GST_VALIDATE_OUTPUT="/validate-output" +ENV GST_VALIDATE_TESTSUITES="$GST_VALIDATE_OUTPUT/gst-integration-testsuites" +# FIXME Enabled gst-libav. +ENV GST_BUILD_MESON_ARGS="-Dpython=disabled -Dlibav=disabled -Dugly=enabled -Dbad=enabled -Ddevtools=enabled -Dges=enabled -Drtsp_server=enabled -Domx=disabled -Dvaapi=disabled -Dsharp=disabled" # Configure git for various usage -RUN git config --global user.email "gstbuild@gstreamer.net" && git config --global user.name "Gstbuild Runner" +RUN git config --global user.email "gst-build@gstreamer.net" && git config --global user.name "Gstbuild Runner" + +# get gst-build and make all subprojects available +RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ + cd $GST_BUILD_PATH && \ + meson build/ $GST_BUILD_MESON_ARGS \ No newline at end of file diff --git a/docker/build-base-images/Dockerfile-meson-fedora b/docker/build-base-images/Dockerfile-meson-fedora deleted file mode 100644 index f6eba9db3b..0000000000 --- a/docker/build-base-images/Dockerfile-meson-fedora +++ /dev/null @@ -1,12 +0,0 @@ -FROM gstreamer/build-base-fedora:latest - -USER root - -RUN dnf install -y \ - meson - -USER gstbuild - -RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build - -WORKDIR /gstbuild/gst-build diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu index ca137c5d48..cb879157b1 100644 --- a/docker/build-base-images/Dockerfile-ubuntu +++ b/docker/build-base-images/Dockerfile-ubuntu @@ -2,62 +2,40 @@ FROM registry.freedesktop.org/thiblahute/gst-ci/ubuntu-runtime # make source packages available in order to figure out build dependencies RUN \ - sed -i "s/# deb-src/deb-src/g" /etc/apt/sources.list && \ - apt-get update && \ - apt-get -y build-dep \ - orc \ - gstreamer1.0 \ - gst-plugins-base1.0 \ - gst-plugins-good1.0 \ - gst-plugins-bad1.0 \ - gst-plugins-ugly1.0 \ - gst-libav1.0 \ - gst-rtsp-server1.0 \ - gst-python1.0 \ - gstreamer-vaapi \ - gstreamer-editing-services1.0 && \ - apt-get -y install \ - ccache \ - gdb \ - git \ - xvfb \ - wget + sed -i "s/# deb-src/deb-src/g" /etc/apt/sources.list && \ + apt-get update && \ + apt-get -y build-dep \ + orc \ + gstreamer1.0 \ + gst-plugins-base1.0 \ + gst-plugins-good1.0 \ + gst-plugins-bad1.0 \ + gst-plugins-ugly1.0 \ + gst-libav1.0 \ + gst-rtsp-server1.0 \ + gst-python1.0 \ + gstreamer-vaapi \ + gstreamer-editing-services1.0 && \ + apt-get -y install \ + ccache \ + gdb \ + git \ + xvfb \ + python3-pip \ + wget -# Clone copies of all repositories to avoid extra downloads -RUN mkdir /repositories && \ - cd /repositories && \ - for i in orc \ - common \ - gstreamer \ - gst-plugins-base \ - gst-plugins-good \ - gst-plugins-bad \ - gst-plugins-ugly \ - gst-libav \ - gst-build \ - gst-devtools\ - gst-docs \ - gst-editing-services \ - gst-omx \ - gst-python \ - gst-rtsp-server; \ - do \ - git clone --mirror https://anongit.freedesktop.org/git/gstreamer/$i; \ - done +RUN pip3 install meson -# Install repo tool -RUN wget -nd https://storage.googleapis.com/git-repo-downloads/repo -O /usr/bin/repo && chmod a+x /usr/bin/repo - -# Add the user UID:1000, GID:1000, home at /gstbuild -RUN groupadd -r gstbuild -g 1000 && useradd -u 1000 -r -g gstbuild -m -d /gstbuild -s /sbin/nologin -c "Gstbuild user" gstbuild && \ - chmod 755 /gstbuild - -# Set the working directory to gstbuild home directory -WORKDIR /gstbuild - -# Specify the user to execute all commands below -USER gstbuild +ENV GST_BUILD_PATH="/gst-build/" +ENV GST_VALIDATE_OUTPUT="/validate-output" +ENV GST_VALIDATE_TESTSUITES="$GST_VALIDATE_OUTPUT/gst-integration-testsuites" +# FIXME Enabled gst-libav. +ENV GST_BUILD_MESON_ARGS="-Dpython=disabled -Dlibav=disabled -Dugly=enabled -Dbad=enabled -Ddevtools=enabled -Dges=enabled -Drtsp_server=enabled -Domx=disabled -Dvaapi=disabled -Dsharp=disabled" # Configure git for various usage -RUN git config --global user.email "gstbuild@gstreamer.net" && git config --global user.name "Gstbuild Runner" +RUN git config --global user.email "gst-build@gstreamer.net" && git config --global user.name "Gstbuild Runner" +# get gst-build and make all subprojects available +RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ + cd $GST_BUILD_PATH && \ + meson build/ $GST_BUILD_MESON_ARGS \ No newline at end of file From d9604ca8d21396e900cdcb3c9a36edb9d9fbd197 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 5 Nov 2018 20:52:37 +0200 Subject: [PATCH 062/605] Build the docker images and push them to the gitalb registry --- .gitlab-ci.yml | 53 +++++++++++++++++++++- docker/build-base-images/Dockerfile-fedora | 2 +- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3497e7d25c..d32d7de227 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,7 @@ stages: - test + - "build runtime" + - "build base" test manifest: image: "fedora" @@ -12,4 +14,53 @@ test manifest: artifacts: reports: junit: - - "junit.xml" \ No newline at end of file + - "junit.xml" + +.base: + image: "docker:stable" + services: + - docker:dind + variables: + # When using dind, it's wise to use the overlayfs driver for + # improved performance. + DOCKER_DRIVER: "overlay2" + script: + - export IMAGE="${CI_REGISTRY_IMAGE}/${ARCH}/${TAG}" + - export REF="${IMAGE}:${CI_COMMIT_SHA}" + - export LATEST="${IMAGE}:latest" + + - docker build --pull -f ${DOCKERFILE} -t ${REF} ${CONTEXT_DIR} + - docker image tag ${REF} ${LATEST} + + # Push the images + - | + if [ "$CI_COMMIT_REF_NAME" = "master" ]; then + docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + docker push ${REF} + docker push ${LATEST} + fi + only: + changes: + - ".gitlab-ci.yml" + - "docker/**" + - "docker/*/**" + + +fedora amd64 run: + stage: "build runtime" + variables: + ARCH: "amd64" + TAG: "fedora-runtime" + CONTEXT_DIR: "docker/runtime-images/" + DOCKERFILE: "docker/runtime-images/Dockerfile-fedora" + extends: .base + +fedora amd64 build: + stage: "build base" + variables: + ARCH: "amd64" + TAG: "fedora-build" + CONTEXT_DIR: "docker/build-base-images/" + DOCKERFILE: "docker/build-base-images/Dockerfile-fedora" + extends: .base + diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index 14faa1cf67..852080c42c 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -1,4 +1,4 @@ -FROM registry.freedesktop.org/thiblahute/gst-ci/fedora-runtime +FROM registry.freedesktop.org/thiblahute/gst-ci/amd64/fedora-runtime # Add rpmfusion repositories and install all build dependencies of GStreamer RUN dnf builddep -y \ From 17e6d0673538afe047e02294ec9e5842967a4cd6 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 5 Nov 2018 21:34:55 +0200 Subject: [PATCH 063/605] Docker Fedora: Add more dependencies Also make the indentation 4-space widgth and consinstant --- docker/build-base-images/Dockerfile-fedora | 85 +++++++++++++--------- docker/runtime-images/Dockerfile-fedora | 73 +++++++++++-------- 2 files changed, 95 insertions(+), 63 deletions(-) diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index 852080c42c..153469b098 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -1,43 +1,59 @@ FROM registry.freedesktop.org/thiblahute/gst-ci/amd64/fedora-runtime - # Add rpmfusion repositories and install all build dependencies of GStreamer RUN dnf builddep -y \ - gstreamer1 \ - gstreamer1-plugins-base \ - gstreamer1-plugins-good \ - gstreamer1-plugins-good-extras \ - gstreamer1-plugins-ugly \ - gstreamer1-plugins-ugly-free \ - gstreamer1-plugins-bad-nonfree \ - gstreamer1-plugins-bad-free \ - gstreamer1-plugins-bad-free-extras \ - gstreamer1-plugins-bad-freeworld \ - gstreamer1-libav \ - gstreamer1-rtsp-server \ - python3-gstreamer1 \ + gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-plugins-good \ + gstreamer1-plugins-good-extras \ + gstreamer1-plugins-ugly \ + gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-nonfree \ + gstreamer1-plugins-bad-free \ + gstreamer1-plugins-bad-free-extras \ + gstreamer1-plugins-bad-freeworld \ + gstreamer1-libav \ + gstreamer1-rtsp-server \ + python3-gstreamer1 \ && \ rpm -e \ - gstreamer1-devel \ - gstreamer1-plugins-base-devel \ + gstreamer1-devel \ + gstreamer1-plugins-base-devel \ && \ dnf install -y \ - ccache \ - gcc-c++ \ - gdb \ - git \ - json-glib-devel \ - libunwind-devel \ - opencv-devel \ - openjpeg2-devel \ - patch \ - redhat-rpm-config \ - xorg-x11-server-Xvfb \ - xorg-x11-server-utils \ - ninja-build \ - wget && \ - dnf clean all - -RUN pip3 install meson + ccache \ + gcc \ + gcc-c++ \ + gdb \ + git \ + ffmpeg \ + ffmpeg-libs \ + ffmpeg-devel \ + procps-ng \ + patch \ + redhat-rpm-config \ + json-glib \ + json-glib-devel \ + libnice \ + libnice-devel \ + libunwind \ + libunwind-devel \ + opencv \ + opencv-devel \ + openjpeg2 \ + openjpeg2-devel \ + x264 \ + x264-libs \ + x264-devel \ + pygobject3-devel \ + python3-gobject \ + python3-cairo \ + python3-cairo-devel \ + xorg-x11-server-utils \ + xorg-x11-server-Xvfb \ + ninja-build && \ + dnf clean all && \ + pip3 install meson && \ + export PATH="$PATH:/usr/local/lib/python3.6/site-packages" ENV GST_BUILD_PATH="/gst-build/" ENV GST_VALIDATE_OUTPUT="/validate-output" @@ -51,4 +67,5 @@ RUN git config --global user.email "gst-build@gstreamer.net" && git config --glo # get gst-build and make all subprojects available RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ cd $GST_BUILD_PATH && \ - meson build/ $GST_BUILD_MESON_ARGS \ No newline at end of file + meson build/ $GST_BUILD_MESON_ARGS && \ + rm -rf build/ \ No newline at end of file diff --git a/docker/runtime-images/Dockerfile-fedora b/docker/runtime-images/Dockerfile-fedora index c552c743d6..936ffb9514 100644 --- a/docker/runtime-images/Dockerfile-fedora +++ b/docker/runtime-images/Dockerfile-fedora @@ -1,5 +1,8 @@ FROM fedora:29 +# install the config manager +RUN dnf install -y dnf-plugins-core + # Add rpmfusion repositories and install all build dependencies of GStreamer # and then remove the gst packages themselves RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ @@ -9,37 +12,49 @@ RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-29 && \ dnf upgrade -y && \ dnf install -y \ - dnf-plugins-core \ - json-glib \ - libunwind \ - opencv \ - openjpeg2 \ - procps-ng \ + dnf-plugins-core \ + git \ + meson \ + json-glib \ + libnice \ + libunwind \ + openjpeg2 \ + procps-ng \ + x264 \ + x264-libs \ + python3-gobject \ + python3-cairo \ + ffmpeg \ + ffmpeg-libs \ + xorg-x11-server-utils \ + xorg-x11-server-Xvfb \ && \ dnf install -y \ - gstreamer1 \ - gstreamer1-plugins-base \ - gstreamer1-plugins-good \ - gstreamer1-plugins-good-extras \ - gstreamer1-plugins-ugly \ - gstreamer1-plugins-ugly-free \ - gstreamer1-plugins-bad-nonfree \ - gstreamer1-plugins-bad-free \ - gstreamer1-plugins-bad-free-extras \ - gstreamer1-plugins-bad-freeworld \ - gstreamer1-libav \ - gstreamer1-rtsp-server \ + gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-plugins-good \ + gstreamer1-plugins-good-extras \ + gstreamer1-plugins-ugly \ + gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-nonfree \ + gstreamer1-plugins-bad-free \ + gstreamer1-plugins-bad-free-extras \ + gstreamer1-plugins-bad-freeworld \ + gstreamer1-libav \ + gstreamer1-rtsp-server \ && \ rpm -e \ - gstreamer1-plugins-good \ - gstreamer1-plugins-good-extras \ - gstreamer1-plugins-ugly \ - gstreamer1-plugins-ugly-free \ - gstreamer1-plugins-bad-nonfree \ - gstreamer1-plugins-bad-free \ - gstreamer1-plugins-bad-free-extras \ - gstreamer1-plugins-bad-freeworld \ - gstreamer1-rtsp-server \ - gstreamer1-libav \ + gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-plugins-good \ + gstreamer1-plugins-good-extras \ + gstreamer1-plugins-ugly \ + gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-nonfree \ + gstreamer1-plugins-bad-free \ + gstreamer1-plugins-bad-free-extras \ + gstreamer1-plugins-bad-freeworld \ + gstreamer1-rtsp-server \ + gstreamer1-libav \ && \ - dnf clean all + dnf clean all \ No newline at end of file From dcbd3f91d4b8106e4512044f309d94bbc82f17e7 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 8 Nov 2018 12:28:45 +0200 Subject: [PATCH 064/605] Do not hardcode env variables --- docker/build-base-images/Dockerfile-fedora | 6 +----- docker/build-base-images/Dockerfile-ubuntu | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index 153469b098..80f3c01cfa 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -56,10 +56,6 @@ RUN dnf builddep -y \ export PATH="$PATH:/usr/local/lib/python3.6/site-packages" ENV GST_BUILD_PATH="/gst-build/" -ENV GST_VALIDATE_OUTPUT="/validate-output" -ENV GST_VALIDATE_TESTSUITES="$GST_VALIDATE_OUTPUT/gst-integration-testsuites" -# FIXME Enabled gst-libav. -ENV GST_BUILD_MESON_ARGS="-Dpython=disabled -Dlibav=disabled -Dugly=enabled -Dbad=enabled -Ddevtools=enabled -Dges=enabled -Drtsp_server=enabled -Domx=disabled -Dvaapi=disabled -Dsharp=disabled" # Configure git for various usage RUN git config --global user.email "gst-build@gstreamer.net" && git config --global user.name "Gstbuild Runner" @@ -67,5 +63,5 @@ RUN git config --global user.email "gst-build@gstreamer.net" && git config --glo # get gst-build and make all subprojects available RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ cd $GST_BUILD_PATH && \ - meson build/ $GST_BUILD_MESON_ARGS && \ + meson build/ && \ rm -rf build/ \ No newline at end of file diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu index cb879157b1..7b6cdab0ef 100644 --- a/docker/build-base-images/Dockerfile-ubuntu +++ b/docker/build-base-images/Dockerfile-ubuntu @@ -27,10 +27,6 @@ RUN \ RUN pip3 install meson ENV GST_BUILD_PATH="/gst-build/" -ENV GST_VALIDATE_OUTPUT="/validate-output" -ENV GST_VALIDATE_TESTSUITES="$GST_VALIDATE_OUTPUT/gst-integration-testsuites" -# FIXME Enabled gst-libav. -ENV GST_BUILD_MESON_ARGS="-Dpython=disabled -Dlibav=disabled -Dugly=enabled -Dbad=enabled -Ddevtools=enabled -Dges=enabled -Drtsp_server=enabled -Domx=disabled -Dvaapi=disabled -Dsharp=disabled" # Configure git for various usage RUN git config --global user.email "gst-build@gstreamer.net" && git config --global user.name "Gstbuild Runner" @@ -38,4 +34,4 @@ RUN git config --global user.email "gst-build@gstreamer.net" && git config --glo # get gst-build and make all subprojects available RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ cd $GST_BUILD_PATH && \ - meson build/ $GST_BUILD_MESON_ARGS \ No newline at end of file + meson build/ && rm -rf build/ \ No newline at end of file From 9e0abc3c3be3cf09d0f2272710439b41fbbafdaf Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 9 Nov 2018 15:05:38 +0200 Subject: [PATCH 065/605] docker: Use the upstream registry to build images --- docker/build-base-images/Dockerfile-fedora | 2 +- docker/build-base-images/Dockerfile-ubuntu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index 80f3c01cfa..7260991102 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -1,4 +1,4 @@ -FROM registry.freedesktop.org/thiblahute/gst-ci/amd64/fedora-runtime +FROM registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-runtime # Add rpmfusion repositories and install all build dependencies of GStreamer RUN dnf builddep -y \ gstreamer1 \ diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu index 7b6cdab0ef..3e2864c933 100644 --- a/docker/build-base-images/Dockerfile-ubuntu +++ b/docker/build-base-images/Dockerfile-ubuntu @@ -1,4 +1,4 @@ -FROM registry.freedesktop.org/thiblahute/gst-ci/ubuntu-runtime +FROM registry.freedesktop.org/gstreamer/gst-ci/ubuntu-runtime # make source packages available in order to figure out build dependencies RUN \ From 7a51b846f5d3efb13faff281c959525719ef7941 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Fri, 9 Nov 2018 12:09:42 +0100 Subject: [PATCH 066/605] build-manifest: Remove reference to anongit And make the gitlab one the "origin" --- gitlab/build_manifest.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 9b1e208404..18343d99a6 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -32,8 +32,7 @@ GSTREAMER_MODULES: List[Tuple[str, int]] = [ MANIFEST_TEMPLATE: str = """ - - + {} """ @@ -201,14 +200,14 @@ def find_repository_sha(module: Tuple[str, int], branchname: str) -> Tuple[str, if branch is not None: print("Found mathcing branch in upstream project") print(f"gstreamer/{branchname}") - return 'gstreamer', branch['commit']['id'] + return 'origin', branch['commit']['id'] # Fallback to using upstream master branch branch = get_project_branch(module[1], 'master') if branch is not None: print("Falling back to master branch on upstream project") print(f"gstreamer/master") - return 'gstreamer', branch['commit']['id'] + return 'origin', branch['commit']['id'] # This should never occur given the upstream fallback above print("If something reaches that point, please file a bug") @@ -229,11 +228,11 @@ def test_find_repository_sha(): # This should fallback to upstream master branch since no matching branch was found remote, git_ref = find_repository_sha(("gst-plugins-good", 1353), "totally-valid-branch-name") - assert remote == "gstreamer" + assert remote == "origin" # This should fallback to upstream master branch since no repository was found remote, git_ref = find_repository_sha(("totally-valid-project-name", 42), "1.2") - assert remote == "gstreamer" + assert remote == "origin" # This is now the sha of the last commit # assert git_ref == "master" From ece0ea97bffaa6a2bcf01c692abee7a171049606 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 8 Nov 2018 12:32:14 +0200 Subject: [PATCH 067/605] Add an initial template of a gitlab-ci config This is meant to be used with the `include:` featrue of gitlab [1] in order to provide the same CI pipeline for all the gstreamer modules. [1] https://docs.gitlab.com/ce/ci/yaml/README.html#include --- gitlab/ci_template.yml | 69 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 gitlab/ci_template.yml diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml new file mode 100644 index 0000000000..d618677373 --- /dev/null +++ b/gitlab/ci_template.yml @@ -0,0 +1,69 @@ +stages: + - "manifest" + - "build" + +variables: + GIT_STRATEGY: none + +manifest: + image: "fedora" + stage: "manifest" + script: + - dnf install -y python3-requests + - curl -o build_manifest.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/master/gitlab/build_manifest.py + - python3 build_manifest.py + - cat manifest.xml + artifacts: + expire_in: "2 days" + paths: + - "manifest.xml" + +.build: + stage: "build" + dependencies: + - "manifest" + variables: + CC: "ccache gcc" + CXX: "ccache g++" + CCACHE_BASEDIR: "${CI_PROJECT_DIR}" + CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" + MAIN_DIR: '${CI_PROJECT_DIR}' + MESON_ARGS: > + -Dpython=enabled + -Dlibav=enabled + -Dugly=enabled + -Dbad=enabled + -Ddevtools=enabled + -Dges=enabled + -Drtsp_server=enabled + -Dvaapi=disabled + -Dsharp=disabled + + script: + # Not sure why, but permission errors else + # https://gitlab.freedesktop.org/alatiera/gstreamer/-/jobs/41441 + - cp -r /gst-build/ . && cd gst-build + - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" + - meson build/ $MESON_ARGS + - ninja -C build/ + after_script: + - cd gst-build/ + # Clean the artifacts packages to avoid copying "useless" build products. + - rm $(find build -name "*.o") $(find -name "*.a") + # Clean the .git repos since we won't need them anymore + - rm -rf subprojects/*/.git/ + - rm -rf build/subprojects/*/.git/ + cache: + paths: + - "${CCACHE_DIR}" + artifacts: + expire_in: 6hours + when: always + paths: + - "manifest.xml" + - "gst-build/" + +build fedora x86_64: + extends: .build + image: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:9e0abc3c3be3cf09d0f2272710439b41fbbafdaf" + From 6ddb35c3c400586520b0f1e39f6f5369d0f61b44 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 9 Nov 2018 21:29:41 +0200 Subject: [PATCH 068/605] ci_template.yml: Temporarly disable builddir artifact We won't use it till the tests land, and the fd.o runners are getting overloading causing jobs to fail with 500 error during artifact upload. [1] Thus disable this feature temporarly. [1]: https://gitlab.freedesktop.org/freedesktop/freedesktop/issues/44 --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d618677373..64c2860e4e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -61,7 +61,7 @@ manifest: when: always paths: - "manifest.xml" - - "gst-build/" + # - "gst-build/" build fedora x86_64: extends: .build From 8b34f1e9f61e6a552a40342c2f7fabb379cb31c0 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 9 Nov 2018 15:57:45 -0300 Subject: [PATCH 069/605] Also run the main CI template in .gitlab-ci.yml Rename some job otherwise some names where too similare. --- .gitlab-ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d32d7de227..51d1165c5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,8 +2,12 @@ stages: - test - "build runtime" - "build base" + - "manifest" + - "build" test manifest: + variables: + GIT_STRATEGY: fetch image: "fedora" stage: "test" before_script: @@ -24,6 +28,7 @@ test manifest: # When using dind, it's wise to use the overlayfs driver for # improved performance. DOCKER_DRIVER: "overlay2" + GIT_STRATEGY: fetch script: - export IMAGE="${CI_REGISTRY_IMAGE}/${ARCH}/${TAG}" - export REF="${IMAGE}:${CI_COMMIT_SHA}" @@ -46,7 +51,7 @@ test manifest: - "docker/*/**" -fedora amd64 run: +fedora amd64 run docker: stage: "build runtime" variables: ARCH: "amd64" @@ -55,7 +60,7 @@ fedora amd64 run: DOCKERFILE: "docker/runtime-images/Dockerfile-fedora" extends: .base -fedora amd64 build: +fedora amd64 build docker: stage: "build base" variables: ARCH: "amd64" @@ -64,3 +69,4 @@ fedora amd64 build: DOCKERFILE: "docker/build-base-images/Dockerfile-fedora" extends: .base +include: "gitlab/ci_template.yml" From ba98b4138a014feee2fcaa421e66b6fa68409dd9 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sat, 10 Nov 2018 20:34:21 -0500 Subject: [PATCH 070/605] gitlab-ci: Use quote for "test" stage This is just for consistency, no really needed. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51d1165c5b..926232d514 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ stages: - - test + - "test" - "build runtime" - "build base" - "manifest" From 64e3b154cdbf4f0213c733fb51d81abeda99ea09 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sat, 10 Nov 2018 20:35:10 -0500 Subject: [PATCH 071/605] gitlab-ci: Only run scripts test on related code Right now we only have one test against build_manifest.py, it is not needed to run this test if that code haven't changed. It's really easy to add more file or set a wild card in the future. --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 926232d514..a3a657f478 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,9 @@ test manifest: - dnf install -y python3-pytest python3-pytest-cov python3-requests script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py + only: + changes: + - "gitlab/build_manifest.py" coverage: '/TOTAL.*\s+(\d+%)$/' artifacts: reports: From 345afee677770f4de89b1c4b01bc3021500d335c Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 12 Nov 2018 14:37:16 +0200 Subject: [PATCH 072/605] CI: Overwrite the build job from the template Make it so when we build a new image in `gst-ci` it gets picked up automatically by the job in this repo. For the template itself we want the images to me be versioned and reproducible. --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a3a657f478..a3ab7814d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,3 +73,10 @@ fedora amd64 build docker: extends: .base include: "gitlab/ci_template.yml" + +# Overwrite the job with the same name that comes from the include template, +# in order to use the `:latest` tag of the image built +build fedora x86_64: + # This comes from the `include:` above + extends: .build + image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" \ No newline at end of file From 8fbce56d694be9eaf5700af627d8b89576cc98ac Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 12 Nov 2018 15:22:18 +0200 Subject: [PATCH 073/605] CI: Change the push policy of the docker images If we are in a fork of the project, we would like to be able to overwrite the `:latest` tag if the registry from any branch so we won't have to manually overwrite the image tag in the build/test jobs in the .gitlab-ci.yml file --- .gitlab-ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a3ab7814d1..ba7b3cba1a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,9 +40,17 @@ test manifest: - docker build --pull -f ${DOCKERFILE} -t ${REF} ${CONTEXT_DIR} - docker image tag ${REF} ${LATEST} - # Push the images + # If we are in a fork, push the image to the reigstry regardless the branch - | - if [ "$CI_COMMIT_REF_NAME" = "master" ]; then + if [ "$CI_PROJECT_NAMESPACE" != "gstreamer" ]; then + docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + docker push ${REF} + docker push ${LATEST} + fi + + # Push the images to the upstream registry + - | + if [ "$CI_COMMIT_REF_NAME" = "master" && "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY docker push ${REF} docker push ${LATEST} From b335b4416b2a32278552dea4980af3ec41ab3b7f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 12 Nov 2018 16:37:10 +0200 Subject: [PATCH 074/605] build_manifest.py: Silence irrelevant logs The group API querries where removed in 2320cfaf but the log was forgotten. The headers print was meant to be just for debugging. --- gitlab/build_manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 18343d99a6..21d10d489e 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -72,7 +72,7 @@ def request_raw(path: str, headers: Dict[str, str], project_url: str) -> List[Di base_url: str = urlparse(project_url).hostname url: str = f"https://{base_url}/api/v4/{path}" print(f"GET {url}") - print(f"Headers: {headers}") + # print(f"Headers: {headers}") resp = requests.get(url, headers=headers) print(f"Request returned: {resp.status_code}") @@ -94,7 +94,7 @@ def request(path: str) -> List[Dict[str, str]]: try: headers: Dict[str, str] = {'Private-Token': os.environ["READ_PROJECTS_TOKEN"] } except KeyError: - print("Custom token was not set, group api querries will fail") + # print("Custom token was not set, group api querries will fail") # JOB_TOKEN is the default placeholder of CI_JOB_TOKEN headers: Dict[str, str] = {'JOB_TOKEN': "xxxxxxxxxxxxxxxxxxxx" } From 2ec33592e98f257195ddea5f68d9c43a3c25df09 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 12 Nov 2018 18:01:00 +0200 Subject: [PATCH 075/605] CI: Use the correct bash syntax 345afee677770f4de89b1c4b01bc3021500d335c broke the bash extreppsion that evaluates the strings, causing images to never be pushed. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba7b3cba1a..d64d98a5c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ test manifest: # Push the images to the upstream registry - | - if [ "$CI_COMMIT_REF_NAME" = "master" && "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then + if [ "$CI_COMMIT_REF_NAME" = "master" -a "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY docker push ${REF} docker push ${LATEST} From 8cb78a4c383f6b23a8928bdd903c0ebfa14bc0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Mon, 12 Nov 2018 13:52:31 +0100 Subject: [PATCH 076/605] docker: Add gstreamer-vaapi dependencies --- docker/build-base-images/Dockerfile-fedora | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index 7260991102..6b462a9871 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -13,11 +13,13 @@ RUN dnf builddep -y \ gstreamer1-plugins-bad-freeworld \ gstreamer1-libav \ gstreamer1-rtsp-server \ + gstreamer1-vaapi \ python3-gstreamer1 \ && \ rpm -e \ gstreamer1-devel \ gstreamer1-plugins-base-devel \ + gstreamer1-plugins-bad-free-devel \ && \ dnf install -y \ ccache \ @@ -64,4 +66,4 @@ RUN git config --global user.email "gst-build@gstreamer.net" && git config --glo RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ cd $GST_BUILD_PATH && \ meson build/ && \ - rm -rf build/ \ No newline at end of file + rm -rf build/ From d1b80b835b3f198d501f6187333dacaa8f8fd4b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Mon, 12 Nov 2018 13:59:57 +0100 Subject: [PATCH 077/605] build_manifest.py: Add gstreamer-vaapi --- gitlab/build_manifest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 21d10d489e..14884fca74 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -27,6 +27,7 @@ GSTREAMER_MODULES: List[Tuple[str, int]] = [ ('gst-omx', 1350), ('gst-python', 1355), ('gst-rtsp-server', 1362), + ('gstreamer-vaapi', 1359), ] MANIFEST_TEMPLATE: str = """ From 1858a709e6d400716241c9b63dc50b28fa71c48a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Mon, 12 Nov 2018 14:00:18 +0100 Subject: [PATCH 078/605] ci_template.yml: enable gstreamer-vaapi --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 64c2860e4e..0c61dff418 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -36,7 +36,7 @@ manifest: -Ddevtools=enabled -Dges=enabled -Drtsp_server=enabled - -Dvaapi=disabled + -Dvaapi=enabled -Dsharp=disabled script: From e3d2c345b6338040817652aaca37b34763515a5d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 12 Nov 2018 18:57:42 +0200 Subject: [PATCH 079/605] ci_template: Update docker image tags --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0c61dff418..9118ad2924 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -65,5 +65,5 @@ manifest: build fedora x86_64: extends: .build - image: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:9e0abc3c3be3cf09d0f2272710439b41fbbafdaf" + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:1858a709e6d400716241c9b63dc50b28fa71c48a' From 48f3063dafb299066351eec118f2e7887d909364 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 13 Nov 2018 12:52:17 +0200 Subject: [PATCH 080/605] build_manifest.py: Use CI_PROJECT_NAMESPACE instead of GITLAB_USER GITLAB_USER_* variables represent the user *that triggered* the job, which might not match the user that owns the namespace of the fork. This happens frequently with rebase, where the one who triggers the rebase is the one that appears in the GITLAB_USER_* vars. Instead use the CI_PROJECT_NAMESPACE variable to search the user's projects. If CI_PROJECT_NAMESPACE has the value of a Group namespace search_user_namespace returns 404. --- gitlab/build_manifest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 14884fca74..669c4ddee8 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -45,7 +45,7 @@ def preserve_ci_vars(func): def wrapper(): try: url = os.environ["CI_PROJECT_URL"] - user = os.environ["GITLAB_USER_LOGIN"] + user = os.environ["CI_PROJECT_NAMESPACE"] except KeyError: url = "invalid" user = "" @@ -57,7 +57,7 @@ def preserve_ci_vars(func): func() os.environ["CI_PROJECT_URL"] = url - os.environ["GITLAB_USER_LOGIN"] = user + os.environ["CI_PROJECT_NAMESPACE"] = user if private: os.environ["READ_PROJECTS_TOKEN"] = private @@ -177,10 +177,14 @@ def test_search_user_namespace(): res = search_user_namespace("alatiera", "404-project-not-found") assert res is None + # Passing a group namespace instead of user should return None + res = search_user_namespace("gstreamer", "gst-plugins-good") + assert res is None + def find_repository_sha(module: Tuple[str, int], branchname: str) -> Tuple[str, str]: - user_login: str = os.environ["GITLAB_USER_LOGIN"] - project = search_user_namespace(user_login, module[0]) + namespace: str = os.environ["CI_PROJECT_NAMESPACE"] + project = search_user_namespace(namespace, module[0]) # Find a fork in the User's namespace if project: @@ -219,7 +223,7 @@ def find_repository_sha(module: Tuple[str, int], branchname: str) -> Tuple[str, @preserve_ci_vars def test_find_repository_sha(): os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" - os.environ["GITLAB_USER_LOGIN"] = "alatiera" + os.environ["CI_PROJECT_NAMESPACE"] = "alatiera" del os.environ["READ_PROJECTS_TOKEN"] # This should find the repository in the user namespace From 529764a3d7c784a86ae69104fe09990f12599e37 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 11 Nov 2018 09:22:42 -0300 Subject: [PATCH 081/605] gitlab: ci: Run build_manifest tests only if the script changes --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d64d98a5c4..01d96bf04f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,6 +22,9 @@ test manifest: reports: junit: - "junit.xml" + only: + changes: + - gitlab/build_manifest.py .base: image: "docker:stable" From 7ede1f54aabdd5c6c55424bb3c5d6c0ab8641b7c Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sat, 10 Nov 2018 23:49:57 -0300 Subject: [PATCH 082/605] Add a simple alpine based Dockerfile for images to build manifest --- .gitlab-ci.yml | 8 ++++++++ docker/runtime-images/Dockerfile-manifest-builder | 5 +++++ 2 files changed, 13 insertions(+) create mode 100644 docker/runtime-images/Dockerfile-manifest-builder diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01d96bf04f..29f1449af7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,6 +64,14 @@ test manifest: - "docker/**" - "docker/*/**" +alpine amd64 manifest builder docker: + stage: "build runtime" + variables: + ARCH: "amd64" + TAG: "alpine-manifest-build" + CONTEXT_DIR: "docker/runtime-images/" + DOCKERFILE: "docker/runtime-images/Dockerfile-manifest-builder" + extends: .base fedora amd64 run docker: stage: "build runtime" diff --git a/docker/runtime-images/Dockerfile-manifest-builder b/docker/runtime-images/Dockerfile-manifest-builder new file mode 100644 index 0000000000..a0b85bb35e --- /dev/null +++ b/docker/runtime-images/Dockerfile-manifest-builder @@ -0,0 +1,5 @@ +FROM python:3.7.1-alpine + +RUN pip install requests +RUN apk add git +RUN cd / && git clone https://gitlab.freedesktop.org/gstreamer/gst-ci.git \ No newline at end of file From 0cfe2cee41c7bf7186953a8a603d956af1d8ebb1 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 11 Nov 2018 09:33:13 -0300 Subject: [PATCH 083/605] gitlab: Handle checking out the right gst-ci branch to call build_manifest --- gitlab/build_manifest.py | 26 ++++++++++++++++++++++---- gitlab/ci_template.yml | 10 +++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 669c4ddee8..a0bac12398 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import argparse import os import requests import sys +import subprocess from typing import Dict, Tuple, List from urllib.parse import urlparse @@ -243,17 +245,33 @@ def test_find_repository_sha(): if __name__ == "__main__": - projects: str = '' - project_template: str = " \n" + parser = argparse.ArgumentParser() + parser.add_argument("--self-update", action="store_true", default=False) + parser.add_argument(dest="output", default='manifest.xml', nargs='?') + options = parser.parse_args() + + current_branch: str = os.environ['CI_COMMIT_REF_NAME'] user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL']) if not user_remote_url.endswith('/'): user_remote_url += '/' + if options.self_update: + remote, sha = find_repository_sha(("gst-ci", "1343"), current_branch) + if remote == 'user': + remote = user_remote_url + 'gst-ci' + else: + remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci" + + subprocess.check_call(['git', 'fetch', remote, sha]) + subprocess.check_call(['git', 'checkout', '--detach', 'FETCH_HEAD']) + sys.exit(0) + + projects: str = '' + project_template: str = " \n" for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') - current_branch: str = os.environ['CI_COMMIT_REF_NAME'] remote, revision = find_repository_sha(module, current_branch) projects += project_template.format(module[0], remote, revision) - with open('manifest.xml', mode='w') as manifest: + with open(options.output, mode='w') as manifest: print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9118ad2924..6556f47829 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -6,13 +6,13 @@ variables: GIT_STRATEGY: none manifest: - image: "fedora" + image: registry.freedesktop.org/gstreamer/gst-ci/amd64/alpine-manifest-build:latest stage: "manifest" script: - - dnf install -y python3-requests - - curl -o build_manifest.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/master/gitlab/build_manifest.py - - python3 build_manifest.py - - cat manifest.xml + - cd /gst-ci + - gitlab/build_manifest.py --self-update + - gitlab/build_manifest.py ${CI_PROJECT_DIR}/manifest.xml + - cat ${CI_PROJECT_DIR}/manifest.xml artifacts: expire_in: "2 days" paths: From d02fd933d301dfb37acbb2256122650a9848fe1e Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 11 Nov 2018 08:57:35 -0300 Subject: [PATCH 084/605] Revert me when merged Stopping the circular dependency. --- gitlab/ci_template.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6556f47829..d9e616f7b6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -6,10 +6,11 @@ variables: GIT_STRATEGY: none manifest: - image: registry.freedesktop.org/gstreamer/gst-ci/amd64/alpine-manifest-build:latest + image: registry.freedesktop.org/thiblahute/gst-ci/amd64/alpine-manifest-build:latest stage: "manifest" script: - cd /gst-ci + - git fetch https://gitlab.freedesktop.org/thiblahute/gst-ci.git simple_manifest && git checkout FETCH_HEAD # REMOVEME! - gitlab/build_manifest.py --self-update - gitlab/build_manifest.py ${CI_PROJECT_DIR}/manifest.xml - cat ${CI_PROJECT_DIR}/manifest.xml From 12d067f249f472d8e4dacaa2512538d841850a61 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 13 Nov 2018 14:13:23 -0300 Subject: [PATCH 085/605] Revert "Revert me when merged" This reverts commit 5cff46553e913e5ab13e63561edaa519a4ca904f. --- gitlab/ci_template.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d9e616f7b6..6556f47829 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -6,11 +6,10 @@ variables: GIT_STRATEGY: none manifest: - image: registry.freedesktop.org/thiblahute/gst-ci/amd64/alpine-manifest-build:latest + image: registry.freedesktop.org/gstreamer/gst-ci/amd64/alpine-manifest-build:latest stage: "manifest" script: - cd /gst-ci - - git fetch https://gitlab.freedesktop.org/thiblahute/gst-ci.git simple_manifest && git checkout FETCH_HEAD # REMOVEME! - gitlab/build_manifest.py --self-update - gitlab/build_manifest.py ${CI_PROJECT_DIR}/manifest.xml - cat ${CI_PROJECT_DIR}/manifest.xml From d8ebbea1f0f3f5cb974b854bbc4f37c0dce9a2ee Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 13 Nov 2018 23:34:20 -0300 Subject: [PATCH 086/605] build_manifest: Make sure to respect GitLab currently built SHA --- gitlab/build_manifest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index a0bac12398..01e1b7d473 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -186,8 +186,11 @@ def test_search_user_namespace(): def find_repository_sha(module: Tuple[str, int], branchname: str) -> Tuple[str, str]: namespace: str = os.environ["CI_PROJECT_NAMESPACE"] - project = search_user_namespace(namespace, module[0]) + if module[0] == os.environ['CI_PROJECT_NAME']: + return 'user', os.environ['CI_COMMIT_SHA'] + + project = search_user_namespace(namespace, module[0]) # Find a fork in the User's namespace if project: id = project['id'] @@ -243,6 +246,13 @@ def test_find_repository_sha(): # This is now the sha of the last commit # assert git_ref == "master" + os.environ["CI_PROJECT_NAME"] = "the_project" + os.environ["CI_COMMIT_SHA"] = "MySha" + + remote, git_ref = find_repository_sha(("the_project", 199), "whatever") + assert remote == "user" + assert git_ref == "MySha" + if __name__ == "__main__": parser = argparse.ArgumentParser() From 10ae6d0326e9d3198e479e4a7444ce7c4b158ddd Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 14 Nov 2018 10:16:39 -0300 Subject: [PATCH 087/605] build_manifest: Never use users 'master' branch for repos we are not explicitely building People very often do not update them leading to random and weird failures --- gitlab/build_manifest.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 01e1b7d473..dad5263af7 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -190,20 +190,21 @@ def find_repository_sha(module: Tuple[str, int], branchname: str) -> Tuple[str, if module[0] == os.environ['CI_PROJECT_NAME']: return 'user', os.environ['CI_COMMIT_SHA'] - project = search_user_namespace(namespace, module[0]) - # Find a fork in the User's namespace - if project: - id = project['id'] - print(f"User project found, id: {id}") - # If we have a branch with same name, use it. - branch = get_project_branch(id, branchname) - if branch is not None: - path = project['namespace']['path'] - print("Found mathcing branch in user's namespace") - print(f"{path}/{branchname}") + if branchname != "master": + project = search_user_namespace(namespace, module[0]) + # Find a fork in the User's namespace + if project: + id = project['id'] + print(f"User project found, id: {id}") + # If we have a branch with same name, use it. + branch = get_project_branch(id, branchname) + if branch is not None: + path = project['namespace']['path'] + print("Found matching branch in user's namespace") + print(f"{path}/{branchname}") - return 'user', branch['commit']['id'] - print(f"Did not found user branch named {branchname}") + return 'user', branch['commit']['id'] + print(f"Did not find user branch named {branchname}") # Check upstream project for a branch branch = get_project_branch(module[1], branchname) From 10ffa6f1af6fed20a8857d2c7426d81cf162e3d1 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 14 Nov 2018 10:20:16 -0300 Subject: [PATCH 088/605] build_manifest: Add the to repo names to avoid spamming about redirections --- gitlab/build_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index dad5263af7..6aab445240 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -278,7 +278,7 @@ if __name__ == "__main__": sys.exit(0) projects: str = '' - project_template: str = " \n" + project_template: str = " \n" for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') remote, revision = find_repository_sha(module, current_branch) From 2414895a53408ed8a0410a52560f8090b3f74696 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 14 Nov 2018 10:20:16 -0300 Subject: [PATCH 089/605] build_manifest: Add the "path" attribute to projects When the local path and the name are different, both have to be specified, it is now required since we add the '.git' to the name. --- gitlab/build_manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 6aab445240..9ad506f0fd 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -278,11 +278,11 @@ if __name__ == "__main__": sys.exit(0) projects: str = '' - project_template: str = " \n" + project_template: str = " \n" for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') remote, revision = find_repository_sha(module, current_branch) - projects += project_template.format(module[0], remote, revision) + projects += project_template % {'name': module[0], 'remote': remote, 'revision': revision} with open(options.output, mode='w') as manifest: print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest) From 01beb70022af7fcce45636a4c84618fa40632b17 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 21 Nov 2018 21:25:30 -0300 Subject: [PATCH 090/605] gitlab: Update the docker image we use so we have latest fix for gst-build git-update script --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6556f47829..e31b632c7b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -65,5 +65,5 @@ manifest: build fedora x86_64: extends: .build - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:1858a709e6d400716241c9b63dc50b28fa71c48a' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' From c930a3a22990eb0e3051f48ebc1b3a74ad91972b Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 21 Nov 2018 15:47:18 +0200 Subject: [PATCH 091/605] gitlab/ci_template.yml: Use correct yaml syntax single quotes in yaml don't escape special charters which causes the env variable to never be resolved to its proper path. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e31b632c7b..4659623306 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -27,7 +27,7 @@ manifest: CXX: "ccache g++" CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" - MAIN_DIR: '${CI_PROJECT_DIR}' + MAIN_DIR: "${CI_PROJECT_DIR}" MESON_ARGS: > -Dpython=enabled -Dlibav=enabled From bf92771e458adbe72435643fa662fb3e73034b03 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 21 Nov 2018 17:25:29 +0200 Subject: [PATCH 092/605] gitlab/ci_template.yml: Use tagged docker image for the manifest job --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4659623306..1bfe6ba263 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -6,8 +6,8 @@ variables: GIT_STRATEGY: none manifest: - image: registry.freedesktop.org/gstreamer/gst-ci/amd64/alpine-manifest-build:latest - stage: "manifest" + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/alpine-manifest-build:10ffa6f1af6fed20a8857d2c7426d81cf162e3d1' + stage: 'manifest' script: - cd /gst-ci - gitlab/build_manifest.py --self-update From cc5ff7d8f001740b9fc092a13ca06fff31961077 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 21 Nov 2018 16:55:02 +0200 Subject: [PATCH 093/605] CI: Test both the ci_template as well as local changes Instead of just overwritting the template build jobs, test both so we avoid regressions. See [1] for more [1] https://gitlab.freedesktop.org/gstreamer/gst-ci/commit/1858a709e6d400716241c9b63dc50b28fa71c48a --- .gitlab-ci.yml | 12 ++++++------ gitlab/ci_template.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29f1449af7..87e9a2d46b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,9 +93,9 @@ fedora amd64 build docker: include: "gitlab/ci_template.yml" -# Overwrite the job with the same name that comes from the include template, -# in order to use the `:latest` tag of the image built -build fedora x86_64: - # This comes from the `include:` above - extends: .build - image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" \ No newline at end of file +# Test the build job against the latest build image tag and the local manifest +build fedora x86_64 local: + extends: '.build' + image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" + dependencies: + - 'manifest' \ No newline at end of file diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1bfe6ba263..91205a5f2e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -20,8 +20,6 @@ manifest: .build: stage: "build" - dependencies: - - "manifest" variables: CC: "ccache gcc" CXX: "ccache g++" @@ -64,6 +62,8 @@ manifest: # - "gst-build/" build fedora x86_64: - extends: .build + extends: '.build' image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' + dependencies: + - "manifest" From 9b45a95fe0115f9a3d2a0243a45aab7707840fbf Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 22 Nov 2018 13:01:48 +0200 Subject: [PATCH 094/605] CI: only build the images that changed previously when something in the docker/ direcotry was changed, all of the images would be rebuilt. --- .gitlab-ci.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87e9a2d46b..506f6718de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,11 +58,6 @@ test manifest: docker push ${REF} docker push ${LATEST} fi - only: - changes: - - ".gitlab-ci.yml" - - "docker/**" - - "docker/*/**" alpine amd64 manifest builder docker: stage: "build runtime" @@ -72,6 +67,10 @@ alpine amd64 manifest builder docker: CONTEXT_DIR: "docker/runtime-images/" DOCKERFILE: "docker/runtime-images/Dockerfile-manifest-builder" extends: .base + only: + changes: + - 'docker/runtime-images/Dockerfile-manifest-builder' + - 'gitlab/build_manifest.py' fedora amd64 run docker: stage: "build runtime" @@ -81,6 +80,9 @@ fedora amd64 run docker: CONTEXT_DIR: "docker/runtime-images/" DOCKERFILE: "docker/runtime-images/Dockerfile-fedora" extends: .base + only: + changes: + - 'docker/runtime-images/Dockerfile-fedora' fedora amd64 build docker: stage: "build base" @@ -90,6 +92,9 @@ fedora amd64 build docker: CONTEXT_DIR: "docker/build-base-images/" DOCKERFILE: "docker/build-base-images/Dockerfile-fedora" extends: .base + only: + changes: + - 'docker/build-base-images/Dockerfile-fedora' include: "gitlab/ci_template.yml" From 03a1e07fa3447b1213de0923ce7e6e1c3539b362 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 22 Nov 2018 13:03:23 +0200 Subject: [PATCH 095/605] CI: remove duplicate entry --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 506f6718de..58d06fe725 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,9 +14,6 @@ test manifest: - dnf install -y python3-pytest python3-pytest-cov python3-requests script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py - only: - changes: - - "gitlab/build_manifest.py" coverage: '/TOTAL.*\s+(\d+%)$/' artifacts: reports: @@ -24,7 +21,7 @@ test manifest: - "junit.xml" only: changes: - - gitlab/build_manifest.py + - 'gitlab/build_manifest.py' .base: image: "docker:stable" From 71b1b352eea12f04f49da95f4f4c62adb5cf6f52 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 22 Nov 2018 11:37:51 -0500 Subject: [PATCH 096/605] Build ubuntu runtime docker --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58d06fe725..928db3596d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,6 +93,18 @@ fedora amd64 build docker: changes: - 'docker/build-base-images/Dockerfile-fedora' +ubuntu amd64 run docker: + stage: "build runtime" + variables: + ARCH: "amd64" + TAG: "ubuntu-runtime" + CONTEXT_DIR: "docker/runtime-images/" + DOCKERFILE: "docker/runtime-images/Dockerfile-ubuntu" + extends: .base + only: + changes: + - 'docker/runtime-images/Dockerfile-ubuntu' + include: "gitlab/ci_template.yml" # Test the build job against the latest build image tag and the local manifest From af66e26160b104982c5988ea0d7b005fe7d41de0 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 22 Nov 2018 12:00:26 -0500 Subject: [PATCH 097/605] Make a change in Dockerfile-ubuntu to force rebuild of docker image --- docker/runtime-images/Dockerfile-ubuntu | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/runtime-images/Dockerfile-ubuntu b/docker/runtime-images/Dockerfile-ubuntu index 8856d7ca23..6d1809f7a5 100644 --- a/docker/runtime-images/Dockerfile-ubuntu +++ b/docker/runtime-images/Dockerfile-ubuntu @@ -3,7 +3,6 @@ FROM ubuntu:18.04 ENV DEBIAN_FRONTEND=noninteractive # upgrade and install all runtime dependencies for gstreamer - RUN apt-get update && \ apt-get -y upgrade && \ apt-get -y install --no-install-recommends\ From 68a71431b8adc4443863f252ee6618c3e2f3378f Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 22 Nov 2018 13:24:26 -0500 Subject: [PATCH 098/605] Build Ubuntu base image --- .gitlab-ci.yml | 22 +++++++++++++++++++++- docker/build-base-images/Dockerfile-ubuntu | 2 +- gitlab/ci_template.yml | 21 +++++++++++---------- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 928db3596d..b1b0930e3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -105,6 +105,18 @@ ubuntu amd64 run docker: changes: - 'docker/runtime-images/Dockerfile-ubuntu' +ubuntu amd64 build docker: + stage: "build base" + variables: + ARCH: "amd64" + TAG: "ubuntu-build" + CONTEXT_DIR: "docker/build-base-images/" + DOCKERFILE: "docker/build-base-images/Dockerfile-ubuntu" + extends: .base + only: + changes: + - 'docker/build-base-images/Dockerfile-ubuntu' + include: "gitlab/ci_template.yml" # Test the build job against the latest build image tag and the local manifest @@ -112,4 +124,12 @@ build fedora x86_64 local: extends: '.build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" dependencies: - - 'manifest' \ No newline at end of file + - 'manifest' + +build ubuntu x86_64 local: + extends: '.build' + variables: + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dlibav=disabled" + image: "${CI_REGISTRY_IMAGE}/amd64/ubuntu-build:latest" + dependencies: + - 'manifest' diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu index 3e2864c933..0b9671a56f 100644 --- a/docker/build-base-images/Dockerfile-ubuntu +++ b/docker/build-base-images/Dockerfile-ubuntu @@ -1,4 +1,4 @@ -FROM registry.freedesktop.org/gstreamer/gst-ci/ubuntu-runtime +FROM registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu-runtime # make source packages available in order to figure out build dependencies RUN \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 91205a5f2e..6db97bd292 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -4,6 +4,16 @@ stages: variables: GIT_STRATEGY: none + DEFAULT_MESON_ARGS: > + -Dpython=enabled + -Dlibav=enabled + -Dugly=enabled + -Dbad=enabled + -Ddevtools=enabled + -Dges=enabled + -Drtsp_server=enabled + -Dvaapi=enabled + -Dsharp=disabled manifest: image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/alpine-manifest-build:10ffa6f1af6fed20a8857d2c7426d81cf162e3d1' @@ -26,16 +36,7 @@ manifest: CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" MAIN_DIR: "${CI_PROJECT_DIR}" - MESON_ARGS: > - -Dpython=enabled - -Dlibav=enabled - -Dugly=enabled - -Dbad=enabled - -Ddevtools=enabled - -Dges=enabled - -Drtsp_server=enabled - -Dvaapi=enabled - -Dsharp=disabled + MESON_ARGS: "${DEFAULT_MESON_ARGS}" script: # Not sure why, but permission errors else From a1726fee5841b776fcdaeda2e0ad628ce037a3c6 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 22 Nov 2018 13:45:10 -0500 Subject: [PATCH 099/605] Add Android NDK toolchain in Ubuntu docker image --- docker/build-base-images/Dockerfile-ubuntu | 10 +++- .../android-create-toolchain.sh | 49 +++++++++++++++++++ .../build-base-images/android-download-ndk.sh | 31 ++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100755 docker/build-base-images/android-create-toolchain.sh create mode 100755 docker/build-base-images/android-download-ndk.sh diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu index 0b9671a56f..d2dfd72003 100644 --- a/docker/build-base-images/Dockerfile-ubuntu +++ b/docker/build-base-images/Dockerfile-ubuntu @@ -22,7 +22,8 @@ RUN \ git \ xvfb \ python3-pip \ - wget + wget \ + unzip RUN pip3 install meson @@ -31,6 +32,13 @@ ENV GST_BUILD_PATH="/gst-build/" # Configure git for various usage RUN git config --global user.email "gst-build@gstreamer.net" && git config --global user.name "Gstbuild Runner" +# Setup Android toolchain +COPY android-download-ndk.sh . +COPY android-create-toolchain.sh . +RUN ./android-download-ndk.sh +RUN ./android-create-toolchain.sh arm64 28 +RUN rm -rf /android-ndk + # get gst-build and make all subprojects available RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ cd $GST_BUILD_PATH && \ diff --git a/docker/build-base-images/android-create-toolchain.sh b/docker/build-base-images/android-create-toolchain.sh new file mode 100755 index 0000000000..bf909b64e5 --- /dev/null +++ b/docker/build-base-images/android-create-toolchain.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# +# Copyright 2018 Collabora ltd. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# +# Author: Xavier Claessens +# + +set -e + +arch=$1 +api=$2 +toolchain_path=/android-toolchain-$arch-$api + +# Create standalone toolchains +/android-ndk/build/tools/make_standalone_toolchain.py --arch $arch --api $api --install-dir $toolchain_path + +# Create a cross file that can be passed to meson +cat > android_${arch}_${api}.txt <<- EOM +[host_machine] +system = 'android' +cpu_family = 'aarch64' +cpu = '${arch}' +endian = 'little' + +[properties] +c_args = ['--sysroot', '${toolchain_path}/sysroot'] +cpp_args = ['--sysroot', '${toolchain_path}/sysroot'] + +[binaries] +c = '${toolchain_path}/bin/aarch64-linux-android-clang' +cpp = '${toolchain_path}/bin/aarch64-linux-android-clang++' +ar = '${toolchain_path}/bin/aarch64-linux-android-ar' +strip = '${toolchain_path}/bin/aarch64-linux-android-strip' +pkgconfig = 'false' +EOM diff --git a/docker/build-base-images/android-download-ndk.sh b/docker/build-base-images/android-download-ndk.sh new file mode 100755 index 0000000000..0e07407422 --- /dev/null +++ b/docker/build-base-images/android-download-ndk.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# +# Copyright 2018 Collabora ltd. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# +# Author: Xavier Claessens +# + +set -e + +# Download Android NDK +ANDROID_NDK_VERSION="r18b" +ANDROID_NDK_SHA512="a35ab95ece52819194a3874fd210abe5c25905212c4aafe5d75c465c14739a46340d1ff0944ad93ffbbc9c0d86107119399d4f60ec6c5f080758008e75c19617" +wget --quiet https://dl.google.com/android/repository/android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip +echo "$ANDROID_NDK_SHA512 android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip" | sha512sum -c +unzip android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip +rm android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip +mv android-ndk-$ANDROID_NDK_VERSION /android-ndk From 1f326e2693046252959d9abb2be80ece175e8691 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 22 Nov 2018 14:21:20 -0500 Subject: [PATCH 100/605] Add Android arm64 cross build --- gitlab/ci_template.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6db97bd292..ddf63aed9d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -68,3 +68,13 @@ build fedora x86_64: dependencies: - "manifest" +build android arm64 api28: + extends: '.build' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu-build:a1726fee5841b776fcdaeda2e0ad628ce037a3c6' + variables: + MESON_ARGS: > + -Dbad=enabled + -Dbad:androidmedia=enabled + --cross-file /android_arm64_28.txt + dependencies: + - "manifest" From 6bdd1ead99e7d17ba5f8cdbba2668cdebc3bdb43 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 22 Nov 2018 16:47:35 -0500 Subject: [PATCH 101/605] Avoid duplication between ci_template.yml and .gitlab-ci.yml --- .gitlab-ci.yml | 12 +++--------- gitlab/ci_template.yml | 14 ++++++++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1b0930e3f..d25b68964e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,15 +121,9 @@ include: "gitlab/ci_template.yml" # Test the build job against the latest build image tag and the local manifest build fedora x86_64 local: - extends: '.build' + extends: '.build fedora x86_64' image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" - dependencies: - - 'manifest' -build ubuntu x86_64 local: - extends: '.build' - variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dlibav=disabled" +build android arm64 api28 local: + extends: '.build android arm64 api28' image: "${CI_REGISTRY_IMAGE}/amd64/ubuntu-build:latest" - dependencies: - - 'manifest' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ddf63aed9d..c64a0d9b67 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -62,15 +62,13 @@ manifest: - "manifest.xml" # - "gst-build/" -build fedora x86_64: +.build fedora x86_64: extends: '.build' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' dependencies: - "manifest" -build android arm64 api28: +.build android arm64 api28: extends: '.build' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu-build:a1726fee5841b776fcdaeda2e0ad628ce037a3c6' variables: MESON_ARGS: > -Dbad=enabled @@ -78,3 +76,11 @@ build android arm64 api28: --cross-file /android_arm64_28.txt dependencies: - "manifest" + +build fedora x86_64: + extends: '.build fedora x86_64' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' + +build android arm64 api28: + extends: '.build android arm64 api28' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu-build:a1726fee5841b776fcdaeda2e0ad628ce037a3c6' From 67fcaad29ea0e028fa92740c8bd9e47db2a9417d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 23 Nov 2018 18:48:11 +0200 Subject: [PATCH 102/605] Revert "Avoid duplication between ci_template.yml and .gitlab-ci.yml" This reverts commit 6bdd1ead99e7d17ba5f8cdbba2668cdebc3bdb43. This does not actually avoid duplication... [1] [1] https://gitlab.freedesktop.org/gstreamer/gst-ci/commit/6bdd1ead99e7d17ba5f8cdbba2668cdebc3bdb43#note_81275 --- .gitlab-ci.yml | 12 +++++++++--- gitlab/ci_template.yml | 14 ++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d25b68964e..b1b0930e3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,9 +121,15 @@ include: "gitlab/ci_template.yml" # Test the build job against the latest build image tag and the local manifest build fedora x86_64 local: - extends: '.build fedora x86_64' + extends: '.build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" + dependencies: + - 'manifest' -build android arm64 api28 local: - extends: '.build android arm64 api28' +build ubuntu x86_64 local: + extends: '.build' + variables: + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dlibav=disabled" image: "${CI_REGISTRY_IMAGE}/amd64/ubuntu-build:latest" + dependencies: + - 'manifest' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c64a0d9b67..ddf63aed9d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -62,13 +62,15 @@ manifest: - "manifest.xml" # - "gst-build/" -.build fedora x86_64: +build fedora x86_64: extends: '.build' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' dependencies: - "manifest" -.build android arm64 api28: +build android arm64 api28: extends: '.build' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu-build:a1726fee5841b776fcdaeda2e0ad628ce037a3c6' variables: MESON_ARGS: > -Dbad=enabled @@ -76,11 +78,3 @@ manifest: --cross-file /android_arm64_28.txt dependencies: - "manifest" - -build fedora x86_64: - extends: '.build fedora x86_64' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' - -build android arm64 api28: - extends: '.build android arm64 api28' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu-build:a1726fee5841b776fcdaeda2e0ad628ce037a3c6' From 2bb429bbf01f5fdf43f30bf20f007d74645e33c8 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 23 Nov 2018 18:59:04 +0200 Subject: [PATCH 103/605] gitlab/ci_template.yml: add dependencies block to the .build template --- .gitlab-ci.yml | 4 ---- gitlab/ci_template.yml | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1b0930e3f..93419cc035 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -123,13 +123,9 @@ include: "gitlab/ci_template.yml" build fedora x86_64 local: extends: '.build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" - dependencies: - - 'manifest' build ubuntu x86_64 local: extends: '.build' variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dlibav=disabled" image: "${CI_REGISTRY_IMAGE}/amd64/ubuntu-build:latest" - dependencies: - - 'manifest' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ddf63aed9d..4e5ddf0e47 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -30,6 +30,8 @@ manifest: .build: stage: "build" + dependencies: + - "manifest" variables: CC: "ccache gcc" CXX: "ccache g++" @@ -65,8 +67,6 @@ manifest: build fedora x86_64: extends: '.build' image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' - dependencies: - - "manifest" build android arm64 api28: extends: '.build' @@ -76,5 +76,3 @@ build android arm64 api28: -Dbad=enabled -Dbad:androidmedia=enabled --cross-file /android_arm64_28.txt - dependencies: - - "manifest" From e89fad1455d7d54353dda42993fc8fdef9e1a246 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 23 Nov 2018 18:45:31 +0200 Subject: [PATCH 104/605] gitlab/ci_template.yml: rework the CI stages Idea is that in order to not consume many resources for broken builds, we will have a basic stage where just one simple set of build and test jobs is run. If that passes we will continue with the rest of the Pipeline suite. --- .gitlab-ci.yml | 11 ++++++++--- gitlab/ci_template.yml | 14 ++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93419cc035..52c371c3af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,19 @@ stages: - - "test" - "build runtime" - "build base" - - "manifest" + - "preparation" + # Test just one basic build, if succeds procced to test the rest - "build" + - "test" + # Run multiple builds and tests, multi-distro, multi-arch + - "full builds" + - "full tests" test manifest: variables: GIT_STRATEGY: fetch image: "fedora" - stage: "test" + stage: "preparation" before_script: - dnf install -y python3-pytest python3-pytest-cov python3-requests script: @@ -122,6 +126,7 @@ include: "gitlab/ci_template.yml" # Test the build job against the latest build image tag and the local manifest build fedora x86_64 local: extends: '.build' + stage: 'build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" build ubuntu x86_64 local: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4e5ddf0e47..5b66b6bb5c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1,6 +1,11 @@ stages: - - "manifest" - - "build" + - 'preparation' + # Test just one basic build, if it succeeds proceed to test the rest + - 'build' + - 'test' + # Run multiple builds and tests, multi-distro, multi-arch + - 'full builds' + - 'full tests' variables: GIT_STRATEGY: none @@ -17,7 +22,7 @@ variables: manifest: image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/alpine-manifest-build:10ffa6f1af6fed20a8857d2c7426d81cf162e3d1' - stage: 'manifest' + stage: 'preparation' script: - cd /gst-ci - gitlab/build_manifest.py --self-update @@ -29,7 +34,7 @@ manifest: - "manifest.xml" .build: - stage: "build" + stage: 'full builds' dependencies: - "manifest" variables: @@ -66,6 +71,7 @@ manifest: build fedora x86_64: extends: '.build' + stage: 'build' image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' build android arm64 api28: From b48405a455ea4596eee87e55034e8c92e190bf6e Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sat, 24 Nov 2018 09:40:55 -0300 Subject: [PATCH 105/605] docker:fedora: Add required package to build and upload the documentation --- docker/build-base-images/Dockerfile-fedora | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index 6b462a9871..b1edbeeb9c 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -53,8 +53,10 @@ RUN dnf builddep -y \ xorg-x11-server-utils \ xorg-x11-server-Xvfb \ ninja-build && \ + cmake && \ + rsync && \ dnf clean all && \ - pip3 install meson && \ + pip3 install meson hotdoc && \ export PATH="$PATH:/usr/local/lib/python3.6/site-packages" ENV GST_BUILD_PATH="/gst-build/" From 5321d73fa9d07ef80a736e17d0316e385df18b47 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 25 Nov 2018 13:25:37 -0300 Subject: [PATCH 106/605] Add a script to import the GStreamer documentation inside the fdo server --- gitlab/freedesktop_doc_importer.sh | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 gitlab/freedesktop_doc_importer.sh diff --git a/gitlab/freedesktop_doc_importer.sh b/gitlab/freedesktop_doc_importer.sh new file mode 100755 index 0000000000..f93d23f631 --- /dev/null +++ b/gitlab/freedesktop_doc_importer.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -e + +BRANCH=master +NAMESPACE=gstreamer + +WORK_DIR=`mktemp -d -p "$DIR"` + +# deletes the temp directory +function cleanup { + rm -rf "$WORK_DIR" + echo "Deleted temp working directory $WORK_DIR" +} + +# register the cleanup function to be called on the EXIT signal +trap cleanup EXIT + +echo "" +echo "============================================================================================================================" +echo "Updating documentation from: https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=deploy" + +date + +cd $WORK_DIR +wget https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=deploy -O gstdocs.zip + +unzip gstdocs.zip + +DOC_BASE="/srv/gstreamer.freedesktop.org/public_html/documentation" + +rsync -rvaz --links --delete documentation/ $DOC_BASE || /bin/true +chmod -R g+w $DOC_BASE; chgrp -R gstreamer $DOC_BASE + +echo "Done updating documentation" +echo "" \ No newline at end of file From d2cfcef5bb21f4f1431b8799f3e27a1a6772157d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 25 Nov 2018 16:15:59 -0300 Subject: [PATCH 107/605] Fix previous commit and do not install rsync rsync is not needed in the end. --- docker/build-base-images/Dockerfile-fedora | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora index b1edbeeb9c..2c15aec403 100644 --- a/docker/build-base-images/Dockerfile-fedora +++ b/docker/build-base-images/Dockerfile-fedora @@ -52,9 +52,8 @@ RUN dnf builddep -y \ python3-cairo-devel \ xorg-x11-server-utils \ xorg-x11-server-Xvfb \ - ninja-build && \ + ninja-build \ cmake && \ - rsync && \ dnf clean all && \ pip3 install meson hotdoc && \ export PATH="$PATH:/usr/local/lib/python3.6/site-packages" From ed9ae8bb2ba8b16dea9cdad58c12e2f8b0035919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 25 Nov 2018 23:26:20 +0000 Subject: [PATCH 108/605] ci-template: build with --werror by default --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 5b66b6bb5c..ebbea4ac52 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -10,6 +10,7 @@ stages: variables: GIT_STRATEGY: none DEFAULT_MESON_ARGS: > + --werror -Dpython=enabled -Dlibav=enabled -Dugly=enabled From 3cc8d5efb54a13f19ad6c8129c3a48d7ceace26a Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sun, 25 Nov 2018 00:48:50 +0200 Subject: [PATCH 109/605] Docker registry: only tag images for the upstream repository The tags are only used to version the images that are meant to be used in the gitlab/ci_template.yml file. Thus tags are not needed when you hack on a patch from a forked repository. Lets just keep a :latest tag. --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52c371c3af..f076999936 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,20 +41,21 @@ test manifest: - export REF="${IMAGE}:${CI_COMMIT_SHA}" - export LATEST="${IMAGE}:latest" - - docker build --pull -f ${DOCKERFILE} -t ${REF} ${CONTEXT_DIR} - - docker image tag ${REF} ${LATEST} + - docker build --pull -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR} # If we are in a fork, push the image to the reigstry regardless the branch - | if [ "$CI_PROJECT_NAMESPACE" != "gstreamer" ]; then docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker push ${REF} docker push ${LATEST} fi # Push the images to the upstream registry - | if [ "$CI_COMMIT_REF_NAME" = "master" -a "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then + # Tag the image with the git ref + docker image tag ${LATEST} ${REF} + # Push the tags docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY docker push ${REF} docker push ${LATEST} From 72ec8cd33fcee090897ef22d36778ac8c80bfd73 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sun, 25 Nov 2018 02:03:29 +0200 Subject: [PATCH 110/605] docker: Add a unified image for fedora This is meant to replace both the runtime and build images. Spliting it in two increasing the maintainance required without any big benefits. --- .gitlab-ci.yml | 28 +++----- docker/build-base-images/Dockerfile-fedora | 70 ------------------ docker/fedora/Dockerfile | 6 ++ docker/fedora/cleanup.sh | 7 ++ docker/fedora/prepare.sh | 84 ++++++++++++++++++++++ docker/runtime-images/Dockerfile-fedora | 60 ---------------- 6 files changed, 106 insertions(+), 149 deletions(-) delete mode 100644 docker/build-base-images/Dockerfile-fedora create mode 100644 docker/fedora/Dockerfile create mode 100644 docker/fedora/cleanup.sh create mode 100644 docker/fedora/prepare.sh delete mode 100644 docker/runtime-images/Dockerfile-fedora diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f076999936..971027ff7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,8 @@ stages: - "build runtime" - "build base" + # Eventually will replace both of the above stages + - "build docker" - "preparation" # Test just one basic build, if succeds procced to test the rest - "build" @@ -74,29 +76,17 @@ alpine amd64 manifest builder docker: - 'docker/runtime-images/Dockerfile-manifest-builder' - 'gitlab/build_manifest.py' -fedora amd64 run docker: - stage: "build runtime" +fedora amd64 docker: + stage: "build docker" variables: ARCH: "amd64" - TAG: "fedora-runtime" - CONTEXT_DIR: "docker/runtime-images/" - DOCKERFILE: "docker/runtime-images/Dockerfile-fedora" + TAG: "fedora" + CONTEXT_DIR: "docker/fedora/" + DOCKERFILE: "docker/fedora/Dockerfile" extends: .base only: changes: - - 'docker/runtime-images/Dockerfile-fedora' - -fedora amd64 build docker: - stage: "build base" - variables: - ARCH: "amd64" - TAG: "fedora-build" - CONTEXT_DIR: "docker/build-base-images/" - DOCKERFILE: "docker/build-base-images/Dockerfile-fedora" - extends: .base - only: - changes: - - 'docker/build-base-images/Dockerfile-fedora' + - 'docker/fedora/*' ubuntu amd64 run docker: stage: "build runtime" @@ -128,7 +118,7 @@ include: "gitlab/ci_template.yml" build fedora x86_64 local: extends: '.build' stage: 'build' - image: "${CI_REGISTRY_IMAGE}/amd64/fedora-build:latest" + image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest" build ubuntu x86_64 local: extends: '.build' diff --git a/docker/build-base-images/Dockerfile-fedora b/docker/build-base-images/Dockerfile-fedora deleted file mode 100644 index 2c15aec403..0000000000 --- a/docker/build-base-images/Dockerfile-fedora +++ /dev/null @@ -1,70 +0,0 @@ -FROM registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-runtime -# Add rpmfusion repositories and install all build dependencies of GStreamer -RUN dnf builddep -y \ - gstreamer1 \ - gstreamer1-plugins-base \ - gstreamer1-plugins-good \ - gstreamer1-plugins-good-extras \ - gstreamer1-plugins-ugly \ - gstreamer1-plugins-ugly-free \ - gstreamer1-plugins-bad-nonfree \ - gstreamer1-plugins-bad-free \ - gstreamer1-plugins-bad-free-extras \ - gstreamer1-plugins-bad-freeworld \ - gstreamer1-libav \ - gstreamer1-rtsp-server \ - gstreamer1-vaapi \ - python3-gstreamer1 \ - && \ - rpm -e \ - gstreamer1-devel \ - gstreamer1-plugins-base-devel \ - gstreamer1-plugins-bad-free-devel \ - && \ - dnf install -y \ - ccache \ - gcc \ - gcc-c++ \ - gdb \ - git \ - ffmpeg \ - ffmpeg-libs \ - ffmpeg-devel \ - procps-ng \ - patch \ - redhat-rpm-config \ - json-glib \ - json-glib-devel \ - libnice \ - libnice-devel \ - libunwind \ - libunwind-devel \ - opencv \ - opencv-devel \ - openjpeg2 \ - openjpeg2-devel \ - x264 \ - x264-libs \ - x264-devel \ - pygobject3-devel \ - python3-gobject \ - python3-cairo \ - python3-cairo-devel \ - xorg-x11-server-utils \ - xorg-x11-server-Xvfb \ - ninja-build \ - cmake && \ - dnf clean all && \ - pip3 install meson hotdoc && \ - export PATH="$PATH:/usr/local/lib/python3.6/site-packages" - -ENV GST_BUILD_PATH="/gst-build/" - -# Configure git for various usage -RUN git config --global user.email "gst-build@gstreamer.net" && git config --global user.name "Gstbuild Runner" - -# get gst-build and make all subprojects available -RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ - cd $GST_BUILD_PATH && \ - meson build/ && \ - rm -rf build/ diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile new file mode 100644 index 0000000000..0e80c32fa0 --- /dev/null +++ b/docker/fedora/Dockerfile @@ -0,0 +1,6 @@ +FROM fedora:29 + +COPY prepare.sh cleanup.sh /root/ + +RUN /usr/bin/sh /root/prepare.sh && \ + /usr/bin/sh /root/cleanup.sh diff --git a/docker/fedora/cleanup.sh b/docker/fedora/cleanup.sh new file mode 100644 index 0000000000..73d62c773e --- /dev/null +++ b/docker/fedora/cleanup.sh @@ -0,0 +1,7 @@ +set -eu + +echo "Removing DNF cache" +dnf clean all + +rm -R /root/* +rm -rf /var/cache/dnf /var/log/dnf* diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh new file mode 100644 index 0000000000..9de5b6d6a1 --- /dev/null +++ b/docker/fedora/prepare.sh @@ -0,0 +1,84 @@ +set -eu + +dnf install -y git ninja-build dnf-plugins-core + +# Configure git for various usage +git config --global user.email "gst-build@gstreamer.net" +git config --global user.name "Gstbuild Runner" + +# Add rpm fusion repositories in order to access all of the gst plugins +dnf install -y "http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-29.noarch.rpm" \ + "http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-29.noarch.rpm" + +rpm --import "/etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-nonfree-fedora-29" +rpm --import "/etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-29" +dnf upgrade -y + +# Enable the cisco openh264 repo +dnf config-manager --set-enabled fedora-cisco-openh264 + +# install rest of the extra deps +dnf install -y ccache \ + cmake \ + elfutils \ + gcc \ + gcc-c++ \ + gdb \ + gtk3 \ + gtk3-devel \ + ffmpeg \ + ffmpeg-libs \ + ffmpeg-devel \ + procps-ng \ + patch \ + redhat-rpm-config \ + json-glib \ + json-glib-devel \ + libnice \ + libnice-devel \ + libunwind \ + libunwind-devel \ + opencv \ + opencv-devel \ + openjpeg2 \ + openjpeg2-devel \ + openh264 \ + openh264-devel \ + x264 \ + x264-libs \ + x264-devel \ + python3-gobject \ + python3-cairo \ + python3-cairo-devel \ + vulkan \ + vulkan-devel \ + xorg-x11-server-utils \ + xorg-x11-server-Xvfb + +pip3 install meson +# Add the pip3 installation to the path +export PATH="$PATH:/usr/local/lib/python3.7/site-packages" + +# Install the dependencies of gstreamer +dnf builddep -y gstreamer1 \ + gstreamer1-plugins-base \ + gstreamer1-plugins-good \ + gstreamer1-plugins-good-extras \ + gstreamer1-plugins-ugly \ + gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-nonfree \ + gstreamer1-plugins-bad-free \ + gstreamer1-plugins-bad-free-extras \ + gstreamer1-plugins-bad-freeworld \ + gstreamer1-libav \ + gstreamer1-rtsp-server \ + gstreamer1-vaapi \ + python3-gstreamer1 \ + -x meson + +# Remove gst-devel packages installed by builddep above +dnf remove -y "gstreamer1*devel" + +# get gst-build and make all subprojects available +git clone git://anongit.freedesktop.org/gstreamer/gst-build /gst-build/ +cd /gst-build && meson build/ && rm -rf build/ \ No newline at end of file diff --git a/docker/runtime-images/Dockerfile-fedora b/docker/runtime-images/Dockerfile-fedora deleted file mode 100644 index 936ffb9514..0000000000 --- a/docker/runtime-images/Dockerfile-fedora +++ /dev/null @@ -1,60 +0,0 @@ -FROM fedora:29 - -# install the config manager -RUN dnf install -y dnf-plugins-core - -# Add rpmfusion repositories and install all build dependencies of GStreamer -# and then remove the gst packages themselves -RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ - dnf install -y http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-29.noarch.rpm \ - http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-29.noarch.rpm && \ - rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-nonfree-fedora-29 && \ - rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-29 && \ - dnf upgrade -y && \ - dnf install -y \ - dnf-plugins-core \ - git \ - meson \ - json-glib \ - libnice \ - libunwind \ - openjpeg2 \ - procps-ng \ - x264 \ - x264-libs \ - python3-gobject \ - python3-cairo \ - ffmpeg \ - ffmpeg-libs \ - xorg-x11-server-utils \ - xorg-x11-server-Xvfb \ - && \ - dnf install -y \ - gstreamer1 \ - gstreamer1-plugins-base \ - gstreamer1-plugins-good \ - gstreamer1-plugins-good-extras \ - gstreamer1-plugins-ugly \ - gstreamer1-plugins-ugly-free \ - gstreamer1-plugins-bad-nonfree \ - gstreamer1-plugins-bad-free \ - gstreamer1-plugins-bad-free-extras \ - gstreamer1-plugins-bad-freeworld \ - gstreamer1-libav \ - gstreamer1-rtsp-server \ - && \ - rpm -e \ - gstreamer1 \ - gstreamer1-plugins-base \ - gstreamer1-plugins-good \ - gstreamer1-plugins-good-extras \ - gstreamer1-plugins-ugly \ - gstreamer1-plugins-ugly-free \ - gstreamer1-plugins-bad-nonfree \ - gstreamer1-plugins-bad-free \ - gstreamer1-plugins-bad-free-extras \ - gstreamer1-plugins-bad-freeworld \ - gstreamer1-rtsp-server \ - gstreamer1-libav \ - && \ - dnf clean all \ No newline at end of file From d17930eabb28709349c91df0223a8486e7842854 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 26 Nov 2018 15:43:59 +0200 Subject: [PATCH 111/605] docker: move the alpine image to a different directory --- .gitlab-ci.yml | 10 +++++----- .../Dockerfile} | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename docker/{runtime-images/Dockerfile-manifest-builder => build_manifest/Dockerfile} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 971027ff7f..16635f9fc3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,16 +64,16 @@ test manifest: fi alpine amd64 manifest builder docker: - stage: "build runtime" + stage: "build docker" variables: ARCH: "amd64" - TAG: "alpine-manifest-build" - CONTEXT_DIR: "docker/runtime-images/" - DOCKERFILE: "docker/runtime-images/Dockerfile-manifest-builder" + TAG: "build-manifest" + CONTEXT_DIR: "docker/build_manifest/" + DOCKERFILE: "docker/build_manifest/Dockerfile" extends: .base only: changes: - - 'docker/runtime-images/Dockerfile-manifest-builder' + - 'docker/build_manifest/*' - 'gitlab/build_manifest.py' fedora amd64 docker: diff --git a/docker/runtime-images/Dockerfile-manifest-builder b/docker/build_manifest/Dockerfile similarity index 100% rename from docker/runtime-images/Dockerfile-manifest-builder rename to docker/build_manifest/Dockerfile From f085d34a32cb8cac95cbd420344efeaa647d08d5 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 26 Nov 2018 16:23:58 +0200 Subject: [PATCH 112/605] docker: Move android into a unified image --- .gitlab-ci.yml | 37 ++++++--------- docker/android/Dockerfile | 8 ++++ .../android-create-toolchain.sh | 2 +- .../android-download-ndk.sh | 2 +- docker/android/cleanup.sh | 5 +++ docker/android/prepare.sh | 41 +++++++++++++++++ docker/build-base-images/Dockerfile-ubuntu | 45 ------------------- docker/runtime-images/Dockerfile-ubuntu | 22 --------- gitlab/ci_template.yml | 2 + 9 files changed, 71 insertions(+), 93 deletions(-) create mode 100644 docker/android/Dockerfile rename docker/{build-base-images => android}/android-create-toolchain.sh (99%) rename docker/{build-base-images => android}/android-download-ndk.sh (99%) create mode 100644 docker/android/cleanup.sh create mode 100644 docker/android/prepare.sh delete mode 100644 docker/build-base-images/Dockerfile-ubuntu delete mode 100644 docker/runtime-images/Dockerfile-ubuntu diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 16635f9fc3..3b08107971 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,4 @@ stages: - - "build runtime" - - "build base" - # Eventually will replace both of the above stages - "build docker" - "preparation" # Test just one basic build, if succeds procced to test the rest @@ -88,29 +85,18 @@ fedora amd64 docker: changes: - 'docker/fedora/*' -ubuntu amd64 run docker: - stage: "build runtime" +# Cross compile image +android 28 amd64 docker: + stage: "build docker" variables: ARCH: "amd64" - TAG: "ubuntu-runtime" - CONTEXT_DIR: "docker/runtime-images/" - DOCKERFILE: "docker/runtime-images/Dockerfile-ubuntu" + TAG: "android_28" + CONTEXT_DIR: "docker/android/" + DOCKERFILE: "docker/android/Dockerfile" extends: .base only: changes: - - 'docker/runtime-images/Dockerfile-ubuntu' - -ubuntu amd64 build docker: - stage: "build base" - variables: - ARCH: "amd64" - TAG: "ubuntu-build" - CONTEXT_DIR: "docker/build-base-images/" - DOCKERFILE: "docker/build-base-images/Dockerfile-ubuntu" - extends: .base - only: - changes: - - 'docker/build-base-images/Dockerfile-ubuntu' + - 'docker/android/*' include: "gitlab/ci_template.yml" @@ -120,8 +106,11 @@ build fedora x86_64 local: stage: 'build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest" -build ubuntu x86_64 local: +build android 28 local: extends: '.build' variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dlibav=disabled" - image: "${CI_REGISTRY_IMAGE}/amd64/ubuntu-build:latest" + MESON_ARGS: > + -Dbad=enabled + -Dbad:androidmedia=enabled + --cross-file /android_arm64_28.txt + image: "${CI_REGISTRY_IMAGE}/amd64/android_28:latest" diff --git a/docker/android/Dockerfile b/docker/android/Dockerfile new file mode 100644 index 0000000000..67a2095974 --- /dev/null +++ b/docker/android/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:18.04 + +COPY prepare.sh cleanup.sh android-download-ndk.sh android-create-toolchain.sh /root/ + +ENV DEBIAN_FRONTEND=noninteractive + +RUN sh /root/prepare.sh && \ + sh /root/cleanup.sh diff --git a/docker/build-base-images/android-create-toolchain.sh b/docker/android/android-create-toolchain.sh similarity index 99% rename from docker/build-base-images/android-create-toolchain.sh rename to docker/android/android-create-toolchain.sh index bf909b64e5..d15287d5b9 100755 --- a/docker/build-base-images/android-create-toolchain.sh +++ b/docker/android/android-create-toolchain.sh @@ -19,7 +19,7 @@ # Author: Xavier Claessens # -set -e +set -eu arch=$1 api=$2 diff --git a/docker/build-base-images/android-download-ndk.sh b/docker/android/android-download-ndk.sh similarity index 99% rename from docker/build-base-images/android-download-ndk.sh rename to docker/android/android-download-ndk.sh index 0e07407422..487b58315e 100755 --- a/docker/build-base-images/android-download-ndk.sh +++ b/docker/android/android-download-ndk.sh @@ -19,7 +19,7 @@ # Author: Xavier Claessens # -set -e +set -eu # Download Android NDK ANDROID_NDK_VERSION="r18b" diff --git a/docker/android/cleanup.sh b/docker/android/cleanup.sh new file mode 100644 index 0000000000..33032b178e --- /dev/null +++ b/docker/android/cleanup.sh @@ -0,0 +1,5 @@ +set -eu + +echo "Removing apt cache" +rm -R /root/* +rm -R /var/lib/apt/ /var/log/apt/ diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh new file mode 100644 index 0000000000..cf6a726282 --- /dev/null +++ b/docker/android/prepare.sh @@ -0,0 +1,41 @@ +set -eu + +# make source packages available in order to figure out build dependencies +sed -i "s/# deb-src/deb-src/g" /etc/apt/sources.list + +apt update +apt build-dep -y \ + orc \ + gstreamer1.0 \ + gst-plugins-base1.0 \ + gst-plugins-good1.0 \ + gst-plugins-bad1.0 \ + gst-plugins-ugly1.0 \ + gst-libav1.0 \ + gst-rtsp-server1.0 \ + gst-python1.0 \ + gstreamer-vaapi \ + gstreamer-editing-services1.0 + +apt install -y \ + ccache \ + gdb \ + git \ + xvfb \ + python3-pip \ + wget \ + unzip + +pip3 install meson + +# Configure git for various usage +git config --global user.email "gst-build@gstreamer.net" +git config --global user.name "Gstbuild Runner" + +# Setup Android toolchain +./root/android-download-ndk.sh +./root/android-create-toolchain.sh arm64 28 + +# get gst-build and make all subprojects available +git clone git://anongit.freedesktop.org/gstreamer/gst-build '/gst-build/' +cd '/gst-build/' && meson build/ && rm -rf build/ \ No newline at end of file diff --git a/docker/build-base-images/Dockerfile-ubuntu b/docker/build-base-images/Dockerfile-ubuntu deleted file mode 100644 index d2dfd72003..0000000000 --- a/docker/build-base-images/Dockerfile-ubuntu +++ /dev/null @@ -1,45 +0,0 @@ -FROM registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu-runtime - -# make source packages available in order to figure out build dependencies -RUN \ - sed -i "s/# deb-src/deb-src/g" /etc/apt/sources.list && \ - apt-get update && \ - apt-get -y build-dep \ - orc \ - gstreamer1.0 \ - gst-plugins-base1.0 \ - gst-plugins-good1.0 \ - gst-plugins-bad1.0 \ - gst-plugins-ugly1.0 \ - gst-libav1.0 \ - gst-rtsp-server1.0 \ - gst-python1.0 \ - gstreamer-vaapi \ - gstreamer-editing-services1.0 && \ - apt-get -y install \ - ccache \ - gdb \ - git \ - xvfb \ - python3-pip \ - wget \ - unzip - -RUN pip3 install meson - -ENV GST_BUILD_PATH="/gst-build/" - -# Configure git for various usage -RUN git config --global user.email "gst-build@gstreamer.net" && git config --global user.name "Gstbuild Runner" - -# Setup Android toolchain -COPY android-download-ndk.sh . -COPY android-create-toolchain.sh . -RUN ./android-download-ndk.sh -RUN ./android-create-toolchain.sh arm64 28 -RUN rm -rf /android-ndk - -# get gst-build and make all subprojects available -RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ - cd $GST_BUILD_PATH && \ - meson build/ && rm -rf build/ \ No newline at end of file diff --git a/docker/runtime-images/Dockerfile-ubuntu b/docker/runtime-images/Dockerfile-ubuntu deleted file mode 100644 index 6d1809f7a5..0000000000 --- a/docker/runtime-images/Dockerfile-ubuntu +++ /dev/null @@ -1,22 +0,0 @@ -FROM ubuntu:18.04 - -ENV DEBIAN_FRONTEND=noninteractive - -# upgrade and install all runtime dependencies for gstreamer -RUN apt-get update && \ - apt-get -y upgrade && \ - apt-get -y install --no-install-recommends\ - gstreamer1.0-libav \ - gstreamer1.0-plugins-bad \ - gstreamer1.0-plugins-bad-faad \ - gstreamer1.0-plugins-bad-videoparsers \ - gstreamer1.0-plugins-ugly \ - gstreamer1.0-plugins-ugly-amr \ - gstreamer1.0-rtsp \ - gstreamer1.0-alsa \ - gstreamer1.0-plugins-base \ - gstreamer1.0-plugins-good \ - gstreamer1.0-pulseaudio \ - gstreamer1.0-x \ - libgstreamer-plugins-base1.0-0 \ - libgstreamer1.0 diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ebbea4ac52..8121f1f3a8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -83,3 +83,5 @@ build android arm64 api28: -Dbad=enabled -Dbad:androidmedia=enabled --cross-file /android_arm64_28.txt + before_script: + - cat /android_arm64_28.txt From e89c402e14096d6ca398e664c477ba75dfc67d2b Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Wed, 28 Nov 2018 13:39:16 -0500 Subject: [PATCH 113/605] Update docker images to latest --- gitlab/ci_template.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8121f1f3a8..9f9ed5d497 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -22,7 +22,7 @@ variables: -Dsharp=disabled manifest: - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/alpine-manifest-build:10ffa6f1af6fed20a8857d2c7426d81cf162e3d1' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:f085d34a32cb8cac95cbd420344efeaa647d08d5' stage: 'preparation' script: - cd /gst-ci @@ -73,11 +73,11 @@ manifest: build fedora x86_64: extends: '.build' stage: 'build' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora-build:2414895a53408ed8a0410a52560f8090b3f74696' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:f085d34a32cb8cac95cbd420344efeaa647d08d5' build android arm64 api28: extends: '.build' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu-build:a1726fee5841b776fcdaeda2e0ad628ce037a3c6' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android_28:f085d34a32cb8cac95cbd420344efeaa647d08d5' variables: MESON_ARGS: > -Dbad=enabled From e87e1fb6ca05f56f14476fb1e4823a42badd086f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 30 Nov 2018 19:57:13 +0200 Subject: [PATCH 114/605] docker: Add a new image for the gst-indent job This is meant run GNU indent along with the gstreamer wrapper through the codebase and error out for missconfigured code. Part of https://gitlab.freedesktop.org/gstreamer/gstreamer-project/issues/33 --- .gitlab-ci.yml | 12 ++++++++++++ docker/indent/Dockerfile | 5 +++++ 2 files changed, 17 insertions(+) create mode 100644 docker/indent/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3b08107971..375dec4c52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,6 +73,18 @@ alpine amd64 manifest builder docker: - 'docker/build_manifest/*' - 'gitlab/build_manifest.py' +gst-indent amd64 docker: + stage: "build docker" + variables: + ARCH: "amd64" + TAG: "gst-indent" + CONTEXT_DIR: "docker/indent/" + DOCKERFILE: "docker/indent/Dockerfile" + extends: .base + only: + changes: + - 'docker/indent/*' + fedora amd64 docker: stage: "build docker" variables: diff --git a/docker/indent/Dockerfile b/docker/indent/Dockerfile new file mode 100644 index 0000000000..eff3c85f19 --- /dev/null +++ b/docker/indent/Dockerfile @@ -0,0 +1,5 @@ +FROM debian:stretch-slim + +RUN apt update -yqq \ + && apt install -y curl indent git findutils \ + && rm -R /var/lib/apt/ /var/log/apt/ \ No newline at end of file From 8683c2940b200fe9ceaa98f3ff63c719dc46aacc Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 30 Nov 2018 20:43:39 +0200 Subject: [PATCH 115/605] docker: add an image to run the tests of the manifest So in the futuere we can avoid installing pytest each time --- .gitlab-ci.yml | 12 ++++++++++++ docker/test_manifest/Dockerfile | 7 +++++++ 2 files changed, 19 insertions(+) create mode 100644 docker/test_manifest/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 375dec4c52..edd3061a62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,6 +73,18 @@ alpine amd64 manifest builder docker: - 'docker/build_manifest/*' - 'gitlab/build_manifest.py' +test manifest amd64 docker: + stage: "build docker" + variables: + ARCH: "amd64" + TAG: "build-manifest" + CONTEXT_DIR: "docker/test_manifest/" + DOCKERFILE: "docker/test_manifest/Dockerfile" + extends: .base + only: + changes: + - 'docker/test_manifest/*' + gst-indent amd64 docker: stage: "build docker" variables: diff --git a/docker/test_manifest/Dockerfile b/docker/test_manifest/Dockerfile new file mode 100644 index 0000000000..6ac815891c --- /dev/null +++ b/docker/test_manifest/Dockerfile @@ -0,0 +1,7 @@ +FROM fedora + +RUN dnf install -y \ + python3-pytest \ + python3-requests \ + python3-pytest-cov \ + && rm -rf /var/cache/dnf /var/log/dnf* \ No newline at end of file From 032c8cc01ee4a6caa7ea6600965e704265f9e94f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 29 Nov 2018 12:43:55 +0200 Subject: [PATCH 116/605] docker fedora: add extra deps featuring: - flite - elfuitls - gsl - couple of mesa packages - sdl2 - graphene - aom - neon - sbc --- docker/fedora/prepare.sh | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 9de5b6d6a1..29b26ab32e 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -18,17 +18,30 @@ dnf upgrade -y dnf config-manager --set-enabled fedora-cisco-openh264 # install rest of the extra deps -dnf install -y ccache \ +dnf install -y \ + aom \ + aom-extra-tools \ + libaom \ + libaom-devel \ + ccache \ cmake \ elfutils \ + elfutils-libs \ + elfutils-devel \ gcc \ gcc-c++ \ gdb \ gtk3 \ gtk3-devel \ + graphene \ + graphene-devel \ + gsl \ + gsl-devel \ ffmpeg \ ffmpeg-libs \ ffmpeg-devel \ + flite \ + flite-devel \ procps-ng \ patch \ redhat-rpm-config \ @@ -38,12 +51,18 @@ dnf install -y ccache \ libnice-devel \ libunwind \ libunwind-devel \ + neon \ + neon-devel \ opencv \ opencv-devel \ openjpeg2 \ openjpeg2-devel \ openh264 \ openh264-devel \ + SDL2 \ + SDL2-devel \ + sbc \ + sbc-devel \ x264 \ x264-libs \ x264-devel \ @@ -52,6 +71,22 @@ dnf install -y ccache \ python3-cairo-devel \ vulkan \ vulkan-devel \ + mesa-omx-drivers \ + mesa-libGL \ + mesa-libGL-devel \ + mesa-libGLU \ + mesa-libGLU-devel \ + mesa-libGLES \ + mesa-libGLES-devel \ + mesa-libOpenCL \ + mesa-libOpenCL-devel \ + mesa-libgbm \ + mesa-libgbm-devel \ + mesa-libd3d \ + mesa-libd3d-devel \ + mesa-libOSMesa \ + mesa-libOSMesa-devel \ + mesa-vulkan-drivers \ xorg-x11-server-utils \ xorg-x11-server-Xvfb From 6f7e01e1e30a73efa880acdc8e911f1f20c58dbb Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sat, 1 Dec 2018 13:17:46 +0200 Subject: [PATCH 117/605] Docker: fix a type in the test-manifest image --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index edd3061a62..7e8d445dca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,7 +77,7 @@ test manifest amd64 docker: stage: "build docker" variables: ARCH: "amd64" - TAG: "build-manifest" + TAG: "test-manifest" CONTEXT_DIR: "docker/test_manifest/" DOCKERFILE: "docker/test_manifest/Dockerfile" extends: .base From 06c76e1d63a19d1448be753d13da48c30a111a9e Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sat, 1 Dec 2018 15:20:38 +0200 Subject: [PATCH 118/605] CI: Update image uris --- .gitlab-ci.yml | 4 +--- gitlab/ci_template.yml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e8d445dca..ee703ba4c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,10 +11,8 @@ stages: test manifest: variables: GIT_STRATEGY: fetch - image: "fedora" + image: "registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb" stage: "preparation" - before_script: - - dnf install -y python3-pytest python3-pytest-cov python3-requests script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py coverage: '/TOTAL.*\s+(\d+%)$/' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9f9ed5d497..b62f79ca1c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -73,7 +73,7 @@ manifest: build fedora x86_64: extends: '.build' stage: 'build' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:f085d34a32cb8cac95cbd420344efeaa647d08d5' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' build android arm64 api28: extends: '.build' From 597544cd6116a6e7cac2cdcf6f7bf06f153c3ad5 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sat, 1 Dec 2018 11:33:44 -0500 Subject: [PATCH 119/605] build_manifest: Increase the number of search results This increase the number of search results when looking up user branches to 100 items. This should be sufficient for all of our users. Fixes #3 --- gitlab/build_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 9ad506f0fd..9e16f16ca2 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -108,7 +108,7 @@ def request(path: str) -> List[Dict[str, str]]: def get_project_branch(project_id: int, name: str) -> Dict[str, str]: print(f"Searching for {name} branch in project {project_id}") - path = f"projects/{project_id}/repository/branches?search={name}" + path = f"projects/{project_id}/repository/branches?search={name}&per_page=100" results = request(path) if not results: From 9e8d215f9001eb5f8b65bfff38d510c1bd3ebfd7 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 13 Nov 2018 13:43:42 +0200 Subject: [PATCH 120/605] ci_template: Add a gst-indent job --- gitlab/ci_template.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b62f79ca1c..61b094e377 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -34,6 +34,30 @@ manifest: paths: - "manifest.xml" +gst indent: + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + stage: 'preparation' + variables: + GIT_STRATEGY: 'fetch' + script: + # man indent. grep RETURN VALUE, grab a beer on my behalf... + - indent --version || true + - curl -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent + - chmod +x gst-indent + - ./gst-indent $(find . -name "*.c") + - | + if [[ $(git diff) ]]; then + git diff --color=always + echo 'style diverges, please run gst-indent first' + exit 1 + else + echo "Code is properly formatted" + fi + except: + variables: + # No point on trying to format C files in those repositories + - $CI_PROJECT_NAME == "gstreamer-sharp" + .build: stage: 'full builds' dependencies: From 61074f6d8dff5a3d7071854fb3f4cedc3c44df09 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 28 Nov 2018 12:09:28 +0200 Subject: [PATCH 121/605] Run gst-indent through the files This is required before we enabled an indent test in the CI. https://gitlab.freedesktop.org/gstreamer/gstreamer-project/issues/33 --- fuzzing/gst-discoverer.c | 65 ++++++++++++++++++++-------------------- fuzzing/typefind.c | 10 +++---- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/fuzzing/gst-discoverer.c b/fuzzing/gst-discoverer.c index 17c91590b9..0440606a4a 100644 --- a/fuzzing/gst-discoverer.c +++ b/fuzzing/gst-discoverer.c @@ -28,13 +28,13 @@ #include #ifndef LOCAL_FUZZ_BUILD -GST_PLUGIN_STATIC_DECLARE(coreelements); -GST_PLUGIN_STATIC_DECLARE(playback); -GST_PLUGIN_STATIC_DECLARE(typefindfunctions); -GST_PLUGIN_STATIC_DECLARE(app); -GST_PLUGIN_STATIC_DECLARE(ogg); -GST_PLUGIN_STATIC_DECLARE(theora); -GST_PLUGIN_STATIC_DECLARE(vorbis); +GST_PLUGIN_STATIC_DECLARE (coreelements); +GST_PLUGIN_STATIC_DECLARE (playback); +GST_PLUGIN_STATIC_DECLARE (typefindfunctions); +GST_PLUGIN_STATIC_DECLARE (app); +GST_PLUGIN_STATIC_DECLARE (ogg); +GST_PLUGIN_STATIC_DECLARE (theora); +GST_PLUGIN_STATIC_DECLARE (vorbis); #endif /* push-based discoverer fuzzing target @@ -57,34 +57,34 @@ const guint8 *fuzztesting_data; size_t fuzztesting_size; static void -appsrc_configuration (GstDiscoverer *dc, GstElement *source, gpointer data) +appsrc_configuration (GstDiscoverer * dc, GstElement * source, gpointer data) { GstBuffer *buf; GstFlowReturn ret; - + /* Create buffer from fuzztesting_data which shouldn't be freed */ - buf = gst_buffer_new_wrapped_full (0, (gpointer) fuzztesting_data, fuzztesting_size, - 0, fuzztesting_size, NULL, NULL); + buf = + gst_buffer_new_wrapped_full (0, (gpointer) fuzztesting_data, + fuzztesting_size, 0, fuzztesting_size, NULL, NULL); g_object_set (G_OBJECT (source), "size", fuzztesting_size, NULL); - g_signal_emit_by_name (G_OBJECT(source), "push-buffer", buf, &ret); + g_signal_emit_by_name (G_OBJECT (source), "push-buffer", buf, &ret); gst_buffer_unref (buf); } static void -custom_logger (const gchar *log_domain, - GLogLevelFlags log_level, - const gchar *message, - gpointer unused_data) +custom_logger (const gchar * log_domain, + GLogLevelFlags log_level, const gchar * message, gpointer unused_data) { if (log_level & G_LOG_LEVEL_CRITICAL) { g_printerr ("CRITICAL ERROR : %s\n", message); - abort(); + abort (); } else if (log_level & G_LOG_LEVEL_WARNING) { g_printerr ("WARNING : %s\n", message); } } -int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) +int +LLVMFuzzerTestOneInput (const guint8 * data, size_t size) { GError *err = NULL; GstDiscoverer *dc; @@ -94,25 +94,25 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) if (!initialized) { /* We want critical warnings to assert so we can fix them */ - g_log_set_always_fatal(G_LOG_LEVEL_CRITICAL); + g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL); g_log_set_default_handler (custom_logger, NULL); /* Only initialize and register plugins once */ gst_init (NULL, NULL); #ifndef LOCAL_FUZZ_BUILD - GST_PLUGIN_STATIC_REGISTER(coreelements); - GST_PLUGIN_STATIC_REGISTER(playback); - GST_PLUGIN_STATIC_REGISTER(typefindfunctions); - GST_PLUGIN_STATIC_REGISTER(app); - GST_PLUGIN_STATIC_REGISTER(ogg); - GST_PLUGIN_STATIC_REGISTER(theora); - GST_PLUGIN_STATIC_REGISTER(vorbis); + GST_PLUGIN_STATIC_REGISTER (coreelements); + GST_PLUGIN_STATIC_REGISTER (playback); + GST_PLUGIN_STATIC_REGISTER (typefindfunctions); + GST_PLUGIN_STATIC_REGISTER (app); + GST_PLUGIN_STATIC_REGISTER (ogg); + GST_PLUGIN_STATIC_REGISTER (theora); + GST_PLUGIN_STATIC_REGISTER (vorbis); #endif initialized = TRUE; } - + dc = gst_discoverer_new (timeout * GST_SECOND, &err); if (G_UNLIKELY (dc == NULL)) { g_print ("Error initializing: %s\n", err->message); @@ -122,17 +122,16 @@ int LLVMFuzzerTestOneInput(const guint8 *data, size_t size) fuzztesting_data = data; fuzztesting_size = size; - + /* Connect to source-setup signal to give the data */ g_signal_connect (dc, "source-setup", (GCallback) appsrc_configuration, NULL); - + info = gst_discoverer_discover_uri (dc, "appsrc://", &err); g_clear_error (&err); if (info) gst_discoverer_info_unref (info); - + g_object_unref (dc); - + return 0; - } - +} diff --git a/fuzzing/typefind.c b/fuzzing/typefind.c index 3d5d14cfec..999ba337fc 100644 --- a/fuzzing/typefind.c +++ b/fuzzing/typefind.c @@ -44,14 +44,12 @@ GST_PLUGIN_STATIC_DECLARE (app); * **/ static void -custom_logger (const gchar *log_domain, - GLogLevelFlags log_level, - const gchar *message, - gpointer unused_data) +custom_logger (const gchar * log_domain, + GLogLevelFlags log_level, const gchar * message, gpointer unused_data) { if (log_level & G_LOG_LEVEL_CRITICAL) { g_printerr ("CRITICAL ERROR : %s\n", message); - abort(); + abort (); } else if (log_level & G_LOG_LEVEL_WARNING) { g_printerr ("WARNING : %s\n", message); } @@ -69,7 +67,7 @@ LLVMFuzzerTestOneInput (const guint8 * data, size_t size) if (!initialized) { /* We want critical warnings to assert so we can fix them */ - g_log_set_always_fatal(G_LOG_LEVEL_CRITICAL); + g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL); g_log_set_default_handler (custom_logger, NULL); /* Only initialize and register plugins once */ From 5b664cd00fce7968f1f64458246772777542b542 Mon Sep 17 00:00:00 2001 From: "Jan Alexander Steffens (heftig)" Date: Wed, 5 Dec 2018 22:08:49 +0100 Subject: [PATCH 122/605] ci_template: Don't use command substitution with find GNU find has operations like -exec and -delete that operate without the brittleness of applying shell word splitting to its output. In addition, it won't fail if no files were matched. --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 61b094e377..380343ff30 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -44,7 +44,7 @@ gst indent: - indent --version || true - curl -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent - chmod +x gst-indent - - ./gst-indent $(find . -name "*.c") + - find . -name '*.c' -exec ./gst-indent {} + - | if [[ $(git diff) ]]; then git diff --color=always @@ -80,7 +80,7 @@ gst indent: after_script: - cd gst-build/ # Clean the artifacts packages to avoid copying "useless" build products. - - rm $(find build -name "*.o") $(find -name "*.a") + - test -d build && find build -name '*.[ao]' -delete # Clean the .git repos since we won't need them anymore - rm -rf subprojects/*/.git/ - rm -rf build/subprojects/*/.git/ From 4493ab0bb8ddfa52ed5b54137cc28013a6e11888 Mon Sep 17 00:00:00 2001 From: "Jan Alexander Steffens (heftig)" Date: Wed, 5 Dec 2018 22:12:09 +0100 Subject: [PATCH 123/605] ci_template: Use git diff --quiet for checking for changes Checking the exit code of git diff --quiet is preferred over looking whether git diff produces any output. --- gitlab/ci_template.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 380343ff30..7052006c1b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -46,12 +46,12 @@ gst indent: - chmod +x gst-indent - find . -name '*.c' -exec ./gst-indent {} + - | - if [[ $(git diff) ]]; then + if git diff --quiet; then + echo "Code is properly formatted" + else git diff --color=always echo 'style diverges, please run gst-indent first' exit 1 - else - echo "Code is properly formatted" fi except: variables: From 93ddefcc20d95af164576331b83075089e9e24fc Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 5 Dec 2018 22:46:57 -0300 Subject: [PATCH 124/605] Install git in the test_manifest docker as necessary now --- docker/test_manifest/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test_manifest/Dockerfile b/docker/test_manifest/Dockerfile index 6ac815891c..15ffbdafe5 100644 --- a/docker/test_manifest/Dockerfile +++ b/docker/test_manifest/Dockerfile @@ -4,4 +4,5 @@ RUN dnf install -y \ python3-pytest \ python3-requests \ python3-pytest-cov \ + git \ && rm -rf /var/cache/dnf /var/log/dnf* \ No newline at end of file From 8932df0c2d0e7adea4792b727dc094b9de6e209b Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 5 Dec 2018 22:03:53 -0300 Subject: [PATCH 125/605] buildmanifest: Move away from the gitlab API and use plain git to retrieve branch names Makes the code much simpler --- gitlab/build_manifest.py | 269 ++++++++++++--------------------------- 1 file changed, 84 insertions(+), 185 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 9e16f16ca2..2a54fc3e4d 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -10,26 +10,22 @@ from typing import Dict, Tuple, List from urllib.parse import urlparse # from pprint import pprint -# Each item is a Tuple of (project-path, project-id) -# ex. https://gitlab.freedesktop.org/gstreamer/gst-build -# has project path 'gst-build' and project-id '1342' -# TODO: Named tuples are awesome -GSTREAMER_MODULES: List[Tuple[str, int]] = [ - # ('orc', 1360), - ('gst-build', 1342), - ('gstreamer', 1357), - ('gst-plugins-base', 1352), - ('gst-plugins-good', 1353), - ('gst-plugins-bad', 1351), - ('gst-plugins-ugly', 1354), - ('gst-libav', 1349), - ('gst-devtools', 1344), - ('gst-docs', 1345), - ('gst-editing-services', 1346), - ('gst-omx', 1350), - ('gst-python', 1355), - ('gst-rtsp-server', 1362), - ('gstreamer-vaapi', 1359), +GSTREAMER_MODULES: List[str] = [ + # 'orc', + 'gst-build', + 'gstreamer', + 'gst-plugins-base', + 'gst-plugins-good', + 'gst-plugins-bad', + 'gst-plugins-ugly', + 'gst-libav', + 'gst-devtools', + 'gst-docs', + 'gst-editing-services', + 'gst-omx', + 'gst-python', + 'gst-rtsp-server', + 'gstreamer-vaapi', ] MANIFEST_TEMPLATE: str = """ @@ -40,6 +36,51 @@ MANIFEST_TEMPLATE: str = """ """ +def git(*args, repository_path='.'): + return subprocess.check_output(["git"] + list(args), cwd=repository_path, + ).decode() + + +def get_repository_sha_in_namespace(module: str, namespace: str, branches: List[str]) -> str: + print(branches) + res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', *branches) + if not res: + return None + + for branch in branches: + for line in res.split('\n'): + if line.endswith('/' + branch): + return res.split('\t')[0] + + +def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: + namespace: str = os.environ["CI_PROJECT_NAMESPACE"] + + if module == os.environ['CI_PROJECT_NAME']: + return 'user', os.environ['CI_COMMIT_SHA'] + + if branchname != "master": + sha = get_repository_sha_in_namespace(module, namespace, [branchname]) + if sha is not None: + return 'user', sha + + print(f"Did not find user branch named {branchname}") + + # Check upstream project for a branch + sha = get_repository_sha_in_namespace(module, 'gstreamer', [branchname, 'master']) + if sha is not None: + print("Found mathcing branch in upstream project") + print(f"gstreamer/{branchname}") + return 'origin', sha + + # This should never occur given the upstream fallback above + print(f"Could not find anything for {module}:{branchname}") + print("If something reaches that point, please file a bug") + print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues") + assert False + + +# --- Unit tests --- # # Basically, pytest will happily let a test mutate a variable, and then run # the next tests one the same environment without reset the vars. def preserve_ci_vars(func): @@ -69,163 +110,6 @@ def preserve_ci_vars(func): return wrapper - -def request_raw(path: str, headers: Dict[str, str], project_url: str) -> List[Dict[str, str]]: - # ex. base_url = "gitlab.freedesktop.org" - base_url: str = urlparse(project_url).hostname - url: str = f"https://{base_url}/api/v4/{path}" - print(f"GET {url}") - # print(f"Headers: {headers}") - resp = requests.get(url, headers=headers) - - print(f"Request returned: {resp.status_code}") - if not resp.ok: - return None - - return resp.json() - - -def request(path: str) -> List[Dict[str, str]]: - # Check if there is a custom token set - # API calls to Group namespaces need to be authenticated - # regardless if the group/projects are public or not. - # CI_JOB_TOKEN has an actuall value only for private jobs - # and that's also an Gitlab EE feature. - # Which means no matter what we need to give the runner - # an actuall token if we want to query even the public - # gitlab.fd.o/gstreamer group - try: - headers: Dict[str, str] = {'Private-Token': os.environ["READ_PROJECTS_TOKEN"] } - except KeyError: - # print("Custom token was not set, group api querries will fail") - # JOB_TOKEN is the default placeholder of CI_JOB_TOKEN - headers: Dict[str, str] = {'JOB_TOKEN': "xxxxxxxxxxxxxxxxxxxx" } - - # mock: "https://gitlab.freedesktop.org/gstreamer/gstreamer" - project_url: str = os.environ['CI_PROJECT_URL'] - return request_raw(path, headers, project_url) - - -def get_project_branch(project_id: int, name: str) -> Dict[str, str]: - print(f"Searching for {name} branch in project {project_id}") - path = f"projects/{project_id}/repository/branches?search={name}&per_page=100" - results = request(path) - - if not results: - return None - - # The api returns a list of projects that match the search - # we want the exact match, which might not be the first on the list - for project_res in results: - if project_res["name"] == name: - return project_res - - return None - - -@preserve_ci_vars -def test_get_project_branch(): - id = 1353 - os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" - del os.environ["READ_PROJECTS_TOKEN"] - - twelve = get_project_branch(id, '1.12') - assert twelve is not None - assert twelve['name'] == '1.12' - - fourteen = get_project_branch(id, '1.14') - assert fourteen is not None - assert fourteen['name'] == '1.14' - - failure = get_project_branch(id, 'why-would-anyone-chose-this-branch-name') - assert failure is None - - failure2 = get_project_branch("invalid-id", '1.12') - assert failure2 is None - - -# Documentation: https://docs.gitlab.com/ce/api/projects.html#list-user-projects -def search_user_namespace(user: str, project: str) -> Dict[str, str]: - print(f"Searching for {project} project in @{user} user's namespace") - path = f"users/{user}/projects?search={project}" - results = request(path) - - if not results: - return None - - # The api returns a list of projects that match the search - # we want the exact match, which might not be the first on the list - for project_res in results: - if project_res["path"] == project: - return project_res - - return None - - -@preserve_ci_vars -def test_search_user_namespace(): - os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/alatiera/gst-plugins-good" - del os.environ["READ_PROJECTS_TOKEN"] - user = "alatiera" - - gst = search_user_namespace("alatiera", "gstreamer") - assert gst is not None - assert gst['path'] == 'gstreamer' - - gst_good = search_user_namespace("alatiera", "gst-plugins-good") - assert gst_good is not None - assert gst_good['path'] == 'gst-plugins-good' - - res = search_user_namespace("alatiera", "404-project-not-found") - assert res is None - - # Passing a group namespace instead of user should return None - res = search_user_namespace("gstreamer", "gst-plugins-good") - assert res is None - - -def find_repository_sha(module: Tuple[str, int], branchname: str) -> Tuple[str, str]: - namespace: str = os.environ["CI_PROJECT_NAMESPACE"] - - if module[0] == os.environ['CI_PROJECT_NAME']: - return 'user', os.environ['CI_COMMIT_SHA'] - - if branchname != "master": - project = search_user_namespace(namespace, module[0]) - # Find a fork in the User's namespace - if project: - id = project['id'] - print(f"User project found, id: {id}") - # If we have a branch with same name, use it. - branch = get_project_branch(id, branchname) - if branch is not None: - path = project['namespace']['path'] - print("Found matching branch in user's namespace") - print(f"{path}/{branchname}") - - return 'user', branch['commit']['id'] - print(f"Did not find user branch named {branchname}") - - # Check upstream project for a branch - branch = get_project_branch(module[1], branchname) - if branch is not None: - print("Found mathcing branch in upstream project") - print(f"gstreamer/{branchname}") - return 'origin', branch['commit']['id'] - - # Fallback to using upstream master branch - branch = get_project_branch(module[1], 'master') - if branch is not None: - print("Falling back to master branch on upstream project") - print(f"gstreamer/master") - return 'origin', branch['commit']['id'] - - # This should never occur given the upstream fallback above - print("If something reaches that point, please file a bug") - print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues") - assert False - - @preserve_ci_vars def test_find_repository_sha(): os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" @@ -233,16 +117,16 @@ def test_find_repository_sha(): del os.environ["READ_PROJECTS_TOKEN"] # This should find the repository in the user namespace - remote, git_ref = find_repository_sha(("gst-plugins-good", 1353), "1.2") + remote, git_ref = find_repository_sha("gst-plugins-good", "1.2") assert remote == "user" assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325" # This should fallback to upstream master branch since no matching branch was found - remote, git_ref = find_repository_sha(("gst-plugins-good", 1353), "totally-valid-branch-name") + remote, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name") assert remote == "origin" # This should fallback to upstream master branch since no repository was found - remote, git_ref = find_repository_sha(("totally-valid-project-name", 42), "1.2") + remote, git_ref = find_repository_sha("totally-valid-project-name", "1.2") assert remote == "origin" # This is now the sha of the last commit # assert git_ref == "master" @@ -250,11 +134,26 @@ def test_find_repository_sha(): os.environ["CI_PROJECT_NAME"] = "the_project" os.environ["CI_COMMIT_SHA"] = "MySha" - remote, git_ref = find_repository_sha(("the_project", 199), "whatever") + remote, git_ref = find_repository_sha("the_project", "whatever") assert remote == "user" assert git_ref == "MySha" +@preserve_ci_vars +def test_get_project_branch(): + os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" + os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_" + del os.environ["READ_PROJECTS_TOKEN"] + + remote, twelve = find_repository_sha('gst-plugins-good', '1.12') + assert twelve is not None + assert remote == 'origin' + + remote, fourteen = find_repository_sha('gst-plugins-good', '1.12') + assert fourteen is not None + assert remote == 'origin' + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--self-update", action="store_true", default=False) @@ -267,14 +166,14 @@ if __name__ == "__main__": user_remote_url += '/' if options.self_update: - remote, sha = find_repository_sha(("gst-ci", "1343"), current_branch) + remote, sha = find_repository_sha("gst-ci", current_branch) if remote == 'user': remote = user_remote_url + 'gst-ci' else: remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci" - subprocess.check_call(['git', 'fetch', remote, sha]) - subprocess.check_call(['git', 'checkout', '--detach', 'FETCH_HEAD']) + git('fetch', remote, sha) + git('checkout', '--detach', 'FETCH_HEAD') sys.exit(0) projects: str = '' @@ -282,7 +181,7 @@ if __name__ == "__main__": for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') remote, revision = find_repository_sha(module, current_branch) - projects += project_template % {'name': module[0], 'remote': remote, 'revision': revision} + projects += project_template % {'name': module, 'remote': remote, 'revision': revision} with open(options.output, mode='w') as manifest: print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest) From 7cb3feec1902d82c4a536e7007e8faff84674379 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 5 Dec 2018 22:23:47 -0300 Subject: [PATCH 126/605] build_manifest: Include remote refname in the manifest --- gitlab/build_manifest.py | 75 ++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 2a54fc3e4d..d9b481fa87 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -36,42 +36,49 @@ MANIFEST_TEMPLATE: str = """ """ +# Disallow git prompting for a username/password +os.environ['GIT_TERMINAL_PROMPT'] = '0' def git(*args, repository_path='.'): - return subprocess.check_output(["git"] + list(args), cwd=repository_path, - ).decode() + return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() -def get_repository_sha_in_namespace(module: str, namespace: str, branches: List[str]) -> str: - print(branches) - res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', *branches) +def get_branches_info(module: str, namespace: str, branches: List[str]) -> Tuple[str, str]: + try: + res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', *branches) + except subprocess.CalledProcessError: + return None, None + if not res: - return None + return None, None + lines = res.split('\n') for branch in branches: - for line in res.split('\n'): + for line in lines: if line.endswith('/' + branch): - return res.split('\t')[0] + try: + sha, refname = line.split('\t') + except ValueError: + continue + return refname.strip(), sha + + return None, None -def find_repository_sha(module: str, branchname: str) -> Tuple[str, str]: +def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]: namespace: str = os.environ["CI_PROJECT_NAMESPACE"] if module == os.environ['CI_PROJECT_NAME']: - return 'user', os.environ['CI_COMMIT_SHA'] + return 'user', branchname, os.environ['CI_COMMIT_SHA'] if branchname != "master": - sha = get_repository_sha_in_namespace(module, namespace, [branchname]) + remote_refname, sha = get_branches_info(module, namespace, [branchname]) if sha is not None: - return 'user', sha - - print(f"Did not find user branch named {branchname}") + return 'user', remote_refname, sha # Check upstream project for a branch - sha = get_repository_sha_in_namespace(module, 'gstreamer', [branchname, 'master']) + remote_refname, sha = get_branches_info(module, 'gstreamer', [branchname, 'master']) if sha is not None: - print("Found mathcing branch in upstream project") - print(f"gstreamer/{branchname}") - return 'origin', sha + return 'origin', remote_refname, sha # This should never occur given the upstream fallback above print(f"Could not find anything for {module}:{branchname}") @@ -112,46 +119,48 @@ def preserve_ci_vars(func): @preserve_ci_vars def test_find_repository_sha(): + os.environ["CI_PROJECT_NAME"] = "some-random-project" os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" os.environ["CI_PROJECT_NAMESPACE"] = "alatiera" del os.environ["READ_PROJECTS_TOKEN"] # This should find the repository in the user namespace - remote, git_ref = find_repository_sha("gst-plugins-good", "1.2") + remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2") assert remote == "user" assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325" + assert refname == "refs/heads/1.2" # This should fallback to upstream master branch since no matching branch was found - remote, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name") + remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name") assert remote == "origin" - - # This should fallback to upstream master branch since no repository was found - remote, git_ref = find_repository_sha("totally-valid-project-name", "1.2") - assert remote == "origin" - # This is now the sha of the last commit - # assert git_ref == "master" + assert refname == "refs/heads/master" os.environ["CI_PROJECT_NAME"] = "the_project" os.environ["CI_COMMIT_SHA"] = "MySha" - remote, git_ref = find_repository_sha("the_project", "whatever") + remote, refname, git_ref = find_repository_sha("the_project", "whatever") assert remote == "user" assert git_ref == "MySha" + assert refname == "whatever" @preserve_ci_vars def test_get_project_branch(): + os.environ["CI_PROJECT_NAME"] = "some-random-project" + os.environ["CI_COMMIT_SHA"] = "dwbuiw" os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_" del os.environ["READ_PROJECTS_TOKEN"] - remote, twelve = find_repository_sha('gst-plugins-good', '1.12') + remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12') assert twelve is not None assert remote == 'origin' + assert refname == "refs/heads/1.12" - remote, fourteen = find_repository_sha('gst-plugins-good', '1.12') + remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14') assert fourteen is not None assert remote == 'origin' + assert refname == "refs/heads/1.14" if __name__ == "__main__": @@ -166,7 +175,7 @@ if __name__ == "__main__": user_remote_url += '/' if options.self_update: - remote, sha = find_repository_sha("gst-ci", current_branch) + remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch) if remote == 'user': remote = user_remote_url + 'gst-ci' else: @@ -177,11 +186,11 @@ if __name__ == "__main__": sys.exit(0) projects: str = '' - project_template: str = " \n" for module in GSTREAMER_MODULES: print(f"Checking {module}:", end=' ') - remote, revision = find_repository_sha(module, current_branch) - projects += project_template % {'name': module, 'remote': remote, 'revision': revision} + remote, refname, revision = find_repository_sha(module, current_branch) + print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'") + projects += f" \n" with open(options.output, mode='w') as manifest: print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest) From 5aad1e149de0c2e7e48433cf37757f7aa84f9f23 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 6 Dec 2018 00:03:25 -0300 Subject: [PATCH 127/605] Run test_manifest when the docker is changed --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee703ba4c1..f804113fa6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,7 @@ test manifest: only: changes: - 'gitlab/build_manifest.py' + - 'docker/test_manifest/*' .base: image: "docker:stable" From 4717e446a9d8b4dc0c2a02ddf0630d7c65d78a37 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 6 Dec 2018 00:00:21 -0300 Subject: [PATCH 128/605] Revert me --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f804113fa6..1fa6bd7e7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ stages: test manifest: variables: GIT_STRATEGY: fetch - image: "registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb" + image: "registry.freedesktop.org/thiblahute/gst-ci/amd64/test-manifest:latest" stage: "preparation" script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py From 5856b05f3010136cebcaa9a90058572e008ae3f6 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 6 Dec 2018 07:23:20 -0300 Subject: [PATCH 129/605] build_manifest: fetch remote refname before checking out right commit Working around https://gitlab.freedesktop.org/freedesktop/freedesktop/issues/95 --- gitlab/build_manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index d9b481fa87..732ca4ab45 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -181,8 +181,8 @@ if __name__ == "__main__": else: remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci" - git('fetch', remote, sha) - git('checkout', '--detach', 'FETCH_HEAD') + git('fetch', remote, remote_refname) + git('checkout', '--detach', sha) sys.exit(0) projects: str = '' From 96c5f89d798e8a773684581d2e2e8fe8ff31d5c6 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 5 Dec 2018 00:06:00 +0200 Subject: [PATCH 130/605] CI: do not run local jobs for master branch They are identical to the normal jobs --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1fa6bd7e7f..5826c358ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -128,6 +128,9 @@ build fedora x86_64 local: extends: '.build' stage: 'build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest" + except: + refs: + - "master@gstreamer/gst-ci" build android 28 local: extends: '.build' @@ -137,3 +140,6 @@ build android 28 local: -Dbad:androidmedia=enabled --cross-file /android_arm64_28.txt image: "${CI_REGISTRY_IMAGE}/amd64/android_28:latest" + except: + refs: + - "master@gstreamer/gst-ci" From bb2c819c775ac47688696821e4d8963a0267992a Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 6 Dec 2018 10:30:42 -0300 Subject: [PATCH 131/605] Revert "Revert me" and fix test_manifest image to be used This reverts commit 4717e446a9d8b4dc0c2a02ddf0630d7c65d78a37. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5826c358ff..afd103262f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ stages: test manifest: variables: GIT_STRATEGY: fetch - image: "registry.freedesktop.org/thiblahute/gst-ci/amd64/test-manifest:latest" + image: "registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:5856b05f3010136cebcaa9a90058572e008ae3f6" stage: "preparation" script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py From 932a9c2137846e0a722a11e99268550a667ee4d9 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sun, 9 Dec 2018 04:34:35 +0200 Subject: [PATCH 132/605] ci_template: Use a cache key specific to each job --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7052006c1b..de01720581 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -85,6 +85,7 @@ gst indent: - rm -rf subprojects/*/.git/ - rm -rf build/subprojects/*/.git/ cache: + key: "${CI_JOB_NAME}" paths: - "${CCACHE_DIR}" artifacts: From f9435d7312cefc0c49f52c99ea564addd227f193 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 13 Nov 2018 20:13:13 -0500 Subject: [PATCH 133/605] docker: Create Cerbero Fedora Docker --- .gitlab-ci.yml | 12 ++++++++++++ docker/cerbero/Dockerfile-fedora | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 docker/cerbero/Dockerfile-fedora diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index afd103262f..0a6e28bda7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,6 +121,18 @@ android 28 amd64 docker: changes: - 'docker/android/*' +cerbero fedora amd64 docker: + stage: "build docker" + variables: + ARCH: "amd64" + TAG: "cerbero-fedora" + CONTEXT_DIR: "docker/cerbero/" + DOCKERFILE: "docker/cerbero/Dockerfile-fedora" + extends: .base + only: + changes: + - 'docker/cerbero/Dockerfile-fedora' + include: "gitlab/ci_template.yml" # Test the build job against the latest build image tag and the local manifest diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora new file mode 100644 index 0000000000..92ea2e7a29 --- /dev/null +++ b/docker/cerbero/Dockerfile-fedora @@ -0,0 +1,19 @@ +FROM fedora:29 + +# System setup +RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ + dnf upgrade -y && \ + dnf install -y sudo git rsync + +# Configure git for various usage +RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ + git config --global user.name "Cerbero Build System" + +RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ + mkdir $HOME/.cerbero && \ + echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \ + echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \ + cd cerbero && \ + ./cerbero-uninstalled bootstrap -y --system-only && \ + cd .. && \ + dnf clean all From b1e3b9e6ab297bd1f1119f7634ae08f1b5918588 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 15 Nov 2018 21:47:34 -0500 Subject: [PATCH 134/605] gitlab: Add cerbero build template and x86_64 build --- .gitlab-ci.yml | 7 +++ gitlab/ci_template.yml | 105 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a6e28bda7..a41266cd94 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,3 +155,10 @@ build android 28 local: except: refs: - "master@gstreamer/gst-ci" + +build cerbero fedora x86_64 local: + extends: '.build cerbero fedora x86_64' + image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + except: + refs: + - "master@gstreamer/gst-ci" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index de01720581..617552c232 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -110,3 +110,108 @@ build android arm64 api28: --cross-file /android_arm64_28.txt before_script: - cat /android_arm64_28.txt + +# Template for Cerbero GStreamer Build +# +# Parameters: +# CONFIG: The name of the configuration file to use +# +# Produces runtime and devel tarball packages. +.cerbero: + stage: "full builds" + dependencies: + - "manifest" + variables: + CCACHE_BASEDIR: "${CI_PROJECT_DIR}" + CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" + CERBERO_HOME: "${CI_PROJECT_DIR}/cerbero-build" + CERBERO_SOURCES: "${CI_PROJECT_DIR}/cerbero-sources" + CERBERO_DEPS: "cerbero-deps.tar.gz" + CERBERO: ./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml + before_script: + # FIXME Wrong namespace + # Workaround build-tools having hardcoded internal path + - mkdir -p /builds/gstreamer + - ln -sf ${CI_PROJECT_DIR} /builds/gstreamer/cerbero + - rsync -aH /cerbero/ . + - mkdir -p ${CERBERO_HOME} + - test -f ${CERBERO_DEPS} && tar -C ${CERBERO_HOME} -xf ${CERBERO_DEPS} + - echo "home_dir = \"${CERBERO_HOME}\"" >> localconf.cbc + - echo "local_sources = \"${CERBERO_SOURCES}\"" >> localconf.cbc + - ./cerbero-uninstalled --self-update manifest.xml + - ccache -z + script: + - $CERBERO show-config + - $CERBERO fetch-bootstrap --build-tools-only + - $CERBERO fetch-package --deps --reset-rdeps --full-reset gstreamer-1.0 + - $CERBERO fetch-cache + - $CERBERO bootstrap --offline --build-tools-only + - $CERBERO package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0 + - ccache -s + cache: + paths: + - "${CCACHE_DIR}" + - "${CERBERO_SOURCES}" + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '2 days' + when: 'always' + paths: + - "manifest.xml" + - "cerbero-build/logs" + - "*.tar.bz2" + dependencies: + - "manifest" + +# Template for Cerbero GStreamer Deps +# +# This template is used by cerbero/ project to pre-built the GStreamer +# depedencies. When available, the .cerbero jobs will download this artifact +# in order to speed up the build. +# +# Parameters: +# CONFIG: The name of the configuration file to use +# ARCH: The architecture name (Cerbero naming) +# +# Produce an artifact with the dist/ and .cache for the ARCH along +# with the associated build-tools. +.cerbero deps: + extends: .cerbero + stage: "build" + script: + - $CERBERO show-config + - $CERBERO fetch-bootstrap --build-tools-only + - $CERBERO fetch-package --deps gstreamer-1.0 + - $CERBERO bootstrap --offline --build-tools-only + - $CERBERO build-deps --offline + gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 + gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 + gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice + - $CERBERO fetch-cache --skip-fetch --job-id=${CI_JOB_ID} + - tar -C ${CERBERO_HOME} -czf $CERBERO_DEPS + build-tools build-tools.cache + dist/${ARCH} ${ARCH}.cache + - ccache -s + only: + variables: + - $CI_PROJECT_NAME == "cerbero" + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '10 days' + when: 'always' + paths: + - "manifest.xml" + - "cerbero-build/logs" + - "cerbero-build/cerbero-deps.log" + - "${CERBERO_DEPS}" + +.build cerbero fedora x86_64: + extends: '.cerbero' + variables: + CONFIG: "linux.config" + +# FIXME uncomment and set the SHA when docker is ready +# This indirection allow overriding the image without passing again CONFIG and ARCH +#build cerbero fedora x86_64: +# image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:SHA' +# extends: '.build cerbero fedora x86_64' From d7322403f2ed8df3045c807bbbc74e039105bd16 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 21 Nov 2018 23:05:16 -0500 Subject: [PATCH 135/605] gitlab: Add orc and cerbero to manifest --- gitlab/build_manifest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 732ca4ab45..fcce1b7f3b 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -11,7 +11,8 @@ from urllib.parse import urlparse # from pprint import pprint GSTREAMER_MODULES: List[str] = [ - # 'orc', + 'orc', + 'cerbero', 'gst-build', 'gstreamer', 'gst-plugins-base', From e2595530ad3552f4daaf74ce2c434b7a7b7b3ef1 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Fri, 23 Nov 2018 22:52:59 -0500 Subject: [PATCH 136/605] gitlab: Don't build gst-build in cerbero repository --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 617552c232..f16d45b490 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -57,6 +57,7 @@ gst indent: variables: # No point on trying to format C files in those repositories - $CI_PROJECT_NAME == "gstreamer-sharp" + - $CI_PROJECT_NAME == "cerbero" .build: stage: 'full builds' @@ -94,6 +95,9 @@ gst indent: paths: - "manifest.xml" # - "gst-build/" + except: + variables: + - $CI_PROJECT_NAME == "cerbero" build fedora x86_64: extends: '.build' From fcbfb3dc68f22f962e25ea8361a535f707af13ba Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sat, 8 Dec 2018 21:26:25 -0500 Subject: [PATCH 137/605] gitlab: Use cache key for cerbero builds The key is set to cerbero_${ARCH}. This should help maintain a smaller cache size and will avoid other jobs from overriding our cerbero-sources cache. --- gitlab/ci_template.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f16d45b490..e8d6d74890 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -119,6 +119,7 @@ build android arm64 api28: # # Parameters: # CONFIG: The name of the configuration file to use +# ARCH: The cerbero _ (used in cache key) # # Produces runtime and devel tarball packages. .cerbero: @@ -153,6 +154,7 @@ build android arm64 api28: - $CERBERO package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0 - ccache -s cache: + key: "cerbero_${ARCH}" paths: - "${CCACHE_DIR}" - "${CERBERO_SOURCES}" @@ -213,6 +215,7 @@ build android arm64 api28: extends: '.cerbero' variables: CONFIG: "linux.config" + ARCH: "linux_x86_64" # FIXME uncomment and set the SHA when docker is ready # This indirection allow overriding the image without passing again CONFIG and ARCH From d19082b72667fb3382bdc3621520c4d26e258b2e Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sat, 8 Dec 2018 21:35:26 -0500 Subject: [PATCH 138/605] gitlab: Limit cerbero ccache to 1.7G Testing showed that we need 1.6G to store all the objects. This limit will prevent the cache from growing over time. Compressing the cache can take a lot of time. --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e8d6d74890..0a017b0ac2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -129,6 +129,7 @@ build android arm64 api28: variables: CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" + CCACHE_MAXSIZE: "1.7G" CERBERO_HOME: "${CI_PROJECT_DIR}/cerbero-build" CERBERO_SOURCES: "${CI_PROJECT_DIR}/cerbero-sources" CERBERO_DEPS: "cerbero-deps.tar.gz" From 4291812f757b30b298d761f787630ebf16830c36 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sun, 9 Dec 2018 15:11:35 -0500 Subject: [PATCH 139/605] CI: Add cerbero docker SHA No job activated yet. --- gitlab/ci_template.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0a017b0ac2..362644c4a8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -218,8 +218,15 @@ build android arm64 api28: CONFIG: "linux.config" ARCH: "linux_x86_64" +.cerbero deps fedora x86_64: + extends: '.cerbero deps' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:d19082b72667fb3382bdc3621520c4d26e258b2e' + variables: + CONFIG: "linux.config" + ARCH: "linux_x86_64" + # FIXME uncomment and set the SHA when docker is ready # This indirection allow overriding the image without passing again CONFIG and ARCH #build cerbero fedora x86_64: -# image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:SHA' +# image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:d19082b72667fb3382bdc3621520c4d26e258b2e' # extends: '.build cerbero fedora x86_64' From 00b67289b1aec1b356346f60d98904fc2d9976ae Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sun, 9 Dec 2018 16:29:46 -0500 Subject: [PATCH 140/605] CI: Place cerbero docker image: in the template This avoids having to duplicate that path into cerbero/ project and will ease later updates. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 362644c4a8..8c8046db8a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -214,6 +214,7 @@ build android arm64 api28: .build cerbero fedora x86_64: extends: '.cerbero' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:d19082b72667fb3382bdc3621520c4d26e258b2e' variables: CONFIG: "linux.config" ARCH: "linux_x86_64" @@ -228,5 +229,4 @@ build android arm64 api28: # FIXME uncomment and set the SHA when docker is ready # This indirection allow overriding the image without passing again CONFIG and ARCH #build cerbero fedora x86_64: -# image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:d19082b72667fb3382bdc3621520c4d26e258b2e' # extends: '.build cerbero fedora x86_64' From d399288f1566ea1acbf419acfe4da87019e58878 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sun, 9 Dec 2018 18:19:06 -0500 Subject: [PATCH 141/605] CI: Enable cerbero builds for all projects --- gitlab/ci_template.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8c8046db8a..e55ace47ca 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -226,7 +226,6 @@ build android arm64 api28: CONFIG: "linux.config" ARCH: "linux_x86_64" -# FIXME uncomment and set the SHA when docker is ready # This indirection allow overriding the image without passing again CONFIG and ARCH -#build cerbero fedora x86_64: -# extends: '.build cerbero fedora x86_64' +build cerbero fedora x86_64: + extends: '.build cerbero fedora x86_64' From 830b24df3e692859fd96e1496efaf992e3397ae4 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 10 Dec 2018 11:50:49 +0200 Subject: [PATCH 142/605] ci_template: Remove duplicate key Looks like this slipped past the review of !47 --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e55ace47ca..c204144cb6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -167,8 +167,6 @@ build android arm64 api28: - "manifest.xml" - "cerbero-build/logs" - "*.tar.bz2" - dependencies: - - "manifest" # Template for Cerbero GStreamer Deps # From c9ee22d4c3dcf077e1650bdc8c918d133d601768 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 21 Nov 2018 15:49:51 +0200 Subject: [PATCH 143/605] Revert "ci_template.yml: Temporarly disable builddir artifact" This reverts commit 6ddb35c3c400586520b0f1e39f6f5369d0f61b44. A hotfix has been deployed, lets see how it goes. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c204144cb6..909cfc7c2e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -94,7 +94,7 @@ gst indent: when: always paths: - "manifest.xml" - # - "gst-build/" + - "gst-build/" except: variables: - $CI_PROJECT_NAME == "cerbero" From 7151af0270d8199bc64ec2c2875ccbbbcb922e31 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 21 Nov 2018 17:24:00 +0200 Subject: [PATCH 144/605] ci_template.yml: Run the check test suite!!! Add a fedora job that runs the 'check' tests with gst-validate-launcher. Its fairly well abstracted so the same template can be used to add the rest of the test-suites fairly soon. --- .gitlab-ci.yml | 14 ++++++++++++++ gitlab/ci_template.yml | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a41266cd94..700f1458ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,3 +162,17 @@ build cerbero fedora x86_64 local: except: refs: - "master@gstreamer/gst-ci" + +.test fedora x86_64 local: + image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest' + extends: '.test' + dependencies: + - build fedora x86_64 local + except: + refs: + - "master@gstreamer/gst-ci" + +check fedora local: + extends: '.test fedora x86_64 local' + variables: + TEST_SUITE: 'check.gst*' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 909cfc7c2e..377e36c9d4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -68,7 +68,7 @@ gst indent: CXX: "ccache g++" CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" - MAIN_DIR: "${CI_PROJECT_DIR}" + MAIN_DIR: "${CI_PROJECT_DIR}/validate-output/" MESON_ARGS: "${DEFAULT_MESON_ARGS}" script: @@ -115,6 +115,46 @@ build android arm64 api28: before_script: - cat /android_arm64_28.txt +.test: + stage: 'test' + variables: + MAIN_DIR: "${CI_PROJECT_DIR}/validate-output/" + # Disable colored output to avoid weird rendering issues + GST_DEBUG_COLOR: 'no' + script: + - cd gst-build/ + - > + ./gst-uninstalled.py + gst-validate-launcher ${TEST_SUITE} + -fs + -j=1 + --mute + --shuffle + --no-display + --meson-no-rebuild + -M "${CI_PROJECT_DIR}/validate-output/" + --xunit-file "${CI_PROJECT_DIR}/validate-output/logs/xunit.xml" + artifacts: + expire_in: 2 days + when: always + paths: + - 'gst-build/build/meson-logs/' + - 'validate-output/logs' + reports: + junit: + - "validate-output/logs/*.xml" + +.test fedora x86_64: + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + extends: '.test' + dependencies: + - build fedora x86_64 + +check fedora: + extends: '.test fedora x86_64' + variables: + TEST_SUITE: "check.gst*" + # Template for Cerbero GStreamer Build # # Parameters: From 830507f2021330c403c03afe234715be0ca83e19 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 6 Dec 2018 22:20:14 +0200 Subject: [PATCH 145/605] ci_template: Blacklist problematic tests --- gitlab/ci_template.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 377e36c9d4..b58885bde6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -121,6 +121,21 @@ build android arm64 api28: MAIN_DIR: "${CI_PROJECT_DIR}/validate-output/" # Disable colored output to avoid weird rendering issues GST_DEBUG_COLOR: 'no' + + # note the -b at the start of each line + # Can't comment inline sadly + # FIXME: get this into gst-validate-launcher KNOWN_ISSUES + # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 + # h264parse: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/842 + # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 + # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 + # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 + BLACKLIST: > + -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream + -b check.gst-plugins-bad.elements_h264parse.test_parse_packetized + -b check.gst-plugins-bad.elements_dtls.test_data_transfer + -b check.gst-plugins-bad.elements_dtls.test_create_and_unref + -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp script: - cd gst-build/ - > @@ -134,6 +149,7 @@ build android arm64 api28: --meson-no-rebuild -M "${CI_PROJECT_DIR}/validate-output/" --xunit-file "${CI_PROJECT_DIR}/validate-output/logs/xunit.xml" + ${BLACKLIST} artifacts: expire_in: 2 days when: always From a529757decdf5416c828c28291898d8c23235129 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 10 Dec 2018 20:35:29 +0200 Subject: [PATCH 146/605] ci_template: blacklist a problematic test in check Also pass --dump-on-failure in the job. While the logs are artifacts and they are integrated in the MRs, its not as convinient to findout out what is failling if you are looking into just a job log, of a commit. Thus make it show logs are also priting on stdout/stderr. --- gitlab/ci_template.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b58885bde6..62925ebfa8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -125,12 +125,14 @@ build android arm64 api28: # note the -b at the start of each line # Can't comment inline sadly # FIXME: get this into gst-validate-launcher KNOWN_ISSUES + # check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 # h264parse: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/842 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 BLACKLIST: > + -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream -b check.gst-plugins-bad.elements_h264parse.test_parse_packetized -b check.gst-plugins-bad.elements_dtls.test_data_transfer @@ -146,6 +148,7 @@ build android arm64 api28: --mute --shuffle --no-display + --dump-on-failure --meson-no-rebuild -M "${CI_PROJECT_DIR}/validate-output/" --xunit-file "${CI_PROJECT_DIR}/validate-output/logs/xunit.xml" From d25392f3e10e208734ea1dfaf50bae9c38f00ce3 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 10 Dec 2018 21:35:38 +0200 Subject: [PATCH 147/605] ci_template: Report ccache stats for build jobs --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 62925ebfa8..3f2b0f81a4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -72,12 +72,14 @@ gst indent: MESON_ARGS: "${DEFAULT_MESON_ARGS}" script: + - ccache -z # Not sure why, but permission errors else # https://gitlab.freedesktop.org/alatiera/gstreamer/-/jobs/41441 - cp -r /gst-build/ . && cd gst-build - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" - meson build/ $MESON_ARGS - ninja -C build/ + - ccache -s after_script: - cd gst-build/ # Clean the artifacts packages to avoid copying "useless" build products. From 5eb652177c6bb019133e24b9f70994062c08b0bd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 11 Dec 2018 13:25:37 +0200 Subject: [PATCH 148/605] ci_template: do not .test in the cerbero repo We already disable the .build inhereted build jobs, and thus the tests fail to run since they depend on them. Cerbero doesn't need to run them anyway. --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 3f2b0f81a4..a6ef6c7ff3 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -164,6 +164,10 @@ build android arm64 api28: reports: junit: - "validate-output/logs/*.xml" + # We disable the .build above, which this job usually depends upon for cerbero + except: + variables: + - $CI_PROJECT_NAME == "cerbero" .test fedora x86_64: image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' From f32f70d79bfd0dca52dbd40feb23175fcac966e5 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 11 Dec 2018 20:33:18 +0200 Subject: [PATCH 149/605] ci_template: Blacklist another test in check gst-plugins-base.elements_multisocketsink.test_client_next_keyframe is occasionally failing somehow in the CI. For more see: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 --- gitlab/ci_template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a6ef6c7ff3..31cccac9bf 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -127,7 +127,8 @@ build android arm64 api28: # note the -b at the start of each line # Can't comment inline sadly # FIXME: get this into gst-validate-launcher KNOWN_ISSUES - # check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 + # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 + # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 # h264parse: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/842 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 @@ -135,6 +136,7 @@ build android arm64 api28: # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories + -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream -b check.gst-plugins-bad.elements_h264parse.test_parse_packetized -b check.gst-plugins-bad.elements_dtls.test_data_transfer From 1c68f94c68533fe1b53d6225a15dfd88b7b2b5f6 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 11 Dec 2018 22:57:46 +0200 Subject: [PATCH 150/605] ci_template: another blacklisted test in check gst-plugins-good.elements_rtpbin.test_sender_eos is occasionally failing. For more see: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 31cccac9bf..5e4def9ff4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -129,6 +129,7 @@ build android arm64 api28: # FIXME: get this into gst-validate-launcher KNOWN_ISSUES # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 + # rtpbin.test_sender_eos: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 # h264parse: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/842 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 @@ -138,6 +139,7 @@ build android arm64 api28: -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream + -b check.gst-plugins-good.elements_rtpbin.test_sender_eos -b check.gst-plugins-bad.elements_h264parse.test_parse_packetized -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref From c8c9cad6855d3d77379bd01c849b74e854688ba9 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 10 Dec 2018 17:35:57 +0200 Subject: [PATCH 151/605] CI: Make the docker build and local jobs manual While they are very useful, each time we create a branch, gitlab tries to build all the images which is very resource intesinve. Thus make all the local images and everything that depends upon them a manual job and only trigger them before merging an MR. --- .gitlab-ci.yml | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 700f1458ae..4d67da0d10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,15 +20,12 @@ test manifest: reports: junit: - "junit.xml" - only: - changes: - - 'gitlab/build_manifest.py' - - 'docker/test_manifest/*' .base: image: "docker:stable" services: - docker:dind + when: 'manual' variables: # When using dind, it's wise to use the overlayfs driver for # improved performance. @@ -67,10 +64,6 @@ alpine amd64 manifest builder docker: CONTEXT_DIR: "docker/build_manifest/" DOCKERFILE: "docker/build_manifest/Dockerfile" extends: .base - only: - changes: - - 'docker/build_manifest/*' - - 'gitlab/build_manifest.py' test manifest amd64 docker: stage: "build docker" @@ -80,9 +73,6 @@ test manifest amd64 docker: CONTEXT_DIR: "docker/test_manifest/" DOCKERFILE: "docker/test_manifest/Dockerfile" extends: .base - only: - changes: - - 'docker/test_manifest/*' gst-indent amd64 docker: stage: "build docker" @@ -92,9 +82,6 @@ gst-indent amd64 docker: CONTEXT_DIR: "docker/indent/" DOCKERFILE: "docker/indent/Dockerfile" extends: .base - only: - changes: - - 'docker/indent/*' fedora amd64 docker: stage: "build docker" @@ -104,9 +91,6 @@ fedora amd64 docker: CONTEXT_DIR: "docker/fedora/" DOCKERFILE: "docker/fedora/Dockerfile" extends: .base - only: - changes: - - 'docker/fedora/*' # Cross compile image android 28 amd64 docker: @@ -117,9 +101,6 @@ android 28 amd64 docker: CONTEXT_DIR: "docker/android/" DOCKERFILE: "docker/android/Dockerfile" extends: .base - only: - changes: - - 'docker/android/*' cerbero fedora amd64 docker: stage: "build docker" @@ -129,15 +110,16 @@ cerbero fedora amd64 docker: CONTEXT_DIR: "docker/cerbero/" DOCKERFILE: "docker/cerbero/Dockerfile-fedora" extends: .base - only: - changes: - - 'docker/cerbero/Dockerfile-fedora' include: "gitlab/ci_template.yml" +.build local: + extends: '.build' + when: 'manual' + # Test the build job against the latest build image tag and the local manifest build fedora x86_64 local: - extends: '.build' + extends: '.build local' stage: 'build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest" except: @@ -145,7 +127,7 @@ build fedora x86_64 local: - "master@gstreamer/gst-ci" build android 28 local: - extends: '.build' + extends: '.build local' variables: MESON_ARGS: > -Dbad=enabled @@ -159,6 +141,7 @@ build android 28 local: build cerbero fedora x86_64 local: extends: '.build cerbero fedora x86_64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + when: 'manual' except: refs: - "master@gstreamer/gst-ci" @@ -168,6 +151,7 @@ build cerbero fedora x86_64 local: extends: '.test' dependencies: - build fedora x86_64 local + when: 'manual' except: refs: - "master@gstreamer/gst-ci" From 5a3cf8360c670ca0e3785a0760089741808deefe Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sun, 9 Dec 2018 19:42:21 -0500 Subject: [PATCH 152/605] CI: Don't reset cache on fetch This is not needed since we only do run without cache or from a deps cache for the specific cerbero commit we are building. This will speed up builds in cerbero/ project when the source cache is missing. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 5e4def9ff4..925bc890a6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -218,7 +218,7 @@ check fedora: script: - $CERBERO show-config - $CERBERO fetch-bootstrap --build-tools-only - - $CERBERO fetch-package --deps --reset-rdeps --full-reset gstreamer-1.0 + - $CERBERO fetch-package --deps gstreamer-1.0 - $CERBERO fetch-cache - $CERBERO bootstrap --offline --build-tools-only - $CERBERO package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0 From 39f535986b5d99369fc0150bd9097f72ec250901 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 13 Dec 2018 17:18:40 +0200 Subject: [PATCH 153/605] ci_template: enable h264parse.test_parse_packetized test again Remove check.gst-plugins-bad.elements_h264parse.test_parse_packetized from the blacklist since the relevant issue [1] seems to have been resolved with the following MR [2] [1] https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/842 [2] https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/merge_requests/69 --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 925bc890a6..41e951b65b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -131,7 +131,6 @@ build android arm64 api28: # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 # rtpbin.test_sender_eos: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 - # h264parse: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/842 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 @@ -140,7 +139,6 @@ build android arm64 api28: -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream -b check.gst-plugins-good.elements_rtpbin.test_sender_eos - -b check.gst-plugins-bad.elements_h264parse.test_parse_packetized -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp From f7331ab4f87206087db5b33b95168a558a60a577 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 29 Nov 2018 15:01:55 -0500 Subject: [PATCH 154/605] Move Android NDK to /opt It was installed in '/' which doesn't feel right. Installing it in /root is also not correct because we want to run the build as user instead of root in the future and cleanup.sh removes everything in /root. /opt seems the best place because that's also the default location when installing Android Studio. --- .gitlab-ci.yml | 2 +- docker/android/android-create-toolchain.sh | 6 +++--- docker/android/android-download-ndk.sh | 2 +- docker/android/prepare.sh | 5 +++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d67da0d10..57c6eab1c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -132,7 +132,7 @@ build android 28 local: MESON_ARGS: > -Dbad=enabled -Dbad:androidmedia=enabled - --cross-file /android_arm64_28.txt + --cross-file /opt/android_arm64_28.txt image: "${CI_REGISTRY_IMAGE}/amd64/android_28:latest" except: refs: diff --git a/docker/android/android-create-toolchain.sh b/docker/android/android-create-toolchain.sh index d15287d5b9..f57711c304 100755 --- a/docker/android/android-create-toolchain.sh +++ b/docker/android/android-create-toolchain.sh @@ -23,13 +23,13 @@ set -eu arch=$1 api=$2 -toolchain_path=/android-toolchain-$arch-$api +toolchain_path=/opt/android-toolchain-$arch-$api # Create standalone toolchains -/android-ndk/build/tools/make_standalone_toolchain.py --arch $arch --api $api --install-dir $toolchain_path +/opt/android-ndk/build/tools/make_standalone_toolchain.py --arch $arch --api $api --install-dir $toolchain_path # Create a cross file that can be passed to meson -cat > android_${arch}_${api}.txt <<- EOM +cat > /opt/android_${arch}_${api}.txt <<- EOM [host_machine] system = 'android' cpu_family = 'aarch64' diff --git a/docker/android/android-download-ndk.sh b/docker/android/android-download-ndk.sh index 487b58315e..566543efb2 100755 --- a/docker/android/android-download-ndk.sh +++ b/docker/android/android-download-ndk.sh @@ -28,4 +28,4 @@ wget --quiet https://dl.google.com/android/repository/android-ndk-$ANDROID_NDK_V echo "$ANDROID_NDK_SHA512 android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip" | sha512sum -c unzip android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip rm android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip -mv android-ndk-$ANDROID_NDK_VERSION /android-ndk +mv android-ndk-$ANDROID_NDK_VERSION /opt/android-ndk diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh index cf6a726282..5559ede629 100644 --- a/docker/android/prepare.sh +++ b/docker/android/prepare.sh @@ -33,8 +33,9 @@ git config --global user.email "gst-build@gstreamer.net" git config --global user.name "Gstbuild Runner" # Setup Android toolchain -./root/android-download-ndk.sh -./root/android-create-toolchain.sh arm64 28 +/root/android-download-ndk.sh +/root/android-create-toolchain.sh arm64 28 +rm -rf /opt/android-ndk # get gst-build and make all subprojects available git clone git://anongit.freedesktop.org/gstreamer/gst-build '/gst-build/' From 791757de6c23a41e4decb6921576411f1e385a54 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 29 Nov 2018 15:12:51 -0500 Subject: [PATCH 155/605] Rename 'android' docker image back to 'ubuntu' There is no point in doing a different docker image for each build we are going to do inside an Ubuntu distro. We can later use the same image for native ubuntu builds, or other cross builds (e.g. android API 21, etc). --- .gitlab-ci.yml | 9 ++++----- docker/{android => ubuntu}/Dockerfile | 0 docker/{android => ubuntu}/android-create-toolchain.sh | 0 docker/{android => ubuntu}/android-download-ndk.sh | 0 docker/{android => ubuntu}/cleanup.sh | 0 docker/{android => ubuntu}/prepare.sh | 0 6 files changed, 4 insertions(+), 5 deletions(-) rename docker/{android => ubuntu}/Dockerfile (100%) rename docker/{android => ubuntu}/android-create-toolchain.sh (100%) rename docker/{android => ubuntu}/android-download-ndk.sh (100%) rename docker/{android => ubuntu}/cleanup.sh (100%) rename docker/{android => ubuntu}/prepare.sh (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57c6eab1c3..59863018c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,14 +92,13 @@ fedora amd64 docker: DOCKERFILE: "docker/fedora/Dockerfile" extends: .base -# Cross compile image -android 28 amd64 docker: +ubuntu amd64 docker: stage: "build docker" variables: ARCH: "amd64" - TAG: "android_28" - CONTEXT_DIR: "docker/android/" - DOCKERFILE: "docker/android/Dockerfile" + TAG: "ubuntu" + CONTEXT_DIR: "docker/ubuntu/" + DOCKERFILE: "docker/ubuntu/Dockerfile" extends: .base cerbero fedora amd64 docker: diff --git a/docker/android/Dockerfile b/docker/ubuntu/Dockerfile similarity index 100% rename from docker/android/Dockerfile rename to docker/ubuntu/Dockerfile diff --git a/docker/android/android-create-toolchain.sh b/docker/ubuntu/android-create-toolchain.sh similarity index 100% rename from docker/android/android-create-toolchain.sh rename to docker/ubuntu/android-create-toolchain.sh diff --git a/docker/android/android-download-ndk.sh b/docker/ubuntu/android-download-ndk.sh similarity index 100% rename from docker/android/android-download-ndk.sh rename to docker/ubuntu/android-download-ndk.sh diff --git a/docker/android/cleanup.sh b/docker/ubuntu/cleanup.sh similarity index 100% rename from docker/android/cleanup.sh rename to docker/ubuntu/cleanup.sh diff --git a/docker/android/prepare.sh b/docker/ubuntu/prepare.sh similarity index 100% rename from docker/android/prepare.sh rename to docker/ubuntu/prepare.sh From aa13631e38cb4e1e4ee1c869aa523952dcd8cca7 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 13 Dec 2018 09:45:58 -0500 Subject: [PATCH 156/605] android: Use cross file shipped by gst-build We want to test that cross files shipped by gst-build are working, instead of a generating our own. --- .gitlab-ci.yml | 4 ++-- docker/ubuntu/android-create-toolchain.sh | 24 +---------------------- docker/ubuntu/prepare.sh | 2 +- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 59863018c7..0d535ac697 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -131,8 +131,8 @@ build android 28 local: MESON_ARGS: > -Dbad=enabled -Dbad:androidmedia=enabled - --cross-file /opt/android_arm64_28.txt - image: "${CI_REGISTRY_IMAGE}/amd64/android_28:latest" + --cross-file cross-files/android_arm64_api28.txt + image: "${CI_REGISTRY_IMAGE}/amd64/ubuntu:latest" except: refs: - "master@gstreamer/gst-ci" diff --git a/docker/ubuntu/android-create-toolchain.sh b/docker/ubuntu/android-create-toolchain.sh index f57711c304..996dfe231d 100755 --- a/docker/ubuntu/android-create-toolchain.sh +++ b/docker/ubuntu/android-create-toolchain.sh @@ -23,27 +23,5 @@ set -eu arch=$1 api=$2 -toolchain_path=/opt/android-toolchain-$arch-$api - -# Create standalone toolchains +toolchain_path=/opt/android-$arch-api$api /opt/android-ndk/build/tools/make_standalone_toolchain.py --arch $arch --api $api --install-dir $toolchain_path - -# Create a cross file that can be passed to meson -cat > /opt/android_${arch}_${api}.txt <<- EOM -[host_machine] -system = 'android' -cpu_family = 'aarch64' -cpu = '${arch}' -endian = 'little' - -[properties] -c_args = ['--sysroot', '${toolchain_path}/sysroot'] -cpp_args = ['--sysroot', '${toolchain_path}/sysroot'] - -[binaries] -c = '${toolchain_path}/bin/aarch64-linux-android-clang' -cpp = '${toolchain_path}/bin/aarch64-linux-android-clang++' -ar = '${toolchain_path}/bin/aarch64-linux-android-ar' -strip = '${toolchain_path}/bin/aarch64-linux-android-strip' -pkgconfig = 'false' -EOM diff --git a/docker/ubuntu/prepare.sh b/docker/ubuntu/prepare.sh index 5559ede629..c5b74a040b 100644 --- a/docker/ubuntu/prepare.sh +++ b/docker/ubuntu/prepare.sh @@ -39,4 +39,4 @@ rm -rf /opt/android-ndk # get gst-build and make all subprojects available git clone git://anongit.freedesktop.org/gstreamer/gst-build '/gst-build/' -cd '/gst-build/' && meson build/ && rm -rf build/ \ No newline at end of file +meson subprojects download --sourcedir /gst-build From 6542818ffb8018b8719ecc1b16aee225576742e6 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 13 Dec 2018 13:37:38 -0500 Subject: [PATCH 157/605] android: Use the new docker image --- gitlab/ci_template.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 41e951b65b..333e4ffb6f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -108,14 +108,12 @@ build fedora x86_64: build android arm64 api28: extends: '.build' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android_28:f085d34a32cb8cac95cbd420344efeaa647d08d5' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:aa13631e38cb4e1e4ee1c869aa523952dcd8cca7' variables: MESON_ARGS: > -Dbad=enabled -Dbad:androidmedia=enabled - --cross-file /android_arm64_28.txt - before_script: - - cat /android_arm64_28.txt + --cross-file cross-files/android_arm64_api28.txt .test: stage: 'test' From c2c72edc9b5732cd598fb222a880a79c6d17a20a Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 13 Dec 2018 21:18:40 +0200 Subject: [PATCH 158/605] ci_template.yml: blacklist another test in check check.gst-plugins-good.elements_flvmux.test_incrementing_timestamps occasianlly fails. https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 333e4ffb6f..fac41cad17 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -127,6 +127,7 @@ build android arm64 api28: # FIXME: get this into gst-validate-launcher KNOWN_ISSUES # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 + # flvmux.test_incrementing_timestamps: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 # rtpbin.test_sender_eos: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 @@ -137,6 +138,7 @@ build android arm64 api28: -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream -b check.gst-plugins-good.elements_rtpbin.test_sender_eos + -b check.gst-plugins-good.elements_flvmux.test_incrementing_timestamps -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp From 7f03e972eec5e94c66467b1e45e2ba9761f7bcc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 14 Dec 2018 11:59:48 +0200 Subject: [PATCH 159/605] ci_template.yml: blacklist another test in check gstpad.test_pad_probe_block_add_remove occasionally fails. https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/339 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index fac41cad17..3e2d69b96b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -133,6 +133,7 @@ build android arm64 api28: # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 + # gstpad.test_pad_probe_block_add_remove: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/339 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe @@ -142,6 +143,7 @@ build android arm64 api28: -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp + -b check.gstreamer.gst_gstpad.test_pad_probe_block_add_remove script: - cd gst-build/ - > From 17e25299e8dfefc99467aa7c17ae4d2eab250967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 14 Dec 2018 12:34:21 +0200 Subject: [PATCH 160/605] ci_template.yml: blacklist two more tests in check check.gst-plugins-bad.elements_shm.test_shm_live check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async occasionally fail. https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 3e2d69b96b..f2e50cba8c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -134,6 +134,8 @@ build android arm64 api28: # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 # gstpad.test_pad_probe_block_add_remove: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/339 + # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 + # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe @@ -144,6 +146,8 @@ build android arm64 api28: -b check.gst-plugins-bad.elements_dtls.test_create_and_unref -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp -b check.gstreamer.gst_gstpad.test_pad_probe_block_add_remove + -b check.gst-plugins-bad.elements_shm.test_shm_live + -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async script: - cd gst-build/ - > From d922144894e136b5237a20094411e15dd03be17e Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 14 Dec 2018 13:33:48 +0200 Subject: [PATCH 161/605] ci_template.yml: blacklist another test in check check.gstreamer.gst_gstpad.test_pad_probe_block_add_remove occasianlly fails. https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/339 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f2e50cba8c..975239d3a2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -125,6 +125,7 @@ build android arm64 api28: # note the -b at the start of each line # Can't comment inline sadly # FIXME: get this into gst-validate-launcher KNOWN_ISSUES + # gst_gstpad.test_pad_probe_block_add_remove: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/339 # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 # flvmux.test_incrementing_timestamps: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 @@ -137,6 +138,7 @@ build android arm64 api28: # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 BLACKLIST: > + -b check.gstreamer.gst_gstpad.test_pad_probe_block_add_remove -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream From 31991e6e0dcf4b2ca292c090073e08148a88916e Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 14 Dec 2018 14:11:17 +0200 Subject: [PATCH 162/605] Revert "ci_template.yml: blacklist another test in check" This reverts commit d922144894e136b5237a20094411e15dd03be17e. This was duplicate of 7f03e972eec5e94c66467b1e45e2ba9761f7bcc6 --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 975239d3a2..f2e50cba8c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -125,7 +125,6 @@ build android arm64 api28: # note the -b at the start of each line # Can't comment inline sadly # FIXME: get this into gst-validate-launcher KNOWN_ISSUES - # gst_gstpad.test_pad_probe_block_add_remove: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/339 # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 # flvmux.test_incrementing_timestamps: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 @@ -138,7 +137,6 @@ build android arm64 api28: # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 BLACKLIST: > - -b check.gstreamer.gst_gstpad.test_pad_probe_block_add_remove -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream From 0592481638414e6911dfc589b30ab1089c4528ed Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 14 Dec 2018 14:09:10 +0200 Subject: [PATCH 163/605] ci_template: bump the expiration time of artifacts build job artifacts are the ones that consume much space, and thus only set to 5 days, might lower it if space is an issue. Otherwise rest job artifacts are just logs, text and xml files. --- gitlab/ci_template.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f2e50cba8c..817d080a06 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -30,7 +30,7 @@ manifest: - gitlab/build_manifest.py ${CI_PROJECT_DIR}/manifest.xml - cat ${CI_PROJECT_DIR}/manifest.xml artifacts: - expire_in: "2 days" + expire_in: "7 days" paths: - "manifest.xml" @@ -92,7 +92,7 @@ gst indent: paths: - "${CCACHE_DIR}" artifacts: - expire_in: 6hours + expire_in: '5 days' when: always paths: - "manifest.xml" @@ -164,7 +164,7 @@ build android arm64 api28: --xunit-file "${CI_PROJECT_DIR}/validate-output/logs/xunit.xml" ${BLACKLIST} artifacts: - expire_in: 2 days + expire_in: '14 days' when: always paths: - 'gst-build/build/meson-logs/' @@ -234,7 +234,7 @@ check fedora: - "${CERBERO_SOURCES}" artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" - expire_in: '2 days' + expire_in: '5 days' when: 'always' paths: - "manifest.xml" From da4edd69508f8a5c61c018b3987f75baa86b568b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 14 Dec 2018 20:39:14 +0200 Subject: [PATCH 164/605] ci_template.yml: Unblacklist test in check gstpad.test_pad_probe_block_add_remove does not fail anymore. https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/339 --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 817d080a06..58f7396272 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -133,7 +133,6 @@ build android arm64 api28: # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 - # gstpad.test_pad_probe_block_add_remove: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/339 # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 BLACKLIST: > @@ -145,7 +144,6 @@ build android arm64 api28: -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp - -b check.gstreamer.gst_gstpad.test_pad_probe_block_add_remove -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async script: From 57ec9441d2ea0f6cd4460b2e38cad69502381b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 17 Dec 2018 11:43:34 +0200 Subject: [PATCH 165/605] ci_template.yml: blacklist one more tests in check check.gst-plugins-bad.elements_netsim.netsim_stress occasionally fails. https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 58f7396272..53e519a90c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -135,6 +135,7 @@ build android arm64 api28: # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 + # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe @@ -146,6 +147,7 @@ build android arm64 api28: -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async + -b check.gst-plugins-bad.elements_netsim.netsim_stress script: - cd gst-build/ - > From 8eec47d8fe701a68990fc434127c629b98478e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 18 Dec 2018 11:07:04 +0000 Subject: [PATCH 166/605] ci_template: blacklist flaky gst-devtools.validate_padmonitor.buffer test https://gitlab.freedesktop.org/gstreamer/gst-devtools/issues/34 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 53e519a90c..c72c25d410 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -136,6 +136,7 @@ build android arm64 api28: # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 + # validate_padmonitor.buffer: https://gitlab.freedesktop.org/gstreamer/gst-devtools/issues/34 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe @@ -148,6 +149,7 @@ build android arm64 api28: -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async -b check.gst-plugins-bad.elements_netsim.netsim_stress + -b check.gst-devtools.validate_padmonitor.buffer script: - cd gst-build/ - > From 24811579312cd55ab818836bcaff3ba2c777e4dd Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 17 Dec 2018 14:07:56 -0500 Subject: [PATCH 167/605] docker: cerbero: Add in-docker sources caching As our runner cache in most of the time cold, and is always cold for new contributors, which should greatly speedup the build. --- docker/cerbero/Dockerfile-fedora | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 92ea2e7a29..14adbd0ed7 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -13,7 +13,10 @@ RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ mkdir $HOME/.cerbero && \ echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \ echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \ + echo "local_sources=\"/cerbero/cerbero-sources\"" >> $HOME/.cerbero/cerbero.cbc && \ cd cerbero && \ ./cerbero-uninstalled bootstrap -y --system-only && \ + ./cerbero-uninstalled fetch-bootstrap --build-tools-only && \ + ./cerbero-uninstalled fetch-package gstreamer-1.0 && \ cd .. && \ dnf clean all From ea7a482326eb484491c12e610a31f3af7a1fd242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 18 Dec 2018 18:52:24 +0200 Subject: [PATCH 168/605] ci_template.yml: blacklist one more tests in check check.gst-editing-services.nle_complex.test_one_expandable_another occasionally fails. https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c72c25d410..10c0adaa20 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -137,6 +137,7 @@ build android arm64 api28: # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 # validate_padmonitor.buffer: https://gitlab.freedesktop.org/gstreamer/gst-devtools/issues/34 + # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe @@ -150,6 +151,7 @@ build android arm64 api28: -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async -b check.gst-plugins-bad.elements_netsim.netsim_stress -b check.gst-devtools.validate_padmonitor.buffer + -b check.gst-editing-services.nle_complex.test_one_expandable_another script: - cd gst-build/ - > From 00d2fd776caba8a664fa19c0b3c3d530ec9a31d7 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 18 Dec 2018 14:05:37 -0500 Subject: [PATCH 169/605] ci: Update cerbero docker image This new image contains source cache, which should speed first fork and cold cache cases. --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 10c0adaa20..4c04a3cf75 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -289,14 +289,14 @@ check fedora: .build cerbero fedora x86_64: extends: '.cerbero' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:d19082b72667fb3382bdc3621520c4d26e258b2e' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:24811579312cd55ab818836bcaff3ba2c777e4dd' variables: CONFIG: "linux.config" ARCH: "linux_x86_64" .cerbero deps fedora x86_64: extends: '.cerbero deps' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:d19082b72667fb3382bdc3621520c4d26e258b2e' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:24811579312cd55ab818836bcaff3ba2c777e4dd' variables: CONFIG: "linux.config" ARCH: "linux_x86_64" From 652af108402ae5956b85bf3e43882eb08ca54019 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 3 Jan 2019 19:31:18 +0100 Subject: [PATCH 170/605] gitlab: Stop blacklisting check.gst-devtools.validate_padmonitor.buffer https://gitlab.freedesktop.org/gstreamer/gst-devtools/issues/34 has been closed --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4c04a3cf75..41aacc91e9 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -136,7 +136,6 @@ build android arm64 api28: # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 - # validate_padmonitor.buffer: https://gitlab.freedesktop.org/gstreamer/gst-devtools/issues/34 # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories @@ -150,7 +149,6 @@ build android arm64 api28: -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async -b check.gst-plugins-bad.elements_netsim.netsim_stress - -b check.gst-devtools.validate_padmonitor.buffer -b check.gst-editing-services.nle_complex.test_one_expandable_another script: - cd gst-build/ From 14a4f360f299f8ea9f7091bea3854a05e0e1ae45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 7 Jan 2019 14:33:50 +0200 Subject: [PATCH 171/605] gitlab: Blacklist 4 more tests check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 check.gst-rtsp-server.gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/542 --- gitlab/ci_template.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 41aacc91e9..4bda02f8a4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -137,6 +137,10 @@ build android arm64 api28: # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 + # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 + # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 + # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 + # elements_splitmux.test_splitmuxsrc_sparse_streams: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/542 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe @@ -150,6 +154,10 @@ build android arm64 api28: -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async -b check.gst-plugins-bad.elements_netsim.netsim_stress -b check.gst-editing-services.nle_complex.test_one_expandable_another + -b check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical + -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps + -b check.gst-rtsp-server.gst_rtspclientsink.test_record + -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams script: - cd gst-build/ - > From 5cd4bea651a24ec771907b1531ddf3f73e570f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 7 Jan 2019 14:39:32 +0200 Subject: [PATCH 172/605] gitlab: Unblacklist check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams Was caused by a bug in a MR. --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4bda02f8a4..1726481e69 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -140,7 +140,6 @@ build android arm64 api28: # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 - # elements_splitmux.test_splitmuxsrc_sparse_streams: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/542 BLACKLIST: > -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe @@ -157,7 +156,6 @@ build android arm64 api28: -b check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps -b check.gst-rtsp-server.gst_rtspclientsink.test_record - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams script: - cd gst-build/ - > From b16ec7445ff7874d905eb5759ceaee5f93d29e66 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 17 Dec 2018 21:38:06 -0500 Subject: [PATCH 173/605] docker: cerbero: Add MingW cross toolchains This runs the cross-win32 and cross-win64 bootstrap. --- docker/cerbero/Dockerfile-fedora | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 14adbd0ed7..7f3d3b306b 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -14,9 +14,19 @@ RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \ echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \ echo "local_sources=\"/cerbero/cerbero-sources\"" >> $HOME/.cerbero/cerbero.cbc && \ + echo "home_dir=\"/cerbero/cerbero-build\"" >> $HOME/.cerbero/cerbero.cbc && \ cd cerbero && \ ./cerbero-uninstalled bootstrap -y --system-only && \ - ./cerbero-uninstalled fetch-bootstrap --build-tools-only && \ + ./cerbero-uninstalled fetch-bootstrap && \ ./cerbero-uninstalled fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -c config/cross-win32.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -c config/cross-win32.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -c config/cross-win64.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -c config/cross-win64.cbc bootstrap -y --system-only && \ cd .. && \ + rm /cerbero/cerbero-sources/mingw-w*.tar.xz && \ + rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \ + rm -f /cerbero/cerbero-build/{linux,windows}*.cache && \ dnf clean all From f072577905f9d2997941c85b15bfc17287a20ceb Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 8 Jan 2019 08:13:07 +0200 Subject: [PATCH 174/605] gitlab/ci_template: Factor out the image uris to variables This avoids duplication and makes them easier to update and keep in sync. --- gitlab/ci_template.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1726481e69..7b5fff03fd 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,6 +8,12 @@ stages: - 'full tests' variables: + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:24811579312cd55ab818836bcaff3ba2c777e4dd' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:aa13631e38cb4e1e4ee1c869aa523952dcd8cca7' + INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:f085d34a32cb8cac95cbd420344efeaa647d08d5' + GIT_STRATEGY: none DEFAULT_MESON_ARGS: > --werror @@ -22,7 +28,7 @@ variables: -Dsharp=disabled manifest: - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:f085d34a32cb8cac95cbd420344efeaa647d08d5' + image: $MANIFEST_IMAGE stage: 'preparation' script: - cd /gst-ci @@ -35,7 +41,7 @@ manifest: - "manifest.xml" gst indent: - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + image: $INDENT_IMAGE stage: 'preparation' variables: GIT_STRATEGY: 'fetch' @@ -104,11 +110,11 @@ gst indent: build fedora x86_64: extends: '.build' stage: 'build' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + image: $FEDORA_IMAGE build android arm64 api28: extends: '.build' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:aa13631e38cb4e1e4ee1c869aa523952dcd8cca7' + image: $ANDROID_IMAGE variables: MESON_ARGS: > -Dbad=enabled @@ -186,7 +192,7 @@ build android arm64 api28: - $CI_PROJECT_NAME == "cerbero" .test fedora x86_64: - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + image: $FEDORA_IMAGE extends: '.test' dependencies: - build fedora x86_64 @@ -293,14 +299,14 @@ check fedora: .build cerbero fedora x86_64: extends: '.cerbero' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:24811579312cd55ab818836bcaff3ba2c777e4dd' + image: $CERBERO_IMAGE variables: CONFIG: "linux.config" ARCH: "linux_x86_64" .cerbero deps fedora x86_64: extends: '.cerbero deps' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:24811579312cd55ab818836bcaff3ba2c777e4dd' + image: $CERBERO_IMAGE variables: CONFIG: "linux.config" ARCH: "linux_x86_64" From 6f8e23359a28e60b90a9b641953f6ef6e1d29a4c Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 8 Jan 2019 08:15:29 +0200 Subject: [PATCH 175/605] gitlab/ci_template: update the sha of the images --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7b5fff03fd..a3421c4129 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,8 +8,8 @@ stages: - 'full tests' variables: - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:24811579312cd55ab818836bcaff3ba2c777e4dd' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:b16ec7445ff7874d905eb5759ceaee5f93d29e66' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:b16ec7445ff7874d905eb5759ceaee5f93d29e66' ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:aa13631e38cb4e1e4ee1c869aa523952dcd8cca7' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:f085d34a32cb8cac95cbd420344efeaa647d08d5' From 5a55d5103d0c30aa958e58fef641067f2486f11c Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 17 Dec 2018 21:58:39 -0500 Subject: [PATCH 176/605] CI: Add Win32 and Win64 cross builds --- .gitlab-ci.yml | 16 ++++++++++++++ gitlab/ci_template.yml | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d535ac697..e7709c7a58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -145,6 +145,22 @@ build cerbero fedora x86_64 local: refs: - "master@gstreamer/gst-ci" +build cerbero cross win32 local: + extends: '.build cerbero cross win32' + image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + when: 'manual' + except: + refs: + - "master@gstreamer/gst-ci" + +build cerbero cross win64 local: + extends: '.build cerbero cross win64' + image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + when: 'manual' + except: + refs: + - "master@gstreamer/gst-ci" + .test fedora x86_64 local: image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest' extends: '.test' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a3421c4129..17e934ef95 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -206,11 +206,11 @@ check fedora: # # Parameters: # CONFIG: The name of the configuration file to use -# ARCH: The cerbero _ (used in cache key) # # Produces runtime and devel tarball packages. .cerbero: stage: "full builds" + image: $CERBERO_IMAGE dependencies: - "manifest" variables: @@ -242,7 +242,7 @@ check fedora: - $CERBERO package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0 - ccache -s cache: - key: "cerbero_${ARCH}" + key: "${CI_JOB_NAME}" paths: - "${CCACHE_DIR}" - "${CERBERO_SOURCES}" @@ -263,9 +263,8 @@ check fedora: # # Parameters: # CONFIG: The name of the configuration file to use -# ARCH: The architecture name (Cerbero naming) # -# Produce an artifact with the dist/ and .cache for the ARCH along +# Produce an artifact with the dist/ and .cache along # with the associated build-tools. .cerbero deps: extends: .cerbero @@ -281,8 +280,7 @@ check fedora: gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice - $CERBERO fetch-cache --skip-fetch --job-id=${CI_JOB_ID} - tar -C ${CERBERO_HOME} -czf $CERBERO_DEPS - build-tools build-tools.cache - dist/${ARCH} ${ARCH}.cache + build-tools/ dist/ *.cache - ccache -s only: variables: @@ -297,20 +295,48 @@ check fedora: - "cerbero-build/cerbero-deps.log" - "${CERBERO_DEPS}" +# +# Cerbero Linux X86_64 build +# .build cerbero fedora x86_64: extends: '.cerbero' - image: $CERBERO_IMAGE variables: CONFIG: "linux.config" - ARCH: "linux_x86_64" .cerbero deps fedora x86_64: extends: '.cerbero deps' - image: $CERBERO_IMAGE variables: CONFIG: "linux.config" - ARCH: "linux_x86_64" -# This indirection allow overriding the image without passing again CONFIG and ARCH +# This indirection allow overriding the image without passing again CONFIG build cerbero fedora x86_64: extends: '.build cerbero fedora x86_64' + +# +# Cerbero Cross Windows builds +# +.build cerbero cross win32: + extends: '.cerbero' + variables: + CONFIG: "cross-win32.cbc" + +.cerbero deps cross win32: + extends: '.cerbero deps' + variables: + CONFIG: "cross-win32.cbc" + +build cerbero cross win32: + extends: '.build cerbero cross win32' + +.build cerbero cross win64: + extends: '.cerbero' + variables: + CONFIG: "cross-win64.cbc" + +.cerbero deps cross win64: + extends: '.cerbero deps' + variables: + CONFIG: "cross-win64.cbc" + +build cerbero cross win64: + extends: '.build cerbero cross win64' From 2396e1835849582dff3e65e68dcbac7c859b094a Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 8 Jan 2019 13:22:02 -0500 Subject: [PATCH 177/605] gitlab: Re-add ARCH parameter to cerbero deps jobs This is required and was removed without proper testing. --- gitlab/ci_template.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 17e934ef95..777e0f1a86 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -263,6 +263,7 @@ check fedora: # # Parameters: # CONFIG: The name of the configuration file to use +# ARCH: The cerbero _ (used in cache key) # # Produce an artifact with the dist/ and .cache along # with the associated build-tools. @@ -280,7 +281,8 @@ check fedora: gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice - $CERBERO fetch-cache --skip-fetch --job-id=${CI_JOB_ID} - tar -C ${CERBERO_HOME} -czf $CERBERO_DEPS - build-tools/ dist/ *.cache + build-tools build-tools.cache + dist/${ARCH} ${ARCH}.cache - ccache -s only: variables: @@ -307,6 +309,7 @@ check fedora: extends: '.cerbero deps' variables: CONFIG: "linux.config" + ARCH: "linux_x86_64" # This indirection allow overriding the image without passing again CONFIG build cerbero fedora x86_64: @@ -324,6 +327,7 @@ build cerbero fedora x86_64: extends: '.cerbero deps' variables: CONFIG: "cross-win32.cbc" + ARCH: "windows_x86" build cerbero cross win32: extends: '.build cerbero cross win32' @@ -337,6 +341,7 @@ build cerbero cross win32: extends: '.cerbero deps' variables: CONFIG: "cross-win64.cbc" + ARCH: "windows_x86_64" build cerbero cross win64: extends: '.build cerbero cross win64' From 0530c4e0e7304409f9d093876981e3e8482fd998 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 8 Jan 2019 17:10:19 +0200 Subject: [PATCH 178/605] gitlab/ci_template: Blacklist another flvmux test See https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/543 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 777e0f1a86..6b1c3cb787 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -134,6 +134,7 @@ build android arm64 api28: # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 # flvmux.test_incrementing_timestamps: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 + # flvmux.test_video_caps_late: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/543 # rtpbin.test_sender_eos: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 @@ -152,6 +153,7 @@ build android arm64 api28: -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream -b check.gst-plugins-good.elements_rtpbin.test_sender_eos -b check.gst-plugins-good.elements_flvmux.test_incrementing_timestamps + -b check.gst-plugins-good.elements_flvmux.test_video_caps_late -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp From b22e113e7a06b61f1b67600e3668c796a5e4672f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 8 Jan 2019 22:31:06 +0200 Subject: [PATCH 179/605] gitlab/ci_template: blacklist another splitmux test https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/544 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6b1c3cb787..fc972d0951 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -142,6 +142,7 @@ build android arm64 api28: # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 + # splitmux.test_splitmuxsrc_sparse_streams: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/544 # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 @@ -154,6 +155,7 @@ build android arm64 api28: -b check.gst-plugins-good.elements_rtpbin.test_sender_eos -b check.gst-plugins-good.elements_flvmux.test_incrementing_timestamps -b check.gst-plugins-good.elements_flvmux.test_video_caps_late + -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp From c54fe7bc89d48ece8501d207a10bd13e2696295c Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 10 Jan 2019 10:41:52 +0200 Subject: [PATCH 180/605] gitlab/ci_template: Blacklist more tests pipelines_parse_launch.delayed_link: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/345 gst_gstsystemclock.test_async_sync_interaction: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/346 gst_gstsystemclock.test_periodic_multi: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/347 rtpbin.test_cleanup_recv: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/546 --- gitlab/ci_template.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index fc972d0951..e6b57ff978 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -131,11 +131,15 @@ build android arm64 api28: # note the -b at the start of each line # Can't comment inline sadly # FIXME: get this into gst-validate-launcher KNOWN_ISSUES + # gstreamer.pipelines_parse_launch.delayed_link: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/345 + # gstreamer.gst_gstsystemclock.test_async_sync_interaction: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/346 + # gstreamer.gst_gstsystemclock.test_periodic_multi: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/347 # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 # flvmux.test_incrementing_timestamps: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 # flvmux.test_video_caps_late: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/543 # rtpbin.test_sender_eos: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 + # rtpbin.test_cleanup_recv: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/546 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 @@ -149,10 +153,14 @@ build android arm64 api28: # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 BLACKLIST: > + -b check.gstreamer.pipelines_parse_launch.delayed_link + -b check.gstreamer.gst_gstsystemclock.test_async_sync_interaction + -b check.gstreamer.gst_gstsystemclock.test_periodic_multi -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream -b check.gst-plugins-good.elements_rtpbin.test_sender_eos + -b check.gst-plugins-good.elements_rtpbin.test_cleanup_recv -b check.gst-plugins-good.elements_flvmux.test_incrementing_timestamps -b check.gst-plugins-good.elements_flvmux.test_video_caps_late -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams From eaa7497fa45951f3ddce5a409a954ff457f51c3d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 10 Jan 2019 11:39:13 +0200 Subject: [PATCH 181/605] gitlab/ci_template: blacklist another test camerabin.test_image_video_cycle seems racy: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 --- gitlab/ci_template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e6b57ff978..f989422cc8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -152,6 +152,7 @@ build android arm64 api28: # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 + # camerabin.test_image_video_cycle: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 BLACKLIST: > -b check.gstreamer.pipelines_parse_launch.delayed_link -b check.gstreamer.gst_gstsystemclock.test_async_sync_interaction @@ -166,6 +167,7 @@ build android arm64 api28: -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref + -b check.gst-plugins-bad.elements_camerabin.test_image_video_cycle -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async @@ -235,7 +237,7 @@ check fedora: CERBERO: ./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml before_script: # FIXME Wrong namespace - # Workaround build-tools having hardcoded internal path + # Workaround build-tools having hardcoded internal path - mkdir -p /builds/gstreamer - ln -sf ${CI_PROJECT_DIR} /builds/gstreamer/cerbero - rsync -aH /cerbero/ . From db62f95ba2194f3e4e246fbbaf84bfdf465a7c77 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 10 Jan 2019 16:24:31 -0500 Subject: [PATCH 182/605] gitlab: Remove uneeded mkdir The cerbero_home is already part of the /cerbero folder that we rsync at start. --- gitlab/ci_template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f989422cc8..6526668be5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -241,7 +241,6 @@ check fedora: - mkdir -p /builds/gstreamer - ln -sf ${CI_PROJECT_DIR} /builds/gstreamer/cerbero - rsync -aH /cerbero/ . - - mkdir -p ${CERBERO_HOME} - test -f ${CERBERO_DEPS} && tar -C ${CERBERO_HOME} -xf ${CERBERO_DEPS} - echo "home_dir = \"${CERBERO_HOME}\"" >> localconf.cbc - echo "local_sources = \"${CERBERO_SOURCES}\"" >> localconf.cbc From bd83865236a1cd5b1694adefee24bc1a6b4f1ec4 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 10 Jan 2019 16:30:57 -0500 Subject: [PATCH 183/605] docker: Use a local config for cerbero local_source and home_dir Ohterwise the cerbero.cbc seems to take precendence on the localconf.cbc during builds, which cause caching issues, since data does not get stored at the right place. --- docker/cerbero/Dockerfile-fedora | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 7f3d3b306b..12df38c383 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -13,18 +13,18 @@ RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ mkdir $HOME/.cerbero && \ echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \ echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \ - echo "local_sources=\"/cerbero/cerbero-sources\"" >> $HOME/.cerbero/cerbero.cbc && \ - echo "home_dir=\"/cerbero/cerbero-build\"" >> $HOME/.cerbero/cerbero.cbc && \ cd cerbero && \ - ./cerbero-uninstalled bootstrap -y --system-only && \ - ./cerbero-uninstalled fetch-bootstrap && \ - ./cerbero-uninstalled fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -c config/cross-win32.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -c config/cross-win32.cbc bootstrap -y --system-only && \ - ./cerbero-uninstalled -c config/cross-win64.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -c config/cross-win64.cbc bootstrap -y --system-only && \ + echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \ + echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \ + ./cerbero-uninstalled -c localconf.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -c localconf.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -c localconf.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \ cd .. && \ rm /cerbero/cerbero-sources/mingw-w*.tar.xz && \ rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \ From c021fb83101497a43f1cb185928a7840707c5081 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 10 Jan 2019 22:04:11 -0500 Subject: [PATCH 184/605] gitlab: Update to latest cerbero docker image --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6526668be5..fd52786fb8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,7 +8,7 @@ stages: - 'full tests' variables: - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:b16ec7445ff7874d905eb5759ceaee5f93d29e66' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:bd83865236a1cd5b1694adefee24bc1a6b4f1ec4' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:b16ec7445ff7874d905eb5759ceaee5f93d29e66' ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:aa13631e38cb4e1e4ee1c869aa523952dcd8cca7' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' From dcf237a438eff2e4b8b58e5704aa34e743616e04 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 31 Dec 2018 08:47:02 +0200 Subject: [PATCH 185/605] ci_template: Remove the j = 1 restriction from check Previously we were forcing tests to run with j 1 to avoid random failures/races that occurred due to tests being racy. Majority of those tests have been blacklisted and this should no longer be needed. --- gitlab/ci_template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index fd52786fb8..0523f2ebb9 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -182,7 +182,6 @@ build android arm64 api28: ./gst-uninstalled.py gst-validate-launcher ${TEST_SUITE} -fs - -j=1 --mute --shuffle --no-display From 0dd20f61ea6dfb9cb76dac3ee5200aa3e30704c6 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 8 Jan 2019 18:02:11 +0200 Subject: [PATCH 186/605] gitlab/ci_template: Blacklist some more problematic tests These tests are somewhat hard to reproduce and don't have an issue associated with them yet, but they seem to fail quite often in scenarios where they are run as part of a bigger group of tests See !61 for more --- gitlab/ci_template.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0523f2ebb9..4eb68181d8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -153,17 +153,29 @@ build android arm64 api28: # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 # camerabin.test_image_video_cycle: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 + # + # TO FURTHER INVESTIGATE: + # check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer + # check.gstreamer.gst_gstelement.test_foreach_pad + # check.gstreamer.libs_baseparse.parser_pull_short_read + # check.gstreamer.pipelines_seek.test_loopback_2 + # check.gst-plugins-base.elements_appsrc.test_appsrc_blocked_on_caps BLACKLIST: > -b check.gstreamer.pipelines_parse_launch.delayed_link -b check.gstreamer.gst_gstsystemclock.test_async_sync_interaction -b check.gstreamer.gst_gstsystemclock.test_periodic_multi + -b check.gstreamer.pipelines_seek.test_loopback_2 + -b check.gstreamer.gst_gstelement.test_foreach_pad + -b check.gstreamer.libs_baseparse.parser_pull_short_read -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe + -b check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream -b check.gst-plugins-good.elements_rtpbin.test_sender_eos -b check.gst-plugins-good.elements_rtpbin.test_cleanup_recv -b check.gst-plugins-good.elements_flvmux.test_incrementing_timestamps -b check.gst-plugins-good.elements_flvmux.test_video_caps_late + -b check.gst-plugins-base.elements_appsrc.test_appsrc_blocked_on_caps -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref From f951999359ce894be41be0fc52af27ad7b386dcd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 10 Jan 2019 16:26:03 +0200 Subject: [PATCH 187/605] gitlab/ci_template: blacklist even more tests! gstreamer.gst_gstsystemclock.test_periodic_shot: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/348 splitmux.test_splitmuxsrc_caps_change: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/547 nle_simple.test_simplest: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/57 ges_basic.test_ges_pipeline_change_stat: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/58 camerabin.test_single_video_recording: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101558 camerabin.test_multiple_video_recordings: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101646 --- gitlab/ci_template.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4eb68181d8..a65c354b9c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -134,6 +134,7 @@ build android arm64 api28: # gstreamer.pipelines_parse_launch.delayed_link: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/345 # gstreamer.gst_gstsystemclock.test_async_sync_interaction: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/346 # gstreamer.gst_gstsystemclock.test_periodic_multi: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/347 + # gstreamer.gst_gstsystemclock.test_periodic_shot: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/348 # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 # flvmux.test_incrementing_timestamps: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 @@ -146,13 +147,18 @@ build android arm64 api28: # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 + # splitmux.test_splitmuxsrc_caps_change: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/547 # splitmux.test_splitmuxsrc_sparse_streams: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/544 # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 + # nle_simple.test_simplest: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/57 + # ges_basic.test_ges_pipeline_change_stat: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/58 # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 # camerabin.test_image_video_cycle: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 + # camerabin.test_single_video_recording: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101558 + # camerabin.test_multiple_video_recordings: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101646 # # TO FURTHER INVESTIGATE: # check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer @@ -164,6 +170,7 @@ build android arm64 api28: -b check.gstreamer.pipelines_parse_launch.delayed_link -b check.gstreamer.gst_gstsystemclock.test_async_sync_interaction -b check.gstreamer.gst_gstsystemclock.test_periodic_multi + -b check.gstreamer.gst_gstsystemclock.test_periodic_shot -b check.gstreamer.pipelines_seek.test_loopback_2 -b check.gstreamer.gst_gstelement.test_foreach_pad -b check.gstreamer.libs_baseparse.parser_pull_short_read @@ -177,14 +184,19 @@ build android arm64 api28: -b check.gst-plugins-good.elements_flvmux.test_video_caps_late -b check.gst-plugins-base.elements_appsrc.test_appsrc_blocked_on_caps -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams + -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_caps_change -b check.gst-plugins-bad.elements_dtls.test_data_transfer -b check.gst-plugins-bad.elements_dtls.test_create_and_unref -b check.gst-plugins-bad.elements_camerabin.test_image_video_cycle + -b check.gst-plugins-bad.elements_camerabin.test_single_video_recording + -b check.gst-plugins-bad.elements_camerabin.test_multiple_video_recordings -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async -b check.gst-plugins-bad.elements_netsim.netsim_stress -b check.gst-editing-services.nle_complex.test_one_expandable_another + -b check.gst-editing-services.nle_simple.test_simplest + -b check.gst-editing-services.ges_basic.test_ges_pipeline_change_state -b check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps -b check.gst-rtsp-server.gst_rtspclientsink.test_record From 092f6ddd84c7aa919ff8a7a6d66b153988cc4010 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 10 Jan 2019 20:14:08 +0200 Subject: [PATCH 188/605] gitlab/ci_template: blacklist extra tests rtpsession.test_multiple_senders_roundrobin_rbs: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/548 elements_audiotestsrc.test_layout: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/535 --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a65c354b9c..cd1a535be1 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -143,6 +143,7 @@ build android arm64 api28: # rtpbin.test_cleanup_recv: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/546 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 + # rtpsession.test_multiple_senders_roundrobin_rbs: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/548 # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 @@ -156,6 +157,7 @@ build android arm64 api28: # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 + # elements_audiotestsrc.test_layout: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/535 # camerabin.test_image_video_cycle: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 # camerabin.test_single_video_recording: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101558 # camerabin.test_multiple_video_recordings: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101646 @@ -177,6 +179,7 @@ build android arm64 api28: -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe -b check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer + -b check.gst-plugins-base.elements_audiotestsrc.test_layout -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream -b check.gst-plugins-good.elements_rtpbin.test_sender_eos -b check.gst-plugins-good.elements_rtpbin.test_cleanup_recv @@ -191,6 +194,7 @@ build android arm64 api28: -b check.gst-plugins-bad.elements_camerabin.test_single_video_recording -b check.gst-plugins-bad.elements_camerabin.test_multiple_video_recordings -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp + -b check.gst-plugins-good.elements_rtpsession.test_multiple_senders_roundrobin_rbs -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async -b check.gst-plugins-bad.elements_netsim.netsim_stress From e6a39a4ebaa51b0ef25223660c84cdedb65f8246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Wed, 16 Jan 2019 15:46:56 +0000 Subject: [PATCH 189/605] gitlab/ci_template: blacklist multisocketsink test_add_client test Unexpected critical/warning: gsignal.c:2641: instance '0x144a770' has no handler with id '2' Stack trace: gst_debug_get_stack_trace (gstinfo.c:2788) gst_check_log_critical_func (gstcheck.c:281) g_logv (/usr/lib64/libglib-2.0.so.0.5800.2:0x7f40d144e4fb) g_log (/usr/lib64/libglib-2.0.so.0.5800.2:0x7f40d144e6ef) g_signal_handler_disconnect (/usr/lib64/libgobject-2.0.so.0.5800.2:0x7f40d13c5a03) g_cancellable_disconnect (/usr/lib64/libgio-2.0.so.0.5800.2:0x7f40d1238dae) ?? (/usr/lib64/libgio-2.0.so.0.5800.2:0x7f40d1238df8) ?? (/usr/lib64/libglib-2.0.so.0.5800.2:0x7f40d1444045) ?? (/usr/lib64/libglib-2.0.so.0.5800.2:0x7f40d1445407) ensure_condition (gstmultisocketsink.c:999) gst_multi_handle_sink_remove_client_link (gstmultihandlesink.c:971) gst_multi_handle_sink_clear (gstmultihandlesink.c:847) gst_multi_handle_sink_change_state (gstmultihandlesink.c:2200) gst_element_change_state (gstelement.c:2952) gst_element_change_state (gstelement.c:2991) gst_element_change_state (gstelement.c:2991) gst_element_set_state_func (gstelement.c:2906) test_add_client (multisocketsink.c:230) Likely same root cause as: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/534 --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index cd1a535be1..ffb243a9c3 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -178,6 +178,7 @@ build android arm64 api28: -b check.gstreamer.libs_baseparse.parser_pull_short_read -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe + -b check.gst-plugins-base.elements_multisocketsink.test_add_client -b check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer -b check.gst-plugins-base.elements_audiotestsrc.test_layout -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream From b289400a04540c0fd32a4ca54404f7863aedd94b Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 31 Jan 2019 11:18:12 -0500 Subject: [PATCH 190/605] Update docker hash While cleaning up, I got confused since some docker were rebuilt without but unused. So I broke the build, this should fix it. --- .gitlab-ci.yml | 2 +- gitlab/ci_template.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e7709c7a58..8812467577 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ stages: test manifest: variables: GIT_STRATEGY: fetch - image: "registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:5856b05f3010136cebcaa9a90058572e008ae3f6" + image: "registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:e6a39a4ebaa51b0ef25223660c84cdedb65f8246" stage: "preparation" script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ffb243a9c3..d62d56563c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -10,9 +10,9 @@ stages: variables: CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:bd83865236a1cd5b1694adefee24bc1a6b4f1ec4' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:b16ec7445ff7874d905eb5759ceaee5f93d29e66' - ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:aa13631e38cb4e1e4ee1c869aa523952dcd8cca7' + ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:b16ec7445ff7874d905eb5759ceaee5f93d29e66' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' - MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:f085d34a32cb8cac95cbd420344efeaa647d08d5' + MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' GIT_STRATEGY: none DEFAULT_MESON_ARGS: > From fb5e6c259536f2733ac2901f04fffffb35c510b5 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 17 Dec 2018 21:38:06 -0500 Subject: [PATCH 191/605] docker: cerbero: Add Android cross toolchain This runs the cross-android-universal bootstrap. Note that we remove the Android NDK source as it is no longer needed during builds. --- docker/cerbero/Dockerfile-fedora | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 12df38c383..554e2ae276 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -25,8 +25,12 @@ RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \ ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \ cd .. && \ + rm /cerbero/cerbero-sources/android-ndk-*.zip && \ rm /cerbero/cerbero-sources/mingw-w*.tar.xz && \ rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \ - rm -f /cerbero/cerbero-build/{linux,windows}*.cache && \ + rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \ dnf clean all From 81765960caca853cbca443307d6cde2e840b10fe Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Tue, 5 Feb 2019 10:27:39 +0000 Subject: [PATCH 192/605] gitlab: Blacklist the rtsp-server test_shared_udp test This test sometimes fails, as reported in https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/61 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d62d56563c..09a4897037 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -143,6 +143,7 @@ build android arm64 api28: # rtpbin.test_cleanup_recv: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/546 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 + # rtspserver.test_shared_udp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/61 # rtpsession.test_multiple_senders_roundrobin_rbs: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/548 # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 @@ -205,6 +206,7 @@ build android arm64 api28: -b check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps -b check.gst-rtsp-server.gst_rtspclientsink.test_record + -b check.gst-rtsp-server.gst_rtspserver.test_shared_udp script: - cd gst-build/ - > From 0cde909af17b07f042c96143334511827a4a6597 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Tue, 5 Feb 2019 11:31:30 +0000 Subject: [PATCH 193/605] gitlab: Blacklist another ges test python.test_timeline.TestTransitions.test_transition_type fails as reported in https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/62 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 09a4897037..c64f777695 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -155,6 +155,7 @@ build android arm64 api28: # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 # nle_simple.test_simplest: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/57 # ges_basic.test_ges_pipeline_change_stat: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/58 + # gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_transition_type: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/62 # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 @@ -203,6 +204,7 @@ build android arm64 api28: -b check.gst-editing-services.nle_complex.test_one_expandable_another -b check.gst-editing-services.nle_simple.test_simplest -b check.gst-editing-services.ges_basic.test_ges_pipeline_change_state + -b check.gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_transition_type -b check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps -b check.gst-rtsp-server.gst_rtspclientsink.test_record From c1d7f69fd52461cebadbeb6420e945f3f73abd00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 12 Feb 2019 11:26:03 +0200 Subject: [PATCH 194/605] Blacklist racy audiomixer.test_flush_start_flush_stop test See https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/555 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c64f777695..d697470954 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -163,6 +163,7 @@ build android arm64 api28: # camerabin.test_image_video_cycle: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 # camerabin.test_single_video_recording: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101558 # camerabin.test_multiple_video_recordings: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101646 + # audiomixer.test_flush_start_flush_stop: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/555 # # TO FURTHER INVESTIGATE: # check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer @@ -209,6 +210,7 @@ build android arm64 api28: -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps -b check.gst-rtsp-server.gst_rtspclientsink.test_record -b check.gst-rtsp-server.gst_rtspserver.test_shared_udp + -b check.gst-plugins-base.elements_audiomixer.test_flush_start_flush_stop script: - cd gst-build/ - > From 001e05ab3ade37eb2bbdc6b9e5202374514c7999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Mon, 18 Feb 2019 14:12:35 +0000 Subject: [PATCH 195/605] gitlab/ci_template: blacklist ges test_auto_transition test Fails too often. Reported as https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/63 --- gitlab/ci_template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d697470954..39b78d9b85 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -154,8 +154,9 @@ build android arm64 api28: # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 # nle_simple.test_simplest: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/57 - # ges_basic.test_ges_pipeline_change_stat: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/58 + # ges_basic.test_ges_pipeline_change_state: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/58 # gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_transition_type: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/62 + # gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_auto_transition: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/63 # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 @@ -206,6 +207,7 @@ build android arm64 api28: -b check.gst-editing-services.nle_simple.test_simplest -b check.gst-editing-services.ges_basic.test_ges_pipeline_change_state -b check.gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_transition_type + -b check.gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_auto_transition -b check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps -b check.gst-rtsp-server.gst_rtspclientsink.test_record From 42335f3100c56d7038fb36cc449f6eaf9f9b4886 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 22 Feb 2019 14:52:53 +0200 Subject: [PATCH 196/605] ci_template: unblacklist test Should be fixed by the following MR https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/merge_requests/20 --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 39b78d9b85..4f381a6561 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -142,7 +142,6 @@ build android arm64 api28: # rtpbin.test_sender_eos: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 # rtpbin.test_cleanup_recv: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/546 # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 - # rtspserver.test_shared_tcp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/54 # rtspserver.test_shared_udp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/61 # rtpsession.test_multiple_senders_roundrobin_rbs: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/548 # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 @@ -198,7 +197,6 @@ build android arm64 api28: -b check.gst-plugins-bad.elements_camerabin.test_image_video_cycle -b check.gst-plugins-bad.elements_camerabin.test_single_video_recording -b check.gst-plugins-bad.elements_camerabin.test_multiple_video_recordings - -b check.gst-rtsp-server.gst_rtspserver.test_shared_tcp -b check.gst-plugins-good.elements_rtpsession.test_multiple_senders_roundrobin_rbs -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async From b8c24cc7f4cfb19813a324cbbb65c5e89b55eebc Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 25 Feb 2019 18:00:02 -0300 Subject: [PATCH 197/605] Add depencencies to build gstreamer-sharp in our image --- docker/fedora/prepare.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 29b26ab32e..3dff9540fc 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -42,6 +42,7 @@ dnf install -y \ ffmpeg-devel \ flite \ flite-devel \ + mono-devel \ procps-ng \ patch \ redhat-rpm-config \ @@ -53,6 +54,7 @@ dnf install -y \ libunwind-devel \ neon \ neon-devel \ + nunit \ opencv \ opencv-devel \ openjpeg2 \ From f87d553bc4c89cbdb0107b6b414e02f6f8c59ff2 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 25 Feb 2019 19:17:11 -0300 Subject: [PATCH 198/605] Enable GstSharp --- gitlab/ci_template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4f381a6561..2c3dadac93 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,7 +9,7 @@ stages: variables: CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:bd83865236a1cd5b1694adefee24bc1a6b4f1ec4' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:b16ec7445ff7874d905eb5759ceaee5f93d29e66' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:b8c24cc7f4cfb19813a324cbbb65c5e89b55eebc' ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:b16ec7445ff7874d905eb5759ceaee5f93d29e66' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' @@ -111,6 +111,8 @@ build fedora x86_64: extends: '.build' stage: 'build' image: $FEDORA_IMAGE + variables: + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled" build android arm64 api28: extends: '.build' From badb75f4ed4105f1216789e7d96d28b93adfdc2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 3 Mar 2019 21:57:16 +0000 Subject: [PATCH 199/605] gitlab/ci_template: skip cerbero builds for gst-build changes --- gitlab/ci_template.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2c3dadac93..839f17ddc5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -290,6 +290,9 @@ check fedora: - $CERBERO bootstrap --offline --build-tools-only - $CERBERO package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0 - ccache -s + except: + variables: + - $CI_PROJECT_NAME == "gst-build" cache: key: "${CI_JOB_NAME}" paths: From 907400a2a0ee0b1eb3573cac7e8fcdbd1607977e Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 6 Mar 2019 10:47:13 -0300 Subject: [PATCH 200/605] Blacklist check.gstreamer-sharp.SdpTests --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 839f17ddc5..4e77397756 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -166,6 +166,7 @@ build android arm64 api28: # camerabin.test_single_video_recording: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101558 # camerabin.test_multiple_video_recordings: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101646 # audiomixer.test_flush_start_flush_stop: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/555 + # check.gstreamer-sharp.SdpTests: https://gitlab.freedesktop.org/gstreamer/gstreamer-sharp/issues/17 # # TO FURTHER INVESTIGATE: # check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer @@ -213,6 +214,7 @@ build android arm64 api28: -b check.gst-rtsp-server.gst_rtspclientsink.test_record -b check.gst-rtsp-server.gst_rtspserver.test_shared_udp -b check.gst-plugins-base.elements_audiomixer.test_flush_start_flush_stop + -b check.gstreamer-sharp.SdpTests script: - cd gst-build/ - > From d86a39a65b55c9332263f9a82afaa65aaea5d03f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 10 Mar 2019 23:51:36 +1100 Subject: [PATCH 201/605] gitlab: update cerbero-fedora image to the latest --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4e77397756..c04334a6e7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,7 +8,7 @@ stages: - 'full tests' variables: - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:bd83865236a1cd5b1694adefee24bc1a6b4f1ec4' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:fb5e6c259536f2733ac2901f04fffffb35c510b5' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:b8c24cc7f4cfb19813a324cbbb65c5e89b55eebc' ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:b16ec7445ff7874d905eb5759ceaee5f93d29e66' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' From 98176bd5bfa32f8121b1dc128229b258d23c6909 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 11 Mar 2019 14:38:02 +0200 Subject: [PATCH 202/605] docker/fedora/prepare.sh: add valgrind --- docker/fedora/prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 3dff9540fc..7187983762 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -71,6 +71,7 @@ dnf install -y \ python3-gobject \ python3-cairo \ python3-cairo-devel \ + valgrind \ vulkan \ vulkan-devel \ mesa-omx-drivers \ From ed534a2f5ac2a4a686807428491bc2a140cebb46 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 11 Mar 2019 20:12:01 +1100 Subject: [PATCH 203/605] cerbero: enable the werror variant by default --- gitlab/ci_template.yml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c04334a6e7..1eea2611bb 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -27,6 +27,9 @@ variables: -Dvaapi=enabled -Dsharp=disabled + DEFAULT_CERBERO_ARGS: > + --variants werror + manifest: image: $MANIFEST_IMAGE stage: 'preparation' @@ -272,7 +275,8 @@ check fedora: CERBERO_HOME: "${CI_PROJECT_DIR}/cerbero-build" CERBERO_SOURCES: "${CI_PROJECT_DIR}/cerbero-sources" CERBERO_DEPS: "cerbero-deps.tar.gz" - CERBERO: ./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS}" + CERBERO: "./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml" before_script: # FIXME Wrong namespace # Workaround build-tools having hardcoded internal path @@ -285,12 +289,12 @@ check fedora: - ./cerbero-uninstalled --self-update manifest.xml - ccache -z script: - - $CERBERO show-config - - $CERBERO fetch-bootstrap --build-tools-only - - $CERBERO fetch-package --deps gstreamer-1.0 - - $CERBERO fetch-cache - - $CERBERO bootstrap --offline --build-tools-only - - $CERBERO package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0 + - $CERBERO $CERBERO_ARGS show-config + - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only + - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 + - $CERBERO $CERBERO_ARGS fetch-cache + - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only + - $CERBERO $CERBERO_ARGS package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0 - ccache -s except: variables: @@ -325,15 +329,15 @@ check fedora: extends: .cerbero stage: "build" script: - - $CERBERO show-config - - $CERBERO fetch-bootstrap --build-tools-only - - $CERBERO fetch-package --deps gstreamer-1.0 - - $CERBERO bootstrap --offline --build-tools-only - - $CERBERO build-deps --offline + - $CERBERO $CERBERO_ARGS show-config + - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only + - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 + - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only + - $CERBERO $CERBERO_ARGS build-deps --offline gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice - - $CERBERO fetch-cache --skip-fetch --job-id=${CI_JOB_ID} + - $CERBERO $CERBERO_ARGS fetch-cache --skip-fetch --job-id=${CI_JOB_ID} - tar -C ${CERBERO_HOME} -czf $CERBERO_DEPS build-tools build-tools.cache dist/${ARCH} ${ARCH}.cache From aa8205abdb26e922416f671c04bae18f964999f8 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 12 Mar 2019 13:48:59 +0200 Subject: [PATCH 204/605] docker/fedora: Include common debug symbols in the image Part of #14 --- docker/fedora/prepare.sh | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 7187983762..e8883fa704 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -93,6 +93,63 @@ dnf install -y \ xorg-x11-server-utils \ xorg-x11-server-Xvfb +# Install common debug symbols +dnf debuginfo-install -y gtk3 \ + glib2 \ + glibc \ + freetype \ + openjpeg \ + gobject-introspection \ + python3 \ + python3-gobject \ + libjpeg-turbo \ + glib-networking \ + libcurl \ + libsoup \ + nss \ + nss-softokn \ + nss-softokn-freebl \ + nss-sysinit \ + nss-util \ + openssl \ + openssl-libs \ + openssl-pkcs11 \ + brotli \ + bzip2-libs \ + gpm-libs \ + harfbuzz \ + harfbuzz-icu \ + json-c \ + json-glib \ + libbabeltrace \ + libffi \ + libsrtp \ + libunwind \ + mpg123-libs \ + neon \ + orc-compiler \ + orc \ + pixman \ + pulseaudio-libs \ + pulseaudio-libs-glib2 \ + wavpack \ + webrtc-audio-processing \ + ffmpeg \ + ffmpeg-libs \ + faad2-libs \ + libavdevice \ + libmpeg2 \ + faac \ + fdk-aac \ + x264 \ + x264-libs \ + x265 \ + x265-libs \ + xz \ + xz-libs \ + zip \ + zlib + pip3 install meson # Add the pip3 installation to the path export PATH="$PATH:/usr/local/lib/python3.7/site-packages" From ae80b863c346174c8c8e4fbdc7c637151c590d9a Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 12 Mar 2019 21:33:58 -0400 Subject: [PATCH 205/605] gitlab: Move cerbero jobs in gst-ci This will allow extending the CI at one place. --- .gitlab-ci.yml | 6 ++-- gitlab/ci_template.yml | 75 ++++++++++++++++++++++++++++++------------ 2 files changed, 57 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8812467577..1e79df29a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -138,7 +138,7 @@ build android 28 local: - "master@gstreamer/gst-ci" build cerbero fedora x86_64 local: - extends: '.build cerbero fedora x86_64' + extends: '.cerbero fedora x86_64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" when: 'manual' except: @@ -146,7 +146,7 @@ build cerbero fedora x86_64 local: - "master@gstreamer/gst-ci" build cerbero cross win32 local: - extends: '.build cerbero cross win32' + extends: '.cerbero cross win32' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" when: 'manual' except: @@ -154,7 +154,7 @@ build cerbero cross win32 local: - "master@gstreamer/gst-ci" build cerbero cross win64 local: - extends: '.build cerbero cross win64' + extends: '.cerbero cross win64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" when: 'manual' except: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1eea2611bb..23ec2fd1a1 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -358,48 +358,81 @@ check fedora: # # Cerbero Linux X86_64 build # -.build cerbero fedora x86_64: - extends: '.cerbero' - variables: - CONFIG: "linux.config" - -.cerbero deps fedora x86_64: +cerbero deps fedora x86_64: extends: '.cerbero deps' variables: CONFIG: "linux.config" ARCH: "linux_x86_64" -# This indirection allow overriding the image without passing again CONFIG +.cerbero fedora x86_64: + extends: '.cerbero' + variables: + CONFIG: "linux.config" + +cerbero fedora x86_64: + extends: '.cerbero fedora x86_64' + dependencies: + - "cerbero deps fedora x86_64" + only: + variables: + - $CI_PROJECT_NAME == "cerbero" + build cerbero fedora x86_64: - extends: '.build cerbero fedora x86_64' + extends: '.cerbero fedora x86_64' + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + # # Cerbero Cross Windows builds # -.build cerbero cross win32: - extends: '.cerbero' - variables: - CONFIG: "cross-win32.cbc" - -.cerbero deps cross win32: +cerbero deps windows x86: extends: '.cerbero deps' variables: CONFIG: "cross-win32.cbc" ARCH: "windows_x86" -build cerbero cross win32: - extends: '.build cerbero cross win32' - -.build cerbero cross win64: +.cerbero cross win32: extends: '.cerbero' variables: - CONFIG: "cross-win64.cbc" + CONFIG: "cross-win32.cbc" -.cerbero deps cross win64: +cerbero cross win32: + extends: '.cerbero cross win32' + dependencies: + - "cerbero deps windows x86" + only: + variables: + - $CI_PROJECT_NAME == "cerbero" + +build cerbero cross win32: + extends: '.cerbero cross win32' + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + +cerbero deps windows x86_64: extends: '.cerbero deps' variables: CONFIG: "cross-win64.cbc" ARCH: "windows_x86_64" +.cerbero cross win64: + extends: '.cerbero' + variables: + CONFIG: "cross-win64.cbc" + +cerbero cross win64: + extends: '.cerbero cross win64' + dependencies: + - "cerbero deps windows x86_64" + only: + variables: + - $CI_PROJECT_NAME == "cerbero" + build cerbero cross win64: - extends: '.build cerbero cross win64' + extends: '.cerbero cross win64' + except: + variables: + - $CI_PROJECT_NAME == "cerbero" From 4221153836f441788b538e82795727173985f6cd Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 17 Dec 2018 21:58:39 -0500 Subject: [PATCH 206/605] CI: Add Android universal builds --- .gitlab-ci.yml | 8 ++++++++ gitlab/ci_template.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e79df29a1..dbdf73a784 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -145,6 +145,14 @@ build cerbero fedora x86_64 local: refs: - "master@gstreamer/gst-ci" +build cerbero android universal local: + extends: '.cerbero android universal' + image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + when: 'manual' + except: + refs: + - "master@gstreamer/gst-ci" + build cerbero cross win32 local: extends: '.cerbero cross win32' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 23ec2fd1a1..6e97ef5f12 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -384,6 +384,34 @@ build cerbero fedora x86_64: - $CI_PROJECT_NAME == "cerbero" +# +# Cerbero Android Universal build +# +cerbero deps android universal: + extends: '.cerbero deps' + variables: + CONFIG: "cross-android-universal.cbc" + ARCH: "android_universal" + +.cerbero android universal: + extends: '.cerbero' + variables: + CONFIG: "cross-android-universal.cbc" + +cerbero android universal: + extends: '.cerbero android universal' + dependencies: + - "cerbero deps android universal" + only: + variables: + - $CI_PROJECT_NAME == "cerbero" + +build cerbero android universal: + extends: '.cerbero android universal' + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + # # Cerbero Cross Windows builds # From d6779a1d077c7ea6ffc88328e7ec6494145880a4 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 12 Mar 2019 12:32:15 +1100 Subject: [PATCH 207/605] gitlab/cerbero-android: build without werror --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6e97ef5f12..3d777203c0 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -390,12 +390,14 @@ build cerbero fedora x86_64: cerbero deps android universal: extends: '.cerbero deps' variables: + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" CONFIG: "cross-android-universal.cbc" ARCH: "android_universal" .cerbero android universal: extends: '.cerbero' variables: + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" CONFIG: "cross-android-universal.cbc" cerbero android universal: From 775c0b35208088c70b08cc62469b9e8003c39bf3 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 12 Mar 2019 23:59:26 +1100 Subject: [PATCH 208/605] gitlab: remove the android gst-build target android is now being tested using cerbero for the binaries we produce. --- .gitlab-ci.yml | 21 ------------ docker/ubuntu/Dockerfile | 8 ----- docker/ubuntu/android-create-toolchain.sh | 27 --------------- docker/ubuntu/android-download-ndk.sh | 31 ----------------- docker/ubuntu/cleanup.sh | 5 --- docker/ubuntu/prepare.sh | 42 ----------------------- gitlab/ci_template.yml | 10 ------ 7 files changed, 144 deletions(-) delete mode 100644 docker/ubuntu/Dockerfile delete mode 100755 docker/ubuntu/android-create-toolchain.sh delete mode 100755 docker/ubuntu/android-download-ndk.sh delete mode 100644 docker/ubuntu/cleanup.sh delete mode 100644 docker/ubuntu/prepare.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbdf73a784..5a865231bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,15 +92,6 @@ fedora amd64 docker: DOCKERFILE: "docker/fedora/Dockerfile" extends: .base -ubuntu amd64 docker: - stage: "build docker" - variables: - ARCH: "amd64" - TAG: "ubuntu" - CONTEXT_DIR: "docker/ubuntu/" - DOCKERFILE: "docker/ubuntu/Dockerfile" - extends: .base - cerbero fedora amd64 docker: stage: "build docker" variables: @@ -125,18 +116,6 @@ build fedora x86_64 local: refs: - "master@gstreamer/gst-ci" -build android 28 local: - extends: '.build local' - variables: - MESON_ARGS: > - -Dbad=enabled - -Dbad:androidmedia=enabled - --cross-file cross-files/android_arm64_api28.txt - image: "${CI_REGISTRY_IMAGE}/amd64/ubuntu:latest" - except: - refs: - - "master@gstreamer/gst-ci" - build cerbero fedora x86_64 local: extends: '.cerbero fedora x86_64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" diff --git a/docker/ubuntu/Dockerfile b/docker/ubuntu/Dockerfile deleted file mode 100644 index 67a2095974..0000000000 --- a/docker/ubuntu/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:18.04 - -COPY prepare.sh cleanup.sh android-download-ndk.sh android-create-toolchain.sh /root/ - -ENV DEBIAN_FRONTEND=noninteractive - -RUN sh /root/prepare.sh && \ - sh /root/cleanup.sh diff --git a/docker/ubuntu/android-create-toolchain.sh b/docker/ubuntu/android-create-toolchain.sh deleted file mode 100755 index 996dfe231d..0000000000 --- a/docker/ubuntu/android-create-toolchain.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# -# Copyright 2018 Collabora ltd. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# -# Author: Xavier Claessens -# - -set -eu - -arch=$1 -api=$2 -toolchain_path=/opt/android-$arch-api$api -/opt/android-ndk/build/tools/make_standalone_toolchain.py --arch $arch --api $api --install-dir $toolchain_path diff --git a/docker/ubuntu/android-download-ndk.sh b/docker/ubuntu/android-download-ndk.sh deleted file mode 100755 index 566543efb2..0000000000 --- a/docker/ubuntu/android-download-ndk.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# -# Copyright 2018 Collabora ltd. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# -# Author: Xavier Claessens -# - -set -eu - -# Download Android NDK -ANDROID_NDK_VERSION="r18b" -ANDROID_NDK_SHA512="a35ab95ece52819194a3874fd210abe5c25905212c4aafe5d75c465c14739a46340d1ff0944ad93ffbbc9c0d86107119399d4f60ec6c5f080758008e75c19617" -wget --quiet https://dl.google.com/android/repository/android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip -echo "$ANDROID_NDK_SHA512 android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip" | sha512sum -c -unzip android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip -rm android-ndk-$ANDROID_NDK_VERSION-linux-x86_64.zip -mv android-ndk-$ANDROID_NDK_VERSION /opt/android-ndk diff --git a/docker/ubuntu/cleanup.sh b/docker/ubuntu/cleanup.sh deleted file mode 100644 index 33032b178e..0000000000 --- a/docker/ubuntu/cleanup.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -eu - -echo "Removing apt cache" -rm -R /root/* -rm -R /var/lib/apt/ /var/log/apt/ diff --git a/docker/ubuntu/prepare.sh b/docker/ubuntu/prepare.sh deleted file mode 100644 index c5b74a040b..0000000000 --- a/docker/ubuntu/prepare.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -eu - -# make source packages available in order to figure out build dependencies -sed -i "s/# deb-src/deb-src/g" /etc/apt/sources.list - -apt update -apt build-dep -y \ - orc \ - gstreamer1.0 \ - gst-plugins-base1.0 \ - gst-plugins-good1.0 \ - gst-plugins-bad1.0 \ - gst-plugins-ugly1.0 \ - gst-libav1.0 \ - gst-rtsp-server1.0 \ - gst-python1.0 \ - gstreamer-vaapi \ - gstreamer-editing-services1.0 - -apt install -y \ - ccache \ - gdb \ - git \ - xvfb \ - python3-pip \ - wget \ - unzip - -pip3 install meson - -# Configure git for various usage -git config --global user.email "gst-build@gstreamer.net" -git config --global user.name "Gstbuild Runner" - -# Setup Android toolchain -/root/android-download-ndk.sh -/root/android-create-toolchain.sh arm64 28 -rm -rf /opt/android-ndk - -# get gst-build and make all subprojects available -git clone git://anongit.freedesktop.org/gstreamer/gst-build '/gst-build/' -meson subprojects download --sourcedir /gst-build diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 3d777203c0..912a153751 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -10,7 +10,6 @@ stages: variables: CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:fb5e6c259536f2733ac2901f04fffffb35c510b5' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:b8c24cc7f4cfb19813a324cbbb65c5e89b55eebc' - ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/ubuntu:b16ec7445ff7874d905eb5759ceaee5f93d29e66' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' @@ -117,15 +116,6 @@ build fedora x86_64: variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled" -build android arm64 api28: - extends: '.build' - image: $ANDROID_IMAGE - variables: - MESON_ARGS: > - -Dbad=enabled - -Dbad:androidmedia=enabled - --cross-file cross-files/android_arm64_api28.txt - .test: stage: 'test' variables: From a116f19eeb1d6cc40b2b9310c50c404ce407fcdd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 14 Mar 2019 02:42:15 +0200 Subject: [PATCH 209/605] docker/fedora: use meson from fedora repos Fedora is still on version 0.49.2 while pip is on 0.50. With the version from pip we can no longer run some gst-sharp tests so we essentially revert back to 0.49 https://github.com/mesonbuild/meson/issues/5087 --- docker/fedora/prepare.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index e8883fa704..4f31888c3c 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -1,6 +1,6 @@ set -eu -dnf install -y git ninja-build dnf-plugins-core +dnf install -y git meson ninja-build dnf-plugins-core # Configure git for various usage git config --global user.email "gst-build@gstreamer.net" @@ -150,10 +150,6 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson -# Add the pip3 installation to the path -export PATH="$PATH:/usr/local/lib/python3.7/site-packages" - # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ gstreamer1-plugins-base \ @@ -168,8 +164,7 @@ dnf builddep -y gstreamer1 \ gstreamer1-libav \ gstreamer1-rtsp-server \ gstreamer1-vaapi \ - python3-gstreamer1 \ - -x meson + python3-gstreamer1 # Remove gst-devel packages installed by builddep above dnf remove -y "gstreamer1*devel" From a9e17aad526bfaa1a51277e25427bf3efb32b24b Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 14 Mar 2019 16:51:14 +0200 Subject: [PATCH 210/605] docker/fedora: use git-core instead of git We don't need the full blown git pakcage and we can avoid a couple of perl deps that way --- docker/fedora/prepare.sh | 2 +- docker/test_manifest/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 4f31888c3c..491ddc2b64 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -1,6 +1,6 @@ set -eu -dnf install -y git meson ninja-build dnf-plugins-core +dnf install -y git-core meson ninja-build dnf-plugins-core # Configure git for various usage git config --global user.email "gst-build@gstreamer.net" diff --git a/docker/test_manifest/Dockerfile b/docker/test_manifest/Dockerfile index 15ffbdafe5..2a43a392e2 100644 --- a/docker/test_manifest/Dockerfile +++ b/docker/test_manifest/Dockerfile @@ -4,5 +4,5 @@ RUN dnf install -y \ python3-pytest \ python3-requests \ python3-pytest-cov \ - git \ + git-core \ && rm -rf /var/cache/dnf /var/log/dnf* \ No newline at end of file From 71ef6dd8047b8886a5b0c515a0fed2324adcc70b Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 14 Mar 2019 16:54:11 +0200 Subject: [PATCH 211/605] docker/fedora: use upstream registry to retch the base image Fedora is now hosting its official docker images at https://registry.fedoraproject.org --- docker/cerbero/Dockerfile-fedora | 2 +- docker/fedora/Dockerfile | 2 +- docker/test_manifest/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 554e2ae276..f40b48080e 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -1,4 +1,4 @@ -FROM fedora:29 +FROM registry.fedoraproject.org/fedora:29 # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index 0e80c32fa0..41911102dd 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:29 +FROM registry.fedoraproject.org/fedora:29 COPY prepare.sh cleanup.sh /root/ diff --git a/docker/test_manifest/Dockerfile b/docker/test_manifest/Dockerfile index 2a43a392e2..efadb5af83 100644 --- a/docker/test_manifest/Dockerfile +++ b/docker/test_manifest/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora +FROM registry.fedoraproject.org/fedora:29 RUN dnf install -y \ python3-pytest \ From fb91ee18309ab981289aa818ab2a7824ef0567a1 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 14 Mar 2019 17:19:49 +0200 Subject: [PATCH 212/605] docker/fedora: use the meson subprojects cmd to download subprojects Introduced in 0.49.0 so we can avoid the need to configure in order to cache the subproejct repos. --- docker/fedora/prepare.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 491ddc2b64..245465b692 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -171,4 +171,6 @@ dnf remove -y "gstreamer1*devel" # get gst-build and make all subprojects available git clone git://anongit.freedesktop.org/gstreamer/gst-build /gst-build/ -cd /gst-build && meson build/ && rm -rf build/ \ No newline at end of file +cd /gst-build +meson subprojects download + From 0831f87561860f85880190e86e524f87d42d54c1 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 12 Mar 2019 18:41:10 +0200 Subject: [PATCH 213/605] gitlab/ci_template: refresh the image builds --- .gitlab-ci.yml | 2 +- gitlab/ci_template.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5a865231bc..4ff583505f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ stages: test manifest: variables: GIT_STRATEGY: fetch - image: "registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:e6a39a4ebaa51b0ef25223660c84cdedb65f8246" + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:fb91ee18309ab981289aa818ab2a7824ef0567a1' stage: "preparation" script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 912a153751..6b92af4a5d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,8 +8,8 @@ stages: - 'full tests' variables: - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:fb5e6c259536f2733ac2901f04fffffb35c510b5' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:b8c24cc7f4cfb19813a324cbbb65c5e89b55eebc' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:fb91ee18309ab981289aa818ab2a7824ef0567a1' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:fb91ee18309ab981289aa818ab2a7824ef0567a1' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' From 3f3031eafb932d290fd44b07f7c5c74f9bab3f88 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 1 Feb 2019 13:36:28 -0300 Subject: [PATCH 214/605] fedora: Install git-lfs --- docker/fedora/prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 245465b692..a328fb256b 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -31,6 +31,7 @@ dnf install -y \ gcc \ gcc-c++ \ gdb \ + git-lfs \ gtk3 \ gtk3-devel \ graphene \ From 74a954ca9162701ba82ee702859bd3055c151240 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 13 Mar 2019 19:10:18 -0300 Subject: [PATCH 215/605] Track gst-integration-testsuites --- gitlab/build_manifest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index fcce1b7f3b..0619847ae1 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -27,6 +27,7 @@ GSTREAMER_MODULES: List[str] = [ 'gst-python', 'gst-rtsp-server', 'gstreamer-vaapi', + 'gst-integration-testsuites', ] MANIFEST_TEMPLATE: str = """ From e229b559d44b6316c7d5c17fddceed3a180a8fdf Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 15 Mar 2019 08:53:54 -0300 Subject: [PATCH 216/605] Revert "docker/fedora: use meson from fedora repos" This reverts commit a116f19eeb1d6cc40b2b9310c50c404ce407fcdd. --- docker/fedora/prepare.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index a328fb256b..d0000ccc9c 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -1,6 +1,6 @@ set -eu -dnf install -y git-core meson ninja-build dnf-plugins-core +dnf install -y git-core ninja-build dnf-plugins-core # Configure git for various usage git config --global user.email "gst-build@gstreamer.net" @@ -151,6 +151,10 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib +pip3 install meson +# Add the pip3 installation to the path +export PATH="$PATH:/usr/local/lib/python3.7/site-packages" + # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ gstreamer1-plugins-base \ @@ -165,7 +169,8 @@ dnf builddep -y gstreamer1 \ gstreamer1-libav \ gstreamer1-rtsp-server \ gstreamer1-vaapi \ - python3-gstreamer1 + python3-gstreamer1 \ + -x meson # Remove gst-devel packages installed by builddep above dnf remove -y "gstreamer1*devel" From f04c2e7f38535e7c03d58165f66f7633d9a901e0 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 15 Mar 2019 08:55:13 -0300 Subject: [PATCH 217/605] fedora: Install meson 0.49.2 Working around https://github.com/mesonbuild/meson/issues/5087 --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index d0000ccc9c..7cecd39f33 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -151,7 +151,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson +pip3 install meson==0.49.2 # Add the pip3 installation to the path export PATH="$PATH:/usr/local/lib/python3.7/site-packages" From 452f64bb5e7ff58a0753fe95eb76fd5a6597d277 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 21 Mar 2019 20:53:32 -0300 Subject: [PATCH 218/605] docker:fedora: No need to add site-packages to PATH --- docker/fedora/prepare.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 7cecd39f33..b9115550d6 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -152,8 +152,6 @@ dnf debuginfo-install -y gtk3 \ zlib pip3 install meson==0.49.2 -# Add the pip3 installation to the path -export PATH="$PATH:/usr/local/lib/python3.7/site-packages" # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From d1181c94d56de703294666d180a312ada03a1069 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 20 Mar 2019 21:45:31 +0200 Subject: [PATCH 219/605] gitlab/ci_template: add a build that disabled debug logs --- gitlab/ci_template.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6b92af4a5d..0128e12270 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -116,6 +116,13 @@ build fedora x86_64: variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled" +build_disable_debug fedora x86_64: + extends: '.build' + stage: 'build' + image: $FEDORA_IMAGE + variables: + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false" + .test: stage: 'test' variables: From 472901ba171b579c2461cac7fa1cd1788ff3c693 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 22 Mar 2019 18:19:59 -0300 Subject: [PATCH 220/605] docker:fedora: Remove Qt5 devel packages as we haven't tested building it and it leads to build issues in examples. --- docker/fedora/prepare.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index b9115550d6..3a49e895a4 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -173,6 +173,10 @@ dnf builddep -y gstreamer1 \ # Remove gst-devel packages installed by builddep above dnf remove -y "gstreamer1*devel" +# Remove Qt5 devel packages as we haven't tested building it and +# it leads to build issues in examples. +dnf remove -y "qt5-qtbase-devel" + # get gst-build and make all subprojects available git clone git://anongit.freedesktop.org/gstreamer/gst-build /gst-build/ cd /gst-build From b0affb257e92c1e4fe942d016232b49fb66ebb3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sat, 23 Mar 2019 20:04:38 +0000 Subject: [PATCH 221/605] gitlab/ci_template: blacklist curlhttpsrc test_multiple_http_requests Too flaky. https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/932 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0128e12270..ff3ffe849b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -165,6 +165,7 @@ build_disable_debug fedora x86_64: # camerabin.test_image_video_cycle: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 # camerabin.test_single_video_recording: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101558 # camerabin.test_multiple_video_recordings: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101646 + # curlhttpsrc.test_multiple_http_requests: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/932 # audiomixer.test_flush_start_flush_stop: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/555 # check.gstreamer-sharp.SdpTests: https://gitlab.freedesktop.org/gstreamer/gstreamer-sharp/issues/17 # @@ -200,6 +201,7 @@ build_disable_debug fedora x86_64: -b check.gst-plugins-bad.elements_camerabin.test_image_video_cycle -b check.gst-plugins-bad.elements_camerabin.test_single_video_recording -b check.gst-plugins-bad.elements_camerabin.test_multiple_video_recordings + -b check.gst-plugins-bad.elements_curlhttpsrc.test_multiple_http_requests -b check.gst-plugins-good.elements_rtpsession.test_multiple_senders_roundrobin_rbs -b check.gst-plugins-bad.elements_shm.test_shm_live -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async From 43d8affe31efe579d369f420afe97d57f994de91 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 25 Mar 2019 09:32:27 -0300 Subject: [PATCH 222/605] Do not run indent on gst-integration-testsuites --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ff3ffe849b..f12f130568 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -65,6 +65,7 @@ gst indent: variables: # No point on trying to format C files in those repositories - $CI_PROJECT_NAME == "gstreamer-sharp" + - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "cerbero" .build: From fc2b4a657ce9b8f76b0bd0f600a9901b6e3cf868 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 25 Mar 2019 16:29:57 +0530 Subject: [PATCH 223/605] docker/cerbero: Always install lbzip2 for use in packaging See: https://gitlab.freedesktop.org/gstreamer/cerbero/issues/132 Needed for https://gitlab.freedesktop.org/gstreamer/cerbero/merge_requests/139 --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index f40b48080e..b76f75e0b8 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -3,7 +3,7 @@ FROM registry.fedoraproject.org/fedora:29 # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ dnf upgrade -y && \ - dnf install -y sudo git rsync + dnf install -y sudo git lbzip2 rsync # Configure git for various usage RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ From ad598c3f024d7fcff4caeb060e4e83a473ed881a Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 25 Mar 2019 17:23:37 +0530 Subject: [PATCH 224/605] Always print timestamps when running Cerbero This is useful when doing performance improvements or comparisons. --- docker/cerbero/Dockerfile-fedora | 24 ++++++++++++------------ gitlab/ci_template.yml | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index b76f75e0b8..cde56e0b40 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -16,18 +16,18 @@ RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ cd cerbero && \ echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \ echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \ - ./cerbero-uninstalled -c localconf.cbc bootstrap -y --system-only && \ - ./cerbero-uninstalled -c localconf.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -c localconf.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --system-only && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -t -c localconf.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \ cd .. && \ rm /cerbero/cerbero-sources/android-ndk-*.zip && \ rm /cerbero/cerbero-sources/mingw-w*.tar.xz && \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f12f130568..cf008bcb49 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -27,7 +27,8 @@ variables: -Dsharp=disabled DEFAULT_CERBERO_ARGS: > - --variants werror + --variants werror + --timestamps manifest: image: $MANIFEST_IMAGE From 21733f40570a1dbbef7938713082d2c3a2cac118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Wed, 6 Mar 2019 19:01:48 +0000 Subject: [PATCH 225/605] gitlab/ci_template: add manual valgrind jobs --- gitlab/ci_template.yml | 154 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index cf008bcb49..99e77df30c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -233,6 +233,7 @@ build_disable_debug fedora x86_64: -M "${CI_PROJECT_DIR}/validate-output/" --xunit-file "${CI_PROJECT_DIR}/validate-output/logs/xunit.xml" ${BLACKLIST} + ${EXTRA_VALIDATE_ARGS} artifacts: expire_in: '14 days' when: always @@ -258,6 +259,159 @@ check fedora: variables: TEST_SUITE: "check.gst*" +# Valgrind +.valgrind fedora x86_64: + extends: '.test fedora x86_64' + stage: test + when: manual + variables: + SUPPRESSION_FILES: > + gst-build/subprojects/gstreamer/tests/check/gstreamer.supp + gst-build/subprojects/gst-plugins-base/tests/check/gst-plugins-base.supp + gst-build/subprojects/gst-plugins-good/tests/check/gst-plugins-good.supp + gst-build/subprojects/gst-plugins-bad/tests/check/gst-plugins-bad.supp + gst-build/subprojects/gst-plugins-ugly/tests/check/gst-plugins-ugly.supp + gst-build/subprojects/gst-libav/tests/check/gst-libav.supp + gst-build/subprojects/gst-devtools/validate/data/gstvalidate.supp + gst-build/subprojects/libnice/tests/libnice.supp + gst-build/subprojects/libsoup/tests/libsoup.supp + gst-build/subprojects/glib/glib.supp + # These tests take very long compared to what they add, so let's skip them. + # - check.*.generic_states.* - enough to run one of the sequences + # - check.gstreamer.gst_gstelement.test_foreach_pad - 48s + # - check.gstreamer.gst_gstinfo.info_post_gst_init_category_registration - 21s + # - check.gstreamer.gst_gstsystemclock.test_resolution - 60s + # - check.gstreamer.libs_aggregator.test_infinite_seek - 20s + # - check.gstreamer.libs_aggregator.test_infinite_seek_50_src - 20s + # - check.gstreamer.libs_gstharness.test_harness_element_ref - 20s + # - check.gstreamer.pipelines_simple_launch_lines.test_2_elements - 58s + # - check.gstreamer.pipelines_stress.test_stress - 54s + # - check.gstreamer.pipelines_stress.test_stress_preroll - 27s + # - check.gst-plugins-base.elements_appsrc.test_appsrc_block_deadlock - 265.595s + # - check.gst-plugins-base.elements_audioresample.test_fft - 91.247s + # - check.gst-plugins-base.elements_audioresample.test_timestamp_drift - 141.784s + # - check.gst-plugins-base.elements-videoscale - superlong + # - check.gst-plugins-base.libs_video.test_overlay_blend - 74.096s + # - check.gst-plugins-base.libs_video.test_video_color_convert - 345.271s + # - check.gst-plugins-base.libs_video.test_video_formats - 70.987s + # - check.gst-plugins-base.libs_video.test_video_size_convert - 56.387s + # - check.gst-plugins-base.elements_audiointerleave.test_audiointerleave_2ch_pipeline_ - 5 * 51.069s + # - check.gst-plugins-base.elements_multifdsink.test_client_kick - 46.909s + # - check.gst-plugins-base.elements_videotestsrc.test_all_patterns + # - check.gst-plugins-base.elements_videotestsrc.test_patterns_are_deterministic + # - check.gst-plugins-good.elements_shapewipe.test_general - 325s + # - check.gst-plugins-good.elements_videocrop.test_cropping - 245s + # - check.gst-plugins-good.elements_videomixer - 30s (also deprecated) + # - check.gst-plugins-good.elements_rtp_payloading.rtp_jpeg_packet_loss - 109s + # - check.gst-plugins-good.elements_videomixer.test_play_twice_then_add_and_play_again - 55s + # - check.gst-plugins-good.pipelines_effectv.test_quarktv - 53s + # - check.gst-plugins-good.elements_deinterlace.test_mode_disabled_passthrough - 52s + # - check.gst-plugins-good.elements_deinterlace.test_mode_auto_deinterlaced_passthrough - 28s + # - check.gst-plugins-good.elements_deinterleave.test_2_channels_caps_change - 30s + # - check.gst-plugins-good.elements_deinterleave.test_2_channels - 22s + # - check.gst-plugins-good.elements_rtpjitterbuffer.test_fill_queue - 22s + # - check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async - 20s + # - check.gst-plugins-good.elements_videomixer.test_play_twice - 22s + VALGRIND_SKIPLIST: > + -b check.[a-z-]*.generic_states.test_state_changes_down_seq + -b check.[a-z-]*.generic_states.test_state_changes_up_seq + -b check.gstreamer.gst_gstelement.test_foreach_pad + -b check.gstreamer.gst_gstinfo.info_post_gst_init_category_registration + -b check.gstreamer.gst_gstsystemclock.test_resolution + -b check.gstreamer.libs_aggregator.test_infinite_seek + -b check.gstreamer.libs_aggregator.test_infinite_seek_50_src + -b check.gstreamer.libs_gstharness.test_harness_element_ref + -b check.gstreamer.pipelines_simple_launch_lines.test_2_elements + -b check.gstreamer.pipelines_stress.test_stress + -b check.gstreamer.pipelines_stress.test_stress_preroll + -b check.gst-plugins-base.elements_appsrc.test_appsrc_block_deadlock + -b check.gst-plugins-base.elements_audioresample.test_fft + -b check.gst-plugins-base.elements_audioresample.test_timestamp_drift + -b check.gst-plugins-base.elements-videoscale + -b check.gst-plugins-base.libs_video.test_overlay_blend + -b check.gst-plugins-base.libs_video.test_video_color_convert + -b check.gst-plugins-base.libs_video.test_video_formats + -b check.gst-plugins-base.libs_video.test_video_size_convert + -b check.gst-plugins-base.elements_audiointerleave.test_audiointerleave_2ch_pipeline_ + -b check.gst-plugins-base.elements_multifdsink.test_client_kick + -b check.gst-plugins-base.elements_videotestsrc.test_all_patterns + -b check.gst-plugins-base.elements_videotestsrc.test_patterns_are_deterministic + -b check.gst-plugins-good.elements_shapewipe.test_general + -b check.gst-plugins-good.elements_videocrop.test_cropping + -b check.gst-plugins-good.elements_videomixer + -b check.gst-plugins-good.elements_rtp_payloading.rtp_jpeg_packet_loss + -b check.gst-plugins-good.elements_videomixer.test_play_twice_then_add_and_play_again + -b check.gst-plugins-good.pipelines_effectv.test_quarktv + -b check.gst-plugins-good.elements_deinterlace.test_mode_disabled_passthrough + -b check.gst-plugins-good.elements_deinterlace.test_mode_auto_deinterlaced_passthrough + -b check.gst-plugins-good.elements_deinterleave.test_2_channels_caps_change + -b check.gst-plugins-good.elements_deinterleave.test_2_channels + -b check.gst-plugins-good.elements_rtpjitterbuffer.test_fill_queue + -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async + -b check.gst-plugins-good.elements_videomixer.test_play_twice + # - check.gst-plugins-good.elements_rtpjitterbuffer.test_push_* - flaky in valgrind + # - check.gst-plugins-base.pipelines_gl_launch_lines - driver leaks / memory access + # - check.gst-plugins-base.libs_gstgl - driver leaks / memory access + # - check.gst-plugins-base.elements_gl - driver leaks / memory access + # - check.gst-plugins-base.elements_libvisual - uninitialized memory access + # - check.gst-plugins-base.generic_states - need to add gl elements to ignore list but only if using valgrind + VALGRIND_BLACKLIST: > + -b check.gstreamer.gst_gstsystemclock.test_stress_cleanup_unschedule + -b check.gstreamer.gst_gstsystemclock.test_stress_reschedule + -b check.gstreamer.tools_gstinspect + -b check.gst-plugins-base.elements_videoscale + -b check.gst-plugins-base.pipelines_gl_launch_lines + -b check.gst-plugins-base.libs_gstgl + -b check.gst-plugins-base.elements_gl + -b check.gst-plugins-base.elements_libvisual + -b check.gst-plugins-base.generic_states + -b check.gst-plugins-good.elements_rtpjitterbuffer.test_push_backward_seq + -b check.gst-plugins-good.elements_rtpjitterbuffer.test_push_unordered + -b check.gst-plugins-bad.elements_assrender + -b check.gst-plugins-bad.elements_camerabin + -b check.gst-plugins-bad.elements_line21 + -b check.gst-plugins-bad.elements_mpeg2enc + -b check.gst-plugins-bad.elements_mplex + -b check.gst-plugins-bad.elements_mxfmux + -b check.gst-plugins-bad.elements_x265enc + -b check.gst-plugins-bad.elements_zbar + -b check.gst-libav.generic_plugin_test + -b check.gst-libav.generic_libavcodec_locking + -b check.gst-libav.elements_avdemux_ape + VALGRIND_OPTS: "--suppressions=/tmp/gstreamer-ci.supp" + EXTRA_VALIDATE_ARGS: "--valgrind ${VALGRIND_BLACKLIST} ${VALGRIND_SKIPLIST}" + ORC_CODE: "backup" + # Some suppression files are missing a newline at the end which messes things + # up when concatenating them. awk will add missing newlines (unlike cat) + before_script: + - awk '{print}' ${SUPPRESSION_FILES} > /tmp/gstreamer-ci.supp + +valgrind core: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gstreamer\\..*" + +valgrind base: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-plugins-base\\..*" + +valgrind good: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-plugins-good\\..*" + +valgrind ugly: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-plugins-ugly\\..*" + +valgrind bad: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-plugins-bad\\..*" + + # Template for Cerbero GStreamer Build # # Parameters: From 7ed820aa1fcf95938cc161e397b1bf455f1f5b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 21 Mar 2019 21:39:03 +0000 Subject: [PATCH 226/605] gitlab/ci_template: run valgrind tests automatically Don't valgrind everything for all changes though, but only those modules most likely to be affected by changes in the current ci project. So, valgrind - gstreamer only for gstreamer core changes - gst-plugins-base for core/base changes - gst-plugins-good for core/base/good changes - gst-plugins-ugly for core/base/ugly changes - gst-plugins-bad for core/base/good changes In other words: don't valgrind core/base if it's good/bad/ugly that's being changed, for example. Don't valgrind -good or -ugly for changes in -bad, etc. Meaning: - for changes in core, valgrind core/base/good/ugly/bad - for changes in base, valgrind base/good/ugly/bad - for changes in good, valgrind good/bad - for changes in ugly, valgrind ugly - for changes in bad, valgrind bad More modules to be added later once they're valgrind clean on the CI. https://gitlab.freedesktop.org/gstreamer/gst-ci/merge_requests/80 --- gitlab/ci_template.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 99e77df30c..61bb998dcb 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -262,8 +262,7 @@ check fedora: # Valgrind .valgrind fedora x86_64: extends: '.test fedora x86_64' - stage: test - when: manual + stage: 'full tests' variables: SUPPRESSION_FILES: > gst-build/subprojects/gstreamer/tests/check/gstreamer.supp @@ -390,26 +389,41 @@ valgrind core: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gstreamer\\..*" + only: + variables: + - $CI_PROJECT_NAME == "gstreamer" valgrind base: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-base\\..*" + only: + variables: + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base)$/ valgrind good: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-good\\..*" + only: + variables: + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good)$/ valgrind ugly: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-ugly\\..*" + only: + variables: + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly)$/ valgrind bad: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-bad\\..*" + only: + variables: + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad)$/ # Template for Cerbero GStreamer Build From 01624a5051fa3eb0f5387d067c63e7693da5f994 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 26 Mar 2019 00:25:07 +0000 Subject: [PATCH 227/605] gitab/ci_template: Update cerbero image --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 61bb998dcb..5bc3bdc61d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,7 +8,7 @@ stages: - 'full tests' variables: - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:fb91ee18309ab981289aa818ab2a7824ef0567a1' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:fb91ee18309ab981289aa818ab2a7824ef0567a1' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' From 497415544b1a8235f0cff6af247e6e1c7eee8fb9 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 24 Mar 2019 21:41:31 +1100 Subject: [PATCH 228/605] docker: add android-specific image pre-includes the NDK/SDK --- .gitlab-ci.yml | 9 +++++++++ docker/android/Dockerfile | 9 +++++++++ docker/android/cleanup.sh | 7 +++++++ docker/android/prepare.sh | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 docker/android/Dockerfile create mode 100644 docker/android/cleanup.sh create mode 100644 docker/android/prepare.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ff583505f..6b52a41b73 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -101,6 +101,15 @@ cerbero fedora amd64 docker: DOCKERFILE: "docker/cerbero/Dockerfile-fedora" extends: .base +android docker: + stage: "build docker" + variables: + ARCH: "amd64" + TAG: "android" + CONTEXT_DIR: "docker/android/" + DOCKERFILE: "docker/android/Dockerfile" + extends: .base + include: "gitlab/ci_template.yml" .build local: diff --git a/docker/android/Dockerfile b/docker/android/Dockerfile new file mode 100644 index 0000000000..b95dd87dea --- /dev/null +++ b/docker/android/Dockerfile @@ -0,0 +1,9 @@ +FROM registry.fedoraproject.org/fedora:29 + +ENV ANDROID_HOME=/android/sdk +ENV ANDROID_NDK_HOME=/android/ndk + +COPY prepare.sh cleanup.sh /root/ + +RUN /usr/bin/sh /root/prepare.sh && \ + /usr/bin/sh /root/cleanup.sh diff --git a/docker/android/cleanup.sh b/docker/android/cleanup.sh new file mode 100644 index 0000000000..73d62c773e --- /dev/null +++ b/docker/android/cleanup.sh @@ -0,0 +1,7 @@ +set -eu + +echo "Removing DNF cache" +dnf clean all + +rm -R /root/* +rm -rf /var/cache/dnf /var/log/dnf* diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh new file mode 100644 index 0000000000..58335ad32b --- /dev/null +++ b/docker/android/prepare.sh @@ -0,0 +1,32 @@ +set -eu + +dnf install -y \ + file \ + git-core \ + java-1.8.0-openjdk-devel \ + lbzip2 \ + make \ + pkg-config \ + unzip \ + which + +mkdir -p /android/sources + +curl -o /android/sources/android-ndk.zip https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip +unzip /android/sources/android-ndk.zip -d ${ANDROID_NDK_HOME}/ +# remove the intermediate versioned directory +mv ${ANDROID_NDK_HOME}/*/* ${ANDROID_NDK_HOME}/ + +curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip +unzip /android/sources/android-sdk-tools.zip -d ${ANDROID_HOME}/ +mkdir -p ${ANDROID_HOME}/licenses + +rm -rf /android/sources + +# Accept licenses. Values taken from: +# ANDROID_HOME=/path/to/android/sdk-tools $ANDROID_HOME/tools/bin/sdkmanager --licenses +echo "601085b94cd77f0b54ff86406957099ebe79c4d6" > ${ANDROID_HOME}/licenses/android-googletv-license +echo "24333f8a63b6825ea9c5514f83c2829b004d1fee" > ${ANDROID_HOME}/licenses/android-sdk-license +echo "84831b9409646a918e30573bab4c9c91346d8abd" > ${ANDROID_HOME}/licenses/android-sdk-preview-license +echo "33b6a2b64607f11b759f320ef9dff4ae5c47d97a" > ${ANDROID_HOME}/licenses/google-gdk-license +echo "e9acab5b5fbb560a72cfaecce8946896ff6aab9d" > ${ANDROID_HOME}/licenses/mips-android-sysimage-license From bc2bd98a7540184778de5ac9edf88f6902674664 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 26 Mar 2019 13:46:09 +0200 Subject: [PATCH 229/605] gitlab/ci_template: disable another test in check close #12 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 5bc3bdc61d..5abde30b41 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -177,6 +177,7 @@ build_disable_debug fedora x86_64: # check.gstreamer.libs_baseparse.parser_pull_short_read # check.gstreamer.pipelines_seek.test_loopback_2 # check.gst-plugins-base.elements_appsrc.test_appsrc_blocked_on_caps + # check.gst-devtools.validate.launcher_tests.test_validate.launch_pipeline.not_negotiated.caps_query_failure.play_15s BLACKLIST: > -b check.gstreamer.pipelines_parse_launch.delayed_link -b check.gstreamer.gst_gstsystemclock.test_async_sync_interaction @@ -219,6 +220,7 @@ build_disable_debug fedora x86_64: -b check.gst-rtsp-server.gst_rtspserver.test_shared_udp -b check.gst-plugins-base.elements_audiomixer.test_flush_start_flush_stop -b check.gstreamer-sharp.SdpTests + -b check.gst-devtools.validate.launcher_tests.test_validate.launch_pipeline.not_negotiated.caps_query_failure.play_15s script: - cd gst-build/ - > From 5fe97b9f1c5fb12281be8686579176536568ba0c Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 26 Mar 2019 14:00:31 +0200 Subject: [PATCH 230/605] registry: Change the way in which images are tagged Previously we tried using the git commit ref as a UID for the images. This does not work though cause multiple jobs that rebuild the image can be triggered and override the same image tag. Instead use the CI_JOB_ID to provide better semantics wrt naming conflicts. Additionally add the date as part of the tag to better indicate the age of the image. Gitlab WEBGUI doens't indicate the age in a good way nor makes it easy to short the images. --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6b52a41b73..60c9cf5c2e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,8 +32,9 @@ test manifest: DOCKER_DRIVER: "overlay2" GIT_STRATEGY: fetch script: + - export DATE=$(date +"%Y-%m-%d") - export IMAGE="${CI_REGISTRY_IMAGE}/${ARCH}/${TAG}" - - export REF="${IMAGE}:${CI_COMMIT_SHA}" + - export UID="${IMAGE}:${DATE}-${CI_JOB_ID}" - export LATEST="${IMAGE}:latest" - docker build --pull -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR} @@ -49,10 +50,10 @@ test manifest: - | if [ "$CI_COMMIT_REF_NAME" = "master" -a "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then # Tag the image with the git ref - docker image tag ${LATEST} ${REF} + docker image tag ${LATEST} ${UID} # Push the tags docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker push ${REF} + docker push ${UID} docker push ${LATEST} fi From ab873b5e58a58f7904b1c72c5ed3520fe2f0c2cb Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 26 Mar 2019 17:21:32 +1100 Subject: [PATCH 231/605] gitlab: add a script to clone a repository using manifest details --- gitlab/clone_manifest_ref.py | 86 ++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100755 gitlab/clone_manifest_ref.py diff --git a/gitlab/clone_manifest_ref.py b/gitlab/clone_manifest_ref.py new file mode 100755 index 0000000000..cbaf75deb1 --- /dev/null +++ b/gitlab/clone_manifest_ref.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess + +from collections import namedtuple +import xml.etree.ElementTree as ET + +# Disallow git prompting for a username/password +os.environ['GIT_TERMINAL_PROMPT'] = '0' +def git(*args, repository_path='.'): + return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() + +class Manifest(object): + ''' + Parse and store the content of a manifest file + ''' + + remotes = {} + projects = {} + default_remote = 'origin' + default_revision = 'refs/heads/master' + + def __init__(self, manifest_path): + self.manifest_path = manifest_path + + def parse(self): + try: + tree = ET.parse(self.manifest_path) + except Exception as ex: + raise Exception("Error loading manifest %s in file %s" % (self.manifest_path, ex)) + + root = tree.getroot() + + for child in root: + if child.tag == 'remote': + self.remotes[child.attrib['name']] = child.attrib['fetch'] + if child.tag == 'default': + self.default_remote = child.attrib['remote'] or self.default_remote + self.default_revision = child.attrib['revision'] or self.default_revision + if child.tag == 'project': + project = namedtuple('Project', ['name', 'remote', + 'revision', 'fetch_uri']) + + project.name = child.attrib['name'] + if project.name.endswith('.git'): + project.name = project.name[:-4] + project.remote = child.attrib.get('remote') or self.default_remote + project.revision = child.attrib.get('revision') or self.default_revision + project.fetch_uri = self.remotes[project.remote] + project.name + '.git' + + self.projects[project.name] = project + + def find_project(self, name): + try: + return self.projects[name] + except KeyError as ex: + raise Exception("Could not find project %s in manifest %s" % (name, self.manifest_path)) + + def get_fetch_uri(self, project, remote): + fetch = self.remotes[remote] + return fetch + project.name + '.git' + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--project", action="store", type=str) + parser.add_argument("--destination", action="store", type=str, default='.') + parser.add_argument("--manifest", action="store", type=str) + options = parser.parse_args() + + if not options.project: + raise ValueError("--project argument not provided") + if not options.manifest: + raise ValueError("--manifest argument not provided") + + manifest = Manifest(options.manifest) + manifest.parse() + project = manifest.find_project(options.project) + + dest = options.destination + if dest == '.': + dest = os.path.join (os.getcwd(), project.name) + + git('clone', project.fetch_uri, dest) + git('checkout', '--detach', project.revision, repository_path=dest) From c4a3df759d65fafdc9932935dbbc8e6638e4d6d0 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 24 Mar 2019 17:56:01 +1100 Subject: [PATCH 232/605] add CI for gst-examples android build --- .gitlab-ci.yml | 12 +++++++++ gitlab/build_manifest.py | 1 + gitlab/ci_template.yml | 53 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 60c9cf5c2e..fe6844ab62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,8 @@ stages: # Run multiple builds and tests, multi-distro, multi-arch - "full builds" - "full tests" + # build some apps to check that cross-platform binaries are usable + - "apps" test manifest: variables: @@ -172,3 +174,13 @@ check fedora local: extends: '.test fedora x86_64 local' variables: TEST_SUITE: 'check.gst*' + +android universal examples local: + extends: '.android universal examples' + image: "${CI_REGISTRY_IMAGE}/amd64/android:latest" + dependencies: + - "build cerbero android universal local" + when: 'manual' + except: + refs: + - "master@gstreamer/gst-ci" diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 0619847ae1..ec8901a063 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -28,6 +28,7 @@ GSTREAMER_MODULES: List[str] = [ 'gst-rtsp-server', 'gstreamer-vaapi', 'gst-integration-testsuites', + 'gst-examples', ] MANIFEST_TEMPLATE: str = """ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 5abde30b41..dea8fe1b49 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -6,8 +6,11 @@ stages: # Run multiple builds and tests, multi-distro, multi-arch - 'full builds' - 'full tests' + # build some apps to check that cross-platform binaries are usable + - 'apps' variables: + ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:fb91ee18309ab981289aa818ab2a7824ef0567a1' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' @@ -637,3 +640,53 @@ build cerbero cross win64: except: variables: - $CI_PROJECT_NAME == "cerbero" + +# +# Build an Android App using the android binaries +# +.android universal examples: + image: $ANDROID_IMAGE + stage: 'apps' + variables: + EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples + GSTREAMER_ROOT_ANDROID: ${CI_PROJECT_DIR}/examples/cerbero-android-universal + script: + - mkdir -p ${EXAMPLES_HOME} + - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/master/gitlab/clone_manifest_ref.py + - chmod +x clone_manifest_ref.py + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples + - rm clone_manifest_ref.py + + # extract our binaries + - rm -f gstreamer-1.0-android-universal-*-runtime.tar.bz2 + - mkdir ${GSTREAMER_ROOT_ANDROID} + - tar -C ${GSTREAMER_ROOT_ANDROID} -xf gstreamer-1.0-android-universal-*.tar.bz2 + + - chmod +x ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew + - ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/playback/player/android assembleDebug + after_script: + - rm -rf ${GSTREAMER_ROOT_ANDROID} + - mv gstreamer-1.0-android-universal-*.tar.bz2 ${EXAMPLES_HOME}/ + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${EXAMPLES_HOME}" + +android universal examples: + extends: ".android universal examples" + dependencies: + - "build cerbero android universal" + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + +cerbero android universal examples: + extends: ".android universal examples" + dependencies: + - "cerbero android universal" + only: + variables: + - $CI_PROJECT_NAME == "cerbero" From be70dc373565e1d81adb3e0c60fc5050d0cc993d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 27 Mar 2019 13:26:17 +1100 Subject: [PATCH 233/605] gitlab: disable check target in gst-examples Not particularly useful there --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index dea8fe1b49..76189c50e3 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -252,6 +252,7 @@ build_disable_debug fedora x86_64: except: variables: - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-examples" .test fedora x86_64: image: $FEDORA_IMAGE From 42c1dcdf59da9dac5c99ba252d4348247f58c706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Wed, 27 Mar 2019 15:17:23 +0000 Subject: [PATCH 234/605] gitlab/ci_template: shorten name for gstreamer nodebug build Shorter and nicer to read with spaces than underscores. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 76189c50e3..2821674f92 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -121,7 +121,7 @@ build fedora x86_64: variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled" -build_disable_debug fedora x86_64: +build nodebug fedora x86_64: extends: '.build' stage: 'build' image: $FEDORA_IMAGE From f398562cb3fb2d8667d98fa86ed09c78d4cc9692 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 28 Mar 2019 11:25:05 +1100 Subject: [PATCH 235/605] gitlab: also build-test the android tutorials from gst-docs --- gitlab/ci_template.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2821674f92..234f9a6014 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -656,6 +656,7 @@ build cerbero cross win64: - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/master/gitlab/clone_manifest_ref.py - chmod +x clone_manifest_ref.py - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs - rm clone_manifest_ref.py # extract our binaries @@ -665,6 +666,8 @@ build cerbero cross win64: - chmod +x ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew - ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/playback/player/android assembleDebug + - chmod +x ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew + - ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android assembleDebug after_script: - rm -rf ${GSTREAMER_ROOT_ANDROID} - mv gstreamer-1.0-android-universal-*.tar.bz2 ${EXAMPLES_HOME}/ From caec83aa1f419be373a4d595ca76652dbf5c949f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 28 Mar 2019 12:15:18 +1100 Subject: [PATCH 236/605] gitlab/android-examples: only output the resulting apks The full build directory while useful for debugging is rather large (almost 1GB per app depending on plugins used). --- gitlab/ci_template.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 234f9a6014..fb24c138dc 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -652,7 +652,7 @@ build cerbero cross win64: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples GSTREAMER_ROOT_ANDROID: ${CI_PROJECT_DIR}/examples/cerbero-android-universal script: - - mkdir -p ${EXAMPLES_HOME} + - mkdir -p ${EXAMPLES_HOME}/outputs - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/master/gitlab/clone_manifest_ref.py - chmod +x clone_manifest_ref.py - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples @@ -664,20 +664,25 @@ build cerbero cross win64: - mkdir ${GSTREAMER_ROOT_ANDROID} - tar -C ${GSTREAMER_ROOT_ANDROID} -xf gstreamer-1.0-android-universal-*.tar.bz2 + # gst-examples - chmod +x ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew - ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/playback/player/android assembleDebug + - cp ${EXAMPLES_HOME}/gst-examples/playback/player/android/app/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ + + # gst-docs android tutorials - chmod +x ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew - ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android assembleDebug + - cp ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/android-tutorial-*/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ after_script: - rm -rf ${GSTREAMER_ROOT_ANDROID} - - mv gstreamer-1.0-android-universal-*.tar.bz2 ${EXAMPLES_HOME}/ + - rm -rf ${EXAMPLES_HOME}/gst-examples ${EXAMPLES_HOME}/gst-docs artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" expire_in: '5 days' when: 'always' paths: - "manifest.xml" - - "${EXAMPLES_HOME}" + - "${EXAMPLES_HOME}/outputs" android universal examples: extends: ".android universal examples" From d3ecc6d7419d8b12928df587f75e25ace7d015b3 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 2 Apr 2019 23:27:59 +1100 Subject: [PATCH 237/605] gitlab: don't build cerbero jobs in gst-build Fixes https://gitlab.freedesktop.org/gstreamer/gst-ci/issues/17 --- gitlab/ci_template.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index fb24c138dc..9750fa23eb 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -557,6 +557,7 @@ build cerbero fedora x86_64: except: variables: - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-build" # @@ -588,6 +589,7 @@ build cerbero android universal: except: variables: - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-build" # # Cerbero Cross Windows builds @@ -616,6 +618,7 @@ build cerbero cross win32: except: variables: - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-build" cerbero deps windows x86_64: extends: '.cerbero deps' @@ -641,6 +644,7 @@ build cerbero cross win64: except: variables: - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-build" # # Build an Android App using the android binaries @@ -691,6 +695,7 @@ android universal examples: except: variables: - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-build" cerbero android universal examples: extends: ".android universal examples" From e06f324cf124f475514e138debecfd59970e1142 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 1 Feb 2019 14:24:40 -0300 Subject: [PATCH 238/605] Update to latest fedora 64 image --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9750fa23eb..f7f1940529 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -12,7 +12,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:fb91ee18309ab981289aa818ab2a7824ef0567a1' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' From edd8ff52e391a62c63b16c90e17fff431e49de52 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 27 Mar 2019 16:44:36 -0300 Subject: [PATCH 239/605] ci: Run 'validate' and 'ges' integration testsuites And use new validate feature to print debug logs URL directly in the logs instead of dumping the whole files. --- .gitlab-ci.yml | 9 +++++++++ gitlab/ci_template.yml | 25 +++++++++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fe6844ab62..e8d4044b65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -184,3 +184,12 @@ android universal examples local: except: refs: - "master@gstreamer/gst-ci" + +integration testsuites fedora local: + extends: '.test fedora x86_64 local' + before_script: + - rm -f gst-build/build/subprojects/gstreamer-vaapi/gst/vaapi/libgstvaapi.so + variables: + EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures" + TEST_SUITE: "validate ges" + diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f7f1940529..cdb06bcd66 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -81,7 +81,6 @@ gst indent: CXX: "ccache g++" CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" - MAIN_DIR: "${CI_PROJECT_DIR}/validate-output/" MESON_ARGS: "${DEFAULT_MESON_ARGS}" script: @@ -131,9 +130,10 @@ build nodebug fedora x86_64: .test: stage: 'test' variables: - MAIN_DIR: "${CI_PROJECT_DIR}/validate-output/" # Disable colored output to avoid weird rendering issues - GST_DEBUG_COLOR: 'no' + GST_DEBUG_NO_COLOR: "true" + CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" + GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" # note the -b at the start of each line # Can't comment inline sadly @@ -226,6 +226,7 @@ build nodebug fedora x86_64: -b check.gst-devtools.validate.launcher_tests.test_validate.launch_pipeline.not_negotiated.caps_query_failure.play_15s script: - cd gst-build/ + - echo "-> Running ${TEST_SUITE} testsuite." - > ./gst-uninstalled.py gst-validate-launcher ${TEST_SUITE} @@ -233,10 +234,10 @@ build nodebug fedora x86_64: --mute --shuffle --no-display - --dump-on-failure --meson-no-rebuild - -M "${CI_PROJECT_DIR}/validate-output/" - --xunit-file "${CI_PROJECT_DIR}/validate-output/logs/xunit.xml" + --fail-on-testlist-change + -l "${CI_PROJECT_DIR}/validate-logs/" + --xunit-file "${CI_PROJECT_DIR}/validate-logs/xunit.xml" ${BLACKLIST} ${EXTRA_VALIDATE_ARGS} artifacts: @@ -244,10 +245,10 @@ build nodebug fedora x86_64: when: always paths: - 'gst-build/build/meson-logs/' - - 'validate-output/logs' + - 'validate-logs' reports: junit: - - "validate-output/logs/*.xml" + - "validate-logs/*.xml" # We disable the .build above, which this job usually depends upon for cerbero except: variables: @@ -265,6 +266,14 @@ check fedora: variables: TEST_SUITE: "check.gst*" +integration testsuites fedora: + extends: '.test fedora x86_64' + before_script: + - rm -f gst-build/build/subprojects/gstreamer-vaapi/gst/vaapi/libgstvaapi.so + variables: + EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures" + TEST_SUITE: "validate ges" + # Valgrind .valgrind fedora x86_64: extends: '.test fedora x86_64' From 9d32304eb378483e2382739b050876349a98e5fd Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 27 Mar 2019 18:20:04 -0300 Subject: [PATCH 240/605] gitlab: Do not run check and apps job on gst-integration-testsuites --- gitlab/ci_template.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index cdb06bcd66..333124abfd 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -265,6 +265,9 @@ check fedora: extends: '.test fedora x86_64' variables: TEST_SUITE: "check.gst*" + except: + variables: + - $CI_PROJECT_NAME == "gst-integration-testsuites" integration testsuites fedora: extends: '.test fedora x86_64' @@ -696,6 +699,9 @@ build cerbero cross win64: paths: - "manifest.xml" - "${EXAMPLES_HOME}/outputs" + except: + variables: + - $CI_PROJECT_NAME == "gst-integration-testsuites" android universal examples: extends: ".android universal examples" From 4cbf9a6ed506a669eccc46821b57284dffa8b71c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 4 Apr 2019 21:32:46 +1100 Subject: [PATCH 241/605] gitlab: fix gst-integration-testuite exceptions Fixes error introduced in https://gitlab.freedesktop.org/gstreamer/gst-ci/commit/9d32304eb378483e2382739b050876349a98e5fd https://gitlab.freedesktop.org/gstreamer/gst-ci/merge_requests/94 --- gitlab/ci_template.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 333124abfd..4dde732f0f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -268,6 +268,8 @@ check fedora: except: variables: - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-examples" integration testsuites fedora: extends: '.test fedora x86_64' @@ -699,9 +701,6 @@ build cerbero cross win64: paths: - "manifest.xml" - "${EXAMPLES_HOME}/outputs" - except: - variables: - - $CI_PROJECT_NAME == "gst-integration-testsuites" android universal examples: extends: ".android universal examples" @@ -711,6 +710,7 @@ android universal examples: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-integration-testsuites" cerbero android universal examples: extends: ".android universal examples" From 1a3f1f66cd95891931c7f54db3791ae8879a89cc Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 4 Apr 2019 17:23:14 -0300 Subject: [PATCH 242/605] gitlab: The launcher now generate the right suppression files list See https://gitlab.freedesktop.org/gstreamer/gst-devtools/commit/787939f7505407949a104539688905bc4057d2d4 --- gitlab/ci_template.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4dde732f0f..d67e820ab4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -284,17 +284,6 @@ integration testsuites fedora: extends: '.test fedora x86_64' stage: 'full tests' variables: - SUPPRESSION_FILES: > - gst-build/subprojects/gstreamer/tests/check/gstreamer.supp - gst-build/subprojects/gst-plugins-base/tests/check/gst-plugins-base.supp - gst-build/subprojects/gst-plugins-good/tests/check/gst-plugins-good.supp - gst-build/subprojects/gst-plugins-bad/tests/check/gst-plugins-bad.supp - gst-build/subprojects/gst-plugins-ugly/tests/check/gst-plugins-ugly.supp - gst-build/subprojects/gst-libav/tests/check/gst-libav.supp - gst-build/subprojects/gst-devtools/validate/data/gstvalidate.supp - gst-build/subprojects/libnice/tests/libnice.supp - gst-build/subprojects/libsoup/tests/libsoup.supp - gst-build/subprojects/glib/glib.supp # These tests take very long compared to what they add, so let's skip them. # - check.*.generic_states.* - enough to run one of the sequences # - check.gstreamer.gst_gstelement.test_foreach_pad - 48s @@ -397,13 +386,10 @@ integration testsuites fedora: -b check.gst-libav.generic_plugin_test -b check.gst-libav.generic_libavcodec_locking -b check.gst-libav.elements_avdemux_ape - VALGRIND_OPTS: "--suppressions=/tmp/gstreamer-ci.supp" EXTRA_VALIDATE_ARGS: "--valgrind ${VALGRIND_BLACKLIST} ${VALGRIND_SKIPLIST}" ORC_CODE: "backup" # Some suppression files are missing a newline at the end which messes things # up when concatenating them. awk will add missing newlines (unlike cat) - before_script: - - awk '{print}' ${SUPPRESSION_FILES} > /tmp/gstreamer-ci.supp valgrind core: extends: '.valgrind fedora x86_64' From 8fb9cb147b0745542dcf06373e61f69cac8fb625 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 5 Apr 2019 13:14:21 +0300 Subject: [PATCH 243/605] gitlab/ci_template: Build gst-omx as well --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d67e820ab4..9dc7473550 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -118,14 +118,14 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic" build nodebug fedora x86_64: extends: '.build' stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic" .test: stage: 'test' From 57848d32f33e7c39fe5fd2717febced0e53996d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sat, 6 Apr 2019 17:45:22 +0100 Subject: [PATCH 244/605] gitlab/ci_template: skip cerbero builds and integration tests for gst-omx and the integration test for gstreamer-vaapi (we explicitly remove the vaapi plugin when running integration tests). --- gitlab/ci_template.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9dc7473550..b3db880262 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -270,6 +270,7 @@ check fedora: - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-examples" + - $CI_PROJECT_NAME == "gst-omx" integration testsuites fedora: extends: '.test fedora x86_64' @@ -278,6 +279,10 @@ integration testsuites fedora: variables: EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures" TEST_SUITE: "validate ges" + except: + variables: + - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gstreamer-vaapi" # Valgrind .valgrind fedora x86_64: @@ -558,6 +563,7 @@ build cerbero fedora x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-omx" # @@ -590,6 +596,7 @@ build cerbero android universal: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-omx" # # Cerbero Cross Windows builds @@ -619,6 +626,7 @@ build cerbero cross win32: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-omx" cerbero deps windows x86_64: extends: '.cerbero deps' @@ -645,6 +653,7 @@ build cerbero cross win64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-omx" # # Build an Android App using the android binaries @@ -696,6 +705,7 @@ android universal examples: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-integration-testsuites" cerbero android universal examples: From 22a3bfeeb523ced9b4be1ea2f35c14078498d997 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sun, 7 Apr 2019 18:00:02 -0400 Subject: [PATCH 245/605] gitlab: Disable integration test in cerbero repository This is minimal fix for this job, we don't build the required dependency. --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b3db880262..aa40dace87 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -283,6 +283,7 @@ integration testsuites fedora: variables: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "cerbero" # Valgrind .valgrind fedora x86_64: From 69f1f7760bb08c6252686b646ae880c9478dd350 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 7 Apr 2019 09:48:42 -0400 Subject: [PATCH 246/605] fedora: Install temrinal markdown viewer To enhance the launcher output --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 3a49e895a4..fecdd84912 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -151,7 +151,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.49.2 +pip3 install meson==0.49.2 git+https://github.com/axiros/terminal_markdown_viewer # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From 074a0eecbf2be0d4e600cdb98b6c2804128005ee Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 7 Apr 2019 09:49:14 -0400 Subject: [PATCH 247/605] gitlab: Dump launcher logs on failures --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index aa40dace87..36ee155081 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -230,6 +230,7 @@ build nodebug fedora x86_64: - > ./gst-uninstalled.py gst-validate-launcher ${TEST_SUITE} + --dump-on-failure -fs --mute --shuffle From b55b0f7a7450b8ef16bb00d20402b1f9fd13ca39 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 12 Apr 2019 18:26:38 -0400 Subject: [PATCH 248/605] gitlab: Run valgrind on ges tests blacklisting python tests for now as those are still leaky/need some python suppressions --- gitlab/ci_template.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 36ee155081..c11d0786ae 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -226,7 +226,7 @@ build nodebug fedora x86_64: -b check.gst-devtools.validate.launcher_tests.test_validate.launch_pipeline.not_negotiated.caps_query_failure.play_15s script: - cd gst-build/ - - echo "-> Running ${TEST_SUITE} testsuite." + - echo "-> Running ${TEST_SUITE} testsuite with ${BLACKLIST} and ${EXTRA_VALIDATE_ARGS}" - > ./gst-uninstalled.py gst-validate-launcher ${TEST_SUITE} @@ -327,6 +327,7 @@ integration testsuites fedora: # - check.gst-plugins-good.elements_rtpjitterbuffer.test_fill_queue - 22s # - check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async - 20s # - check.gst-plugins-good.elements_videomixer.test_play_twice - 22s + # - check.gst-editing-services.nle_simple.test_one_after_other - 40s VALGRIND_SKIPLIST: > -b check.[a-z-]*.generic_states.test_state_changes_down_seq -b check.[a-z-]*.generic_states.test_state_changes_up_seq @@ -364,12 +365,14 @@ integration testsuites fedora: -b check.gst-plugins-good.elements_rtpjitterbuffer.test_fill_queue -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async -b check.gst-plugins-good.elements_videomixer.test_play_twice + -b check.gst-editing-services.nle_simple.test_one_after_other # - check.gst-plugins-good.elements_rtpjitterbuffer.test_push_* - flaky in valgrind # - check.gst-plugins-base.pipelines_gl_launch_lines - driver leaks / memory access # - check.gst-plugins-base.libs_gstgl - driver leaks / memory access # - check.gst-plugins-base.elements_gl - driver leaks / memory access # - check.gst-plugins-base.elements_libvisual - uninitialized memory access # - check.gst-plugins-base.generic_states - need to add gl elements to ignore list but only if using valgrind + # - "check.gst-editing-services.pythontests.*" - Need to figure out how to introduce python suppressions VALGRIND_BLACKLIST: > -b check.gstreamer.gst_gstsystemclock.test_stress_cleanup_unschedule -b check.gstreamer.gst_gstsystemclock.test_stress_reschedule @@ -393,6 +396,7 @@ integration testsuites fedora: -b check.gst-libav.generic_plugin_test -b check.gst-libav.generic_libavcodec_locking -b check.gst-libav.elements_avdemux_ape + -b check.gst-editing-services.pythontests EXTRA_VALIDATE_ARGS: "--valgrind ${VALGRIND_BLACKLIST} ${VALGRIND_SKIPLIST}" ORC_CODE: "backup" # Some suppression files are missing a newline at the end which messes things @@ -438,6 +442,13 @@ valgrind bad: variables: - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad)$/ +valgrind ges: + extends: '.valgrind fedora x86_64' + variables: + TEST_SUITE: "check.gst-editing-services\\..*" + only: + variables: + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services)$/ # Template for Cerbero GStreamer Build # From 5c5554622bc12ba6b0a2932312ac598549981b6e Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 17 Apr 2019 10:06:32 -0400 Subject: [PATCH 249/605] gitlab: Blacklist moved to the check testsuite directly --- gitlab/ci_template.yml | 201 +---------------------------------------- 1 file changed, 2 insertions(+), 199 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c11d0786ae..a5803b7ae7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -134,99 +134,9 @@ build nodebug fedora x86_64: GST_DEBUG_NO_COLOR: "true" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" - - # note the -b at the start of each line - # Can't comment inline sadly - # FIXME: get this into gst-validate-launcher KNOWN_ISSUES - # gstreamer.pipelines_parse_launch.delayed_link: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/345 - # gstreamer.gst_gstsystemclock.test_async_sync_interaction: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/346 - # gstreamer.gst_gstsystemclock.test_periodic_multi: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/347 - # gstreamer.gst_gstsystemclock.test_periodic_shot: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/348 - # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 - # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 - # flvmux.test_incrementing_timestamps: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 - # flvmux.test_video_caps_late: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/543 - # rtpbin.test_sender_eos: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 - # rtpbin.test_cleanup_recv: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/546 - # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 - # rtspserver.test_shared_udp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/61 - # rtpsession.test_multiple_senders_roundrobin_rbs: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/548 - # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 - # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 - # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 - # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 - # splitmux.test_splitmuxsrc_caps_change: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/547 - # splitmux.test_splitmuxsrc_sparse_streams: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/544 - # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 - # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 - # nle_simple.test_simplest: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/57 - # ges_basic.test_ges_pipeline_change_state: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/58 - # gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_transition_type: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/62 - # gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_auto_transition: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/63 - # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 - # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 - # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 - # elements_audiotestsrc.test_layout: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/535 - # camerabin.test_image_video_cycle: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 - # camerabin.test_single_video_recording: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101558 - # camerabin.test_multiple_video_recordings: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101646 - # curlhttpsrc.test_multiple_http_requests: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/932 - # audiomixer.test_flush_start_flush_stop: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/555 - # check.gstreamer-sharp.SdpTests: https://gitlab.freedesktop.org/gstreamer/gstreamer-sharp/issues/17 - # - # TO FURTHER INVESTIGATE: - # check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer - # check.gstreamer.gst_gstelement.test_foreach_pad - # check.gstreamer.libs_baseparse.parser_pull_short_read - # check.gstreamer.pipelines_seek.test_loopback_2 - # check.gst-plugins-base.elements_appsrc.test_appsrc_blocked_on_caps - # check.gst-devtools.validate.launcher_tests.test_validate.launch_pipeline.not_negotiated.caps_query_failure.play_15s - BLACKLIST: > - -b check.gstreamer.pipelines_parse_launch.delayed_link - -b check.gstreamer.gst_gstsystemclock.test_async_sync_interaction - -b check.gstreamer.gst_gstsystemclock.test_periodic_multi - -b check.gstreamer.gst_gstsystemclock.test_periodic_shot - -b check.gstreamer.pipelines_seek.test_loopback_2 - -b check.gstreamer.gst_gstelement.test_foreach_pad - -b check.gstreamer.libs_baseparse.parser_pull_short_read - -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories - -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe - -b check.gst-plugins-base.elements_multisocketsink.test_add_client - -b check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer - -b check.gst-plugins-base.elements_audiotestsrc.test_layout - -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream - -b check.gst-plugins-good.elements_rtpbin.test_sender_eos - -b check.gst-plugins-good.elements_rtpbin.test_cleanup_recv - -b check.gst-plugins-good.elements_flvmux.test_incrementing_timestamps - -b check.gst-plugins-good.elements_flvmux.test_video_caps_late - -b check.gst-plugins-base.elements_appsrc.test_appsrc_blocked_on_caps - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_caps_change - -b check.gst-plugins-bad.elements_dtls.test_data_transfer - -b check.gst-plugins-bad.elements_dtls.test_create_and_unref - -b check.gst-plugins-bad.elements_camerabin.test_image_video_cycle - -b check.gst-plugins-bad.elements_camerabin.test_single_video_recording - -b check.gst-plugins-bad.elements_camerabin.test_multiple_video_recordings - -b check.gst-plugins-bad.elements_curlhttpsrc.test_multiple_http_requests - -b check.gst-plugins-good.elements_rtpsession.test_multiple_senders_roundrobin_rbs - -b check.gst-plugins-bad.elements_shm.test_shm_live - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async - -b check.gst-plugins-bad.elements_netsim.netsim_stress - -b check.gst-editing-services.nle_complex.test_one_expandable_another - -b check.gst-editing-services.nle_simple.test_simplest - -b check.gst-editing-services.ges_basic.test_ges_pipeline_change_state - -b check.gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_transition_type - -b check.gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_auto_transition - -b check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical - -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps - -b check.gst-rtsp-server.gst_rtspclientsink.test_record - -b check.gst-rtsp-server.gst_rtspserver.test_shared_udp - -b check.gst-plugins-base.elements_audiomixer.test_flush_start_flush_stop - -b check.gstreamer-sharp.SdpTests - -b check.gst-devtools.validate.launcher_tests.test_validate.launch_pipeline.not_negotiated.caps_query_failure.play_15s script: - cd gst-build/ - - echo "-> Running ${TEST_SUITE} testsuite with ${BLACKLIST} and ${EXTRA_VALIDATE_ARGS}" + - echo "-> Running ${TEST_SUITE}" - > ./gst-uninstalled.py gst-validate-launcher ${TEST_SUITE} @@ -239,7 +149,6 @@ build nodebug fedora x86_64: --fail-on-testlist-change -l "${CI_PROJECT_DIR}/validate-logs/" --xunit-file "${CI_PROJECT_DIR}/validate-logs/xunit.xml" - ${BLACKLIST} ${EXTRA_VALIDATE_ARGS} artifacts: expire_in: '14 days' @@ -291,113 +200,7 @@ integration testsuites fedora: extends: '.test fedora x86_64' stage: 'full tests' variables: - # These tests take very long compared to what they add, so let's skip them. - # - check.*.generic_states.* - enough to run one of the sequences - # - check.gstreamer.gst_gstelement.test_foreach_pad - 48s - # - check.gstreamer.gst_gstinfo.info_post_gst_init_category_registration - 21s - # - check.gstreamer.gst_gstsystemclock.test_resolution - 60s - # - check.gstreamer.libs_aggregator.test_infinite_seek - 20s - # - check.gstreamer.libs_aggregator.test_infinite_seek_50_src - 20s - # - check.gstreamer.libs_gstharness.test_harness_element_ref - 20s - # - check.gstreamer.pipelines_simple_launch_lines.test_2_elements - 58s - # - check.gstreamer.pipelines_stress.test_stress - 54s - # - check.gstreamer.pipelines_stress.test_stress_preroll - 27s - # - check.gst-plugins-base.elements_appsrc.test_appsrc_block_deadlock - 265.595s - # - check.gst-plugins-base.elements_audioresample.test_fft - 91.247s - # - check.gst-plugins-base.elements_audioresample.test_timestamp_drift - 141.784s - # - check.gst-plugins-base.elements-videoscale - superlong - # - check.gst-plugins-base.libs_video.test_overlay_blend - 74.096s - # - check.gst-plugins-base.libs_video.test_video_color_convert - 345.271s - # - check.gst-plugins-base.libs_video.test_video_formats - 70.987s - # - check.gst-plugins-base.libs_video.test_video_size_convert - 56.387s - # - check.gst-plugins-base.elements_audiointerleave.test_audiointerleave_2ch_pipeline_ - 5 * 51.069s - # - check.gst-plugins-base.elements_multifdsink.test_client_kick - 46.909s - # - check.gst-plugins-base.elements_videotestsrc.test_all_patterns - # - check.gst-plugins-base.elements_videotestsrc.test_patterns_are_deterministic - # - check.gst-plugins-good.elements_shapewipe.test_general - 325s - # - check.gst-plugins-good.elements_videocrop.test_cropping - 245s - # - check.gst-plugins-good.elements_videomixer - 30s (also deprecated) - # - check.gst-plugins-good.elements_rtp_payloading.rtp_jpeg_packet_loss - 109s - # - check.gst-plugins-good.elements_videomixer.test_play_twice_then_add_and_play_again - 55s - # - check.gst-plugins-good.pipelines_effectv.test_quarktv - 53s - # - check.gst-plugins-good.elements_deinterlace.test_mode_disabled_passthrough - 52s - # - check.gst-plugins-good.elements_deinterlace.test_mode_auto_deinterlaced_passthrough - 28s - # - check.gst-plugins-good.elements_deinterleave.test_2_channels_caps_change - 30s - # - check.gst-plugins-good.elements_deinterleave.test_2_channels - 22s - # - check.gst-plugins-good.elements_rtpjitterbuffer.test_fill_queue - 22s - # - check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async - 20s - # - check.gst-plugins-good.elements_videomixer.test_play_twice - 22s - # - check.gst-editing-services.nle_simple.test_one_after_other - 40s - VALGRIND_SKIPLIST: > - -b check.[a-z-]*.generic_states.test_state_changes_down_seq - -b check.[a-z-]*.generic_states.test_state_changes_up_seq - -b check.gstreamer.gst_gstelement.test_foreach_pad - -b check.gstreamer.gst_gstinfo.info_post_gst_init_category_registration - -b check.gstreamer.gst_gstsystemclock.test_resolution - -b check.gstreamer.libs_aggregator.test_infinite_seek - -b check.gstreamer.libs_aggregator.test_infinite_seek_50_src - -b check.gstreamer.libs_gstharness.test_harness_element_ref - -b check.gstreamer.pipelines_simple_launch_lines.test_2_elements - -b check.gstreamer.pipelines_stress.test_stress - -b check.gstreamer.pipelines_stress.test_stress_preroll - -b check.gst-plugins-base.elements_appsrc.test_appsrc_block_deadlock - -b check.gst-plugins-base.elements_audioresample.test_fft - -b check.gst-plugins-base.elements_audioresample.test_timestamp_drift - -b check.gst-plugins-base.elements-videoscale - -b check.gst-plugins-base.libs_video.test_overlay_blend - -b check.gst-plugins-base.libs_video.test_video_color_convert - -b check.gst-plugins-base.libs_video.test_video_formats - -b check.gst-plugins-base.libs_video.test_video_size_convert - -b check.gst-plugins-base.elements_audiointerleave.test_audiointerleave_2ch_pipeline_ - -b check.gst-plugins-base.elements_multifdsink.test_client_kick - -b check.gst-plugins-base.elements_videotestsrc.test_all_patterns - -b check.gst-plugins-base.elements_videotestsrc.test_patterns_are_deterministic - -b check.gst-plugins-good.elements_shapewipe.test_general - -b check.gst-plugins-good.elements_videocrop.test_cropping - -b check.gst-plugins-good.elements_videomixer - -b check.gst-plugins-good.elements_rtp_payloading.rtp_jpeg_packet_loss - -b check.gst-plugins-good.elements_videomixer.test_play_twice_then_add_and_play_again - -b check.gst-plugins-good.pipelines_effectv.test_quarktv - -b check.gst-plugins-good.elements_deinterlace.test_mode_disabled_passthrough - -b check.gst-plugins-good.elements_deinterlace.test_mode_auto_deinterlaced_passthrough - -b check.gst-plugins-good.elements_deinterleave.test_2_channels_caps_change - -b check.gst-plugins-good.elements_deinterleave.test_2_channels - -b check.gst-plugins-good.elements_rtpjitterbuffer.test_fill_queue - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async - -b check.gst-plugins-good.elements_videomixer.test_play_twice - -b check.gst-editing-services.nle_simple.test_one_after_other - # - check.gst-plugins-good.elements_rtpjitterbuffer.test_push_* - flaky in valgrind - # - check.gst-plugins-base.pipelines_gl_launch_lines - driver leaks / memory access - # - check.gst-plugins-base.libs_gstgl - driver leaks / memory access - # - check.gst-plugins-base.elements_gl - driver leaks / memory access - # - check.gst-plugins-base.elements_libvisual - uninitialized memory access - # - check.gst-plugins-base.generic_states - need to add gl elements to ignore list but only if using valgrind - # - "check.gst-editing-services.pythontests.*" - Need to figure out how to introduce python suppressions - VALGRIND_BLACKLIST: > - -b check.gstreamer.gst_gstsystemclock.test_stress_cleanup_unschedule - -b check.gstreamer.gst_gstsystemclock.test_stress_reschedule - -b check.gstreamer.tools_gstinspect - -b check.gst-plugins-base.elements_videoscale - -b check.gst-plugins-base.pipelines_gl_launch_lines - -b check.gst-plugins-base.libs_gstgl - -b check.gst-plugins-base.elements_gl - -b check.gst-plugins-base.elements_libvisual - -b check.gst-plugins-base.generic_states - -b check.gst-plugins-good.elements_rtpjitterbuffer.test_push_backward_seq - -b check.gst-plugins-good.elements_rtpjitterbuffer.test_push_unordered - -b check.gst-plugins-bad.elements_assrender - -b check.gst-plugins-bad.elements_camerabin - -b check.gst-plugins-bad.elements_line21 - -b check.gst-plugins-bad.elements_mpeg2enc - -b check.gst-plugins-bad.elements_mplex - -b check.gst-plugins-bad.elements_mxfmux - -b check.gst-plugins-bad.elements_x265enc - -b check.gst-plugins-bad.elements_zbar - -b check.gst-libav.generic_plugin_test - -b check.gst-libav.generic_libavcodec_locking - -b check.gst-libav.elements_avdemux_ape - -b check.gst-editing-services.pythontests - EXTRA_VALIDATE_ARGS: "--valgrind ${VALGRIND_BLACKLIST} ${VALGRIND_SKIPLIST}" + EXTRA_VALIDATE_ARGS: "--valgrind" ORC_CODE: "backup" # Some suppression files are missing a newline at the end which messes things # up when concatenating them. awk will add missing newlines (unlike cat) From 3c6bf43ce3b5b0b371293669ad7835c2e10ec046 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 17 Apr 2019 11:05:11 -0400 Subject: [PATCH 250/605] gitlab: ORC_CODE=backup is now set in validate GstCheck application --- gitlab/ci_template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a5803b7ae7..10e15179c7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -201,7 +201,6 @@ integration testsuites fedora: stage: 'full tests' variables: EXTRA_VALIDATE_ARGS: "--valgrind" - ORC_CODE: "backup" # Some suppression files are missing a newline at the end which messes things # up when concatenating them. awk will add missing newlines (unlike cat) From 4c1a551c912e36df7c87c3b037578b52e5868095 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 12 Feb 2019 13:49:30 +0200 Subject: [PATCH 251/605] Fedora docker: Bump to f30 Needed to get an upgraded ffmpeg. Also required to have a new enough version of gobject-introspection and remove all warnings --- docker/fedora/Dockerfile | 2 +- docker/fedora/prepare.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index 41911102dd..31bdd58c63 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:29 +FROM registry.fedoraproject.org/fedora:30 COPY prepare.sh cleanup.sh /root/ diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index fecdd84912..fdfadbd501 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -7,11 +7,11 @@ git config --global user.email "gst-build@gstreamer.net" git config --global user.name "Gstbuild Runner" # Add rpm fusion repositories in order to access all of the gst plugins -dnf install -y "http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-29.noarch.rpm" \ - "http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-29.noarch.rpm" +dnf install -y "http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-30.noarch.rpm" \ + "http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-30.noarch.rpm" -rpm --import "/etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-nonfree-fedora-29" -rpm --import "/etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-29" +rpm --import "/etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-nonfree-fedora-30" +rpm --import "/etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-30" dnf upgrade -y # Enable the cisco openh264 repo From fee11e9374168372d63d4e19f845aae134601a5d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sat, 10 Nov 2018 23:34:59 -0300 Subject: [PATCH 252/605] docker: Install hotdoc and required packages to build the doc --- docker/fedora/prepare.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index fdfadbd501..68d6034c46 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -25,6 +25,7 @@ dnf install -y \ libaom-devel \ ccache \ cmake \ + clang-devel \ elfutils \ elfutils-libs \ elfutils-devel \ @@ -53,9 +54,15 @@ dnf install -y \ libnice-devel \ libunwind \ libunwind-devel \ + libyaml-devel \ + libxml2-devel \ + libxslt-devel \ + llvm-devel \ + make \ neon \ neon-devel \ nunit \ + npm \ opencv \ opencv-devel \ openjpeg2 \ @@ -69,6 +76,7 @@ dnf install -y \ x264 \ x264-libs \ x264-devel \ + python3-devel \ python3-gobject \ python3-cairo \ python3-cairo-devel \ @@ -151,7 +159,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.49.2 git+https://github.com/axiros/terminal_markdown_viewer +pip3 install meson==0.49.2 git+https://github.com/axiros/terminal_markdown_viewer hotdoc==0.9.1 # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ @@ -177,8 +185,11 @@ dnf remove -y "gstreamer1*devel" # it leads to build issues in examples. dnf remove -y "qt5-qtbase-devel" +# FIXME: Why does installing directly with dnf *fails* ? +dnf download glib2-doc +rpm -i glib2-doc*.rpm + # get gst-build and make all subprojects available git clone git://anongit.freedesktop.org/gstreamer/gst-build /gst-build/ cd /gst-build meson subprojects download - From 1c249d24aebb4fe34309ca4962cec3e0468c45a1 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 22 Apr 2019 12:25:29 -0400 Subject: [PATCH 253/605] gitlab: Do not pass --force-sync in the launcher We let subproject handle it for us now --- gitlab/ci_template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 10e15179c7..adaa80a386 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -141,7 +141,6 @@ build nodebug fedora x86_64: ./gst-uninstalled.py gst-validate-launcher ${TEST_SUITE} --dump-on-failure - -fs --mute --shuffle --no-display From e2db59567889fc14c8ed92941e9cdafbf087ae07 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 22 Apr 2019 12:56:06 -0400 Subject: [PATCH 254/605] docker:fedora: Use meson 0.50.1 --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 68d6034c46..61f15ca7e5 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -159,7 +159,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.49.2 git+https://github.com/axiros/terminal_markdown_viewer hotdoc==0.9.1 +pip3 install meson==0.50.1 git+https://github.com/axiros/terminal_markdown_viewer hotdoc==0.9.1 # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From 09c9517c64ea00cc99bd23a6d34137d18a61b194 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 25 Apr 2019 16:48:02 -0400 Subject: [PATCH 255/605] docker: Explicitely install flex before installing hotdoc --- docker/fedora/prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 61f15ca7e5..76e9ff3c48 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -42,6 +42,7 @@ dnf install -y \ ffmpeg \ ffmpeg-libs \ ffmpeg-devel \ + flex \ flite \ flite-devel \ mono-devel \ From a659ed8f2ed5befd8ce449a510fa08bb4a2e2aee Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 1 May 2019 09:50:58 -0400 Subject: [PATCH 256/605] gitlab: Check bugs resolutions when running the validate/ges testsuite --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index adaa80a386..b8c20ad850 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -186,7 +186,7 @@ integration testsuites fedora: before_script: - rm -f gst-build/build/subprojects/gstreamer-vaapi/gst/vaapi/libgstvaapi.so variables: - EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures" + EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs" TEST_SUITE: "validate ges" except: variables: From 1e27cc81b167a6f0c8cfb3dcdd65b8d37d8c8ee3 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 2 May 2019 17:28:42 -0400 Subject: [PATCH 257/605] gitlab: manifest: Use a cerbero ref with build For non-cerbero builds, pick a cerbero reference for which a build has completed. This will reduce the number of cache miss, hence reduce the number of timeouts and slow build we are facing each time cerbero is updated. Fixes #16 --- gitlab/build_manifest.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index ec8901a063..f8c7b016fb 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -2,12 +2,14 @@ import argparse import os -import requests import sys import subprocess +import urllib.error +import urllib.parse +import urllib.request +import json from typing import Dict, Tuple, List -from urllib.parse import urlparse # from pprint import pprint GSTREAMER_MODULES: List[str] = [ @@ -44,6 +46,23 @@ os.environ['GIT_TERMINAL_PROMPT'] = '0' def git(*args, repository_path='.'): return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() +def get_cerbero_last_build_info (namespace : str, branch : str): + base_url = f"https://gitlab.freedesktop.org/{namespace}/cerbero/-/jobs" + url = f"{base_url}/artifacts/{branch}/raw/cerbero-build/cerbero-deps.log" + deps = [{'commit': None}] + + try: + # The logs are only available if all jobs have passed so it does not + # matter which distro/arch is picked. + values = { 'job': "cerbero deps fedora x86_64" } + data = urllib.parse.urlencode(values) + req = urllib.request.Request(f"{url}?{data}") + resp = urllib.request.urlopen(req); + deps = json.loads(resp.read()) + except urllib.error.URLError: + return None + + return deps[0]['commit'] def get_branches_info(module: str, namespace: str, branches: List[str]) -> Tuple[str, str]: try: @@ -54,6 +73,13 @@ def get_branches_info(module: str, namespace: str, branches: List[str]) -> Tuple if not res: return None, None + # Special case cerbero to avoid cache misses + if module == 'cerbero': + for branch in branches: + sha = get_cerbero_last_build_info(namespace, branch) + if sha is not None: + return sha, sha + lines = res.split('\n') for branch in branches: for line in lines: From ebb38fc3a0695a82315eca0a4eaf1c9780aa83b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 2 May 2019 09:25:57 +0100 Subject: [PATCH 258/605] gitlab/ci_template: no need to do cerbero builds for gstreamer-vaapi --- gitlab/ci_template.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b8c20ad850..5af0fa01b9 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -378,6 +378,7 @@ build cerbero fedora x86_64: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gstreamer-vaapi" # @@ -411,6 +412,7 @@ build cerbero android universal: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gstreamer-vaapi" # # Cerbero Cross Windows builds @@ -441,6 +443,7 @@ build cerbero cross win32: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gstreamer-vaapi" cerbero deps windows x86_64: extends: '.cerbero deps' @@ -468,6 +471,7 @@ build cerbero cross win64: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gstreamer-vaapi" # # Build an Android App using the android binaries @@ -521,6 +525,7 @@ android universal examples: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gstreamer-vaapi" cerbero android universal examples: extends: ".android universal examples" From f7bcf8dd6a8a298b0a6307da2475344ec38cd931 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 13 May 2019 11:26:23 -0400 Subject: [PATCH 259/605] docker: Update hotdoc to latest release --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 76e9ff3c48..e433f0bc6a 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -160,7 +160,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.50.1 git+https://github.com/axiros/terminal_markdown_viewer hotdoc==0.9.1 +pip3 install meson==0.50.1 git+https://github.com/axiros/terminal_markdown_viewer hotdoc==0.9.3 # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From 9e62bd32c31ebdb35033438b5c72621d5c537238 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 13 May 2019 13:56:06 -0400 Subject: [PATCH 260/605] docker: Always get latest hotdoc release We had another little mistake to fix and that might happen often in the future so let's use latest releases for now. --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index e433f0bc6a..633b69239f 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -160,7 +160,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.50.1 git+https://github.com/axiros/terminal_markdown_viewer hotdoc==0.9.3 +pip3 install meson==0.50.1 git+https://github.com/axiros/terminal_markdown_viewer hotdoc # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From 58861fd41092c625caf9f577d9f84574654a7cca Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sat, 10 Nov 2018 23:34:59 -0300 Subject: [PATCH 261/605] Add support for building the doc --- .gitlab-ci.yml | 2 ++ gitlab/ci_template.yml | 32 +++++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8d4044b65..7a232632a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,8 @@ stages: - "full tests" # build some apps to check that cross-platform binaries are usable - "apps" + # Deploy the documentation + - "deploy" test manifest: variables: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 5af0fa01b9..fb3979783a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,11 +8,12 @@ stages: - 'full tests' # build some apps to check that cross-platform binaries are usable - 'apps' + - 'deploy' variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-13-295487' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' @@ -71,6 +72,7 @@ gst indent: - $CI_PROJECT_NAME == "gstreamer-sharp" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" .build: stage: 'full builds' @@ -118,7 +120,7 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled" build nodebug fedora x86_64: extends: '.build' @@ -163,6 +165,7 @@ build nodebug fedora x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-examples" + - $CI_PROJECT_NAME == "gst-docs" .test fedora x86_64: image: $FEDORA_IMAGE @@ -180,6 +183,7 @@ check fedora: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-examples" - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gst-docs" integration testsuites fedora: extends: '.test fedora x86_64' @@ -193,6 +197,7 @@ integration testsuites fedora: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" # Valgrind .valgrind fedora x86_64: @@ -293,6 +298,7 @@ valgrind ges: except: variables: - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-docs" cache: key: "${CI_JOB_NAME}" paths: @@ -379,7 +385,7 @@ build cerbero fedora x86_64: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - + - $CI_PROJECT_NAME == "gst-docs" # # Cerbero Android Universal build @@ -413,6 +419,7 @@ build cerbero android universal: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" # # Cerbero Cross Windows builds @@ -444,6 +451,7 @@ build cerbero cross win32: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" cerbero deps windows x86_64: extends: '.cerbero deps' @@ -472,6 +480,7 @@ build cerbero cross win64: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" # # Build an Android App using the android binaries @@ -526,6 +535,7 @@ android universal examples: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" cerbero android universal examples: extends: ".android universal examples" @@ -534,3 +544,19 @@ cerbero android universal examples: only: variables: - $CI_PROJECT_NAME == "cerbero" + +documentation: + image: $FEDORA_IMAGE + dependencies: + - 'build fedora x86_64' + stage: deploy + + script: + - cd gst-build/ + - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings + - cd - + - mv gst-build/build/subprojects/gst-docs/GStreamer-doc/html documentation/ + + artifacts: + paths: + - documentation/ From 7ecf44d2327555bd5df8197caf2acc99102f97df Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 19 Apr 2019 17:42:14 -0400 Subject: [PATCH 262/605] gitlab: Do not "build nodebug fedora" on gst-docs And do not build at all for gst-examples either --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index fb3979783a..9361c440f6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -128,6 +128,10 @@ build nodebug fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic" + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" .test: stage: 'test' From 4e5707d2b3f88833282a3fca1ea45792ea3ab78d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 13 May 2019 19:35:55 -0400 Subject: [PATCH 263/605] doc-importer: Update job name now that we build the API doc --- gitlab/freedesktop_doc_importer.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gitlab/freedesktop_doc_importer.sh b/gitlab/freedesktop_doc_importer.sh index f93d23f631..be68222d0f 100755 --- a/gitlab/freedesktop_doc_importer.sh +++ b/gitlab/freedesktop_doc_importer.sh @@ -4,6 +4,7 @@ set -e BRANCH=master NAMESPACE=gstreamer +JOB=documentation WORK_DIR=`mktemp -d -p "$DIR"` @@ -18,12 +19,12 @@ trap cleanup EXIT echo "" echo "============================================================================================================================" -echo "Updating documentation from: https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=deploy" +echo "Updating documentation from: https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=$JOB" date cd $WORK_DIR -wget https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=deploy -O gstdocs.zip +wget https://gitlab.freedesktop.org/$NAMESPACE/gst-docs/-/jobs/artifacts/$BRANCH/download?job=$JOB -O gstdocs.zip unzip gstdocs.zip From 68fa0afea7529eb4c16396d48e715465074e6e40 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 13 May 2019 21:48:36 -0400 Subject: [PATCH 264/605] Revert "Add support for building the doc" This reverts commit 58861fd41092c625caf9f577d9f84574654a7cca. --- .gitlab-ci.yml | 2 -- gitlab/ci_template.yml | 32 +++----------------------------- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a232632a0..e8d4044b65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,6 @@ stages: - "full tests" # build some apps to check that cross-platform binaries are usable - "apps" - # Deploy the documentation - - "deploy" test manifest: variables: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9361c440f6..ecba540c1c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,12 +8,11 @@ stages: - 'full tests' # build some apps to check that cross-platform binaries are usable - 'apps' - - 'deploy' variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-13-295487' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' @@ -72,7 +71,6 @@ gst indent: - $CI_PROJECT_NAME == "gstreamer-sharp" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" .build: stage: 'full builds' @@ -120,7 +118,7 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic" build nodebug fedora x86_64: extends: '.build' @@ -169,7 +167,6 @@ build nodebug fedora x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-examples" - - $CI_PROJECT_NAME == "gst-docs" .test fedora x86_64: image: $FEDORA_IMAGE @@ -187,7 +184,6 @@ check fedora: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-examples" - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gst-docs" integration testsuites fedora: extends: '.test fedora x86_64' @@ -201,7 +197,6 @@ integration testsuites fedora: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" # Valgrind .valgrind fedora x86_64: @@ -302,7 +297,6 @@ valgrind ges: except: variables: - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-docs" cache: key: "${CI_JOB_NAME}" paths: @@ -389,7 +383,7 @@ build cerbero fedora x86_64: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" + # # Cerbero Android Universal build @@ -423,7 +417,6 @@ build cerbero android universal: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" # # Cerbero Cross Windows builds @@ -455,7 +448,6 @@ build cerbero cross win32: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" cerbero deps windows x86_64: extends: '.cerbero deps' @@ -484,7 +476,6 @@ build cerbero cross win64: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" # # Build an Android App using the android binaries @@ -539,7 +530,6 @@ android universal examples: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" cerbero android universal examples: extends: ".android universal examples" @@ -548,19 +538,3 @@ cerbero android universal examples: only: variables: - $CI_PROJECT_NAME == "cerbero" - -documentation: - image: $FEDORA_IMAGE - dependencies: - - 'build fedora x86_64' - stage: deploy - - script: - - cd gst-build/ - - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings - - cd - - - mv gst-build/build/subprojects/gst-docs/GStreamer-doc/html documentation/ - - artifacts: - paths: - - documentation/ From b159e08f89b5497ab5212a08ad8b807079dd7f1d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 13 May 2019 21:51:15 -0400 Subject: [PATCH 265/605] gitlab: Run all valgrind jobs when running in gst-ci Everything needs to be tested there as we might be updating the docker image --- gitlab/ci_template.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ecba540c1c..17e2eb88b7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -213,7 +213,7 @@ valgrind core: TEST_SUITE: "check.gstreamer\\..*" only: variables: - - $CI_PROJECT_NAME == "gstreamer" + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-ci)$/ valgrind base: extends: '.valgrind fedora x86_64' @@ -221,7 +221,7 @@ valgrind base: TEST_SUITE: "check.gst-plugins-base\\..*" only: variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base)$/ + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-ci)$/ valgrind good: extends: '.valgrind fedora x86_64' @@ -229,7 +229,7 @@ valgrind good: TEST_SUITE: "check.gst-plugins-good\\..*" only: variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good)$/ + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci)$/ valgrind ugly: extends: '.valgrind fedora x86_64' @@ -237,7 +237,7 @@ valgrind ugly: TEST_SUITE: "check.gst-plugins-ugly\\..*" only: variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly)$/ + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci)$/ valgrind bad: extends: '.valgrind fedora x86_64' @@ -245,7 +245,7 @@ valgrind bad: TEST_SUITE: "check.gst-plugins-bad\\..*" only: variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad)$/ + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci)$/ valgrind ges: extends: '.valgrind fedora x86_64' @@ -253,7 +253,7 @@ valgrind ges: TEST_SUITE: "check.gst-editing-services\\..*" only: variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services)$/ + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci)$/ # Template for Cerbero GStreamer Build # From 3c5db7d4495700382fb54cd6e2063191852700ec Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 14 May 2019 12:03:49 -0400 Subject: [PATCH 266/605] Add support to build the documentation Since we can't run valgrind tests on fedora 30 yet (see https://gitlab.freedesktop.org/gstreamer/gst-ci/merge_requests/125), we build the nodebug variant with the F30 image and use the artifacts of that build to build the documentation and not the main one. --- .gitlab-ci.yml | 2 ++ gitlab/ci_template.yml | 42 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8d4044b65..7a232632a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,8 @@ stages: - "full tests" # build some apps to check that cross-platform binaries are usable - "apps" + # Deploy the documentation + - "deploy" test manifest: variables: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 17e2eb88b7..1a9f62463a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,11 +8,13 @@ stages: - 'full tests' # build some apps to check that cross-platform binaries are usable - 'apps' + - 'deploy' variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223' + FEDORA30_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-13-295487' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' @@ -71,6 +73,7 @@ gst indent: - $CI_PROJECT_NAME == "gstreamer-sharp" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" .build: stage: 'full builds' @@ -119,13 +122,15 @@ build fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic" + except: + - $CI_PROJECT_NAME == "gst-docs" build nodebug fedora x86_64: extends: '.build' stage: 'build' - image: $FEDORA_IMAGE + image: $FEDORA30_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled" except: variables: - $CI_PROJECT_NAME == "cerbero" @@ -167,6 +172,7 @@ build nodebug fedora x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-examples" + - $CI_PROJECT_NAME == "gst-docs" .test fedora x86_64: image: $FEDORA_IMAGE @@ -184,6 +190,7 @@ check fedora: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-examples" - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gst-docs" integration testsuites fedora: extends: '.test fedora x86_64' @@ -197,6 +204,7 @@ integration testsuites fedora: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" # Valgrind .valgrind fedora x86_64: @@ -297,6 +305,7 @@ valgrind ges: except: variables: - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-docs" cache: key: "${CI_JOB_NAME}" paths: @@ -383,7 +392,7 @@ build cerbero fedora x86_64: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - + - $CI_PROJECT_NAME == "gst-docs" # # Cerbero Android Universal build @@ -417,6 +426,7 @@ build cerbero android universal: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" # # Cerbero Cross Windows builds @@ -448,6 +458,7 @@ build cerbero cross win32: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" cerbero deps windows x86_64: extends: '.cerbero deps' @@ -476,6 +487,7 @@ build cerbero cross win64: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" # # Build an Android App using the android binaries @@ -530,6 +542,7 @@ android universal examples: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" cerbero android universal examples: extends: ".android universal examples" @@ -538,3 +551,26 @@ cerbero android universal examples: only: variables: - $CI_PROJECT_NAME == "cerbero" + +documentation: + image: $FEDORA30_IMAGE + dependencies: + - 'build nodebug fedora x86_64' + stage: deploy + + script: + - cd gst-build/ + - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings + - cd - + - mv gst-build/build/subprojects/gst-docs/GStreamer-doc/html documentation/ + + artifacts: + paths: + - documentation/ + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-integration-testsuite" + - $CI_PROJECT_NAME == "gst-python" + - $CI_PROJECT_NAME == "gst-sharp" + - $CI_PROJECT_NAME == "gst-build" \ No newline at end of file From 917ff85856e5c80fead9000674aaaef80f63ff60 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 16 May 2019 14:03:09 -0400 Subject: [PATCH 267/605] gitlab: Fix mixup in excepting gst-docs to build the nodebug job --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1a9f62463a..b0c9476db6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -123,7 +123,8 @@ build fedora x86_64: variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic" except: - - $CI_PROJECT_NAME == "gst-docs" + variables: + - $CI_PROJECT_NAME == "gst-docs" build nodebug fedora x86_64: extends: '.build' @@ -134,7 +135,6 @@ build nodebug fedora x86_64: except: variables: - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" .test: stage: 'test' From e6cf53329ee346d2e0243d4de544c5d9bd45f63b Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Apr 2019 09:58:31 +0300 Subject: [PATCH 268/605] gitlab/build_manifest: add gst-sharp to the module list --- gitlab/build_manifest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index f8c7b016fb..836b8c5998 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -28,6 +28,7 @@ GSTREAMER_MODULES: List[str] = [ 'gst-omx', 'gst-python', 'gst-rtsp-server', + 'gstreamer-sharp', 'gstreamer-vaapi', 'gst-integration-testsuites', 'gst-examples', From 274f1839556944187be883f7b26b3f1c905d0bff Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 24 May 2019 18:32:16 +0300 Subject: [PATCH 269/605] gitlab/ci_template: make sure that the fedora build job is never run on cerbero We exclude cerbero in the .build template but that key is getting overwritten since !126. Valgrind needs to be fixed first in order to remove this and the previous workarounds. --- gitlab/ci_template.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b0c9476db6..b713dc76b5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -125,6 +125,7 @@ build fedora x86_64: except: variables: - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "cerbero" build nodebug fedora x86_64: extends: '.build' @@ -132,9 +133,6 @@ build nodebug fedora x86_64: image: $FEDORA30_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled" - except: - variables: - - $CI_PROJECT_NAME == "cerbero" .test: stage: 'test' @@ -573,4 +571,4 @@ documentation: - $CI_PROJECT_NAME == "gst-integration-testsuite" - $CI_PROJECT_NAME == "gst-python" - $CI_PROJECT_NAME == "gst-sharp" - - $CI_PROJECT_NAME == "gst-build" \ No newline at end of file + - $CI_PROJECT_NAME == "gst-build" From 9afac85d0d34f976b68ae20c0a17200a764281b5 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 28 May 2019 22:59:19 -0400 Subject: [PATCH 270/605] fedora: Install gtk3 and gdk-pixbuf documentation --- docker/fedora/prepare.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 633b69239f..d0516534a9 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -186,9 +186,11 @@ dnf remove -y "gstreamer1*devel" # it leads to build issues in examples. dnf remove -y "qt5-qtbase-devel" -# FIXME: Why does installing directly with dnf *fails* ? -dnf download glib2-doc -rpm -i glib2-doc*.rpm +# FIXME: Why does installing directly with dnf doesn't actually install +# the documentation files? +dnf download glib2-doc gdk-pixbuf2-devel*x86_64* gtk3-devel-docs +rpm -i --reinstall *.rpm +rm -f *.rpm # get gst-build and make all subprojects available git clone git://anongit.freedesktop.org/gstreamer/gst-build /gst-build/ From a30652954cd6ff1027f01d623bd0eed0ece5b589 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 27 May 2019 10:38:17 -0400 Subject: [PATCH 271/605] doc: Use latest image with update hotdoc --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b713dc76b5..b303c34aaa 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -14,7 +14,7 @@ variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223' - FEDORA30_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-13-295487' + FEDORA30_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' From 35be7d2b91c47cf7dc8f56fdd295a8af2662b1f1 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 31 May 2019 20:12:55 -0400 Subject: [PATCH 272/605] gitlab: Always run hotdoc from git We are moving too fast in hotdoc and the overhead of releasing and updating docker images is too big --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b303c34aaa..95c3ccab9b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -557,6 +557,7 @@ documentation: stage: deploy script: + - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git - cd gst-build/ - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings - cd - From 798945be7a40207459a36631c1067d90942ce9c0 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 25 May 2019 13:48:12 +0200 Subject: [PATCH 273/605] gitlab: Add static builds for gst-build --- gitlab/ci_template.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 95c3ccab9b..45965f1c02 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -19,6 +19,7 @@ variables: MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' GIT_STRATEGY: none + MESON_BUILDTYPE_ARGS: --default-library=both DEFAULT_MESON_ARGS: > --werror -Dpython=enabled @@ -84,7 +85,7 @@ gst indent: CXX: "ccache g++" CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" - MESON_ARGS: "${DEFAULT_MESON_ARGS}" + MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS}" script: - ccache -z @@ -121,7 +122,7 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS}" except: variables: - $CI_PROJECT_NAME == "gst-docs" @@ -132,7 +133,17 @@ build nodebug fedora x86_64: stage: 'build' image: $FEDORA30_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled ${MESON_BUILDTYPE_ARGS}" + +build static fedora x86_64: + extends: 'build fedora x86_64' + variables: + MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" + +build static nodebug fedora x86_64: + extends: 'build nodebug fedora x86_64' + variables: + MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" .test: stage: 'test' From 326e3a5ac596fa914f60b3d1020071396f215253 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 1 Apr 2019 13:24:46 +1100 Subject: [PATCH 274/605] add macos CI --- gitlab/ci_template.yml | 98 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 84 insertions(+), 14 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 45965f1c02..1c60577b6d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -276,8 +276,9 @@ valgrind ges: # # Parameters: # CONFIG: The name of the configuration file to use +# ARCH: The cerbero _ (used in cache key) # -# Produces runtime and devel tarball packages. +# Produces runtime and devel tarball packages for linux/android or .pkg for macos .cerbero: stage: "full builds" image: $CERBERO_IMAGE @@ -287,30 +288,41 @@ valgrind ges: CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" CCACHE_MAXSIZE: "1.7G" - CERBERO_HOME: "${CI_PROJECT_DIR}/cerbero-build" - CERBERO_SOURCES: "${CI_PROJECT_DIR}/cerbero-sources" + CERBERO_HOME: "cerbero-build" + CERBERO_SOURCES: "cerbero-sources" CERBERO_DEPS: "cerbero-deps.tar.gz" CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS}" CERBERO: "./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml" + CERBERO_PACKAGE_ARGS: "-t" + HAVE_CCACHE: "yes" + # used by macos packages as we only ever install to a fixed directory + CERBERO_OVERRIDDEN_DIST_DIR: "" before_script: # FIXME Wrong namespace # Workaround build-tools having hardcoded internal path - - mkdir -p /builds/gstreamer - - ln -sf ${CI_PROJECT_DIR} /builds/gstreamer/cerbero + - pwd + - mkdir -p ../../gstreamer + - ln -sf $(pwd) ../../gstreamer/cerbero + - mkdir -p ../../${CI_PROJECT_NAMESPACE} + - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero - rsync -aH /cerbero/ . - test -f ${CERBERO_DEPS} && tar -C ${CERBERO_HOME} -xf ${CERBERO_DEPS} - - echo "home_dir = \"${CERBERO_HOME}\"" >> localconf.cbc - - echo "local_sources = \"${CERBERO_SOURCES}\"" >> localconf.cbc + - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" + && test -d ${CERBERO_HOME}/dist/${ARCH} + && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR} + && rsync -aH ${CERBERO_HOME}/dist/${ARCH}/ ${CERBERO_OVERRIDDEN_DIST_DIR} + - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc + - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc - ./cerbero-uninstalled --self-update manifest.xml - - ccache -z script: + - test "x${HAVE_CCACHE}" = "xyes" && ccache -z - $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - $CERBERO $CERBERO_ARGS fetch-cache - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only - - $CERBERO $CERBERO_ARGS package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0 - - ccache -s + - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o $(pwd) gstreamer-1.0 + - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test except: variables: - $CI_PROJECT_NAME == "gst-build" @@ -326,7 +338,7 @@ valgrind ges: when: 'always' paths: - "manifest.xml" - - "cerbero-build/logs" + - "${CERBERO_HOME}/logs" - "*.tar.bz2" # Template for Cerbero GStreamer Deps @@ -354,10 +366,12 @@ valgrind ges: gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice - $CERBERO $CERBERO_ARGS fetch-cache --skip-fetch --job-id=${CI_JOB_ID} + - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" + && mkdir -p ${CERBERO_HOME}/dist/${ARCH} + && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH} - tar -C ${CERBERO_HOME} -czf $CERBERO_DEPS build-tools build-tools.cache dist/${ARCH} ${ARCH}.cache - - ccache -s only: variables: - $CI_PROJECT_NAME == "cerbero" @@ -367,8 +381,8 @@ valgrind ges: when: 'always' paths: - "manifest.xml" - - "cerbero-build/logs" - - "cerbero-build/cerbero-deps.log" + - "${CERBERO_HOME}/logs" + - "${CERBERO_HOME}/cerbero-deps.log" - "${CERBERO_DEPS}" # @@ -561,6 +575,62 @@ cerbero android universal examples: variables: - $CI_PROJECT_NAME == "cerbero" +# +# Cerbero macOS X86_64 build +# +cerbero deps macos x86_64: + extends: '.cerbero deps' + stage: "build" + variables: + ARCH: "darwin_x86_64" + CONFIG: "osx-x86-64.cbc" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" + HAVE_CCACHE: "" + CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" + tags: + - gst-macos-10.14 + +.cerbero macos x86_64: + extends: '.cerbero' + variables: + ARCH: "darwin_x86_64" + CONFIG: "osx-x86-64.cbc" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" + CERBERO_PACKAGE_ARGS: "" + HAVE_CCACHE: "" + CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" + tags: + - gst-macos-10.14 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "gstreamer-1.0-1.*.pkg" + - "gstreamer-1.0-devel-1.*.pkg" + +cerbero macos x86_64: + extends: '.cerbero macos x86_64' + dependencies: + - "cerbero deps macos x86_64" + only: + variables: + - $CI_PROJECT_NAME == "cerbero" + +build cerbero macos x86_64: + extends: '.cerbero macos x86_64' + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-integration-testsuite" + - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-sharp" + documentation: image: $FEDORA30_IMAGE dependencies: From bc44b55c4a55b79f96e55c3c784a0c596e023dfd Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 5 Jun 2019 15:49:13 +1000 Subject: [PATCH 275/605] gitlab-ci: move overriden dist dir copy to after cache fetch Otherwise, the cache will not be copied to the overriden dist dir for the normal fast builds. --- gitlab/ci_template.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1c60577b6d..2a3b0de8f2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -307,10 +307,6 @@ valgrind ges: - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero - rsync -aH /cerbero/ . - test -f ${CERBERO_DEPS} && tar -C ${CERBERO_HOME} -xf ${CERBERO_DEPS} - - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" - && test -d ${CERBERO_HOME}/dist/${ARCH} - && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR} - && rsync -aH ${CERBERO_HOME}/dist/${ARCH}/ ${CERBERO_OVERRIDDEN_DIST_DIR} - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc - ./cerbero-uninstalled --self-update manifest.xml @@ -320,6 +316,10 @@ valgrind ges: - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - $CERBERO $CERBERO_ARGS fetch-cache + - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" + && test -d ${CERBERO_HOME}/dist/${ARCH} + && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR} + && rsync -aH ${CERBERO_HOME}/dist/${ARCH}/ ${CERBERO_OVERRIDDEN_DIST_DIR} - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o $(pwd) gstreamer-1.0 - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test From 478686faa76917bef6df32c73c538e05fb8f6813 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 5 Jun 2019 16:12:56 +1000 Subject: [PATCH 276/605] gitlab: build libnice as part of cerbero deps Building the same libnice from the same tarball every time is not useful. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2a3b0de8f2..2204a7c0a5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -364,7 +364,7 @@ valgrind ges: - $CERBERO $CERBERO_ARGS build-deps --offline gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 - gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice + gst-libav-1.0 gst-validate gst-editing-services-1.0 - $CERBERO $CERBERO_ARGS fetch-cache --skip-fetch --job-id=${CI_JOB_ID} - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && mkdir -p ${CERBERO_HOME}/dist/${ARCH} From 24f24828d0e90650e527fd88f726ec1b13ffcefd Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 5 Jun 2019 13:54:17 +0000 Subject: [PATCH 277/605] Revert "gitlab: build libnice as part of cerbero deps" This reverts commit 478686faa76917bef6df32c73c538e05fb8f6813 --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2204a7c0a5..2a3b0de8f2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -364,7 +364,7 @@ valgrind ges: - $CERBERO $CERBERO_ARGS build-deps --offline gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 - gst-libav-1.0 gst-validate gst-editing-services-1.0 + gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice - $CERBERO $CERBERO_ARGS fetch-cache --skip-fetch --job-id=${CI_JOB_ID} - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && mkdir -p ${CERBERO_HOME}/dist/${ARCH} From 2cde4bef1e5b205b86792cd182b4b26894e14e32 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 24 Apr 2019 08:42:27 +0300 Subject: [PATCH 278/605] gitlab/build_manifest: allow for upstream branch to be specified When the pipeline is based on top of a stable branch, we want to track that branch isntead of the primary development branch. This patch makes it so the upstream branch can be specified with an env var. part of #11 --- gitlab/build_manifest.py | 39 ++++++++++++++++++++++----------------- gitlab/ci_template.yml | 8 ++++++-- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 836b8c5998..28ba6c6fe6 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -65,9 +65,9 @@ def get_cerbero_last_build_info (namespace : str, branch : str): return deps[0]['commit'] -def get_branches_info(module: str, namespace: str, branches: List[str]) -> Tuple[str, str]: +def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]: try: - res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', *branches) + res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch) except subprocess.CalledProcessError: return None, None @@ -76,37 +76,39 @@ def get_branches_info(module: str, namespace: str, branches: List[str]) -> Tuple # Special case cerbero to avoid cache misses if module == 'cerbero': - for branch in branches: - sha = get_cerbero_last_build_info(namespace, branch) - if sha is not None: - return sha, sha + sha = get_cerbero_last_build_info(namespace, branch) + if sha is not None: + return sha, sha lines = res.split('\n') - for branch in branches: - for line in lines: - if line.endswith('/' + branch): - try: - sha, refname = line.split('\t') - except ValueError: - continue - return refname.strip(), sha + for line in lines: + if line.endswith('/' + branch): + try: + sha, refname = line.split('\t') + except ValueError: + continue + return refname.strip(), sha return None, None def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]: namespace: str = os.environ["CI_PROJECT_NAMESPACE"] + ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master') + + if module == "orc": + ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master') if module == os.environ['CI_PROJECT_NAME']: return 'user', branchname, os.environ['CI_COMMIT_SHA'] - if branchname != "master": - remote_refname, sha = get_branches_info(module, namespace, [branchname]) + if branchname != ups_branch: + remote_refname, sha = get_branch_info(module, namespace, branchname) if sha is not None: return 'user', remote_refname, sha # Check upstream project for a branch - remote_refname, sha = get_branches_info(module, 'gstreamer', [branchname, 'master']) + remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch) if sha is not None: return 'origin', remote_refname, sha @@ -152,6 +154,7 @@ def test_find_repository_sha(): os.environ["CI_PROJECT_NAME"] = "some-random-project" os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good" os.environ["CI_PROJECT_NAMESPACE"] = "alatiera" + os.environ["GST_UPSTREAM_BRANCH"] = "master" del os.environ["READ_PROJECTS_TOKEN"] # This should find the repository in the user namespace @@ -182,11 +185,13 @@ def test_get_project_branch(): os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_" del os.environ["READ_PROJECTS_TOKEN"] + os.environ['GST_UPSTREAM_BRANCH'] = '1.12' remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12') assert twelve is not None assert remote == 'origin' assert refname == "refs/heads/1.12" + os.environ['GST_UPSTREAM_BRANCH'] = '1.14' remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14') assert fourteen is not None assert remote == 'origin' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2a3b0de8f2..31383853d9 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -18,6 +18,10 @@ variables: INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' + # Branch to track for modules that have no ref specified in the manifest + GST_UPSTREAM_BRANCH: 'master' + ORC_UPSTREAM_BRANCH: 'master' + GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both DEFAULT_MESON_ARGS: > @@ -57,7 +61,7 @@ gst indent: script: # man indent. grep RETURN VALUE, grab a beer on my behalf... - indent --version || true - - curl -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent + - curl -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/${GST_UPSTREAM_BRANCH}/tools/gst-indent - chmod +x gst-indent - find . -name '*.c' -exec ./gst-indent {} + - | @@ -523,7 +527,7 @@ build cerbero cross win64: GSTREAMER_ROOT_ANDROID: ${CI_PROJECT_DIR}/examples/cerbero-android-universal script: - mkdir -p ${EXAMPLES_HOME}/outputs - - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/master/gitlab/clone_manifest_ref.py + - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py - chmod +x clone_manifest_ref.py - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs From 8a20ec0606c3d3bf3b48cc960206dcc027f01ebd Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 4 Jun 2019 22:28:46 -0400 Subject: [PATCH 279/605] docker: cerbero: Fix for mingw tarball rename --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index cde56e0b40..838528082b 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -30,7 +30,7 @@ RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \ cd .. && \ rm /cerbero/cerbero-sources/android-ndk-*.zip && \ - rm /cerbero/cerbero-sources/mingw-w*.tar.xz && \ + rm /cerbero/cerbero-sources/mingw-*.tar.xz && \ rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \ rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \ dnf clean all From 69656b287cceb8b512718729d6c881b369b36705 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Fri, 7 Jun 2019 10:15:06 -0400 Subject: [PATCH 280/605] gitlab: Update cerbero docker to the new MingW --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 31383853d9..fe573a547b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -12,7 +12,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-06-06-343857' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223' FEDORA30_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' From 142f24043d9edf0306679b615f53e7790df164e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 16 Apr 2019 13:06:38 +0100 Subject: [PATCH 281/605] gitlab/ci_template: only run valgrind for MRs not master and stable branches The valgrind runs are there to spot obvious problems during the dev phase, not sure we really need to run them in full after each merge. Should reduce load on the build bots a little. If any problems slip in they will be picked up soon enough by the MR jobs again. --- gitlab/ci_template.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index fe573a547b..2b304e1962 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -227,6 +227,9 @@ integration testsuites fedora: EXTRA_VALIDATE_ARGS: "--valgrind" # Some suppression files are missing a newline at the end which messes things # up when concatenating them. awk will add missing newlines (unlike cat) + except: + variables: + - $CI_PROJECT_NAMESPACE == 'gstreamer' valgrind core: extends: '.valgrind fedora x86_64' From 292f9a425062af9310cc16bfa070b516de8f787d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 11 Jun 2019 15:50:41 +1000 Subject: [PATCH 282/605] gst-build: clone gst-build directly instead of relying on the docker version The git-update that is performed attempts to update gst-build however will not use the updated git-update script for further operations. This causes the CI to not use any updates to the git-update which is always stuck on the version provided by the backing docker image. --- gitlab/ci_template.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2b304e1962..741298d745 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -93,9 +93,12 @@ gst indent: script: - ccache -z - # Not sure why, but permission errors else - # https://gitlab.freedesktop.org/alatiera/gstreamer/-/jobs/41441 - - cp -r /gst-build/ . && cd gst-build + - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py + - chmod +x clone_manifest_ref.py + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build + - cd gst-build/ + # only copy immediate directories. We don't want to modify .wrap files + - find /gst-build/subprojects/ -mindepth 1 -maxdepth 1 -type d -exec cp -r \{\} subprojects/ \; - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" - meson build/ $MESON_ARGS - ninja -C build/ From ef82e5a97140de674f78a3fc197a95db729b4fb8 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 12 Jun 2019 14:26:10 +1000 Subject: [PATCH 283/605] docker/android: cache gradle deps inside the docker image --- docker/android/prepare.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh index 58335ad32b..efc5b8a123 100644 --- a/docker/android/prepare.sh +++ b/docker/android/prepare.sh @@ -8,7 +8,8 @@ dnf install -y \ make \ pkg-config \ unzip \ - which + which \ + xz mkdir -p /android/sources @@ -21,8 +22,6 @@ curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/rep unzip /android/sources/android-sdk-tools.zip -d ${ANDROID_HOME}/ mkdir -p ${ANDROID_HOME}/licenses -rm -rf /android/sources - # Accept licenses. Values taken from: # ANDROID_HOME=/path/to/android/sdk-tools $ANDROID_HOME/tools/bin/sdkmanager --licenses echo "601085b94cd77f0b54ff86406957099ebe79c4d6" > ${ANDROID_HOME}/licenses/android-googletv-license @@ -30,3 +29,22 @@ echo "24333f8a63b6825ea9c5514f83c2829b004d1fee" > ${ANDROID_HOME}/licenses/andro echo "84831b9409646a918e30573bab4c9c91346d8abd" > ${ANDROID_HOME}/licenses/android-sdk-preview-license echo "33b6a2b64607f11b759f320ef9dff4ae5c47d97a" > ${ANDROID_HOME}/licenses/google-gdk-license echo "e9acab5b5fbb560a72cfaecce8946896ff6aab9d" > ${ANDROID_HOME}/licenses/mips-android-sysimage-license + +# pre-cache deps +export GSTREAMER_ROOT_ANDROID=/android/sources/gstreamer-android +curl -o /android/sources/gstreamer-android.tar.xz https://gstreamer.freedesktop.org/data/pkg/android/1.16.0/gstreamer-1.0-android-universal-1.16.0.tar.xz +mkdir $GSTREAMER_ROOT_ANDROID +tar -xvf /android/sources/gstreamer-android.tar.xz -C $GSTREAMER_ROOT_ANDROID +ls $GSTREAMER_ROOT_ANDROID + +git clone https://gitlab.freedesktop.org/gstreamer/gst-examples.git /android/sources/gst-examples +chmod +x /android/sources/gst-examples/playback/player/android/gradlew +/android/sources/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies + +git clone https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs +chmod +x /android/sources/gst-docs/examples/tutorials/android/gradlew +/android/sources/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies + +unset GSTREAMER_ROOT_ANDROID + +rm -rf /android/sources From f00192da0c768e3ebedc1c69961a61d6cd49f74c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 13 Jun 2019 01:00:04 +1000 Subject: [PATCH 284/605] gitlab: update android docker image to the latest --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 741298d745..9d5118d268 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -11,7 +11,7 @@ stages: - 'deploy' variables: - ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225' + ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-06-12-354260' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-06-06-343857' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223' FEDORA30_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578' From a9cd95f7ba283352e8c28dec558c52b395f6feaf Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 12 Jun 2019 15:38:14 +1000 Subject: [PATCH 285/605] gitlab: consolidate stages This now optimizes for a successful build. --- .gitlab-ci.yml | 11 +++-------- gitlab/ci_template.yml | 23 +++++++++-------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a232632a0..498683a4ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,11 @@ stages: - "build docker" - "preparation" - # Test just one basic build, if succeds procced to test the rest + - "pre-build" - "build" - "test" - # Run multiple builds and tests, multi-distro, multi-arch - - "full builds" - - "full tests" - # build some apps to check that cross-platform binaries are usable - - "apps" - # Deploy the documentation - - "deploy" + # use the binaries in some way + - "integrate" test manifest: variables: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9d5118d268..8fbebf9069 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1,14 +1,10 @@ stages: - 'preparation' - # Test just one basic build, if it succeeds proceed to test the rest + - 'pre-build' - 'build' - 'test' - # Run multiple builds and tests, multi-distro, multi-arch - - 'full builds' - - 'full tests' - # build some apps to check that cross-platform binaries are usable - - 'apps' - - 'deploy' + # Use the resulting binaries + - 'integrate' variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-06-12-354260' @@ -81,7 +77,7 @@ gst indent: - $CI_PROJECT_NAME == "gst-docs" .build: - stage: 'full builds' + stage: 'build' dependencies: - "manifest" variables: @@ -225,7 +221,7 @@ integration testsuites fedora: # Valgrind .valgrind fedora x86_64: extends: '.test fedora x86_64' - stage: 'full tests' + stage: 'test' variables: EXTRA_VALIDATE_ARGS: "--valgrind" # Some suppression files are missing a newline at the end which messes things @@ -290,7 +286,7 @@ valgrind ges: # # Produces runtime and devel tarball packages for linux/android or .pkg for macos .cerbero: - stage: "full builds" + stage: "build" image: $CERBERO_IMAGE dependencies: - "manifest" @@ -365,7 +361,7 @@ valgrind ges: # with the associated build-tools. .cerbero deps: extends: .cerbero - stage: "build" + stage: "pre-build" script: - $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only @@ -527,7 +523,7 @@ build cerbero cross win64: # .android universal examples: image: $ANDROID_IMAGE - stage: 'apps' + stage: 'integrate' variables: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples GSTREAMER_ROOT_ANDROID: ${CI_PROJECT_DIR}/examples/cerbero-android-universal @@ -590,7 +586,6 @@ cerbero android universal examples: # cerbero deps macos x86_64: extends: '.cerbero deps' - stage: "build" variables: ARCH: "darwin_x86_64" CONFIG: "osx-x86-64.cbc" @@ -645,7 +640,7 @@ documentation: image: $FEDORA30_IMAGE dependencies: - 'build nodebug fedora x86_64' - stage: deploy + stage: integrate script: - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git From 802dd4eb2b39f150a32f89dcd493cbdc099db1c2 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 2 Jun 2019 17:17:04 +1000 Subject: [PATCH 286/605] add iOS CI --- gitlab/ci_template.yml | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8fbebf9069..02396123f5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -636,6 +636,58 @@ build cerbero macos x86_64: - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-sharp" +# +# Cerbero iOS build +# +cerbero deps ios universal: + extends: '.cerbero deps' + variables: + ARCH: "ios_universal" + CONFIG: "cross-ios-universal.cbc" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" + HAVE_CCACHE: "" + tags: + - gst-ios-12.2 + +.cerbero ios universal: + extends: '.cerbero' + variables: + ARCH: "ios_universal" + CONFIG: "cross-ios-universal.cbc" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" + CERBERO_PACKAGE_ARGS: "" + HAVE_CCACHE: "" + tags: + - gst-ios-12.2 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "gstreamer-1.0-*-ios-universal.pkg" + +cerbero ios universal: + extends: '.cerbero ios universal' + dependencies: + - "cerbero deps ios universal" + only: + variables: + - $CI_PROJECT_NAME == "cerbero" + +build cerbero ios universal: + extends: '.cerbero ios universal' + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-integration-testsuite" + - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-sharp" + documentation: image: $FEDORA30_IMAGE dependencies: From 34cb58f294881e59e0728de18913d4bab188ef62 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sat, 15 Jun 2019 16:49:49 -0400 Subject: [PATCH 287/605] CI: Disable build cerbero ios universal This fails because the master CI of cerbero/ never passed with, as a result an older cerbero is being used. Related to #29 --- gitlab/ci_template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 02396123f5..383389dabe 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -676,7 +676,9 @@ cerbero ios universal: variables: - $CI_PROJECT_NAME == "cerbero" -build cerbero ios universal: +# Disabled until the cerbero/ master CI succeed uploading deps +# see #29 +.build cerbero ios universal: extends: '.cerbero ios universal' except: variables: From 7c344aeb17e894f8f5caf3c0f27b57d64723fef1 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 16 Jun 2019 13:50:23 +1000 Subject: [PATCH 288/605] Revert "CI: Disable build cerbero ios universal" This reverts commit 34cb58f294881e59e0728de18913d4bab188ef62. --- gitlab/ci_template.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 383389dabe..02396123f5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -676,9 +676,7 @@ cerbero ios universal: variables: - $CI_PROJECT_NAME == "cerbero" -# Disabled until the cerbero/ master CI succeed uploading deps -# see #29 -.build cerbero ios universal: +build cerbero ios universal: extends: '.cerbero ios universal' except: variables: From 545b023050c47896d797c411e928193c118717b7 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 16 Jun 2019 13:51:39 +1000 Subject: [PATCH 289/605] gitlab: make non-cerbero iOS build manual --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 02396123f5..95fffc63d7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -678,6 +678,7 @@ cerbero ios universal: build cerbero ios universal: extends: '.cerbero ios universal' + when: 'manual' except: variables: - $CI_PROJECT_NAME == "cerbero" From a774ea6ce90aa052184b66ae369f726703634e61 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 14 Jun 2019 20:59:19 +1000 Subject: [PATCH 290/605] gitlab/cerbero: use the correct branch on fetch-cache --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 95fffc63d7..73b4d1a825 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -321,7 +321,7 @@ valgrind ges: - $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - - $CERBERO $CERBERO_ARGS fetch-cache + - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && test -d ${CERBERO_HOME}/dist/${ARCH} && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR} @@ -371,7 +371,7 @@ valgrind ges: gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice - - $CERBERO $CERBERO_ARGS fetch-cache --skip-fetch --job-id=${CI_JOB_ID} + - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} --skip-fetch --job-id=${CI_JOB_ID} - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && mkdir -p ${CERBERO_HOME}/dist/${ARCH} && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH} From a110f8399e1dedc70ea1c4093afc9be218a5ea9c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 5 Jul 2019 13:19:41 +1000 Subject: [PATCH 291/605] docker/fedora: add glslc to the install list Enables building the vulkan plugin with shaders --- docker/fedora/prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index d0516534a9..15467bd7cb 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -33,6 +33,7 @@ dnf install -y \ gcc-c++ \ gdb \ git-lfs \ + glslc \ gtk3 \ gtk3-devel \ graphene \ From 7e88d99f9504dfdda7fa8a31e75f560c285cda3a Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 15 Jul 2019 12:06:38 +0300 Subject: [PATCH 292/605] docker/fedora: avoid hardcoding the fedora version in the setup script --- docker/fedora/prepare.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 15467bd7cb..e0b9873d9a 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -7,11 +7,10 @@ git config --global user.email "gst-build@gstreamer.net" git config --global user.name "Gstbuild Runner" # Add rpm fusion repositories in order to access all of the gst plugins -dnf install -y "http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-30.noarch.rpm" \ - "http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-30.noarch.rpm" +sudo dnf install -y \ + "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm" \ + "https://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm" -rpm --import "/etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-nonfree-fedora-30" -rpm --import "/etc/pki/rpm-gpg/RPM-GPG-KEY-rpmfusion-free-fedora-30" dnf upgrade -y # Enable the cisco openh264 repo From 47d07c48990fc296a3a4078404a2547164a67250 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 7 Aug 2019 12:40:27 -0400 Subject: [PATCH 293/605] Move to Fedora 30 --- gitlab/ci_template.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 73b4d1a825..01fc6d0414 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,8 +9,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-06-12-354260' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-06-06-343857' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223' - FEDORA30_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' @@ -134,7 +133,7 @@ build fedora x86_64: build nodebug fedora x86_64: extends: '.build' stage: 'build' - image: $FEDORA30_IMAGE + image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled ${MESON_BUILDTYPE_ARGS}" @@ -690,7 +689,7 @@ build cerbero ios universal: - $CI_PROJECT_NAME == "gst-sharp" documentation: - image: $FEDORA30_IMAGE + image: $FEDORA_IMAGE dependencies: - 'build nodebug fedora x86_64' stage: integrate From f4be7840784b565a992bf1e0e54cd2a2c432c61d Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Tue, 6 Aug 2019 11:06:46 +0100 Subject: [PATCH 294/605] docker/fedora: Enable WPEWebKit support These dependencies should enable the WPE plugin build in -bad. See https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/839 --- docker/fedora/prepare.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index e0b9873d9a..97a3f8e0f3 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -100,6 +100,10 @@ dnf install -y \ mesa-libOSMesa \ mesa-libOSMesa-devel \ mesa-vulkan-drivers \ + wpewebkit \ + wpewebkit-devel \ + wpebackend-fdo \ + wpebackend-fdo-devel \ xorg-x11-server-utils \ xorg-x11-server-Xvfb From 157f94fb62da7582fce90c85bf2338999d83b7cc Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 9 Aug 2019 11:36:00 -0400 Subject: [PATCH 295/605] gitlab: Set COLUMNS and width variable to make terminal_markdown_view happy --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 01fc6d0414..7768a7d576 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -154,6 +154,8 @@ build static nodebug fedora x86_64: GST_DEBUG_NO_COLOR: "true" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" + width: 100 + COLUMNS: 100 script: - cd gst-build/ - echo "-> Running ${TEST_SUITE}" From fd89f33d88595907ac5953a77efb753c6126af49 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Sun, 25 Nov 2018 13:35:54 -0500 Subject: [PATCH 296/605] Do not specify the compiler to use Meson will autodetect ccache when available, and we don't always have gcc available (upcoming MSVC job). --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7768a7d576..ac6f1980f2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -80,8 +80,6 @@ gst indent: dependencies: - "manifest" variables: - CC: "ccache gcc" - CXX: "ccache g++" CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS}" From 31cec0a2459b38ae64d8bbb3f2060d2d86f8f86f Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Sun, 25 Nov 2018 13:38:06 -0500 Subject: [PATCH 297/605] Add Windows MSVC and MSYS2 builds This adds 3 new jobs that build against msys2 x86_64, msvc 2017 x86 and msvc 2017 x86_64. For the msvc build, some subprojects (like libnice) don't satisfy all their deps, and are getting automatically disabled. This doesn't add jobs that run the test suite also. Will hopefully get implemented later on. --- docker/windows/Dockerfile | 9 ++++++ docker/windows/cleanup.ps1 | 0 docker/windows/prepare.ps1 | 46 ++++++++++++++++++++++++++++ gitlab/ci_template.yml | 61 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 docker/windows/Dockerfile create mode 100644 docker/windows/cleanup.ps1 create mode 100644 docker/windows/prepare.ps1 diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile new file mode 100644 index 0000000000..32eaccc152 --- /dev/null +++ b/docker/windows/Dockerfile @@ -0,0 +1,9 @@ +# escape=` + +FROM 'mcr.microsoft.com/windows/servercore:1607' + +# Make sure any failure in PowerShell scripts is fatal +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] + +COPY prepare.ps1 cleanup.ps1 C:\ +RUN C:\prepare.ps1 ; C:\cleanup.ps1 diff --git a/docker/windows/cleanup.ps1 b/docker/windows/cleanup.ps1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/windows/prepare.ps1 b/docker/windows/prepare.ps1 new file mode 100644 index 0000000000..9bf2cfe71b --- /dev/null +++ b/docker/windows/prepare.ps1 @@ -0,0 +1,46 @@ +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +Write-Host "Installing VisualStudio" +Invoke-WebRequest -Uri 'https://aka.ms/vs/15/release/vs_buildtools.exe' -OutFile C:\vs_buildtools.exe +Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait +Remove-Item C:\vs_buildtools.exe -Force + +Write-Host "Installing Python" +Invoke-WebRequest -Uri 'https://www.python.org/ftp/python/3.7.0/python-3.7.0.exe' -OutFile C:\python-3.7.0.exe +Start-Process C:\python-3.7.0.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait +Remove-Item C:\python-3.7.0.exe -Force + +Write-Host "Installing Git" +Invoke-WebRequest -Uri 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip' -OutFile C:\mingit.zip +Expand-Archive C:\mingit.zip -DestinationPath c:\mingit +Remove-Item C:\mingit.zip -Force +$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd' +[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine) + +Write-Host "Installing 7zip" +Invoke-WebRequest -Uri 'https://www.7-zip.org/a/7z1805-x64.exe' -OutFile C:\7z-x64.exe +Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait +Remove-Item C:\7z-x64.exe -Force + +Write-Host "Installing MSYS2" +Invoke-WebRequest -Uri 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' -OutFile C:\msys2-x86_64.tar.xz +C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait +C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\" +Remove-Item C:\msys2-x86_64.tar.xz -Force +Remove-Item C:\msys2-x86_64.tar -Force +Remove-Item C:\7zip -Recurse -Force + +# FIXME: This works but then docker fails to save the image. Needs to investigate why. +#$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin" +#C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys" +#C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" +#C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" + +pip install meson + +git config --global user.email "gst-build@gstreamer.net" +git config --global user.name "Gstbuild Runner" + +# Download gst-build and all its subprojects +git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +meson subprojects download --sourcedir C:\gst-build diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ac6f1980f2..197dd9977f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -20,7 +20,6 @@ variables: GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both DEFAULT_MESON_ARGS: > - --werror -Dpython=enabled -Dlibav=enabled -Dugly=enabled @@ -82,7 +81,7 @@ gst indent: variables: CCACHE_BASEDIR: "${CI_PROJECT_DIR}" CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" - MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS}" + MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} --werror" script: - ccache -z @@ -346,6 +345,64 @@ valgrind ges: - "${CERBERO_HOME}/logs" - "*.tar.bz2" +.build windows: + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v4' + stage: 'build' + dependencies: + - 'manifest' + tags: + - 'docker' + - 'windows' + - '1607' + variables: + MESON_ARGS: > + ${DEFAULT_MESON_ARGS} + -Dpython=disabled + -Dlibav=disabled + -Dvaapi=disabled + -Ddevtools=disabled + script: + # For some reason, options are separated by newline instead of space, so we + # have to replace them first. + - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") + # Gitlab executes PowerShell in docker, but VsDevCmd.bat is a batch script. + # Environment variables substitutions is done by PowerShell before calling + # cmd.exe, that's why we use $env:FOO instead of %FOO% + - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && + cd C:\gst-build && + python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml && + meson build $env:MESON_ARGS && + ninja -C build" + after_script: + # FIXME: cleanup build dir for artifacts. + +build vs2017 amd64: + extends: '.build windows' + variables: + ARCH: 'amd64' + +build vs2017 x86: + extends: '.build windows' + variables: + ARCH: 'x86' + +build msys2 : + extends: '.build windows' + script: + - $env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin" + # FIXME: This should be done in Dockerfile, but doesn't work there. Needs to investigate why. + - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true" + - C:\msys64\usr\bin\bash -c "pacman -Syu --noconfirm" + - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" + # For some reason, options are separated by newline instead of space, so we + # have to replace them first. + - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") + - C:\msys64\usr\bin\bash -c "cd /c/gst-build && + python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml && + meson build $env:MESON_ARGS && + ninja -C build" + + # Template for Cerbero GStreamer Deps # # This template is used by cerbero/ project to pre-built the GStreamer From 072cf0122a3774e41f971b1d819282a4aa009c4d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 1 Jul 2019 18:55:46 +0300 Subject: [PATCH 298/605] docker/windows: refactorings to get it to work with the new runner * Install git-lfs as its required now by gst-integration-suites * Clone gst-build eache time to avoid dated gst-build checkouts and overwritting .wrap files. Similar to !137 * Split the dockerfile and add a second run stage refresh the powershell env inbetween calls * Remove the msys2 workaround as its not needed anymore --- docker/windows/Dockerfile | 7 ++-- docker/windows/cleanup.ps1 | 0 docker/windows/install_toolchain.ps1 | 52 ++++++++++++++++++++++++++++ docker/windows/prepare.ps1 | 46 ------------------------ docker/windows/prepare_gst_env.ps1 | 18 ++++++++++ gitlab/ci_template.yml | 25 ++++++++----- 6 files changed, 92 insertions(+), 56 deletions(-) delete mode 100644 docker/windows/cleanup.ps1 create mode 100644 docker/windows/install_toolchain.ps1 delete mode 100644 docker/windows/prepare.ps1 create mode 100644 docker/windows/prepare_gst_env.ps1 diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 32eaccc152..e0b8244141 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -5,5 +5,8 @@ FROM 'mcr.microsoft.com/windows/servercore:1607' # Make sure any failure in PowerShell scripts is fatal SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] -COPY prepare.ps1 cleanup.ps1 C:\ -RUN C:\prepare.ps1 ; C:\cleanup.ps1 +COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\ + +RUN C:\install_toolchain.ps1 + +RUN C:\prepare_gst_env.ps1 \ No newline at end of file diff --git a/docker/windows/cleanup.ps1 b/docker/windows/cleanup.ps1 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 new file mode 100644 index 0000000000..ac7a3ecc03 --- /dev/null +++ b/docker/windows/install_toolchain.ps1 @@ -0,0 +1,52 @@ +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +$python_dl_url = 'https://www.python.org/ftp/python/3.7.3/python-3.7.3.exe' +$msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' +$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip' +$zip_url = 'https://www.7-zip.org/a/7z1805-x64.exe' +$msys_url = 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' + +Write-Host "Installing VisualStudio" +Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe +Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait +Remove-Item C:\vs_buildtools.exe -Force + +Write-Host "Installing Python" +Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3.exe +Start-Process C:\python3.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait +Remove-Item C:\python3.exe -Force + +Write-Host "Installing Git" +Invoke-WebRequest -Uri $git_url -OutFile C:\mingit.zip +Expand-Archive C:\mingit.zip -DestinationPath c:\mingit +Remove-Item C:\mingit.zip -Force +$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd' +[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine) + +Write-Host "Installing 7zip" +Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe +Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait +Remove-Item C:\7z-x64.exe -Force + +Write-Host "Installing MSYS2" +Invoke-WebRequest -Uri $msys_url -OutFile C:\msys2-x86_64.tar.xz +C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait +C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\" +Remove-Item C:\msys2-x86_64.tar.xz -Force +Remove-Item C:\msys2-x86_64.tar -Force +Remove-Item C:\7zip -Recurse -Force + +Write-Host "Installing Choco" +iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) +refreshenv + +Write-Host "Installing git-lfs" +choco install -y git-lfs +refreshenv + +$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin" +C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys" +C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" +C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" + +pip install meson \ No newline at end of file diff --git a/docker/windows/prepare.ps1 b/docker/windows/prepare.ps1 deleted file mode 100644 index 9bf2cfe71b..0000000000 --- a/docker/windows/prepare.ps1 +++ /dev/null @@ -1,46 +0,0 @@ -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; - -Write-Host "Installing VisualStudio" -Invoke-WebRequest -Uri 'https://aka.ms/vs/15/release/vs_buildtools.exe' -OutFile C:\vs_buildtools.exe -Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait -Remove-Item C:\vs_buildtools.exe -Force - -Write-Host "Installing Python" -Invoke-WebRequest -Uri 'https://www.python.org/ftp/python/3.7.0/python-3.7.0.exe' -OutFile C:\python-3.7.0.exe -Start-Process C:\python-3.7.0.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait -Remove-Item C:\python-3.7.0.exe -Force - -Write-Host "Installing Git" -Invoke-WebRequest -Uri 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip' -OutFile C:\mingit.zip -Expand-Archive C:\mingit.zip -DestinationPath c:\mingit -Remove-Item C:\mingit.zip -Force -$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd' -[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine) - -Write-Host "Installing 7zip" -Invoke-WebRequest -Uri 'https://www.7-zip.org/a/7z1805-x64.exe' -OutFile C:\7z-x64.exe -Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait -Remove-Item C:\7z-x64.exe -Force - -Write-Host "Installing MSYS2" -Invoke-WebRequest -Uri 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' -OutFile C:\msys2-x86_64.tar.xz -C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait -C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\" -Remove-Item C:\msys2-x86_64.tar.xz -Force -Remove-Item C:\msys2-x86_64.tar -Force -Remove-Item C:\7zip -Recurse -Force - -# FIXME: This works but then docker fails to save the image. Needs to investigate why. -#$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin" -#C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys" -#C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" -#C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" - -pip install meson - -git config --global user.email "gst-build@gstreamer.net" -git config --global user.name "Gstbuild Runner" - -# Download gst-build and all its subprojects -git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build -meson subprojects download --sourcedir C:\gst-build diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1 new file mode 100644 index 0000000000..833f7dceac --- /dev/null +++ b/docker/windows/prepare_gst_env.ps1 @@ -0,0 +1,18 @@ +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +git config --global user.email "gst-build@gstreamer.net" +git config --global user.name "Gstbuild Runner" + +# Download gst-build and all its subprojects +git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build + +# download the subprojects to try and cache them +meson subprojects download --sourcedir C:\gst-build + +# Remove files that will conflict with a fresh clone on the runner side +Remove-Item -Force 'C:/gst-build/subprojects/*.wrap' +Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-nasm' +Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-flex-bison-binaries' + +Move-Item C:\gst-build\subprojects C:\subprojects +Remove-Item -Recurse -Force C:\gst-build diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 197dd9977f..a3049dedb1 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -12,6 +12,7 @@ variables: FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v6' # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'master' @@ -346,7 +347,7 @@ valgrind ges: - "*.tar.bz2" .build windows: - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v4' + image: $WINDOWS_IMAGE stage: 'build' dependencies: - 'manifest' @@ -362,6 +363,9 @@ valgrind ges: -Dvaapi=disabled -Ddevtools=disabled script: + - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build + - cd $env:CI_PROJECT_DIR/gst-build + - cp -r C:/subprojects/* subprojects/ # For some reason, options are separated by newline instead of space, so we # have to replace them first. - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") @@ -369,12 +373,17 @@ valgrind ges: # Environment variables substitutions is done by PowerShell before calling # cmd.exe, that's why we use $env:FOO instead of %FOO% - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && - cd C:\gst-build && python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml && meson build $env:MESON_ARGS && ninja -C build" + # FIXME: extract builddir for tests + # There's a bug that prevents us from exporting artifacts with docker-windows + # executors. It has since been fixed in gitlab 12.1, but + # we are blocked from upgrading currently. + # + # Gitlab Runner issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/4291 + # Blocked upgrade issue: https://gitlab.freedesktop.org/gstreamer/gst-ci/issues/6#note_192780 after_script: - # FIXME: cleanup build dir for artifacts. build vs2017 amd64: extends: '.build windows' @@ -389,15 +398,15 @@ build vs2017 x86: build msys2 : extends: '.build windows' script: + - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build + - cd $env:CI_PROJECT_DIR/gst-build + - cp -r C:/subprojects/* subprojects/ + - $env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin" - # FIXME: This should be done in Dockerfile, but doesn't work there. Needs to investigate why. - - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true" - - C:\msys64\usr\bin\bash -c "pacman -Syu --noconfirm" - - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" # For some reason, options are separated by newline instead of space, so we # have to replace them first. - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") - - C:\msys64\usr\bin\bash -c "cd /c/gst-build && + - C:\msys64\usr\bin\bash -c "cd $env:CI_PROJECT_DIR/gst-build && python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml && meson build $env:MESON_ARGS && ninja -C build" From e5228dc810d664efcf95785e14feaf3379d5257a Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 26 Aug 2019 21:56:40 +1000 Subject: [PATCH 299/605] gitlab: renable gst-build werror after msvc CI merge 31cec0a2459b38ae64d8bbb3f2060d2d86f8f86f removed --werror from the default meson args but did not add it back in all the correct places --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a3049dedb1..df104da330 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -122,7 +122,7 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS}" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS} --werror" except: variables: - $CI_PROJECT_NAME == "gst-docs" @@ -133,7 +133,7 @@ build nodebug fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled ${MESON_BUILDTYPE_ARGS}" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled ${MESON_BUILDTYPE_ARGS} --werror" build static fedora x86_64: extends: 'build fedora x86_64' From a596162ab5dd35bef96ef4a61b251825001b8ed9 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 27 Aug 2019 21:26:28 +1000 Subject: [PATCH 300/605] gitlab: use the new needs yaml key Allows implementing a DAG where a dependant job can be built before the entirety of the previous stage has completed. --- .gitlab-ci.yml | 4 +++- gitlab/ci_template.yml | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 498683a4ff..38aeb9346b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,6 +115,8 @@ include: "gitlab/ci_template.yml" .build local: extends: '.build' when: 'manual' + needs: + - 'manifest' # Test the build job against the latest build image tag and the local manifest build fedora x86_64 local: @@ -161,7 +163,7 @@ build cerbero cross win64 local: image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest' extends: '.test' dependencies: - - build fedora x86_64 local + - 'build fedora x86_64 local' when: 'manual' except: refs: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index df104da330..b08a472cdc 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -77,6 +77,8 @@ gst indent: .build: stage: 'build' + needs: + - "manifest" dependencies: - "manifest" variables: @@ -188,6 +190,8 @@ build static nodebug fedora x86_64: .test fedora x86_64: image: $FEDORA_IMAGE extends: '.test' + needs: + - 'build fedora x86_64' dependencies: - build fedora x86_64 @@ -287,6 +291,8 @@ valgrind ges: .cerbero: stage: "build" image: $CERBERO_IMAGE + needs: + - "manifest" dependencies: - "manifest" variables: @@ -349,6 +355,8 @@ valgrind ges: .build windows: image: $WINDOWS_IMAGE stage: 'build' + needs: + - 'manifest' dependencies: - 'manifest' tags: @@ -472,6 +480,9 @@ cerbero deps fedora x86_64: cerbero fedora x86_64: extends: '.cerbero fedora x86_64' + needs: + - "cerbero deps fedora x86_64" + - "manifest" dependencies: - "cerbero deps fedora x86_64" only: @@ -506,6 +517,9 @@ cerbero deps android universal: cerbero android universal: extends: '.cerbero android universal' + needs: + - "cerbero deps android universal" + - "manifest" dependencies: - "cerbero deps android universal" only: @@ -538,6 +552,9 @@ cerbero deps windows x86: cerbero cross win32: extends: '.cerbero cross win32' + needs: + - "cerbero deps windows x86" + - "manifest" dependencies: - "cerbero deps windows x86" only: @@ -567,6 +584,9 @@ cerbero deps windows x86_64: cerbero cross win64: extends: '.cerbero cross win64' + needs: + - "cerbero deps windows x86_64" + - "manifest" dependencies: - "cerbero deps windows x86_64" only: @@ -627,6 +647,9 @@ build cerbero cross win64: android universal examples: extends: ".android universal examples" + needs: + - "build cerbero android universal" + - "manifest" dependencies: - "build cerbero android universal" except: @@ -640,6 +663,9 @@ android universal examples: cerbero android universal examples: extends: ".android universal examples" + needs: + - "cerbero android universal" + - "manifest" dependencies: - "cerbero android universal" only: @@ -683,6 +709,9 @@ cerbero deps macos x86_64: cerbero macos x86_64: extends: '.cerbero macos x86_64' + needs: + - "cerbero deps macos x86_64" + - "manifest" dependencies: - "cerbero deps macos x86_64" only: @@ -735,6 +764,9 @@ cerbero deps ios universal: cerbero ios universal: extends: '.cerbero ios universal' + needs: + - "cerbero deps ios universal" + - "manifest" dependencies: - "cerbero deps ios universal" only: @@ -756,6 +788,9 @@ build cerbero ios universal: documentation: image: $FEDORA_IMAGE + needs: + - "build nodebug fedora x86_64" + - "manifest" dependencies: - 'build nodebug fedora x86_64' stage: integrate From 1b48188e200d93bee4315237a53e7889e6444687 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 26 Aug 2019 20:57:16 +1000 Subject: [PATCH 301/605] gitlab: add a clang gst-build job --- gitlab/ci_template.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b08a472cdc..df1efea8f4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -147,6 +147,12 @@ build static nodebug fedora x86_64: variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" +build clang fedora x86_64: + extends: 'build fedora x86_64' + variables: + CC: 'ccache clang' + CXX: 'ccache clang++' + .test: stage: 'test' variables: From ee6141699838f3cd22f98a30331472ca292c7cc4 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 26 Aug 2019 17:46:28 +1000 Subject: [PATCH 302/605] gitlab: enable werror for android universal builds --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index df1efea8f4..277aa270f7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -511,14 +511,12 @@ build cerbero fedora x86_64: cerbero deps android universal: extends: '.cerbero deps' variables: - CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" CONFIG: "cross-android-universal.cbc" ARCH: "android_universal" .cerbero android universal: extends: '.cerbero' variables: - CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" CONFIG: "cross-android-universal.cbc" cerbero android universal: From 5a5f9dbb862b09845144c79e4910a3b937fe8f4d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 30 Aug 2019 11:57:06 +1000 Subject: [PATCH 303/605] gitlab: enable werror for macos builds --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 277aa270f7..9d202876d9 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -684,7 +684,6 @@ cerbero deps macos x86_64: variables: ARCH: "darwin_x86_64" CONFIG: "osx-x86-64.cbc" - CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" HAVE_CCACHE: "" CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" tags: @@ -695,7 +694,6 @@ cerbero deps macos x86_64: variables: ARCH: "darwin_x86_64" CONFIG: "osx-x86-64.cbc" - CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" CERBERO_PACKAGE_ARGS: "" HAVE_CCACHE: "" CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" From 11ff98a757c3bb59e955c010b810655642198b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Thu, 29 Aug 2019 13:23:51 +0200 Subject: [PATCH 304/605] gitlab: no need of vaapi nor omx on windows builds --- gitlab/ci_template.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9d202876d9..c253f19e20 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -398,6 +398,11 @@ valgrind ges: # Gitlab Runner issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/4291 # Blocked upgrade issue: https://gitlab.freedesktop.org/gstreamer/gst-ci/issues/6#note_192780 after_script: + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gstreamer-vaapi" build vs2017 amd64: extends: '.build windows' From 6e39d68c4e711942de092640c396f39a3b5496d4 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 27 Aug 2019 19:30:49 -0400 Subject: [PATCH 305/605] ci: Cleanup gst-docs build Make sure to build against the plain fedora build and avoid building useless things when inside gst-docs --- gitlab/ci_template.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c253f19e20..61ec29ca73 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -124,10 +124,9 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled ${MESON_BUILDTYPE_ARGS} --werror" except: variables: - - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "cerbero" build nodebug fedora x86_64: @@ -135,23 +134,39 @@ build nodebug fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS} --werror" + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" build static fedora x86_64: extends: 'build fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" build static nodebug fedora x86_64: extends: 'build nodebug fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" build clang fedora x86_64: extends: 'build fedora x86_64' variables: CC: 'ccache clang' CXX: 'ccache clang++' + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-docs" .test: stage: 'test' @@ -403,6 +418,7 @@ valgrind ges: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-docs" build vs2017 amd64: extends: '.build windows' @@ -434,7 +450,7 @@ build msys2 : # Template for Cerbero GStreamer Deps # # This template is used by cerbero/ project to pre-built the GStreamer -# depedencies. When available, the .cerbero jobs will download this artifact +# dependencies. When available, the .cerbero jobs will download this artifact # in order to speed up the build. # # Parameters: @@ -796,10 +812,10 @@ build cerbero ios universal: documentation: image: $FEDORA_IMAGE needs: - - "build nodebug fedora x86_64" + - "build fedora x86_64" - "manifest" dependencies: - - 'build nodebug fedora x86_64' + - 'build fedora x86_64' stage: integrate script: From 20f5d7d646e7c941d71bb7264a9ba6b868982be3 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 9 Aug 2019 14:31:07 +0300 Subject: [PATCH 306/605] ci_template: Only export artifacts for the main fedora build job We are only using the builddir of the main fedora job to run tests, the rest where exported by accident. Its especially problematic cause static build eat a bunch of space and take an eternity to be uploaded. Part of #32 --- gitlab/ci_template.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 61ec29ca73..ae4e2ae43b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -109,12 +109,6 @@ gst indent: key: "${CI_JOB_NAME}" paths: - "${CCACHE_DIR}" - artifacts: - expire_in: '5 days' - when: always - paths: - - "manifest.xml" - - "gst-build/" except: variables: - $CI_PROJECT_NAME == "cerbero" @@ -125,6 +119,12 @@ build fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled ${MESON_BUILDTYPE_ARGS} --werror" + artifacts: + expire_in: '5 days' + when: always + paths: + - "manifest.xml" + - "gst-build/" except: variables: - $CI_PROJECT_NAME == "cerbero" From c732a9234cb8da2deb531f803b9d60a7947106a8 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Thu, 5 Sep 2019 17:25:37 +0200 Subject: [PATCH 307/605] docker: Upgrade cerbero image to use fedora 30 --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 838528082b..b8b2e3f7d5 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:29 +FROM registry.fedoraproject.org/fedora:30 # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ From c6420d91643e62c6b5fba1884869375da022ecc7 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 29 Aug 2019 02:00:28 +1000 Subject: [PATCH 308/605] gitlab: add ios app build testing --- gitlab/ci_template.yml | 67 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ae4e2ae43b..e99b3b7d86 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -835,3 +835,70 @@ documentation: - $CI_PROJECT_NAME == "gst-python" - $CI_PROJECT_NAME == "gst-sharp" - $CI_PROJECT_NAME == "gst-build" + +# +# Build an iOS App using the iOS binaries +# +.ios universal examples: + stage: 'integrate' + when: 'manual' + variables: + EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples + # disable codesigning so we don't need developer identities on the CI + # machines + XCODE_BUILD_ARGS: > + CODE_SIGNING_REQUIRED="NO" + CODE_SIGN_IDENTITY="" + CODE_SIGNING_ALLOWED="NO" + CODE_SIGN_ENTITLEMENTS="" + script: + # install the binaries + - installer -pkg gstreamer-1.0-devel-*-ios-universal.pkg -target CurrentUserHomeDirectory -verbose + + - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py + - chmod +x clone_manifest_ref.py + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples + - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs + - rm clone_manifest_ref.py + + # dump some useful information + - xcodebuild -version + - xcodebuild -showsdks + + # gst-docs ios tutorials + - xcodebuild -showBuildSettings -alltargets -project ${EXAMPLES_HOME}/gst-docs/examples/tutorials/xcode\ iOS/GStreamer\ iOS\ Tutorials.xcodeproj + - xcodebuild -alltargets -destination generic/platform=iOS -project ${EXAMPLES_HOME}/gst-docs/examples/tutorials/xcode\ iOS/GStreamer\ iOS\ Tutorials.xcodeproj ${XCODE_BUILD_ARGS} + + # gst-examples + - xcodebuild -showBuildSettings -alltargets -project ${EXAMPLES_HOME}/gst-examples/playback/player/ios/GstPlay.xcodeproj + - xcodebuild -alltargets -destination generic/platform=iOS -project ${EXAMPLES_HOME}/gst-examples/playback/player/ios/GstPlay.xcodeproj ${XCODE_BUILD_ARGS} + after_script: + - rm -rf ${EXAMPLES_HOME}/gst-examples ${EXAMPLES_HOME}/gst-docs + tags: + - gst-ios-12.2 + +ios universal examples: + extends: ".ios universal examples" + needs: + - "build cerbero ios universal" + - "manifest" + dependencies: + - "build cerbero ios universal" + except: + variables: + - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-omx" + - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gstreamer-vaapi" + +cerbero ios universal examples: + extends: ".ios universal examples" + needs: + - "cerbero ios universal" + - "manifest" + dependencies: + - "cerbero ios universal" + only: + variables: + - $CI_PROJECT_NAME == "cerbero" From ed5343d3cef0a0968f0ef8bb2879ba09408e3147 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 10 Sep 2019 17:45:17 +1000 Subject: [PATCH 309/605] gitlab: remove needs from ios examples job needs and manual jobs do not really get along https://gitlab.com/gitlab-org/gitlab-ce/issues/66602 --- gitlab/ci_template.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e99b3b7d86..f259e57aef 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -879,9 +879,6 @@ documentation: ios universal examples: extends: ".ios universal examples" - needs: - - "build cerbero ios universal" - - "manifest" dependencies: - "build cerbero ios universal" except: @@ -894,9 +891,6 @@ ios universal examples: cerbero ios universal examples: extends: ".ios universal examples" - needs: - - "cerbero ios universal" - - "manifest" dependencies: - "cerbero ios universal" only: From 53abd5e8fd8885da95e73e94df7705cf583ad447 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 10 Sep 2019 14:38:18 +0530 Subject: [PATCH 310/605] gitlab: Windows Cerbero jobs are cross Change the name to avoid confusion. --- gitlab/ci_template.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f259e57aef..d116f261c5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -564,7 +564,7 @@ build cerbero android universal: # # Cerbero Cross Windows builds # -cerbero deps windows x86: +cerbero deps cross-windows x86: extends: '.cerbero deps' variables: CONFIG: "cross-win32.cbc" @@ -578,10 +578,10 @@ cerbero deps windows x86: cerbero cross win32: extends: '.cerbero cross win32' needs: - - "cerbero deps windows x86" + - "cerbero deps cross-windows x86" - "manifest" dependencies: - - "cerbero deps windows x86" + - "cerbero deps cross-windows x86" only: variables: - $CI_PROJECT_NAME == "cerbero" @@ -596,7 +596,7 @@ build cerbero cross win32: - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" -cerbero deps windows x86_64: +cerbero deps cross-windows x86_64: extends: '.cerbero deps' variables: CONFIG: "cross-win64.cbc" @@ -610,10 +610,10 @@ cerbero deps windows x86_64: cerbero cross win64: extends: '.cerbero cross win64' needs: - - "cerbero deps windows x86_64" + - "cerbero deps cross-windows x86_64" - "manifest" dependencies: - - "cerbero deps windows x86_64" + - "cerbero deps cross-windows x86_64" only: variables: - $CI_PROJECT_NAME == "cerbero" From 17fc9a66d4f1648db2f90066ded11f4419d2c229 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 11 Sep 2019 13:15:21 +0530 Subject: [PATCH 311/605] gitlab: Rename all cross jobs to have a cross- prefix For consistency, and to differentiate, f.ex., cross-windows jobs from native windows jobs. --- .gitlab-ci.yml | 10 +++---- gitlab/ci_template.yml | 64 +++++++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38aeb9346b..794c5de348 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -135,8 +135,8 @@ build cerbero fedora x86_64 local: refs: - "master@gstreamer/gst-ci" -build cerbero android universal local: - extends: '.cerbero android universal' +build cerbero cross-android universal local: + extends: '.cerbero cross-android universal' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" when: 'manual' except: @@ -174,11 +174,11 @@ check fedora local: variables: TEST_SUITE: 'check.gst*' -android universal examples local: - extends: '.android universal examples' +cross-android universal examples local: + extends: '.cross-android universal examples' image: "${CI_REGISTRY_IMAGE}/amd64/android:latest" dependencies: - - "build cerbero android universal local" + - "build cerbero cross-android universal local" when: 'manual' except: refs: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d116f261c5..1a56f05f7d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -529,30 +529,30 @@ build cerbero fedora x86_64: # # Cerbero Android Universal build # -cerbero deps android universal: +cerbero deps cross-android universal: extends: '.cerbero deps' variables: CONFIG: "cross-android-universal.cbc" ARCH: "android_universal" -.cerbero android universal: +.cerbero cross-android universal: extends: '.cerbero' variables: CONFIG: "cross-android-universal.cbc" -cerbero android universal: - extends: '.cerbero android universal' +cerbero cross-android universal: + extends: '.cerbero cross-android universal' needs: - - "cerbero deps android universal" + - "cerbero deps cross-android universal" - "manifest" dependencies: - - "cerbero deps android universal" + - "cerbero deps cross-android universal" only: variables: - $CI_PROJECT_NAME == "cerbero" -build cerbero android universal: - extends: '.cerbero android universal' +build cerbero cross-android universal: + extends: '.cerbero cross-android universal' except: variables: - $CI_PROJECT_NAME == "cerbero" @@ -631,7 +631,7 @@ build cerbero cross win64: # # Build an Android App using the android binaries # -.android universal examples: +.cross-android universal examples: image: $ANDROID_IMAGE stage: 'integrate' variables: @@ -670,13 +670,13 @@ build cerbero cross win64: - "manifest.xml" - "${EXAMPLES_HOME}/outputs" -android universal examples: - extends: ".android universal examples" +cross-android universal examples: + extends: ".cross-android universal examples" needs: - - "build cerbero android universal" + - "build cerbero cross-android universal" - "manifest" dependencies: - - "build cerbero android universal" + - "build cerbero cross-android universal" except: variables: - $CI_PROJECT_NAME == "cerbero" @@ -686,13 +686,13 @@ android universal examples: - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" -cerbero android universal examples: - extends: ".android universal examples" +cerbero cross-android universal examples: + extends: ".cross-android universal examples" needs: - - "cerbero android universal" + - "cerbero cross-android universal" - "manifest" dependencies: - - "cerbero android universal" + - "cerbero cross-android universal" only: variables: - $CI_PROJECT_NAME == "cerbero" @@ -756,7 +756,7 @@ build cerbero macos x86_64: # # Cerbero iOS build # -cerbero deps ios universal: +cerbero deps cross-ios universal: extends: '.cerbero deps' variables: ARCH: "ios_universal" @@ -766,7 +766,7 @@ cerbero deps ios universal: tags: - gst-ios-12.2 -.cerbero ios universal: +.cerbero cross-ios universal: extends: '.cerbero' variables: ARCH: "ios_universal" @@ -785,19 +785,19 @@ cerbero deps ios universal: - "${CERBERO_HOME}/logs" - "gstreamer-1.0-*-ios-universal.pkg" -cerbero ios universal: - extends: '.cerbero ios universal' +cerbero cross-ios universal: + extends: '.cerbero cross-ios universal' needs: - - "cerbero deps ios universal" + - "cerbero deps cross-ios universal" - "manifest" dependencies: - - "cerbero deps ios universal" + - "cerbero deps cross-ios universal" only: variables: - $CI_PROJECT_NAME == "cerbero" -build cerbero ios universal: - extends: '.cerbero ios universal' +build cerbero cross-ios universal: + extends: '.cerbero cross-ios universal' when: 'manual' except: variables: @@ -839,7 +839,7 @@ documentation: # # Build an iOS App using the iOS binaries # -.ios universal examples: +.cross-ios universal examples: stage: 'integrate' when: 'manual' variables: @@ -877,10 +877,10 @@ documentation: tags: - gst-ios-12.2 -ios universal examples: - extends: ".ios universal examples" +cross-ios universal examples: + extends: ".cross-ios universal examples" dependencies: - - "build cerbero ios universal" + - "build cerbero cross-ios universal" except: variables: - $CI_PROJECT_NAME == "cerbero" @@ -889,10 +889,10 @@ ios universal examples: - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gstreamer-vaapi" -cerbero ios universal examples: - extends: ".ios universal examples" +cerbero cross-ios universal examples: + extends: ".cross-ios universal examples" dependencies: - - "cerbero ios universal" + - "cerbero cross-ios universal" only: variables: - $CI_PROJECT_NAME == "cerbero" From 5acede7214924012f4092b8f083f1858e17c932e Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 12 Sep 2019 13:05:37 +1000 Subject: [PATCH 312/605] gitlab: build the ios examples in the cerbero repository --- gitlab/ci_template.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1a56f05f7d..31760fda37 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -841,7 +841,6 @@ documentation: # .cross-ios universal examples: stage: 'integrate' - when: 'manual' variables: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples # disable codesigning so we don't need developer identities on the CI @@ -879,6 +878,7 @@ documentation: cross-ios universal examples: extends: ".cross-ios universal examples" + when: 'manual' dependencies: - "build cerbero cross-ios universal" except: @@ -891,6 +891,9 @@ cross-ios universal examples: cerbero cross-ios universal examples: extends: ".cross-ios universal examples" + needs: + - "cerbero cross-ios universal" + - "manifest" dependencies: - "cerbero cross-ios universal" only: From 470b1516205d026f1e3303ab9020c7865f664775 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 6 Sep 2019 18:22:55 +0300 Subject: [PATCH 313/605] fedora/docker: remove the fedora openh264 package The cisco repository has been problematic for a while causing dnf to error due to some gpg issue and break the pipe in non-interactive ttys. ``` Importing GPG key 0xCFC659B9: Userid : "Fedora (30) " Fingerprint: F1D8 EC98 F241 AAF2 0DF6 9420 EF3C 111F CFC6 59B9 From : /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-30-x86_64 The command '/bin/sh -c /usr/bin/sh /root/prepare.sh && /usr/bin/sh /root/cleanup.sh' returned a non-zero code: 141 ``` https://gitlab.freedesktop.org/gstreamer/gst-ci/-/jobs/573870 We do have a meson subproject for openh264 nowdays in gst-build and it will get enabled automatically once the distro package is no longer there. --- docker/fedora/prepare.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 97a3f8e0f3..8e4af4fbd4 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -13,9 +13,6 @@ sudo dnf install -y \ dnf upgrade -y -# Enable the cisco openh264 repo -dnf config-manager --set-enabled fedora-cisco-openh264 - # install rest of the extra deps dnf install -y \ aom \ @@ -68,8 +65,6 @@ dnf install -y \ opencv-devel \ openjpeg2 \ openjpeg2-devel \ - openh264 \ - openh264-devel \ SDL2 \ SDL2-devel \ sbc \ From c98b4accb32ded2bf3120e0094105937aad17487 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 11 Sep 2019 16:50:27 +0300 Subject: [PATCH 314/605] .gitlab-ci.yml: export artifacts for the fedora build job 20f5d7d646e7c941d71bb7264a9ba6b868982be3 stopped exporting artifacts from the .build template which broke the local jobs since they no longer exported the builddir. --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 794c5de348..14c75dd632 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -123,6 +123,12 @@ build fedora x86_64 local: extends: '.build local' stage: 'build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest" + artifacts: + expire_in: '5 days' + when: always + paths: + - "manifest.xml" + - "gst-build/" except: refs: - "master@gstreamer/gst-ci" From 1e063c8539bcb7b31638d5ad9428348bd877ddbf Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 5 Sep 2019 22:53:20 +0300 Subject: [PATCH 315/605] .gitlab-ci.yml: Rename the UID variable in the docker build job UID is a read-only protected variable in bash 5.0+ --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14c75dd632..d2bc1ca84f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,7 +33,7 @@ test manifest: script: - export DATE=$(date +"%Y-%m-%d") - export IMAGE="${CI_REGISTRY_IMAGE}/${ARCH}/${TAG}" - - export UID="${IMAGE}:${DATE}-${CI_JOB_ID}" + - export _UID="${IMAGE}:${DATE}-${CI_JOB_ID}" - export LATEST="${IMAGE}:latest" - docker build --pull -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR} @@ -49,10 +49,10 @@ test manifest: - | if [ "$CI_COMMIT_REF_NAME" = "master" -a "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then # Tag the image with the git ref - docker image tag ${LATEST} ${UID} + docker image tag ${LATEST} ${_UID} # Push the tags docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker push ${UID} + docker push ${_UID} docker push ${LATEST} fi From d08496b1428eafa2ea8adceda4c2574e454e4f8d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 11 Sep 2019 17:41:06 +0300 Subject: [PATCH 316/605] docker/fedora: install debuginfo for python3-libs as well --- docker/fedora/prepare.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 8e4af4fbd4..71e8711964 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -72,7 +72,9 @@ dnf install -y \ x264 \ x264-libs \ x264-devel \ + python3 \ python3-devel \ + python3-libs \ python3-gobject \ python3-cairo \ python3-cairo-devel \ @@ -110,6 +112,7 @@ dnf debuginfo-install -y gtk3 \ openjpeg \ gobject-introspection \ python3 \ + python3-libs \ python3-gobject \ libjpeg-turbo \ glib-networking \ From e1d7b721e0be026aebef7d5512e4413ab89b2682 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 9 Sep 2019 16:01:08 +0300 Subject: [PATCH 317/605] docker: improve logging of the setup scripts --- docker/android/cleanup.sh | 2 +- docker/android/prepare.sh | 2 +- docker/fedora/cleanup.sh | 2 +- docker/fedora/prepare.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/android/cleanup.sh b/docker/android/cleanup.sh index 73d62c773e..a60a9fcf4b 100644 --- a/docker/android/cleanup.sh +++ b/docker/android/cleanup.sh @@ -1,4 +1,4 @@ -set -eu +set -eux echo "Removing DNF cache" dnf clean all diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh index efc5b8a123..32843922be 100644 --- a/docker/android/prepare.sh +++ b/docker/android/prepare.sh @@ -1,4 +1,4 @@ -set -eu +set -eux dnf install -y \ file \ diff --git a/docker/fedora/cleanup.sh b/docker/fedora/cleanup.sh index 73d62c773e..a60a9fcf4b 100644 --- a/docker/fedora/cleanup.sh +++ b/docker/fedora/cleanup.sh @@ -1,4 +1,4 @@ -set -eu +set -eux echo "Removing DNF cache" dnf clean all diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 71e8711964..b4c5069059 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -1,4 +1,4 @@ -set -eu +set -eux dnf install -y git-core ninja-build dnf-plugins-core From 4a8beec41044e21ffeec73b7e58d0afaf112e689 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 24 Sep 2019 11:33:46 +1000 Subject: [PATCH 318/605] docker/cerbero/fedora install openssl-devel libraries Temporary workaround until cerbero installs the library itself --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index b8b2e3f7d5..7dc80fc0fb 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -3,7 +3,7 @@ FROM registry.fedoraproject.org/fedora:30 # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ dnf upgrade -y && \ - dnf install -y sudo git lbzip2 rsync + dnf install -y sudo git lbzip2 rsync openssl-devel # Configure git for various usage RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ From 62ec9173e61057b9e7904ef82de08a54790f7672 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 20 Sep 2019 01:47:52 +1000 Subject: [PATCH 319/605] gitlab: update cerbero docker image used --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 31760fda37..af0ed80d32 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,7 +8,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-06-12-354260' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-06-06-343857' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-09-24-648163' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' From bcb1ae6cf280559d3ba1fbf6e0aed6ce6185afe9 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 24 Sep 2019 12:39:11 +1000 Subject: [PATCH 320/605] Revert "docker/cerbero/fedora install openssl-devel libraries" This reverts commit 4a8beec41044e21ffeec73b7e58d0afaf112e689. --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 7dc80fc0fb..b8b2e3f7d5 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -3,7 +3,7 @@ FROM registry.fedoraproject.org/fedora:30 # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ dnf upgrade -y && \ - dnf install -y sudo git lbzip2 rsync openssl-devel + dnf install -y sudo git lbzip2 rsync # Configure git for various usage RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ From 92709d5542ed83418ed72ae9833739241d875a23 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 11 Sep 2019 17:39:14 +0300 Subject: [PATCH 321/605] ci_template: Add an auto-retry policy Its common that docker will throw transient errors, especially on the windows runner. If such the case, automatically retry the job. --- gitlab/ci_template.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index af0ed80d32..19b00794c2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -35,8 +35,17 @@ variables: --variants werror --timestamps +.global_retry_policy: + retry: + max: 2 + when: + - 'runner_system_failure' + - 'api_failure' + manifest: image: $MANIFEST_IMAGE + extends: + - '.global_retry_policy' stage: 'preparation' script: - cd /gst-ci @@ -50,6 +59,8 @@ manifest: gst indent: image: $INDENT_IMAGE + extends: + - '.global_retry_policy' stage: 'preparation' variables: GIT_STRATEGY: 'fetch' @@ -77,6 +88,8 @@ gst indent: .build: stage: 'build' + extends: + - '.global_retry_policy' needs: - "manifest" dependencies: @@ -170,6 +183,8 @@ build clang fedora x86_64: .test: stage: 'test' + extends: + - '.global_retry_policy' variables: # Disable colored output to avoid weird rendering issues GST_DEBUG_NO_COLOR: "true" @@ -312,6 +327,8 @@ valgrind ges: .cerbero: stage: "build" image: $CERBERO_IMAGE + extends: + - '.global_retry_policy' needs: - "manifest" dependencies: @@ -375,6 +392,8 @@ valgrind ges: .build windows: image: $WINDOWS_IMAGE + extends: + - '.global_retry_policy' stage: 'build' needs: - 'manifest' @@ -633,6 +652,8 @@ build cerbero cross win64: # .cross-android universal examples: image: $ANDROID_IMAGE + extends: + - '.global_retry_policy' stage: 'integrate' variables: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples @@ -811,6 +832,8 @@ build cerbero cross-ios universal: documentation: image: $FEDORA_IMAGE + extends: + - '.global_retry_policy' needs: - "build fedora x86_64" - "manifest" @@ -841,6 +864,8 @@ documentation: # .cross-ios universal examples: stage: 'integrate' + extends: + - '.global_retry_policy' variables: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples # disable codesigning so we don't need developer identities on the CI From 32e64c4ac369b038588f8462f474335113e5cfe8 Mon Sep 17 00:00:00 2001 From: "U. Artie Eoff" Date: Wed, 9 Oct 2019 08:37:01 -0700 Subject: [PATCH 322/605] ci_template: avoid check.gstreamer-vaapi Check tests are being added to gstreamer/gstreamer-vaapi!181. However, gstreamer-vaapi inherently requires specific hardware drivers and platforms to function. The CI does not provide this level of driver/platform selection. Thus, avoid running any check tests in gstreamer-vaapi. --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 19b00794c2..ab4c5d20ca 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -242,6 +242,7 @@ check fedora: - $CI_PROJECT_NAME == "gst-examples" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gstreamer-vaapi" integration testsuites fedora: extends: '.test fedora x86_64' From 2a5d7674246cabfd4dccc115bb496b0ccfe85fe8 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 9 Aug 2019 13:27:37 +0300 Subject: [PATCH 323/605] ci_template: update the fedora image to a newer build Among other things, it includes the wpe dependencies !148 and glslc !145 --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ab4c5d20ca..da14f67b12 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,7 +9,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-06-12-354260' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-09-24-648163' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-09-23-643905' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v6' From 20c2bb8080709e680d942483e45b0a0d67e6e149 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 24 Oct 2019 15:13:03 +0300 Subject: [PATCH 324/605] ci_template: refresh the docker images used for the CI --- .gitlab-ci.yml | 2 +- gitlab/ci_template.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2bc1ca84f..ad22b328da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ stages: test manifest: variables: GIT_STRATEGY: fetch - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:fb91ee18309ab981289aa818ab2a7824ef0567a1' + image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:2019-10-23-793476' stage: "preparation" script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index da14f67b12..ecc2c3916a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -7,11 +7,11 @@ stages: - 'integrate' variables: - ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-06-12-354260' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-09-24-648163' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-09-23-643905' + ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-10-23-793480' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-10-23-793479' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-10-23-793478' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' - MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e' + MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v6' # Branch to track for modules that have no ref specified in the manifest From e8abe05e021d98c29d5129804e27ecd51b483046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 25 Oct 2019 17:38:43 +0200 Subject: [PATCH 325/605] ci_template: remove gstreamer-vaapi control in test tag Since now gstreamer-vaapi is blacklisted in validate (https://gitlab.freedesktop.org/gstreamer/gst-devtools/merge_requests/134) there is no need to make exceptions in the CI. This patch removes those exceptions. --- gitlab/ci_template.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ecc2c3916a..d9a80f64d6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -242,19 +242,15 @@ check fedora: - $CI_PROJECT_NAME == "gst-examples" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gstreamer-vaapi" integration testsuites fedora: extends: '.test fedora x86_64' - before_script: - - rm -f gst-build/build/subprojects/gstreamer-vaapi/gst/vaapi/libgstvaapi.so variables: EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs" TEST_SUITE: "validate ges" except: variables: - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" From c8d0102561132b702c1eec307c5625f648f4cfb2 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 12 Aug 2019 17:15:47 -0400 Subject: [PATCH 326/605] Revert "gitlab: Set COLUMNS and width variable to make terminal_markdown_view happy" This reverts commit 157f94fb62da7582fce90c85bf2338999d83b7cc. --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d9a80f64d6..ff853079a8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -190,8 +190,6 @@ build clang fedora x86_64: GST_DEBUG_NO_COLOR: "true" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" - width: 100 - COLUMNS: 100 script: - cd gst-build/ - echo "-> Running ${TEST_SUITE}" From b0b76594c3386e2ee90abfb5b5605ba3acee37ca Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 12 Aug 2019 17:17:03 -0400 Subject: [PATCH 327/605] docker: Stop using mdv and use bat to colorize markdown --- docker/fedora/prepare.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index b4c5069059..bb94a468f9 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -17,6 +17,7 @@ dnf upgrade -y dnf install -y \ aom \ aom-extra-tools \ + bat \ libaom \ libaom-devel \ ccache \ @@ -162,7 +163,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.50.1 git+https://github.com/axiros/terminal_markdown_viewer hotdoc +pip3 install meson==0.50.1 hotdoc # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From bd66c756ef375c5432281bc0bbe2b48f3ad05783 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 7 Nov 2019 11:51:29 +0100 Subject: [PATCH 328/605] citemplate: Tag test jobs to only run on the dedicated runners Avoid spreading the jobs on lower power runners which ends up trippling the execution time of the testsuites and/or timing out. --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ff853079a8..28cd3deb0b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -224,6 +224,7 @@ build clang fedora x86_64: .test fedora x86_64: image: $FEDORA_IMAGE extends: '.test' + tags: ['gstreamer'] needs: - 'build fedora x86_64' dependencies: From 4e93f927831b5ac1e38163b35ea21b54092a551b Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sun, 10 Nov 2019 04:27:14 +0530 Subject: [PATCH 329/605] docker/cerbero: Temporarily add nasm to the system setup We will later install it as part of the cerbero bootstrap step when generating the image after https://gitlab.freedesktop.org/gstreamer/cerbero/merge_requests/320 is merged. --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index b8b2e3f7d5..61f7b368c1 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -3,7 +3,7 @@ FROM registry.fedoraproject.org/fedora:30 # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ dnf upgrade -y && \ - dnf install -y sudo git lbzip2 rsync + dnf install -y sudo git lbzip2 rsync nasm # Configure git for various usage RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ From 6b48c9c003c1385b1382cc339bd972a4b6bf0b8d Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sun, 10 Nov 2019 15:26:48 +0530 Subject: [PATCH 330/605] ci_template.yml: Use the new docker cerbero-fedora image This has nasm temporarily added in it using https://gitlab.freedesktop.org/gstreamer/gst-ci/merge_requests/179 --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 28cd3deb0b..87e901765d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,7 +8,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-10-23-793480' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-10-23-793479' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-11-10-917246' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-10-23-793478' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' From d19360b37e28374a196a2e9a0dc02a5beef5b4e6 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 19 Nov 2019 15:47:38 +0200 Subject: [PATCH 331/605] ci_template: set the TIMEOUT_FACTOR of gst-validate-launcher to 2 We have lots of tests that timeout on the CI due to a high load of jobs on the CI runners. Let's try giving them a bit more time and see how its going. --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 87e901765d..798c2e9d9a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -190,6 +190,7 @@ build clang fedora x86_64: GST_DEBUG_NO_COLOR: "true" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" + TIMEOUT_FACTOR: "2" script: - cd gst-build/ - echo "-> Running ${TEST_SUITE}" @@ -201,6 +202,7 @@ build clang fedora x86_64: --shuffle --no-display --meson-no-rebuild + --timeout-factor "${TIMEOUT_FACTOR}" --fail-on-testlist-change -l "${CI_PROJECT_DIR}/validate-logs/" --xunit-file "${CI_PROJECT_DIR}/validate-logs/xunit.xml" From 3b700eb09ace2c65f082a8339df74c7a0c5aa218 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 21 Nov 2019 16:21:50 +0200 Subject: [PATCH 332/605] ci_template: skip unnecessary jobs for gst-integration-testsuites Close #42 --- gitlab/ci_template.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 798c2e9d9a..2e2cdf7347 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -152,6 +152,7 @@ build nodebug fedora x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" build static fedora x86_64: extends: 'build fedora x86_64' @@ -161,6 +162,7 @@ build static fedora x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" build static nodebug fedora x86_64: extends: 'build nodebug fedora x86_64' @@ -170,6 +172,7 @@ build static nodebug fedora x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" build clang fedora x86_64: extends: 'build fedora x86_64' @@ -180,6 +183,7 @@ build clang fedora x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" .test: stage: 'test' @@ -266,6 +270,7 @@ integration testsuites fedora: except: variables: - $CI_PROJECT_NAMESPACE == 'gstreamer' + - $CI_PROJECT_NAME == "gst-integration-testsuites" valgrind core: extends: '.valgrind fedora x86_64' @@ -374,6 +379,7 @@ valgrind ges: variables: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" cache: key: "${CI_JOB_NAME}" paths: @@ -436,6 +442,7 @@ valgrind ges: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" build vs2017 amd64: extends: '.build windows' @@ -542,6 +549,7 @@ build cerbero fedora x86_64: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" # # Cerbero Android Universal build @@ -577,6 +585,7 @@ build cerbero cross-android universal: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" # # Cerbero Cross Windows builds @@ -612,6 +621,7 @@ build cerbero cross win32: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" cerbero deps cross-windows x86_64: extends: '.cerbero deps' @@ -644,6 +654,7 @@ build cerbero cross win64: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-integration-testsuites" # # Build an Android App using the android binaries @@ -766,7 +777,7 @@ build cerbero macos x86_64: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-integration-testsuite" + - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" @@ -822,7 +833,7 @@ build cerbero cross-ios universal: variables: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-integration-testsuite" + - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" @@ -852,7 +863,7 @@ documentation: except: variables: - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-integration-testsuite" + - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gst-python" - $CI_PROJECT_NAME == "gst-sharp" - $CI_PROJECT_NAME == "gst-build" From 2fadd6e3d0e99312ee75965ac75819ad2d1b59ac Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 22 Nov 2019 15:44:56 +0200 Subject: [PATCH 333/605] docker/windows: update the version of various deps we install --- docker/windows/install_toolchain.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index ac7a3ecc03..6c224e05d6 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -1,9 +1,9 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; -$python_dl_url = 'https://www.python.org/ftp/python/3.7.3/python-3.7.3.exe' +$python_dl_url = 'https://www.python.org/ftp/python/3.7.4/python-3.7.4.exe' $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' -$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip' -$zip_url = 'https://www.7-zip.org/a/7z1805-x64.exe' +$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.0.windows.2/MinGit-2.24.0.2-64-bit.zip' +$zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe' $msys_url = 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' Write-Host "Installing VisualStudio" @@ -49,4 +49,4 @@ C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && p C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" -pip install meson \ No newline at end of file +pip install meson From 4b20b2dbb3c184e4f84f794027d428c3e6e274ea Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 22 Nov 2019 17:23:15 +0200 Subject: [PATCH 334/605] ci_template.yml: Update the windows image used --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2e2cdf7347..fb8f8e90e2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -12,7 +12,7 @@ variables: FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-10-23-793478' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v6' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v7' # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'master' From d153e0d7ed4ed9ff0b212116ba304d072e7d5490 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 28 Nov 2019 12:55:51 +1100 Subject: [PATCH 335/605] update for macos 10.15 making / read-only We've moved the host cached cerbero directory to /Users/gst-ci/cerbero instead now. --- gitlab/ci_template.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index fb8f8e90e2..d08f0ae6fe 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -349,6 +349,8 @@ valgrind ges: HAVE_CCACHE: "yes" # used by macos packages as we only ever install to a fixed directory CERBERO_OVERRIDDEN_DIST_DIR: "" + # location where cerbero is cached on the host + CERBERO_HOST_DIR: "/cerbero/" before_script: # FIXME Wrong namespace # Workaround build-tools having hardcoded internal path @@ -357,7 +359,7 @@ valgrind ges: - ln -sf $(pwd) ../../gstreamer/cerbero - mkdir -p ../../${CI_PROJECT_NAMESPACE} - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero - - rsync -aH /cerbero/ . + - rsync -aH ${CERBERO_HOST_DIR} . - test -f ${CERBERO_DEPS} && tar -C ${CERBERO_HOME} -xf ${CERBERO_DEPS} - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc @@ -737,8 +739,9 @@ cerbero deps macos x86_64: CONFIG: "osx-x86-64.cbc" HAVE_CCACHE: "" CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" + CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" tags: - - gst-macos-10.14 + - gst-macos-10.15 .cerbero macos x86_64: extends: '.cerbero' @@ -748,8 +751,9 @@ cerbero deps macos x86_64: CERBERO_PACKAGE_ARGS: "" HAVE_CCACHE: "" CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" + CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" tags: - - gst-macos-10.14 + - gst-macos-10.15 artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" expire_in: '5 days' @@ -793,8 +797,9 @@ cerbero deps cross-ios universal: CONFIG: "cross-ios-universal.cbc" CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" HAVE_CCACHE: "" + CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" tags: - - gst-ios-12.2 + - gst-ios-13.2 .cerbero cross-ios universal: extends: '.cerbero' @@ -804,8 +809,9 @@ cerbero deps cross-ios universal: CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" CERBERO_PACKAGE_ARGS: "" HAVE_CCACHE: "" + CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" tags: - - gst-ios-12.2 + - gst-ios-13.2 artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" expire_in: '5 days' @@ -908,7 +914,7 @@ documentation: after_script: - rm -rf ${EXAMPLES_HOME}/gst-examples ${EXAMPLES_HOME}/gst-docs tags: - - gst-ios-12.2 + - gst-ios-13.2 cross-ios universal examples: extends: ".cross-ios universal examples" From be6d2b4cf21c8d0f7a16716fc4451837d6202a6c Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 25 Nov 2019 15:51:52 +0200 Subject: [PATCH 336/605] ci_template: Use a host-mapped volume for the ccache directory Use hardcoded path for the cache, so it will be shared by all jobs of all gstreamer projects running on the same runner host. This should increase the hit rate and decrease the worst case storage used. Adaptation of https://gitlab.freedesktop.org/mesa/mesa/merge_requests/2818 See https://gitlab.freedesktop.org/freedesktop/freedesktop/issues/120 --- gitlab/ci_template.yml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d08f0ae6fe..536066c359 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -95,12 +95,17 @@ gst indent: dependencies: - "manifest" variables: - CCACHE_BASEDIR: "${CI_PROJECT_DIR}" - CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" + CCACHE_COMPILERCHECK: "content" + CCACHE_COMPRESS: "true" + CCACHE_BASEDIR: "/cache/gstreamer/gst-build" + CCACHE_DIR: "/cache/gstreamer/gst-build/ccache/" + # shared across everything really + CCACHE_MAXSIZE: "10G" + MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} --werror" script: - - ccache -z + - ccache --show-stats - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py - chmod +x clone_manifest_ref.py - ./clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build @@ -110,7 +115,7 @@ gst indent: - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" - meson build/ $MESON_ARGS - ninja -C build/ - - ccache -s + - ccache --show-stats after_script: - cd gst-build/ # Clean the artifacts packages to avoid copying "useless" build products. @@ -118,10 +123,6 @@ gst indent: # Clean the .git repos since we won't need them anymore - rm -rf subprojects/*/.git/ - rm -rf build/subprojects/*/.git/ - cache: - key: "${CI_JOB_NAME}" - paths: - - "${CCACHE_DIR}" except: variables: - $CI_PROJECT_NAME == "cerbero" @@ -337,9 +338,13 @@ valgrind ges: dependencies: - "manifest" variables: - CCACHE_BASEDIR: "${CI_PROJECT_DIR}" - CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" - CCACHE_MAXSIZE: "1.7G" + CCACHE_COMPILERCHECK: "content" + CCACHE_COMPRESS: "true" + CCACHE_BASEDIR: "/cache/gstreamer/cerbero/" + CCACHE_DIR: "/cache/gstreamer/cerbero/ccache/" + # shared across everything really + CCACHE_MAXSIZE: "50G" + CERBERO_HOME: "cerbero-build" CERBERO_SOURCES: "cerbero-sources" CERBERO_DEPS: "cerbero-deps.tar.gz" @@ -365,7 +370,7 @@ valgrind ges: - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc - ./cerbero-uninstalled --self-update manifest.xml script: - - test "x${HAVE_CCACHE}" = "xyes" && ccache -z + - test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats - $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 @@ -385,7 +390,6 @@ valgrind ges: cache: key: "${CI_JOB_NAME}" paths: - - "${CCACHE_DIR}" - "${CERBERO_SOURCES}" artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" From ae90b9a0424a535cae9b256e84a03353287c7220 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 28 Nov 2019 14:59:30 +0200 Subject: [PATCH 337/605] ci_template: set per-job timeouts for builds In case a build gets stuck for whatever reason, happens from time to time on windows, try to baild out quickly. For cerbero builds, set the timeout to 3h which according to the docs should also be able to override the project defined timeout Fix #19 https://docs.gitlab.com/ce/ci/yaml/README.html#timeout --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 536066c359..9b698e4134 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -94,6 +94,8 @@ gst indent: - "manifest" dependencies: - "manifest" + # Taking into account the slowest shared runner + time needed to upload the binaries to artifacts + timeout: '30min' variables: CCACHE_COMPILERCHECK: "content" CCACHE_COMPRESS: "true" @@ -337,6 +339,7 @@ valgrind ges: - "manifest" dependencies: - "manifest" + timeout: '3h' variables: CCACHE_COMPILERCHECK: "content" CCACHE_COMPRESS: "true" @@ -413,6 +416,7 @@ valgrind ges: - 'docker' - 'windows' - '1607' + timeout: '10min' variables: MESON_ARGS: > ${DEFAULT_MESON_ARGS} From d382cb0b759d344e88c15cd3374a161cad3184ba Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sun, 1 Dec 2019 14:46:27 +0200 Subject: [PATCH 338/605] ci_template: Bump the timeout limit from 10 to 15min for windows --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9b698e4134..0d104fde90 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -416,7 +416,7 @@ valgrind ges: - 'docker' - 'windows' - '1607' - timeout: '10min' + timeout: '15min' variables: MESON_ARGS: > ${DEFAULT_MESON_ARGS} From 626be8a012b88104a5d9277b9855410d5d560c66 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 3 Dec 2019 00:53:02 +0200 Subject: [PATCH 339/605] ci_template: Increase the timeout of gst-builds more Need to take into account I/O needed to upload artficats and download docker images. --- gitlab/ci_template.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0d104fde90..4c06b2c572 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -95,7 +95,8 @@ gst indent: dependencies: - "manifest" # Taking into account the slowest shared runner + time needed to upload the binaries to artifacts - timeout: '30min' + # Also need to take into account I/O of pulling docker images and uploading artifacts + timeout: '45min' variables: CCACHE_COMPILERCHECK: "content" CCACHE_COMPRESS: "true" From 9c7c52f5b9ab72d036d2c30cec191c4fe57f1f6f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 3 Dec 2019 10:37:16 +1100 Subject: [PATCH 340/605] gitlab: build cerbero android/ios for gst-docs examples --- gitlab/ci_template.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4c06b2c572..e516024a23 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -386,11 +386,6 @@ valgrind ges: - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o $(pwd) gstreamer-1.0 - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test - except: - variables: - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" cache: key: "${CI_JOB_NAME}" paths: @@ -595,7 +590,6 @@ build cerbero cross-android universal: - $CI_PROJECT_NAME == "gst-build" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" # @@ -725,7 +719,6 @@ cross-android universal examples: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" cerbero cross-android universal examples: extends: ".cross-android universal examples" @@ -851,7 +844,6 @@ build cerbero cross-ios universal: - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-sharp" documentation: From b3aa9b5286af1d27ed68249a540fac98d846a720 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 2 Dec 2019 18:23:27 +0200 Subject: [PATCH 341/605] windows: Migrate to windows 1809 base Rebuild the windows docker image against the current ltsc [1] of server 2019. This requires moving some of the msys setup to the runner job cause it causes docker build to hang Switch the job tags so they now use the 1809 runner, instead of 1607. Tweak the PATHs in the msys job so bash doesn't complain about slashes.. Lastly, increase the timeout of the windows jobs, as msys2 installs its deps at runtime [1] https://docs.microsoft.com/en-us/windows-server/get-started-19/servicing-channels-19 --- docker/windows/Dockerfile | 4 ++-- docker/windows/install_toolchain.ps1 | 8 +++----- gitlab/ci_template.yml | 29 ++++++++++++++++++---------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index e0b8244141..e8430606a9 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -1,6 +1,6 @@ # escape=` -FROM 'mcr.microsoft.com/windows/servercore:1607' +FROM 'mcr.microsoft.com/windows/servercore:1809' # Make sure any failure in PowerShell scripts is fatal SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] @@ -9,4 +9,4 @@ COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\ RUN C:\install_toolchain.ps1 -RUN C:\prepare_gst_env.ps1 \ No newline at end of file +RUN C:\prepare_gst_env.ps1 diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index 6c224e05d6..2d4260ec0c 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -44,9 +44,7 @@ Write-Host "Installing git-lfs" choco install -y git-lfs refreshenv -$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin" -C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys" -C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" -C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" - +Write-Host "Installing Meson" pip install meson + +Write-Host "Complete" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e516024a23..7466700422 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -12,7 +12,7 @@ variables: FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-10-23-793478' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v7' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v8' # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'master' @@ -411,8 +411,8 @@ valgrind ges: tags: - 'docker' - 'windows' - - '1607' - timeout: '15min' + - '1809' + timeout: '45min' variables: MESON_ARGS: > ${DEFAULT_MESON_ARGS} @@ -424,6 +424,8 @@ valgrind ges: - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build - cd $env:CI_PROJECT_DIR/gst-build - cp -r C:/subprojects/* subprojects/ + # Run the git-update script and feed it the manifest to setup the environment + - cd $env:CI_PROJECT_DIR/gst-build && python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml # For some reason, options are separated by newline instead of space, so we # have to replace them first. - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") @@ -431,7 +433,6 @@ valgrind ges: # Environment variables substitutions is done by PowerShell before calling # cmd.exe, that's why we use $env:FOO instead of %FOO% - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && - python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml && meson build $env:MESON_ARGS && ninja -C build" # FIXME: extract builddir for tests @@ -463,16 +464,24 @@ build vs2017 x86: build msys2 : extends: '.build windows' script: - - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build - - cd $env:CI_PROJECT_DIR/gst-build - - cp -r C:/subprojects/* subprojects/ + # For some reason docker build hangs if this is included in the image, needs more troubleshooting + - $env:PATH += ';C:\msys64\usr\bin;C:\msys64\mingw64\bin;C:\msys64\mingw32\bin' + - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true" + - C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" + - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" - - $env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin" + - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR\gst-build + - cd $env:CI_PROJECT_DIR\gst-build + - cp -r C:\subprojects\* subprojects\ + + # Run the git-update script and feed it the manifest to setup the environment + - cd $env:CI_PROJECT_DIR/gst-build && python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml # For some reason, options are separated by newline instead of space, so we # have to replace them first. - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") - - C:\msys64\usr\bin\bash -c "cd $env:CI_PROJECT_DIR/gst-build && - python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml && + # Replace forward slashes with backwards so bash doesn't complain + - $env:_PROJECT_DIR = $env:CI_PROJECT_DIR.replace('\','/') + - C:\msys64\usr\bin\bash -c "cd $env:_PROJECT_DIR/gst-build && meson build $env:MESON_ARGS && ninja -C build" From 5c0ad2facdfc418c7af640ecf6eb4449d3f6b04f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 5 Dec 2019 00:52:45 +0200 Subject: [PATCH 342/605] ci_template: make sure powershell hard exists on errors We set the env var as the shell of the image, but looks like the gitlab runner is overriding that. --- docker/windows/Dockerfile | 1 + gitlab/ci_template.yml | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index e8430606a9..913b19732b 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -4,6 +4,7 @@ FROM 'mcr.microsoft.com/windows/servercore:1809' # Make sure any failure in PowerShell scripts is fatal SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] +ENV ErrorActionPreference='Stop' COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7466700422..e846c6169a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -421,6 +421,10 @@ valgrind ges: -Dvaapi=disabled -Ddevtools=disabled script: + # Make sure powershell exists on errors + # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 + - $ErrorActionPreference = "Stop" + - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build - cd $env:CI_PROJECT_DIR/gst-build - cp -r C:/subprojects/* subprojects/ @@ -464,6 +468,10 @@ build vs2017 x86: build msys2 : extends: '.build windows' script: + # Make sure powershell exists on errors + # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 + - $ErrorActionPreference = "Stop" + # For some reason docker build hangs if this is included in the image, needs more troubleshooting - $env:PATH += ';C:\msys64\usr\bin;C:\msys64\mingw64\bin;C:\msys64\mingw32\bin' - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true" From bc7f2feb5b6e93fac19fe7f0fc7ed3cc13cd233a Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 5 Dec 2019 00:49:40 +0200 Subject: [PATCH 343/605] windows: fix invalid powershell syntax ``` At line:1 char:34 + cd $env:CI_PROJECT_DIR/gst-build && python git-update --no-interactio ... + ~~ The token '&&' is not a valid statement separator in this version. + CategoryInfo : ParserError: (:) [], ParentContainsErrorRecordEx ception + FullyQualifiedErrorId : InvalidEndOfLine ``` This is not bash, but powershell, hue hue hue --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e846c6169a..625f06ab06 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -429,7 +429,7 @@ valgrind ges: - cd $env:CI_PROJECT_DIR/gst-build - cp -r C:/subprojects/* subprojects/ # Run the git-update script and feed it the manifest to setup the environment - - cd $env:CI_PROJECT_DIR/gst-build && python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml + - cd $env:CI_PROJECT_DIR/gst-build; python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml # For some reason, options are separated by newline instead of space, so we # have to replace them first. - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") @@ -483,7 +483,7 @@ build msys2 : - cp -r C:\subprojects\* subprojects\ # Run the git-update script and feed it the manifest to setup the environment - - cd $env:CI_PROJECT_DIR/gst-build && python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml + - cd $env:CI_PROJECT_DIR/gst-build; python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml # For some reason, options are separated by newline instead of space, so we # have to replace them first. - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") From 926fcc52b9a5962b64bd21062451e23015626f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 5 Dec 2019 00:37:20 +0000 Subject: [PATCH 344/605] ci_template: bump timeout for msys2 job --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 625f06ab06..5e40800655 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -467,6 +467,7 @@ build vs2017 x86: build msys2 : extends: '.build windows' + timeout: '60min' script: # Make sure powershell exists on errors # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 From aa18f639b517d5d0a5fe74610c51a148a2820b07 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 6 Dec 2019 00:53:24 +1100 Subject: [PATCH 345/605] gitlab: enable ios builds by default They are small enough now to not impact the overall running time of a CI pipeline. --- gitlab/ci_template.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 5e40800655..8f66a8811e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -854,7 +854,6 @@ cerbero cross-ios universal: build cerbero cross-ios universal: extends: '.cerbero cross-ios universal' - when: 'manual' except: variables: - $CI_PROJECT_NAME == "cerbero" @@ -937,7 +936,9 @@ documentation: cross-ios universal examples: extends: ".cross-ios universal examples" - when: 'manual' + needs: + - "build cerbero cross-ios universal" + - "manifest" dependencies: - "build cerbero cross-ios universal" except: From 51ff579cecb194ddfa54b0689032b78025deed41 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 11 Dec 2019 17:34:19 +0200 Subject: [PATCH 346/605] ci_template: limit cerbero jobs to the GStreamer tagged runners CCache tends to consume a lot of space which taxes heavily some of the shared runners. Limit the mahcines the job can run to those were we can ensure they will not have issues with the storage. --- gitlab/ci_template.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8f66a8811e..56730a4359 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -340,6 +340,9 @@ valgrind ges: - "manifest" dependencies: - "manifest" + # Ensure that the runners it will be executed on + # will have plenty of space for the cache + tags: ['gstreamer'] timeout: '3h' variables: CCACHE_COMPILERCHECK: "content" From f4d335bc8c5b3149eda3a706157b60220a6992c7 Mon Sep 17 00:00:00 2001 From: Eric Marks Date: Tue, 17 Dec 2019 10:13:05 -0600 Subject: [PATCH 347/605] docker/fedora: Install libcaca and aalib Add libcaca-devel and aalib-devel to dependencies so that cacasink and aasink can be built. --- docker/fedora/prepare.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index bb94a468f9..d80e69b905 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -15,11 +15,13 @@ dnf upgrade -y # install rest of the extra deps dnf install -y \ + aalib-devel \ aom \ aom-extra-tools \ bat \ libaom \ libaom-devel \ + libcaca-devel \ ccache \ cmake \ clang-devel \ From 0b3645f7fdcf6c87fb0248f079c5a522af94b127 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Wed, 18 Dec 2019 10:48:14 -0500 Subject: [PATCH 348/605] docker/fedora: Update meson to 0.52.1 --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index d80e69b905..aba45b9669 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -165,7 +165,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.50.1 hotdoc +pip3 install meson==0.52.1 hotdoc # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From 511ffe9fa14f4ddc741087841a26f17908bc7ad1 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Wed, 18 Dec 2019 13:47:01 -0500 Subject: [PATCH 349/605] Update fedora image --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 56730a4359..565151ff8f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,7 +9,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-10-23-793480' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-11-10-917246' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-10-23-793478' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-12-18-1191711' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v8' From af9982cffefef50273f70e0cbd6da153492b2819 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 18 Dec 2019 09:41:59 +0530 Subject: [PATCH 350/605] docker/fedora: install Rust Needed to build gst-plugins-rs, see gst-plugins-rs!205 --- docker/fedora/Dockerfile | 2 ++ docker/fedora/prepare.sh | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index 31bdd58c63..fdd972432d 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -1,5 +1,7 @@ FROM registry.fedoraproject.org/fedora:30 +ENV PATH=/usr/local/cargo/bin:$PATH + COPY prepare.sh cleanup.sh /root/ RUN /usr/bin/sh /root/prepare.sh && \ diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index aba45b9669..b773c04045 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -197,6 +197,28 @@ dnf download glib2-doc gdk-pixbuf2-devel*x86_64* gtk3-devel-docs rpm -i --reinstall *.rpm rm -f *.rpm +# Install Rust +RUSTUP_VERSION=1.21.0 +RUST_VERSION=1.40.0 +RUST_ARCH="x86_64-unknown-linux-gnu" + +# rustup-init uses those variables as install paths +export RUSTUP_HOME=/usr/local/rustup +export CARGO_HOME=/usr/local/cargo +dnf install -y wget +RUSTUP_URL=https://static.rust-lang.org/rustup/archive/$RUSTUP_VERSION/$RUST_ARCH/rustup-init +wget $RUSTUP_URL +dnf remove -y wget + +chmod +x rustup-init; +./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; +rm rustup-init; +chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +rustup --version +cargo --version +rustc --version + # get gst-build and make all subprojects available git clone git://anongit.freedesktop.org/gstreamer/gst-build /gst-build/ cd /gst-build From 29760465061f7b41d3df29565393a546bf138145 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Fri, 20 Dec 2019 15:45:17 +0530 Subject: [PATCH 351/605] gitlab: update fedora image Use image with Rust. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 565151ff8f..b34184ffcb 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,7 +9,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-10-23-793480' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-11-10-917246' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-12-18-1191711' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-12-20-1211878' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v8' From 8a85aa6eba87773c154087ee0ff6e80ea5974659 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Mon, 23 Dec 2019 11:52:20 +0530 Subject: [PATCH 352/605] docker: export RUSTUP/CARGO_HOME in container The rust binaries (rustc) needs those variables to be defined to work properly. --- docker/fedora/Dockerfile | 4 +++- docker/fedora/prepare.sh | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index fdd972432d..56f9279c44 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -1,6 +1,8 @@ FROM registry.fedoraproject.org/fedora:30 -ENV PATH=/usr/local/cargo/bin:$PATH +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH COPY prepare.sh cleanup.sh /root/ diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index b773c04045..eec0275e1d 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -202,9 +202,6 @@ RUSTUP_VERSION=1.21.0 RUST_VERSION=1.40.0 RUST_ARCH="x86_64-unknown-linux-gnu" -# rustup-init uses those variables as install paths -export RUSTUP_HOME=/usr/local/rustup -export CARGO_HOME=/usr/local/cargo dnf install -y wget RUSTUP_URL=https://static.rust-lang.org/rustup/archive/$RUSTUP_VERSION/$RUST_ARCH/rustup-init wget $RUSTUP_URL From d175858badc59e44ffb55f0ad1b363c941ccc7e7 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Mon, 23 Dec 2019 14:24:17 +0530 Subject: [PATCH 353/605] gitlab: update Fedora image Include Rust environnement fix. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b34184ffcb..157d46438e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,7 +9,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-10-23-793480' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-11-10-917246' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-12-20-1211878' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-12-23-1224753' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v8' From d0fbcac3dfe7ce5ca8b9c655741d42ac824736ec Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 25 Dec 2019 15:08:59 +0530 Subject: [PATCH 354/605] gitlab/gst-build: Disable building rust plugins gst-build now includes gst-plugins-rs, but gst-plugins-rs CI does not run the gst-build CI jobs, so MRs can easily break gst-build CI. F.ex: https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/merge_requests/186#note_371696 Disable this till that's resolved. --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 157d46438e..983e7275b8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -135,7 +135,7 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" artifacts: expire_in: '5 days' when: always @@ -151,7 +151,7 @@ build nodebug fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" except: variables: - $CI_PROJECT_NAME == "cerbero" From e41650913e938f6f9ab47170d66aceb7418b467b Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 2 Jan 2020 11:59:07 +0530 Subject: [PATCH 355/605] gitlab: disable most jobs for gst-plugins-rs No need to run jobs which are not building gst-plugins-rs. --- gitlab/ci_template.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 983e7275b8..682af430d8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -157,6 +157,7 @@ build nodebug fedora x86_64: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" build static fedora x86_64: extends: 'build fedora x86_64' @@ -167,6 +168,7 @@ build static fedora x86_64: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" build static nodebug fedora x86_64: extends: 'build nodebug fedora x86_64' @@ -177,6 +179,7 @@ build static nodebug fedora x86_64: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" build clang fedora x86_64: extends: 'build fedora x86_64' @@ -188,6 +191,7 @@ build clang fedora x86_64: - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" .test: stage: 'test' @@ -251,6 +255,7 @@ check fedora: - $CI_PROJECT_NAME == "gst-examples" - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-plugins-rs" integration testsuites fedora: extends: '.test fedora x86_64' @@ -262,6 +267,7 @@ integration testsuites fedora: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "cerbero" - $CI_PROJECT_NAME == "gst-docs" + - $CI_PROJECT_NAME == "gst-plugins-rs" # Valgrind .valgrind fedora x86_64: @@ -457,6 +463,7 @@ valgrind ges: - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" build vs2017 amd64: extends: '.build windows' @@ -577,6 +584,7 @@ build cerbero fedora x86_64: - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" # # Cerbero Android Universal build @@ -612,6 +620,7 @@ build cerbero cross-android universal: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" # # Cerbero Cross Windows builds @@ -648,6 +657,7 @@ build cerbero cross win32: - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" cerbero deps cross-windows x86_64: extends: '.cerbero deps' @@ -681,6 +691,7 @@ build cerbero cross win64: - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-integration-testsuites" + - $CI_PROJECT_NAME == "gst-plugins-rs" # # Build an Android App using the android binaries @@ -740,6 +751,7 @@ cross-android universal examples: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-plugins-rs" cerbero cross-android universal examples: extends: ".cross-android universal examples" @@ -809,6 +821,7 @@ build cerbero macos x86_64: - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-sharp" + - $CI_PROJECT_NAME == "gst-plugins-rs" # # Cerbero iOS build @@ -865,6 +878,7 @@ build cerbero cross-ios universal: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gstreamer-vaapi" - $CI_PROJECT_NAME == "gst-sharp" + - $CI_PROJECT_NAME == "gst-plugins-rs" documentation: image: $FEDORA_IMAGE @@ -894,6 +908,7 @@ documentation: - $CI_PROJECT_NAME == "gst-python" - $CI_PROJECT_NAME == "gst-sharp" - $CI_PROJECT_NAME == "gst-build" + - $CI_PROJECT_NAME == "gst-plugins-rs" # # Build an iOS App using the iOS binaries @@ -951,6 +966,7 @@ cross-ios universal examples: - $CI_PROJECT_NAME == "gst-omx" - $CI_PROJECT_NAME == "gst-integration-testsuites" - $CI_PROJECT_NAME == "gstreamer-vaapi" + - $CI_PROJECT_NAME == "gst-plugins-rs" cerbero cross-ios universal examples: extends: ".cross-ios universal examples" From 6b09187d21cb53261847114c98cd330424dd4778 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 7 Jan 2020 02:37:55 +0530 Subject: [PATCH 356/605] docker/cerbero: Do not manually install nasm when building image This is pulled in correctly via cerbero's bootstrap deps now. --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 61f7b368c1..b8b2e3f7d5 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -3,7 +3,7 @@ FROM registry.fedoraproject.org/fedora:30 # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ dnf upgrade -y && \ - dnf install -y sudo git lbzip2 rsync nasm + dnf install -y sudo git lbzip2 rsync # Configure git for various usage RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ From eedde1884aee095ae3d4d28e77f1c3c8b427de55 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 7 Jan 2020 12:58:21 +0530 Subject: [PATCH 357/605] gitlab-ci.yml: Allow tagging images built on upstream branches If we're on the gstreamer namespace, we should push all images built on that namespace to the registry. This is needed to, f.ex., update the docker image used in stable builds. This is needed for: https://gitlab.freedesktop.org/gstreamer/cerbero/merge_requests/367 --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad22b328da..6484336cbc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,9 +47,9 @@ test manifest: # Push the images to the upstream registry - | - if [ "$CI_COMMIT_REF_NAME" = "master" -a "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then + if [ "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then # Tag the image with the git ref - docker image tag ${LATEST} ${_UID} + docker image tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} # Push the tags docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY docker push ${_UID} From f46a0d33c0b576c7119bf39e43eb4b291e14fd6c Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 19 Dec 2019 12:50:14 +0530 Subject: [PATCH 358/605] gitlab: build gst-omx with zynq as target Build just the minimum to be able to build gst-omx with this specific target. --- gitlab/ci_template.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 682af430d8..0b5a59becc 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -978,3 +978,13 @@ cerbero cross-ios universal examples: only: variables: - $CI_PROJECT_NAME == "cerbero" + +build gst-omx zynq fedora x86_64: + extends: 'build fedora x86_64' + variables: + MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + before_script: + - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2019.2 ${CI_PROJECT_DIR}/vcu-omx-il + only: + variables: + - $CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/ From 10326927d5fee2000cfb88745226bf13d47a9806 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 8 Jan 2020 23:44:56 +0530 Subject: [PATCH 359/605] gitlab-ci: Fix docker tag when pushing image Missed in https://gitlab.freedesktop.org/gstreamer/gst-ci/merge_requests/213 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6484336cbc..760b7a10a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,7 +52,7 @@ test manifest: docker image tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} # Push the tags docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker push ${_UID} + docker push ${_UID}-${CI_COMMIT_REF_NAME} docker push ${LATEST} fi From 5fc211d107f3fa87290959bb8d2f1e7db7ab6729 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 8 Jan 2020 16:21:09 +0530 Subject: [PATCH 360/605] gitlab: don't run gst-plugins-rs tests They are not working with CI atm, see #46 --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0b5a59becc..4666e82403 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -247,6 +247,7 @@ build clang fedora x86_64: check fedora: extends: '.test fedora x86_64' variables: + EXTRA_VALIDATE_ARGS: "-b check.gst-plugins-rs.*" TEST_SUITE: "check.gst*" except: variables: From 8e1b1c43c75c4ebb273f08b7136f145dcf6dc48a Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 2 Jan 2020 14:03:54 +0530 Subject: [PATCH 361/605] gitlab: enable gst-plugins-rs in build fedora jobs --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4666e82403..8735f54c6e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -135,7 +135,7 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=enabled ${MESON_BUILDTYPE_ARGS} --werror" artifacts: expire_in: '5 days' when: always @@ -151,7 +151,7 @@ build nodebug fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=enabled ${MESON_BUILDTYPE_ARGS} --werror" except: variables: - $CI_PROJECT_NAME == "cerbero" From b9c68041b2d4cfe5ae61fbdeea5ef758174439fb Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 15 Jan 2020 11:29:01 +0530 Subject: [PATCH 362/605] gitlab: cache CARGO_HOME in shared cache --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8735f54c6e..79486bae62 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -104,6 +104,7 @@ gst indent: CCACHE_DIR: "/cache/gstreamer/gst-build/ccache/" # shared across everything really CCACHE_MAXSIZE: "10G" + CARGO_HOME: "/cache/gstreamer/cargo" MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} --werror" @@ -203,6 +204,7 @@ build clang fedora x86_64: CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" TIMEOUT_FACTOR: "2" + CARGO_HOME: "/cache/gstreamer/cargo" script: - cd gst-build/ - echo "-> Running ${TEST_SUITE}" From 9e7b8f9392ff47cd289d27350585d241e3b74a79 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sat, 11 Jan 2020 23:09:04 -0500 Subject: [PATCH 363/605] CI: integration: Split this test in 4 jobs That job was the slowest, now each jobs takes about 12 minutes, which makes it slightly faster then msys2 jobs, and sometime iOS due to low bandwidth and low availibility of OSX runners. --- gitlab/ci_template.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 79486bae62..ee68c51930 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -262,8 +262,9 @@ check fedora: integration testsuites fedora: extends: '.test fedora x86_64' + parallel: 4 variables: - EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs" + EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs --parts=${CI_NODE_TOTAL} --part-index=${CI_NODE_INDEX}" TEST_SUITE: "validate ges" except: variables: From 7fb6c56784dd5d9f1118eeabd09dfde55ad25f93 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 15 Jan 2020 10:09:55 +0530 Subject: [PATCH 364/605] gitlab: move gst-plugins-rs to its own job Building Rust code is significantly slowing down ci so do it only for projects which could make gst-plugins-rs regress. --- gitlab/ci_template.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ee68c51930..3d79105222 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -136,7 +136,7 @@ build fedora x86_64: stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=enabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" artifacts: expire_in: '5 days' when: always @@ -146,13 +146,14 @@ build fedora x86_64: except: variables: - $CI_PROJECT_NAME == "cerbero" + - $CI_PROJECT_NAME == "gst-plugins-rs" build nodebug fedora x86_64: extends: '.build' stage: 'build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=enabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" except: variables: - $CI_PROJECT_NAME == "cerbero" @@ -249,7 +250,6 @@ build clang fedora x86_64: check fedora: extends: '.test fedora x86_64' variables: - EXTRA_VALIDATE_ARGS: "-b check.gst-plugins-rs.*" TEST_SUITE: "check.gst*" except: variables: @@ -992,3 +992,11 @@ build gst-omx zynq fedora x86_64: only: variables: - $CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/ + +build rust fedora x86_64: + extends: 'build fedora x86_64' + variables: + MESON_ARGS: "-Domx=disabled -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=enabled ${MESON_BUILDTYPE_ARGS} --werror" + only: + variables: + - $CI_PROJECT_NAME =~ /^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build)$/ From 44c286fffdb3033dafe3053d16681771bbafe7dd Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 15 Jan 2020 14:05:34 +0530 Subject: [PATCH 365/605] docker: fedora: install libsodium Used by a Rust plugin. --- docker/fedora/prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index eec0275e1d..0eb88237f7 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -53,6 +53,7 @@ dnf install -y \ json-glib-devel \ libnice \ libnice-devel \ + libsodium-devel \ libunwind \ libunwind-devel \ libyaml-devel \ From 44f421e9e546dea8e983cd524860cc3f342ac1e7 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Fri, 24 Jan 2020 11:26:03 +0530 Subject: [PATCH 366/605] gitlab: override 'except' on build rust job The parent job has an except rule on gst-plugins-rs which was inherited preventing the job to be run on gst-plugins-rs updates. Fix #51 --- gitlab/ci_template.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 3d79105222..c2c4dbe088 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1000,3 +1000,6 @@ build rust fedora x86_64: only: variables: - $CI_PROJECT_NAME =~ /^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build)$/ + except: + variables: + - $CI_PROJECT_NAME == "cerbero" From 8ea2989e0b0aa5db921e7988acb98ab527e69ef6 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 15 Jan 2020 10:33:29 +0530 Subject: [PATCH 367/605] gitlab: run gst-plugins-rs tests Fix #46 --- gitlab/ci_template.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c2c4dbe088..12b10f8a02 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -996,10 +996,22 @@ build gst-omx zynq fedora x86_64: build rust fedora x86_64: extends: 'build fedora x86_64' variables: - MESON_ARGS: "-Domx=disabled -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=enabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "-Domx=disabled -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=enabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=enabled ${MESON_BUILDTYPE_ARGS} --werror" only: variables: - - $CI_PROJECT_NAME =~ /^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build)$/ + - $CI_PROJECT_NAME =~ /^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/ except: variables: - $CI_PROJECT_NAME == "cerbero" + +check rust fedora: + extends: '.test fedora x86_64' + needs: + - 'build rust fedora x86_64' + dependencies: + - build rust fedora x86_64 + variables: + TEST_SUITE: "check.gst-plugins-rs.*" + only: + variables: + - $CI_PROJECT_NAME =~ /^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/ From a86020a9bff23bb016de19cab635c9e4fe9ab958 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 25 Dec 2019 11:08:08 +0100 Subject: [PATCH 368/605] docker/fedora: Install libdav1d This library is required for the gst-plugins-rs rsdav1d plugin build. --- docker/fedora/prepare.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 0eb88237f7..1d63005de9 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -22,6 +22,8 @@ dnf install -y \ libaom \ libaom-devel \ libcaca-devel \ + libdav1d \ + libdav1d-devel \ ccache \ cmake \ clang-devel \ @@ -122,6 +124,7 @@ dnf debuginfo-install -y gtk3 \ glib-networking \ libcurl \ libsoup \ + nasm \ nss \ nss-softokn \ nss-softokn-freebl \ From ddf9f8962fa590abca740e8bfc4ca22a7e9f1a3c Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 12 Dec 2019 10:19:53 +0530 Subject: [PATCH 369/605] docker/windows: Download Python installer as python3-installer Having a binary called 'python3' anywhere that might accidentally end up in PATH is asking for trouble. --- docker/windows/install_toolchain.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index 2d4260ec0c..5c50e1b195 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -12,9 +12,9 @@ Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --n Remove-Item C:\vs_buildtools.exe -Force Write-Host "Installing Python" -Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3.exe -Start-Process C:\python3.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait -Remove-Item C:\python3.exe -Force +Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3-installer.exe +Start-Process C:\python3-installer.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait +Remove-Item C:\python3-installer.exe -Force Write-Host "Installing Git" Invoke-WebRequest -Uri $git_url -OutFile C:\mingit.zip From cec8de9054395902d8f7f7ec90a3fc61ee1ba87b Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 12 Dec 2019 10:20:55 +0530 Subject: [PATCH 370/605] docker/windows: Install Python to a path without spaces This is needed for Cerbero since Autotools cannot handle paths to binaries that have spaces in them. --- docker/windows/install_toolchain.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index 5c50e1b195..72f3a07984 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -13,7 +13,7 @@ Remove-Item C:\vs_buildtools.exe -Force Write-Host "Installing Python" Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3-installer.exe -Start-Process C:\python3-installer.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait +Start-Process C:\python3-installer.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 TargetDir=C:\Python37' -Wait Remove-Item C:\python3-installer.exe -Force Write-Host "Installing Git" From b722709cad5d5832dfc67d0d0db0b639364d6cf7 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 12 Dec 2019 10:22:06 +0530 Subject: [PATCH 371/605] docker/windows: Use load-balancing for sourceforge URLs Instead of using a specific mirror, which may be slow or might disappear. --- docker/windows/install_toolchain.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index 72f3a07984..ccc4b29b6f 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -4,7 +4,7 @@ $python_dl_url = 'https://www.python.org/ftp/python/3.7.4/python-3.7.4.exe' $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' $git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.0.windows.2/MinGit-2.24.0.2-64-bit.zip' $zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe' -$msys_url = 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' +$msys_url = 'https://download.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' Write-Host "Installing VisualStudio" Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe From 1ae8f6491f2a3a8c94e79d6e2f180e33500e9506 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 12 Dec 2019 10:22:43 +0530 Subject: [PATCH 372/605] docker/windows: Update python, git, and MSYS2 The git update is particularly important because of security vulnerabilities on Windows related to NTFS. --- docker/windows/install_toolchain.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index ccc4b29b6f..8e6e9e6ab0 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -1,10 +1,10 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; -$python_dl_url = 'https://www.python.org/ftp/python/3.7.4/python-3.7.4.exe' +$python_dl_url = 'https://www.python.org/ftp/python/3.7.5/python-3.7.5.exe' $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' -$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.0.windows.2/MinGit-2.24.0.2-64-bit.zip' +$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.1.windows.2/MinGit-2.24.1.2-64-bit.zip' $zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe' -$msys_url = 'https://download.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' +$msys_url = 'https://download.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' Write-Host "Installing VisualStudio" Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe From cfb15f47b81cb8da35f430d0e1d88503e723d84b Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 13 Dec 2019 06:00:04 +0530 Subject: [PATCH 373/605] docker/windows: Clarify some MSYS2 install steps --- docker/windows/install_toolchain.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index 8e6e9e6ab0..67bf1ff9d9 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -4,7 +4,7 @@ $python_dl_url = 'https://www.python.org/ftp/python/3.7.5/python-3.7.5.exe' $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' $git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.1.windows.2/MinGit-2.24.1.2-64-bit.zip' $zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe' -$msys_url = 'https://download.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' +$msys2_url = 'https://download.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' Write-Host "Installing VisualStudio" Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe @@ -28,8 +28,8 @@ Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait Remove-Item C:\7z-x64.exe -Force -Write-Host "Installing MSYS2" -Invoke-WebRequest -Uri $msys_url -OutFile C:\msys2-x86_64.tar.xz +Write-Host "Installing MSYS2 into C:/msys64" +Invoke-WebRequest -Uri $msys2_url -OutFile C:\msys2-x86_64.tar.xz C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\" Remove-Item C:\msys2-x86_64.tar.xz -Force From 6267810f3a90de93a0cd80b0770ceb2e9aba4027 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 13 Dec 2019 06:21:36 +0530 Subject: [PATCH 374/605] docker/windows: Fix 7zip extract command for MSYS2 -Wait is only needed for Start-Process. Also, explicitly extract the tar file into C:\ -- don't assume the current directory is C:\ --- docker/windows/install_toolchain.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index 67bf1ff9d9..cebacce6bf 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -30,7 +30,7 @@ Remove-Item C:\7z-x64.exe -Force Write-Host "Installing MSYS2 into C:/msys64" Invoke-WebRequest -Uri $msys2_url -OutFile C:\msys2-x86_64.tar.xz -C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait +C:\7zip\7z e C:\msys2-x86_64.tar.xz -o"C:\\" C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\" Remove-Item C:\msys2-x86_64.tar.xz -Force Remove-Item C:\msys2-x86_64.tar -Force From d8e87dc89a8b5c196986aabf0d041a6d0186c7ed Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 13 Dec 2019 06:23:17 +0530 Subject: [PATCH 375/605] docker/windows: Install CMake for Cerbero --- docker/windows/install_toolchain.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index cebacce6bf..f1ddb8398f 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -4,6 +4,7 @@ $python_dl_url = 'https://www.python.org/ftp/python/3.7.5/python-3.7.5.exe' $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' $git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.1.windows.2/MinGit-2.24.1.2-64-bit.zip' $zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe' +$cmake_url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.msi' $msys2_url = 'https://download.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' Write-Host "Installing VisualStudio" @@ -23,6 +24,11 @@ Remove-Item C:\mingit.zip -Force $env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd' [Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine) +Write-Host "Installing CMake for Cerbero" +Invoke-WebRequest -Uri $cmake_url -OutFile C:\cmake-x64.msi +Start-Process msiexec -ArgumentList '-i C:\cmake-x64.msi -quiet -norestart ADD_CMAKE_TO_PATH=System' -Wait +Remove-Item C:\cmake-x64.msi -Force + Write-Host "Installing 7zip" Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait From 21df205069a73103b5061569ea4b87d600f448c1 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 13 Dec 2019 06:23:42 +0530 Subject: [PATCH 376/605] docker/windows: Download and install MSYS for Cerbero --- docker/windows/install_toolchain.ps1 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index f1ddb8398f..e9e00ae8ba 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -6,6 +6,7 @@ $git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.1.win $zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe' $cmake_url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.msi' $msys2_url = 'https://download.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' +$msys_mingw_get_url = 'https://osdn.net/projects/mingw/downloads/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' Write-Host "Installing VisualStudio" Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe @@ -34,6 +35,16 @@ Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait Remove-Item C:\7z-x64.exe -Force +Write-Host "Downloading and extracting mingw-get for MSYS" +Invoke-WebRequest -Uri $msys_mingw_get_url -OutFile C:\mingw-get.tar.xz +C:\7zip\7z e C:\mingw-get.tar.xz -o"C:\\" +C:\7zip\7z x C:\mingw-get.tar -o"C:\\MinGW" +Remove-Item C:\mingw-get.tar.xz -Force +Remove-Item C:\mingw-get.tar -Force + +Write-Host "Installing MSYS for Cerbero into C:/MinGW using mingw-get" +Start-Process C:\MinGW\bin\mingw-get.exe -ArgumentList 'install msys-base mingw32-base mingw-developer-toolkit' -Wait + Write-Host "Installing MSYS2 into C:/msys64" Invoke-WebRequest -Uri $msys2_url -OutFile C:\msys2-x86_64.tar.xz C:\7zip\7z e C:\msys2-x86_64.tar.xz -o"C:\\" From c0caae072298992b768a4606503423b0a5ea765f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 13 Dec 2019 12:34:38 +0000 Subject: [PATCH 377/605] docker/windows: use direct download links from mirrors Else Invoke-WebRequest gets redirected and ends up downloading html page. Possibly due to an unkown user-agent? who knows. --- docker/windows/install_toolchain.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index e9e00ae8ba..e1150c6da2 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -5,8 +5,8 @@ $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' $git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.1.windows.2/MinGit-2.24.1.2-64-bit.zip' $zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe' $cmake_url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.msi' -$msys2_url = 'https://download.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' -$msys_mingw_get_url = 'https://osdn.net/projects/mingw/downloads/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' +$msys2_url = 'https://netcologne.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' +$msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' Write-Host "Installing VisualStudio" Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe From df6ebdeaaf04d449fa82ac23b90a62e3ff7ccb26 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 21 Jan 2020 15:27:19 +0200 Subject: [PATCH 378/605] docker/windows: install git, gitlfs, 7zip and python3 with choco Move them into a seperate script to avoid needing to refresh the env, and make it simpler. Adapted from patches by Xavier Claessens! --- docker/windows/Dockerfile | 4 ++- docker/windows/install_choco.ps1 | 4 +++ docker/windows/install_toolchain.ps1 | 37 +++------------------------- 3 files changed, 11 insertions(+), 34 deletions(-) create mode 100644 docker/windows/install_choco.ps1 diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 913b19732b..5d8d101fc3 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -6,8 +6,10 @@ FROM 'mcr.microsoft.com/windows/servercore:1809' SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] ENV ErrorActionPreference='Stop' -COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\ +COPY install_choco.ps1 C:\ +RUN C:\install_choco.ps1 +COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\ RUN C:\install_toolchain.ps1 RUN C:\prepare_gst_env.ps1 diff --git a/docker/windows/install_choco.ps1 b/docker/windows/install_choco.ps1 new file mode 100644 index 0000000000..8ab6474043 --- /dev/null +++ b/docker/windows/install_choco.ps1 @@ -0,0 +1,4 @@ +Write-Host "Installing Choco" +Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) +Write-Host "Installing Choco packages" +choco install -y python3 git git-lfs 7zip diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index e1150c6da2..ed3fff394e 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -1,9 +1,6 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; -$python_dl_url = 'https://www.python.org/ftp/python/3.7.5/python-3.7.5.exe' $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' -$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.1.windows.2/MinGit-2.24.1.2-64-bit.zip' -$zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe' $cmake_url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.msi' $msys2_url = 'https://netcologne.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' $msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' @@ -13,32 +10,15 @@ Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait Remove-Item C:\vs_buildtools.exe -Force -Write-Host "Installing Python" -Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3-installer.exe -Start-Process C:\python3-installer.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 TargetDir=C:\Python37' -Wait -Remove-Item C:\python3-installer.exe -Force - -Write-Host "Installing Git" -Invoke-WebRequest -Uri $git_url -OutFile C:\mingit.zip -Expand-Archive C:\mingit.zip -DestinationPath c:\mingit -Remove-Item C:\mingit.zip -Force -$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd' -[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine) - Write-Host "Installing CMake for Cerbero" Invoke-WebRequest -Uri $cmake_url -OutFile C:\cmake-x64.msi Start-Process msiexec -ArgumentList '-i C:\cmake-x64.msi -quiet -norestart ADD_CMAKE_TO_PATH=System' -Wait Remove-Item C:\cmake-x64.msi -Force -Write-Host "Installing 7zip" -Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe -Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait -Remove-Item C:\7z-x64.exe -Force - Write-Host "Downloading and extracting mingw-get for MSYS" Invoke-WebRequest -Uri $msys_mingw_get_url -OutFile C:\mingw-get.tar.xz -C:\7zip\7z e C:\mingw-get.tar.xz -o"C:\\" -C:\7zip\7z x C:\mingw-get.tar -o"C:\\MinGW" +7z e C:\mingw-get.tar.xz -o"C:\\" +7z x C:\mingw-get.tar -o"C:\\MinGW" Remove-Item C:\mingw-get.tar.xz -Force Remove-Item C:\mingw-get.tar -Force @@ -47,19 +27,10 @@ Start-Process C:\MinGW\bin\mingw-get.exe -ArgumentList 'install msys-base mingw3 Write-Host "Installing MSYS2 into C:/msys64" Invoke-WebRequest -Uri $msys2_url -OutFile C:\msys2-x86_64.tar.xz -C:\7zip\7z e C:\msys2-x86_64.tar.xz -o"C:\\" -C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\" +7z e C:\msys2-x86_64.tar.xz -o"C:\\" +7z x C:\msys2-x86_64.tar -o"C:\\" Remove-Item C:\msys2-x86_64.tar.xz -Force Remove-Item C:\msys2-x86_64.tar -Force -Remove-Item C:\7zip -Recurse -Force - -Write-Host "Installing Choco" -iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) -refreshenv - -Write-Host "Installing git-lfs" -choco install -y git-lfs -refreshenv Write-Host "Installing Meson" pip install meson From 6fb7b6744c5055fa781cb8656df74b7fd0aa01f1 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 21 Jan 2020 15:29:01 +0200 Subject: [PATCH 379/605] docker/windows: copy the install scripts just before running them This results into more docker layers, but it avoid invalidating the previous layers when changing anything in the script, making build times faster. Adapted from patches by Xavier Claessens! --- docker/windows/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 5d8d101fc3..580ab2e239 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -9,7 +9,8 @@ ENV ErrorActionPreference='Stop' COPY install_choco.ps1 C:\ RUN C:\install_choco.ps1 -COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\ +COPY install_toolchain.ps1 C:\ RUN C:\install_toolchain.ps1 +COPY prepare_gst_env.ps1 C:\ RUN C:\prepare_gst_env.ps1 From 0320050e063004de972ba88de9d3b2bd5c9138f9 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 21 Jan 2020 20:22:56 +0200 Subject: [PATCH 380/605] docker/windows: install cmake with choco as well --- docker/windows/install_choco.ps1 | 3 ++- docker/windows/install_toolchain.ps1 | 6 ------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docker/windows/install_choco.ps1 b/docker/windows/install_choco.ps1 index 8ab6474043..f1a7c380a3 100644 --- a/docker/windows/install_choco.ps1 +++ b/docker/windows/install_choco.ps1 @@ -1,4 +1,5 @@ Write-Host "Installing Choco" Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) Write-Host "Installing Choco packages" -choco install -y python3 git git-lfs 7zip +choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' +choco install -y python3 git git-lfs 7zip \ No newline at end of file diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index ed3fff394e..66a4df73b0 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -1,7 +1,6 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' -$cmake_url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.msi' $msys2_url = 'https://netcologne.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' $msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' @@ -10,11 +9,6 @@ Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait Remove-Item C:\vs_buildtools.exe -Force -Write-Host "Installing CMake for Cerbero" -Invoke-WebRequest -Uri $cmake_url -OutFile C:\cmake-x64.msi -Start-Process msiexec -ArgumentList '-i C:\cmake-x64.msi -quiet -norestart ADD_CMAKE_TO_PATH=System' -Wait -Remove-Item C:\cmake-x64.msi -Force - Write-Host "Downloading and extracting mingw-get for MSYS" Invoke-WebRequest -Uri $msys_mingw_get_url -OutFile C:\mingw-get.tar.xz 7z e C:\mingw-get.tar.xz -o"C:\\" From 23f303eb9407f5810080b750483f3dc9e3f46548 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 22 Jan 2020 13:59:44 +0200 Subject: [PATCH 381/605] docker/windows: work around a python ssl issue Python fails to validate github.com SSL certificate, unless we first run a dummy download to force refreshing Windows' CA database. See: https://bugs.python.org/issue36137 Adapted from patches by Xavier Claessens! --- docker/windows/prepare_gst_env.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1 index 833f7dceac..a930c5dc19 100644 --- a/docker/windows/prepare_gst_env.ps1 +++ b/docker/windows/prepare_gst_env.ps1 @@ -3,6 +3,11 @@ git config --global user.email "gst-build@gstreamer.net" git config --global user.name "Gstbuild Runner" +# FIXME: Python fails to validate github.com SSL certificate, unless we first +# run a dummy download to force refreshing Windows' CA database. +# See: https://bugs.python.org/issue36137 +(New-Object System.Net.WebClient).DownloadString("https://github.com") >$null + # Download gst-build and all its subprojects git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build From ad6f4df17f8b4df7a2f6f0081ca4cada012e094e Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 22 Jan 2020 17:06:08 +0200 Subject: [PATCH 382/605] ci_template.yml: Update the windows image version --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 12b10f8a02..cfe81052b0 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -12,7 +12,7 @@ variables: FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-12-23-1224753' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v8' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v9' # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'master' From 08b346d96c1c44e1b21d7c13cb34f8982230ac89 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 30 Jan 2020 14:59:57 +0530 Subject: [PATCH 383/605] docker: fedora: install Tizonia dep Need to build gst-omx with Tizonia as target (#20). --- docker/fedora/prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 1d63005de9..a071ee4029 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -62,6 +62,7 @@ dnf install -y \ libxml2-devel \ libxslt-devel \ llvm-devel \ + log4c-devel \ make \ neon \ neon-devel \ From cb22d4c4f9b16e32b236f8ef7a7e25d2ea91fbc3 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 4 Feb 2020 00:16:09 +0530 Subject: [PATCH 384/605] docker/cerbero: Install wine in system setup This is now an optional runtime dependency that cerbero can make use of for doing runtime testing of the built binaries. Needed for building a new image so that !227 can be merged. --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index b8b2e3f7d5..ae5c17682f 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -3,7 +3,7 @@ FROM registry.fedoraproject.org/fedora:30 # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ dnf upgrade -y && \ - dnf install -y sudo git lbzip2 rsync + dnf install -y sudo git lbzip2 rsync wine # Configure git for various usage RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ From eeaa92fac5af122b8ed668646ef54a5c248bb21c Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 3 Feb 2020 02:32:53 +0530 Subject: [PATCH 385/605] gitlab/cerbero: Run gst-inspect on the built binaries Using the latest cerbero image to get Wine for cross-win32|64. --- gitlab/ci_template.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index cfe81052b0..8adaac40c5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,7 +8,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-10-23-793480' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2019-11-10-917246' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-02-03-1534422-master' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-12-23-1224753' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' @@ -368,6 +368,8 @@ valgrind ges: CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS}" CERBERO: "./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml" CERBERO_PACKAGE_ARGS: "-t" + CERBERO_RUN_WRAPPER: "" # 'wine' on cross-winXX + CERBERO_RUN_SUFFIX: "" # '.exe' on cross-winXX HAVE_CCACHE: "yes" # used by macos packages as we only ever install to a fixed directory CERBERO_OVERRIDDEN_DIST_DIR: "" @@ -398,6 +400,11 @@ valgrind ges: && rsync -aH ${CERBERO_HOME}/dist/${ARCH}/ ${CERBERO_OVERRIDDEN_DIST_DIR} - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o $(pwd) gstreamer-1.0 + # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. + - |- + [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version + - |- + [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test cache: key: "${CI_JOB_NAME}" @@ -639,6 +646,8 @@ cerbero deps cross-windows x86: extends: '.cerbero' variables: CONFIG: "cross-win32.cbc" + CERBERO_RUN_WRAPPER: "wine" + CERBERO_RUN_SUFFIX: ".exe" cerbero cross win32: extends: '.cerbero cross win32' @@ -673,6 +682,8 @@ cerbero deps cross-windows x86_64: extends: '.cerbero' variables: CONFIG: "cross-win64.cbc" + CERBERO_RUN_WRAPPER: "wine" + CERBERO_RUN_SUFFIX: ".exe" cerbero cross win64: extends: '.cerbero cross win64' From 2f40cb1418eb6507c77e950551de88b1cbc93df3 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Fri, 31 Jan 2020 20:32:59 +0530 Subject: [PATCH 386/605] gitlab: update fedora image --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8adaac40c5..d48836cfb5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,7 +9,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-10-23-793480' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-02-03-1534422-master' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-12-23-1224753' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-01-31-1518299-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v9' From 3aa542dfb8539daff706d2f9b19627e88a2f19d5 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 22 Jan 2020 15:22:41 +0530 Subject: [PATCH 387/605] gitlab: use libsodium from system The Rust crate can either build its own libsodium or use the one from the system. Do the latter to reduce build time. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d48836cfb5..e7db06d187 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1007,7 +1007,7 @@ build gst-omx zynq fedora x86_64: build rust fedora x86_64: extends: 'build fedora x86_64' variables: - MESON_ARGS: "-Domx=disabled -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=enabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=enabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "-Domx=disabled -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=enabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=enabled -Dgst-plugins-rs:sodium=system ${MESON_BUILDTYPE_ARGS} --werror" only: variables: - $CI_PROJECT_NAME =~ /^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/ From def2e2d80bbcd7c92f5d909026372bdfb7a200fc Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 29 Jan 2020 16:35:20 +0200 Subject: [PATCH 388/605] docker/fedora: add libappstream-glib to the image Needed to validate the appstream files of GstDebugViewer in gst-devtools. https://gitlab.freedesktop.org/gstreamer/gst-devtools/merge_requests/155 --- docker/fedora/prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index a071ee4029..793cc25491 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -121,6 +121,7 @@ dnf debuginfo-install -y gtk3 \ python3 \ python3-libs \ python3-gobject \ + libappstream-glib-devel \ libjpeg-turbo \ glib-networking \ libcurl \ From 65c18304c4518cf39249b44ff2176f8f7de37fce Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 29 Jan 2020 16:43:34 +0200 Subject: [PATCH 389/605] docker/fedora: upgrade base images to f31 All the images except the gst-build fedora image, due to further changes being needed --- docker/android/Dockerfile | 2 +- docker/cerbero/Dockerfile-fedora | 4 +++- docker/test_manifest/Dockerfile | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/android/Dockerfile b/docker/android/Dockerfile index b95dd87dea..05a7ab49ae 100644 --- a/docker/android/Dockerfile +++ b/docker/android/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:29 +FROM registry.fedoraproject.org/fedora:31 ENV ANDROID_HOME=/android/sdk ENV ANDROID_NDK_HOME=/android/ndk diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index ae5c17682f..b3d22d9200 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -1,4 +1,6 @@ -FROM registry.fedoraproject.org/fedora:30 +FROM registry.fedoraproject.org/fedora:31 + +ENV TERM="dumb" # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ diff --git a/docker/test_manifest/Dockerfile b/docker/test_manifest/Dockerfile index efadb5af83..cdc369c822 100644 --- a/docker/test_manifest/Dockerfile +++ b/docker/test_manifest/Dockerfile @@ -1,8 +1,8 @@ -FROM registry.fedoraproject.org/fedora:29 +FROM registry.fedoraproject.org/fedora:31 RUN dnf install -y \ python3-pytest \ python3-requests \ python3-pytest-cov \ git-core \ - && rm -rf /var/cache/dnf /var/log/dnf* \ No newline at end of file + && rm -rf /var/cache/dnf /var/log/dnf* From f7a83c65ab223da016913979b32317dbfcf31da0 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 10 Feb 2020 13:53:27 +0200 Subject: [PATCH 390/605] ci_template: update the docker image versions --- gitlab/ci_template.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e7db06d187..7cff5838cf 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -7,11 +7,11 @@ stages: - 'integrate' variables: - ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-10-23-793480' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-02-03-1534422-master' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-01-31-1518299-master' + ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-02-10-1589547-master' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-02-10-1589546-master' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-02-10-1589545-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' - MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2019-10-23-793475' + MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v9' # Branch to track for modules that have no ref specified in the manifest From 316848d43fb4ce55a9a34e982cce9d5941f28eee Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Mon, 24 Feb 2020 09:50:47 +0530 Subject: [PATCH 391/605] docker: fedora: update to meson 0.53.1 Needed to build Tizonia (gst-omx backend). --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 793cc25491..5b79d7ac2f 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -171,7 +171,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.52.1 hotdoc +pip3 install meson==0.53.1 hotdoc # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From ad25b174a4aff0168be6ca4f4db997734821badc Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Mon, 24 Feb 2020 12:01:53 +0530 Subject: [PATCH 392/605] gitlab: update fedora image Update to meson 0.53.1 --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7cff5838cf..039752c10f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,7 +9,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-02-10-1589547-master' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-02-10-1589546-master' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-02-10-1589545-master' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-02-24-1712085-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v9' From 28633f2fcfb34fe62f572d935d8902a7c8255c55 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 25 Feb 2020 14:01:45 +0200 Subject: [PATCH 393/605] windows: try to avoid random failures due to pre-existing gst-build clone This is slightly weird, cause I am not sure what causes the clone to be there, since gitlab-runner supposedly always either use a clean volume or at least runs git clean on the existing ones. But its there and so we have to deal with failures like so https://gitlab.freedesktop.org/tpm/gstreamer-sharp/-/jobs/1672137 --- gitlab/ci_template.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 039752c10f..f30fd6208c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -441,6 +441,12 @@ valgrind ges: -Dvaapi=disabled -Ddevtools=disabled script: + # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or + # from a previous job due to some gitlab bug or implicit behavior? + # So let's always check and clear it out if its there + # https://gitlab.freedesktop.org/tpm/gstreamer-sharp/-/jobs/1672137 + - if (Test-Path $env:CI_PROJECT_DIR/gst-build) { Remove-Item -Recurse -Force $env:CI_PROJECT_DIR/gst-build } + # Make sure powershell exists on errors # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 - $ErrorActionPreference = "Stop" @@ -490,6 +496,12 @@ build msys2 : extends: '.build windows' timeout: '60min' script: + # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or + # from a previous job due to some gitlab bug or implicit behavior? + # So let's always check and clear it out if its there + # https://gitlab.freedesktop.org/tpm/gstreamer-sharp/-/jobs/1672137 + - if (Test-Path $env:CI_PROJECT_DIR/gst-build) { Remove-Item -Recurse -Force $env:CI_PROJECT_DIR/gst-build } + # Make sure powershell exists on errors # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 - $ErrorActionPreference = "Stop" From a73a597f12cf31f14dbef79e303dcfe4075cfa9d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 20 Feb 2020 18:27:47 -0300 Subject: [PATCH 394/605] gitlab: Run ges valgrind tests on devtools changes We have validate 'unit test' there --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f30fd6208c..32d79a839e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -332,7 +332,7 @@ valgrind ges: TEST_SUITE: "check.gst-editing-services\\..*" only: variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci)$/ + - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci|gst-devtools)$/ # Template for Cerbero GStreamer Build # From c7466e2edcae17142fb09b78ca6622b0a92af495 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 26 Feb 2020 14:03:40 +0200 Subject: [PATCH 395/605] windows: checkout the correct branch of gst-build for stable branch Previously we would always checkout from master, which doesn't end up working well for the stable branch. We already have a mechanism to specify the correct branch to clone from each template. [1] [1] https://gitlab.freedesktop.org/gstreamer/gst-ci/-/merge_requests/118 --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 32d79a839e..79d2addbfd 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -451,7 +451,7 @@ valgrind ges: # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 - $ErrorActionPreference = "Stop" - - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build + - git clone -b $GST_UPSTREAM_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR\gst-build - cd $env:CI_PROJECT_DIR/gst-build - cp -r C:/subprojects/* subprojects/ # Run the git-update script and feed it the manifest to setup the environment @@ -512,7 +512,7 @@ build msys2 : - C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" - - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR\gst-build + - git clone -b $GST_UPSTREAM_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR\gst-build - cd $env:CI_PROJECT_DIR\gst-build - cp -r C:\subprojects\* subprojects\ From fbd1eb9283754e5ec72306ea49bd8d13d812f11f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 4 Mar 2020 11:46:50 +1100 Subject: [PATCH 396/605] cerbero-android: don't upload the '-runtime' package It's useless for android --- gitlab/ci_template.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 79d2addbfd..30c2b061c1 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -622,6 +622,14 @@ cerbero deps cross-android universal: extends: '.cerbero' variables: CONFIG: "cross-android-universal.cbc" + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "*[0-9].tar.bz2" cerbero cross-android universal: extends: '.cerbero cross-android universal' From 7d31018ce981420cf7f9d771e2ebc8f4999b43cd Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 4 Mar 2020 05:37:12 +0530 Subject: [PATCH 397/605] gitlab/cerbero: Do not add tmp files to cerbero-deps tarball This is where the WINEPREFIX is now in Cerbero. This used to be share/wine, but was moved to var/tmp/wine for clarity. It was causing two problems: 1. The size of these generated files are ~1GB, which were ~500MB after tar.gz, and they were completely useless since they can just be regenerated by Wine the next time it's run. Let's not waste egress bandwidth. 2. Random build failures because wineserver and associated processes would not always exit before we started tarring up the prefix, then write to the directory on exit while tar was reading the directory causing `tar -czf` to fail: ``` $ tar -C ${CERBERO_HOME} -czf $CERBERO_DEPS build-tools build-tools.cache dist/${ARCH} ${ARCH}.cache tar: build-tools/share/wine: file changed as we read it Uploading artifacts... manifest.xml: found 1 matching files cerbero-build/logs: found 461 matching files cerbero-build/cerbero-deps.log: found 1 matching files cerbero-deps.tar.gz: found 1 matching files Uploading artifacts to coordinator... ok id=1807197 responseStatus=201 Created token=4_qFUP8z ERROR: Job failed: exit code 1 ``` --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 30c2b061c1..daea1e640d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -556,7 +556,7 @@ build msys2 : - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && mkdir -p ${CERBERO_HOME}/dist/${ARCH} && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH} - - tar -C ${CERBERO_HOME} -czf $CERBERO_DEPS + - tar -C ${CERBERO_HOME} --exclude=var/tmp -czf $CERBERO_DEPS build-tools build-tools.cache dist/${ARCH} ${ARCH}.cache only: From 28881dddb443fc37001645cb630b7a86b22e0d29 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Mon, 16 Dec 2019 16:04:53 -0500 Subject: [PATCH 398/605] Add integration test for gstreamer-full-1.0 --- gitlab/ci_template.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index daea1e640d..86bda2b07a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -273,6 +273,27 @@ integration testsuites fedora: - $CI_PROJECT_NAME == "gst-docs" - $CI_PROJECT_NAME == "gst-plugins-rs" +gstreamer-full: + extends: 'build static fedora x86_64' + stage: integrate + variables: + MESON_BUILDTYPE_ARGS: > + --default-library=static + -Dintrospection=disabled + -Ddoc=disabled + -Dgtk_doc=disabled + -Dgst-plugins-rs=disabled + -Dpython=disabled + after_script: + - cd gst-build/ + - ninja -C build install + - export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib64/gstreamer-1.0/pkgconfig + - export LD_LIBRARY_PATH=/usr/local/lib64 + - cd examples/static-plugins + - meson _build + - ninja -C _build + - meson test -C _build -v + # Valgrind .valgrind fedora x86_64: extends: '.test fedora x86_64' From 3608d69c1fce1fbb3f591ba29b5ba311d9fcd29e Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 2 Mar 2020 14:39:43 +0530 Subject: [PATCH 399/605] gitlab/cerbero: Use new prefix for jobs built with MinGW --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 86bda2b07a..191063362a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -681,7 +681,7 @@ cerbero deps cross-windows x86: extends: '.cerbero deps' variables: CONFIG: "cross-win32.cbc" - ARCH: "windows_x86" + ARCH: "mingw_x86" .cerbero cross win32: extends: '.cerbero' @@ -717,7 +717,7 @@ cerbero deps cross-windows x86_64: extends: '.cerbero deps' variables: CONFIG: "cross-win64.cbc" - ARCH: "windows_x86_64" + ARCH: "mingw_x86_64" .cerbero cross win64: extends: '.cerbero' From a99cdc890911a54285206a04ea5f36e56fa3f775 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 10 Feb 2020 15:09:49 +0200 Subject: [PATCH 400/605] docker: allow to specify the branch to pull for various repos Declare an docker build-arg [1] and use it whenever cloning one of our repositories. If the buildarg is not specified, the variable defaults back to 'master' and thus the current behavior doesn't change. From the .gitlab-ci.yml file, when building pass the GST_UPSTREAM_BRANCH that's defined from the ci_template as the buildarg so we will be building the corresponding branches for the docker images. Close #33 [1] https://docs.docker.com/engine/reference/builder/#arg --- .gitlab-ci.yml | 6 +++--- docker/android/Dockerfile | 2 ++ docker/android/prepare.sh | 4 ++-- docker/build_manifest/Dockerfile | 4 +++- docker/cerbero/Dockerfile-fedora | 4 +++- docker/fedora/Dockerfile | 2 ++ docker/fedora/prepare.sh | 2 +- 7 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 760b7a10a1..b55554ed8d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,5 @@ +include: "gitlab/ci_template.yml" + stages: - "build docker" - "preparation" @@ -36,7 +38,7 @@ test manifest: - export _UID="${IMAGE}:${DATE}-${CI_JOB_ID}" - export LATEST="${IMAGE}:latest" - - docker build --pull -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR} + - docker build --pull --build-arg DEFAULT_BRANCH=${GST_UPSTREAM_BRANCH} -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR} # If we are in a fork, push the image to the reigstry regardless the branch - | @@ -110,8 +112,6 @@ android docker: DOCKERFILE: "docker/android/Dockerfile" extends: .base -include: "gitlab/ci_template.yml" - .build local: extends: '.build' when: 'manual' diff --git a/docker/android/Dockerfile b/docker/android/Dockerfile index 05a7ab49ae..f874ff01d2 100644 --- a/docker/android/Dockerfile +++ b/docker/android/Dockerfile @@ -3,6 +3,8 @@ FROM registry.fedoraproject.org/fedora:31 ENV ANDROID_HOME=/android/sdk ENV ANDROID_NDK_HOME=/android/ndk +ARG DEFAULT_BRANCH="master" + COPY prepare.sh cleanup.sh /root/ RUN /usr/bin/sh /root/prepare.sh && \ diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh index 32843922be..4b01162692 100644 --- a/docker/android/prepare.sh +++ b/docker/android/prepare.sh @@ -37,11 +37,11 @@ mkdir $GSTREAMER_ROOT_ANDROID tar -xvf /android/sources/gstreamer-android.tar.xz -C $GSTREAMER_ROOT_ANDROID ls $GSTREAMER_ROOT_ANDROID -git clone https://gitlab.freedesktop.org/gstreamer/gst-examples.git /android/sources/gst-examples +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-examples.git /android/sources/gst-examples chmod +x /android/sources/gst-examples/playback/player/android/gradlew /android/sources/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies -git clone https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs chmod +x /android/sources/gst-docs/examples/tutorials/android/gradlew /android/sources/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies diff --git a/docker/build_manifest/Dockerfile b/docker/build_manifest/Dockerfile index a0b85bb35e..e3c4921e6f 100644 --- a/docker/build_manifest/Dockerfile +++ b/docker/build_manifest/Dockerfile @@ -1,5 +1,7 @@ FROM python:3.7.1-alpine +ARG DEFAULT_BRANCH="master" + RUN pip install requests RUN apk add git -RUN cd / && git clone https://gitlab.freedesktop.org/gstreamer/gst-ci.git \ No newline at end of file +RUN cd / && git clone -b ${DEFAULT_BRANCH:=master} https://gitlab.freedesktop.org/gstreamer/gst-ci.git diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index b3d22d9200..2347399ebc 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -2,6 +2,8 @@ FROM registry.fedoraproject.org/fedora:31 ENV TERM="dumb" +ARG DEFAULT_BRANCH="master" + # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ dnf upgrade -y && \ @@ -11,7 +13,7 @@ RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ git config --global user.name "Cerbero Build System" -RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ +RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ mkdir $HOME/.cerbero && \ echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \ echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \ diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index 56f9279c44..82abdd3a40 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -4,6 +4,8 @@ ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ PATH=/usr/local/cargo/bin:$PATH +ARG DEFAULT_BRANCH="master" + COPY prepare.sh cleanup.sh /root/ RUN /usr/bin/sh /root/prepare.sh && \ diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 5b79d7ac2f..e52d8ccc3d 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -223,6 +223,6 @@ cargo --version rustc --version # get gst-build and make all subprojects available -git clone git://anongit.freedesktop.org/gstreamer/gst-build /gst-build/ +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-build.git /gst-build/ cd /gst-build meson subprojects download From 604928a40bc1783f82f45cf4c51146d00b94e16c Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 19 Dec 2019 12:50:14 +0530 Subject: [PATCH 401/605] gitlab: build gst-omx with tizonia as target Build just the minimum to be able to build gst-omx with this specific target. --- gitlab/ci_template.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 191063362a..726919aff2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1045,6 +1045,22 @@ build gst-omx zynq fedora x86_64: variables: - $CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/ +build gst-omx tizonia fedora x86_64: + extends: 'build fedora x86_64' + variables: + MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}:${CI_PROJECT_DIR}/tizonia-install/lib64/pkgconfig/ + before_script: + - git clone https://github.com/tizonia/tizonia-openmax-il --branch v0.20.2 + - cd tizonia-openmax-il + - meson build -Dclients=false -Dplugins='' -Dplayer=false -Dprefix=${CI_PROJECT_DIR}/tizonia-install + - ninja -C build + - ninja -C build install + - cd .. + only: + variables: + - $CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/ + build rust fedora x86_64: extends: 'build fedora x86_64' variables: From d8c5432793bd6c902fcc77562d7e5c3dd283f103 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 9 Mar 2020 15:27:25 +0200 Subject: [PATCH 402/605] ci_template: extend the scenarios where retries are attempted The windows runner has become a bit unstable lately, might be due to some recent update. It frequently timeouts while waiting to pick up a job or sometimes it goes missing in the middle of a job. --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 726919aff2..c9cefb4381 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -40,6 +40,8 @@ variables: max: 2 when: - 'runner_system_failure' + - 'stuck_or_timeout_failure' + - 'scheduler_failure' - 'api_failure' manifest: From b0eaeb69f3b8cedcdd1cd3457ac03933beaae8df Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Mon, 16 Mar 2020 15:48:41 -0400 Subject: [PATCH 403/605] gst-full: Only build 1 plugin and disable all artifacts This is enough to validate that it works for now. Fixes: #60 --- gitlab/ci_template.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c9cefb4381..0225cd3f60 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -279,13 +279,10 @@ gstreamer-full: extends: 'build static fedora x86_64' stage: integrate variables: - MESON_BUILDTYPE_ARGS: > + MESON_ARGS: > --default-library=static - -Dintrospection=disabled - -Ddoc=disabled - -Dgtk_doc=disabled - -Dgst-plugins-rs=disabled - -Dpython=disabled + -Dauto_features=disabled + -Dgst-plugins-good:alpha=enabled after_script: - cd gst-build/ - ninja -C build install @@ -295,6 +292,10 @@ gstreamer-full: - meson _build - ninja -C _build - meson test -C _build -v + artifacts: + paths: + - 'gst-build/build/meson-logs/' + - 'gst-build/build/gstinitstaticplugins.c' # Valgrind .valgrind fedora x86_64: From e12ea82a23ce82a62de7e0268e0076a790663ff3 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 4 Mar 2020 09:58:38 -0500 Subject: [PATCH 404/605] Port from only/except to rules Rules is a new feature that replaces only/except and allow for finer grain control on the workflow. With rules, we gain finer grain to pipeline and merge request pipelines. --- .gitlab-ci.yml | 33 +--- gitlab/ci_template.yml | 347 ++++++++++------------------------------- 2 files changed, 88 insertions(+), 292 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b55554ed8d..8ded44087e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -114,7 +114,10 @@ android docker: .build local: extends: '.build' - when: 'manual' + rules: + - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_PATH == "gstreamer/gst-ci"' + when: 'never' + - when: 'manual' needs: - 'manifest' @@ -129,51 +132,28 @@ build fedora x86_64 local: paths: - "manifest.xml" - "gst-build/" - except: - refs: - - "master@gstreamer/gst-ci" build cerbero fedora x86_64 local: extends: '.cerbero fedora x86_64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" - when: 'manual' - except: - refs: - - "master@gstreamer/gst-ci" build cerbero cross-android universal local: extends: '.cerbero cross-android universal' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" - when: 'manual' - except: - refs: - - "master@gstreamer/gst-ci" build cerbero cross win32 local: extends: '.cerbero cross win32' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" - when: 'manual' - except: - refs: - - "master@gstreamer/gst-ci" build cerbero cross win64 local: extends: '.cerbero cross win64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" - when: 'manual' - except: - refs: - - "master@gstreamer/gst-ci" .test fedora x86_64 local: image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest' extends: '.test' dependencies: - 'build fedora x86_64 local' - when: 'manual' - except: - refs: - - "master@gstreamer/gst-ci" check fedora local: extends: '.test fedora x86_64 local' @@ -185,10 +165,6 @@ cross-android universal examples local: image: "${CI_REGISTRY_IMAGE}/amd64/android:latest" dependencies: - "build cerbero cross-android universal local" - when: 'manual' - except: - refs: - - "master@gstreamer/gst-ci" integration testsuites fedora local: extends: '.test fedora x86_64 local' @@ -197,4 +173,3 @@ integration testsuites fedora local: variables: EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures" TEST_SUITE: "validate ges" - diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0225cd3f60..423d6ed9e4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -35,7 +35,22 @@ variables: --variants werror --timestamps -.global_retry_policy: +# +# Global CI policy +# +# This can be used to configure global behaviour our our jobs. It also +# supports some variable to affect the job when: +# +# EXCEPT_PROJECT_REGEX: Will mark to never if that regex match +# ONLY_PROJECT_REGEX: Will mark to never if that regex do not match +# +.global_ci_policy: + rules: + - if: '$EXCEPT_PROJECT_REGEX && $CI_PROJECT_NAME =~ $EXCEPT_PROJECT_REGEX' + when: 'never' + - if: '$ONLY_PROJECT_REGEX && $CI_PROJECT_NAME !~ $ONLY_PROJECT_REGEX' + when: 'never' + - when: 'on_success' retry: max: 2 when: @@ -47,7 +62,7 @@ variables: manifest: image: $MANIFEST_IMAGE extends: - - '.global_retry_policy' + - '.global_ci_policy' stage: 'preparation' script: - cd /gst-ci @@ -62,10 +77,11 @@ manifest: gst indent: image: $INDENT_IMAGE extends: - - '.global_retry_policy' + - '.global_ci_policy' stage: 'preparation' variables: GIT_STRATEGY: 'fetch' + EXCEPT_REGEX: '/^(gstreamer-sharp|gst-integration-testsuites|cerbero|gst-docs)$/' script: # man indent. grep RETURN VALUE, grab a beer on my behalf... - indent --version || true @@ -80,22 +96,13 @@ gst indent: echo 'style diverges, please run gst-indent first' exit 1 fi - except: - variables: - # No point on trying to format C files in those repositories - - $CI_PROJECT_NAME == "gstreamer-sharp" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" .build: stage: 'build' extends: - - '.global_retry_policy' + - '.global_ci_policy' needs: - "manifest" - dependencies: - - "manifest" # Taking into account the slowest shared runner + time needed to upload the binaries to artifacts # Also need to take into account I/O of pulling docker images and uploading artifacts timeout: '45min' @@ -109,6 +116,7 @@ gst indent: CARGO_HOME: "/cache/gstreamer/cargo" MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} --werror" + EXCEPT_PROJECT_REGEX: '/^(cerbero)$/' script: - ccache --show-stats @@ -129,9 +137,6 @@ gst indent: # Clean the .git repos since we won't need them anymore - rm -rf subprojects/*/.git/ - rm -rf build/subprojects/*/.git/ - except: - variables: - - $CI_PROJECT_NAME == "cerbero" build fedora x86_64: extends: '.build' @@ -139,16 +144,13 @@ build fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-plugins-rs)$/' artifacts: expire_in: '5 days' when: always paths: - "manifest.xml" - "gst-build/" - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-plugins-rs" build nodebug fedora x86_64: extends: '.build' @@ -156,51 +158,31 @@ build nodebug fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' build static fedora x86_64: extends: 'build fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' build static nodebug fedora x86_64: extends: 'build nodebug fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' build clang fedora x86_64: extends: 'build fedora x86_64' variables: CC: 'ccache clang' CXX: 'ccache clang++' - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' .test: stage: 'test' extends: - - '.global_retry_policy' + - '.global_ci_policy' variables: # Disable colored output to avoid weird rendering issues GST_DEBUG_NO_COLOR: "true" @@ -208,6 +190,7 @@ build clang fedora x86_64: GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" TIMEOUT_FACTOR: "2" CARGO_HOME: "/cache/gstreamer/cargo" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-examples|gst-docs)$/' script: - cd gst-build/ - echo "-> Running ${TEST_SUITE}" @@ -233,12 +216,6 @@ build clang fedora x86_64: reports: junit: - "validate-logs/*.xml" - # We disable the .build above, which this job usually depends upon for cerbero - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-examples" - - $CI_PROJECT_NAME == "gst-docs" .test fedora x86_64: image: $FEDORA_IMAGE @@ -246,21 +223,12 @@ build clang fedora x86_64: tags: ['gstreamer'] needs: - 'build fedora x86_64' - dependencies: - - build fedora x86_64 check fedora: extends: '.test fedora x86_64' variables: TEST_SUITE: "check.gst*" - except: - variables: - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-examples" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-plugins-rs" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-examples|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' integration testsuites fedora: extends: '.test fedora x86_64' @@ -268,12 +236,7 @@ integration testsuites fedora: variables: EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs --parts=${CI_NODE_TOTAL} --part-index=${CI_NODE_INDEX}" TEST_SUITE: "validate ges" - except: - variables: - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-plugins-rs" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-omx|gst-plugins-rs)$/' gstreamer-full: extends: 'build static fedora x86_64' @@ -303,60 +266,45 @@ gstreamer-full: stage: 'test' variables: EXTRA_VALIDATE_ARGS: "--valgrind" - # Some suppression files are missing a newline at the end which messes things - # up when concatenating them. awk will add missing newlines (unlike cat) - except: - variables: - - $CI_PROJECT_NAMESPACE == 'gstreamer' - - $CI_PROJECT_NAME == "gst-integration-testsuites" + # Some suppression files are missing a newline at the end which messes things + # up when concatenating them. awk will add missing newlines (unlike cat) + EXCEPT_PROJECT_REGEX: '/^(gstreamer|gst-integration-testsuites)$/' valgrind core: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gstreamer\\..*" - only: - variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-ci)$/ + ONLY_PROJECT_REGEX: '/^(gstreamer|gst-ci)$/' valgrind base: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-base\\..*" - only: - variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-ci)$/ + ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-ci)$/' valgrind good: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-good\\..*" - only: - variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci)$/ + ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci)$/' valgrind ugly: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-ugly\\..*" - only: - variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci)$/ + ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci)$/' valgrind bad: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-bad\\..*" - only: - variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci)$/ + ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci)$/' valgrind ges: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-editing-services\\..*" - only: - variables: - - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci|gst-devtools)$/ + ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci|gst-devtools)$/' # Template for Cerbero GStreamer Build # @@ -369,11 +317,9 @@ valgrind ges: stage: "build" image: $CERBERO_IMAGE extends: - - '.global_retry_policy' + - '.global_ci_policy' needs: - "manifest" - dependencies: - - "manifest" # Ensure that the runners it will be executed on # will have plenty of space for the cache tags: ['gstreamer'] @@ -446,12 +392,10 @@ valgrind ges: .build windows: image: $WINDOWS_IMAGE extends: - - '.global_retry_policy' + - '.global_ci_policy' stage: 'build' needs: - 'manifest' - dependencies: - - 'manifest' tags: - 'docker' - 'windows' @@ -464,6 +408,7 @@ valgrind ges: -Dlibav=disabled -Dvaapi=disabled -Ddevtools=disabled + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' script: # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or # from a previous job due to some gitlab bug or implicit behavior? @@ -497,14 +442,6 @@ valgrind ges: # Gitlab Runner issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/4291 # Blocked upgrade issue: https://gitlab.freedesktop.org/gstreamer/gst-ci/issues/6#note_192780 after_script: - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" build vs2017 amd64: extends: '.build windows' @@ -566,6 +503,8 @@ build msys2 : # with the associated build-tools. .cerbero deps: extends: .cerbero + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' stage: "pre-build" script: - $CERBERO $CERBERO_ARGS show-config @@ -583,9 +522,6 @@ build msys2 : - tar -C ${CERBERO_HOME} --exclude=var/tmp -czf $CERBERO_DEPS build-tools build-tools.cache dist/${ARCH} ${ARCH}.cache - only: - variables: - - $CI_PROJECT_NAME == "cerbero" artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" expire_in: '10 days' @@ -612,26 +548,15 @@ cerbero deps fedora x86_64: cerbero fedora x86_64: extends: '.cerbero fedora x86_64' + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps fedora x86_64" - - "manifest" - dependencies: - - "cerbero deps fedora x86_64" - only: - variables: - - $CI_PROJECT_NAME == "cerbero" build cerbero fedora x86_64: extends: '.cerbero fedora x86_64' - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" + variables: + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' # # Cerbero Android Universal build @@ -657,25 +582,15 @@ cerbero deps cross-android universal: cerbero cross-android universal: extends: '.cerbero cross-android universal' + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps cross-android universal" - - "manifest" - dependencies: - - "cerbero deps cross-android universal" - only: - variables: - - $CI_PROJECT_NAME == "cerbero" build cerbero cross-android universal: extends: '.cerbero cross-android universal' - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" + variables: + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' # # Cerbero Cross Windows builds @@ -695,26 +610,15 @@ cerbero deps cross-windows x86: cerbero cross win32: extends: '.cerbero cross win32' + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps cross-windows x86" - - "manifest" - dependencies: - - "cerbero deps cross-windows x86" - only: - variables: - - $CI_PROJECT_NAME == "cerbero" build cerbero cross win32: extends: '.cerbero cross win32' - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" + variables: + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' cerbero deps cross-windows x86_64: extends: '.cerbero deps' @@ -731,26 +635,15 @@ cerbero deps cross-windows x86_64: cerbero cross win64: extends: '.cerbero cross win64' + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps cross-windows x86_64" - - "manifest" - dependencies: - - "cerbero deps cross-windows x86_64" - only: - variables: - - $CI_PROJECT_NAME == "cerbero" build cerbero cross win64: extends: '.cerbero cross win64' - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-plugins-rs" + variables: + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' # # Build an Android App using the android binaries @@ -758,7 +651,7 @@ build cerbero cross win64: .cross-android universal examples: image: $ANDROID_IMAGE extends: - - '.global_retry_policy' + - '.global_ci_policy' stage: 'integrate' variables: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples @@ -798,30 +691,17 @@ build cerbero cross win64: cross-android universal examples: extends: ".cross-android universal examples" + variables: + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' needs: - "build cerbero cross-android universal" - - "manifest" - dependencies: - - "build cerbero cross-android universal" - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-plugins-rs" cerbero cross-android universal examples: extends: ".cross-android universal examples" + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero cross-android universal" - - "manifest" - dependencies: - - "cerbero cross-android universal" - only: - variables: - - $CI_PROJECT_NAME == "cerbero" # # Cerbero macOS X86_64 build @@ -860,27 +740,15 @@ cerbero deps macos x86_64: cerbero macos x86_64: extends: '.cerbero macos x86_64' + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps macos x86_64" - - "manifest" - dependencies: - - "cerbero deps macos x86_64" - only: - variables: - - $CI_PROJECT_NAME == "cerbero" build cerbero macos x86_64: extends: '.cerbero macos x86_64' - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-docs" - - $CI_PROJECT_NAME == "gst-sharp" - - $CI_PROJECT_NAME == "gst-plugins-rs" + variables: + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' # # Cerbero iOS build @@ -918,36 +786,24 @@ cerbero deps cross-ios universal: cerbero cross-ios universal: extends: '.cerbero cross-ios universal' + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps cross-ios universal" - - "manifest" - dependencies: - - "cerbero deps cross-ios universal" - only: - variables: - - $CI_PROJECT_NAME == "cerbero" build cerbero cross-ios universal: extends: '.cerbero cross-ios universal' - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-sharp" - - $CI_PROJECT_NAME == "gst-plugins-rs" + variables: + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' documentation: image: $FEDORA_IMAGE extends: - - '.global_retry_policy' + - '.global_ci_policy' + variables: + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-sharp|gst-integration-testsuites|gst-plugins-rs|gst-python)$/' needs: - "build fedora x86_64" - - "manifest" - dependencies: - - 'build fedora x86_64' stage: integrate script: @@ -960,14 +816,6 @@ documentation: artifacts: paths: - documentation/ - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gst-python" - - $CI_PROJECT_NAME == "gst-sharp" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-plugins-rs" # # Build an iOS App using the iOS binaries @@ -975,7 +823,7 @@ documentation: .cross-ios universal examples: stage: 'integrate' extends: - - '.global_retry_policy' + - '.global_ci_policy' variables: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples # disable codesigning so we don't need developer identities on the CI @@ -985,6 +833,7 @@ documentation: CODE_SIGN_IDENTITY="" CODE_SIGNING_ALLOWED="NO" CODE_SIGN_ENTITLEMENTS="" + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' script: # install the binaries - installer -pkg gstreamer-1.0-devel-*-ios-universal.pkg -target CurrentUserHomeDirectory -verbose @@ -1015,44 +864,28 @@ cross-ios universal examples: extends: ".cross-ios universal examples" needs: - "build cerbero cross-ios universal" - - "manifest" - dependencies: - - "build cerbero cross-ios universal" - except: - variables: - - $CI_PROJECT_NAME == "cerbero" - - $CI_PROJECT_NAME == "gst-build" - - $CI_PROJECT_NAME == "gst-omx" - - $CI_PROJECT_NAME == "gst-integration-testsuites" - - $CI_PROJECT_NAME == "gstreamer-vaapi" - - $CI_PROJECT_NAME == "gst-plugins-rs" cerbero cross-ios universal examples: extends: ".cross-ios universal examples" + variables: + ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero cross-ios universal" - - "manifest" - dependencies: - - "cerbero cross-ios universal" - only: - variables: - - $CI_PROJECT_NAME == "cerbero" build gst-omx zynq fedora x86_64: extends: 'build fedora x86_64' variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + ONLY_PROJECT_REGEX: '/^(gst-omx|gst-ci)$/' before_script: - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2019.2 ${CI_PROJECT_DIR}/vcu-omx-il - only: - variables: - - $CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/ build gst-omx tizonia fedora x86_64: extends: 'build fedora x86_64' variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}:${CI_PROJECT_DIR}/tizonia-install/lib64/pkgconfig/ + ONLY_PROJECT_REGEX: '/^(gst-omx|gst-ci)$/' before_script: - git clone https://github.com/tizonia/tizonia-openmax-il --branch v0.20.2 - cd tizonia-openmax-il @@ -1060,29 +893,17 @@ build gst-omx tizonia fedora x86_64: - ninja -C build - ninja -C build install - cd .. - only: - variables: - - $CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/ build rust fedora x86_64: extends: 'build fedora x86_64' variables: MESON_ARGS: "-Domx=disabled -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=enabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=enabled -Dgst-plugins-rs:sodium=system ${MESON_BUILDTYPE_ARGS} --werror" - only: - variables: - - $CI_PROJECT_NAME =~ /^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/ - except: - variables: - - $CI_PROJECT_NAME == "cerbero" + ONLY_PROJECT_REGEX: '/^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/' check rust fedora: extends: '.test fedora x86_64' needs: - 'build rust fedora x86_64' - dependencies: - - build rust fedora x86_64 variables: TEST_SUITE: "check.gst-plugins-rs.*" - only: - variables: - - $CI_PROJECT_NAME =~ /^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/ + ONLY_PROJECT_REGEX: '/^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/' From f551f4a7beefa7516ca6db3d3ac9903afe5166cd Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 4 Mar 2020 16:46:57 -0500 Subject: [PATCH 405/605] Make jobs interruptible by default This was added in 12.3 and is false by default. Make it to true, so that we stop the previous pipelien on force push. --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 423d6ed9e4..53b4c76658 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -58,6 +58,7 @@ variables: - 'stuck_or_timeout_failure' - 'scheduler_failure' - 'api_failure' + interruptible: true manifest: image: $MANIFEST_IMAGE From 2fca5b798121e20226a97214e385fd2488020a3d Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 4 Mar 2020 17:30:06 -0500 Subject: [PATCH 406/605] Only run CI automatically on merge_request --- .gitlab-ci.yml | 29 ++++++++++++++++++++++++----- gitlab/ci_template.yml | 6 ++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ded44087e..99c1fe7b1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,8 @@ stages: test manifest: variables: GIT_STRATEGY: fetch + rules: + - when: 'always' image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:2019-10-23-793476' stage: "preparation" script: @@ -24,9 +26,13 @@ test manifest: .base: image: "docker:stable" + extends: + - '.global_ci_policy' + rules: + - when: 'manual' + allow_failure: true services: - docker:dind - when: 'manual' variables: # When using dind, it's wise to use the overlayfs driver for # improved performance. @@ -112,18 +118,21 @@ android docker: DOCKERFILE: "docker/android/Dockerfile" extends: .base -.build local: - extends: '.build' +.local_rules: &local_rules rules: - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_PATH == "gstreamer/gst-ci"' when: 'never' - when: 'manual' + allow_failure: true + +.local_template: &local_template + <<: *local_rules needs: - 'manifest' # Test the build job against the latest build image tag and the local manifest build fedora x86_64 local: - extends: '.build local' + extends: '.build' stage: 'build' image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest" artifacts: @@ -132,26 +141,35 @@ build fedora x86_64 local: paths: - "manifest.xml" - "gst-build/" + <<: *local_template build cerbero fedora x86_64 local: extends: '.cerbero fedora x86_64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + <<: *local_template build cerbero cross-android universal local: extends: '.cerbero cross-android universal' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + <<: *local_template build cerbero cross win32 local: extends: '.cerbero cross win32' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + <<: *local_template build cerbero cross win64 local: extends: '.cerbero cross win64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" + <<: *local_template +# Note: dependencies: will be deprecated in the future, but current manual +# jobs with needs: (even if they are allowed to fail) will leave the pipeline +# pending. .test fedora x86_64 local: image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest' extends: '.test' + <<: *local_rules dependencies: - 'build fedora x86_64 local' @@ -163,8 +181,9 @@ check fedora local: cross-android universal examples local: extends: '.cross-android universal examples' image: "${CI_REGISTRY_IMAGE}/amd64/android:latest" + <<: *local_rules dependencies: - - "build cerbero cross-android universal local" + - 'build cerbero cross-android universal local' integration testsuites fedora local: extends: '.test fedora x86_64 local' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 53b4c76658..4f697216c9 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -64,6 +64,12 @@ manifest: image: $MANIFEST_IMAGE extends: - '.global_ci_policy' + rules: + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + when: 'always' + - if: '$CI_PROJECT_PATH == "gstreamer/cerbero"' + when: 'always' + - when: 'manual' stage: 'preparation' script: - cd /gst-ci From 8b8f1e0dfe00ba3310e3ee729c494d278c91e7f0 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 16 Mar 2020 20:29:26 -0400 Subject: [PATCH 407/605] ci: Cleanup gst-omx from unneeded jobs --- gitlab/ci_template.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4f697216c9..72bc0a74fe 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -151,7 +151,7 @@ build fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-plugins-rs)$/' + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-omx|gst-plugins-rs)$/' artifacts: expire_in: '5 days' when: always @@ -165,26 +165,26 @@ build nodebug fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' build static fedora x86_64: extends: 'build fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' build static nodebug fedora x86_64: extends: 'build nodebug fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' build clang fedora x86_64: extends: 'build fedora x86_64' variables: CC: 'ccache clang' CXX: 'ccache clang++' - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' .test: stage: 'test' @@ -253,6 +253,8 @@ gstreamer-full: --default-library=static -Dauto_features=disabled -Dgst-plugins-good:alpha=enabled + EXCEPT_PROJECT_REGEX: '' + ONLY_PROJECT_REGEX: '/^(gst-omx|gst-ci)$/' after_script: - cd gst-build/ - ninja -C build install @@ -808,7 +810,7 @@ documentation: extends: - '.global_ci_policy' variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-sharp|gst-integration-testsuites|gst-plugins-rs|gst-python)$/' + EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-sharp|gst-integration-testsuites|gst-omx|gst-plugins-rs|gst-python)$/' needs: - "build fedora x86_64" stage: integrate @@ -883,6 +885,7 @@ build gst-omx zynq fedora x86_64: extends: 'build fedora x86_64' variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + EXCEPT_PROJECT_REGEX: '' ONLY_PROJECT_REGEX: '/^(gst-omx|gst-ci)$/' before_script: - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2019.2 ${CI_PROJECT_DIR}/vcu-omx-il @@ -892,6 +895,7 @@ build gst-omx tizonia fedora x86_64: variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}:${CI_PROJECT_DIR}/tizonia-install/lib64/pkgconfig/ + EXCEPT_PROJECT_REGEX: '' ONLY_PROJECT_REGEX: '/^(gst-omx|gst-ci)$/' before_script: - git clone https://github.com/tizonia/tizonia-openmax-il --branch v0.20.2 @@ -905,6 +909,7 @@ build rust fedora x86_64: extends: 'build fedora x86_64' variables: MESON_ARGS: "-Domx=disabled -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=enabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=enabled -Dgst-plugins-rs:sodium=system ${MESON_BUILDTYPE_ARGS} --werror" + EXCEPT_PROJECT_REGEX: '' ONLY_PROJECT_REGEX: '/^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/' check rust fedora: @@ -913,4 +918,5 @@ check rust fedora: - 'build rust fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-rs.*" + EXCEPT_PROJECT_REGEX: '' ONLY_PROJECT_REGEX: '/^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/' From ee77993ccd859e96b50a92f5036b62a994778538 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 17 Mar 2020 11:58:33 -0400 Subject: [PATCH 408/605] ci: Remove outdated commemt We no longer use awk or cat int hat context. --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 72bc0a74fe..1fd30468bf 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -275,8 +275,6 @@ gstreamer-full: stage: 'test' variables: EXTRA_VALIDATE_ARGS: "--valgrind" - # Some suppression files are missing a newline at the end which messes things - # up when concatenating them. awk will add missing newlines (unlike cat) EXCEPT_PROJECT_REGEX: '/^(gstreamer|gst-integration-testsuites)$/' valgrind core: From 45901ee69b0af07e12be8187fd2bc10328f56e8f Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 17 Mar 2020 11:59:34 -0400 Subject: [PATCH 409/605] ci: Remove spurious except regex in valgrind test This was ported but unused from the previous code. In the new code it was preventing valgrind jobs from being run in gstreamer repository. --- gitlab/ci_template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1fd30468bf..f394bc268b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -275,7 +275,6 @@ gstreamer-full: stage: 'test' variables: EXTRA_VALIDATE_ARGS: "--valgrind" - EXCEPT_PROJECT_REGEX: '/^(gstreamer|gst-integration-testsuites)$/' valgrind core: extends: '.valgrind fedora x86_64' From c02cb97dc2509913cfa1bde3f4c86b696a11ca91 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 17 Mar 2020 21:27:09 -0400 Subject: [PATCH 410/605] ci: Fix exceptions for gst indent job The name of the variable passing the exception regex wasn't correct. As a side effect, gst-indent was not run on all repository. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f394bc268b..dd3ea3ea4e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -88,7 +88,7 @@ gst indent: stage: 'preparation' variables: GIT_STRATEGY: 'fetch' - EXCEPT_REGEX: '/^(gstreamer-sharp|gst-integration-testsuites|cerbero|gst-docs)$/' + EXCEPT_PROJECT_REGEX: '/^(gstreamer-sharp|gst-integration-testsuites|cerbero|gst-docs)$/' script: # man indent. grep RETURN VALUE, grab a beer on my behalf... - indent --version || true From 21584e46105d0f358ee41baa7ad75a7da3d70386 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 17 Mar 2020 21:38:34 -0400 Subject: [PATCH 411/605] ci: Fix gstreamer-full job to run in gst-build It was set to gst-omx by accident instead. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index dd3ea3ea4e..39f7ff340e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -254,7 +254,7 @@ gstreamer-full: -Dauto_features=disabled -Dgst-plugins-good:alpha=enabled EXCEPT_PROJECT_REGEX: '' - ONLY_PROJECT_REGEX: '/^(gst-omx|gst-ci)$/' + ONLY_PROJECT_REGEX: '/^(gst-build|gst-ci)$/' after_script: - cd gst-build/ - ninja -C build install From bbc624175d5e63c8e493404fc5c478f5d4bcb3e6 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 18 Mar 2020 08:57:12 +0100 Subject: [PATCH 412/605] ci: disable Rust jobs for now They are not reliable enough (#53) and generated artifacts are too big (#61) so disable them until those issues are solved. Fix https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/issues/97 --- gitlab/ci_template.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 39f7ff340e..6bf6a80870 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -901,19 +901,3 @@ build gst-omx tizonia fedora x86_64: - ninja -C build - ninja -C build install - cd .. - -build rust fedora x86_64: - extends: 'build fedora x86_64' - variables: - MESON_ARGS: "-Domx=disabled -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=enabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=enabled -Dgst-plugins-rs:sodium=system ${MESON_BUILDTYPE_ARGS} --werror" - EXCEPT_PROJECT_REGEX: '' - ONLY_PROJECT_REGEX: '/^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/' - -check rust fedora: - extends: '.test fedora x86_64' - needs: - - 'build rust fedora x86_64' - variables: - TEST_SUITE: "check.gst-plugins-rs.*" - EXCEPT_PROJECT_REGEX: '' - ONLY_PROJECT_REGEX: '/^(gst-plugins-rs|gstreamer|gst-plugins-base|gst-ci|gst-build|gst-devtools)$/' From a8941a856612cc9c88175261a3568f9fed0ce0fe Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 17 Mar 2020 21:58:54 -0400 Subject: [PATCH 413/605] ci: Replace REGEX variables Passing regex as variable does not really works, we ended up matching the regex as a string instead. Replace all REGEX variable with rules: override. It is longer but more reliable. Related to !247 Fixes #63 --- gitlab/ci_template.yml | 137 ++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 69 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6bf6a80870..ce32469aee 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -38,19 +38,9 @@ variables: # # Global CI policy # -# This can be used to configure global behaviour our our jobs. It also -# supports some variable to affect the job when: -# -# EXCEPT_PROJECT_REGEX: Will mark to never if that regex match -# ONLY_PROJECT_REGEX: Will mark to never if that regex do not match +# This can be used to configure global behaviour our our jobs. # .global_ci_policy: - rules: - - if: '$EXCEPT_PROJECT_REGEX && $CI_PROJECT_NAME =~ $EXCEPT_PROJECT_REGEX' - when: 'never' - - if: '$ONLY_PROJECT_REGEX && $CI_PROJECT_NAME !~ $ONLY_PROJECT_REGEX' - when: 'never' - - when: 'on_success' retry: max: 2 when: @@ -88,7 +78,8 @@ gst indent: stage: 'preparation' variables: GIT_STRATEGY: 'fetch' - EXCEPT_PROJECT_REGEX: '/^(gstreamer-sharp|gst-integration-testsuites|cerbero|gst-docs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(gstreamer-sharp|gst-integration-testsuites|cerbero|gst-docs)$/' script: # man indent. grep RETURN VALUE, grab a beer on my behalf... - indent --version || true @@ -123,7 +114,8 @@ gst indent: CARGO_HOME: "/cache/gstreamer/cargo" MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} --werror" - EXCEPT_PROJECT_REGEX: '/^(cerbero)$/' + rules: + - if: '$CI_PROJECT_NAME != "cerbero"' script: - ccache --show-stats @@ -151,7 +143,8 @@ build fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-omx|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-omx|gst-plugins-rs)$/' artifacts: expire_in: '5 days' when: always @@ -165,26 +158,30 @@ build nodebug fedora x86_64: image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' build static fedora x86_64: extends: 'build fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' build static nodebug fedora x86_64: extends: 'build nodebug fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' build clang fedora x86_64: extends: 'build fedora x86_64' variables: CC: 'ccache clang' CXX: 'ccache clang++' - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' .test: stage: 'test' @@ -197,7 +194,8 @@ build clang fedora x86_64: GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" TIMEOUT_FACTOR: "2" CARGO_HOME: "/cache/gstreamer/cargo" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-examples|gst-docs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-examples|gst-docs)$/' script: - cd gst-build/ - echo "-> Running ${TEST_SUITE}" @@ -235,7 +233,8 @@ check fedora: extends: '.test fedora x86_64' variables: TEST_SUITE: "check.gst*" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-examples|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' integration testsuites fedora: extends: '.test fedora x86_64' @@ -243,7 +242,8 @@ integration testsuites fedora: variables: EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs --parts=${CI_NODE_TOTAL} --part-index=${CI_NODE_INDEX}" TEST_SUITE: "validate ges" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-omx|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-omx|gst-plugins-rs)$/' gstreamer-full: extends: 'build static fedora x86_64' @@ -253,8 +253,9 @@ gstreamer-full: --default-library=static -Dauto_features=disabled -Dgst-plugins-good:alpha=enabled - EXCEPT_PROJECT_REGEX: '' - ONLY_PROJECT_REGEX: '/^(gst-build|gst-ci)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gst-build|gst-ci)$/' + after_script: - cd gst-build/ - ninja -C build install @@ -280,37 +281,43 @@ valgrind core: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gstreamer\\..*" - ONLY_PROJECT_REGEX: '/^(gstreamer|gst-ci)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-ci)$/' valgrind base: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-base\\..*" - ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-ci)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-ci)$/' valgrind good: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-good\\..*" - ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci)$/' valgrind ugly: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-ugly\\..*" - ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci)$/' valgrind bad: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-bad\\..*" - ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci)$/' valgrind ges: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-editing-services\\..*" - ONLY_PROJECT_REGEX: '/^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci|gst-devtools)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci|gst-devtools)$/' # Template for Cerbero GStreamer Build # @@ -324,6 +331,8 @@ valgrind ges: image: $CERBERO_IMAGE extends: - '.global_ci_policy' + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' needs: - "manifest" # Ensure that the runners it will be executed on @@ -414,7 +423,8 @@ valgrind ges: -Dlibav=disabled -Dvaapi=disabled -Ddevtools=disabled - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' script: # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or # from a previous job due to some gitlab bug or implicit behavior? @@ -509,8 +519,8 @@ build msys2 : # with the associated build-tools. .cerbero deps: extends: .cerbero - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' stage: "pre-build" script: - $CERBERO $CERBERO_ARGS show-config @@ -554,15 +564,13 @@ cerbero deps fedora x86_64: cerbero fedora x86_64: extends: '.cerbero fedora x86_64' - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps fedora x86_64" build cerbero fedora x86_64: extends: '.cerbero fedora x86_64' - variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' # # Cerbero Android Universal build @@ -588,15 +596,13 @@ cerbero deps cross-android universal: cerbero cross-android universal: extends: '.cerbero cross-android universal' - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps cross-android universal" build cerbero cross-android universal: extends: '.cerbero cross-android universal' - variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' # # Cerbero Cross Windows builds @@ -616,15 +622,13 @@ cerbero deps cross-windows x86: cerbero cross win32: extends: '.cerbero cross win32' - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps cross-windows x86" build cerbero cross win32: extends: '.cerbero cross win32' - variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' cerbero deps cross-windows x86_64: extends: '.cerbero deps' @@ -641,15 +645,13 @@ cerbero deps cross-windows x86_64: cerbero cross win64: extends: '.cerbero cross win64' - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps cross-windows x86_64" build cerbero cross win64: extends: '.cerbero cross win64' - variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' # # Build an Android App using the android binaries @@ -697,15 +699,15 @@ build cerbero cross win64: cross-android universal examples: extends: ".cross-android universal examples" - variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' needs: - "build cerbero cross-android universal" cerbero cross-android universal examples: extends: ".cross-android universal examples" - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' needs: - "cerbero cross-android universal" @@ -746,15 +748,13 @@ cerbero deps macos x86_64: cerbero macos x86_64: extends: '.cerbero macos x86_64' - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps macos x86_64" build cerbero macos x86_64: extends: '.cerbero macos x86_64' - variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' # # Cerbero iOS build @@ -792,22 +792,20 @@ cerbero deps cross-ios universal: cerbero cross-ios universal: extends: '.cerbero cross-ios universal' - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' needs: - "cerbero deps cross-ios universal" build cerbero cross-ios universal: extends: '.cerbero cross-ios universal' - variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' documentation: image: $FEDORA_IMAGE extends: - '.global_ci_policy' - variables: - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-sharp|gst-integration-testsuites|gst-omx|gst-plugins-rs|gst-python)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-sharp|gst-integration-testsuites|gst-omx|gst-plugins-rs|gst-python)$/' needs: - "build fedora x86_64" stage: integrate @@ -839,7 +837,8 @@ documentation: CODE_SIGN_IDENTITY="" CODE_SIGNING_ALLOWED="NO" CODE_SIGN_ENTITLEMENTS="" - EXCEPT_PROJECT_REGEX: '/^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' script: # install the binaries - installer -pkg gstreamer-1.0-devel-*-ios-universal.pkg -target CurrentUserHomeDirectory -verbose @@ -873,8 +872,8 @@ cross-ios universal examples: cerbero cross-ios universal examples: extends: ".cross-ios universal examples" - variables: - ONLY_PROJECT_REGEX: '/^cerbero$/' + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' needs: - "cerbero cross-ios universal" @@ -882,8 +881,8 @@ build gst-omx zynq fedora x86_64: extends: 'build fedora x86_64' variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" - EXCEPT_PROJECT_REGEX: '' - ONLY_PROJECT_REGEX: '/^(gst-omx|gst-ci)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/' before_script: - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2019.2 ${CI_PROJECT_DIR}/vcu-omx-il @@ -892,8 +891,8 @@ build gst-omx tizonia fedora x86_64: variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}:${CI_PROJECT_DIR}/tizonia-install/lib64/pkgconfig/ - EXCEPT_PROJECT_REGEX: '' - ONLY_PROJECT_REGEX: '/^(gst-omx|gst-ci)$/' + rules: + - if: '$CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/' before_script: - git clone https://github.com/tizonia/tizonia-openmax-il --branch v0.20.2 - cd tizonia-openmax-il From cf21e723e345cd080516b3e12c5a8585e661b282 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Fri, 20 Mar 2020 07:57:41 -0400 Subject: [PATCH 414/605] ci: Switch msys2 job to manual This will reduce the excessive load on the runners which are having issues with this job in particuliar. We will revisit when we better understand the runners issues. --- gitlab/ci_template.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ce32469aee..1090b9b8b2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -469,8 +469,13 @@ build vs2017 x86: variables: ARCH: 'x86' +# Disabled until the runner issues has been resolved. build msys2 : extends: '.build windows' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' + allow_failure: true + when: manual timeout: '60min' script: # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or From d016fa74531452c20d76f15d4fbfb31476837766 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Fri, 7 Feb 2020 13:37:06 +0530 Subject: [PATCH 415/605] docker: fedora: update to Fedora 31 aom-extra-tools has been removed from Fedora: https://src.fedoraproject.org/rpms/aom/c/038d7b2164e15c9067af3d2d2e0b50a4d890b218?branch=master gstreamer1-plugins-bad-nonfree has been removed from rpmfusion: https://pkgs.rpmfusion.org/cgit/nonfree/gstreamer1-plugins-bad-nonfree.git/commit/?id=30641c2ded63d12e099d32cb814bab8244f48c90 It was only used of its build deps of which only faac-devel was not already installed, so I added it instead. Need to manually install pip3 now. --- docker/fedora/Dockerfile | 2 +- docker/fedora/prepare.sh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index 82abdd3a40..624cfcdcb8 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:30 +FROM registry.fedoraproject.org/fedora:31 ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index e52d8ccc3d..ec1722ea1a 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -1,6 +1,6 @@ set -eux -dnf install -y git-core ninja-build dnf-plugins-core +dnf install -y git-core ninja-build dnf-plugins-core python3-pip # Configure git for various usage git config --global user.email "gst-build@gstreamer.net" @@ -17,7 +17,6 @@ dnf upgrade -y dnf install -y \ aalib-devel \ aom \ - aom-extra-tools \ bat \ libaom \ libaom-devel \ @@ -41,6 +40,7 @@ dnf install -y \ graphene-devel \ gsl \ gsl-devel \ + faac-devel \ ffmpeg \ ffmpeg-libs \ ffmpeg-devel \ @@ -180,7 +180,6 @@ dnf builddep -y gstreamer1 \ gstreamer1-plugins-good-extras \ gstreamer1-plugins-ugly \ gstreamer1-plugins-ugly-free \ - gstreamer1-plugins-bad-nonfree \ gstreamer1-plugins-bad-free \ gstreamer1-plugins-bad-free-extras \ gstreamer1-plugins-bad-freeworld \ From 9405f9245b1063896f1c02a3b0468a1ecc870193 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Fri, 7 Feb 2020 12:48:44 +0530 Subject: [PATCH 416/605] docker: install intel-mediasdk on Fedora image Will be used to build gst-msdk in gst-plugins-bad. Fix #52 --- docker/fedora/prepare.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index ec1722ea1a..a52672ad7d 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -51,6 +51,7 @@ dnf install -y \ procps-ng \ patch \ redhat-rpm-config \ + intel-mediasdk-devel \ json-glib \ json-glib-devel \ libnice \ From 70ac883275fea45d4efb20ba089e8a065c80989b Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 25 Mar 2020 14:00:20 -0400 Subject: [PATCH 417/605] ci: Remove uneeded cerbero jobs from gst-docs CI --- gitlab/ci_template.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1090b9b8b2..400ff10d53 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -607,7 +607,7 @@ cerbero cross-android universal: build cerbero cross-android universal: extends: '.cerbero cross-android universal' rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' # # Cerbero Cross Windows builds @@ -633,7 +633,7 @@ cerbero cross win32: build cerbero cross win32: extends: '.cerbero cross win32' rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' cerbero deps cross-windows x86_64: extends: '.cerbero deps' @@ -705,7 +705,7 @@ build cerbero cross win64: cross-android universal examples: extends: ".cross-android universal examples" rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' needs: - "build cerbero cross-android universal" @@ -842,8 +842,6 @@ documentation: CODE_SIGN_IDENTITY="" CODE_SIGNING_ALLOWED="NO" CODE_SIGN_ENTITLEMENTS="" - rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' script: # install the binaries - installer -pkg gstreamer-1.0-devel-*-ios-universal.pkg -target CurrentUserHomeDirectory -verbose @@ -872,6 +870,8 @@ documentation: cross-ios universal examples: extends: ".cross-ios universal examples" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' needs: - "build cerbero cross-ios universal" From 54d94f7444153df68b63059715151f97a9668beb Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 25 Mar 2020 14:36:48 -0400 Subject: [PATCH 418/605] ci: Always run CI for gst-docs We have an external cron that rely on this to pull and update the doc on the webpage. Fixes #64 --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 400ff10d53..7afd2efd84 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -59,6 +59,8 @@ manifest: when: 'always' - if: '$CI_PROJECT_PATH == "gstreamer/cerbero"' when: 'always' + - if: '$CI_PROJECT_PATH == "gstreamer/gst-docs"' + when: 'always' - when: 'manual' stage: 'preparation' script: From 8ba527416cb1ffbfdecb9d9c24db4965361bce24 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 25 Mar 2020 17:22:09 -0400 Subject: [PATCH 419/605] ci: Re-enable example builds for MR As we should still build the examples to make sure they are not broken. Reported-by: Matthew Waters --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7afd2efd84..9a76e4eb5c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -609,6 +609,7 @@ cerbero cross-android universal: build cerbero cross-android universal: extends: '.cerbero cross-android universal' rules: + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' # @@ -707,6 +708,7 @@ build cerbero cross win64: cross-android universal examples: extends: ".cross-android universal examples" rules: + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' needs: - "build cerbero cross-android universal" @@ -805,6 +807,7 @@ cerbero cross-ios universal: build cerbero cross-ios universal: extends: '.cerbero cross-ios universal' rules: + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' documentation: @@ -873,6 +876,7 @@ documentation: cross-ios universal examples: extends: ".cross-ios universal examples" rules: + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' needs: - "build cerbero cross-ios universal" From 3d06b08d4965bf9407021148e060decf8c870234 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 26 Mar 2020 17:20:03 +0530 Subject: [PATCH 420/605] gst-build: Always include meson-logs in the artifacts Can't figure out build errors without this. --- gitlab/ci_template.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9a76e4eb5c..805da43c6d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -138,6 +138,11 @@ gst indent: # Clean the .git repos since we won't need them anymore - rm -rf subprojects/*/.git/ - rm -rf build/subprojects/*/.git/ + artifacts: + expire_in: "7 days" + when: "always" + paths: + - 'gst-build/build/meson-logs/' build fedora x86_64: extends: '.build' @@ -268,6 +273,8 @@ gstreamer-full: - ninja -C _build - meson test -C _build -v artifacts: + expire_in: "7 days" + when: "always" paths: - 'gst-build/build/meson-logs/' - 'gst-build/build/gstinitstaticplugins.c' @@ -453,13 +460,12 @@ valgrind ges: meson build $env:MESON_ARGS && ninja -C build" # FIXME: extract builddir for tests - # There's a bug that prevents us from exporting artifacts with docker-windows - # executors. It has since been fixed in gitlab 12.1, but - # we are blocked from upgrading currently. - # - # Gitlab Runner issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/4291 - # Blocked upgrade issue: https://gitlab.freedesktop.org/gstreamer/gst-ci/issues/6#note_192780 - after_script: + #after_script: + artifacts: + expire_in: "7 days" + when: "always" + paths: + - 'gst-build/build/meson-logs/' build vs2017 amd64: extends: '.build windows' From 32c91139a4dac04e447119aa1b9dfdc204dc86eb Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 26 Mar 2020 19:07:19 +0530 Subject: [PATCH 421/605] gst-build: Only use gstreamer runners for windows jobs Till we can update the container image to the Feb 11 security update, all x86 executables will crash because of: https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t So restrict to the gstreamer runners that are running an older Windows version. For example: ``` subprojects\gstreamer\gst\parse\meson.build:10:0: ERROR: Command "c:\python38\python.exe C:\builds\thaytan\gst-plugins-good\gst-build\subprojects\gstreamer\gst\parse\get_flex_version.py C:\builds\thaytan\gst-plugins-good\gst-build\subprojects\win-flex-bison-binaries\win_flex.exe" failed with status 1. ``` https://gitlab.freedesktop.org/thaytan/gst-plugins-good/-/jobs/2066909 ``` meson.build:1:0: ERROR: Executables created by c compiler cl are not runnable. ``` https://gitlab.freedesktop.org/nirbheek/gst-ci/-/jobs/2070394 --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 805da43c6d..74da5a97fb 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -424,6 +424,7 @@ valgrind ges: - 'docker' - 'windows' - '1809' + - 'gstreamer-windows' timeout: '45min' variables: MESON_ARGS: > From a5686ae478e86a0e0e80c0a342acf8064589fad8 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 27 Mar 2020 20:45:32 +0530 Subject: [PATCH 422/605] gst-build: Disable vs2017 x86 job too This might be related to the same issue described in the previous commit: Till we can update the container image to the Feb 11 security update, x86 executables and in general the container image will behave badly because of: https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t vs2017 x86 has been failing with a runner system failure while uploading artifacts / submitting job status: ``` Uploading artifacts... gst-build/build/meson-logs/: found 2 matching files WARNING: Failed to load system CertPool: crypto/x509: system root pool is not available on Windows ERROR: Job failed (system failure): aborted: ``` https://gitlab.freedesktop.org/slomo/gst-plugins-good/-/jobs/2084184 Disable it for now. --- gitlab/ci_template.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 74da5a97fb..1cbb15cd17 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -475,16 +475,19 @@ build vs2017 amd64: build vs2017 x86: extends: '.build windows' + # Disabled until the container image can be updated + rules: + - allow_failure: true + when: 'manual' variables: ARCH: 'x86' -# Disabled until the runner issues has been resolved. build msys2 : extends: '.build windows' + # Disabled until the container image can be updated rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' - allow_failure: true - when: manual + - allow_failure: true + when: 'manual' timeout: '60min' script: # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or From e0544b256f3c9fd18b7c081199b3179671bf8f9d Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Sat, 28 Mar 2020 17:40:18 +0900 Subject: [PATCH 423/605] ci_template: Increase timeout of -good valgrind splitmux unit test includes encoding task which might require more time to finish test --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1cbb15cd17..cc04fbd068 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -304,6 +304,8 @@ valgrind good: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-good\\..*" + # take longer time due to splitmux unit test + TIMEOUT_FACTOR: "4" rules: - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci)$/' From 6c379762c24201def9533f941e26b81ae4dd5dd7 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 28 Mar 2020 16:34:03 +0530 Subject: [PATCH 424/605] gst-build: Disabling uploading of artifacts for now It seems to be timing out with high frequency only on Windows runners. ``` Version: 12.8.0 00:47 Git revision: 1b659122 Git branch: 12-8-stable GO version: go1.13.7 Built: 2020-02-22T03:03:07+0000 OS/Arch: windows/amd64 Uploading artifacts... gst-build/build/meson-logs/: found 2 matching files WARNING: Failed to load system CertPool: crypto/x509: system root pool is not available on Windows ERROR: Job failed (system failure): aborted: ``` See: https://gitlab.freedesktop.org/gstreamer/gst-ci/-/merge_requests/261 --- gitlab/ci_template.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index cc04fbd068..8852d302f7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -464,11 +464,12 @@ valgrind ges: ninja -C build" # FIXME: extract builddir for tests #after_script: - artifacts: - expire_in: "7 days" - when: "always" - paths: - - 'gst-build/build/meson-logs/' + # XXX: Re-enable when uploading stops timing out + #artifacts: + # expire_in: "7 days" + # when: "always" + # paths: + # - 'gst-build/build/meson-logs/' build vs2017 amd64: extends: '.build windows' From 18f285f15c7beab6a0f00357fe27a3ba9d17574a Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 20 Jun 2019 18:52:17 +1000 Subject: [PATCH 425/605] gitlab: compress deps using xz on master Results in a better compression for artifacts that are going to be used multiple times. --- gitlab/ci_template.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8852d302f7..bbd74b1967 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -360,7 +360,6 @@ valgrind ges: CERBERO_HOME: "cerbero-build" CERBERO_SOURCES: "cerbero-sources" - CERBERO_DEPS: "cerbero-deps.tar.gz" CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS}" CERBERO: "./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml" CERBERO_PACKAGE_ARGS: "-t" @@ -380,7 +379,15 @@ valgrind ges: - mkdir -p ../../${CI_PROJECT_NAMESPACE} - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero - rsync -aH ${CERBERO_HOST_DIR} . - - test -f ${CERBERO_DEPS} && tar -C ${CERBERO_HOME} -xf ${CERBERO_DEPS} + - | + if test -f cerbero-deps.tar.xz ; then + echo "Extracting xz deps tarball" + export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible + time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf cerbero-deps.tar.xz + elif test -f cerbero-deps.tar.gz; then + echo "Extracting gz deps tarball" + time tar -C ${CERBERO_HOME} -xf cerbero-deps.tar.gz + fi - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc - ./cerbero-uninstalled --self-update manifest.xml @@ -413,7 +420,7 @@ valgrind ges: paths: - "manifest.xml" - "${CERBERO_HOME}/logs" - - "*.tar.bz2" + - "*.tar.*" .build windows: image: $WINDOWS_IMAGE @@ -555,7 +562,9 @@ build msys2 : - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && mkdir -p ${CERBERO_HOME}/dist/${ARCH} && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH} - - tar -C ${CERBERO_HOME} --exclude=var/tmp -czf $CERBERO_DEPS + - export CERBERO_DEPS="cerbero-deps.tar.xz" + - export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible + - time tar -C ${CERBERO_HOME} --exclude=var/tmp --use-compress-program=xz -cf $CERBERO_DEPS build-tools build-tools.cache dist/${ARCH} ${ARCH}.cache artifacts: @@ -566,7 +575,7 @@ build msys2 : - "manifest.xml" - "${CERBERO_HOME}/logs" - "${CERBERO_HOME}/cerbero-deps.log" - - "${CERBERO_DEPS}" + - "cerbero-deps.tar.*" # # Cerbero Linux X86_64 build @@ -612,7 +621,7 @@ cerbero deps cross-android universal: paths: - "manifest.xml" - "${CERBERO_HOME}/logs" - - "*[0-9].tar.bz2" + - "*[0-9].tar.*" cerbero cross-android universal: extends: '.cerbero cross-android universal' From b00c68e69e42d5df2dfa4ce095279ec00581cfb1 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 2 Apr 2020 23:42:41 +1100 Subject: [PATCH 426/605] gitlab: don't upload gst-build artifacts we never use --- gitlab/ci_template.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index bbd74b1967..e8d861b4cf 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -144,14 +144,16 @@ gst indent: paths: - 'gst-build/build/meson-logs/' -build fedora x86_64: +.build fedora x86_64: extends: '.build' - stage: 'build' image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-omx|gst-plugins-rs)$/' + +build fedora x86_64: + extends: '.build fedora x86_64' artifacts: expire_in: '5 days' when: always @@ -161,7 +163,6 @@ build fedora x86_64: build nodebug fedora x86_64: extends: '.build' - stage: 'build' image: $FEDORA_IMAGE variables: MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" @@ -169,7 +170,7 @@ build nodebug fedora x86_64: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' build static fedora x86_64: - extends: 'build fedora x86_64' + extends: '.build fedora x86_64' variables: MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled" rules: @@ -183,7 +184,7 @@ build static nodebug fedora x86_64: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' build clang fedora x86_64: - extends: 'build fedora x86_64' + extends: '.build fedora x86_64' variables: CC: 'ccache clang' CXX: 'ccache clang++' From e137093defd950206ff634a91ea026d4ce120e20 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 2 Apr 2020 17:40:15 -0400 Subject: [PATCH 427/605] ci: Update to use new external storage for cerbero deps The goal of this change is to reduce egress on the FDO servers. --- gitlab/build_manifest.py | 15 ++++++--------- gitlab/ci_template.yml | 21 ++++++++------------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index 28ba6c6fe6..99f73c0911 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -47,17 +47,14 @@ os.environ['GIT_TERMINAL_PROMPT'] = '0' def git(*args, repository_path='.'): return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() -def get_cerbero_last_build_info (namespace : str, branch : str): - base_url = f"https://gitlab.freedesktop.org/{namespace}/cerbero/-/jobs" - url = f"{base_url}/artifacts/{branch}/raw/cerbero-build/cerbero-deps.log" +def get_cerbero_last_build_info (branch : str): + # Take the log from slowest build to reduce cache misses, the logs are + # uploaded as soon as they are ready. + url = "https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/cross-ios/universal/cerbero-deps.log" deps = [{'commit': None}] try: - # The logs are only available if all jobs have passed so it does not - # matter which distro/arch is picked. - values = { 'job': "cerbero deps fedora x86_64" } - data = urllib.parse.urlencode(values) - req = urllib.request.Request(f"{url}?{data}") + req = urllib.request.Request(url) resp = urllib.request.urlopen(req); deps = json.loads(resp.read()) except urllib.error.URLError: @@ -76,7 +73,7 @@ def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str] # Special case cerbero to avoid cache misses if module == 'cerbero': - sha = get_cerbero_last_build_info(namespace, branch) + sha = get_cerbero_last_build_info(branch) if sha is not None: return sha, sha diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e8d861b4cf..f9537b2381 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -371,6 +371,7 @@ valgrind ges: CERBERO_OVERRIDDEN_DIST_DIR: "" # location where cerbero is cached on the host CERBERO_HOST_DIR: "/cerbero/" + CERBERO_DEPS: "${CERBERO_HOME}/cerbero-deps.tar.xz" before_script: # FIXME Wrong namespace # Workaround build-tools having hardcoded internal path @@ -381,13 +382,10 @@ valgrind ges: - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero - rsync -aH ${CERBERO_HOST_DIR} . - | - if test -f cerbero-deps.tar.xz ; then + if test -f ${CERBERO_DEPS} ; then echo "Extracting xz deps tarball" export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible - time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf cerbero-deps.tar.xz - elif test -f cerbero-deps.tar.gz; then - echo "Extracting gz deps tarball" - time tar -C ${CERBERO_HOME} -xf cerbero-deps.tar.gz + time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf ${CERBERO_DEPS} fi - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc @@ -397,7 +395,7 @@ valgrind ges: - $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} + - test -f ${CERBERO_DEPS} || $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && test -d ${CERBERO_HOME}/dist/${ARCH} && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR} @@ -559,15 +557,12 @@ build msys2 : gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice - - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} --skip-fetch --job-id=${CI_JOB_ID} - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && mkdir -p ${CERBERO_HOME}/dist/${ARCH} && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH} - - export CERBERO_DEPS="cerbero-deps.tar.xz" - - export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible - - time tar -C ${CERBERO_HOME} --exclude=var/tmp --use-compress-program=xz -cf $CERBERO_DEPS - build-tools build-tools.cache - dist/${ARCH} ${ARCH}.cache + - $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH} + - test "x${CERBERO_PRIVATE_SSH_KEY}" = "x" + || $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH} artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" expire_in: '10 days' @@ -576,7 +571,7 @@ build msys2 : - "manifest.xml" - "${CERBERO_HOME}/logs" - "${CERBERO_HOME}/cerbero-deps.log" - - "cerbero-deps.tar.*" + - "${CERBERO_DEPS}" # # Cerbero Linux X86_64 build From afab7cc48b27cc04dd81c43074643d61ede4be29 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 2 Apr 2020 11:26:04 +1100 Subject: [PATCH 428/605] docker: update android image container for ndk-21 --- docker/android/prepare.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh index 4b01162692..3f3632cd5c 100644 --- a/docker/android/prepare.sh +++ b/docker/android/prepare.sh @@ -13,18 +13,21 @@ dnf install -y \ mkdir -p /android/sources -curl -o /android/sources/android-ndk.zip https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip +curl -o /android/sources/android-ndk.zip https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip unzip /android/sources/android-ndk.zip -d ${ANDROID_NDK_HOME}/ # remove the intermediate versioned directory mv ${ANDROID_NDK_HOME}/*/* ${ANDROID_NDK_HOME}/ -curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip +curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-6200805_latest.zip unzip /android/sources/android-sdk-tools.zip -d ${ANDROID_HOME}/ mkdir -p ${ANDROID_HOME}/licenses # Accept licenses. Values taken from: -# ANDROID_HOME=/path/to/android/sdk-tools $ANDROID_HOME/tools/bin/sdkmanager --licenses +# $ANDROID_HOME/tools/bin/sdkmanager --sdk_root=$ANDROID_HOME --licenses +# cd $ANDROID_HOME +# for f in licenses/*; do echo "echo \"$(cat $f | tr -d '\n')\" > \${ANDROID_HOME}/$f"; done echo "601085b94cd77f0b54ff86406957099ebe79c4d6" > ${ANDROID_HOME}/licenses/android-googletv-license +echo "859f317696f67ef3d7f30a50a5560e7834b43903" > ${ANDROID_HOME}/licenses/android-sdk-arm-dbt-license echo "24333f8a63b6825ea9c5514f83c2829b004d1fee" > ${ANDROID_HOME}/licenses/android-sdk-license echo "84831b9409646a918e30573bab4c9c91346d8abd" > ${ANDROID_HOME}/licenses/android-sdk-preview-license echo "33b6a2b64607f11b759f320ef9dff4ae5c47d97a" > ${ANDROID_HOME}/licenses/google-gdk-license @@ -32,7 +35,7 @@ echo "e9acab5b5fbb560a72cfaecce8946896ff6aab9d" > ${ANDROID_HOME}/licenses/mips- # pre-cache deps export GSTREAMER_ROOT_ANDROID=/android/sources/gstreamer-android -curl -o /android/sources/gstreamer-android.tar.xz https://gstreamer.freedesktop.org/data/pkg/android/1.16.0/gstreamer-1.0-android-universal-1.16.0.tar.xz +curl -o /android/sources/gstreamer-android.tar.xz https://gstreamer.freedesktop.org/data/pkg/android/1.16.2/gstreamer-1.0-android-universal-1.16.2.tar.xz mkdir $GSTREAMER_ROOT_ANDROID tar -xvf /android/sources/gstreamer-android.tar.xz -C $GSTREAMER_ROOT_ANDROID ls $GSTREAMER_ROOT_ANDROID From 9ed6305e1fc139edb50c24ea00169fc481b0517d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 2 Apr 2020 11:28:09 +1100 Subject: [PATCH 429/605] docker/cerbero: checkout proposed ndk-21 cerbero branch --- docker/cerbero/Dockerfile-fedora | 2 +- gitlab/ci_template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 2347399ebc..3c30ef2cbb 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -13,7 +13,7 @@ RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ git config --global user.name "Cerbero Build System" -RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ +RUN git clone -b ndk-21 https://gitlab.freedesktop.org/ystreet/cerbero.git && \ mkdir $HOME/.cerbero && \ echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \ echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f9537b2381..192ebdaf61 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -8,7 +8,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-02-10-1589547-master' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-02-10-1589546-master' + CERBERO_IMAGE: 'registry.freedesktop.org/ystreet/gst-ci/amd64/cerbero-fedora:latest' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-02-24-1712085-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' From 3b273b01e8411db5b6e23f28156127de0d6da64d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 6 Apr 2020 18:38:41 +1000 Subject: [PATCH 430/605] Partial revert of 9ed6305e1fc139edb50c24ea00169fc481b0517d For post-merge of android ndk-21 as part of: https://gitlab.freedesktop.org/gstreamer/gst-ci/-/merge_requests/266 https://gitlab.freedesktop.org/gstreamer/cerbero/-/merge_requests/463 --- docker/cerbero/Dockerfile-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 3c30ef2cbb..2347399ebc 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -13,7 +13,7 @@ RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ git config --global user.name "Cerbero Build System" -RUN git clone -b ndk-21 https://gitlab.freedesktop.org/ystreet/cerbero.git && \ +RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git && \ mkdir $HOME/.cerbero && \ echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \ echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \ From 45d4e8ebc028f2a2e338b6a975dd5cb0b4ce412d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 6 Apr 2020 21:52:59 +1000 Subject: [PATCH 431/605] Update cerbero and android docker images Partial revert of 9ed6305e1fc139edb50c24ea00169fc481b0517d For post-merge of android ndk-21 as part of: https://gitlab.freedesktop.org/gstreamer/gst-ci/-/merge_requests/266 https://gitlab.freedesktop.org/gstreamer/cerbero/-/merge_requests/463 --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 192ebdaf61..9f21f97d71 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -7,8 +7,8 @@ stages: - 'integrate' variables: - ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-02-10-1589547-master' - CERBERO_IMAGE: 'registry.freedesktop.org/ystreet/gst-ci/amd64/cerbero-fedora:latest' + ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-04-06-2193279-master' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-04-06-2193307-master' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-02-24-1712085-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' From ae1b75b51f20513fc27b4996ad2ed790cbde7230 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 7 Apr 2020 14:45:08 +0300 Subject: [PATCH 432/605] windows: switch the image tag to ltsc2019 This should have no functional difference as 1809 is an alias to ltsc2019 but it makes it more explicit that what branch we are tracking --- docker/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 580ab2e239..7c86152f3b 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -1,6 +1,6 @@ # escape=` -FROM 'mcr.microsoft.com/windows/servercore:1809' +FROM 'mcr.microsoft.com/windows/servercore:ltsc2019' # Make sure any failure in PowerShell scripts is fatal SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] From 42830e91f6bf565cbde289e1a148ee42cfe3c727 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 7 Apr 2020 14:52:37 +0300 Subject: [PATCH 433/605] windows: udpate the docker image This is a rebuild of the image to acount for an issue with a security update during February 2020 https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t This also means we can drop the gstreamer-windows tag that was added in 32c91139a4dac04e447119aa1b9dfdc204dc86eb --- gitlab/ci_template.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9f21f97d71..40f6d7b924 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -12,7 +12,7 @@ variables: FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-02-24-1712085-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v9' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v10' # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'master' @@ -432,7 +432,6 @@ valgrind ges: - 'docker' - 'windows' - '1809' - - 'gstreamer-windows' timeout: '45min' variables: MESON_ARGS: > From 2871b293e1f693c6a8379119ba3632b68b4abdfa Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 10 Apr 2020 20:42:57 +0300 Subject: [PATCH 434/605] ci_template: reduce the size of exported artifacts in gst-build jobs This is initial work looking over the deps subprojects and cleaning up the low hanging fruits. Saves about 90mb compressed. --- gitlab/ci_template.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 40f6d7b924..71dbadfa1e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -138,6 +138,21 @@ gst indent: # Clean the .git repos since we won't need them anymore - rm -rf subprojects/*/.git/ - rm -rf build/subprojects/*/.git/ + # Clean the big files that are not useful to us to reduce the artifact size + - rm -rf subprojects/FFmpeg/tests + - rm -rf subprojects/FFmpeg/doc + - rm -rf subprojects/openh264/res + - rm -rf subprojects/libxml2-2.9.7/result + - rm -rf subprojects/libxml2-2.9.7/doc + - rm -rf subprojects/libxml2-2.9.7/test + - rm -rf subprojects/libpsl/fuzz + - rm -rf subprojects/glib/gio/tests + - rm -rf subprojects/glib/glib/tests + - rm -rf subprojects/glib/gobject/tests + - rm -rf subprojects/glib/docs/tests + - rm -rf subprojects/glib/tests + - rm -rf subprojects/glib/po + - rm -rf subprojects/glib/docs artifacts: expire_in: "7 days" when: "always" From d692406ab17a14a2835f0069ab4353759a6d7a62 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 16 Apr 2020 12:02:36 -0400 Subject: [PATCH 435/605] ci: Let Marge the merge bot trigger our CI We have notice that a lot of CI activity is cause by user pushing to their branch after having created an MR. To reduce our CI foot-print, the CI will now only be automatically triggered when a reviewer assign the MR to the merge bot. It will still be possible to run the CI manually but the result of that CI won't be used by Marge. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 71dbadfa1e..8d2723fba5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -55,7 +55,7 @@ manifest: extends: - '.global_ci_policy' rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + - if: '$GITLAB_USER_LOGIN == "gstreamer-merge-bot"' when: 'always' - if: '$CI_PROJECT_PATH == "gstreamer/cerbero"' when: 'always' From 26c45d6ca32934e6610e3be70f898b6e0e54c768 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 17 Apr 2020 08:08:15 +0300 Subject: [PATCH 436/605] clone_manifest_ref: add the ability to fetch instead of cloning the repo --- gitlab/clone_manifest_ref.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gitlab/clone_manifest_ref.py b/gitlab/clone_manifest_ref.py index cbaf75deb1..b26043b510 100755 --- a/gitlab/clone_manifest_ref.py +++ b/gitlab/clone_manifest_ref.py @@ -67,6 +67,7 @@ if __name__ == "__main__": parser.add_argument("--project", action="store", type=str) parser.add_argument("--destination", action="store", type=str, default='.') parser.add_argument("--manifest", action="store", type=str) + parser.add_argument("--fetch", action="store_true", default=False) options = parser.parse_args() if not options.project: @@ -82,5 +83,10 @@ if __name__ == "__main__": if dest == '.': dest = os.path.join (os.getcwd(), project.name) - git('clone', project.fetch_uri, dest) + if options.fetch: + assert os.path.exists(dest) == True + git('fetch', project.fetch_uri, project.revision, repository_path=dest) + else: + git('clone', project.fetch_uri, dest) + git('checkout', '--detach', project.revision, repository_path=dest) From 8cb3a84d97871e67e2fccdca1ed3e49f9b49f84d Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Fri, 17 Apr 2020 13:47:22 -0400 Subject: [PATCH 437/605] Don't let marge run post-merge pipeline This is a small regression from previous change. --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8d2723fba5..012085a49c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -55,7 +55,7 @@ manifest: extends: - '.global_ci_policy' rules: - - if: '$GITLAB_USER_LOGIN == "gstreamer-merge-bot"' + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $GITLAB_USER_LOGIN == "gstreamer-merge-bot"' when: 'always' - if: '$CI_PROJECT_PATH == "gstreamer/cerbero"' when: 'always' From 76763636457a7831506dc1243f849e1358734349 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 17 Apr 2020 07:29:16 +0300 Subject: [PATCH 438/605] ci_template: avoid building libnice tests and examples in the subproject We don't run the libnice testsuite, and when binaries are built they consume ~45mb of space. This increases the size of the artifacts we export from the gst-build job for the testsuite and drives up the storage and bandwith costs when re-downloading the artifacts. Similary disable the test targets of couple other subprojects as well --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 012085a49c..6e1a5faf4c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,6 +21,10 @@ variables: GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both DEFAULT_MESON_ARGS: > + -Dlibnice:tests=disabled + -Dlibnice:examples=disabled + -Dopenh264:tests=disabled + -Dpygobject:tests=false -Dpython=enabled -Dlibav=enabled -Dugly=enabled From 815976e4aa42bc0d19c40a13c9cba68fbe0643c4 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 16 Apr 2020 21:11:36 +0300 Subject: [PATCH 439/605] ci_template: avoid exporting the mediafiles into build artifacts Previously we where accidently exporting the whole repo of gst-integration-testsuites which includes 350mb of raw media files and made the artifacts storage explode through the roof along with the CI bills fd.o had to pay for uploading and redownloading the artifacts To deal with this, we clean all the media files from the builddir and when needed we copy them over from the cache in the docker image, and then git fetch the repo. Close #69 --- gitlab/ci_template.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6e1a5faf4c..4e4d901791 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -157,11 +157,18 @@ gst indent: - rm -rf subprojects/glib/tests - rm -rf subprojects/glib/po - rm -rf subprojects/glib/docs + + # Clear gst-intergrations-testsuites report as it contains raw media + # files which is only needed for validate jobs. When needed we copy the + # cached repo in the docker image and fetch. + - rm -rf subprojects/gst-integration-testsuites artifacts: expire_in: "7 days" when: "always" paths: - 'gst-build/build/meson-logs/' + # Needed by the integration tests + - 'manifest.xml' .build fedora x86_64: extends: '.build' @@ -271,6 +278,18 @@ integration testsuites fedora: TEST_SUITE: "validate ges" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-omx|gst-plugins-rs)$/' + before_script: + # Avoid cloning the repo since it contains media files, + # instead copy it from the cache and do a git fetch + # We remove this copy first since it had its mediafiles stripped + - cp -r /gst-build/subprojects/gst-integration-testsuites/ "${CI_PROJECT_DIR}/gst-build/subprojects/gst-integration-testsuites/" + - curl -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" + - >- + python3 clone_manifest_ref.py + --fetch + --manifest manifest.xml + --project gst-integration-testsuites + --destination gst-build/subprojects/gst-integration-testsuites gstreamer-full: extends: 'build static fedora x86_64' From 95f9b25683c146231cfa2b3d392a84f881a28332 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sun, 19 Apr 2020 14:30:43 -0400 Subject: [PATCH 440/605] ci: Always allow redirect when using curl This done by specifying -L parameter. Some of the FDO infrastructure is using redirect. --- gitlab/ci_template.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4e4d901791..910999ae80 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -89,7 +89,7 @@ gst indent: script: # man indent. grep RETURN VALUE, grab a beer on my behalf... - indent --version || true - - curl -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/${GST_UPSTREAM_BRANCH}/tools/gst-indent + - curl -L -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/${GST_UPSTREAM_BRANCH}/tools/gst-indent - chmod +x gst-indent - find . -name '*.c' -exec ./gst-indent {} + - | @@ -125,7 +125,7 @@ gst indent: script: - ccache --show-stats - - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py + - curl -L -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py - chmod +x clone_manifest_ref.py - ./clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build - cd gst-build/ @@ -283,7 +283,7 @@ integration testsuites fedora: # instead copy it from the cache and do a git fetch # We remove this copy first since it had its mediafiles stripped - cp -r /gst-build/subprojects/gst-integration-testsuites/ "${CI_PROJECT_DIR}/gst-build/subprojects/gst-integration-testsuites/" - - curl -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" + - curl -L -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" - >- python3 clone_manifest_ref.py --fetch @@ -729,7 +729,7 @@ build cerbero cross win64: GSTREAMER_ROOT_ANDROID: ${CI_PROJECT_DIR}/examples/cerbero-android-universal script: - mkdir -p ${EXAMPLES_HOME}/outputs - - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py + - curl -L -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py - chmod +x clone_manifest_ref.py - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs @@ -906,7 +906,7 @@ documentation: # install the binaries - installer -pkg gstreamer-1.0-devel-*-ios-universal.pkg -target CurrentUserHomeDirectory -verbose - - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py + - curl -L -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py - chmod +x clone_manifest_ref.py - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs From dd69902fbfe4d2fe84c851d05eaf899fa70f89d2 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 26 Mar 2019 20:40:35 +0200 Subject: [PATCH 441/605] Replace docker-in-docker with buildah This avoids the need of using privilledged namespaces and dind as buildah are able to build images unprivilledged. One thing to note is that buildah inside docker is not a supported configuration and not tested in upstream podman, but the possible fallout is still easier to deal with than dind and requiring privileged runners. --- .gitlab-ci.yml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99c1fe7b1d..9198870e1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,18 +25,15 @@ test manifest: - "junit.xml" .base: - image: "docker:stable" + image: 'registry.fedoraproject.org/fedora:31' extends: - '.global_ci_policy' rules: - when: 'manual' allow_failure: true - services: - - docker:dind variables: - # When using dind, it's wise to use the overlayfs driver for - # improved performance. - DOCKER_DRIVER: "overlay2" + STORAGE_DRIVER: 'vfs' + BUILDAH_FORMAT: 'docker' GIT_STRATEGY: fetch script: - export DATE=$(date +"%Y-%m-%d") @@ -44,24 +41,29 @@ test manifest: - export _UID="${IMAGE}:${DATE}-${CI_JOB_ID}" - export LATEST="${IMAGE}:latest" - - docker build --pull --build-arg DEFAULT_BRANCH=${GST_UPSTREAM_BRANCH} -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR} + - dnf install -y buildah runc + # Newer versions of podman/buildah try to set overlayfs mount options when + # using the vfs driver, and this causes errors. + - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf + + - buildah bud --build-arg DEFAULT_BRANCH=${GST_UPSTREAM_BRANCH} -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR} # If we are in a fork, push the image to the reigstry regardless the branch - | if [ "$CI_PROJECT_NAMESPACE" != "gstreamer" ]; then - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker push ${LATEST} + buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + buildah push ${LATEST} fi # Push the images to the upstream registry - | if [ "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then # Tag the image with the git ref - docker image tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} + buildah image tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} # Push the tags - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker push ${_UID}-${CI_COMMIT_REF_NAME} - docker push ${LATEST} + buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + buildah push ${_UID}-${CI_COMMIT_REF_NAME} + buildah push ${LATEST} fi alpine amd64 manifest builder docker: From f3aedd930dc4d298b64b9df500355c0d6718c515 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 15 Apr 2020 19:54:15 +0300 Subject: [PATCH 442/605] build docker image: tag the jobs to use the gstreamer runners docker builds can be big, and other shared runner have smaller storage space which we should avoid filling if can be avoided. The gst tagged runners are a better fit for such builds since their disk storage is about 1-2 terabytes. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9198870e1e..dbd2171e98 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ test manifest: image: 'registry.fedoraproject.org/fedora:31' extends: - '.global_ci_policy' + tags: ['gstreamer'] rules: - when: 'manual' allow_failure: true From 0c315875be667e4d51040738f794b0e30653706e Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 20 Apr 2020 18:36:28 +1000 Subject: [PATCH 443/605] oss-fuzz: update for autotools removal --- fuzzing/build-oss-fuzz.sh | 42 ++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/fuzzing/build-oss-fuzz.sh b/fuzzing/build-oss-fuzz.sh index a06f4facd5..297c322173 100755 --- a/fuzzing/build-oss-fuzz.sh +++ b/fuzzing/build-oss-fuzz.sh @@ -20,6 +20,10 @@ # Prefix where we will temporarily install everything PREFIX=$WORK/prefix mkdir -p $PREFIX +# always try getting the arguments for static compilation/linking +# Fixes GModule not being picked when gstreamer-1.0.pc is looked up by meson +# more or less https://github.com/mesonbuild/meson/pull/6629 +export PKG_CONFIG="`which pkg-config` --static" export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig export PATH=$PREFIX/bin:$PATH @@ -31,14 +35,27 @@ echo "CFLAGS : " $CFLAGS echo "CXXFLAGS : " $CXXFLAGS PLUGIN_DIR=$PREFIX/lib/gstreamer-1.0 +rm -rf $WORK/* + # Switch to work directory cd $WORK # 1) BUILD GLIB AND GSTREAMER # Note: we build glib ourselves so that we get proper malloc/free backtraces -tar xvJf $SRC/glib-2.54.2.tar.xz -cd glib-2.54.2 -./configure --prefix=$PREFIX --enable-static --disable-shared --disable-libmount --with-pcre=internal && make -j$(nproc) && make install +tar xvJf $SRC/glib-2.64.2.tar.xz +cd glib-2.64.2 +# options taken from glib's oss-fuzz build definition +meson \ + --prefix=$PREFIX \ + --libdir=lib \ + --default-library=static \ + -Db_lundef=false \ + -Doss_fuzz=enabled \ + -Dlibmount=disabled \ + -Dinternal_pcre=true \ + _builddir +ninja -C _builddir +ninja -C _builddir install cd .. # Note: We don't use/build orc since it still seems to be problematic @@ -49,11 +66,18 @@ for i in gstreamer gst-plugins-base; do mkdir -p $i cd $i - $SRC/$i/autogen.sh --prefix=$PREFIX --disable-shared --enable-static --disable-examples \ - --disable-gtk-doc --disable-introspection --enable-static-plugins \ - --disable-gst-tracer-hooks --disable-registry - make -j$(nproc) - make install + meson \ + --prefix=$PREFIX \ + --libdir=lib \ + --default-library=static \ + -Db_lundef=false \ + -Ddoc=disabled \ + -Dexamples=disabled \ + -Dintrospection=disabled \ + -Dtracer_hooks=false \ + -Dregistry=false _builddir $SRC/$i + ninja -C _builddir + ninja -C _builddir install cd .. done @@ -77,7 +101,7 @@ PREDEPS_LDFLAGS="-Wl,-Bdynamic -ldl -lm -pthread -lrt -lpthread" # These are the basic .pc dependencies required to build any of the fuzzing targets # That is : glib, gstreamer core and gst-app # The extra target-specific dependencies are to be specified later -COMMON_DEPS="glib-2.0 gstreamer-1.0 gstreamer-app-1.0" +COMMON_DEPS="glib-2.0 gio-2.0 gstreamer-1.0 gstreamer-app-1.0" # For each target, defined the following: # TARGET_DEPS : Extra .pc dependencies for the target (in addition to $COMMON_DEPS) From b36d813f5b2c4588886641afbf6a7c7f9774fffe Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 21 Apr 2020 11:54:04 +0300 Subject: [PATCH 444/605] ci_template: don't use prebuilt binaries for test jobs Previously we were optimizing for cpu time, so we where building gst-build once and then exporting that to be used by the test jobs. However this meant that we where uploading 200mb (previously 600mb) zipped of artifacts and then re-downloading them for each test job. This caused big costs in terms of cloud egress since the runners aren't hosted on the same cloud as the storage/artifacts instance. Instead we are going to be rebuilding gst-build for each test job from now, it also doesn't take more time than the network i/o would of downloading the artifacts, so the impact of rebuilding shouldn't be noticebly. We are also using pinned git refs the modules we rebuild from the manifest, so the binaries should be reproducible for the most part (minus things like .pyc files). Close #68 Part-of: --- .gitlab-ci.yml | 2 - gitlab/ci_template.yml | 137 +++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 82 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbd2171e98..bfad624a0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -173,8 +173,6 @@ build cerbero cross win64 local: image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest' extends: '.test' <<: *local_rules - dependencies: - - 'build fedora x86_64 local' check fedora local: extends: '.test fedora x86_64 local' diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 910999ae80..58c5264d67 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -101,92 +101,75 @@ gst indent: exit 1 fi +.gst_build_template: &gst_build + - echo $MESON_ARGS + + - curl -L -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" + - python3 clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build + - cd gst-build/ + # only copy immediate directories. We don't want to modify .wrap files + - find /gst-build/subprojects/ -mindepth 1 -maxdepth 1 -type d -exec cp -r \{\} subprojects/ \; + - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" + - meson build/ $MESON_ARGS + - ninja -C build/ + - ccache --show-stats + +.gst_build_ccache_vars: + variables: + CCACHE_COMPILERCHECK: 'content' + CCACHE_COMPRESS: 'true' + CCACHE_BASEDIR: '/cache/gstreamer/gst-build' + CCACHE_DIR: '/cache/gstreamer/gst-build/ccache/' + # shared across everything really + CCACHE_MAXSIZE: '10G' + CARGO_HOME: '/cache/gstreamer/cargo' + +.simple_fedora_build: &simple_build >- + ${DEFAULT_MESON_ARGS} + -Dsharp=enabled + -Domx=enabled + -Dgst-omx:target=generic + -Ddoc=enabled + -Drs=disabled + ${MESON_BUILDTYPE_ARGS} + --werror + .build: stage: 'build' extends: - '.global_ci_policy' + - '.gst_build_ccache_vars' needs: - "manifest" # Taking into account the slowest shared runner + time needed to upload the binaries to artifacts # Also need to take into account I/O of pulling docker images and uploading artifacts timeout: '45min' variables: - CCACHE_COMPILERCHECK: "content" - CCACHE_COMPRESS: "true" - CCACHE_BASEDIR: "/cache/gstreamer/gst-build" - CCACHE_DIR: "/cache/gstreamer/gst-build/ccache/" - # shared across everything really - CCACHE_MAXSIZE: "10G" - CARGO_HOME: "/cache/gstreamer/cargo" - MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} --werror" rules: - if: '$CI_PROJECT_NAME != "cerbero"' - script: - - ccache --show-stats - - curl -L -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py - - chmod +x clone_manifest_ref.py - - ./clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build - - cd gst-build/ - # only copy immediate directories. We don't want to modify .wrap files - - find /gst-build/subprojects/ -mindepth 1 -maxdepth 1 -type d -exec cp -r \{\} subprojects/ \; - - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" - - meson build/ $MESON_ARGS - - ninja -C build/ - - ccache --show-stats + *gst_build after_script: - - cd gst-build/ - # Clean the artifacts packages to avoid copying "useless" build products. - - test -d build && find build -name '*.[ao]' -delete - # Clean the .git repos since we won't need them anymore - - rm -rf subprojects/*/.git/ - - rm -rf build/subprojects/*/.git/ - # Clean the big files that are not useful to us to reduce the artifact size - - rm -rf subprojects/FFmpeg/tests - - rm -rf subprojects/FFmpeg/doc - - rm -rf subprojects/openh264/res - - rm -rf subprojects/libxml2-2.9.7/result - - rm -rf subprojects/libxml2-2.9.7/doc - - rm -rf subprojects/libxml2-2.9.7/test - - rm -rf subprojects/libpsl/fuzz - - rm -rf subprojects/glib/gio/tests - - rm -rf subprojects/glib/glib/tests - - rm -rf subprojects/glib/gobject/tests - - rm -rf subprojects/glib/docs/tests - - rm -rf subprojects/glib/tests - - rm -rf subprojects/glib/po - - rm -rf subprojects/glib/docs - - # Clear gst-intergrations-testsuites report as it contains raw media - # files which is only needed for validate jobs. When needed we copy the - # cached repo in the docker image and fetch. - - rm -rf subprojects/gst-integration-testsuites + - mv gst-build/build/meson-logs/ meson-logs + # Cleanup everything else to reduce the size + # of the docker volume we leave behind + - rm -rf gst-build artifacts: expire_in: "7 days" when: "always" paths: - - 'gst-build/build/meson-logs/' - # Needed by the integration tests + - 'meson-logs/' - 'manifest.xml' .build fedora x86_64: extends: '.build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: *simple_build rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-omx|gst-plugins-rs)$/' -build fedora x86_64: - extends: '.build fedora x86_64' - artifacts: - expire_in: '5 days' - when: always - paths: - - "manifest.xml" - - "gst-build/" - build nodebug fedora x86_64: extends: '.build' image: $FEDORA_IMAGE @@ -221,7 +204,12 @@ build clang fedora x86_64: stage: 'test' extends: - '.global_ci_policy' + - '.gst_build_ccache_vars' + needs: + - "manifest" variables: + MESON_ARGS: *simple_build + # Disable colored output to avoid weird rendering issues GST_DEBUG_NO_COLOR: "true" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" @@ -231,7 +219,8 @@ build clang fedora x86_64: rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-examples|gst-docs)$/' script: - - cd gst-build/ + - *gst_build + - echo "-> Running ${TEST_SUITE}" - > ./gst-uninstalled.py @@ -252,6 +241,7 @@ build clang fedora x86_64: paths: - 'gst-build/build/meson-logs/' - 'validate-logs' + - 'manifest.xml' reports: junit: - "validate-logs/*.xml" @@ -260,8 +250,6 @@ build clang fedora x86_64: image: $FEDORA_IMAGE extends: '.test' tags: ['gstreamer'] - needs: - - 'build fedora x86_64' check fedora: extends: '.test fedora x86_64' @@ -278,18 +266,6 @@ integration testsuites fedora: TEST_SUITE: "validate ges" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-omx|gst-plugins-rs)$/' - before_script: - # Avoid cloning the repo since it contains media files, - # instead copy it from the cache and do a git fetch - # We remove this copy first since it had its mediafiles stripped - - cp -r /gst-build/subprojects/gst-integration-testsuites/ "${CI_PROJECT_DIR}/gst-build/subprojects/gst-integration-testsuites/" - - curl -L -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" - - >- - python3 clone_manifest_ref.py - --fetch - --manifest manifest.xml - --project gst-integration-testsuites - --destination gst-build/subprojects/gst-integration-testsuites gstreamer-full: extends: 'build static fedora x86_64' @@ -505,8 +481,6 @@ valgrind ges: - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && meson build $env:MESON_ARGS && ninja -C build" - # FIXME: extract builddir for tests - #after_script: # XXX: Re-enable when uploading stops timing out #artifacts: # expire_in: "7 days" @@ -869,15 +843,18 @@ documentation: image: $FEDORA_IMAGE extends: - '.global_ci_policy' + - '.gst_build_ccache_vars' rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-sharp|gst-integration-testsuites|gst-omx|gst-plugins-rs|gst-python)$/' needs: - - "build fedora x86_64" + - "manifest" stage: integrate - + variables: + MESON_ARGS: *simple_build script: - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git - - cd gst-build/ + - *gst_build + - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings - cd - - mv gst-build/build/subprojects/gst-docs/GStreamer-doc/html documentation/ @@ -944,7 +921,7 @@ cerbero cross-ios universal examples: - "cerbero cross-ios universal" build gst-omx zynq fedora x86_64: - extends: 'build fedora x86_64' + extends: '.build fedora x86_64' variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" rules: @@ -953,7 +930,7 @@ build gst-omx zynq fedora x86_64: - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2019.2 ${CI_PROJECT_DIR}/vcu-omx-il build gst-omx tizonia fedora x86_64: - extends: 'build fedora x86_64' + extends: '.build fedora x86_64' variables: MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}:${CI_PROJECT_DIR}/tizonia-install/lib64/pkgconfig/ From f546834e6cb3f4d601382ff6f3412d41c0c67aea Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 22 Apr 2020 11:00:02 +0300 Subject: [PATCH 445/605] ci_template: enable the windows x86 and msys native jobs again Part-of: --- gitlab/ci_template.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 58c5264d67..0b73d1e3c6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -495,19 +495,11 @@ build vs2017 amd64: build vs2017 x86: extends: '.build windows' - # Disabled until the container image can be updated - rules: - - allow_failure: true - when: 'manual' variables: ARCH: 'x86' build msys2 : extends: '.build windows' - # Disabled until the container image can be updated - rules: - - allow_failure: true - when: 'manual' timeout: '60min' script: # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or From 2f504610b3a468ebc480197a3d2c3cfdb2b46453 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 23 Apr 2020 16:55:01 +0300 Subject: [PATCH 446/605] ci_tepmlate: disable the msys job again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its still broken and keeps haning randomly... 😢️ Part-of: --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0b73d1e3c6..d6c27c1e9f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -501,6 +501,10 @@ build vs2017 x86: build msys2 : extends: '.build windows' timeout: '60min' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' + allow_failure: true + when: 'manual' script: # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or # from a previous job due to some gitlab bug or implicit behavior? From 38719d0aefd152c3289deacea5ec2521ecdf5b79 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 24 Apr 2020 14:16:23 +1000 Subject: [PATCH 447/605] docker: remove unused Dockerfiles Part-of: --- docker/build-base-images/.dockerignore | 2 -- docker/build-base-images/Dockerfile-archlinux | 25 ------------------- docker/build-base-images/build.sh | 8 ------ docker/runtime-images/.dockerignore | 2 -- docker/runtime-images/Dockerfile-allure | 9 ------- docker/runtime-images/build.sh | 9 ------- 6 files changed, 55 deletions(-) delete mode 100644 docker/build-base-images/.dockerignore delete mode 100644 docker/build-base-images/Dockerfile-archlinux delete mode 100755 docker/build-base-images/build.sh delete mode 100644 docker/runtime-images/.dockerignore delete mode 100644 docker/runtime-images/Dockerfile-allure delete mode 100755 docker/runtime-images/build.sh diff --git a/docker/build-base-images/.dockerignore b/docker/build-base-images/.dockerignore deleted file mode 100644 index 131a0fd029..0000000000 --- a/docker/build-base-images/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -build.sh - diff --git a/docker/build-base-images/Dockerfile-archlinux b/docker/build-base-images/Dockerfile-archlinux deleted file mode 100644 index 6ffc188e27..0000000000 --- a/docker/build-base-images/Dockerfile-archlinux +++ /dev/null @@ -1,25 +0,0 @@ -FROM base/devel:latest - -ENV LANG=en_US.UTF-8 -ENV GST_BUILD_PATH="/gst-build/" -ENV GST_VALIDATE_OUTPUT="/validate-output" -ENV GST_VALIDATE_TESTSUITES="$GST_VALIDATE_OUTPUT/gst-integration-testsuites" -ENV GST_BUILD_MESON_ARGS="-Dpython=disabled -Dlibav=enabled -Dugly=enabled -Dbad=enabled -Ddevtools=enabled -Dges=enabled -Drtsp_server=enabled -Domx=disabled -Dvaapi=disabled -Dsharp=enabled" -ENV PATH="/usr/lib/ccache/bin/:${PATH}" - -RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen - -RUN pacman -Sy --noconfirm python3 libxml2 libxslt cmake libyaml git make nodejs ninja clang llvm flex python-gobject gstreamer gst-plugins-base gst-plugins-bad gst-plugins-ugly gst-libav gst-editing-services python-pip npm json-glib gobject-introspection wget mono unzip glib2-docs xorg-xset xorg-server-xvfb ccache -RUN pip install --upgrade git+https://github.com/hotdoc/hotdoc.git@gst git+https://github.com/hotdoc/hotdoc_c_extension.git@gst meson - -RUN mkdir $GST_VALIDATE_OUTPUT && \ - git clone https://anongit.freedesktop.org/git/gstreamer/gst-integration-testsuites.git $GST_VALIDATE_TESTSUITES && \ - cd $GST_VALIDATE_TESTSUITES/medias/ && \ - python get_files.py defaults/ - -# get gst-build and make all subprojects available -RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \ - cd $GST_BUILD_PATH && \ - meson build/ $GST_BUILD_MESON_ARGS && rm -Rf build/ - -RUN rm -Rf /var/cache/pacman/ \ No newline at end of file diff --git a/docker/build-base-images/build.sh b/docker/build-base-images/build.sh deleted file mode 100755 index c8214c5b5a..0000000000 --- a/docker/build-base-images/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -echo Building gstreamer/build-base-fedora:latest - -docker build -t gstreamer/build-base-fedora:latest . -f Dockerfile-fedora - -echo Building gstreamer/build-base-ubuntu:latest - -docker build -t gstreamer/build-base-ubuntu:latest . -f Dockerfile-ubuntu diff --git a/docker/runtime-images/.dockerignore b/docker/runtime-images/.dockerignore deleted file mode 100644 index 131a0fd029..0000000000 --- a/docker/runtime-images/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -build.sh - diff --git a/docker/runtime-images/Dockerfile-allure b/docker/runtime-images/Dockerfile-allure deleted file mode 100644 index 04818c2ee5..0000000000 --- a/docker/runtime-images/Dockerfile-allure +++ /dev/null @@ -1,9 +0,0 @@ -FROM anapsix/alpine-java:latest - -# Get the allure test report framework -RUN export ALLURE_VERSION=2.7.0 && \ - wget "https://bintray.com/qameta/generic/download_file?file_path=io%2Fqameta%2Fallure%2Fallure%2F2.7.0%2Fallure-${ALLURE_VERSION}.zip" -O allure.zip - unzip allure.zip && \ - mv allure-${ALLURE_VERSION} /opt/allure && \ - chmod +x /opt/allure/bin/allure && \ - rm allure.zip \ No newline at end of file diff --git a/docker/runtime-images/build.sh b/docker/runtime-images/build.sh deleted file mode 100755 index 7cdff2b090..0000000000 --- a/docker/runtime-images/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -echo Building gstreamer/base-fedora:latest - -docker build -t gstreamer/base-fedora:latest . -f Dockerfile-fedora - -echo Building gstreamer/base-ubuntu:latest - -docker build -t gstreamer/base-ubuntu:latest . -f Dockerfile-ubuntu - From edac99f4418ae660e12cef13afd6b364bb365c60 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 23 Apr 2020 18:49:07 +0300 Subject: [PATCH 448/605] windows/docker: allow to specify the branch to pull for gst-build Similar to !232 Part-of: --- docker/windows/Dockerfile | 2 ++ docker/windows/prepare_gst_env.ps1 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 7c86152f3b..cb6c89f6c6 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -6,6 +6,8 @@ FROM 'mcr.microsoft.com/windows/servercore:ltsc2019' SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] ENV ErrorActionPreference='Stop' +ARG DEFAULT_BRANCH="master" + COPY install_choco.ps1 C:\ RUN C:\install_choco.ps1 diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1 index a930c5dc19..61b30e1f29 100644 --- a/docker/windows/prepare_gst_env.ps1 +++ b/docker/windows/prepare_gst_env.ps1 @@ -9,7 +9,7 @@ git config --global user.name "Gstbuild Runner" (New-Object System.Net.WebClient).DownloadString("https://github.com") >$null # Download gst-build and all its subprojects -git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build # download the subprojects to try and cache them meson subprojects download --sourcedir C:\gst-build From 6576f2f7b77db42a0543d6fa5287f1f13bf9be21 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 27 Apr 2020 15:39:45 +0300 Subject: [PATCH 449/605] ci_template: try harder to cleanup after build jobs Let's try extra hard to clean after ourselves, cause gitlab-runner occasionally decides to reuse existing, polluted, volumes... so much for clean build environments... Part-of: --- gitlab/ci_template.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d6c27c1e9f..31741edb5a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -103,6 +103,12 @@ gst indent: .gst_build_template: &gst_build - echo $MESON_ARGS + # Sometimes, gitlab-runner want to reuse + # existing docker volumes without cleaning them up... + # Make sure the docker volume is clean + - rm -rf gst-build || true + - rm -rf meson-logs || true + - rm -rf validate-logs || true - curl -L -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py" - python3 clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build @@ -235,11 +241,16 @@ build clang fedora x86_64: -l "${CI_PROJECT_DIR}/validate-logs/" --xunit-file "${CI_PROJECT_DIR}/validate-logs/xunit.xml" ${EXTRA_VALIDATE_ARGS} + after_script: + - mv gst-build/build/meson-logs/ meson-logs + # Cleanup everything else to reduce the size + # of the docker volume we leave behind + - rm -rf gst-build artifacts: expire_in: '14 days' when: always paths: - - 'gst-build/build/meson-logs/' + - 'meson-logs/' - 'validate-logs' - 'manifest.xml' reports: From b8fa6db8733e25d35d09f53cdb0a0fd66384e894 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 27 Apr 2020 11:10:26 +0300 Subject: [PATCH 450/605] docker/windows: add script to build the image with the correct branch and tag also move the default_branch arg later in the dockerfile to avoid invalidating previous layers Part-of: --- docker/windows/.dockerignore | 1 + docker/windows/Dockerfile | 4 ++-- docker/windows/build_image.ps1 | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 docker/windows/.dockerignore create mode 100644 docker/windows/build_image.ps1 diff --git a/docker/windows/.dockerignore b/docker/windows/.dockerignore new file mode 100644 index 0000000000..76bab46af2 --- /dev/null +++ b/docker/windows/.dockerignore @@ -0,0 +1 @@ +build_image.ps1 diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index cb6c89f6c6..899fcfa33e 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -6,13 +6,13 @@ FROM 'mcr.microsoft.com/windows/servercore:ltsc2019' SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"] ENV ErrorActionPreference='Stop' -ARG DEFAULT_BRANCH="master" - COPY install_choco.ps1 C:\ RUN C:\install_choco.ps1 COPY install_toolchain.ps1 C:\ RUN C:\install_toolchain.ps1 +ARG DEFAULT_BRANCH="master" + COPY prepare_gst_env.ps1 C:\ RUN C:\prepare_gst_env.ps1 diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 new file mode 100644 index 0000000000..9b1e26b856 --- /dev/null +++ b/docker/windows/build_image.ps1 @@ -0,0 +1,5 @@ +$env:DEFAULT_BRANCH='master' +$env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v10-$env:DEFAULT_BRANCH" +echo "Building $env:tag" +docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . +# docker push $tag From c23bdae050a2188f5cba48407443cf02a3d18d9f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 8 May 2020 18:22:34 +1000 Subject: [PATCH 451/605] ci: run CI for the new vulkan android example HACK: pointing to user repo Part-of: --- docker/android/prepare.sh | 3 +++ gitlab/ci_template.yml | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh index 3f3632cd5c..e18ce2b4c6 100644 --- a/docker/android/prepare.sh +++ b/docker/android/prepare.sh @@ -44,6 +44,9 @@ git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-exam chmod +x /android/sources/gst-examples/playback/player/android/gradlew /android/sources/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies +chmod +x /android/sources/gst-examples/vulkan/android/gradlew +/android/sources/gst-examples/vulkan/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-examples/vulkan/android dependencies --refresh-dependencies + git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs chmod +x /android/sources/gst-docs/examples/tutorials/android/gradlew /android/sources/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 31741edb5a..fb61d4f5e3 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -721,11 +721,16 @@ build cerbero cross win64: - mkdir ${GSTREAMER_ROOT_ANDROID} - tar -C ${GSTREAMER_ROOT_ANDROID} -xf gstreamer-1.0-android-universal-*.tar.bz2 - # gst-examples + # gst-examples - player - chmod +x ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew - ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/playback/player/android assembleDebug - cp ${EXAMPLES_HOME}/gst-examples/playback/player/android/app/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ + # gst-examples - vulkan + - chmod +x ${EXAMPLES_HOME}/gst-examples/vulkan/android/gradlew + - ${EXAMPLES_HOME}/gst-examples/vulkan/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/vulkan/android assembleDebug + - cp ${EXAMPLES_HOME}/gst-examples/vulkan/android/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ + # gst-docs android tutorials - chmod +x ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew - ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android assembleDebug From 95b35ceaf162fd51d5c81baddea2d27f65a194c7 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 8 May 2020 19:39:09 +1000 Subject: [PATCH 452/605] android: update to gradle 5.6.4 without --no-search-upwards Part-of: --- docker/android/prepare.sh | 6 +++--- gitlab/ci_template.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/android/prepare.sh b/docker/android/prepare.sh index e18ce2b4c6..953bc5ec5a 100644 --- a/docker/android/prepare.sh +++ b/docker/android/prepare.sh @@ -42,14 +42,14 @@ ls $GSTREAMER_ROOT_ANDROID git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-examples.git /android/sources/gst-examples chmod +x /android/sources/gst-examples/playback/player/android/gradlew -/android/sources/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies +/android/sources/gst-examples/playback/player/android/gradlew --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies chmod +x /android/sources/gst-examples/vulkan/android/gradlew -/android/sources/gst-examples/vulkan/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-examples/vulkan/android dependencies --refresh-dependencies +/android/sources/gst-examples/vulkan/android/gradlew --no-daemon --project-dir /android/sources/gst-examples/vulkan/android dependencies --refresh-dependencies git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs chmod +x /android/sources/gst-docs/examples/tutorials/android/gradlew -/android/sources/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies +/android/sources/gst-docs/examples/tutorials/android/gradlew --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies unset GSTREAMER_ROOT_ANDROID diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index fb61d4f5e3..11c1412500 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -723,17 +723,17 @@ build cerbero cross win64: # gst-examples - player - chmod +x ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew - - ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/playback/player/android assembleDebug + - ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/playback/player/android assembleDebug - cp ${EXAMPLES_HOME}/gst-examples/playback/player/android/app/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ # gst-examples - vulkan - chmod +x ${EXAMPLES_HOME}/gst-examples/vulkan/android/gradlew - - ${EXAMPLES_HOME}/gst-examples/vulkan/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/vulkan/android assembleDebug + - ${EXAMPLES_HOME}/gst-examples/vulkan/android/gradlew --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/vulkan/android assembleDebug - cp ${EXAMPLES_HOME}/gst-examples/vulkan/android/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ # gst-docs android tutorials - chmod +x ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew - - ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android assembleDebug + - ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew --no-daemon --project-dir ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android assembleDebug - cp ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/android-tutorial-*/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/ after_script: - rm -rf ${GSTREAMER_ROOT_ANDROID} From 0fb6955c5e2fbec2687a2820547892fba0d17011 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 14 May 2020 11:40:50 +0300 Subject: [PATCH 453/605] .gitlab-ci.yml: fix typo in buildah images --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bfad624a0a..8854c1930f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,7 +60,7 @@ test manifest: - | if [ "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then # Tag the image with the git ref - buildah image tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} + buildah images tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} # Push the tags buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY buildah push ${_UID}-${CI_COMMIT_REF_NAME} From 61c1680b076e7a182740720d15681eb0f8fbf3c6 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 13 May 2020 15:10:40 +0100 Subject: [PATCH 454/605] docker/fedora: Remove WPEBackend-FDO It should now be built as a gst-build subproject. The version in Fedora is not suitable for CI tests. Part-of: --- docker/fedora/prepare.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index a52672ad7d..0ba80e1595 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -107,8 +107,6 @@ dnf install -y \ mesa-vulkan-drivers \ wpewebkit \ wpewebkit-devel \ - wpebackend-fdo \ - wpebackend-fdo-devel \ xorg-x11-server-utils \ xorg-x11-server-Xvfb From 822706e4733497ed73475710a9d54ce8019486dd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 14 May 2020 12:31:07 +0300 Subject: [PATCH 455/605] .gitlab-ci.yml: fix buildah syntax for tagging images --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8854c1930f..92dca4f5f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,7 +60,7 @@ test manifest: - | if [ "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then # Tag the image with the git ref - buildah images tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} + buildah tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} # Push the tags buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY buildah push ${_UID}-${CI_COMMIT_REF_NAME} From 0928a8912594af730cbd01f7a2e856a9f7bbd6a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Cerveau?= Date: Wed, 20 May 2020 17:15:44 +0200 Subject: [PATCH 456/605] gst-omx: update vcu-omx-il public release branch Use release 2020.01 for the xilinx zynq target from https://github.com/Xilinx/vcu-omx-il Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 11c1412500..b545715099 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -939,7 +939,7 @@ build gst-omx zynq fedora x86_64: rules: - if: '$CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/' before_script: - - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2019.2 ${CI_PROJECT_DIR}/vcu-omx-il + - git clone https://github.com/Xilinx/vcu-omx-il.git --branch=release-2020.1 ${CI_PROJECT_DIR}/vcu-omx-il build gst-omx tizonia fedora x86_64: extends: '.build fedora x86_64' From 92b78216d49f87735a59957d651dc7f1709f96e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Fri, 29 May 2020 13:05:02 +0100 Subject: [PATCH 457/605] ci_template: update cerbero build-deps for gst-validate -> gst-devtools-1.0 rename https://gitlab.freedesktop.org/gstreamer/gst-devtools/-/merge_requests/199 https://gitlab.freedesktop.org/gstreamer/cerbero/-/merge_requests/490 Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b545715099..a4adedf63c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -574,7 +574,7 @@ build msys2 : - $CERBERO $CERBERO_ARGS build-deps --offline gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 - gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice + gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && mkdir -p ${CERBERO_HOME}/dist/${ARCH} && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH} From 929a611889e22ec5e796f482e0b7db60f38b1a89 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 26 May 2020 17:38:45 -0400 Subject: [PATCH 458/605] ci: Check valgrind for all modules on gst-devtools Now that we have [`gst-tester`] all module are starting to be tested with `gst-validate` (core and -base already have validate based unit tests) so we need to ensure that changes there do not break those modules. [gst-tester]: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/461 Part-of: --- gitlab/ci_template.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a4adedf63c..f6eaadd578 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -317,14 +317,14 @@ valgrind core: variables: TEST_SUITE: "check.gstreamer\\..*" rules: - - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-ci)$/' + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-ci|gst-devtools)$/' valgrind base: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-base\\..*" rules: - - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-ci)$/' + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-ci|gst-devtools)$/' valgrind good: extends: '.valgrind fedora x86_64' @@ -333,21 +333,21 @@ valgrind good: # take longer time due to splitmux unit test TIMEOUT_FACTOR: "4" rules: - - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci)$/' + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci|gst-devtools)$/' valgrind ugly: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-ugly\\..*" rules: - - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci)$/' + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci|gst-devtools)$/' valgrind bad: extends: '.valgrind fedora x86_64' variables: TEST_SUITE: "check.gst-plugins-bad\\..*" rules: - - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci)$/' + - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci|gst-devtools)$/' valgrind ges: extends: '.valgrind fedora x86_64' From dbe39079eefd9e38641aae3462e244b92669b9fd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 29 May 2020 14:30:26 +0300 Subject: [PATCH 459/605] ci_template: windows: clone the gst-build specified in the manifest In all our other builds, we are using the clone_manifest_ref script to fetch the revision of gst-build that we discover in the manifest. For the windows job this was missed it seems, but didn't cause any issues till now cause it only affected the gst-build MRs. Part-of: --- gitlab/ci_template.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f6eaadd578..617c96a439 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -478,8 +478,10 @@ valgrind ges: # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6 - $ErrorActionPreference = "Stop" - - git clone -b $GST_UPSTREAM_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR\gst-build + - Invoke-WebRequest -Uri "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/$env:GST_UPSTREAM_BRANCH/gitlab/clone_manifest_ref.py" -OutFile $env:CI_PROJECT_DIR/clone_manifest_ref.py + - python clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination $env:CI_PROJECT_DIR/gst-build - cd $env:CI_PROJECT_DIR/gst-build + - cp -r C:/subprojects/* subprojects/ # Run the git-update script and feed it the manifest to setup the environment - cd $env:CI_PROJECT_DIR/gst-build; python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml @@ -533,8 +535,10 @@ build msys2 : - C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" - - git clone -b $GST_UPSTREAM_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR\gst-build - - cd $env:CI_PROJECT_DIR\gst-build + - Invoke-WebRequest -Uri "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/$env:GST_UPSTREAM_BRANCH/gitlab/clone_manifest_ref.py" -OutFile $env:CI_PROJECT_DIR/clone_manifest_ref.py + - python clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination $env:CI_PROJECT_DIR/gst-build + - cd $env:CI_PROJECT_DIR/gst-build + - cp -r C:\subprojects\* subprojects\ # Run the git-update script and feed it the manifest to setup the environment From 3839f8ab87b6c15b628d08cb8ca5e6bfa35d08d7 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 3 Jun 2020 22:33:50 -0400 Subject: [PATCH 460/605] gitlab: Enable docs only when building the doc --- gitlab/ci_template.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 617c96a439..c098a729eb 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -135,7 +135,7 @@ gst indent: -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic - -Ddoc=enabled + -Ddoc=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror @@ -867,6 +867,7 @@ documentation: stage: integrate variables: MESON_ARGS: *simple_build + MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" script: - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git - *gst_build From 54553208697512fd23c5325b5b3bf70d02312942 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Thu, 4 Jun 2020 17:29:49 +0200 Subject: [PATCH 461/605] ci_template: pin hotdoc to pre gst cache format change --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c098a729eb..90f1e20972 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -869,7 +869,7 @@ documentation: MESON_ARGS: *simple_build MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" script: - - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git + - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git@e363817ccdfd8150674a370ade2df9267074a694 - *gst_build - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings From 1f07629c3c8a3ae5f37c7d8d9e38b101ecb58eb6 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Thu, 4 Jun 2020 17:56:00 +0200 Subject: [PATCH 462/605] ci_template: pin hotdoc to pre fundamental documentation --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 90f1e20972..1c22c13592 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -869,7 +869,7 @@ documentation: MESON_ARGS: *simple_build MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" script: - - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git@e363817ccdfd8150674a370ade2df9267074a694 + - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git@a8bb7483d473885b9dbb84eb34252f425d43d3ff - *gst_build - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings From 99830c2beeb5fc00c625693e8d7625be5db6d743 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 4 Jun 2020 10:48:54 -0400 Subject: [PATCH 463/605] Revert "docker: fedora: update to Fedora 31" This is unused yet because it breaks some things This reverts commit d016fa74531452c20d76f15d4fbfb31476837766. Part-of: --- docker/fedora/Dockerfile | 2 +- docker/fedora/prepare.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index 624cfcdcb8..82abdd3a40 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:31 +FROM registry.fedoraproject.org/fedora:30 ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 0ba80e1595..55f42d0e2b 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -1,6 +1,6 @@ set -eux -dnf install -y git-core ninja-build dnf-plugins-core python3-pip +dnf install -y git-core ninja-build dnf-plugins-core # Configure git for various usage git config --global user.email "gst-build@gstreamer.net" @@ -17,6 +17,7 @@ dnf upgrade -y dnf install -y \ aalib-devel \ aom \ + aom-extra-tools \ bat \ libaom \ libaom-devel \ @@ -40,7 +41,6 @@ dnf install -y \ graphene-devel \ gsl \ gsl-devel \ - faac-devel \ ffmpeg \ ffmpeg-libs \ ffmpeg-devel \ @@ -179,6 +179,7 @@ dnf builddep -y gstreamer1 \ gstreamer1-plugins-good-extras \ gstreamer1-plugins-ugly \ gstreamer1-plugins-ugly-free \ + gstreamer1-plugins-bad-nonfree \ gstreamer1-plugins-bad-free \ gstreamer1-plugins-bad-free-extras \ gstreamer1-plugins-bad-freeworld \ From 3e7e6615b6ee58eb54347f93d971cfc24124218a Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 4 Jun 2020 10:17:32 -0400 Subject: [PATCH 464/605] gitlab: Ensure devhelp files are installed Part-of: --- docker/fedora/prepare.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 55f42d0e2b..b125289099 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -1,5 +1,9 @@ set -eux +# Fedora base image disable installing documentation files. See https://pagure.io/atomic-wg/issue/308 +# We need them to cleanly build our doc. +sed -i "s/tsflags=nodocs//g" /etc/dnf/dnf.conf + dnf install -y git-core ninja-build dnf-plugins-core # Configure git for various usage From c83578dcd2fdf328de1ed0e1e85a608fa839bcf2 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 4 Jun 2020 11:04:53 -0400 Subject: [PATCH 465/605] Revert "docker: install intel-mediasdk on Fedora image" This is not available on F30 This reverts commit 9405f9245b1063896f1c02a3b0468a1ecc870193. Part-of: --- docker/fedora/prepare.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index b125289099..49d931a457 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -55,7 +55,6 @@ dnf install -y \ procps-ng \ patch \ redhat-rpm-config \ - intel-mediasdk-devel \ json-glib \ json-glib-devel \ libnice \ From d72551ddc4fe147906b550b11ce63b4fd6168f51 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 4 Jun 2020 12:34:15 -0400 Subject: [PATCH 466/605] gitlab: Update hotdoc to latest release --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1c22c13592..df31df3610 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -869,7 +869,7 @@ documentation: MESON_ARGS: *simple_build MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" script: - - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc.git@a8bb7483d473885b9dbb84eb34252f425d43d3ff + - pip3 install --upgrade hotdoc==0.11.0 - *gst_build - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings From 3a620ac384c00ec214feda734a256c8d6fda782d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 4 Jun 2020 12:48:31 -0400 Subject: [PATCH 467/605] ci: Update fedora image --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index df31df3610..a283960e2a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -9,7 +9,7 @@ stages: variables: ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-04-06-2193279-master' CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-04-06-2193307-master' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-02-24-1712085-master' + FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-06-04-2958178-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v10' From a36ac34d49551976c7b44e193a3d221cfc065d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 4 Jun 2020 18:44:51 +0100 Subject: [PATCH 468/605] docker: fedora: install nasm properly Add it to install list not debuginfo-install list. Should make openh264 subproject work in gst-build builds. Fixes #76 Part-of: --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 49d931a457..dc5afc1b74 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -68,6 +68,7 @@ dnf install -y \ llvm-devel \ log4c-devel \ make \ + nasm \ neon \ neon-devel \ nunit \ @@ -128,7 +129,6 @@ dnf debuginfo-install -y gtk3 \ glib-networking \ libcurl \ libsoup \ - nasm \ nss \ nss-softokn \ nss-softokn-freebl \ From e4ba9269550764a53a11338f8706603d3934649d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 4 Jun 2020 17:06:38 -0400 Subject: [PATCH 469/605] gitlab: Enforce documentation cache updates --- gitlab/ci_template.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a283960e2a..ad87c3e108 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -868,17 +868,44 @@ documentation: variables: MESON_ARGS: *simple_build MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" + CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - pip3 install --upgrade hotdoc==0.11.0 - *gst_build + - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches + - | + python3 -c "import os, subprocess, sys + subprojects = [os.environ['CI_PROJECT_NAME']] + if subprojects[0] == 'gst-ci': + subprojects = ['gstreamer', 'gst-plugins-ugly', 'gst-omx', 'gst-plugins-good', 'gst-plugins-base', 'gstreamer-vaapi', 'gst-libav', 'gst-editing-services', 'gst-rtsp-server', 'gst-plugins-bad',] + + diffsdir = '../plugins-cache-diffs' + os.makedirs(diffsdir, exist_ok=True) + res = 0 + for subproject in subprojects: + cwd = os.path.join('subprojects', subproject) + try: + subprocess.check_call(['git', 'diff', '--quiet'], cwd=cwd) + except subprocess.CalledProcessError: + diffname = os.path.join(diffsdir, '%s_cache.diff' % subproject) + res += 1 + with open(diffname, 'w') as diff: + subprocess.check_call(['git', 'diff'], stdout=diff, cwd=cwd) + print('\033[91mYou have a diff in the %s documentation cache. Please update with:\033[0m' % subproject) + print(' $ curl %s/%s | git apply -' % (os.environ['CI_ARTIFACTS_URL'], diffname.replace('../', ''))) + if res != 0: + print('(note that it might take a few minutes for artefacts to be available on the server)\n') + sys.exit(res)" - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings - cd - - mv gst-build/build/subprojects/gst-docs/GStreamer-doc/html documentation/ artifacts: + when: always paths: - documentation/ + - plugins-cache-diffs/ # # Build an iOS App using the iOS binaries From 869fa72595ffd8a92dc8795b6202acb04ea85eed Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 2 Jun 2020 11:33:47 +0300 Subject: [PATCH 470/605] docker/windows: update msys snapshot Part-of: --- docker/windows/build_image.ps1 | 3 ++- docker/windows/install_toolchain.ps1 | 2 +- gitlab/ci_template.yml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 9b1e26b856..490ecc6813 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,5 +1,6 @@ $env:DEFAULT_BRANCH='master' -$env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v10-$env:DEFAULT_BRANCH" +$env:VERSION='v11' +$env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" echo "Building $env:tag" docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . # docker push $tag diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index 66a4df73b0..cd6f147981 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -1,7 +1,7 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' -$msys2_url = 'https://netcologne.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20190524.tar.xz' +$msys2_url = 'https://github.com/msys2/msys2-installer/releases/download/2020-05-22/msys2-base-x86_64-20200522.tar.xz' $msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' Write-Host "Installing VisualStudio" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ad87c3e108..6eee99171f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -12,7 +12,7 @@ variables: FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-06-04-2958178-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v10' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v11-master' # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'master' From f0eea49641a0be2473e5dd8acc4e589623b29299 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 9 Jun 2020 15:40:45 -0400 Subject: [PATCH 471/605] CI: Combine deps and build for cerbero builds This change combine the deps and the build step when running cerbero CI in cerbero repository itself. This removes avoidable usage of large artifacts. Part-of: --- .gitlab-ci.yml | 8 +-- gitlab/ci_template.yml | 151 +++++++++++++---------------------------- 2 files changed, 50 insertions(+), 109 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92dca4f5f2..a011ea5463 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -147,22 +147,22 @@ build fedora x86_64 local: <<: *local_template build cerbero fedora x86_64 local: - extends: '.cerbero fedora x86_64' + extends: 'build cerbero fedora x86_64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" <<: *local_template build cerbero cross-android universal local: - extends: '.cerbero cross-android universal' + extends: 'build cerbero cross-android universal' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" <<: *local_template build cerbero cross win32 local: - extends: '.cerbero cross win32' + extends: 'build cerbero cross win32' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" <<: *local_template build cerbero cross win64 local: - extends: '.cerbero cross win64' + extends: 'build cerbero cross win64' image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" <<: *local_template diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6eee99171f..9c5edbfe68 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -363,6 +363,15 @@ valgrind ges: # ARCH: The cerbero _ (used in cache key) # # Produces runtime and devel tarball packages for linux/android or .pkg for macos +.cerbero template: &cerbero_package_and_check + - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o $(pwd) gstreamer-1.0 + # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. + - |- + [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version + - |- + [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX + - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test + .cerbero: stage: "build" image: $CERBERO_IMAGE @@ -396,7 +405,6 @@ valgrind ges: CERBERO_OVERRIDDEN_DIST_DIR: "" # location where cerbero is cached on the host CERBERO_HOST_DIR: "/cerbero/" - CERBERO_DEPS: "${CERBERO_HOME}/cerbero-deps.tar.xz" before_script: # FIXME Wrong namespace # Workaround build-tools having hardcoded internal path @@ -406,12 +414,6 @@ valgrind ges: - mkdir -p ../../${CI_PROJECT_NAMESPACE} - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero - rsync -aH ${CERBERO_HOST_DIR} . - - | - if test -f ${CERBERO_DEPS} ; then - echo "Extracting xz deps tarball" - export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible - time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf ${CERBERO_DEPS} - fi - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc - ./cerbero-uninstalled --self-update manifest.xml @@ -420,19 +422,13 @@ valgrind ges: - $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - - test -f ${CERBERO_DEPS} || $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} + - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" && test -d ${CERBERO_HOME}/dist/${ARCH} && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR} && rsync -aH ${CERBERO_HOME}/dist/${ARCH}/ ${CERBERO_OVERRIDDEN_DIST_DIR} - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only - - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o $(pwd) gstreamer-1.0 - # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. - - |- - [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version - - |- - [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX - - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test + - *cerbero_package_and_check cache: key: "${CI_JOB_NAME}" paths: @@ -569,8 +565,8 @@ build msys2 : extends: .cerbero rules: - if: '$CI_PROJECT_NAME == "cerbero"' - stage: "pre-build" script: + - test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats - $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 @@ -585,15 +581,7 @@ build msys2 : - $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH} - test "x${CERBERO_PRIVATE_SSH_KEY}" = "x" || $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH} - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" - expire_in: '10 days' - when: 'always' - paths: - - "manifest.xml" - - "${CERBERO_HOME}/logs" - - "${CERBERO_HOME}/cerbero-deps.log" - - "${CERBERO_DEPS}" + - *cerbero_package_and_check # # Cerbero Linux X86_64 build @@ -604,34 +592,20 @@ cerbero deps fedora x86_64: CONFIG: "linux.config" ARCH: "linux_x86_64" -.cerbero fedora x86_64: - extends: '.cerbero' - variables: - CONFIG: "linux.config" - -cerbero fedora x86_64: - extends: '.cerbero fedora x86_64' - needs: - - "cerbero deps fedora x86_64" - build cerbero fedora x86_64: - extends: '.cerbero fedora x86_64' + extends: '.cerbero' rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + variables: + CONFIG: "linux.config" # # Cerbero Android Universal build # -cerbero deps cross-android universal: - extends: '.cerbero deps' +.cerbero cross-android universal: variables: CONFIG: "cross-android-universal.cbc" ARCH: "android_universal" - -.cerbero cross-android universal: - extends: '.cerbero' - variables: - CONFIG: "cross-android-universal.cbc" artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" expire_in: '5 days' @@ -641,13 +615,15 @@ cerbero deps cross-android universal: - "${CERBERO_HOME}/logs" - "*[0-9].tar.*" -cerbero cross-android universal: - extends: '.cerbero cross-android universal' - needs: - - "cerbero deps cross-android universal" +cerbero deps cross-android universal: + extends: + - '.cerbero deps' + - '.cerbero cross-android universal' build cerbero cross-android universal: - extends: '.cerbero cross-android universal' + extends: + - '.cerbero' + - '.cerbero cross-android universal' rules: - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' @@ -661,22 +637,15 @@ cerbero deps cross-windows x86: CONFIG: "cross-win32.cbc" ARCH: "mingw_x86" -.cerbero cross win32: +build cerbero cross win32: extends: '.cerbero' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' variables: CONFIG: "cross-win32.cbc" CERBERO_RUN_WRAPPER: "wine" CERBERO_RUN_SUFFIX: ".exe" -cerbero cross win32: - extends: '.cerbero cross win32' - needs: - - "cerbero deps cross-windows x86" - -build cerbero cross win32: - extends: '.cerbero cross win32' - rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' cerbero deps cross-windows x86_64: extends: '.cerbero deps' @@ -684,23 +653,15 @@ cerbero deps cross-windows x86_64: CONFIG: "cross-win64.cbc" ARCH: "mingw_x86_64" -.cerbero cross win64: +build cerbero cross win64: extends: '.cerbero' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' variables: CONFIG: "cross-win64.cbc" CERBERO_RUN_WRAPPER: "wine" CERBERO_RUN_SUFFIX: ".exe" -cerbero cross win64: - extends: '.cerbero cross win64' - needs: - - "cerbero deps cross-windows x86_64" - -build cerbero cross win64: - extends: '.cerbero cross win64' - rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' - # # Build an Android App using the android binaries # @@ -763,24 +724,12 @@ cerbero cross-android universal examples: rules: - if: '$CI_PROJECT_NAME == "cerbero"' needs: - - "cerbero cross-android universal" + - "cerbero deps cross-android universal" # # Cerbero macOS X86_64 build # -cerbero deps macos x86_64: - extends: '.cerbero deps' - variables: - ARCH: "darwin_x86_64" - CONFIG: "osx-x86-64.cbc" - HAVE_CCACHE: "" - CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" - CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" - tags: - - gst-macos-10.15 - .cerbero macos x86_64: - extends: '.cerbero' variables: ARCH: "darwin_x86_64" CONFIG: "osx-x86-64.cbc" @@ -800,32 +749,22 @@ cerbero deps macos x86_64: - "gstreamer-1.0-1.*.pkg" - "gstreamer-1.0-devel-1.*.pkg" -cerbero macos x86_64: - extends: '.cerbero macos x86_64' - needs: - - "cerbero deps macos x86_64" +cerbero deps macos x86_64: + extends: + - '.cerbero deps' + - '.cerbero macos x86_64' build cerbero macos x86_64: - extends: '.cerbero macos x86_64' + extends: + - '.cerbero' + - '.cerbero macos x86_64' rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' # # Cerbero iOS build # -cerbero deps cross-ios universal: - extends: '.cerbero deps' - variables: - ARCH: "ios_universal" - CONFIG: "cross-ios-universal.cbc" - CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror" - HAVE_CCACHE: "" - CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" - tags: - - gst-ios-13.2 - .cerbero cross-ios universal: - extends: '.cerbero' variables: ARCH: "ios_universal" CONFIG: "cross-ios-universal.cbc" @@ -844,13 +783,15 @@ cerbero deps cross-ios universal: - "${CERBERO_HOME}/logs" - "gstreamer-1.0-*-ios-universal.pkg" -cerbero cross-ios universal: - extends: '.cerbero cross-ios universal' - needs: - - "cerbero deps cross-ios universal" +cerbero deps cross-ios universal: + extends: + - '.cerbero deps' + - '.cerbero cross-ios universal' build cerbero cross-ios universal: - extends: '.cerbero cross-ios universal' + extends: + - '.cerbero' + - '.cerbero cross-ios universal' rules: - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' @@ -962,7 +903,7 @@ cerbero cross-ios universal examples: rules: - if: '$CI_PROJECT_NAME == "cerbero"' needs: - - "cerbero cross-ios universal" + - "cerbero deps cross-ios universal" build gst-omx zynq fedora x86_64: extends: '.build fedora x86_64' From 7beb52a6e0137a874aec1b303a59d75602916e11 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 18 Jun 2020 11:14:28 +0300 Subject: [PATCH 472/605] citemplate: upate the cerbero image Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 9c5edbfe68..f88e595acb 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -7,8 +7,8 @@ stages: - 'integrate' variables: - ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-04-06-2193279-master' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-04-06-2193307-master' + ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-06-18-3152521-master' + CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-06-18-3152520-master' FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-06-04-2958178-master' INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' From 78577cffafd59dd3b801da46b5a5c6759681941d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 23 Jun 2020 12:21:20 -0400 Subject: [PATCH 473/605] ci: Update to hotdoc --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f88e595acb..6782b913e7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -811,7 +811,7 @@ documentation: MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - - pip3 install --upgrade hotdoc==0.11.0 + - pip3 install --upgrade hotdoc==0.12.0 - *gst_build - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches - | From f9881240ae46d7c17abcbff7d9b38654859ae201 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 23 Jun 2020 18:03:04 -0400 Subject: [PATCH 474/605] gitlab: Update hotdoc to 0.12.2 Adding Since markers check Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6782b913e7..375bde5fc0 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -811,7 +811,7 @@ documentation: MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - - pip3 install --upgrade hotdoc==0.12.0 + - pip3 install --upgrade hotdoc==0.12.2 - *gst_build - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches - | From 592124a46bd1d2a499ea47754ba1e4b93dac15d4 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 25 Jun 2020 18:20:32 -0400 Subject: [PATCH 475/605] gitlab: Update hotdoc Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 375bde5fc0..5cfc7bbeb4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -811,7 +811,7 @@ documentation: MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - - pip3 install --upgrade hotdoc==0.12.2 + - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@fa3bcd24929717916ecf42a458a8ab36930cf031 - *gst_build - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches - | From 4bacfa410cb80b679d095a3645ef0b6fe7085af2 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 9 Jun 2020 19:00:19 +0300 Subject: [PATCH 476/605] docker/fedora: update meson version to 0.54.3 Part-of: --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index dc5afc1b74..d710ca552c 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -173,7 +173,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.53.1 hotdoc +pip3 install meson==0.54.3 hotdoc # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ From 87208fdf5359f02355c8496fa623087e05aed90c Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 18 Jun 2020 16:29:33 +0300 Subject: [PATCH 477/605] docker/fedora: update rust toolchain to 1.44 Part-of: --- docker/fedora/prepare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index d710ca552c..fe19c4d361 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -206,8 +206,8 @@ rpm -i --reinstall *.rpm rm -f *.rpm # Install Rust -RUSTUP_VERSION=1.21.0 -RUST_VERSION=1.40.0 +RUSTUP_VERSION=1.21.1 +RUST_VERSION=1.44.1 RUST_ARCH="x86_64-unknown-linux-gnu" dnf install -y wget From c731fab0caacaf354dfad0f817f6b4b4edf96c96 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 10 Jun 2020 14:39:32 +0300 Subject: [PATCH 478/605] image_builds: simplify the tags We can use docker labels to pass the metadata we want and make the tag simpoler. also small fixup, make it so buildah it uses chroot isolation and update the base image Part-of: --- .gitlab-ci.yml | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a011ea5463..8067d2e93f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ test manifest: - "junit.xml" .base: - image: 'registry.fedoraproject.org/fedora:31' + image: 'registry.fedoraproject.org/fedora:32' extends: - '.global_ci_policy' tags: ['gstreamer'] @@ -35,42 +35,58 @@ test manifest: variables: STORAGE_DRIVER: 'vfs' BUILDAH_FORMAT: 'docker' + BUILDAH_ISOLATION: 'chroot' GIT_STRATEGY: fetch script: - export DATE=$(date +"%Y-%m-%d") - - export IMAGE="${CI_REGISTRY_IMAGE}/${ARCH}/${TAG}" - - export _UID="${IMAGE}:${DATE}-${CI_JOB_ID}" - - export LATEST="${IMAGE}:latest" + - export IMAGE="$CI_REGISTRY_IMAGE/${ARCH:=amd64}/$TAG" + - export _UID="$IMAGE:$DATE-$GST_UPSTREAM_BRANCH" + - export LATEST="$IMAGE:latest" - dnf install -y buildah runc # Newer versions of podman/buildah try to set overlayfs mount options when # using the vfs driver, and this causes errors. - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf - - buildah bud --build-arg DEFAULT_BRANCH=${GST_UPSTREAM_BRANCH} -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR} + - echo "Building image $_UID" + + - > + buildah bud + --build-arg DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH + --arch=${ARCH:=amd64} + --label ci.job_id=$CI_JOB_ID + --label pipeline.url=$CI_PIPELINE_URL + --label git.ref_name=$CI_COMMIT_REF_NAME + --label git.sha=$CI_COMMIT_SHA + --label gitlab.project_path=$CI_PROJECT_PATH + --label gitlab.project_url=$CI_PROJECT_URL + -f $DOCKERFILE + -t $IMAGE + $CONTEXT_DIR + + - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY # If we are in a fork, push the image to the reigstry regardless the branch - | if [ "$CI_PROJECT_NAMESPACE" != "gstreamer" ]; then - buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - buildah push ${LATEST} + # Tag the image with the git ref + buildah tag $_UID $LATEST + # Only push the :latest so with each push the previous + # image in the forked registry gets overrwritten + # to avoid bloating the registry and making + # pruning easier for the gitlab admins + buildah push $LATEST fi # Push the images to the upstream registry - | if [ "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then - # Tag the image with the git ref - buildah tag ${LATEST} ${_UID}-${CI_COMMIT_REF_NAME} - # Push the tags - buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - buildah push ${_UID}-${CI_COMMIT_REF_NAME} - buildah push ${LATEST} + buildah push $_UID fi alpine amd64 manifest builder docker: stage: "build docker" variables: - ARCH: "amd64" TAG: "build-manifest" CONTEXT_DIR: "docker/build_manifest/" DOCKERFILE: "docker/build_manifest/Dockerfile" @@ -79,7 +95,6 @@ alpine amd64 manifest builder docker: test manifest amd64 docker: stage: "build docker" variables: - ARCH: "amd64" TAG: "test-manifest" CONTEXT_DIR: "docker/test_manifest/" DOCKERFILE: "docker/test_manifest/Dockerfile" @@ -88,7 +103,6 @@ test manifest amd64 docker: gst-indent amd64 docker: stage: "build docker" variables: - ARCH: "amd64" TAG: "gst-indent" CONTEXT_DIR: "docker/indent/" DOCKERFILE: "docker/indent/Dockerfile" @@ -97,7 +111,6 @@ gst-indent amd64 docker: fedora amd64 docker: stage: "build docker" variables: - ARCH: "amd64" TAG: "fedora" CONTEXT_DIR: "docker/fedora/" DOCKERFILE: "docker/fedora/Dockerfile" @@ -106,7 +119,6 @@ fedora amd64 docker: cerbero fedora amd64 docker: stage: "build docker" variables: - ARCH: "amd64" TAG: "cerbero-fedora" CONTEXT_DIR: "docker/cerbero/" DOCKERFILE: "docker/cerbero/Dockerfile-fedora" @@ -115,7 +127,6 @@ cerbero fedora amd64 docker: android docker: stage: "build docker" variables: - ARCH: "amd64" TAG: "android" CONTEXT_DIR: "docker/android/" DOCKERFILE: "docker/android/Dockerfile" From fe959679b84d9bbe0d7e15eb729bba0457e92f86 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 18 Jun 2020 19:18:27 +0300 Subject: [PATCH 479/605] .gitlab-ci.yml: adapt the docker build jobs to be more like citemplates This is the inital step towards migrating our docker images setup to something closer and eventually freedesktop/citemplates [1] The idea is that jobs always run from the registry in your fork. If the image sha/id matches the one from the upstream registry, its copied over else a new one is build, pushed and tested. Only change the fedora job for now while testing. [1]: https://gitlab.freedesktop.org/freedesktop/ci-templates Part-of: --- .gitlab-ci.yml | 110 ++++++++++++++++++++++++++++++++--------- gitlab/ci_template.yml | 1 + 2 files changed, 89 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8067d2e93f..310f8ee2b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,59 @@ test manifest: junit: - "junit.xml" +# have a special definition for the test if the image already exists +.check_image: &check_image_exists | + # if-not-exists steps + set -x + + if [[ -z "$GST_FORCE_REBUILD" ]] + then + # disable exit on failure + set +e + + # check if our image is already in the current registry + # we store the sha of the digest and the layers + skopeo inspect docker://$LOCAL_IMAGE | jq '[.Digest, .Layers]' > local_sha + + # check if our image is already in the upstream registry + if [[ -z "$GST_UPSTREAM_REPO" ]] + then + echo "WARNING! Variable \$GST_UPSTREAM_REPO is undefined, cannot check for images" + else + skopeo inspect docker://$GST_UPSTREAM_IMAGE | jq '[.Digest, .Layers]' > upstream_sha + fi + + # reenable exit on failure + set -e + + # if the upstream repo has an image, ensure we use the same + if [ -s upstream_sha ] + then + + # ensure we use the same image from upstream + diff upstream_sha local_sha && exit 0 || true + + # copy the original image into the current project registry namespace + # we do 2 attempts with skopeo copy at most + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$GST_UPSTREAM_IMAGE \ + docker://$LOCAL_IMAGE || \ + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$GST_UPSTREAM_IMAGE \ + docker://$LOCAL_IMAGE + + exit 0 + fi + + # if we have a local image but none in the upstream repo, use our + if [ -s local_sha ] + then + exit 0 + fi + fi + + set +x + .base: image: 'registry.fedoraproject.org/fedora:32' extends: @@ -36,19 +89,32 @@ test manifest: STORAGE_DRIVER: 'vfs' BUILDAH_FORMAT: 'docker' BUILDAH_ISOLATION: 'chroot' + + # FIXME: Probably needs to be changed to none and clone gst-ci repo each time, sigh + # as these jobs *could* run potentially in a fork of smth like -bad + # though given our current setup, its only possible to tweak the CI from a gst-ci + # fork so it might be fine GIT_STRATEGY: fetch script: - - export DATE=$(date +"%Y-%m-%d") - - export IMAGE="$CI_REGISTRY_IMAGE/${ARCH:=amd64}/$TAG" - - export _UID="$IMAGE:$DATE-$GST_UPSTREAM_BRANCH" - - export LATEST="$IMAGE:latest" + - echo $TAG + - echo $REPO_SUFFIX + + - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$REPO_SUFFIX:$TAG" + - export GST_UPSTREAM_IMAGE="$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG" + + - echo $LOCAL_IMAGE + - echo $GST_UPSTREAM_IMAGE + + # FIXME: make an image since there will be multiple jobs running every pipeline + - dnf install -y buildah skopeo jq runc - - dnf install -y buildah runc # Newer versions of podman/buildah try to set overlayfs mount options when # using the vfs driver, and this causes errors. - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf - - echo "Building image $_UID" + - *check_image_exists + + - echo "Building image $LOCAL_IMAGE" - > buildah bud @@ -60,29 +126,27 @@ test manifest: --label git.sha=$CI_COMMIT_SHA --label gitlab.project_path=$CI_PROJECT_PATH --label gitlab.project_url=$CI_PROJECT_URL + --label fdo.upstream-repo=$GST_UPSTREAM_REPO + --label fdo.expires-after="3w" -f $DOCKERFILE - -t $IMAGE + -t $LOCAL_IMAGE $CONTEXT_DIR - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - # If we are in a fork, push the image to the reigstry regardless the branch + + # Sanity check - | - if [ "$CI_PROJECT_NAMESPACE" != "gstreamer" ]; then - # Tag the image with the git ref - buildah tag $_UID $LATEST - # Only push the :latest so with each push the previous - # image in the forked registry gets overrwritten - # to avoid bloating the registry and making - # pruning easier for the gitlab admins - buildah push $LATEST + if [ "$CI_PROJECT_PATH" = "gstreamer/gst-ci" ]; then + if [ "$LOCAL_IMAGE" != "$GST_UPSTREAM_IMAGE" ]; then + echo "ERROR!!! AAAAA! THE IMAGE URI DOES NOT MATCH THE EXPECTED UPSTREAM ONE" + echo $LOCAL_IMAGE + echo $GST_UPSTRAM_IMAGE + exit 1 + fi fi - # Push the images to the upstream registry - - | - if [ "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then - buildah push $_UID - fi + - buildah push $LOCAL_IMAGE alpine amd64 manifest builder docker: stage: "build docker" @@ -111,7 +175,9 @@ gst-indent amd64 docker: fedora amd64 docker: stage: "build docker" variables: - TAG: "fedora" + REPO_SUFFIX: "amd64/fedora" + TAG: "2020-06-18.0-$GST_UPSTREAM_BRANCH" + CONTEXT_DIR: "docker/fedora/" DOCKERFILE: "docker/fedora/Dockerfile" extends: .base diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 5cfc7bbeb4..684ef0fe4b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -17,6 +17,7 @@ variables: # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'master' ORC_UPSTREAM_BRANCH: 'master' + GST_UPSTRAM_REPO: 'gstreamer/gst-ci' GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From e73ced3676701946f486624df00e2d799848d58d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 19 Jun 2020 11:09:38 +0300 Subject: [PATCH 480/605] .gitlab-ci.yml: kill local jobs The idea behind local jobs was that you would have jobs that share the same setup as the citemplate ones but would always be pointing to your forked registry with your custom build of the image, since the citemplate hardcoded the image it was running against. With the changes introduced in the previous commit we now mirror the setup from freedesktop/ci-templates and the idea is that jobs always run from the registry in your fork. If the image sha/id matches the one from the upstream registry, its copied over else a new one is build, pushed and tested. Part-of: --- .gitlab-ci.yml | 73 -------------------------------------------------- 1 file changed, 73 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 310f8ee2b2..20ba074a4e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -197,76 +197,3 @@ android docker: CONTEXT_DIR: "docker/android/" DOCKERFILE: "docker/android/Dockerfile" extends: .base - -.local_rules: &local_rules - rules: - - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_PATH == "gstreamer/gst-ci"' - when: 'never' - - when: 'manual' - allow_failure: true - -.local_template: &local_template - <<: *local_rules - needs: - - 'manifest' - -# Test the build job against the latest build image tag and the local manifest -build fedora x86_64 local: - extends: '.build' - stage: 'build' - image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest" - artifacts: - expire_in: '5 days' - when: always - paths: - - "manifest.xml" - - "gst-build/" - <<: *local_template - -build cerbero fedora x86_64 local: - extends: 'build cerbero fedora x86_64' - image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" - <<: *local_template - -build cerbero cross-android universal local: - extends: 'build cerbero cross-android universal' - image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" - <<: *local_template - -build cerbero cross win32 local: - extends: 'build cerbero cross win32' - image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" - <<: *local_template - -build cerbero cross win64 local: - extends: 'build cerbero cross win64' - image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest" - <<: *local_template - -# Note: dependencies: will be deprecated in the future, but current manual -# jobs with needs: (even if they are allowed to fail) will leave the pipeline -# pending. -.test fedora x86_64 local: - image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest' - extends: '.test' - <<: *local_rules - -check fedora local: - extends: '.test fedora x86_64 local' - variables: - TEST_SUITE: 'check.gst*' - -cross-android universal examples local: - extends: '.cross-android universal examples' - image: "${CI_REGISTRY_IMAGE}/amd64/android:latest" - <<: *local_rules - dependencies: - - 'build cerbero cross-android universal local' - -integration testsuites fedora local: - extends: '.test fedora x86_64 local' - before_script: - - rm -f gst-build/build/subprojects/gstreamer-vaapi/gst/vaapi/libgstvaapi.so - variables: - EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures" - TEST_SUITE: "validate ges" From cece45eacbdf88f77b0af64518a16a818bc42770 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 19 Jun 2020 15:27:36 +0300 Subject: [PATCH 481/605] ci_template: merge docker builds into the normal pipeline Port the build docker jobs to the template and integrate it into the pipeline. Then change the image uris to the new system, and the user forked registries. Part-of: --- .gitlab-ci.yml | 181 +------------------------------ gitlab/ci_template.yml | 236 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 232 insertions(+), 185 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20ba074a4e..bcd734be09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,20 +1,11 @@ include: "gitlab/ci_template.yml" -stages: - - "build docker" - - "preparation" - - "pre-build" - - "build" - - "test" - # use the binaries in some way - - "integrate" - test manifest: variables: GIT_STRATEGY: fetch rules: - when: 'always' - image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:2019-10-23-793476' + image: "$TEST_MANIFEST_IMAGE" stage: "preparation" script: - pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py @@ -24,176 +15,12 @@ test manifest: junit: - "junit.xml" -# have a special definition for the test if the image already exists -.check_image: &check_image_exists | - # if-not-exists steps - set -x - - if [[ -z "$GST_FORCE_REBUILD" ]] - then - # disable exit on failure - set +e - - # check if our image is already in the current registry - # we store the sha of the digest and the layers - skopeo inspect docker://$LOCAL_IMAGE | jq '[.Digest, .Layers]' > local_sha - - # check if our image is already in the upstream registry - if [[ -z "$GST_UPSTREAM_REPO" ]] - then - echo "WARNING! Variable \$GST_UPSTREAM_REPO is undefined, cannot check for images" - else - skopeo inspect docker://$GST_UPSTREAM_IMAGE | jq '[.Digest, .Layers]' > upstream_sha - fi - - # reenable exit on failure - set -e - - # if the upstream repo has an image, ensure we use the same - if [ -s upstream_sha ] - then - - # ensure we use the same image from upstream - diff upstream_sha local_sha && exit 0 || true - - # copy the original image into the current project registry namespace - # we do 2 attempts with skopeo copy at most - skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ - docker://$GST_UPSTREAM_IMAGE \ - docker://$LOCAL_IMAGE || \ - skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ - docker://$GST_UPSTREAM_IMAGE \ - docker://$LOCAL_IMAGE - - exit 0 - fi - - # if we have a local image but none in the upstream repo, use our - if [ -s local_sha ] - then - exit 0 - fi - fi - - set +x - -.base: - image: 'registry.fedoraproject.org/fedora:32' - extends: - - '.global_ci_policy' - tags: ['gstreamer'] - rules: - - when: 'manual' - allow_failure: true - variables: - STORAGE_DRIVER: 'vfs' - BUILDAH_FORMAT: 'docker' - BUILDAH_ISOLATION: 'chroot' - - # FIXME: Probably needs to be changed to none and clone gst-ci repo each time, sigh - # as these jobs *could* run potentially in a fork of smth like -bad - # though given our current setup, its only possible to tweak the CI from a gst-ci - # fork so it might be fine - GIT_STRATEGY: fetch - script: - - echo $TAG - - echo $REPO_SUFFIX - - - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$REPO_SUFFIX:$TAG" - - export GST_UPSTREAM_IMAGE="$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG" - - - echo $LOCAL_IMAGE - - echo $GST_UPSTREAM_IMAGE - - # FIXME: make an image since there will be multiple jobs running every pipeline - - dnf install -y buildah skopeo jq runc - - # Newer versions of podman/buildah try to set overlayfs mount options when - # using the vfs driver, and this causes errors. - - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf - - - *check_image_exists - - - echo "Building image $LOCAL_IMAGE" - - - > - buildah bud - --build-arg DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH - --arch=${ARCH:=amd64} - --label ci.job_id=$CI_JOB_ID - --label pipeline.url=$CI_PIPELINE_URL - --label git.ref_name=$CI_COMMIT_REF_NAME - --label git.sha=$CI_COMMIT_SHA - --label gitlab.project_path=$CI_PROJECT_PATH - --label gitlab.project_url=$CI_PROJECT_URL - --label fdo.upstream-repo=$GST_UPSTREAM_REPO - --label fdo.expires-after="3w" - -f $DOCKERFILE - -t $LOCAL_IMAGE - $CONTEXT_DIR - - - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - - # Sanity check - - | - if [ "$CI_PROJECT_PATH" = "gstreamer/gst-ci" ]; then - if [ "$LOCAL_IMAGE" != "$GST_UPSTREAM_IMAGE" ]; then - echo "ERROR!!! AAAAA! THE IMAGE URI DOES NOT MATCH THE EXPECTED UPSTREAM ONE" - echo $LOCAL_IMAGE - echo $GST_UPSTRAM_IMAGE - exit 1 - fi - fi - - - buildah push $LOCAL_IMAGE - -alpine amd64 manifest builder docker: - stage: "build docker" - variables: - TAG: "build-manifest" - CONTEXT_DIR: "docker/build_manifest/" - DOCKERFILE: "docker/build_manifest/Dockerfile" - extends: .base - test manifest amd64 docker: stage: "build docker" variables: - TAG: "test-manifest" + REPO_SUFFIX: "$TEST_MANIFEST_AMD64_SUFFIX" + TAG: "$TEST_MANIFEST_TAG" + CONTEXT_DIR: "docker/test_manifest/" DOCKERFILE: "docker/test_manifest/Dockerfile" extends: .base - -gst-indent amd64 docker: - stage: "build docker" - variables: - TAG: "gst-indent" - CONTEXT_DIR: "docker/indent/" - DOCKERFILE: "docker/indent/Dockerfile" - extends: .base - -fedora amd64 docker: - stage: "build docker" - variables: - REPO_SUFFIX: "amd64/fedora" - TAG: "2020-06-18.0-$GST_UPSTREAM_BRANCH" - - CONTEXT_DIR: "docker/fedora/" - DOCKERFILE: "docker/fedora/Dockerfile" - extends: .base - -cerbero fedora amd64 docker: - stage: "build docker" - variables: - TAG: "cerbero-fedora" - CONTEXT_DIR: "docker/cerbero/" - DOCKERFILE: "docker/cerbero/Dockerfile-fedora" - extends: .base - -android docker: - stage: "build docker" - variables: - TAG: "android" - CONTEXT_DIR: "docker/android/" - DOCKERFILE: "docker/android/Dockerfile" - extends: .base diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 684ef0fe4b..4b0ce96325 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1,4 +1,5 @@ stages: + - 'build docker' - 'preparation' - 'pre-build' - 'build' @@ -7,17 +8,42 @@ stages: - 'integrate' variables: - ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2020-06-18-3152521-master' - CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:2020-06-18-3152520-master' - FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-06-04-2958178-master' - INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' - MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:2020-02-10-1589542-master' - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v11-master' - # Branch to track for modules that have no ref specified in the manifest GST_UPSTREAM_BRANCH: 'master' ORC_UPSTREAM_BRANCH: 'master' - GST_UPSTRAM_REPO: 'gstreamer/gst-ci' + + ### + # IMPORTANT + # These are the version tags for the docker images the CI runs against. + # If you are hacking on them or need a them to rebuild, its enough + # to change any part of the string of the image you want. + ### + FEDORA_TAG: '2020-06-25.0' + CERBERO_TAG: '2020-06-25.0' + ANDROID_TAG: '2020-06-25.0' + MANIFEST_TAG: '2020-06-25.0' + TEST_MANIFEST_TAG: '2020-06-25.0' + + GST_UPSTREAM_REPO: 'gstreamer/gst-ci' + + FEDORA_AMD64_SUFFIX: 'amd64/fedora' + CERBERO_AMD64_SUFFIX: 'amd64/cerbero-fedora' + ANDROID_AMD64_SUFFIX: 'amd64/android-fedora' + MANIFEST_AMD64_SUFFIX: 'amd64/build-manifest' + TEST_MANIFEST_AMD64_SUFFIX: 'amd64/test-manifest' + + FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" + CERBERO_IMAGE: "$CI_REGISTRY_IMAGE/$CERBERO_AMD64_SUFFIX:$CERBERO_TAG-$GST_UPSTREAM_BRANCH" + ANDROID_IMAGE: "$CI_REGISTRY_IMAGE/$ANDROID_AMD64_SUFFIX:$ANDROID_TAG-$GST_UPSTREAM_BRANCH" + MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$MANIFEST_AMD64_SUFFIX:$MANIFEST_TAG-$GST_UPSTREAM_BRANCH" + TEST_MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$TEST_MANIFEST_AMD64_SUFFIX:$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" + + # FIXME: properly append the gst-upstream-branch once the images are rebuilt + INDENT_AMD64_SUFFIX: 'amd64/gst-indent' + INDENT_TAG: '6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' + INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG" + + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v11-master' GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both @@ -55,6 +81,194 @@ variables: - 'api_failure' interruptible: true + +# Script to check if a docker image exists in the upstream registry +# and if so copy it to the forked registry so we can use it +# +# This is copied/adapted from citemplates/templates/fedora.yml +# https://gitlab.freedesktop.org/freedesktop/ci-templates/-/blob/96b621fe9f57ec2464f8d1a0940446afbf6c8f59/templates/fedora.yml +.check_image: &check_image_exists | + # if-not-exists steps + set -x + + if [[ -z "$GST_FORCE_REBUILD" ]] + then + # disable exit on failure + set +e + + # check if our image is already in the current registry + # we store the sha of the digest and the layers + skopeo inspect docker://$LOCAL_IMAGE | jq '[.Digest, .Layers]' > local_sha + + # check if our image is already in the upstream registry + if [[ -z "$GST_UPSTREAM_REPO" ]] + then + echo "WARNING! Variable \$GST_UPSTREAM_REPO is undefined, cannot check for images" + else + skopeo inspect docker://$GST_UPSTREAM_IMAGE | jq '[.Digest, .Layers]' > upstream_sha + fi + + # reenable exit on failure + set -e + + # if the upstream repo has an image, ensure we use the same + if [ -s upstream_sha ] + then + + # ensure we use the same image from upstream + diff upstream_sha local_sha && exit 0 || true + + # copy the original image into the current project registry namespace + # we do 2 attempts with skopeo copy at most + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$GST_UPSTREAM_IMAGE \ + docker://$LOCAL_IMAGE || \ + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$GST_UPSTREAM_IMAGE \ + docker://$LOCAL_IMAGE + + exit 0 + fi + + # if we have a local image but none in the upstream repo, use our + if [ -s local_sha ] + then + exit 0 + fi + fi + + set +x + +# Build docker images from Dockerfiles +# +# This is copied/adapted from citemplates/templates/fedora.yml +# https://gitlab.freedesktop.org/freedesktop/ci-templates/-/blob/96b621fe9f57ec2464f8d1a0940446afbf6c8f59/templates/fedora.yml +# +# CITemplates builds uses buildah run/commit workflow to build the images which doesn't +# fit us atm and our images need further adjustment to make use of it. Porting to +# ci-templates is the next step though. +# +# All the documentation from citemplates should be applicable here, so please refer there +.base: + image: 'registry.fedoraproject.org/fedora:32' + extends: + - '.global_ci_policy' + tags: ['gstreamer'] + rules: + - when: 'always' + variables: + STORAGE_DRIVER: 'vfs' + BUILDAH_FORMAT: 'docker' + BUILDAH_ISOLATION: 'chroot' + + # FIXME: Probably needs to be changed to none and clone gst-ci repo each time, sigh + # as these jobs *could* run potentially in a fork of smth like -bad + # though given our current setup, its only possible to tweak the CI from a gst-ci + # fork so it might be fine + GIT_STRATEGY: fetch + script: + - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$REPO_SUFFIX:$TAG" + - export GST_UPSTREAM_IMAGE="$CI_REGISTRY/$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG" + + # FIXME: make an image since there will be multiple jobs running every pipeline + - dnf install -y skopeo jq diffutils + + # Newer versions of podman/buildah try to set overlayfs mount options when + # using the vfs driver, and this causes errors. + - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf + + - *check_image_exists + + # FIXME: make an image since there will be multiple jobs running every pipeline + - dnf install -y buildah skopeo jq runc + + - echo "Building image $LOCAL_IMAGE" + + - > + buildah bud + --build-arg DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH + --arch=${ARCH:=amd64} + --label ci.job_id=$CI_JOB_ID + --label pipeline.url=$CI_PIPELINE_URL + --label git.ref_name=$CI_COMMIT_REF_NAME + --label git.sha=$CI_COMMIT_SHA + --label gitlab.project_path=$CI_PROJECT_PATH + --label gitlab.project_url=$CI_PROJECT_URL + --label fdo.upstream-repo=$GST_UPSTREAM_REPO + --label fdo.expires-after="3w" + -f $DOCKERFILE + -t $LOCAL_IMAGE + $CONTEXT_DIR + + - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + + # Sanity check + - | + if [ "$CI_PROJECT_PATH" = "gstreamer/gst-ci" ]; then + if [ "$LOCAL_IMAGE" != "$GST_UPSTREAM_IMAGE" ]; then + echo "ERROR!!! AAAAA! THE IMAGE URI DOES NOT MATCH THE EXPECTED UPSTREAM ONE" + echo $LOCAL_IMAGE + echo $GST_UPSTREAM_IMAGE + exit 1 + fi + fi + + - buildah push $LOCAL_IMAGE + +fedora amd64 docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$FEDORA_AMD64_SUFFIX" + TAG: "$FEDORA_TAG-$GST_UPSTREAM_BRANCH" + + CONTEXT_DIR: "./docker/fedora/" + DOCKERFILE: "docker/fedora/Dockerfile" + extends: .base + +cerbero fedora amd64 docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$CERBERO_AMD64_SUFFIX" + TAG: "$CERBERO_TAG-$GST_UPSTREAM_BRANCH" + + CONTEXT_DIR: "docker/cerbero/" + DOCKERFILE: "docker/cerbero/Dockerfile-fedora" + extends: .base + +android docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$ANDROID_AMD64_SUFFIX" + TAG: "$ANDROID_TAG-$GST_UPSTREAM_BRANCH" + + CONTEXT_DIR: "docker/android/" + DOCKERFILE: "docker/android/Dockerfile" + extends: .base + +alpine amd64 manifest builder docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$MANIFEST_AMD64_SUFFIX" + TAG: '$MANIFEST_TAG-$GST_UPSTREAM_BRANCH' + + CONTEXT_DIR: "docker/build_manifest/" + DOCKERFILE: "docker/build_manifest/Dockerfile" + extends: .base + +gst-indent amd64 docker: + stage: "build docker" + variables: + REPO_SUFFIX: "$INDENT_AMD64_SUFFIX" + # FIXME: Add gst-ups-branch to the tag + TAG: "$INDENT_TAG" + + CONTEXT_DIR: "docker/indent/" + DOCKERFILE: "docker/indent/Dockerfile" + extends: .base + +# +# Job to create the manifest.xml to used for our builds +# manifest: image: $MANIFEST_IMAGE extends: @@ -78,6 +292,9 @@ manifest: paths: - "manifest.xml" +# +# gst-indent!! +# gst indent: image: $INDENT_IMAGE extends: @@ -102,6 +319,9 @@ gst indent: exit 1 fi +# +# gst-build setup templates +# .gst_build_template: &gst_build - echo $MESON_ARGS # Sometimes, gitlab-runner want to reuse From 3a004f7ca46b85eb843c2b6002db2e2d640a3717 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 25 Jun 2020 12:49:05 +0300 Subject: [PATCH 482/605] citemplate: use the fd.o pre existing buildah image to build containers Part-of: --- .gitlab-ci.yml | 2 +- gitlab/ci_template.yml | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bcd734be09..048418c0f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ test manifest amd64 docker: stage: "build docker" variables: REPO_SUFFIX: "$TEST_MANIFEST_AMD64_SUFFIX" - TAG: "$TEST_MANIFEST_TAG" + TAG: "$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" CONTEXT_DIR: "docker/test_manifest/" DOCKERFILE: "docker/test_manifest/Dockerfile" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4b0ce96325..c73de12610 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -150,7 +150,7 @@ variables: # # All the documentation from citemplates should be applicable here, so please refer there .base: - image: 'registry.fedoraproject.org/fedora:32' + image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24" extends: - '.global_ci_policy' tags: ['gstreamer'] @@ -170,17 +170,13 @@ variables: - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$REPO_SUFFIX:$TAG" - export GST_UPSTREAM_IMAGE="$CI_REGISTRY/$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG" - # FIXME: make an image since there will be multiple jobs running every pipeline - - dnf install -y skopeo jq diffutils - # Newer versions of podman/buildah try to set overlayfs mount options when # using the vfs driver, and this causes errors. - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf - - *check_image_exists + - dnf upgrade -y buildah skopeo jq runc - # FIXME: make an image since there will be multiple jobs running every pipeline - - dnf install -y buildah skopeo jq runc + - *check_image_exists - echo "Building image $LOCAL_IMAGE" From 5c490de1bd9b431f5c8d112814507e7463ed87b1 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 25 Jun 2020 12:35:10 +0300 Subject: [PATCH 483/605] citemplate: also check the forked registry for image builds In case you are developing a set of changes in a module, in conjunction with a branch in gst-ci, you will end up with a template with a different tag than the upstream repo, which will be refferencing your gst-ci forked registry. But that image won't existin in the namespace the module would be running at, so we need to copy it from there. Part-of: --- gitlab/ci_template.yml | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c73de12610..7a00304153 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -100,6 +100,9 @@ variables: # we store the sha of the digest and the layers skopeo inspect docker://$LOCAL_IMAGE | jq '[.Digest, .Layers]' > local_sha + # check if our image is already in our gst-ci registry fork + skopeo inspect docker://$LOCAL_GST_CI_IMAGE | jq '[.Digest, .Layers]' > local_gst_ci_sha + # check if our image is already in the upstream registry if [[ -z "$GST_UPSTREAM_REPO" ]] then @@ -130,6 +133,25 @@ variables: exit 0 fi + # if the local ci fork repo has an image, ensure we use the same + if [ -s local_gst_ci_sha ] + then + + # ensure we use the same image from upstream + diff local_gst_ci_sha local_sha && exit 0 || true + + # copy the original image into the current project registry namespace + # we do 2 attempts with skopeo copy at most + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$LOCAL_GST_CI_IMAGE \ + docker://$LOCAL_IMAGE || \ + skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ + docker://$LOCAL_GST_CI_IMAGE \ + docker://$LOCAL_IMAGE + + exit 0 + fi + # if we have a local image but none in the upstream repo, use our if [ -s local_sha ] then @@ -161,13 +183,10 @@ variables: BUILDAH_FORMAT: 'docker' BUILDAH_ISOLATION: 'chroot' - # FIXME: Probably needs to be changed to none and clone gst-ci repo each time, sigh - # as these jobs *could* run potentially in a fork of smth like -bad - # though given our current setup, its only possible to tweak the CI from a gst-ci - # fork so it might be fine GIT_STRATEGY: fetch script: - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$REPO_SUFFIX:$TAG" + - export LOCAL_GST_CI_IMAGE="$CI_REGISTRY/$CI_PROJECT_NAMESPACE/gst-ci/$REPO_SUFFIX:$TAG" - export GST_UPSTREAM_IMAGE="$CI_REGISTRY/$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG" # Newer versions of podman/buildah try to set overlayfs mount options when @@ -178,6 +197,15 @@ variables: - *check_image_exists + # This part of the job should only ever be possible to be reach in a gst-ci fork + - | + if [[ "$CI_PROJECT_NAME" != 'gst-ci' ]] + then + echo "ERROR! Something is wrong!" + echo "This part of the job is supposed to be executed only in gst-ci forks, and its probably going to fail else" + exit 1 + fi + - echo "Building image $LOCAL_IMAGE" - > From 0a085866ae37e8141ca4e5d0ec2c3b667eb218b6 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Mon, 15 Jun 2020 14:05:22 +0200 Subject: [PATCH 484/605] docker: move gst-indent script into docker image This script is now very stable, so downloading it everytime is no longer useful. It can also helps avoid network errors when running ci jobs. Fix #70 Part-of: --- docker/indent/Dockerfile | 4 +++- gitlab/ci_template.yml | 11 ++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docker/indent/Dockerfile b/docker/indent/Dockerfile index eff3c85f19..63581e3597 100644 --- a/docker/indent/Dockerfile +++ b/docker/indent/Dockerfile @@ -2,4 +2,6 @@ FROM debian:stretch-slim RUN apt update -yqq \ && apt install -y curl indent git findutils \ - && rm -R /var/lib/apt/ /var/log/apt/ \ No newline at end of file + && rm -R /var/lib/apt/ /var/log/apt/ \ + && curl -L -o /usr/local/bin/gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent \ + && chmod a+x /usr/local/bin/gst-indent diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7a00304153..8da024fa34 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -23,6 +23,7 @@ variables: ANDROID_TAG: '2020-06-25.0' MANIFEST_TAG: '2020-06-25.0' TEST_MANIFEST_TAG: '2020-06-25.0' + INDENT_TAG: '2020-06-29.0' GST_UPSTREAM_REPO: 'gstreamer/gst-ci' @@ -31,17 +32,14 @@ variables: ANDROID_AMD64_SUFFIX: 'amd64/android-fedora' MANIFEST_AMD64_SUFFIX: 'amd64/build-manifest' TEST_MANIFEST_AMD64_SUFFIX: 'amd64/test-manifest' + INDENT_AMD64_SUFFIX: 'amd64/gst-indent' FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" CERBERO_IMAGE: "$CI_REGISTRY_IMAGE/$CERBERO_AMD64_SUFFIX:$CERBERO_TAG-$GST_UPSTREAM_BRANCH" ANDROID_IMAGE: "$CI_REGISTRY_IMAGE/$ANDROID_AMD64_SUFFIX:$ANDROID_TAG-$GST_UPSTREAM_BRANCH" MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$MANIFEST_AMD64_SUFFIX:$MANIFEST_TAG-$GST_UPSTREAM_BRANCH" TEST_MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$TEST_MANIFEST_AMD64_SUFFIX:$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" - - # FIXME: properly append the gst-upstream-branch once the images are rebuilt - INDENT_AMD64_SUFFIX: 'amd64/gst-indent' - INDENT_TAG: '6f7e01e1e30a73efa880acdc8e911f1f20c58dbb' - INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG" + INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v11-master' @@ -283,8 +281,7 @@ gst-indent amd64 docker: stage: "build docker" variables: REPO_SUFFIX: "$INDENT_AMD64_SUFFIX" - # FIXME: Add gst-ups-branch to the tag - TAG: "$INDENT_TAG" + TAG: "$INDENT_TAG-$GST_UPSTREAM_BRANCH" CONTEXT_DIR: "docker/indent/" DOCKERFILE: "docker/indent/Dockerfile" From 41e979e1e41c5a5cf20707e7034d2a7d582dd35d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 29 Jun 2020 16:47:52 +0300 Subject: [PATCH 485/605] citemplate: remove tag restriction for docker builds jobs There isn't anything particular that needs to be on the gstreamer runners, and the other shared runners should be able to handle them fine. Part-of: --- gitlab/ci_template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8da024fa34..e8cdce3fc9 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -173,7 +173,6 @@ variables: image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24" extends: - '.global_ci_policy' - tags: ['gstreamer'] rules: - when: 'always' variables: From e94919137ce622f337dbc3e1bef0cfe5dcdf0c44 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 30 Jun 2020 00:12:45 +0300 Subject: [PATCH 486/605] citemplate: check if the gitlab registry is enabled on the fork If its not CI_REGISTRY_IMAGE will be null and break the job. Instead paste a helpful message with the documentation on how to enable the registry. Part-of: --- gitlab/ci_template.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e8cdce3fc9..ad62558499 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -186,6 +186,16 @@ variables: - export LOCAL_GST_CI_IMAGE="$CI_REGISTRY/$CI_PROJECT_NAMESPACE/gst-ci/$REPO_SUFFIX:$TAG" - export GST_UPSTREAM_IMAGE="$CI_REGISTRY/$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG" + # check if the docker registry is enabled, else the variable will be missing + - | + if [[ -z "$CI_REGISTRY_IMAGE" ]] + then + echo "ERROR! Looks like your repository/fork has disabled Docker Registries." + echo "Pleae enable them by following the documentation from here:" + echo "https://docs.gitlab.com/ee/user/packages/container_registry/#enable-the-container-registry-for-your-project" + exit 1 + fi + # Newer versions of podman/buildah try to set overlayfs mount options when # using the vfs driver, and this causes errors. - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf From 70f6af0e81afd0e4c52e68bed75a9e3d5e34225f Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 30 Jun 2020 17:46:33 +0530 Subject: [PATCH 487/605] msys2: Fix pacman update Match the official steps by MSYS2 upstream for updating on CI: https://github.com/msys2/setup-msys2/blob/master/main.js#L98 Part-of: --- gitlab/ci_template.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ad62558499..c8cc3d60c8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -778,8 +778,16 @@ build msys2 : # For some reason docker build hangs if this is included in the image, needs more troubleshooting - $env:PATH += ';C:\msys64\usr\bin;C:\msys64\mingw64\bin;C:\msys64\mingw32\bin' + # Copied from https://github.com/msys2/setup-msys2/blob/master/main.js#L98 - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true" - - C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm" + - C:\msys64\usr\bin\bash -c "sed -i 's/^CheckSpace/#CheckSpace/g' /etc/pacman.conf" + - echo "Updating MSYS2" + - C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm || echo Update failed, ignoring" + - echo "Killing all MSYS2 processes" + - taskkill /F /FI "MODULES eq msys-2.0.dll" + - echo "Completing MSYS2 update" + - C:\msys64\usr\bin\bash -c "pacman -Suu --noconfirm" + - echo "Installing needed MSYS2 packages" - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja" - Invoke-WebRequest -Uri "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/$env:GST_UPSTREAM_BRANCH/gitlab/clone_manifest_ref.py" -OutFile $env:CI_PROJECT_DIR/clone_manifest_ref.py From 51d0c730745bd1aae2f508c0de49b8027e855fe8 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Wed, 1 Jul 2020 19:02:38 +0200 Subject: [PATCH 488/605] ci_template: update hotdoc --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c8cc3d60c8..70ed490fa5 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1070,7 +1070,7 @@ documentation: MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@fa3bcd24929717916ecf42a458a8ab36930cf031 + - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@0171b68bcb540d2cd74d833f652bf95ebdc4d74f - *gst_build - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches - | From b6ccc2169fd9c3e31a58969a12a32e23663a398d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 1 Jul 2020 19:56:16 -0400 Subject: [PATCH 489/605] gitlab: Update hotdoc --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 70ed490fa5..a74b3b156c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1070,7 +1070,7 @@ documentation: MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@0171b68bcb540d2cd74d833f652bf95ebdc4d74f + - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@95aa79758f64281de3f0e6d2efde45108d8cd2ea - *gst_build - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches - | From dbc4ca1b2ff105868a6d4212f5472b2d5fbb61a6 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Mon, 15 Jun 2020 14:06:59 +0200 Subject: [PATCH 490/605] ci: use gst-indent from docker image Part-of: --- gitlab/ci_template.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a74b3b156c..7f81c55fca 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -337,9 +337,7 @@ gst indent: script: # man indent. grep RETURN VALUE, grab a beer on my behalf... - indent --version || true - - curl -L -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/${GST_UPSTREAM_BRANCH}/tools/gst-indent - - chmod +x gst-indent - - find . -name '*.c' -exec ./gst-indent {} + + - find . -name '*.c' -exec gst-indent {} + - | if git diff --quiet; then echo "Code is properly formatted" From efdb7c545cfaca9161670fa4ccf359935adf82cd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 1 Jul 2020 17:22:33 +0300 Subject: [PATCH 491/605] citemplate: slightly improve logging of docker build jobs Part-of: --- gitlab/ci_template.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7f81c55fca..cf6e808ec2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -116,9 +116,11 @@ variables: if [ -s upstream_sha ] then + echo "Checking if $LOCAL_IMAGE is up to date" # ensure we use the same image from upstream diff upstream_sha local_sha && exit 0 || true + echo "Copying image from gstreamer/gst-ci to local registry" # copy the original image into the current project registry namespace # we do 2 attempts with skopeo copy at most skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ @@ -131,13 +133,16 @@ variables: exit 0 fi + set +x + # if the local ci fork repo has an image, ensure we use the same if [ -s local_gst_ci_sha ] then - + echo "Checking if $LOCAL_GST_CI_IMAGE is up to date" # ensure we use the same image from upstream diff local_gst_ci_sha local_sha && exit 0 || true + echo "Copying image from $CI_PROJECT_NAMESPACE/gst-ci to local registry" # copy the original image into the current project registry namespace # we do 2 attempts with skopeo copy at most skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ @@ -153,11 +158,11 @@ variables: # if we have a local image but none in the upstream repo, use our if [ -s local_sha ] then + echo "Using $LOCAL_IMAGE" exit 0 fi fi - set +x # Build docker images from Dockerfiles # From e18eb064c14976f055ed373cda393c8010599cdc Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 1 Jul 2020 18:32:26 +0300 Subject: [PATCH 492/605] citemplate: run the docs job in more modules Needed to catch regressions in the docs cache files. Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index cf6e808ec2..814b218511 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1064,7 +1064,7 @@ documentation: - '.global_ci_policy' - '.gst_build_ccache_vars' rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-sharp|gst-integration-testsuites|gst-omx|gst-plugins-rs|gst-python)$/' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-integration-testsuites|gst-plugins-rs|gst-python)$/' needs: - "manifest" stage: integrate @@ -1079,7 +1079,7 @@ documentation: - | python3 -c "import os, subprocess, sys subprojects = [os.environ['CI_PROJECT_NAME']] - if subprojects[0] == 'gst-ci': + if subprojects[0] in ['gst-ci', 'gst-build']: subprojects = ['gstreamer', 'gst-plugins-ugly', 'gst-omx', 'gst-plugins-good', 'gst-plugins-base', 'gstreamer-vaapi', 'gst-libav', 'gst-editing-services', 'gst-rtsp-server', 'gst-plugins-bad',] diffsdir = '../plugins-cache-diffs' From f5569ad5dcfdc140e202b9ac49adb8704418e13f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 1 Jul 2020 18:34:07 +0300 Subject: [PATCH 493/605] citemplate: run check in gst-examples but not integration tests gst-example contains tests for check but not integrations suite Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 814b218511..0d8a876670 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -521,7 +521,7 @@ check fedora: variables: TEST_SUITE: "check.gst*" rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' integration testsuites fedora: extends: '.test fedora x86_64' @@ -530,7 +530,7 @@ integration testsuites fedora: EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs --parts=${CI_NODE_TOTAL} --part-index=${CI_NODE_INDEX}" TEST_SUITE: "validate ges" rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-omx|gst-plugins-rs)$/' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-omx|gst-plugins-rs)$/' gstreamer-full: extends: 'build static fedora x86_64' From d57693328bfa7fa016f513dd54e76ccfa1382dad Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 2 Jul 2020 15:43:13 +0300 Subject: [PATCH 494/605] citemplate: avoid the need to upgrade buildah in docker build We only need the newer version from the image, to support the --arch arg, but we don't really use it yet so we could live without it. Part-of: --- gitlab/ci_template.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0d8a876670..7f6923096d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -205,8 +205,6 @@ variables: # using the vfs driver, and this causes errors. - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf - - dnf upgrade -y buildah skopeo jq runc - - *check_image_exists # This part of the job should only ever be possible to be reach in a gst-ci fork @@ -223,7 +221,6 @@ variables: - > buildah bud --build-arg DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH - --arch=${ARCH:=amd64} --label ci.job_id=$CI_JOB_ID --label pipeline.url=$CI_PIPELINE_URL --label git.ref_name=$CI_COMMIT_REF_NAME From a5135b1596405241b73c2f42fc2547986d44f4b8 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 2 Jul 2020 17:21:58 +0300 Subject: [PATCH 495/605] citemplate: use the default keyword for our global policy It was introduced in a recentish release along with pipelines for merge requests, but it required the use of rules. Part-of: --- gitlab/ci_template.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7f6923096d..3cff1e71ff 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -69,7 +69,7 @@ variables: # # This can be used to configure global behaviour our our jobs. # -.global_ci_policy: +default: retry: max: 2 when: @@ -176,8 +176,6 @@ variables: # All the documentation from citemplates should be applicable here, so please refer there .base: image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24" - extends: - - '.global_ci_policy' rules: - when: 'always' variables: @@ -303,8 +301,6 @@ gst-indent amd64 docker: # manifest: image: $MANIFEST_IMAGE - extends: - - '.global_ci_policy' rules: - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $GITLAB_USER_LOGIN == "gstreamer-merge-bot"' when: 'always' @@ -329,8 +325,6 @@ manifest: # gst indent: image: $INDENT_IMAGE - extends: - - '.global_ci_policy' stage: 'preparation' variables: GIT_STRATEGY: 'fetch' @@ -394,7 +388,6 @@ gst indent: .build: stage: 'build' extends: - - '.global_ci_policy' - '.gst_build_ccache_vars' needs: - "manifest" @@ -460,7 +453,6 @@ build clang fedora x86_64: .test: stage: 'test' extends: - - '.global_ci_policy' - '.gst_build_ccache_vars' needs: - "manifest" @@ -626,8 +618,6 @@ valgrind ges: .cerbero: stage: "build" image: $CERBERO_IMAGE - extends: - - '.global_ci_policy' rules: - if: '$CI_PROJECT_NAME == "cerbero"' needs: @@ -695,8 +685,6 @@ valgrind ges: .build windows: image: $WINDOWS_IMAGE - extends: - - '.global_ci_policy' stage: 'build' needs: - 'manifest' @@ -926,8 +914,6 @@ build cerbero cross win64: # .cross-android universal examples: image: $ANDROID_IMAGE - extends: - - '.global_ci_policy' stage: 'integrate' variables: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples @@ -1058,7 +1044,6 @@ build cerbero cross-ios universal: documentation: image: $FEDORA_IMAGE extends: - - '.global_ci_policy' - '.gst_build_ccache_vars' rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-integration-testsuites|gst-plugins-rs|gst-python)$/' @@ -1112,8 +1097,6 @@ documentation: # .cross-ios universal examples: stage: 'integrate' - extends: - - '.global_ci_policy' variables: EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples # disable codesigning so we don't need developer identities on the CI From 3f6395c2e9467a706cf52f280d8d707c7554ff7c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 30 Sep 2019 14:44:10 +1000 Subject: [PATCH 496/605] gst-build: only enable werror for gst recipes It may throws various warning that are treated as errors for other repositories Part-of: --- gitlab/ci_template.yml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 3cff1e71ff..3c846f13c1 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -60,6 +60,23 @@ variables: -Dvaapi=enabled -Dsharp=disabled + MESON_GST_WERROR: > + -Dgstreamer:werror=true + -Dgst-plugins-base:werror=true + -Dgst-plugins-good:werror=true + -Dgst-plugins-ugly:werror=true + -Dgst-plugins-bad:werror=true + -Dgst-rtsp-server:werror=true + -Dgst-libav:werror=true + -Dgst-examples:werror=true + -Dgst-editing-services:werror=true + -Dgst-docs:werror=true + -Dgst-omx:werror=true + -Dgst-devtools:werror=true + -Dgst-python:werror=true + -Dgstreamer-vaapi:werror=true + -Dgstreamer-sharp:werror=true + DEFAULT_CERBERO_ARGS: > --variants werror --timestamps @@ -383,7 +400,7 @@ gst indent: -Ddoc=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} - --werror + ${MESON_GST_WERROR} .build: stage: 'build' @@ -395,7 +412,7 @@ gst indent: # Also need to take into account I/O of pulling docker images and uploading artifacts timeout: '45min' variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} ${MESON_GST_WERROR}" rules: - if: '$CI_PROJECT_NAME != "cerbero"' script: @@ -424,7 +441,7 @@ build nodebug fedora x86_64: extends: '.build' image: $FEDORA_IMAGE variables: - MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS} ${MESON_GST_WERROR}" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/' @@ -529,6 +546,7 @@ gstreamer-full: --default-library=static -Dauto_features=disabled -Dgst-plugins-good:alpha=enabled + $MESON_GST_WERROR rules: - if: '$CI_PROJECT_NAME =~ /^(gst-build|gst-ci)$/' @@ -1150,7 +1168,7 @@ cerbero cross-ios universal examples: build gst-omx zynq fedora x86_64: extends: '.build fedora x86_64' variables: - MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "-Domx=enabled -Dgst-omx:target=zynqultrascaleplus -Dgst-omx:header_path=${CI_PROJECT_DIR}/vcu-omx-il/omx_header -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} $MESON_GST_WERROR" rules: - if: '$CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/' before_script: @@ -1159,7 +1177,7 @@ build gst-omx zynq fedora x86_64: build gst-omx tizonia fedora x86_64: extends: '.build fedora x86_64' variables: - MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} --werror" + MESON_ARGS: "-Domx=enabled -Dgst-omx:target=tizonia -Dpython=disabled -Dlibav=disabled -Dlibnice=disabled -Dugly=disabled -Dbad=disabled -Ddevtools=disabled -Dges=disabled -Drtsp_server=disabled -Dvaapi=disabled -Dsharp=disabled -Dgst-examples=disabled -Drs=disabled ${MESON_BUILDTYPE_ARGS} $MESON_GST_WERROR" PKG_CONFIG_PATH: ${PKG_CONFIG_PATH}:${CI_PROJECT_DIR}/tizonia-install/lib64/pkgconfig/ rules: - if: '$CI_PROJECT_NAME =~ /^(gst-omx|gst-ci)$/' From 4fd36e5d73ff227f48f10fb27d5157cc469fabcf Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 29 Jun 2020 16:05:03 +0300 Subject: [PATCH 497/605] docker/fedora: add gtest to get openh264 to build again Part-of: --- docker/fedora/prepare.sh | 2 ++ gitlab/ci_template.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index fe19c4d361..ad95512772 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -41,6 +41,8 @@ dnf install -y \ glslc \ gtk3 \ gtk3-devel \ + gtest \ + gtest-devel \ graphene \ graphene-devel \ gsl \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 3c846f13c1..ed5035cf6e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -18,7 +18,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2020-06-25.0' + FEDORA_TAG: '2020-07-03.0' CERBERO_TAG: '2020-06-25.0' ANDROID_TAG: '2020-06-25.0' MANIFEST_TAG: '2020-06-25.0' From 13c2d964da0141d111706f7101bfff7a98edac73 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Fri, 10 Jul 2020 01:09:26 +0200 Subject: [PATCH 498/605] gitlab: update hotdoc Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ed5035cf6e..475cf51157 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1073,7 +1073,7 @@ documentation: MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@95aa79758f64281de3f0e6d2efde45108d8cd2ea + - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@28ad9aa59daa49d6b3fd1bc6b144df52af39475b - *gst_build - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches - | From af11a2cd52c4507bfbe2fa540fd8a42d0af4de77 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 10 Jul 2020 22:29:37 +0530 Subject: [PATCH 499/605] gitlab: Support android tarballs compressed with xz Needed for https://gitlab.freedesktop.org/gstreamer/cerbero/-/merge_requests/540. Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 475cf51157..bbb129952c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -945,9 +945,9 @@ build cerbero cross win64: - rm clone_manifest_ref.py # extract our binaries - - rm -f gstreamer-1.0-android-universal-*-runtime.tar.bz2 + - rm -f gstreamer-1.0-android-universal-*-runtime.tar.* - mkdir ${GSTREAMER_ROOT_ANDROID} - - tar -C ${GSTREAMER_ROOT_ANDROID} -xf gstreamer-1.0-android-universal-*.tar.bz2 + - time tar -C ${GSTREAMER_ROOT_ANDROID} -xf gstreamer-1.0-android-universal-*.tar.* # gst-examples - player - chmod +x ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew From a892850eb21f9043e7b385b2fe00b3516d0b02d2 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 10 Jun 2020 17:47:08 +0300 Subject: [PATCH 500/605] docker/fedora: update base to fedora 31 Part-of: --- docker/fedora/Dockerfile | 2 +- docker/fedora/prepare.sh | 5 ++--- gitlab/ci_template.yml | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index 82abdd3a40..624cfcdcb8 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:30 +FROM registry.fedoraproject.org/fedora:31 ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index ad95512772..410f48c089 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -4,7 +4,7 @@ set -eux # We need them to cleanly build our doc. sed -i "s/tsflags=nodocs//g" /etc/dnf/dnf.conf -dnf install -y git-core ninja-build dnf-plugins-core +dnf install -y git-core ninja-build dnf-plugins-core python3-pip # Configure git for various usage git config --global user.email "gst-build@gstreamer.net" @@ -21,7 +21,6 @@ dnf upgrade -y dnf install -y \ aalib-devel \ aom \ - aom-extra-tools \ bat \ libaom \ libaom-devel \ @@ -47,6 +46,7 @@ dnf install -y \ graphene-devel \ gsl \ gsl-devel \ + faac-devel \ ffmpeg \ ffmpeg-libs \ ffmpeg-devel \ @@ -184,7 +184,6 @@ dnf builddep -y gstreamer1 \ gstreamer1-plugins-good-extras \ gstreamer1-plugins-ugly \ gstreamer1-plugins-ugly-free \ - gstreamer1-plugins-bad-nonfree \ gstreamer1-plugins-bad-free \ gstreamer1-plugins-bad-free-extras \ gstreamer1-plugins-bad-freeworld \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index bbb129952c..8b34b16d51 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -18,7 +18,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2020-07-03.0' + FEDORA_TAG: '2020-07-03.1' CERBERO_TAG: '2020-06-25.0' ANDROID_TAG: '2020-06-25.0' MANIFEST_TAG: '2020-06-25.0' From dc486667426d0d31284e25001cfece45879595a8 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 23 Jul 2020 15:16:55 +0300 Subject: [PATCH 501/605] citemplate: revert the docs job to the previous image build The update to f31 from !271 triggered an error in hotdoc it seems, so rollback for now till we figure it out. https://gitlab.freedesktop.org/gdesmott/gst-plugins-base/-/jobs/3764215 Part-of: --- gitlab/ci_template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8b34b16d51..b819b3ba9c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -19,6 +19,7 @@ variables: # to change any part of the string of the image you want. ### FEDORA_TAG: '2020-07-03.1' + FEDORA_DOCS_TAG: '2020-07-03.0' CERBERO_TAG: '2020-06-25.0' ANDROID_TAG: '2020-06-25.0' MANIFEST_TAG: '2020-06-25.0' @@ -35,6 +36,7 @@ variables: INDENT_AMD64_SUFFIX: 'amd64/gst-indent' FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" + FEDORA_DOCS_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_DOCS_TAG-$GST_UPSTREAM_BRANCH" CERBERO_IMAGE: "$CI_REGISTRY_IMAGE/$CERBERO_AMD64_SUFFIX:$CERBERO_TAG-$GST_UPSTREAM_BRANCH" ANDROID_IMAGE: "$CI_REGISTRY_IMAGE/$ANDROID_AMD64_SUFFIX:$ANDROID_TAG-$GST_UPSTREAM_BRANCH" MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$MANIFEST_AMD64_SUFFIX:$MANIFEST_TAG-$GST_UPSTREAM_BRANCH" @@ -1060,7 +1062,7 @@ build cerbero cross-ios universal: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' documentation: - image: $FEDORA_IMAGE + image: $FEDORA_DOCS_IMAGE extends: - '.gst_build_ccache_vars' rules: From fe54d6d8cde8a271262246fb1656e4abeb6416b7 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 24 Jul 2020 18:53:39 +0300 Subject: [PATCH 502/605] citemplate: properly pin the image of the docs job we don't need the image to be copied or rebuilt, as its currently pinned shortly and won't change anytime soon. ammends dc486667426d0d31284e25001cfece45879595a8 !324 Part-of: --- gitlab/ci_template.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index b819b3ba9c..cffaec6343 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -19,7 +19,6 @@ variables: # to change any part of the string of the image you want. ### FEDORA_TAG: '2020-07-03.1' - FEDORA_DOCS_TAG: '2020-07-03.0' CERBERO_TAG: '2020-06-25.0' ANDROID_TAG: '2020-06-25.0' MANIFEST_TAG: '2020-06-25.0' @@ -36,7 +35,7 @@ variables: INDENT_AMD64_SUFFIX: 'amd64/gst-indent' FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" - FEDORA_DOCS_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_DOCS_TAG-$GST_UPSTREAM_BRANCH" + FEDORA_DOCS_IMAGE: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-07-03.0-master" CERBERO_IMAGE: "$CI_REGISTRY_IMAGE/$CERBERO_AMD64_SUFFIX:$CERBERO_TAG-$GST_UPSTREAM_BRANCH" ANDROID_IMAGE: "$CI_REGISTRY_IMAGE/$ANDROID_AMD64_SUFFIX:$ANDROID_TAG-$GST_UPSTREAM_BRANCH" MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$MANIFEST_AMD64_SUFFIX:$MANIFEST_TAG-$GST_UPSTREAM_BRANCH" From c44f415a4bf1f0253aff461ea2bb3e143c94c5e1 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 7 Jul 2020 19:25:19 +0300 Subject: [PATCH 503/605] citemplate: make it so docker builds only run for detached pipelines Currently, if you have a MR open, there 2 pipelines being triggered. One normal, and one detached. Previously, if you were to rebuild an image, the jobs of the docker build stage would be executed concurrently, race and end up both rebuilding the image. Make them manual for normal pipelines to avoid such occurrence and waste of resources. Part-of: --- gitlab/ci_template.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index cffaec6343..7255cbcadf 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -195,7 +195,14 @@ default: .base: image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24" rules: - - when: 'always' + # Always run for detached pipeleines + - if: '$CI_MERGE_REQUEST_ID' + when: 'always' + - if: '$CI_COMMIT_BRANCH == "master"' + when: 'always' + - if: '$CI_COMMIT_BRANCH == "$GST_UPS_BRANCH"' + when: 'always' + - when: 'manual' variables: STORAGE_DRIVER: 'vfs' BUILDAH_FORMAT: 'docker' From 91c85b7ca9fe9d2235bb05f9ede513a263871614 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sun, 26 Jul 2020 13:01:06 +0300 Subject: [PATCH 504/605] citemplate: skip integration testsuites for gstreamer-sharp Close #80 Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7255cbcadf..66264f2197 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -544,7 +544,7 @@ integration testsuites fedora: EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs --parts=${CI_NODE_TOTAL} --part-index=${CI_NODE_INDEX}" TEST_SUITE: "validate ges" rules: - - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-omx|gst-plugins-rs)$/' + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-omx|gstreamer-sharp|gst-plugins-rs)$/' gstreamer-full: extends: 'build static fedora x86_64' From 1ab716b71d79f03f75c4fcea437fbb6ed5f85268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Wed, 5 Aug 2020 00:32:17 +0100 Subject: [PATCH 505/605] citemplate: add support for .indentignore file to indent job See https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1465 Part-of: --- gitlab/ci_template.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 66264f2197..ad15cc62e8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -358,7 +358,20 @@ gst indent: script: # man indent. grep RETURN VALUE, grab a beer on my behalf... - indent --version || true - - find . -name '*.c' -exec gst-indent {} + + - | + filter_cmd="cat" + if test -f ".indentignore"; then + filter_args="" + while read -r line; do + if test -n "$line"; then + filter_args="$filter_args -e $line" + fi + done < ".indentignore" + if test -n "$filter_args"; then + filter_cmd="grep -v $filter_args" + fi + fi + find . -name '*.c' | $filter_cmd | xargs gst-indent - | if git diff --quiet; then echo "Code is properly formatted" From 159e0e81341e15bc10e82378e502e4f2f2e65c9c Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 6 Aug 2020 01:34:18 +0530 Subject: [PATCH 506/605] citemplate: Allow spaces in filenames when indenting Part-of: --- gitlab/ci_template.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ad15cc62e8..147d34225c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -359,19 +359,19 @@ gst indent: # man indent. grep RETURN VALUE, grab a beer on my behalf... - indent --version || true - | - filter_cmd="cat" + filter_cmd=("cat") if test -f ".indentignore"; then - filter_args="" + filter_args=() while read -r line; do if test -n "$line"; then - filter_args="$filter_args -e $line" + filter_args+=("-e" "$line") fi done < ".indentignore" - if test -n "$filter_args"; then - filter_cmd="grep -v $filter_args" + if [[ ${#filter_args[@]} -gt 0 ]]; then + filter_cmd=("grep" "-v" "${filter_args[@]}") fi fi - find . -name '*.c' | $filter_cmd | xargs gst-indent + find . -name '*.c' | "${filter_cmd[@]}" | xargs -d '\n' gst-indent - | if git diff --quiet; then echo "Code is properly formatted" From 524a75e6ff47731915cecd0b9be3d830d1dcc2bd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 27 Aug 2020 15:35:12 +0300 Subject: [PATCH 507/605] ci_template: remove rules from the docker build We can run the jobs always automatically since, most of the time they will be copying/reffing the existance image in the registry. Part-of: --- .gitlab-ci.yml | 2 -- gitlab/ci_template.yml | 9 --------- 2 files changed, 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 048418c0f9..c911c1af4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,8 +3,6 @@ include: "gitlab/ci_template.yml" test manifest: variables: GIT_STRATEGY: fetch - rules: - - when: 'always' image: "$TEST_MANIFEST_IMAGE" stage: "preparation" script: diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 147d34225c..f9197a0075 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -194,15 +194,6 @@ default: # All the documentation from citemplates should be applicable here, so please refer there .base: image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24" - rules: - # Always run for detached pipeleines - - if: '$CI_MERGE_REQUEST_ID' - when: 'always' - - if: '$CI_COMMIT_BRANCH == "master"' - when: 'always' - - if: '$CI_COMMIT_BRANCH == "$GST_UPS_BRANCH"' - when: 'always' - - when: 'manual' variables: STORAGE_DRIVER: 'vfs' BUILDAH_FORMAT: 'docker' From 2a3589dc2df72ae4c91bd9cca284e6ec7342fab4 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 26 Aug 2020 14:55:58 +0300 Subject: [PATCH 508/605] citemplate: rework gitlab rules Rework rules of jobs to prever duplicate pipelines. https://docs.gitlab.com/ce/ci/yaml/README.html#prevent-duplicate-pipelines Part-of: --- gitlab/ci_template.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f9197a0075..d36b4a432e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1,3 +1,6 @@ +include: + - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' + stages: - 'build docker' - 'preparation' @@ -318,13 +321,23 @@ gst-indent amd64 docker: manifest: image: $MANIFEST_IMAGE rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $GITLAB_USER_LOGIN == "gstreamer-merge-bot"' - when: 'always' + # Always have automatic pipeline for branchs in cerbero and docs + # cause they need to update artifacts, like the docs site or cerbero deps - if: '$CI_PROJECT_PATH == "gstreamer/cerbero"' - when: 'always' - if: '$CI_PROJECT_PATH == "gstreamer/gst-docs"' - when: 'always' - - when: 'manual' + + # If the user that triggered the Pipeline is the Merge bot and the branch doesn't match + # the upstream branch set, run the pipeline + - if: '$GITLAB_USER_LOGIN == "gstreamer-merge-bot" && $CI_COMMIT_BRANCH != "$GST_UPS_BRANCH"' + # When the user isn't the merge bot, require an explicit action to trigger the pipeline + # to avoid wasting CI resources + - if: '$GITLAB_USER_LOGIN != "gstreamer-merge-bot"' + when: 'manual' + # If this matches, it means the pipeline is running against either the main + # or a stable branch, so make it manual + - if: '$CI_COMMIT_BRANCH == "$GST_UPS_BRANCH"' + when: 'manual' + stage: 'preparation' script: - cd /gst-ci From dd43637fcea3ea76f2eb4af7cec237fe090fb768 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 9 Sep 2020 16:00:54 +0300 Subject: [PATCH 509/605] ci_template: correctly spell the global var Close #83 Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d36b4a432e..f30bf5534e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -328,14 +328,14 @@ manifest: # If the user that triggered the Pipeline is the Merge bot and the branch doesn't match # the upstream branch set, run the pipeline - - if: '$GITLAB_USER_LOGIN == "gstreamer-merge-bot" && $CI_COMMIT_BRANCH != "$GST_UPS_BRANCH"' + - if: '$GITLAB_USER_LOGIN == "gstreamer-merge-bot" && $CI_COMMIT_BRANCH != "$GST_UPSTREAM_BRANCH"' # When the user isn't the merge bot, require an explicit action to trigger the pipeline # to avoid wasting CI resources - if: '$GITLAB_USER_LOGIN != "gstreamer-merge-bot"' when: 'manual' # If this matches, it means the pipeline is running against either the main # or a stable branch, so make it manual - - if: '$CI_COMMIT_BRANCH == "$GST_UPS_BRANCH"' + - if: '$CI_COMMIT_BRANCH == "$GST_UPSTREAM_BRANCH"' when: 'manual' stage: 'preparation' From 58df3e6f5832ad118a20f9bf229d484e48e487ec Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 9 Sep 2020 17:34:39 +0300 Subject: [PATCH 510/605] citemplate: update the buildah bootstrap image Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f30bf5534e..858b74f845 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -196,7 +196,7 @@ default: # # All the documentation from citemplates should be applicable here, so please refer there .base: - image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24" + image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-07-20.1" variables: STORAGE_DRIVER: 'vfs' BUILDAH_FORMAT: 'docker' From 0f50f3901e02a84dfaf1bc8f1d3b6603697b8939 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 9 Sep 2020 17:31:08 +0300 Subject: [PATCH 511/605] cerbero/fedora: install which into the image We used to get this as transient dependency it seems, which broke rebuilds of the img once w/e pkg was depending on it, stopped. Part-of: --- docker/cerbero/Dockerfile-fedora | 2 +- gitlab/ci_template.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 2347399ebc..2e8c0e3703 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -7,7 +7,7 @@ ARG DEFAULT_BRANCH="master" # System setup RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \ dnf upgrade -y && \ - dnf install -y sudo git lbzip2 rsync wine + dnf install -y sudo git lbzip2 rsync wine which # Configure git for various usage RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 858b74f845..74073edffa 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -22,8 +22,8 @@ variables: # to change any part of the string of the image you want. ### FEDORA_TAG: '2020-07-03.1' - CERBERO_TAG: '2020-06-25.0' - ANDROID_TAG: '2020-06-25.0' + CERBERO_TAG: '2020-09-09.2' + ANDROID_TAG: '2020-09-09.2' MANIFEST_TAG: '2020-06-25.0' TEST_MANIFEST_TAG: '2020-06-25.0' INDENT_TAG: '2020-06-29.0' From 800c46e67c3c024165caa9a623f20a34ff2032a9 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 28 Aug 2020 19:30:41 +0300 Subject: [PATCH 512/605] docker/windows: tweak installation of git Make sure we set autocrlf to false, which is needed by cerbero. Additionally remove useless for the container integrations with the shell. Part-of: --- docker/windows/install_choco.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/windows/install_choco.ps1 b/docker/windows/install_choco.ps1 index f1a7c380a3..9fb9d13b41 100644 --- a/docker/windows/install_choco.ps1 +++ b/docker/windows/install_choco.ps1 @@ -2,4 +2,5 @@ Write-Host "Installing Choco" Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) Write-Host "Installing Choco packages" choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' -choco install -y python3 git git-lfs 7zip \ No newline at end of file +choco install -y git --params "/NoAutoCrlf /NoCredentialManager /NoShellHereIntegration /NoGuiHereIntegration /NoShellIntegration" +choco install -y python3 git-lfs 7zip From 894e72ee87c63a3672c76bfcb22778e32dccce07 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 1 Sep 2020 21:30:29 +0300 Subject: [PATCH 513/605] docker/windows: move the git user config outside the script Part-of: --- docker/windows/Dockerfile | 2 ++ docker/windows/prepare_gst_env.ps1 | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 899fcfa33e..882082ac86 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -9,6 +9,8 @@ ENV ErrorActionPreference='Stop' COPY install_choco.ps1 C:\ RUN C:\install_choco.ps1 +RUN 'git config --global user.email "cirunner@gstreamer.freedesktop.org"; git config --global user.name "GStreamer CI system"' + COPY install_toolchain.ps1 C:\ RUN C:\install_toolchain.ps1 diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1 index 61b30e1f29..f5316e9922 100644 --- a/docker/windows/prepare_gst_env.ps1 +++ b/docker/windows/prepare_gst_env.ps1 @@ -1,8 +1,5 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; -git config --global user.email "gst-build@gstreamer.net" -git config --global user.name "Gstbuild Runner" - # FIXME: Python fails to validate github.com SSL certificate, unless we first # run a dummy download to force refreshing Windows' CA database. # See: https://bugs.python.org/issue36137 From fd7b78a91835e441d685c5cac3cce5747a7dc2a2 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 10 Sep 2020 21:10:37 +0300 Subject: [PATCH 514/605] docker/fedora: install more debug symbols These are needed for the webrtcbin tests Part-of: --- docker/fedora/prepare.sh | 9 +++++++++ gitlab/ci_template.yml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 410f48c089..1024be6571 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -46,6 +46,12 @@ dnf install -y \ graphene-devel \ gsl \ gsl-devel \ + gupnp \ + gupnp-devel \ + gupnp-igd \ + gupnp-igd-devel \ + gssdp \ + gssdp-devel \ faac-devel \ ffmpeg \ ffmpeg-libs \ @@ -120,6 +126,9 @@ dnf install -y \ dnf debuginfo-install -y gtk3 \ glib2 \ glibc \ + gupnp \ + gupnp-igd \ + gssdp \ freetype \ openjpeg \ gobject-introspection \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 74073edffa..ebbb52c7aa 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,7 +21,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2020-07-03.1' + FEDORA_TAG: '2020-09-15.0' CERBERO_TAG: '2020-09-09.2' ANDROID_TAG: '2020-09-09.2' MANIFEST_TAG: '2020-06-25.0' From 050aec524bd5d64ee57a5f9eddef89eca8840cb5 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 15 Sep 2020 16:58:40 +0300 Subject: [PATCH 515/605] citemplate: workaround stale subproject cache The revision of the wrap for pygobject was switched recently [1] [1]: https://gitlab.freedesktop.org/gstreamer/gst-build/-/merge_requests/189 Part-of: --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ebbb52c7aa..cbc064a8f6 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1099,6 +1099,10 @@ documentation: CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@28ad9aa59daa49d6b3fd1bc6b144df52af39475b + + # FIXME: remove this once the docs image is updated + - rm -rf /gst-build/subprojects/pygobject/ + - *gst_build - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches - | From 7d46f1ac5e2c424601555f7522cb95e62b8ea5ba Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 6 Aug 2020 20:21:44 +0300 Subject: [PATCH 516/605] ci_template: move the cerbero scripts from yaml to a new file Its hard to manage ever growing bash scripts in yaml, and its even harder to run them locally to reproduce the environment. This is essentially a copy-paste of the scripts, to make review easier. We can refactor later. Part-of: --- gitlab/cerbero_setup.sh | 72 +++++++++++++++++++++++++++++++++++++++++ gitlab/ci_template.yml | 57 ++++++-------------------------- 2 files changed, 81 insertions(+), 48 deletions(-) create mode 100644 gitlab/cerbero_setup.sh diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh new file mode 100644 index 0000000000..191f4540e7 --- /dev/null +++ b/gitlab/cerbero_setup.sh @@ -0,0 +1,72 @@ +set -ex + +show_ccache_sum() { + if [[ -n ${HAVE_CCACHE} ]]; then + ccache -s + fi +} + +# Produces runtime and devel tarball packages for linux/android or .pkg for macos +cerbero_package_and_check() { + $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd)" gstreamer-1.0 + + # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. + [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version + [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX + test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test +} + +cerbero_before_script() { + # FIXME Wrong namespace + # Workaround build-tools having hardcoded internal path + pwd + mkdir -p "../../gstreamer" + ln -sf "$(pwd)" "../../gstreamer/cerbero" + mkdir -p "../../${CI_PROJECT_NAMESPACE}" + ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + rsync -aH "${CERBERO_HOST_DIR}" . + echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc + echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc + ./cerbero-uninstalled --self-update manifest.xml +} + +cerbero_script() { + test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats + $CERBERO $CERBERO_ARGS show-config + $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only + $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 + $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}" + + if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then + test -d "${CERBERO_HOME}/dist/${ARCH}" + mkdir -p "${CERBERO_OVERRIDDEN_DIST_DIR}" + rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}" + fi + + $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only + cerbero_package_and_check +} + +cerbero_deps_script() { + test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats + $CERBERO $CERBERO_ARGS show-config + $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only + $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 + $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only + $CERBERO $CERBERO_ARGS build-deps --offline \ + gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \ + gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \ + gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice + test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" \ + && mkdir -p "${CERBERO_HOME}/dist/${ARCH}" \ + && rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}" + + $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH} + + test "x${CERBERO_PRIVATE_SSH_KEY}" = "x" \ + || $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH} + cerbero_package_and_check +} + +# Run whichever function is asked of us +eval "$1" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index cbc064a8f6..bcdb6fbeab 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -47,6 +47,10 @@ variables: WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v11-master' + # FIXME: after merging the script + # CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" + CERBERO_SCRIPT_URL: 'https://gitlab.freedesktop.org/alatiera/gst-ci/-/raw/alatiera/cerbero-script-2/gitlab/cerbero_setup.sh' + GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both DEFAULT_MESON_ARGS: > @@ -648,16 +652,6 @@ valgrind ges: # CONFIG: The name of the configuration file to use # ARCH: The cerbero _ (used in cache key) # -# Produces runtime and devel tarball packages for linux/android or .pkg for macos -.cerbero template: &cerbero_package_and_check - - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o $(pwd) gstreamer-1.0 - # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. - - |- - [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version - - |- - [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX - - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test - .cerbero: stage: "build" image: $CERBERO_IMAGE @@ -690,29 +684,11 @@ valgrind ges: # location where cerbero is cached on the host CERBERO_HOST_DIR: "/cerbero/" before_script: - # FIXME Wrong namespace - # Workaround build-tools having hardcoded internal path - - pwd - - mkdir -p ../../gstreamer - - ln -sf $(pwd) ../../gstreamer/cerbero - - mkdir -p ../../${CI_PROJECT_NAMESPACE} - - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero - - rsync -aH ${CERBERO_HOST_DIR} . - - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc - - ./cerbero-uninstalled --self-update manifest.xml + - curl -L -o cerbero_script.sh "${CERBERO_SCRIPT_URL}" + - chmod +x cerbero_script.sh + - ./cerbero_script.sh cerbero_before_script script: - - test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats - - $CERBERO $CERBERO_ARGS show-config - - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} - - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" - && test -d ${CERBERO_HOME}/dist/${ARCH} - && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR} - && rsync -aH ${CERBERO_HOME}/dist/${ARCH}/ ${CERBERO_OVERRIDDEN_DIST_DIR} - - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only - - *cerbero_package_and_check + - ./cerbero_script.sh cerbero_script cache: key: "${CI_JOB_NAME}" paths: @@ -856,22 +832,7 @@ build msys2 : rules: - if: '$CI_PROJECT_NAME == "cerbero"' script: - - test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats - - $CERBERO $CERBERO_ARGS show-config - - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only - - $CERBERO $CERBERO_ARGS build-deps --offline - gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 - gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 - gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice - - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" - && mkdir -p ${CERBERO_HOME}/dist/${ARCH} - && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH} - - $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH} - - test "x${CERBERO_PRIVATE_SSH_KEY}" = "x" - || $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH} - - *cerbero_package_and_check + - ./cerbero_script.sh cerbero_deps_script # # Cerbero Linux X86_64 build From 6ffc323ff5ed88ef3c2ca9aebcfbbfc276b283fd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 6 Aug 2020 20:48:01 +0300 Subject: [PATCH 517/605] gitlab/cerbero_setup.sh: refactor some tests to if/else statments and replace the ccache check Part-of: --- gitlab/cerbero_setup.sh | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 191f4540e7..364aba5d99 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -11,9 +11,12 @@ cerbero_package_and_check() { $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd)" gstreamer-1.0 # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. - [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version - [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true # eat the return value from the failing test + if [[ $CONFIG != *universal* ]]; then + $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version + $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX + fi + + show_ccache_sum } cerbero_before_script() { @@ -31,7 +34,8 @@ cerbero_before_script() { } cerbero_script() { - test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats + show_ccache_sum + $CERBERO $CERBERO_ARGS show-config $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 @@ -48,7 +52,8 @@ cerbero_script() { } cerbero_deps_script() { - test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats + show_ccache_sum + $CERBERO $CERBERO_ARGS show-config $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 @@ -57,14 +62,17 @@ cerbero_deps_script() { gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \ gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \ gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" \ - && mkdir -p "${CERBERO_HOME}/dist/${ARCH}" \ - && rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}" - $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH} + if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then + mkdir -p "${CERBERO_HOME}/dist/${ARCH}" + rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}" + fi + + if [[ -n ${CERBERO_PRIVATE_SSH_KEY} ]]; then + $CERBERO $CERBERO_ARGS gen-cache --branch "${GST_UPSTREAM_BRANCH}" + $CERBERO $CERBERO_ARGS upload-cache --branch "${GST_UPSTREAM_BRANCH}" + fi - test "x${CERBERO_PRIVATE_SSH_KEY}" = "x" \ - || $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH} cerbero_package_and_check } From b857a5ada178563f53dd65bbdbf58b021b4ff3f2 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 7 Aug 2020 15:29:39 +0300 Subject: [PATCH 518/605] citemplate: add initial cerbero msvc builds This commit adds the last bits needed to have a working CI job of cerbero using mingw and msvc17 toolchain. It adds a new stage to the docker image that mirrors the setup of the cerbero fedora docker build, and tweaks the cerbero_setup script run by jobs with a couple windows specific fixes. Close #39 Part-of: --- docker/windows/Dockerfile | 3 ++ docker/windows/build_image.ps1 | 4 +-- docker/windows/prepare_cerbero_env.sh | 29 +++++++++++++++++ gitlab/cerbero_setup.sh | 45 +++++++++++++++++++++++-- gitlab/ci_template.yml | 47 +++++++++++++++++++++++++-- 5 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 docker/windows/prepare_cerbero_env.sh diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 882082ac86..e36e4fa01c 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -18,3 +18,6 @@ ARG DEFAULT_BRANCH="master" COPY prepare_gst_env.ps1 C:\ RUN C:\prepare_gst_env.ps1 + +COPY prepare_cerbero_env.sh C:\ +RUN C:\MinGW\msys\1.0\bin\bash.exe --login -c "C:/prepare_cerbero_env.sh" diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 490ecc6813..c092f6d75b 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,6 +1,6 @@ $env:DEFAULT_BRANCH='master' -$env:VERSION='v11' +$env:VERSION='v12' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" echo "Building $env:tag" docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . -# docker push $tag +# docker push $env:tag diff --git a/docker/windows/prepare_cerbero_env.sh b/docker/windows/prepare_cerbero_env.sh new file mode 100644 index 0000000000..446ed24228 --- /dev/null +++ b/docker/windows/prepare_cerbero_env.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +set -eux + +cd C:/ +git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git +cd cerbero + +echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc +echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc +echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc +echo 'vs_install_version = "vs15"' >> localconf.cbc + +# Fetch all bootstrap requirements +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-bootstrap +# Fetch all package requirements for a mingw gstreamer build +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0 +# Fetch all package requirements for a visualstudio gstreamer build +./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0 + +# Extract the toolchain and fixup the MSYS installation +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --system-only --offline + +# Delete mingw toolchain binary tarball +rm /c/cerbero/cerbero-sources/mingw-*.tar.xz +# Wipe visualstudio package dist, sources, logs, and the build tools recipes +./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc wipe --force --build-tools +# clean the localconf +rm /c/cerbero/localconf.cbc diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 364aba5d99..c4e52513dd 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -8,7 +8,14 @@ show_ccache_sum() { # Produces runtime and devel tarball packages for linux/android or .pkg for macos cerbero_package_and_check() { - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd)" gstreamer-1.0 + # FIXME: mingw translates $(pwd) in a broken way + if [[ $CONFIG == win??.cbc ]]; then + PACKAGE_PATH="$CI_PROJECT_DIR" + else + PACKAGE_PATH=$(pwd) + fi + + $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$PACKAGE_PATH" gstreamer-1.0 # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. if [[ $CONFIG != *universal* ]]; then @@ -27,9 +34,41 @@ cerbero_before_script() { ln -sf "$(pwd)" "../../gstreamer/cerbero" mkdir -p "../../${CI_PROJECT_NAMESPACE}" ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + + # Make sure there isn't a pre-existing config hanging around + rm -v -f localconf.cbc + rm -v -f ${CERBERO_HOST_DIR}/localconf.cbc + + if [[ $CONFIG == win??.cbc ]]; then + # For windows hardcode the path so it doesn't get + # mangled by msys path handling + # FIXME: make the sources point to pwd/$CI_PROJECT_DIR like the rest + echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc + echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc + + # Visual Studio 2017 build tools install path + echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc + echo 'vs_install_version = "vs15"' >> localconf.cbc + else + echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" > localconf.cbc + echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc + fi + + cat localconf.cbc + rsync -aH "${CERBERO_HOST_DIR}" . - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc + + cat localconf.cbc + + # FIXME: if you comment out this line it fails like so, no clue why. Its not windows defender either. + # From https://gitlab.freedesktop.org/gstreamer/cerbero + # b02080cb..d6923e42 master -> origin/master + # Fetching origin + # error: unable to create file cerbero-uninstalled: Permission denied + # fatal: Could not reset index file to revision 'd6923e4216c8a17759527a3db070d15cf7ff10a0'. + # ERROR: Failed to proceed with self update Command Error: Running ['git', 'reset', '--hard', 'd6923e4216c8a17759527a3db070d15cf7ff10a0'] returned 128 + git status + ./cerbero-uninstalled --self-update manifest.xml } diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index bcdb6fbeab..90a31783c1 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -45,11 +45,11 @@ variables: TEST_MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$TEST_MANIFEST_AMD64_SUFFIX:$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v11-master' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v12-master' # FIXME: after merging the script # CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" - CERBERO_SCRIPT_URL: 'https://gitlab.freedesktop.org/alatiera/gst-ci/-/raw/alatiera/cerbero-script-2/gitlab/cerbero_setup.sh' + CERBERO_SCRIPT_URL: 'https://gitlab.freedesktop.org/alatiera/gst-ci/-/raw/alatiera/cerbero-windows-attempt-6/gitlab/cerbero_setup.sh' GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both @@ -1178,3 +1178,46 @@ build gst-omx tizonia fedora x86_64: - ninja -C build - ninja -C build install - cd .. + +# +# Cerbero Native Windows builds +# +.cerbero windows native: + needs: ['manifest'] + image: $WINDOWS_IMAGE + tags: + - 'docker' + - 'windows' + - 'gstreamer-windows' + - '1809' + variables: + CONFIG: 'win64.cbc' + ARCH: 'msvc_x86_64' + # rsync doesn't like windows paths like `C:/cerbero` + CERBERO_HOST_DIR: "/c/cerbero/" + CERBERO_HOME: "C:/cerbero-build" + CERBERO_SOURCES: "C:/cerbero-sources" + CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v visualstudio -v nowerror" + CERBERO_RUN_SUFFIX: ".exe" + HAVE_CCACHE: "" + # FIXME: for some reason the runner is hanging when trying to upload artifacts + artifacts: + paths: [] + before_script: + - $env:CI_PROJECT_DIR = $env:CI_PROJECT_DIR.replace('\', '/') + - $env:CERBERO_SCRIPTS_PATH = "$env:CI_PROJECT_DIR/cerbero_setup.sh" + + - Invoke-WebRequest -Uri $env:CERBERO_SCRIPT_URL -OutFile $env:CERBERO_SCRIPTS_PATH + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_before_script" + +cerbero deps msvc x86_64: + extends: ['.cerbero deps', '.cerbero windows native'] + script: + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_deps_script" + +build cerbero msvc x86_64: + extends: ['.cerbero', '.cerbero windows native'] + script: + - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_script" + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' From 260935e39eb6df8aec6c12b2d175f1b959421137 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 17 Sep 2020 02:14:55 +0300 Subject: [PATCH 519/605] citemplate: post-merge fix for url path switch it so it looks at the stable branch for the cerbero_setup script. Part-of: --- gitlab/ci_template.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 90a31783c1..7fd4d93519 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -47,9 +47,7 @@ variables: WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v12-master' - # FIXME: after merging the script - # CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" - CERBERO_SCRIPT_URL: 'https://gitlab.freedesktop.org/alatiera/gst-ci/-/raw/alatiera/cerbero-windows-attempt-6/gitlab/cerbero_setup.sh' + CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From bdbec90a4939d5c31f800cc9f07646c820a7bd76 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 17 Sep 2020 16:22:06 +0530 Subject: [PATCH 520/605] Switch MSVC job to manual till we can upload deps cache The deps cache job is failing: https://gitlab.freedesktop.org/gstreamer/cerbero/-/jobs/4568410 Because of which MRs in other repos are unable to download the cache and are building everything from scratch, which is inflating CI times to 2 hours: https://gitlab.freedesktop.org/marian/gst-plugins-bad/-/jobs/4567025 Part-of: --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7fd4d93519..d1d9a6404e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1219,3 +1219,5 @@ build cerbero msvc x86_64: - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_script" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + when: 'manual' + allow_failure: true From f24e7b721af338094dc2ced17d9fbcf0da4934a6 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 17 Sep 2020 21:51:49 +0530 Subject: [PATCH 521/605] cerbero ci: Don't expand the ssh private key Because of set -x we were printing the key into the CI logs: https://gitlab.freedesktop.org/gstreamer/cerbero/-/jobs/4572836#L797 Don't ever expand the variable in shell. --- gitlab/cerbero_setup.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index c4e52513dd..48a8a2857a 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -107,7 +107,10 @@ cerbero_deps_script() { rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}" fi - if [[ -n ${CERBERO_PRIVATE_SSH_KEY} ]]; then + # Check that the env var is set. Don't expand this protected variable by + # doing something silly like [[ -n ${CERBERO_...} ]] because it will get + # printed in the CI logs due to set -x + if env | grep -q -e CERBERO_PRIVATE_SSH_KEY; then $CERBERO $CERBERO_ARGS gen-cache --branch "${GST_UPSTREAM_BRANCH}" $CERBERO $CERBERO_ARGS upload-cache --branch "${GST_UPSTREAM_BRANCH}" fi From dab27ee564593777c2bcca614d1fe606867cca8f Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 17 Sep 2020 16:36:59 +0530 Subject: [PATCH 522/605] cerbero ci: Try to fix deps build on the gstreamer namespace Can't ln twice: it does a cp on MSYS. Part-of: --- gitlab/cerbero_setup.sh | 10 ++++++++-- gitlab/ci_template.yml | 6 ++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 48a8a2857a..f9f8f0546f 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -1,3 +1,5 @@ +#!/bin/bash + set -ex show_ccache_sum() { @@ -32,8 +34,12 @@ cerbero_before_script() { pwd mkdir -p "../../gstreamer" ln -sf "$(pwd)" "../../gstreamer/cerbero" - mkdir -p "../../${CI_PROJECT_NAMESPACE}" - ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + # Don't try to symlink twice because on MSYS `ln` does a `cp` since it + # doesn't support the new NTFS symlink feature. + if [[ ${CI_PROJECT_NAMESPACE} != gstreamer ]]; then + mkdir -p "../../${CI_PROJECT_NAMESPACE}" + ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + fi # Make sure there isn't a pre-existing config hanging around rm -v -f localconf.cbc diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d1d9a6404e..2c3f32664d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -47,7 +47,8 @@ variables: WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v12-master' - CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" + # FIXME: reset back to gstreamer/master after merging + CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/nirbheek/gst-ci/raw/fix-msvc-deps-job/gitlab/cerbero_setup.sh" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both @@ -682,6 +683,7 @@ valgrind ges: # location where cerbero is cached on the host CERBERO_HOST_DIR: "/cerbero/" before_script: + - echo "Fetching ${CERBERO_SCRIPT_URL}" - curl -L -o cerbero_script.sh "${CERBERO_SCRIPT_URL}" - chmod +x cerbero_script.sh - ./cerbero_script.sh cerbero_before_script @@ -1204,7 +1206,7 @@ build gst-omx tizonia fedora x86_64: before_script: - $env:CI_PROJECT_DIR = $env:CI_PROJECT_DIR.replace('\', '/') - $env:CERBERO_SCRIPTS_PATH = "$env:CI_PROJECT_DIR/cerbero_setup.sh" - + - echo "Fetching $env:CERBERO_SCRIPT_URL" - Invoke-WebRequest -Uri $env:CERBERO_SCRIPT_URL -OutFile $env:CERBERO_SCRIPTS_PATH - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_before_script" From 555c49f9d17aa7a8b9e650941002b52e4e6dd325 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 17 Sep 2020 23:50:56 +0530 Subject: [PATCH 523/605] cerbero ci: Also disable the deps build There seems to be some issues uploading the cache from the Windows runner. Part-of: --- gitlab/ci_template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2c3f32664d..e90d86f4c2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1214,6 +1214,10 @@ cerbero deps msvc x86_64: extends: ['.cerbero deps', '.cerbero windows native'] script: - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_deps_script" + rules: + - if: '$CI_PROJECT_NAME == "cerbero"' + when: 'manual' + allow_failure: true build cerbero msvc x86_64: extends: ['.cerbero', '.cerbero windows native'] From 01c53ebf020e10a46d8ba91104edf062ab322c66 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 18 Sep 2020 00:36:55 +0530 Subject: [PATCH 524/605] ci: Switch cerbero script URL back to gst-ci master Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e90d86f4c2..88fb040b95 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -47,8 +47,8 @@ variables: WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v12-master' - # FIXME: reset back to gstreamer/master after merging - CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/nirbheek/gst-ci/raw/fix-msvc-deps-job/gitlab/cerbero_setup.sh" + # Can't use $CI_* variables since we use this template in all repos, not just gst-ci + CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From eed018920c85297baa79f3146835c5a55cf89af9 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 19 Sep 2020 11:58:37 +0530 Subject: [PATCH 525/605] cerbero ci: Sprinkle some 'time' commands in the script Sprinkle some 'time' commands so we can measure how long things take and we can know where to spend time optimizing first. Part-of: --- gitlab/cerbero_setup.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index f9f8f0546f..217265eb06 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -33,12 +33,12 @@ cerbero_before_script() { # Workaround build-tools having hardcoded internal path pwd mkdir -p "../../gstreamer" - ln -sf "$(pwd)" "../../gstreamer/cerbero" + time ln -sf "$(pwd)" "../../gstreamer/cerbero" # Don't try to symlink twice because on MSYS `ln` does a `cp` since it # doesn't support the new NTFS symlink feature. if [[ ${CI_PROJECT_NAMESPACE} != gstreamer ]]; then mkdir -p "../../${CI_PROJECT_NAMESPACE}" - ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + time ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" fi # Make sure there isn't a pre-existing config hanging around @@ -62,7 +62,7 @@ cerbero_before_script() { cat localconf.cbc - rsync -aH "${CERBERO_HOST_DIR}" . + time rsync -aH "${CERBERO_HOST_DIR}" . cat localconf.cbc @@ -75,7 +75,7 @@ cerbero_before_script() { # ERROR: Failed to proceed with self update Command Error: Running ['git', 'reset', '--hard', 'd6923e4216c8a17759527a3db070d15cf7ff10a0'] returned 128 git status - ./cerbero-uninstalled --self-update manifest.xml + time ./cerbero-uninstalled --self-update manifest.xml } cerbero_script() { @@ -89,7 +89,7 @@ cerbero_script() { if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then test -d "${CERBERO_HOME}/dist/${ARCH}" mkdir -p "${CERBERO_OVERRIDDEN_DIST_DIR}" - rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}" + time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}" fi $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only @@ -110,15 +110,15 @@ cerbero_deps_script() { if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then mkdir -p "${CERBERO_HOME}/dist/${ARCH}" - rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}" + time rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}" fi # Check that the env var is set. Don't expand this protected variable by # doing something silly like [[ -n ${CERBERO_...} ]] because it will get # printed in the CI logs due to set -x if env | grep -q -e CERBERO_PRIVATE_SSH_KEY; then - $CERBERO $CERBERO_ARGS gen-cache --branch "${GST_UPSTREAM_BRANCH}" - $CERBERO $CERBERO_ARGS upload-cache --branch "${GST_UPSTREAM_BRANCH}" + time $CERBERO $CERBERO_ARGS gen-cache --branch "${GST_UPSTREAM_BRANCH}" + time $CERBERO $CERBERO_ARGS upload-cache --branch "${GST_UPSTREAM_BRANCH}" fi cerbero_package_and_check From 705e3758933ec27198cc9785d0d66cb4e5c10a40 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 21 Sep 2020 19:33:17 +0530 Subject: [PATCH 526/605] Revert "cerbero ci: Also disable the deps build" This reverts commit 555c49f9d17aa7a8b9e650941002b52e4e6dd325. The job is working fine now and takes slightly more time than the iOS build, so it can be re-enabled. Part-of: --- gitlab/ci_template.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 88fb040b95..87d82ed8dd 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1214,10 +1214,6 @@ cerbero deps msvc x86_64: extends: ['.cerbero deps', '.cerbero windows native'] script: - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_deps_script" - rules: - - if: '$CI_PROJECT_NAME == "cerbero"' - when: 'manual' - allow_failure: true build cerbero msvc x86_64: extends: ['.cerbero', '.cerbero windows native'] From 808fc283f3d5f2523ee59226207c322a7341eb0d Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 21 Sep 2020 11:57:26 -0400 Subject: [PATCH 527/605] manifest: Build useful error message This helps detecting issue with optional steps like cerbero deps lookup. Part-of: --- gitlab/build_manifest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) mode change 100755 => 100644 gitlab/build_manifest.py diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py old mode 100755 new mode 100644 index 99f73c0911..d57181fe91 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -57,7 +57,10 @@ def get_cerbero_last_build_info (branch : str): req = urllib.request.Request(url) resp = urllib.request.urlopen(req); deps = json.loads(resp.read()) - except urllib.error.URLError: + except urllib.error.URLError as e: + print(f'Failed to get from URL {url}') + print('WARNING: Could not find a revision with build cache ready: ' + str(e)) + print('Checking cerbero:', end=' ') return None return deps[0]['commit'] From dec76861ea87b5d0c2da1533b7123eb2f55fa66a Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 21 Sep 2020 11:57:59 -0400 Subject: [PATCH 528/605] manifest: Fix URL for cerbero deps log There was a missing 'f' prefix, so the {branch} was not expanded. Part-of: --- gitlab/build_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 gitlab/build_manifest.py diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py old mode 100644 new mode 100755 index d57181fe91..cc091d04e3 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -50,7 +50,7 @@ def git(*args, repository_path='.'): def get_cerbero_last_build_info (branch : str): # Take the log from slowest build to reduce cache misses, the logs are # uploaded as soon as they are ready. - url = "https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/cross-ios/universal/cerbero-deps.log" + url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/cross-ios/universal/cerbero-deps.log' deps = [{'commit': None}] try: From 386e40c22e99ceca2b9977e1a6147029a55d2aad Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 15 Sep 2020 16:38:39 +0300 Subject: [PATCH 529/605] citemplate: set PYTHONFAULTHANDLER for tests In case python segfaults, which might happen for things with a C-FFI like pygobject, make it so we get a proper backtrace. https://docs.python.org/3/using/cmdline.html#envvar-PYTHONFAULTHANDLER Part-of: --- gitlab/ci_template.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 87d82ed8dd..a150164fe1 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -510,6 +510,9 @@ build clang fedora x86_64: GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" TIMEOUT_FACTOR: "2" CARGO_HOME: "/cache/gstreamer/cargo" + # Enable the fault handler so we get backtraces on segfaults. + # any non-empty string will do + PYTHONFAULTHANDLER: "enabled" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-examples|gst-docs)$/' script: From 0c7275212ad98c1a8e66ffc18299e42a9c900307 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 21 Sep 2020 22:26:20 +0530 Subject: [PATCH 530/605] manifest: Check all deps logs when picking cerbero commit Instead of picking the 'slowest build', look at all the deps logs and pick the commit that is mentioned in the maximum number of deps logs. In practice, this will either be the newest commit or the previous commit. If it's not the newest commit, we will warn and use an older one. Part-of: --- gitlab/build_manifest.py | 67 +++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/gitlab/build_manifest.py b/gitlab/build_manifest.py index cc091d04e3..306fe1075e 100755 --- a/gitlab/build_manifest.py +++ b/gitlab/build_manifest.py @@ -12,6 +12,9 @@ import json from typing import Dict, Tuple, List # from pprint import pprint +if sys.version_info < (3, 6): + raise SystemExit('Need Python 3.6 or newer') + GSTREAMER_MODULES: List[str] = [ 'orc', 'cerbero', @@ -42,28 +45,64 @@ MANIFEST_TEMPLATE: str = """ """ +CERBERO_DEPS_LOGS_TARGETS = ( + ('cross-ios', 'universal'), + ('cross-windows-mingw', 'x86'), + ('cross-windows-mingw', 'x86_64'), + ('cross-android', 'universal'), + ('fedora', 'x86_64'), + ('macos', 'x86_64'), + ('windows-msvc', 'x86_64'), +) + # Disallow git prompting for a username/password os.environ['GIT_TERMINAL_PROMPT'] = '0' def git(*args, repository_path='.'): return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() def get_cerbero_last_build_info (branch : str): - # Take the log from slowest build to reduce cache misses, the logs are - # uploaded as soon as they are ready. - url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/cross-ios/universal/cerbero-deps.log' - deps = [{'commit': None}] + # Fetch the deps log for all (distro, arch) targets + all_commits = {} + for distro, arch in CERBERO_DEPS_LOGS_TARGETS: + url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/{distro}/{arch}/cerbero-deps.log' + print(f'Fetching {url}') + try: + req = urllib.request.Request(url) + resp = urllib.request.urlopen(req); + deps = json.loads(resp.read()) + except urllib.error.URLError as e: + print(f'WARNING: Failed to GET {url}: {e!s}') + continue - try: - req = urllib.request.Request(url) - resp = urllib.request.urlopen(req); - deps = json.loads(resp.read()) - except urllib.error.URLError as e: - print(f'Failed to get from URL {url}') - print('WARNING: Could not find a revision with build cache ready: ' + str(e)) - print('Checking cerbero:', end=' ') - return None + for dep in deps: + commit = dep['commit'] + if commit not in all_commits: + all_commits[commit] = [] + all_commits[commit].append((distro, arch)) + + # Fetch the cerbero commit that has the most number of caches + best_commit = None + newest_commit = None + max_caches = 0 + total_caches = len(CERBERO_DEPS_LOGS_TARGETS) + for commit, targets in all_commits.items(): + if newest_commit is None: + newest_commit = commit + have_caches = len(targets) + # If this commit has caches for all targets, just use it + if have_caches == total_caches: + best_commit = commit + break + # Else, try to find the commit with the most caches + if have_caches > max_caches: + max_caches = have_caches + best_commit = commit + if newest_commit is None: + print('WARNING: No deps logs were found, will build from scratch') + if best_commit != newest_commit: + print(f'WARNING: Cache is not up-to-date for commit {newest_commit}, using commit {best_commit} instead') + return best_commit - return deps[0]['commit'] def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]: try: From ef981bafc9540ca795e3ed0a6eb37950859d5da5 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 22 Sep 2020 02:39:25 +0530 Subject: [PATCH 531/605] cerbero ci: Actually use the new cerbero_setup.sh script Revert after merging. Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a150164fe1..6f1fd99686 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -48,7 +48,7 @@ variables: WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v12-master' # Can't use $CI_* variables since we use this template in all repos, not just gst-ci - CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" + CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/nirbheek/gst-ci/raw/refactor-cerbero-setup/gitlab/cerbero_setup.sh" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From 98e3c7d8b81a92f4fab9ea4bf39cb7fd472bb4e9 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 22 Sep 2020 00:04:08 +0530 Subject: [PATCH 532/605] cerbero ci: Refactor cerbero_before_script We were doing a few things suboptimally: 1. Making a symlink from the current directory to ../../gstreamer/cerbero so that build-tools kept working, which should not be necessary since cerbero relocates all binaries inside the cerbero homedir after extracting the cache. 2. Hard-coding the cerbero builddir when building on Windows because `pwd` was outputting an MSYS-style path which Python can't read 3. rsync-ing all the contents of $CERBERO_HOST_DIR, which is bad because we should be building on top of the artifact cache. The only thing we want from $CERBERO_HOST_DIR is cerbero-sources (the network cache), and we can setup the toolchain during bootstrap with the new --system and --toolchains arguments which were split out of --system-only. Part-of: --- docker/cerbero/Dockerfile-fedora | 8 +- docker/windows/prepare_cerbero_env.sh | 4 +- gitlab/cerbero_setup.sh | 120 +++++++++++++++----------- gitlab/ci_template.yml | 7 +- 4 files changed, 80 insertions(+), 59 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 2e8c0e3703..dad3e99e71 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -20,18 +20,18 @@ RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerb cd cerbero && \ echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \ echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \ - ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc fetch-package gstreamer-1.0 && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --build-tools=no --toolchains=no && \ cd .. && \ rm /cerbero/cerbero-sources/android-ndk-*.zip && \ rm /cerbero/cerbero-sources/mingw-*.tar.xz && \ diff --git a/docker/windows/prepare_cerbero_env.sh b/docker/windows/prepare_cerbero_env.sh index 446ed24228..7cafe09afa 100644 --- a/docker/windows/prepare_cerbero_env.sh +++ b/docker/windows/prepare_cerbero_env.sh @@ -18,8 +18,8 @@ echo 'vs_install_version = "vs15"' >> localconf.cbc # Fetch all package requirements for a visualstudio gstreamer build ./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0 -# Extract the toolchain and fixup the MSYS installation -./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --system-only --offline +# Fixup the MSYS installation +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline # Delete mingw toolchain binary tarball rm /c/cerbero/cerbero-sources/mingw-*.tar.xz diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 217265eb06..38f8aea982 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -8,16 +8,60 @@ show_ccache_sum() { fi } +# XXX: This is copied and modified from the cerbero-uninstalled script +# Use `mount` to get a list of MSYS mount points that the MSYS shell uses. +# That's our reference point for translating from MSYS paths to Win32 paths. +# We assume that the MSYS mount point directories are only in the filesystem +# root. This will break if people add their own custom mount points beyond what +# MSYS automatically creates, which is highly unlikely. +# +# /d -> d:/ +# /c -> c:/ +# /d/projects/cerbero -> d:/projects/cerbero/ +# /home/USERNAME/cerbero -> C:\\MinGW\\msys\\1.0/home/USERNAME/ +# /mingw -> C:\\MinGW/ +# /mingw/bin/foobar -> C:\\MinGW\\bin/foobar/ +# /tmp/baz -> C:\\Users\\USERNAME\\AppData\\Local\\Temp/baz/ +msys_dir_to_win32() { + set -e + local msys_path stripped_path mount_point path mounted_path + # If the path is already a native path, just return that + if [[ $1 == ?:/* ]] || [[ $1 == ?:\\* ]]; then + echo $1 + return + fi + # Convert /c or /mingw etc to /c/ or /mingw/ etc; gives us a necessary + # anchor to split the path into components + msys_path="$1/" + # Strip leading slash + stripped_path="${msys_path#/}" + # Get the first path component, which may be a mount point + mount_point="/${stripped_path%%/*}" + # Get the path inside the mountp oint + path="/${stripped_path#*/}" + mounted_path="$(mount | sed -n "s|\(.*\) on $mount_point type.*|\1|p")" + # If it's not a mounted path (like /c or /tmp or /mingw), then it's in the + # general MSYS root mount + if [[ -z $mounted_path ]]; then + mounted_path="$(mount | sed -n "s|\(.*\) on / type.*|\1|p")" + path="$1" + fi + echo ${mounted_path}${path%/} +} + +# Print the working directory in the native OS path format, but with forward +# slashes +pwd_native() { + if [[ -n "$MSYSTEM" ]]; then + msys_dir_to_win32 "$(pwd)" + else + pwd + fi +} + # Produces runtime and devel tarball packages for linux/android or .pkg for macos cerbero_package_and_check() { - # FIXME: mingw translates $(pwd) in a broken way - if [[ $CONFIG == win??.cbc ]]; then - PACKAGE_PATH="$CI_PROJECT_DIR" - else - PACKAGE_PATH=$(pwd) - fi - - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$PACKAGE_PATH" gstreamer-1.0 + $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0 # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course. if [[ $CONFIG != *universal* ]]; then @@ -29,52 +73,30 @@ cerbero_package_and_check() { } cerbero_before_script() { - # FIXME Wrong namespace - # Workaround build-tools having hardcoded internal path pwd - mkdir -p "../../gstreamer" - time ln -sf "$(pwd)" "../../gstreamer/cerbero" - # Don't try to symlink twice because on MSYS `ln` does a `cp` since it - # doesn't support the new NTFS symlink feature. - if [[ ${CI_PROJECT_NAMESPACE} != gstreamer ]]; then - mkdir -p "../../${CI_PROJECT_NAMESPACE}" - time ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + ls -lh + + # Copy cerbero git repo stored on the image + cp -a "${CERBERO_HOST_DIR}/.git" . + git checkout . + git status + + # If there's no cerbero-sources directory in the runner cache, copy it from + # the image cache + if ! [[ -d ${CERBERO_SOURCES} ]]; then + time cp -a "${CERBERO_HOST_DIR}/${CERBERO_SOURCES}" . fi + du -sch "${CERBERO_SOURCES}" - # Make sure there isn't a pre-existing config hanging around - rm -v -f localconf.cbc - rm -v -f ${CERBERO_HOST_DIR}/localconf.cbc - + echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc + echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc if [[ $CONFIG == win??.cbc ]]; then - # For windows hardcode the path so it doesn't get - # mangled by msys path handling - # FIXME: make the sources point to pwd/$CI_PROJECT_DIR like the rest - echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc - echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc - # Visual Studio 2017 build tools install path echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc echo 'vs_install_version = "vs15"' >> localconf.cbc - else - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" > localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc fi - cat localconf.cbc - time rsync -aH "${CERBERO_HOST_DIR}" . - - cat localconf.cbc - - # FIXME: if you comment out this line it fails like so, no clue why. Its not windows defender either. - # From https://gitlab.freedesktop.org/gstreamer/cerbero - # b02080cb..d6923e42 master -> origin/master - # Fetching origin - # error: unable to create file cerbero-uninstalled: Permission denied - # fatal: Could not reset index file to revision 'd6923e4216c8a17759527a3db070d15cf7ff10a0'. - # ERROR: Failed to proceed with self update Command Error: Running ['git', 'reset', '--hard', 'd6923e4216c8a17759527a3db070d15cf7ff10a0'] returned 128 - git status - time ./cerbero-uninstalled --self-update manifest.xml } @@ -82,8 +104,10 @@ cerbero_script() { show_ccache_sum $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only + $CERBERO $CERBERO_ARGS fetch-bootstrap $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 + du -sch "${CERBERO_SOURCES}" + $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}" if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then @@ -92,7 +116,7 @@ cerbero_script() { time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}" fi - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only + $CERBERO $CERBERO_ARGS bootstrap --offline --system=no cerbero_package_and_check } @@ -100,9 +124,9 @@ cerbero_deps_script() { show_ccache_sum $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only + $CERBERO $CERBERO_ARGS fetch-bootstrap $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only + $CERBERO $CERBERO_ARGS bootstrap --offline --system=no $CERBERO $CERBERO_ARGS build-deps --offline \ gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \ gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6f1fd99686..28bb9363b8 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -683,7 +683,7 @@ valgrind ges: HAVE_CCACHE: "yes" # used by macos packages as we only ever install to a fixed directory CERBERO_OVERRIDDEN_DIST_DIR: "" - # location where cerbero is cached on the host + # location where the cerbero git repo is stored on the image CERBERO_HOST_DIR: "/cerbero/" before_script: - echo "Fetching ${CERBERO_SCRIPT_URL}" @@ -1196,10 +1196,7 @@ build gst-omx tizonia fedora x86_64: variables: CONFIG: 'win64.cbc' ARCH: 'msvc_x86_64' - # rsync doesn't like windows paths like `C:/cerbero` - CERBERO_HOST_DIR: "/c/cerbero/" - CERBERO_HOME: "C:/cerbero-build" - CERBERO_SOURCES: "C:/cerbero-sources" + CERBERO_HOST_DIR: "C:/cerbero" CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v visualstudio -v nowerror" CERBERO_RUN_SUFFIX: ".exe" HAVE_CCACHE: "" From 6208a619460318dadf7d7a6adfe8b814b0f44d7c Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 22 Sep 2020 19:36:28 +0530 Subject: [PATCH 533/605] cerbero ci: Add hacks for build tools Instead of putting a hammer-like symlink to ../../gstreamer/cerbero, do recipe-specific relocation hacks. Fixes random errors like: ``` ++ ln -sf /builds/nirbheek/cerbero ../../gstreamer/cerbero ln: ../../gstreamer/cerbero/cerbero: cannot overwrite directory ``` Part-of: --- gitlab/cerbero_setup.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 38f8aea982..0ce12769c4 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -59,6 +59,16 @@ pwd_native() { fi } +fix_build_tools() { + if [[ $(uname) == Darwin ]]; then + # Bison needs these env vars set for the build-tools prefix to be + # relocatable, and we only build it on macOS. On Linux we install it + # using the package manager, and on Windows we use the MSYS Bison. + export M4="$(pwd)/${CERBERO_HOME}/build-tools/bin/m4" + export BISON_PKGDATADIR="$(pwd)/${CERBERO_HOME}/build-tools/share/bison" + fi +} + # Produces runtime and devel tarball packages for linux/android or .pkg for macos cerbero_package_and_check() { $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0 @@ -117,6 +127,8 @@ cerbero_script() { fi $CERBERO $CERBERO_ARGS bootstrap --offline --system=no + fix_build_tools + cerbero_package_and_check } From a22f4d85a3e9c71cbb6dc6e05b6ab900d04911da Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 23 Sep 2020 14:18:57 +0530 Subject: [PATCH 534/605] docker/cerbero: Do not remove toolchain downloads We will use these when running bootstrap in the actual jobs. Part-of: --- docker/cerbero/Dockerfile-fedora | 2 -- docker/windows/prepare_cerbero_env.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index dad3e99e71..b11f6282b6 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -33,8 +33,6 @@ RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerb ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --build-tools=no --toolchains=no && \ cd .. && \ - rm /cerbero/cerbero-sources/android-ndk-*.zip && \ - rm /cerbero/cerbero-sources/mingw-*.tar.xz && \ rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \ rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \ dnf clean all diff --git a/docker/windows/prepare_cerbero_env.sh b/docker/windows/prepare_cerbero_env.sh index 7cafe09afa..7eafd2eb71 100644 --- a/docker/windows/prepare_cerbero_env.sh +++ b/docker/windows/prepare_cerbero_env.sh @@ -21,8 +21,6 @@ echo 'vs_install_version = "vs15"' >> localconf.cbc # Fixup the MSYS installation ./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline -# Delete mingw toolchain binary tarball -rm /c/cerbero/cerbero-sources/mingw-*.tar.xz # Wipe visualstudio package dist, sources, logs, and the build tools recipes ./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc wipe --force --build-tools # clean the localconf From 37bdeb200a1441e9ebe17568b6c4279d0cec08cc Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 24 Sep 2020 05:18:24 +0530 Subject: [PATCH 535/605] Revert "cerbero ci: Actually use the new cerbero_setup.sh script" This reverts commit ef981bafc9540ca795e3ed0a6eb37950859d5da5. Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 28bb9363b8..65464cf9a0 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -48,7 +48,7 @@ variables: WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v12-master' # Can't use $CI_* variables since we use this template in all repos, not just gst-ci - CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/nirbheek/gst-ci/raw/refactor-cerbero-setup/gitlab/cerbero_setup.sh" + CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From ab8e884e1b15dcbeec9177ade4b8536b634c2ff8 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 24 Sep 2020 05:36:46 +0530 Subject: [PATCH 536/605] Revert "Switch MSVC job to manual till we can upload deps cache" This reverts commit bdbec90a4939d5c31f800cc9f07646c820a7bd76. Everything seems to be working fine. Part-of: --- gitlab/ci_template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 65464cf9a0..5fdf2a202a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1221,5 +1221,3 @@ build cerbero msvc x86_64: - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_script" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' - when: 'manual' - allow_failure: true From 8c066ec01ff57e768dabe899d2152f4b1cbbb290 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 28 Sep 2020 07:24:27 +0300 Subject: [PATCH 537/605] citempalte: fix string comparitson in rules When quoting in rules it takes the string as literal and doesn't expand the variable Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 5fdf2a202a..283ad9fa16 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -331,14 +331,14 @@ manifest: # If the user that triggered the Pipeline is the Merge bot and the branch doesn't match # the upstream branch set, run the pipeline - - if: '$GITLAB_USER_LOGIN == "gstreamer-merge-bot" && $CI_COMMIT_BRANCH != "$GST_UPSTREAM_BRANCH"' + - if: '$GITLAB_USER_LOGIN == "gstreamer-merge-bot" && $CI_COMMIT_BRANCH != $GST_UPSTREAM_BRANCH' # When the user isn't the merge bot, require an explicit action to trigger the pipeline # to avoid wasting CI resources - if: '$GITLAB_USER_LOGIN != "gstreamer-merge-bot"' when: 'manual' # If this matches, it means the pipeline is running against either the main # or a stable branch, so make it manual - - if: '$CI_COMMIT_BRANCH == "$GST_UPSTREAM_BRANCH"' + - if: '$CI_COMMIT_BRANCH == $GST_UPSTREAM_BRANCH' when: 'manual' stage: 'preparation' From 3dddfacfbf710fa71777488145594eb0abe11ee4 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 29 Sep 2020 18:07:06 +0300 Subject: [PATCH 538/605] citemplate: set the cerbero msvc build to allow_failure again Very weird things going on.. ``` UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 8608: character maps to ``` https://gitlab.freedesktop.org/Hosang/gst-plugins-bad/-/jobs/4731636 Part-of: --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 283ad9fa16..e825326e7f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1221,3 +1221,4 @@ build cerbero msvc x86_64: - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_script" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' + allow_failure: true From b8e112ed748070a9281c8a27051d0b08b05b2bbf Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 1 Oct 2020 10:00:49 +0530 Subject: [PATCH 539/605] cerbero ci: Wipe in before_script because gitlab doesn't Noticed in this job, which failed because we rebuilt on top of an existing builddir: https://gitlab.freedesktop.org/nirbheek/cerbero/-/jobs/4772728 Part-of: --- gitlab/cerbero_setup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 0ce12769c4..5b319d393d 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -108,6 +108,10 @@ cerbero_before_script() { cat localconf.cbc time ./cerbero-uninstalled --self-update manifest.xml + + # GitLab runner does not always wipe the image after each job, so do that + # to ensure we always have a clean builddir + time $CERBERO $CERBERO_ARGS wipe --keep-sources --build-tools --force } cerbero_script() { From a26129c78cd9096ca92c85fad6a235cd3f3384ee Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 28 Sep 2020 14:00:24 +0300 Subject: [PATCH 540/605] windows: rebuild docker image also: Revert "citemplate: set the cerbero msvc build to allow_failure again" This reverts commit 3dddfacfbf710fa71777488145594eb0abe11ee4. Part-of: --- docker/windows/build_image.ps1 | 2 +- gitlab/ci_template.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index c092f6d75b..4e19de3057 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,5 +1,5 @@ $env:DEFAULT_BRANCH='master' -$env:VERSION='v12' +$env:VERSION='v13' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" echo "Building $env:tag" docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e825326e7f..7026ae525e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -45,7 +45,7 @@ variables: TEST_MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$TEST_MANIFEST_AMD64_SUFFIX:$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v12-master' + WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v13-master' # Can't use $CI_* variables since we use this template in all repos, not just gst-ci CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" @@ -1221,4 +1221,3 @@ build cerbero msvc x86_64: - C:\MinGW\msys\1.0\bin\bash.exe --login -c "cd $env:CI_PROJECT_DIR && $env:CERBERO_SCRIPTS_PATH cerbero_script" rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/' - allow_failure: true From 42696e1654870be08f5a1e846fff1a84b0393141 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 2 Oct 2020 22:17:53 +0530 Subject: [PATCH 541/605] cerbero ci: Make before_script wiping more thorough Will avoid packaging errors due to the old package file still being around from a previous job, such as in: https://gitlab.freedesktop.org/nirbheek/cerbero/-/jobs/4804147 Part-of: --- gitlab/cerbero_setup.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 5b319d393d..d752ce77ec 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -84,7 +84,7 @@ cerbero_package_and_check() { cerbero_before_script() { pwd - ls -lh + ls -lha # Copy cerbero git repo stored on the image cp -a "${CERBERO_HOST_DIR}/.git" . @@ -110,8 +110,10 @@ cerbero_before_script() { time ./cerbero-uninstalled --self-update manifest.xml # GitLab runner does not always wipe the image after each job, so do that - # to ensure we always have a clean builddir - time $CERBERO $CERBERO_ARGS wipe --keep-sources --build-tools --force + # to ensure we don't have any leftover data from a previous job such as + # a dirty builddir, or tarballs/pkg files, leftover files from an old + # cerbero commit, etc. Skip the things we actually need to keep. + time git clean -xdff -e cerbero_setup.sh -e manifest.xml -e localconf.cbc -e "${CERBERO_SOURCES}" } cerbero_script() { From 434d570528dd2348c728ecf40cf534686c077315 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 2 Oct 2020 22:40:30 +0530 Subject: [PATCH 542/605] citemplate: Consistent naming for cerbero_setup.sh script Part-of: --- gitlab/ci_template.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7026ae525e..2ebdc25693 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -687,11 +687,11 @@ valgrind ges: CERBERO_HOST_DIR: "/cerbero/" before_script: - echo "Fetching ${CERBERO_SCRIPT_URL}" - - curl -L -o cerbero_script.sh "${CERBERO_SCRIPT_URL}" - - chmod +x cerbero_script.sh - - ./cerbero_script.sh cerbero_before_script + - curl -L -o cerbero_setup.sh "${CERBERO_SCRIPT_URL}" + - chmod +x cerbero_setup.sh + - ./cerbero_setup.sh cerbero_before_script script: - - ./cerbero_script.sh cerbero_script + - ./cerbero_setup.sh cerbero_script cache: key: "${CI_JOB_NAME}" paths: @@ -835,7 +835,7 @@ build msys2 : rules: - if: '$CI_PROJECT_NAME == "cerbero"' script: - - ./cerbero_script.sh cerbero_deps_script + - ./cerbero_setup.sh cerbero_deps_script # # Cerbero Linux X86_64 build From 19d7ff53424c5bd104b73755f9f640bff8169392 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 2 Oct 2020 22:19:57 +0530 Subject: [PATCH 543/605] citemplate: Actually use the new cerbero_setup.sh script Revert me after merge. Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2ebdc25693..710ee1c3c1 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -48,7 +48,7 @@ variables: WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v13-master' # Can't use $CI_* variables since we use this template in all repos, not just gst-ci - CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" + CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/nirbheek/gst-ci/raw/wipe-more-before-script/gitlab/cerbero_setup.sh" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From ea155d7925f8cc93f709b12e9ebcf6fbcb5028fc Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sun, 4 Oct 2020 13:30:46 +0000 Subject: [PATCH 544/605] Revert "citemplate: Actually use the new cerbero_setup.sh script" This reverts commit 19d7ff53424c5bd104b73755f9f640bff8169392 --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 710ee1c3c1..2ebdc25693 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -48,7 +48,7 @@ variables: WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v13-master' # Can't use $CI_* variables since we use this template in all repos, not just gst-ci - CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/nirbheek/gst-ci/raw/wipe-more-before-script/gitlab/cerbero_setup.sh" + CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From 24db2ef9e75c6edf49b28cd7d46495180fc0f216 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 1 Oct 2020 14:44:14 -0400 Subject: [PATCH 545/605] Enable pango plugin on Windows This tests that all fallbacks works to get pango plugin built with no system dependencies available. Part-of: --- gitlab/ci_template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 2ebdc25693..7317b41420 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -722,6 +722,7 @@ valgrind ges: -Dlibav=disabled -Dvaapi=disabled -Ddevtools=disabled + -Dgst-plugins-base:pango=enabled rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' script: From ad370fecef956a1ddad5e62577ab7b3f6ccd208c Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 8 Oct 2020 13:57:36 +0300 Subject: [PATCH 546/605] ci_template: try to enable merge-request pipelines again Gitlab was slightly bugged in previous version, and we had to use branch-only pipelines. Let's try this again. Part-of: --- gitlab/ci_template.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 7317b41420..a9c73875d9 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1,5 +1,5 @@ include: - - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' + - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' stages: - 'build docker' @@ -329,12 +329,11 @@ manifest: - if: '$CI_PROJECT_PATH == "gstreamer/cerbero"' - if: '$CI_PROJECT_PATH == "gstreamer/gst-docs"' - # If the user that triggered the Pipeline is the Merge bot and the branch doesn't match - # the upstream branch set, run the pipeline - - if: '$GITLAB_USER_LOGIN == "gstreamer-merge-bot" && $CI_COMMIT_BRANCH != $GST_UPSTREAM_BRANCH' - # When the user isn't the merge bot, require an explicit action to trigger the pipeline + # If the MR is assigned to the Merge bot, trigger the pipeline automatically + - if: '$CI_MERGE_REQUEST_ASSIGNEES == "gstreamer-merge-bot"' + # When the assignee isn't the merge bot, require an explicit action to trigger the pipeline # to avoid wasting CI resources - - if: '$GITLAB_USER_LOGIN != "gstreamer-merge-bot"' + - if: '$CI_MERGE_REQUEST_ASSIGNEES != "gstreamer-merge-bot"' when: 'manual' # If this matches, it means the pipeline is running against either the main # or a stable branch, so make it manual @@ -880,7 +879,7 @@ build cerbero cross-android universal: - '.cerbero' - '.cerbero cross-android universal' rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' + - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' # @@ -967,7 +966,7 @@ build cerbero cross win64: cross-android universal examples: extends: ".cross-android universal examples" rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' + - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' needs: - "build cerbero cross-android universal" @@ -1046,7 +1045,7 @@ build cerbero cross-ios universal: - '.cerbero' - '.cerbero cross-ios universal' rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' + - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' documentation: @@ -1147,7 +1146,7 @@ documentation: cross-ios universal examples: extends: ".cross-ios universal examples" rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME && $CI_PROJECT_NAME == "gst-docs"' + - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"' - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' needs: - "build cerbero cross-ios universal" From 2cf083373b72f446f5e5976476231fb40c034d3f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 9 Oct 2020 12:14:32 +0300 Subject: [PATCH 547/605] fedora: rebuild image so it includes a newer subproject cache Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a9c73875d9..c752a638f4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,7 +21,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2020-09-15.0' + FEDORA_TAG: '2020-10-09.0' CERBERO_TAG: '2020-09-09.2' ANDROID_TAG: '2020-09-09.2' MANIFEST_TAG: '2020-06-25.0' From 75129af3b4604968c1b84fdcfd0f671c7a2108f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Cerveau?= Date: Fri, 9 Oct 2020 14:11:31 +0200 Subject: [PATCH 548/605] gstreamer-full: enable unit test Run test-gst-full in gstreamer-full job Part-of: --- gitlab/ci_template.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c752a638f4..afd1b9f1f4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -575,26 +575,20 @@ gstreamer-full: MESON_ARGS: > --default-library=static -Dauto_features=disabled - -Dgst-plugins-good:alpha=enabled + -Dgstreamer:check=enabled + -Dtests=enabled $MESON_GST_WERROR rules: - if: '$CI_PROJECT_NAME =~ /^(gst-build|gst-ci)$/' - after_script: - - cd gst-build/ - - ninja -C build install - - export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib64/gstreamer-1.0/pkgconfig - - export LD_LIBRARY_PATH=/usr/local/lib64 - - cd examples/static-plugins - - meson _build - - ninja -C _build - - meson test -C _build -v + script: + - *gst_build + - meson test -C build -v test-gst-full artifacts: expire_in: "7 days" when: "always" paths: - - 'gst-build/build/meson-logs/' - - 'gst-build/build/gstinitstaticplugins.c' + - 'meson-logs/' # Valgrind .valgrind fedora x86_64: From 5dc5de23960778f5e6601e760395b8ef887d2ccd Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 14 Oct 2020 21:26:35 +0300 Subject: [PATCH 549/605] citemplate: tweak merge-request rules With the MergeRequest rules template, its was impossible to manually create or trigger a pipeline for a branch/commit that was not part of a merge request. This tweaks the workflow:rules such that there will always be a pipeline for each commit (which will be set to a manual job trigger). There is the downside that we will now be triggering 2 pipelines in case of merge requests, but they will require manual action to be started. Part-of: --- gitlab/ci_template.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index afd1b9f1f4..65e69bd7ec 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1,6 +1,3 @@ -include: - - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' - stages: - 'build docker' - 'preparation' @@ -88,6 +85,11 @@ variables: --variants werror --timestamps +workflow: + rules: + - if: $CI_MERGE_REQUEST_IID + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH # # Global CI policy # From 3b59f3ebcb87249539a0ba9f75e3aa0ab292348f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 22 Oct 2020 16:04:01 +0300 Subject: [PATCH 550/605] docker/cerbero: remove bootstrap invocation `bootstrap --build-tools=no --toolchains=no` now do nothing, and just repeat what the bootstrap line above does which is to only run the system bootstraper Part-of: --- docker/cerbero/Dockerfile-fedora | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index b11f6282b6..6009c421ca 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -20,18 +20,15 @@ RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerb cd cerbero && \ echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \ echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \ - ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --build-tools=no --toolchains=no && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \ ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --build-tools=no --toolchains=no && \ cd .. && \ rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \ rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \ From aead7b4eb70fd7823fad26f1efd24f131091229e Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 15 Oct 2020 17:37:53 +0300 Subject: [PATCH 551/605] citemplate: remove fdo.expires-after label This is meant to be used for containers that are going to be periodically rebuilt. For our usecase specifying `fdo.upstream-repo` is enough to garbage collect all the tags from the forks. We also need to rebuild the images, so they don't have include the label anymore. Part-of: --- gitlab/ci_template.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 65e69bd7ec..f0bd2b41c0 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -18,12 +18,12 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2020-10-09.0' - CERBERO_TAG: '2020-09-09.2' - ANDROID_TAG: '2020-09-09.2' - MANIFEST_TAG: '2020-06-25.0' - TEST_MANIFEST_TAG: '2020-06-25.0' - INDENT_TAG: '2020-06-29.0' + FEDORA_TAG: '2020-10-22.0' + CERBERO_TAG: '2020-10-22.0' + ANDROID_TAG: '2020-10-22.0' + MANIFEST_TAG: '2020-10-22.0' + TEST_MANIFEST_TAG: '2020-10-22.0' + INDENT_TAG: '2020-10-22.0' GST_UPSTREAM_REPO: 'gstreamer/gst-ci' @@ -250,7 +250,6 @@ default: --label gitlab.project_path=$CI_PROJECT_PATH --label gitlab.project_url=$CI_PROJECT_URL --label fdo.upstream-repo=$GST_UPSTREAM_REPO - --label fdo.expires-after="3w" -f $DOCKERFILE -t $LOCAL_IMAGE $CONTEXT_DIR From c3b5693736aea7798df87b10de48239ef7e43de7 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 12 Nov 2020 14:03:26 +0530 Subject: [PATCH 552/605] cerbero: Never use cached cerbero git repo Sometimes the git repo cached on the docker image gets corrupted on Windows, in which case there is no way to clean things and resolve that error. For example: https://gitlab.freedesktop.org/bilboed/gst-plugins-bad/-/jobs/5517037 ``` ++ cp -a C:/cerbero/.git . cp: cannot create regular file `./.git/objects/pack/pack-cbdde9dc4dd07bff6761e6715e20d7c0abe62b9e.idx': Permission denied cp: cannot create regular file `./.git/objects/pack/pack-cbdde9dc4dd07bff6761e6715e20d7c0abe62b9e.pack': Permission denied ``` Always delete the cached git repo and copy a pristine copy from the image. This will also avoid accumulation of garbage in the git repo. Part-of: --- gitlab/cerbero_setup.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index d752ce77ec..ce593b8d9d 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -86,7 +86,15 @@ cerbero_before_script() { pwd ls -lha - # Copy cerbero git repo stored on the image + # Copy cerbero git repo stored on the image, delete an existing checkout if + # it exists. Sometimes the git dir gets corrupted on the Windows runner, + # and there's no way to clean it up because gitlab will keep trying to + # reuse the cached image. + if ! [[ -d ${CERBERO_HOST_DIR}/.git ]]; then + mv "${CERBERO_HOST_DIR}/.git" "${CERBERO_HOST_DIR}/.git-old" + # rm -rf is very slow on Windows, so do it in the background + rm -rf "${CERBERO_HOST_DIR}/.git-old" & + fi cp -a "${CERBERO_HOST_DIR}/.git" . git checkout . git status @@ -114,6 +122,9 @@ cerbero_before_script() { # a dirty builddir, or tarballs/pkg files, leftover files from an old # cerbero commit, etc. Skip the things we actually need to keep. time git clean -xdff -e cerbero_setup.sh -e manifest.xml -e localconf.cbc -e "${CERBERO_SOURCES}" + + # Wait for the rm -rf from above if needed + [[ -n $(jobs -p) ]] && fg } cerbero_script() { From 34dec61d8660fa4ecc0522416765d9ae805c786d Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 16 Nov 2020 15:40:16 +0200 Subject: [PATCH 553/605] Revert "cerbero: Never use cached cerbero git repo" This reverts commit c3b5693736aea7798df87b10de48239ef7e43de7. This broke the build and needs further investigation https://gitlab.freedesktop.org/gstreamer/gst-ci/-/jobs/5630095 --- gitlab/cerbero_setup.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index ce593b8d9d..d752ce77ec 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -86,15 +86,7 @@ cerbero_before_script() { pwd ls -lha - # Copy cerbero git repo stored on the image, delete an existing checkout if - # it exists. Sometimes the git dir gets corrupted on the Windows runner, - # and there's no way to clean it up because gitlab will keep trying to - # reuse the cached image. - if ! [[ -d ${CERBERO_HOST_DIR}/.git ]]; then - mv "${CERBERO_HOST_DIR}/.git" "${CERBERO_HOST_DIR}/.git-old" - # rm -rf is very slow on Windows, so do it in the background - rm -rf "${CERBERO_HOST_DIR}/.git-old" & - fi + # Copy cerbero git repo stored on the image cp -a "${CERBERO_HOST_DIR}/.git" . git checkout . git status @@ -122,9 +114,6 @@ cerbero_before_script() { # a dirty builddir, or tarballs/pkg files, leftover files from an old # cerbero commit, etc. Skip the things we actually need to keep. time git clean -xdff -e cerbero_setup.sh -e manifest.xml -e localconf.cbc -e "${CERBERO_SOURCES}" - - # Wait for the rm -rf from above if needed - [[ -n $(jobs -p) ]] && fg } cerbero_script() { From e8d4d1f3ada22ffb0851ba342dcc7a31c259e6f8 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 16 Nov 2020 15:10:28 +0200 Subject: [PATCH 554/605] citemplate: build devtools for windows builds explicitly enable the cairo plugin build as well Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f0bd2b41c0..788d960e61 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -715,8 +715,8 @@ valgrind ges: -Dpython=disabled -Dlibav=disabled -Dvaapi=disabled - -Ddevtools=disabled -Dgst-plugins-base:pango=enabled + -Dgst-plugins-good:cairo=enabled rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/' script: From 12c1689aaaf307feb50f465565f6d752f16a1b0e Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 11 Jun 2020 14:45:24 +0200 Subject: [PATCH 555/605] ci: check commit using ci-fairy See https://freedesktop.pages.freedesktop.org/ci-templates/ci-fairy.html#checking-commits In this setup it will check for any leftover fixup!/squash! commits and that messages don't have the 'Signed-off-by:' tag. Part-of: --- gitlab/ci_template.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 788d960e61..e09faa8bbe 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1,3 +1,6 @@ +include: + - remote: "https://gitlab.freedesktop.org/freedesktop/ci-templates/-/raw/7ea696055e322cc7aa4bcbe5422b56a198c4bdff/templates/ci-fairy.yml" + stages: - 'build docker' - 'preparation' @@ -388,6 +391,18 @@ gst indent: exit 1 fi +# +# Check commit messages with ci-fairy +# +check commits: + extends: '.fdo.ci-fairy' + needs: [] + stage: 'preparation' + variables: + GIT_STRATEGY: 'fetch' + script: + - ci-fairy check-commits --textwidth 0 --no-signed-off-by + # # gst-build setup templates # From 38c16db9ba6ee43e7d401e70cd9c2fee07843a56 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 27 Nov 2020 03:05:11 +0200 Subject: [PATCH 556/605] ci_template: job to complain about allow-edit Make sure merge requests have enabled the "allow-edit" option in the MR so we can rebase them. Close gstreamer/gstreamer-project#34 Part-of: --- gitlab/ci_template.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index e09faa8bbe..1560bf1f2e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -403,6 +403,20 @@ check commits: script: - ci-fairy check-commits --textwidth 0 --no-signed-off-by +# +# Check "allow-edit" checkbox on merge requests with ci-fairy +# +check allow-edit: + extends: '.fdo.ci-fairy' + needs: [] + stage: 'preparation' + variables: + GIT_STRATEGY: 'none' + script: + - ci-fairy check-merge-request --require-allow-collaboration + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + # # gst-build setup templates # From d4e55b655ce2a7d3f3d6c1b6875f8e8fb79cbe75 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 2 Dec 2020 12:38:57 +0200 Subject: [PATCH 557/605] Revert "ci: check commit using ci-fairy" This reverts commit 12c1689aaaf307feb50f465565f6d752f16a1b0e. Revert ci-fairy check-commits job for now as it seems a bit buggy. It picks up commits that are not part of the branch and breaks the CI. Will investigate further and reintroduce it later. --- gitlab/ci_template.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1560bf1f2e..6ec5778f7e 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -391,18 +391,6 @@ gst indent: exit 1 fi -# -# Check commit messages with ci-fairy -# -check commits: - extends: '.fdo.ci-fairy' - needs: [] - stage: 'preparation' - variables: - GIT_STRATEGY: 'fetch' - script: - - ci-fairy check-commits --textwidth 0 --no-signed-off-by - # # Check "allow-edit" checkbox on merge requests with ci-fairy # From 3822c33704348513740c774359d175f16831d044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Cerveau?= Date: Fri, 9 Oct 2020 14:11:31 +0200 Subject: [PATCH 558/605] gstreamer-full: enable features unit test As gst-build can select features to be exposed by gstreamer-full, the test will check that feature are present or not present according to the current configuration. Part-of: --- gitlab/ci_template.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6ec5778f7e..07d6a03252 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -586,6 +586,7 @@ integration testsuites fedora: rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-omx|gstreamer-sharp|gst-plugins-rs)$/' +# gstreamer-full gstreamer-full: extends: 'build static fedora x86_64' stage: integrate @@ -608,6 +609,38 @@ gstreamer-full: paths: - 'meson-logs/' +gstreamer-full-minimal: + extends: 'build static fedora x86_64' + stage: integrate + variables: + MESON_ARGS: > + --default-library=static + -Dauto_features=disabled + -Dgstreamer:check=enabled + -Dtests=enabled + -Dgst-plugins-base:alsa=enabled + -Dgst-plugins-base:typefind=enabled + -Dgst-plugins-base:pbtypes=enabled + -Dgst-full-elements=coreelements:filesrc,fakesink,identity,input-selector + -Dgst-full-typefind-functions=typefindfunctions:wav,flv + -Dgst-full-device-providers=alsa:alsadeviceprovider + -Dgst-full-dynamic-types=pbtypes:video_multiview_flagset + $MESON_GST_WERROR + rules: + - if: '$CI_PROJECT_NAME =~ /^(gst-build|gst-ci)$/' + + script: + - *gst_build + - meson test -C build -v test-gst-full + - meson test -C build test-gst-full-features --test-args "-e filesrc,identity,fakesink -E filesink,capsfilter -t audio/x-wav -T video/vivo -d alsadeviceprovider -D v4l2deviceprovider -l GstVideoMultiviewFlagsSet" + - strip build/libgstreamer-full-1.0.so + - ls -l build/libgstreamer-full-1.0.so + artifacts: + expire_in: "7 days" + when: "always" + paths: + - 'meson-logs/' + # Valgrind .valgrind fedora x86_64: extends: '.test fedora x86_64' From 4afcde9c60ba97690eeed3c281d675b2be6290f1 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 23 Dec 2020 04:46:19 +0200 Subject: [PATCH 559/605] ci_template: re-enable the extraction of logs for windows jobs Most of the networking issues with the runner should be fixed now. Part-of: --- gitlab/ci_template.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 07d6a03252..958aa8232a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -796,12 +796,11 @@ valgrind ges: - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && meson build $env:MESON_ARGS && ninja -C build" - # XXX: Re-enable when uploading stops timing out - #artifacts: - # expire_in: "7 days" - # when: "always" - # paths: - # - 'gst-build/build/meson-logs/' + artifacts: + expire_in: "7 days" + when: "always" + paths: + - 'gst-build/build/meson-logs/' build vs2017 amd64: extends: '.build windows' From 3ec559700a97a8225b634dec5b178a5b9b353d4a Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 6 Jan 2021 20:48:41 +0200 Subject: [PATCH 560/605] ci_template: remove artifacts workaround for cerbero msvc This was fixed 3 weeks ago and should no longer be needed. Part-of: --- gitlab/ci_template.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 958aa8232a..0a3044121b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1244,9 +1244,6 @@ build gst-omx tizonia fedora x86_64: CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v visualstudio -v nowerror" CERBERO_RUN_SUFFIX: ".exe" HAVE_CCACHE: "" - # FIXME: for some reason the runner is hanging when trying to upload artifacts - artifacts: - paths: [] before_script: - $env:CI_PROJECT_DIR = $env:CI_PROJECT_DIR.replace('\', '/') - $env:CERBERO_SCRIPTS_PATH = "$env:CI_PROJECT_DIR/cerbero_setup.sh" From 08cdfca671cc5ad98d5a9af66cf70072e6da7859 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 15 Jan 2021 23:29:33 +1100 Subject: [PATCH 561/605] gitlab: update macos/ios tags for newer vm image Part-of: --- gitlab/ci_template.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 0a3044121b..106a34a0fa 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1034,7 +1034,7 @@ cerbero cross-android universal examples: CERBERO_OVERRIDDEN_DIST_DIR: "/Library/Frameworks/GStreamer.framework/Versions/1.0" CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" tags: - - gst-macos-10.15 + - gst-macos-11.1 artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" expire_in: '5 days' @@ -1069,7 +1069,7 @@ build cerbero macos x86_64: HAVE_CCACHE: "" CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" tags: - - gst-ios-13.2 + - gst-ios-14.3 artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" expire_in: '5 days' @@ -1185,7 +1185,7 @@ documentation: after_script: - rm -rf ${EXAMPLES_HOME}/gst-examples ${EXAMPLES_HOME}/gst-docs tags: - - gst-ios-13.2 + - gst-ios-14.3 cross-ios universal examples: extends: ".cross-ios universal examples" From 8aa80fb7140cff633d99490ce51d57f9147c2767 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 26 Jan 2021 08:39:42 +0200 Subject: [PATCH 562/605] citemplate: only spawn a single pipeline in Merge Requests This is the 3rd or 4th time we are trying this yes, (!331), but hear me out. This time is for Real! Jokes aside, its possible to avoid spawning >1 pipeline per event with gitlab 13.8. https://gitlab.com/gitlab-org/gitlab/-/issues/201845 https://gitlab.com/gitlab-org/gitlab/-/issues/299409 Part-of: --- gitlab/ci_template.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 106a34a0fa..ac69fdc5fa 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -91,8 +91,13 @@ variables: workflow: rules: - if: $CI_MERGE_REQUEST_IID + # don't create a pipeline if its a commit pipeline, on a branch and that branch has + # open merge requests (bc we will get a MR build instead) + - if: $CI_OPEN_MERGE_REQUESTS + when: never - if: $CI_COMMIT_TAG - if: $CI_COMMIT_BRANCH + # # Global CI policy # From dbf14146b95093f217a7295c8a392396821db901 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 16 Dec 2020 17:22:40 +0200 Subject: [PATCH 563/605] citemplate: use meson 0.56 and suproject update --reset The windows build will be updated in a later commit. Close #73 #74 #75 Part-of: --- docker/fedora/prepare.sh | 2 +- gitlab/ci_template.yml | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 1024be6571..60f877de4e 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -184,7 +184,7 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.54.3 hotdoc +pip3 install meson==0.56.2 hotdoc # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ac69fdc5fa..c0133f7a45 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,7 +21,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2020-10-22.0' + FEDORA_TAG: '2021-01-22.0' CERBERO_TAG: '2020-10-22.0' ANDROID_TAG: '2020-10-22.0' MANIFEST_TAG: '2020-10-22.0' @@ -427,7 +427,11 @@ check allow-edit: - cd gst-build/ # only copy immediate directories. We don't want to modify .wrap files - find /gst-build/subprojects/ -mindepth 1 -maxdepth 1 -type d -exec cp -r \{\} subprojects/ \; + + # Reset the state of the subprojects to what gst-build expects + - meson subprojects update --reset - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml" + - meson build/ $MESON_ARGS - ninja -C build/ - ccache --show-stats @@ -1114,7 +1118,7 @@ documentation: - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@28ad9aa59daa49d6b3fd1bc6b144df52af39475b # FIXME: remove this once the docs image is updated - - rm -rf /gst-build/subprojects/pygobject/ + - pip3 install -U meson==0.56.2 - *gst_build - ./gst-uninstalled.py ninja -C build/ plugins_doc_caches From d5136450c2456ce369c6516f8604bd0e9b927e63 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 16 Dec 2020 17:16:39 +0200 Subject: [PATCH 564/605] docker/fedora: run git gc after cloning the subprojects Most of the subprojects specify depth=1 in the wrap file, so this doesn't have a big effect atm. Part-of: --- docker/fedora/prepare.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 60f877de4e..bc13d9d99c 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -238,3 +238,9 @@ rustc --version git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gst-build.git /gst-build/ cd /gst-build meson subprojects download + +# Run git gc to prune unwanted refs and reduce the size of the image +for i in $(find subprojects/ -mindepth 1- maxdepth 1 -type d); +do + git -C $i gc --aggressive || true; +done \ No newline at end of file From 114ce593ef1df9e5e7199968fe07e7fbd947ee31 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Mon, 1 Feb 2021 14:59:04 +0100 Subject: [PATCH 565/605] hotdoc: use latest pip version Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c0133f7a45..6924fe2215 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1115,7 +1115,7 @@ documentation: MESON_BUILDTYPE_ARGS: "-Ddoc=enabled" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - - pip3 install --upgrade git+https://github.com/hotdoc/hotdoc@28ad9aa59daa49d6b3fd1bc6b144df52af39475b + - pip3 install --upgrade hotdoc # FIXME: remove this once the docs image is updated - pip3 install -U meson==0.56.2 From 26a990454c958b0f9771b5c8392564734121e212 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 3 Feb 2021 14:05:19 +0530 Subject: [PATCH 566/605] gitlab: Allow du to fail in cerbero scripts It's purely for informative reasons. `du` will fail on the sources dir if a branch name has unicode in it due to an MSYS/MinGW bug. The long term fix is to from MSYS/MinGW to MSYS/MinGW-W64 or MSYS2/MinGW-W64. Part-of: --- gitlab/cerbero_setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index d752ce77ec..4fd7cea74b 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -96,7 +96,7 @@ cerbero_before_script() { if ! [[ -d ${CERBERO_SOURCES} ]]; then time cp -a "${CERBERO_HOST_DIR}/${CERBERO_SOURCES}" . fi - du -sch "${CERBERO_SOURCES}" + du -sch "${CERBERO_SOURCES}" || true echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc @@ -122,7 +122,7 @@ cerbero_script() { $CERBERO $CERBERO_ARGS show-config $CERBERO $CERBERO_ARGS fetch-bootstrap $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - du -sch "${CERBERO_SOURCES}" + du -sch "${CERBERO_SOURCES}" || true $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}" From 2cd6f5f6b9051eda49f97513b178343f12333c57 Mon Sep 17 00:00:00 2001 From: Biswapriyo Nath Date: Sat, 13 Feb 2021 20:51:35 +0530 Subject: [PATCH 567/605] docker/windows: Update msys2 installer to 2021-02-15 version. This fixes the PGP key installation issue. Part-of: --- docker/windows/build_image.ps1 | 2 +- docker/windows/install_toolchain.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 4e19de3057..8494dfd4d1 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,5 +1,5 @@ $env:DEFAULT_BRANCH='master' -$env:VERSION='v13' +$env:VERSION='v17' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" echo "Building $env:tag" docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index cd6f147981..a29330b32b 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -1,7 +1,7 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' -$msys2_url = 'https://github.com/msys2/msys2-installer/releases/download/2020-05-22/msys2-base-x86_64-20200522.tar.xz' +$msys2_url = 'https://github.com/msys2/msys2-installer/releases/download/2021-02-15/msys2-base-x86_64-20210215.tar.xz' $msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' Write-Host "Installing VisualStudio" From 2c0ab5f8a1756c5a2c8d1e192dcd87da6c9dba48 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 18 Feb 2021 14:49:41 +0530 Subject: [PATCH 568/605] gitlab/cerbero: Don't fail when there's no macOS cache `test -d "${CERBERO_HOME}/dist/${ARCH}"` will cause the build to error out if the dist dir doesn't exist, which will happen if there's no cache, for example: https://gitlab.freedesktop.org/seungha.yang/gst-plugins-bad/-/jobs/7405565 What we actually want to do, is to not perform the rsync if there's no cache, since it means we want to rebuild from scratch; for instance, if the current MR uses a custom cerbero branch of the same name which won't have a prebuilt cache. Part-of: --- gitlab/cerbero_setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index 4fd7cea74b..fbf164faa7 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -126,8 +126,7 @@ cerbero_script() { $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}" - if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then - test -d "${CERBERO_HOME}/dist/${ARCH}" + if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} && -d "${CERBERO_HOME}/dist/${ARCH}" ]]; then mkdir -p "${CERBERO_OVERRIDDEN_DIST_DIR}" time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}" fi From 1f0b3938af243fce1bfa8095ffed9228748d384a Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 16 Dec 2020 17:23:15 +0200 Subject: [PATCH 569/605] ci_template: use meson subproject --rebase for the windows builds as well Similar to !375 Part-of: --- gitlab/ci_template.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 6924fe2215..bfc9044c3d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -793,9 +793,17 @@ valgrind ges: - python clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination $env:CI_PROJECT_DIR/gst-build - cd $env:CI_PROJECT_DIR/gst-build + # Update meson + # FIXME: remove this once the windows image is rebuilt/updated + - pip install -U meson==0.56.2 + - cp -r C:/subprojects/* subprojects/ + - cd $env:CI_PROJECT_DIR/gst-build + # Reset the subprojects to invalidate the cache if needed + - meson subprojects update --reset # Run the git-update script and feed it the manifest to setup the environment - - cd $env:CI_PROJECT_DIR/gst-build; python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml + - python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml + # For some reason, options are separated by newline instead of space, so we # have to replace them first. - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ") From 8f0481f19a4b2f6847bf86a0003cd41ba7e7e67e Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 27 Jan 2021 07:16:23 +0200 Subject: [PATCH 570/605] docker/windows: make the powershell scripts more robust Check the exit codes of various commands and fail appropriatly upon error. Part-of: --- docker/windows/build_image.ps1 | 20 ++++++++++-- docker/windows/install_choco.ps1 | 18 +++++++++-- docker/windows/install_toolchain.ps1 | 47 +++++++++++++++++++++++++--- docker/windows/prepare_gst_env.ps1 | 8 +++++ 4 files changed, 84 insertions(+), 9 deletions(-) diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 8494dfd4d1..95a60d8aeb 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,6 +1,22 @@ $env:DEFAULT_BRANCH='master' -$env:VERSION='v17' +$env:VERSION='v18' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" -echo "Building $env:tag" + +Get-Date +Write-Output "Building $env:tag" docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . +if (!$?) { + Write-Host "Failed to build docker image $env:tag" + Exit 1 +} + +# Get-Date +# Write-Output "Pushing $env:tag" # docker push $env:tag +# if (!$?) { +# Write-Host "Failed to push docker image $env:tag" +# Exit 1 +# } + +Get-Date +Write-Output "Build Finished" \ No newline at end of file diff --git a/docker/windows/install_choco.ps1 b/docker/windows/install_choco.ps1 index 9fb9d13b41..dd435ed131 100644 --- a/docker/windows/install_choco.ps1 +++ b/docker/windows/install_choco.ps1 @@ -1,6 +1,20 @@ -Write-Host "Installing Choco" +Get-Date +Write-Host "Installing Chocolatey" Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) -Write-Host "Installing Choco packages" +Import-Module "$env:ProgramData\chocolatey\helpers\chocolateyProfile.psm1" +Update-SessionEnvironment + +Write-Host "Installing Chocolatey packages" choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' +$cmake_install = $? + choco install -y git --params "/NoAutoCrlf /NoCredentialManager /NoShellHereIntegration /NoGuiHereIntegration /NoShellIntegration" +$git_install = $? + choco install -y python3 git-lfs 7zip +$rest_installs = $? + +if (!($cmake_install -and $git_install -and $rest_installs)) { + Write-Host "Failed to install some dependencies from choco" + Exit 1 +} diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1 index a29330b32b..fd61b42d1c 100644 --- a/docker/windows/install_toolchain.ps1 +++ b/docker/windows/install_toolchain.ps1 @@ -4,29 +4,66 @@ $msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe' $msys2_url = 'https://github.com/msys2/msys2-installer/releases/download/2021-02-15/msys2-base-x86_64-20210215.tar.xz' $msys_mingw_get_url = 'https://dotsrc.dl.osdn.net/osdn/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.tar.xz' -Write-Host "Installing VisualStudio" +Get-Date +Write-Host "Downloading Visual Studio 2017 build tools" Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe -Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait + +Get-Date +Write-Host "Installing Visual Studio 2017" +Start-Process -NoNewWindow -Wait C:\vs_buildtools.exe -ArgumentList '--wait --quiet --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' +if (!$?) { + Write-Host "Failed to install Visual Studio tools" + Exit 1 +} Remove-Item C:\vs_buildtools.exe -Force +Get-Date Write-Host "Downloading and extracting mingw-get for MSYS" Invoke-WebRequest -Uri $msys_mingw_get_url -OutFile C:\mingw-get.tar.xz 7z e C:\mingw-get.tar.xz -o"C:\\" +$res1 = $? 7z x C:\mingw-get.tar -o"C:\\MinGW" +$res2 = $? + +if (!($res1 -and $res2)) { + Write-Host "Failed to extract mingw-get" + Exit 1 +} + Remove-Item C:\mingw-get.tar.xz -Force Remove-Item C:\mingw-get.tar -Force +Get-Date Write-Host "Installing MSYS for Cerbero into C:/MinGW using mingw-get" -Start-Process C:\MinGW\bin\mingw-get.exe -ArgumentList 'install msys-base mingw32-base mingw-developer-toolkit' -Wait +Start-Process -Wait C:\MinGW\bin\mingw-get.exe -ArgumentList 'install msys-base mingw32-base mingw-developer-toolkit' +if (!$?) { + Write-Host "Failed to install Msys for cerbero using MinGW" + Exit 1 +} +Get-Date Write-Host "Installing MSYS2 into C:/msys64" Invoke-WebRequest -Uri $msys2_url -OutFile C:\msys2-x86_64.tar.xz + 7z e C:\msys2-x86_64.tar.xz -o"C:\\" +$res1 = $? 7z x C:\msys2-x86_64.tar -o"C:\\" +$res2 = $? + +if (!($res1 -and $res2)) { + Write-Host "Failed to extract msys2" + Exit 1 +} + Remove-Item C:\msys2-x86_64.tar.xz -Force Remove-Item C:\msys2-x86_64.tar -Force +Get-Date Write-Host "Installing Meson" -pip install meson +pip3 install meson +if (!$?) { + Write-Host "Failed to install meson from pip" + Exit 1 +} -Write-Host "Complete" +Write-Host "Toolchain Install Complete" diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1 index f5316e9922..601e799a64 100644 --- a/docker/windows/prepare_gst_env.ps1 +++ b/docker/windows/prepare_gst_env.ps1 @@ -7,9 +7,17 @@ # Download gst-build and all its subprojects git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +if (!$?) { + Write-Host "Failed to clone gst-build" + Exit 1 +} # download the subprojects to try and cache them meson subprojects download --sourcedir C:\gst-build +if (!$?) { + Write-Host "Failed to download the subprojects" + Exit 1 +} # Remove files that will conflict with a fresh clone on the runner side Remove-Item -Force 'C:/gst-build/subprojects/*.wrap' From bdbef0449c9c20b59c7421d58c80a37cb7ee8d15 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 27 Jan 2021 09:54:59 +0200 Subject: [PATCH 571/605] ci_template: build windows image in the ci Build the images in the ci runners like we do, for the linux containers. Kudos to mesa for the powershell script that reimplements the ci-templates behavior. https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4304 Part-of: --- docker/windows/container.ps1 | 62 ++++++++++++++++++++++++++++++++++++ gitlab/ci_template.yml | 39 +++++++++++++++++++++-- 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 docker/windows/container.ps1 diff --git a/docker/windows/container.ps1 b/docker/windows/container.ps1 new file mode 100644 index 0000000000..fac0087f99 --- /dev/null +++ b/docker/windows/container.ps1 @@ -0,0 +1,62 @@ +# Copied from mesa, big kudos +# +# https://gitlab.freedesktop.org/mesa/mesa/-/blob/master/.gitlab-ci/windows/mesa_container.ps1 +# https://gitlab.freedesktop.org/mesa/mesa/-/blob/34e3e164936d1d3cef267da7780e87f062fedf39/.gitlab-ci/windows/mesa_container.ps1 + +# Implements the equivalent of ci-templates container-ifnot-exists, using +# Docker directly as we don't have buildah/podman/skopeo available under +# Windows, nor can we execute Docker-in-Docker +$registry_uri = $args[0] +$registry_username = $args[1] +$registry_password = $args[2] +$registry_user_image = $args[3] +$registry_central_image = $args[4] +$dockerfile = $args[5] + +Set-Location -Path ".\docker\windows" + +docker --config "windows-docker.conf" login -u "$registry_username" -p "$registry_password" "$registry_uri" +if (!$?) { + Write-Host "docker login failed to $registry_uri" + Exit 1 +} + +# if the image already exists, don't rebuild it +docker --config "windows-docker.conf" pull "$registry_user_image" +if ($?) { + Write-Host "User image $registry_user_image already exists; not rebuilding" + docker --config "windows-docker.conf" logout "$registry_uri" + Exit 0 +} + +# if the image already exists upstream, copy it +docker --config "windows-docker.conf" pull "$registry_central_image" +if ($?) { + Write-Host "Copying central image $registry_central_image to user image $registry_user_image" + docker --config "windows-docker.conf" tag "$registry_central_image" "$registry_user_image" + docker --config "windows-docker.conf" push "$registry_user_image" + $pushstatus = $? + docker --config "windows-docker.conf" logout "$registry_uri" + if (!$pushstatus) { + Write-Host "Pushing image to $registry_user_image failed" + Exit 1 + } + Exit 0 +} + +Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding" +docker --config "windows-docker.conf" build --no-cache -t "$registry_user_image" -f "$dockerfile" . +if (!$?) { + Write-Host "Container build failed" + docker --config "windows-docker.conf" logout "$registry_uri" + Exit 1 +} +Get-Date + +docker --config "windows-docker.conf" push "$registry_user_image" +$pushstatus = $? +docker --config "windows-docker.conf" logout "$registry_uri" +if (!$pushstatus) { + Write-Host "Pushing image to $registry_user_image failed" + Exit 1 +} diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index bfc9044c3d..25765d02c4 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -27,6 +27,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' + WINDOWS_TAG: "2021-02-24.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' @@ -36,6 +37,7 @@ variables: MANIFEST_AMD64_SUFFIX: 'amd64/build-manifest' TEST_MANIFEST_AMD64_SUFFIX: 'amd64/test-manifest' INDENT_AMD64_SUFFIX: 'amd64/gst-indent' + WINDOWS_AMD64_SUFFIX: 'amd64/windows' FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" FEDORA_DOCS_IMAGE: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-07-03.0-master" @@ -44,11 +46,14 @@ variables: MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$MANIFEST_AMD64_SUFFIX:$MANIFEST_TAG-$GST_UPSTREAM_BRANCH" TEST_MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$TEST_MANIFEST_AMD64_SUFFIX:$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH" INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" - - WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v13-master' + WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Can't use $CI_* variables since we use this template in all repos, not just gst-ci CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" + # FIXME: fix the file path url to point to the upstram once merged + # WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" + WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/alatiera/gst-ci/raw/alatiera/windows-build-redux/docker/windows/container.ps1" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both @@ -327,6 +332,36 @@ gst-indent amd64 docker: DOCKERFILE: "docker/indent/Dockerfile" extends: .base +windows amd64 docker: + stage: "build docker" + variables: + GIT_STRATEGY: fetch + REPO_SUFFIX: "$WINDOWS_AMD64_SUFFIX" + TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + + # Unlike the buildah/linux jobs, this file + # needs to be relative to docker/windows/ subdir + # as it makes life easier in the powershell script + # + # We also don't need a CONTEXT_DIR var as its also + # hardcoded to be docker/windows/ + DOCKERFILE: "Dockerfile" + tags: + - windows + - shell + - "1809" + script: + - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" + - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" + - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH + + - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_IMAGE $WINDOWS_UPSTREAM_IMAGE $DOCKERFILE" + - | + if (!($?)) { + echo "Failed to build the image" + Exit 1 + } + # # Job to create the manifest.xml to used for our builds # From 136019bab77c3754e9819efd418178573a780619 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 5 Mar 2021 07:12:35 +0200 Subject: [PATCH 572/605] ci_template: fix WINDOWS_CONTAINER_SCRIPT_URL location Part-of: --- gitlab/ci_template.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 25765d02c4..066a237327 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -51,9 +51,7 @@ variables: # Can't use $CI_* variables since we use this template in all repos, not just gst-ci CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" - # FIXME: fix the file path url to point to the upstram once merged - # WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" - WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/alatiera/gst-ci/raw/alatiera/windows-build-redux/docker/windows/container.ps1" + WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From 8f8cd9ec4b940a01bfbc38be260f9cccfed9b7a4 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 5 Mar 2021 08:28:47 +0200 Subject: [PATCH 573/605] ci_template: use older windows image for cerbero Current windows image doesn't seem to be able to spawn procceses for reason, use a different image until this is debuged ``` Running command 'c:/builds/alatiera/gst-ci/cerbero-build/build-tools/bin/meson --prefix=c:/builds/alatiera/gst-ci/cerbero-build/dist/msvc_x86_64 --libdir=lib -Ddebug=true --default-library=both -Doptimization=2 --backend=ninja --wrap-mode=nodownload -Db_vscrt=md --native-file c:/builds/alatiera/gst-ci/cerbero-build/sources/msvc_x86_64/gstreamer-1.0-1.19.0.1/_builddir/meson-native-file.txt -Dlibunwind=disabled -Ddbghelp=enabled -Dintrospection=disabled -Dexamples=disabled' failed to create process ``` Part-of: --- gitlab/ci_template.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 066a237327..1560cf34c2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -49,6 +49,12 @@ variables: WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + # Current windows image doesn't seem to be able to spawn procceses for reason, + # use a different image until this is debuged + # Running command 'c:/builds/alatiera/gst-ci/cerbero-build/build-tools/bin/meson --prefix=c:/builds/alatiera/gst-ci/cerbero-build/dist/msvc_x86_64 --libdir=lib -Ddebug=true --default-library=both -Doptimization=2 --backend=ninja --wrap-mode=nodownload -Db_vscrt=md --native-file c:/builds/alatiera/gst-ci/cerbero-build/sources/msvc_x86_64/gstreamer-1.0-1.19.0.1/_builddir/meson-native-file.txt -Dlibunwind=disabled -Ddbghelp=enabled -Dintrospection=disabled -Dexamples=disabled' + # failed to create process + CERBERO_WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v13-master' + # Can't use $CI_* variables since we use this template in all repos, not just gst-ci CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" @@ -1281,7 +1287,7 @@ build gst-omx tizonia fedora x86_64: # .cerbero windows native: needs: ['manifest'] - image: $WINDOWS_IMAGE + image: $CERBERO_WINDOWS_IMAGE tags: - 'docker' - 'windows' From 38a1b046371e01145ca8d715fb1f71add4d46dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Cerveau?= Date: Tue, 30 Mar 2021 10:27:33 +0200 Subject: [PATCH 574/605] docker: fedora: add qt5 devel package. add qt5 devel package to build qt plugin Part-of: --- docker/fedora/prepare.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index bc13d9d99c..800b7af605 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -62,6 +62,7 @@ dnf install -y \ mono-devel \ procps-ng \ patch \ + qt5-devel \ redhat-rpm-config \ json-glib \ json-glib-devel \ @@ -205,10 +206,6 @@ dnf builddep -y gstreamer1 \ # Remove gst-devel packages installed by builddep above dnf remove -y "gstreamer1*devel" -# Remove Qt5 devel packages as we haven't tested building it and -# it leads to build issues in examples. -dnf remove -y "qt5-qtbase-devel" - # FIXME: Why does installing directly with dnf doesn't actually install # the documentation files? dnf download glib2-doc gdk-pixbuf2-devel*x86_64* gtk3-devel-docs From ac6890e50f4e8253a7356e88249fa5e349626c47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Cerveau?= Date: Tue, 30 Mar 2021 15:38:03 +0200 Subject: [PATCH 575/605] docker: fedora: update string version Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 1560cf34c2..c19e7b8e9c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,7 +21,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2021-01-22.0' + FEDORA_TAG: '2021-03-30.0' CERBERO_TAG: '2020-10-22.0' ANDROID_TAG: '2020-10-22.0' MANIFEST_TAG: '2020-10-22.0' From 89050a82223a6d7ce5bb888d969b54a104942692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Cerveau?= Date: Thu, 29 Apr 2021 14:19:57 +0200 Subject: [PATCH 576/605] ci_template: gstreamer-full: enable auto features Enable auto features for gstreamer-full but disable "introspection" for now. Part-of: --- gitlab/ci_template.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c19e7b8e9c..f9c8c3e2ab 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -634,16 +634,16 @@ integration testsuites fedora: rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-omx|gstreamer-sharp|gst-plugins-rs)$/' -# gstreamer-full +# gstreamer-full: +# introspection has been disabled as the static build does not support it. +# See https://gitlab.freedesktop.org/gstreamer/gst-build/-/issues/162 gstreamer-full: extends: 'build static fedora x86_64' stage: integrate variables: MESON_ARGS: > --default-library=static - -Dauto_features=disabled - -Dgstreamer:check=enabled - -Dtests=enabled + -Dintrospection=disabled $MESON_GST_WERROR rules: - if: '$CI_PROJECT_NAME =~ /^(gst-build|gst-ci)$/' From c18e08e4ef67c2f2ba0dcbbbe9541b767266c11e Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Sat, 22 May 2021 15:59:54 +0100 Subject: [PATCH 577/605] docker/fedora: Fix rpmfusion urls Part-of: --- docker/fedora/prepare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 800b7af605..716b2e80ff 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -12,8 +12,8 @@ git config --global user.name "Gstbuild Runner" # Add rpm fusion repositories in order to access all of the gst plugins sudo dnf install -y \ - "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm" \ - "https://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm" + "https://mirrors.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm" \ + "https://mirrors.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm" dnf upgrade -y From 0c5838bcdd45545bd47b5ffaeb66df5483756c7a Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Sat, 22 May 2021 16:00:28 +0100 Subject: [PATCH 578/605] docker/fedora: Reverse meson installation Part-of: --- docker/fedora/prepare.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index 716b2e80ff..efb2f2e7ac 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -185,8 +185,6 @@ dnf debuginfo-install -y gtk3 \ zip \ zlib -pip3 install meson==0.56.2 hotdoc - # Install the dependencies of gstreamer dnf builddep -y gstreamer1 \ gstreamer1-plugins-base \ @@ -200,8 +198,10 @@ dnf builddep -y gstreamer1 \ gstreamer1-libav \ gstreamer1-rtsp-server \ gstreamer1-vaapi \ - python3-gstreamer1 \ - -x meson + python3-gstreamer1 + +dnf remove -y meson +pip3 install meson==0.58.0 hotdoc # Remove gst-devel packages installed by builddep above dnf remove -y "gstreamer1*devel" From 8aa710f6b8af5fd4e541077b2c0835e3382a1158 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Sat, 22 May 2021 16:00:43 +0100 Subject: [PATCH 579/605] docker/fedora: Bump to latest rustup Part-of: --- docker/fedora/prepare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index efb2f2e7ac..c645b69b97 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -213,8 +213,8 @@ rpm -i --reinstall *.rpm rm -f *.rpm # Install Rust -RUSTUP_VERSION=1.21.1 -RUST_VERSION=1.44.1 +RUSTUP_VERSION=1.24.2 +RUST_VERSION=1.52.1 RUST_ARCH="x86_64-unknown-linux-gnu" dnf install -y wget From 92c7258a8280842465dd9c5d40ae6c2c5de60b5b Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 4 Jun 2021 12:38:29 +0300 Subject: [PATCH 580/605] docker/fedora: Fix find cli arg in prepare.sh script Part-of: --- docker/fedora/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index c645b69b97..eb053ae34b 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -237,7 +237,7 @@ cd /gst-build meson subprojects download # Run git gc to prune unwanted refs and reduce the size of the image -for i in $(find subprojects/ -mindepth 1- maxdepth 1 -type d); +for i in $(find subprojects/ -mindepth 1 -maxdepth 1 -type d); do git -C $i gc --aggressive || true; done \ No newline at end of file From f03eca8ff778887490a115f152b0bcf2e980c23b Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 4 Jun 2021 12:21:21 +0300 Subject: [PATCH 581/605] docker/fedora: Bump image tag Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f9c8c3e2ab..ed52fe5ef0 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,7 +21,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2021-03-30.0' + FEDORA_TAG: '2021-06-04.0' CERBERO_TAG: '2020-10-22.0' ANDROID_TAG: '2020-10-22.0' MANIFEST_TAG: '2020-10-22.0' From 02dce71d952706418e79c8b6e0798a35045c1b18 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 24 May 2021 23:11:10 +0300 Subject: [PATCH 582/605] ci_template: mark the manifest job as not allowed to fail With gitlab 13.12 the behavior of jobs that depend of manual jobs changes, and they are considered skipped by default. But in our pipeline we want to enforce that the whole pipeline runs and succeds and the manual status is only there to reduce CI load. For more details see: https://about.gitlab.com/blog/2021/05/20/dag-manual-fix/ Part-of: --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ed52fe5ef0..d543c9c835 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -383,10 +383,12 @@ manifest: # to avoid wasting CI resources - if: '$CI_MERGE_REQUEST_ASSIGNEES != "gstreamer-merge-bot"' when: 'manual' + allow_failure: false # If this matches, it means the pipeline is running against either the main # or a stable branch, so make it manual - if: '$CI_COMMIT_BRANCH == $GST_UPSTREAM_BRANCH' when: 'manual' + allow_failure: false stage: 'preparation' script: From cfdc3bdc3185004abb9d878d9359bae5c9d9d9ab Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 7 Jun 2021 16:02:03 +0300 Subject: [PATCH 583/605] ci_template: rebuild some images We need a newer cache for harfbuzz since it renamed master branch to main now and the wrap file has depth=1 https://gitlab.freedesktop.org/gstreamer/gst-build/-/merge_requests/246#note_948717 Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d543c9c835..a5498840da 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,13 +21,13 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2021-06-04.0' + FEDORA_TAG: '2021-06-07.0' CERBERO_TAG: '2020-10-22.0' ANDROID_TAG: '2020-10-22.0' MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-02-24.0" + WINDOWS_TAG: "2021-06-07.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' From 9a559c3705d676d68aace7cc060b937c08d68f4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Mon, 24 May 2021 17:10:11 +0200 Subject: [PATCH 584/605] docker: fedora: Add intel-mediasdk package. Add intel-mediasdk package to build gst-msdk plugin in gst-plugins-bad. Part-of: --- docker/fedora/prepare.sh | 1 + gitlab/ci_template.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/fedora/prepare.sh b/docker/fedora/prepare.sh index eb053ae34b..a85f3e495d 100644 --- a/docker/fedora/prepare.sh +++ b/docker/fedora/prepare.sh @@ -22,6 +22,7 @@ dnf install -y \ aalib-devel \ aom \ bat \ + intel-mediasdk-devel \ libaom \ libaom-devel \ libcaca-devel \ diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index a5498840da..31cb6ae52f 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,7 +21,7 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2021-06-07.0' + FEDORA_TAG: '2021-06-09.0' CERBERO_TAG: '2020-10-22.0' ANDROID_TAG: '2020-10-22.0' MANIFEST_TAG: '2020-10-22.0' From d6fa1847e6ac1059a7d5918d9f591be24265646e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Wed, 30 Jun 2021 16:45:52 +0100 Subject: [PATCH 585/605] ci_template: rebuild some images We need a newer cache for fontconfig since it renamed master branch to main now and the wrap file has depth=1 Part-of: --- gitlab/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 31cb6ae52f..75e1bad7e7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -21,13 +21,13 @@ variables: # If you are hacking on them or need a them to rebuild, its enough # to change any part of the string of the image you want. ### - FEDORA_TAG: '2021-06-09.0' + FEDORA_TAG: '2021-06-30.0' CERBERO_TAG: '2020-10-22.0' ANDROID_TAG: '2020-10-22.0' MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-06-07.0" + WINDOWS_TAG: "2021-06-30.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' From b65eb14636ada0b340e89fadc3261be1fff078a3 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 17 Feb 2021 16:32:38 +1100 Subject: [PATCH 586/605] ci: build universal macos packages instead of x86_64 only Part-of: --- gitlab/ci_template.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 75e1bad7e7..d70530fd5b 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1115,6 +1115,40 @@ build cerbero macos x86_64: rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' +# +# Cerbero macOS Universal build +# +.cerbero cross-macos universal: + variables: + ARCH: "darwin_universal" + CONFIG: "cross-macos-universal.cbc" + CERBERO_PACKAGE_ARGS: "" + HAVE_CCACHE: "" + CERBERO_HOST_DIR: "/Users/gst-ci/cerbero/" + tags: + - gst-macos-11.1 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" + expire_in: '5 days' + when: 'always' + paths: + - "manifest.xml" + - "${CERBERO_HOME}/logs" + - "gstreamer-1.0-1.*.pkg" + - "gstreamer-1.0-devel-1.*.pkg" + +cerbero deps cross-macos universal: + extends: + - '.cerbero deps' + - '.cerbero cross-macos universal' + +build cerbero cross-macos universal: + extends: + - '.cerbero' + - '.cerbero cross-macos universal' + rules: + - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' + # # Cerbero iOS build # From ab5c501b871b73b6d81728ea8e8b833d087b9619 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 29 Jul 2021 22:41:06 +1000 Subject: [PATCH 587/605] gitlab: fix incorrect build-tools path being used on macOS The relevant CI log dyld: Library not loaded: /Users/gst-ci/builds/gstreamer/cerbero/cerbero-build/dist/darwin_universal/x86_64/lib/liborc-0.4.0.dylib Referenced from: /Users/gst-ci/builds/xhaakon/gstreamer-sharp/cerbero-build/dist/darwin_universal/x86_64/bin/orcc Reason: image not found Part-of: --- gitlab/ci_template.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index d70530fd5b..55781089d2 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1146,6 +1146,16 @@ build cerbero cross-macos universal: extends: - '.cerbero' - '.cerbero cross-macos universal' + script: + # We have build-tools that have fixed paths pointing to the upstream + # repository. We need to point + - pwd + - mkdir -p "../../gstreamer" + - ln -sf "$(pwd)" "../../gstreamer/cerbero" + - mkdir -p "../../${CI_PROJECT_NAMESPACE}" + - ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" + + - ./cerbero_setup.sh cerbero_deps_script rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' From 9c7a2df4a8b7c21d4c942f2c7b81518bec45da2e Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 17 Aug 2021 16:04:42 +0530 Subject: [PATCH 588/605] cerbero: Always fetch sources with four parallel jobs The default number of parallel jobs is two, which is too few. We can easily use four or more. Should speed up image builds and also downloading of (new) sources that aren't already cached in the image. Part-of: --- docker/cerbero/Dockerfile-fedora | 16 ++++++++-------- docker/windows/prepare_cerbero_env.sh | 6 +++--- gitlab/cerbero_setup.sh | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/cerbero/Dockerfile-fedora b/docker/cerbero/Dockerfile-fedora index 6009c421ca..c903b9562b 100644 --- a/docker/cerbero/Dockerfile-fedora +++ b/docker/cerbero/Dockerfile-fedora @@ -20,15 +20,15 @@ RUN git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerb cd cerbero && \ echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \ echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \ - ./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -t -c localconf.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap --jobs=4 && \ + ./cerbero-uninstalled -t -c localconf.cbc fetch-package --jobs=4 gstreamer-1.0 && \ ./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \ - ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap --jobs=4 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package --jobs=4 gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap --jobs=4 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package --jobs=4 gstreamer-1.0 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap --jobs=4 && \ + ./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package --jobs=4 gstreamer-1.0 && \ cd .. && \ rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \ rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \ diff --git a/docker/windows/prepare_cerbero_env.sh b/docker/windows/prepare_cerbero_env.sh index 7eafd2eb71..da94acabb3 100644 --- a/docker/windows/prepare_cerbero_env.sh +++ b/docker/windows/prepare_cerbero_env.sh @@ -12,11 +12,11 @@ echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc echo 'vs_install_version = "vs15"' >> localconf.cbc # Fetch all bootstrap requirements -./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-bootstrap +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-bootstrap --jobs=4 # Fetch all package requirements for a mingw gstreamer build -./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0 +./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc fetch-package --jobs=4 gstreamer-1.0 # Fetch all package requirements for a visualstudio gstreamer build -./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0 +./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package --jobs=4 gstreamer-1.0 # Fixup the MSYS installation ./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline diff --git a/gitlab/cerbero_setup.sh b/gitlab/cerbero_setup.sh index fbf164faa7..96fc7390df 100644 --- a/gitlab/cerbero_setup.sh +++ b/gitlab/cerbero_setup.sh @@ -120,8 +120,8 @@ cerbero_script() { show_ccache_sum $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 + $CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4 + $CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0 du -sch "${CERBERO_SOURCES}" || true $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}" @@ -141,8 +141,8 @@ cerbero_deps_script() { show_ccache_sum $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS fetch-bootstrap - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 + $CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4 + $CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0 $CERBERO $CERBERO_ARGS bootstrap --offline --system=no $CERBERO $CERBERO_ARGS build-deps --offline \ gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \ From abe89db32dbe93074f0eb22caa64b6411aa9373b Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 7 Sep 2021 22:47:09 +1000 Subject: [PATCH 589/605] cerbero: fix cross-macos universal building all of cerbero When building outside of cerbero, we only want to build the gst modules as necessary. Regression introduced by 03c5f1db29d751f8ccbe6062d8046c307c92a60d Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 55781089d2..f2968b7f0a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -1155,7 +1155,7 @@ build cerbero cross-macos universal: - mkdir -p "../../${CI_PROJECT_NAMESPACE}" - ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero" - - ./cerbero_setup.sh cerbero_deps_script + - ./cerbero_setup.sh cerbero_script rules: - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gst-sharp|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/' From 37c0ee72806b767a01b4eb60e533e3dc9c4bcbef Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 18 Aug 2021 15:16:24 +0300 Subject: [PATCH 590/605] docker/windows: install vcredist140 explicitly We depend on it for GStreamer builds and it was gettings pulled transiently by python3 install. Part-of: --- docker/windows/install_choco.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/windows/install_choco.ps1 b/docker/windows/install_choco.ps1 index dd435ed131..b19eb7b0a3 100644 --- a/docker/windows/install_choco.ps1 +++ b/docker/windows/install_choco.ps1 @@ -4,6 +4,9 @@ Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://cho Import-Module "$env:ProgramData\chocolatey\helpers\chocolateyProfile.psm1" Update-SessionEnvironment +choco install -y vcredist140 +$vc140_install = $? + Write-Host "Installing Chocolatey packages" choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' $cmake_install = $? @@ -14,7 +17,7 @@ $git_install = $? choco install -y python3 git-lfs 7zip $rest_installs = $? -if (!($cmake_install -and $git_install -and $rest_installs)) { +if (!($vc140_install -and $cmake_install -and $git_install -and $rest_installs)) { Write-Host "Failed to install some dependencies from choco" Exit 1 } From 9c21b05d1bf438d728a69d94c3acae8ddf85d496 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 16 Oct 2020 02:13:07 +0300 Subject: [PATCH 591/605] docker/windows: create a rust image for gst-rs Based on the build image for the existing windows jobs, add an image with rust toolchain and a (stable) gstreamer install for the bindings to build against. See https://gitlab.freedesktop.org/gstreamer/gstreamer-rs/-/merge_requests/701 Part-of: --- docker/windows/build_image.ps1 | 24 ++++++++++- docker/windows/install_gst.ps1 | 76 ++++++++++++++++++++++++++++++++++ docker/windows/rust.Dockerfile | 18 ++++++++ gitlab/ci_template.yml | 2 +- 4 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 docker/windows/install_gst.ps1 create mode 100644 docker/windows/rust.Dockerfile diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 95a60d8aeb..37ffe02ce2 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,15 +1,28 @@ +$env:ErrorActionPreference='Stop' + $env:DEFAULT_BRANCH='master' $env:VERSION='v18' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" +$env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH" + +Set-Location './docker/windows/' Get-Date Write-Output "Building $env:tag" -docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . +docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . if (!$?) { Write-Host "Failed to build docker image $env:tag" Exit 1 } +Get-Date +Write-Output "Building $env:rust_tag" +docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f rust.Dockerfile -t $env:rust_tag . +if (!$?) { + Write-Host "Failed to build docker image $env:rust_tag" + Exit 1 +} + # Get-Date # Write-Output "Pushing $env:tag" # docker push $env:tag @@ -18,5 +31,14 @@ if (!$?) { # Exit 1 # } +# Get-Date +# Write-Output "Pushing $env:rust_tag" +# docker push $env:rust_tag +# if (!$?) { +# Write-Host "Failed to push docker image $env:rust_tag" +# Exit 1 +# } + + Get-Date Write-Output "Build Finished" \ No newline at end of file diff --git a/docker/windows/install_gst.ps1 b/docker/windows/install_gst.ps1 new file mode 100644 index 0000000000..f0002590f0 --- /dev/null +++ b/docker/windows/install_gst.ps1 @@ -0,0 +1,76 @@ +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +# FIXME: Python fails to validate github.com SSL certificate, unless we first +# run a dummy download to force refreshing Windows' CA database. +# See: https://bugs.python.org/issue36137 +(New-Object System.Net.WebClient).DownloadString("https://github.com") >$null + +# Download gst-build and all its subprojects +# git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +# FIXME: need 1.19+ for cairo subproject :/ +# Should use a stable branch instead +git clone -b master --depth 1 https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +if (!$?) { + Write-Host "Failed to clone gst-build" + Exit 1 +} + +Set-Location C:\gst-build + +# Copy the cache we already have in the image to avoid massive redownloads +Move-Item C:/subprojects/* C:\gst-build\subprojects + +if (!$?) { + Write-Host "Failed to copy subprojects cache" + Exit 1 +} + +# Update the subprojects cache +Write-Output "Running meson subproject reset" +meson subprojects update --reset + +if (!$?) { + Write-Host "Failed to reset subprojects state" + Exit 1 +} + +Write-Output "Running git update" +python git-update --no-interaction + +if (!$?) { + Write-Host "Failed to run git-update" + Exit 1 +} + +$env:MESON_ARGS = "-Dglib:installed_tests=false " + + "-Dlibnice:tests=disabled " + + "-Dlibnice:examples=disabled " + + "-Dffmpeg:tests=disabled " + + "-Dopenh264:tests=disabled " + + "-Dpygobject:tests=false " + + "-Dugly=enabled " + + "-Dbad=enabled " + + "-Dges=enabled " + + "-Drtsp_server=enabled " + + "-Ddevtools=enabled " + + "-Dsharp=disabled " + + "-Dpython=disabled " + + "-Dlibav=disabled " + + "-Dvaapi=disabled " + + "-Dgst-plugins-base:pango=enabled " + + "-Dgst-plugins-good:cairo=enabled " + +Write-Output "Building gst" +cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && meson _build $env:MESON_ARGS && meson compile -C _build && ninja -C _build install" + +if (!$?) { + Write-Host "Failed to build and install gst" + Exit 1 +} + +git clean -fdxx + +if (!$?) { + Write-Host "Failed to git clean" + Exit 1 +} \ No newline at end of file diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile new file mode 100644 index 0000000000..7ccfaffee9 --- /dev/null +++ b/docker/windows/rust.Dockerfile @@ -0,0 +1,18 @@ +# escape=` + +FROM 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v16-master' + +ARG DEFAULT_BRANCH="master" +ARG RUST_VERSION="1.52.1" + +COPY install_gst.ps1 C:\ +RUN C:\install_gst.ps1 +RUN choco install -y pkgconfiglite +ENV PKG_CONFIG_PATH="C:/lib/pkgconfig" + +ADD https://win.rustup.rs/x86_64 C:\rustup-init.exe +RUN C:\rustup-init.exe -y --profile minimal --default-toolchain $env:RUST_VERSION + +# Uncomment for easy testing +# RUN git clone --depth 1 https://gitlab.freedesktop.org/gstreamer/gstreamer-rs.git +# RUN cd gstreamer-rs; cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64; cargo build --all; cargo test --all" \ No newline at end of file diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f2968b7f0a..308af0ed0c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -27,7 +27,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-06-30.0" + WINDOWS_TAG: "2021-07-12.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' From 5c536f2601bbb547b566e5835af1131762eb2f66 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Sat, 30 Jan 2021 11:46:07 +0200 Subject: [PATCH 592/605] docker/windows: build rust image in the ci This makes it a bit harder to build an image locally with the script, as now the rust image requires to pass it the base image tag as a --build-arg. This way though we don't need to duplicate and keep the string of the image in sync. Another thing is that now the docker windows image build is properly passing/replacing build args and we can build non-master versions of the image too. Part-of: --- docker/windows/build_image.ps1 | 19 +------------ docker/windows/container.ps1 | 4 +-- docker/windows/rust.Dockerfile | 5 +++- gitlab/ci_template.yml | 50 ++++++++++++++++++++++++++++++++-- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 37ffe02ce2..a4be56c1fa 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,7 +1,7 @@ $env:ErrorActionPreference='Stop' $env:DEFAULT_BRANCH='master' -$env:VERSION='v18' +$env:VERSION='test' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" $env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH" @@ -23,22 +23,5 @@ if (!$?) { Exit 1 } -# Get-Date -# Write-Output "Pushing $env:tag" -# docker push $env:tag -# if (!$?) { -# Write-Host "Failed to push docker image $env:tag" -# Exit 1 -# } - -# Get-Date -# Write-Output "Pushing $env:rust_tag" -# docker push $env:rust_tag -# if (!$?) { -# Write-Host "Failed to push docker image $env:rust_tag" -# Exit 1 -# } - - Get-Date Write-Output "Build Finished" \ No newline at end of file diff --git a/docker/windows/container.ps1 b/docker/windows/container.ps1 index fac0087f99..e5dc2cd835 100644 --- a/docker/windows/container.ps1 +++ b/docker/windows/container.ps1 @@ -13,8 +13,6 @@ $registry_user_image = $args[3] $registry_central_image = $args[4] $dockerfile = $args[5] -Set-Location -Path ".\docker\windows" - docker --config "windows-docker.conf" login -u "$registry_username" -p "$registry_password" "$registry_uri" if (!$?) { Write-Host "docker login failed to $registry_uri" @@ -45,7 +43,7 @@ if ($?) { } Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding" -docker --config "windows-docker.conf" build --no-cache -t "$registry_user_image" -f "$dockerfile" . +docker --config "windows-docker.conf" build $DOCKER_BUILD_ARGS --no-cache -t "$registry_user_image" -f "$dockerfile" "./docker/windows" if (!$?) { Write-Host "Container build failed" docker --config "windows-docker.conf" logout "$registry_uri" diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile index 7ccfaffee9..6d52353788 100644 --- a/docker/windows/rust.Dockerfile +++ b/docker/windows/rust.Dockerfile @@ -1,6 +1,9 @@ # escape=` -FROM 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v16-master' +# Expect this to be set when calling docker build with +# --build-arg BASE_IMAGE="" and make it fail if not set. +ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid" +FROM $BASE_IMAGE ARG DEFAULT_BRANCH="master" ARG RUST_VERSION="1.52.1" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 308af0ed0c..8435c0ed88 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -3,6 +3,7 @@ include: stages: - 'build docker' + - 'build docker stage 2' - 'preparation' - 'pre-build' - 'build' @@ -38,6 +39,7 @@ variables: TEST_MANIFEST_AMD64_SUFFIX: 'amd64/test-manifest' INDENT_AMD64_SUFFIX: 'amd64/gst-indent' WINDOWS_AMD64_SUFFIX: 'amd64/windows' + WINDOWS_RUST_AMD64_SUFFIX: 'amd64/windows-rust' FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" FEDORA_DOCS_IMAGE: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-07-03.0-master" @@ -48,6 +50,8 @@ variables: INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + WINDOWS_RUST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + WINDOWS_RUST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Current windows image doesn't seem to be able to spawn procceses for reason, # use a different image until this is debuged @@ -57,7 +61,9 @@ variables: # Can't use $CI_* variables since we use this template in all repos, not just gst-ci CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" - WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" + # FIXME: fix the file path url to point to the upstram once merged + # WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" + WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/alatiera/gst-ci/raw/alatiera/windows-rs-wip/docker/windows/container.ps1" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both @@ -349,12 +355,15 @@ windows amd64 docker: # # We also don't need a CONTEXT_DIR var as its also # hardcoded to be docker/windows/ - DOCKERFILE: "Dockerfile" + DOCKERFILE: "docker/windows/Dockerfile" tags: - windows - shell - "1809" script: + # We need to pass an array and to resolve the env vars, so we can't use a variable: + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH") + - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH @@ -366,6 +375,43 @@ windows amd64 docker: Exit 1 } +windows rust amd64 docker: + stage: 'build docker stage 2' + needs: + - job: 'windows amd64 docker' + artifacts: false + variables: + GIT_STRATEGY: 'fetch' + REPO_SUFFIX: "$WINDOWS_RUST_AMD64_SUFFIX" + TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + + # Unlike the buildah/linux jobs, this file + # needs to be relative to docker/windows/ subdir + # as it makes life easier in the powershell script + # + # We also don't need a CONTEXT_DIR var as its also + # hardcoded to be docker/windows/ + DOCKERFILE: 'docker/windows/rust.Dockerfile' + tags: + - 'windows' + - 'shell' + - '1809' + script: + # We need to pass an array and to resolve the env vars, so we can't use a variable: + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE") + + - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" + - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" + - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH + + - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_RUST_IMAGE $WINDOWS_RUST_UPSTREAM_IMAGE $DOCKERFILE" + - | + if (!($?)) { + echo "Failed to build the image" + Exit 1 + } + + # # Job to create the manifest.xml to used for our builds # From 3d9ddab20949a581895e6f430aa862432c58ecd5 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 14 May 2021 11:51:35 +0300 Subject: [PATCH 593/605] docker/windows: build two version of the rust image Build 2 images with different rust toolchains so we can test both the latest release, as well as the minimum supported compiler version and catch regressions. Part-of: --- docker/windows/rust.Dockerfile | 2 +- gitlab/ci_template.yml | 36 ++++++++++++++++++++++++---------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile index 6d52353788..5e7425fef9 100644 --- a/docker/windows/rust.Dockerfile +++ b/docker/windows/rust.Dockerfile @@ -6,7 +6,7 @@ ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid" FROM $BASE_IMAGE ARG DEFAULT_BRANCH="master" -ARG RUST_VERSION="1.52.1" +ARG RUST_VERSION="invalid" COPY install_gst.ps1 C:\ RUN C:\install_gst.ps1 diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8435c0ed88..aaee920f8c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -28,7 +28,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-07-12.0" + WINDOWS_TAG: "2021-08-24.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' @@ -50,8 +50,15 @@ variables: INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_RUST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_RUST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + + RUST_MINIMUM_VERSION: '1.54.0' + RUST_LATEST_VERSION: '1.54.0' + + WINDOWS_RUST_MINIMUM_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" + WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" + + WINDOWS_RUST_LATEST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_LATEST_VERSION" + WINDOWS_RUST_LATEST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_LATEST_VERSION" # Current windows image doesn't seem to be able to spawn procceses for reason, # use a different image until this is debuged @@ -346,8 +353,6 @@ windows amd64 docker: stage: "build docker" variables: GIT_STRATEGY: fetch - REPO_SUFFIX: "$WINDOWS_AMD64_SUFFIX" - TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Unlike the buildah/linux jobs, this file # needs to be relative to docker/windows/ subdir @@ -375,15 +380,13 @@ windows amd64 docker: Exit 1 } -windows rust amd64 docker: +.windows rust docker build: stage: 'build docker stage 2' needs: - job: 'windows amd64 docker' artifacts: false variables: GIT_STRATEGY: 'fetch' - REPO_SUFFIX: "$WINDOWS_RUST_AMD64_SUFFIX" - TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Unlike the buildah/linux jobs, this file # needs to be relative to docker/windows/ subdir @@ -398,19 +401,32 @@ windows rust amd64 docker: - '1809' script: # We need to pass an array and to resolve the env vars, so we can't use a variable: - - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE") + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE", "--build-arg", "RUST_VERSION=$RUST_VERSION") - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH - - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_RUST_IMAGE $WINDOWS_RUST_UPSTREAM_IMAGE $DOCKERFILE" + - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $RUST_IMAGE $RUST_UPSTREAM_IMAGE $DOCKERFILE" - | if (!($?)) { echo "Failed to build the image" Exit 1 } +windows rust amd64 docker latest stable: + extends: '.windows rust docker build' + variables: + RUST_IMAGE: !reference [variables, "WINDOWS_RUST_LATEST_IMAGE"] + RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_LATEST_UPSTREAM_IMAGE"] + RUST_VERSION: !reference [variables, "RUST_LATEST_VERSION"] + +windows rust amd64 docker minimum supported version: + extends: '.windows rust docker build' + variables: + RUST_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_IMAGE"] + RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE"] + RUST_VERSION: !reference [variables, "RUST_MINIMUM_VERSION"] # # Job to create the manifest.xml to used for our builds From 3cfedc8a9a6272a4a402a5b14cd095c10a0d75c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordan=20Petrid=D1=96s?= Date: Wed, 8 Sep 2021 06:52:29 +0000 Subject: [PATCH 594/605] Revert "docker/windows: build two version of the rust image" This reverts commit 3d9ddab20949a581895e6f430aa862432c58ecd5 --- docker/windows/rust.Dockerfile | 2 +- gitlab/ci_template.yml | 36 ++++++++++------------------------ 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile index 5e7425fef9..6d52353788 100644 --- a/docker/windows/rust.Dockerfile +++ b/docker/windows/rust.Dockerfile @@ -6,7 +6,7 @@ ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid" FROM $BASE_IMAGE ARG DEFAULT_BRANCH="master" -ARG RUST_VERSION="invalid" +ARG RUST_VERSION="1.52.1" COPY install_gst.ps1 C:\ RUN C:\install_gst.ps1 diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index aaee920f8c..8435c0ed88 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -28,7 +28,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-08-24.0" + WINDOWS_TAG: "2021-07-12.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' @@ -50,15 +50,8 @@ variables: INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" - - RUST_MINIMUM_VERSION: '1.54.0' - RUST_LATEST_VERSION: '1.54.0' - - WINDOWS_RUST_MINIMUM_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" - WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" - - WINDOWS_RUST_LATEST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_LATEST_VERSION" - WINDOWS_RUST_LATEST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_LATEST_VERSION" + WINDOWS_RUST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + WINDOWS_RUST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Current windows image doesn't seem to be able to spawn procceses for reason, # use a different image until this is debuged @@ -353,6 +346,8 @@ windows amd64 docker: stage: "build docker" variables: GIT_STRATEGY: fetch + REPO_SUFFIX: "$WINDOWS_AMD64_SUFFIX" + TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Unlike the buildah/linux jobs, this file # needs to be relative to docker/windows/ subdir @@ -380,13 +375,15 @@ windows amd64 docker: Exit 1 } -.windows rust docker build: +windows rust amd64 docker: stage: 'build docker stage 2' needs: - job: 'windows amd64 docker' artifacts: false variables: GIT_STRATEGY: 'fetch' + REPO_SUFFIX: "$WINDOWS_RUST_AMD64_SUFFIX" + TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Unlike the buildah/linux jobs, this file # needs to be relative to docker/windows/ subdir @@ -401,32 +398,19 @@ windows amd64 docker: - '1809' script: # We need to pass an array and to resolve the env vars, so we can't use a variable: - - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE", "--build-arg", "RUST_VERSION=$RUST_VERSION") + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE") - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH - - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $RUST_IMAGE $RUST_UPSTREAM_IMAGE $DOCKERFILE" + - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_RUST_IMAGE $WINDOWS_RUST_UPSTREAM_IMAGE $DOCKERFILE" - | if (!($?)) { echo "Failed to build the image" Exit 1 } -windows rust amd64 docker latest stable: - extends: '.windows rust docker build' - variables: - RUST_IMAGE: !reference [variables, "WINDOWS_RUST_LATEST_IMAGE"] - RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_LATEST_UPSTREAM_IMAGE"] - RUST_VERSION: !reference [variables, "RUST_LATEST_VERSION"] - -windows rust amd64 docker minimum supported version: - extends: '.windows rust docker build' - variables: - RUST_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_IMAGE"] - RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE"] - RUST_VERSION: !reference [variables, "RUST_MINIMUM_VERSION"] # # Job to create the manifest.xml to used for our builds From 3f17e16c72409be5d9c257aefc1abd09c4e26b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordan=20Petrid=D1=96s?= Date: Wed, 8 Sep 2021 06:54:15 +0000 Subject: [PATCH 595/605] Revert "docker/windows: build rust image in the ci" This reverts commit 5c536f2601bbb547b566e5835af1131762eb2f66 --- docker/windows/build_image.ps1 | 19 ++++++++++++- docker/windows/container.ps1 | 4 ++- docker/windows/rust.Dockerfile | 5 +--- gitlab/ci_template.yml | 50 ++-------------------------------- 4 files changed, 24 insertions(+), 54 deletions(-) diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index a4be56c1fa..37ffe02ce2 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,7 +1,7 @@ $env:ErrorActionPreference='Stop' $env:DEFAULT_BRANCH='master' -$env:VERSION='test' +$env:VERSION='v18' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" $env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH" @@ -23,5 +23,22 @@ if (!$?) { Exit 1 } +# Get-Date +# Write-Output "Pushing $env:tag" +# docker push $env:tag +# if (!$?) { +# Write-Host "Failed to push docker image $env:tag" +# Exit 1 +# } + +# Get-Date +# Write-Output "Pushing $env:rust_tag" +# docker push $env:rust_tag +# if (!$?) { +# Write-Host "Failed to push docker image $env:rust_tag" +# Exit 1 +# } + + Get-Date Write-Output "Build Finished" \ No newline at end of file diff --git a/docker/windows/container.ps1 b/docker/windows/container.ps1 index e5dc2cd835..fac0087f99 100644 --- a/docker/windows/container.ps1 +++ b/docker/windows/container.ps1 @@ -13,6 +13,8 @@ $registry_user_image = $args[3] $registry_central_image = $args[4] $dockerfile = $args[5] +Set-Location -Path ".\docker\windows" + docker --config "windows-docker.conf" login -u "$registry_username" -p "$registry_password" "$registry_uri" if (!$?) { Write-Host "docker login failed to $registry_uri" @@ -43,7 +45,7 @@ if ($?) { } Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding" -docker --config "windows-docker.conf" build $DOCKER_BUILD_ARGS --no-cache -t "$registry_user_image" -f "$dockerfile" "./docker/windows" +docker --config "windows-docker.conf" build --no-cache -t "$registry_user_image" -f "$dockerfile" . if (!$?) { Write-Host "Container build failed" docker --config "windows-docker.conf" logout "$registry_uri" diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile index 6d52353788..7ccfaffee9 100644 --- a/docker/windows/rust.Dockerfile +++ b/docker/windows/rust.Dockerfile @@ -1,9 +1,6 @@ # escape=` -# Expect this to be set when calling docker build with -# --build-arg BASE_IMAGE="" and make it fail if not set. -ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid" -FROM $BASE_IMAGE +FROM 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v16-master' ARG DEFAULT_BRANCH="master" ARG RUST_VERSION="1.52.1" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8435c0ed88..308af0ed0c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -3,7 +3,6 @@ include: stages: - 'build docker' - - 'build docker stage 2' - 'preparation' - 'pre-build' - 'build' @@ -39,7 +38,6 @@ variables: TEST_MANIFEST_AMD64_SUFFIX: 'amd64/test-manifest' INDENT_AMD64_SUFFIX: 'amd64/gst-indent' WINDOWS_AMD64_SUFFIX: 'amd64/windows' - WINDOWS_RUST_AMD64_SUFFIX: 'amd64/windows-rust' FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" FEDORA_DOCS_IMAGE: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-07-03.0-master" @@ -50,8 +48,6 @@ variables: INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_RUST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_RUST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Current windows image doesn't seem to be able to spawn procceses for reason, # use a different image until this is debuged @@ -61,9 +57,7 @@ variables: # Can't use $CI_* variables since we use this template in all repos, not just gst-ci CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" - # FIXME: fix the file path url to point to the upstram once merged - # WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" - WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/alatiera/gst-ci/raw/alatiera/windows-rs-wip/docker/windows/container.ps1" + WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both @@ -355,15 +349,12 @@ windows amd64 docker: # # We also don't need a CONTEXT_DIR var as its also # hardcoded to be docker/windows/ - DOCKERFILE: "docker/windows/Dockerfile" + DOCKERFILE: "Dockerfile" tags: - windows - shell - "1809" script: - # We need to pass an array and to resolve the env vars, so we can't use a variable: - - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH") - - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH @@ -375,43 +366,6 @@ windows amd64 docker: Exit 1 } -windows rust amd64 docker: - stage: 'build docker stage 2' - needs: - - job: 'windows amd64 docker' - artifacts: false - variables: - GIT_STRATEGY: 'fetch' - REPO_SUFFIX: "$WINDOWS_RUST_AMD64_SUFFIX" - TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" - - # Unlike the buildah/linux jobs, this file - # needs to be relative to docker/windows/ subdir - # as it makes life easier in the powershell script - # - # We also don't need a CONTEXT_DIR var as its also - # hardcoded to be docker/windows/ - DOCKERFILE: 'docker/windows/rust.Dockerfile' - tags: - - 'windows' - - 'shell' - - '1809' - script: - # We need to pass an array and to resolve the env vars, so we can't use a variable: - - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE") - - - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" - - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" - - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH - - - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_RUST_IMAGE $WINDOWS_RUST_UPSTREAM_IMAGE $DOCKERFILE" - - | - if (!($?)) { - echo "Failed to build the image" - Exit 1 - } - - # # Job to create the manifest.xml to used for our builds # From b81620c3a75b6fff821e815c65b20f4fe36c677a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordan=20Petrid=D1=96s?= Date: Wed, 8 Sep 2021 07:36:47 +0000 Subject: [PATCH 596/605] Revert "docker/windows: create a rust image for gst-rs" This reverts commit 9c21b05d1bf438d728a69d94c3acae8ddf85d496 --- docker/windows/build_image.ps1 | 24 +---------- docker/windows/install_gst.ps1 | 76 ---------------------------------- docker/windows/rust.Dockerfile | 18 -------- gitlab/ci_template.yml | 2 +- 4 files changed, 2 insertions(+), 118 deletions(-) delete mode 100644 docker/windows/install_gst.ps1 delete mode 100644 docker/windows/rust.Dockerfile diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 37ffe02ce2..95a60d8aeb 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,28 +1,15 @@ -$env:ErrorActionPreference='Stop' - $env:DEFAULT_BRANCH='master' $env:VERSION='v18' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" -$env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH" - -Set-Location './docker/windows/' Get-Date Write-Output "Building $env:tag" -docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . +docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . if (!$?) { Write-Host "Failed to build docker image $env:tag" Exit 1 } -Get-Date -Write-Output "Building $env:rust_tag" -docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f rust.Dockerfile -t $env:rust_tag . -if (!$?) { - Write-Host "Failed to build docker image $env:rust_tag" - Exit 1 -} - # Get-Date # Write-Output "Pushing $env:tag" # docker push $env:tag @@ -31,14 +18,5 @@ if (!$?) { # Exit 1 # } -# Get-Date -# Write-Output "Pushing $env:rust_tag" -# docker push $env:rust_tag -# if (!$?) { -# Write-Host "Failed to push docker image $env:rust_tag" -# Exit 1 -# } - - Get-Date Write-Output "Build Finished" \ No newline at end of file diff --git a/docker/windows/install_gst.ps1 b/docker/windows/install_gst.ps1 deleted file mode 100644 index f0002590f0..0000000000 --- a/docker/windows/install_gst.ps1 +++ /dev/null @@ -1,76 +0,0 @@ -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; - -# FIXME: Python fails to validate github.com SSL certificate, unless we first -# run a dummy download to force refreshing Windows' CA database. -# See: https://bugs.python.org/issue36137 -(New-Object System.Net.WebClient).DownloadString("https://github.com") >$null - -# Download gst-build and all its subprojects -# git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build -# FIXME: need 1.19+ for cairo subproject :/ -# Should use a stable branch instead -git clone -b master --depth 1 https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build -if (!$?) { - Write-Host "Failed to clone gst-build" - Exit 1 -} - -Set-Location C:\gst-build - -# Copy the cache we already have in the image to avoid massive redownloads -Move-Item C:/subprojects/* C:\gst-build\subprojects - -if (!$?) { - Write-Host "Failed to copy subprojects cache" - Exit 1 -} - -# Update the subprojects cache -Write-Output "Running meson subproject reset" -meson subprojects update --reset - -if (!$?) { - Write-Host "Failed to reset subprojects state" - Exit 1 -} - -Write-Output "Running git update" -python git-update --no-interaction - -if (!$?) { - Write-Host "Failed to run git-update" - Exit 1 -} - -$env:MESON_ARGS = "-Dglib:installed_tests=false " + - "-Dlibnice:tests=disabled " + - "-Dlibnice:examples=disabled " + - "-Dffmpeg:tests=disabled " + - "-Dopenh264:tests=disabled " + - "-Dpygobject:tests=false " + - "-Dugly=enabled " + - "-Dbad=enabled " + - "-Dges=enabled " + - "-Drtsp_server=enabled " + - "-Ddevtools=enabled " + - "-Dsharp=disabled " + - "-Dpython=disabled " + - "-Dlibav=disabled " + - "-Dvaapi=disabled " + - "-Dgst-plugins-base:pango=enabled " + - "-Dgst-plugins-good:cairo=enabled " - -Write-Output "Building gst" -cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && meson _build $env:MESON_ARGS && meson compile -C _build && ninja -C _build install" - -if (!$?) { - Write-Host "Failed to build and install gst" - Exit 1 -} - -git clean -fdxx - -if (!$?) { - Write-Host "Failed to git clean" - Exit 1 -} \ No newline at end of file diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile deleted file mode 100644 index 7ccfaffee9..0000000000 --- a/docker/windows/rust.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# escape=` - -FROM 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v16-master' - -ARG DEFAULT_BRANCH="master" -ARG RUST_VERSION="1.52.1" - -COPY install_gst.ps1 C:\ -RUN C:\install_gst.ps1 -RUN choco install -y pkgconfiglite -ENV PKG_CONFIG_PATH="C:/lib/pkgconfig" - -ADD https://win.rustup.rs/x86_64 C:\rustup-init.exe -RUN C:\rustup-init.exe -y --profile minimal --default-toolchain $env:RUST_VERSION - -# Uncomment for easy testing -# RUN git clone --depth 1 https://gitlab.freedesktop.org/gstreamer/gstreamer-rs.git -# RUN cd gstreamer-rs; cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64; cargo build --all; cargo test --all" \ No newline at end of file diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 308af0ed0c..f2968b7f0a 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -27,7 +27,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-07-12.0" + WINDOWS_TAG: "2021-06-30.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' From 6da650c12fe292c5670f5a89603e3d069a31a362 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 8 Sep 2021 15:14:54 +0300 Subject: [PATCH 597/605] Revert "Revert "docker/windows: create a rust image for gst-rs"" This reverts commit b81620c3a75b6fff821e815c65b20f4fe36c677a. Part-of: --- docker/windows/build_image.ps1 | 24 ++++++++++- docker/windows/install_gst.ps1 | 76 ++++++++++++++++++++++++++++++++++ docker/windows/rust.Dockerfile | 18 ++++++++ gitlab/ci_template.yml | 2 +- 4 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 docker/windows/install_gst.ps1 create mode 100644 docker/windows/rust.Dockerfile diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 95a60d8aeb..37ffe02ce2 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,15 +1,28 @@ +$env:ErrorActionPreference='Stop' + $env:DEFAULT_BRANCH='master' $env:VERSION='v18' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" +$env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH" + +Set-Location './docker/windows/' Get-Date Write-Output "Building $env:tag" -docker build --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . +docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f Dockerfile -t $env:tag . if (!$?) { Write-Host "Failed to build docker image $env:tag" Exit 1 } +Get-Date +Write-Output "Building $env:rust_tag" +docker build --isolation=hyperv -m 12g --build-arg DEFAULT_BRANCH=$env:DEFAULT_BRANCH -f rust.Dockerfile -t $env:rust_tag . +if (!$?) { + Write-Host "Failed to build docker image $env:rust_tag" + Exit 1 +} + # Get-Date # Write-Output "Pushing $env:tag" # docker push $env:tag @@ -18,5 +31,14 @@ if (!$?) { # Exit 1 # } +# Get-Date +# Write-Output "Pushing $env:rust_tag" +# docker push $env:rust_tag +# if (!$?) { +# Write-Host "Failed to push docker image $env:rust_tag" +# Exit 1 +# } + + Get-Date Write-Output "Build Finished" \ No newline at end of file diff --git a/docker/windows/install_gst.ps1 b/docker/windows/install_gst.ps1 new file mode 100644 index 0000000000..f0002590f0 --- /dev/null +++ b/docker/windows/install_gst.ps1 @@ -0,0 +1,76 @@ +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + +# FIXME: Python fails to validate github.com SSL certificate, unless we first +# run a dummy download to force refreshing Windows' CA database. +# See: https://bugs.python.org/issue36137 +(New-Object System.Net.WebClient).DownloadString("https://github.com") >$null + +# Download gst-build and all its subprojects +# git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +# FIXME: need 1.19+ for cairo subproject :/ +# Should use a stable branch instead +git clone -b master --depth 1 https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build +if (!$?) { + Write-Host "Failed to clone gst-build" + Exit 1 +} + +Set-Location C:\gst-build + +# Copy the cache we already have in the image to avoid massive redownloads +Move-Item C:/subprojects/* C:\gst-build\subprojects + +if (!$?) { + Write-Host "Failed to copy subprojects cache" + Exit 1 +} + +# Update the subprojects cache +Write-Output "Running meson subproject reset" +meson subprojects update --reset + +if (!$?) { + Write-Host "Failed to reset subprojects state" + Exit 1 +} + +Write-Output "Running git update" +python git-update --no-interaction + +if (!$?) { + Write-Host "Failed to run git-update" + Exit 1 +} + +$env:MESON_ARGS = "-Dglib:installed_tests=false " + + "-Dlibnice:tests=disabled " + + "-Dlibnice:examples=disabled " + + "-Dffmpeg:tests=disabled " + + "-Dopenh264:tests=disabled " + + "-Dpygobject:tests=false " + + "-Dugly=enabled " + + "-Dbad=enabled " + + "-Dges=enabled " + + "-Drtsp_server=enabled " + + "-Ddevtools=enabled " + + "-Dsharp=disabled " + + "-Dpython=disabled " + + "-Dlibav=disabled " + + "-Dvaapi=disabled " + + "-Dgst-plugins-base:pango=enabled " + + "-Dgst-plugins-good:cairo=enabled " + +Write-Output "Building gst" +cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && meson _build $env:MESON_ARGS && meson compile -C _build && ninja -C _build install" + +if (!$?) { + Write-Host "Failed to build and install gst" + Exit 1 +} + +git clean -fdxx + +if (!$?) { + Write-Host "Failed to git clean" + Exit 1 +} \ No newline at end of file diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile new file mode 100644 index 0000000000..7ccfaffee9 --- /dev/null +++ b/docker/windows/rust.Dockerfile @@ -0,0 +1,18 @@ +# escape=` + +FROM 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v16-master' + +ARG DEFAULT_BRANCH="master" +ARG RUST_VERSION="1.52.1" + +COPY install_gst.ps1 C:\ +RUN C:\install_gst.ps1 +RUN choco install -y pkgconfiglite +ENV PKG_CONFIG_PATH="C:/lib/pkgconfig" + +ADD https://win.rustup.rs/x86_64 C:\rustup-init.exe +RUN C:\rustup-init.exe -y --profile minimal --default-toolchain $env:RUST_VERSION + +# Uncomment for easy testing +# RUN git clone --depth 1 https://gitlab.freedesktop.org/gstreamer/gstreamer-rs.git +# RUN cd gstreamer-rs; cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64; cargo build --all; cargo test --all" \ No newline at end of file diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index f2968b7f0a..308af0ed0c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -27,7 +27,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-06-30.0" + WINDOWS_TAG: "2021-07-12.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' From 83e9552bf0ed9ae6ae02f6aaf98a15d8c758d848 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 8 Sep 2021 15:14:57 +0300 Subject: [PATCH 598/605] Revert "Revert "docker/windows: build rust image in the ci"" This reverts commit 3f17e16c72409be5d9c257aefc1abd09c4e26b8b. Part-of: --- docker/windows/build_image.ps1 | 19 +------------ docker/windows/container.ps1 | 4 +-- docker/windows/rust.Dockerfile | 5 +++- gitlab/ci_template.yml | 50 ++++++++++++++++++++++++++++++++-- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/docker/windows/build_image.ps1 b/docker/windows/build_image.ps1 index 37ffe02ce2..a4be56c1fa 100644 --- a/docker/windows/build_image.ps1 +++ b/docker/windows/build_image.ps1 @@ -1,7 +1,7 @@ $env:ErrorActionPreference='Stop' $env:DEFAULT_BRANCH='master' -$env:VERSION='v18' +$env:VERSION='test' $env:tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:$env:VERSION-$env:DEFAULT_BRANCH" $env:rust_tag ="registry.freedesktop.org/gstreamer/gst-ci/amd64/windows-rust:$env:VERSION-$env:DEFAULT_BRANCH" @@ -23,22 +23,5 @@ if (!$?) { Exit 1 } -# Get-Date -# Write-Output "Pushing $env:tag" -# docker push $env:tag -# if (!$?) { -# Write-Host "Failed to push docker image $env:tag" -# Exit 1 -# } - -# Get-Date -# Write-Output "Pushing $env:rust_tag" -# docker push $env:rust_tag -# if (!$?) { -# Write-Host "Failed to push docker image $env:rust_tag" -# Exit 1 -# } - - Get-Date Write-Output "Build Finished" \ No newline at end of file diff --git a/docker/windows/container.ps1 b/docker/windows/container.ps1 index fac0087f99..e5dc2cd835 100644 --- a/docker/windows/container.ps1 +++ b/docker/windows/container.ps1 @@ -13,8 +13,6 @@ $registry_user_image = $args[3] $registry_central_image = $args[4] $dockerfile = $args[5] -Set-Location -Path ".\docker\windows" - docker --config "windows-docker.conf" login -u "$registry_username" -p "$registry_password" "$registry_uri" if (!$?) { Write-Host "docker login failed to $registry_uri" @@ -45,7 +43,7 @@ if ($?) { } Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding" -docker --config "windows-docker.conf" build --no-cache -t "$registry_user_image" -f "$dockerfile" . +docker --config "windows-docker.conf" build $DOCKER_BUILD_ARGS --no-cache -t "$registry_user_image" -f "$dockerfile" "./docker/windows" if (!$?) { Write-Host "Container build failed" docker --config "windows-docker.conf" logout "$registry_uri" diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile index 7ccfaffee9..6d52353788 100644 --- a/docker/windows/rust.Dockerfile +++ b/docker/windows/rust.Dockerfile @@ -1,6 +1,9 @@ # escape=` -FROM 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v16-master' +# Expect this to be set when calling docker build with +# --build-arg BASE_IMAGE="" and make it fail if not set. +ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid" +FROM $BASE_IMAGE ARG DEFAULT_BRANCH="master" ARG RUST_VERSION="1.52.1" diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 308af0ed0c..8435c0ed88 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -3,6 +3,7 @@ include: stages: - 'build docker' + - 'build docker stage 2' - 'preparation' - 'pre-build' - 'build' @@ -38,6 +39,7 @@ variables: TEST_MANIFEST_AMD64_SUFFIX: 'amd64/test-manifest' INDENT_AMD64_SUFFIX: 'amd64/gst-indent' WINDOWS_AMD64_SUFFIX: 'amd64/windows' + WINDOWS_RUST_AMD64_SUFFIX: 'amd64/windows-rust' FEDORA_IMAGE: "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH" FEDORA_DOCS_IMAGE: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-07-03.0-master" @@ -48,6 +50,8 @@ variables: INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + WINDOWS_RUST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + WINDOWS_RUST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Current windows image doesn't seem to be able to spawn procceses for reason, # use a different image until this is debuged @@ -57,7 +61,9 @@ variables: # Can't use $CI_* variables since we use this template in all repos, not just gst-ci CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" - WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" + # FIXME: fix the file path url to point to the upstram once merged + # WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" + WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/alatiera/gst-ci/raw/alatiera/windows-rs-wip/docker/windows/container.ps1" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both @@ -349,12 +355,15 @@ windows amd64 docker: # # We also don't need a CONTEXT_DIR var as its also # hardcoded to be docker/windows/ - DOCKERFILE: "Dockerfile" + DOCKERFILE: "docker/windows/Dockerfile" tags: - windows - shell - "1809" script: + # We need to pass an array and to resolve the env vars, so we can't use a variable: + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH") + - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH @@ -366,6 +375,43 @@ windows amd64 docker: Exit 1 } +windows rust amd64 docker: + stage: 'build docker stage 2' + needs: + - job: 'windows amd64 docker' + artifacts: false + variables: + GIT_STRATEGY: 'fetch' + REPO_SUFFIX: "$WINDOWS_RUST_AMD64_SUFFIX" + TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + + # Unlike the buildah/linux jobs, this file + # needs to be relative to docker/windows/ subdir + # as it makes life easier in the powershell script + # + # We also don't need a CONTEXT_DIR var as its also + # hardcoded to be docker/windows/ + DOCKERFILE: 'docker/windows/rust.Dockerfile' + tags: + - 'windows' + - 'shell' + - '1809' + script: + # We need to pass an array and to resolve the env vars, so we can't use a variable: + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE") + + - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" + - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" + - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH + + - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_RUST_IMAGE $WINDOWS_RUST_UPSTREAM_IMAGE $DOCKERFILE" + - | + if (!($?)) { + echo "Failed to build the image" + Exit 1 + } + + # # Job to create the manifest.xml to used for our builds # From 29596184bedf9f01b8068c96a7d7835a9e968c38 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 8 Sep 2021 15:14:58 +0300 Subject: [PATCH 599/605] Revert "Revert "docker/windows: build two version of the rust image"" This reverts commit 3cfedc8a9a6272a4a402a5b14cd095c10a0d75c6. Part-of: --- docker/windows/rust.Dockerfile | 2 +- gitlab/ci_template.yml | 36 ++++++++++++++++++++++++---------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/docker/windows/rust.Dockerfile b/docker/windows/rust.Dockerfile index 6d52353788..5e7425fef9 100644 --- a/docker/windows/rust.Dockerfile +++ b/docker/windows/rust.Dockerfile @@ -6,7 +6,7 @@ ARG BASE_IMAGE="inavlid.gstreamer.freedesktop.org/invalid" FROM $BASE_IMAGE ARG DEFAULT_BRANCH="master" -ARG RUST_VERSION="1.52.1" +ARG RUST_VERSION="invalid" COPY install_gst.ps1 C:\ RUN C:\install_gst.ps1 diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8435c0ed88..aaee920f8c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -28,7 +28,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-07-12.0" + WINDOWS_TAG: "2021-08-24.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' @@ -50,8 +50,15 @@ variables: INDENT_IMAGE: "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_RUST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" - WINDOWS_RUST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" + + RUST_MINIMUM_VERSION: '1.54.0' + RUST_LATEST_VERSION: '1.54.0' + + WINDOWS_RUST_MINIMUM_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" + WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" + + WINDOWS_RUST_LATEST_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_LATEST_VERSION" + WINDOWS_RUST_LATEST_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_LATEST_VERSION" # Current windows image doesn't seem to be able to spawn procceses for reason, # use a different image until this is debuged @@ -346,8 +353,6 @@ windows amd64 docker: stage: "build docker" variables: GIT_STRATEGY: fetch - REPO_SUFFIX: "$WINDOWS_AMD64_SUFFIX" - TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Unlike the buildah/linux jobs, this file # needs to be relative to docker/windows/ subdir @@ -375,15 +380,13 @@ windows amd64 docker: Exit 1 } -windows rust amd64 docker: +.windows rust docker build: stage: 'build docker stage 2' needs: - job: 'windows amd64 docker' artifacts: false variables: GIT_STRATEGY: 'fetch' - REPO_SUFFIX: "$WINDOWS_RUST_AMD64_SUFFIX" - TAG: "$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" # Unlike the buildah/linux jobs, this file # needs to be relative to docker/windows/ subdir @@ -398,19 +401,32 @@ windows rust amd64 docker: - '1809' script: # We need to pass an array and to resolve the env vars, so we can't use a variable: - - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE") + - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", "--build-arg", "BASE_IMAGE=$WINDOWS_IMAGE", "--build-arg", "RUST_VERSION=$RUST_VERSION") - $env:WINDOWS_CONTAINER_SCRIPT_PATH = "$env:CI_PROJECT_DIR\container.ps1" - echo "Fetching $env:WINDOWS_CONTAINER_SCRIPT_URL" - Invoke-WebRequest -Uri $env:WINDOWS_CONTAINER_SCRIPT_URL -OutFile $env:WINDOWS_CONTAINER_SCRIPT_PATH - - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_RUST_IMAGE $WINDOWS_RUST_UPSTREAM_IMAGE $DOCKERFILE" + - "& $env:WINDOWS_CONTAINER_SCRIPT_PATH $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $RUST_IMAGE $RUST_UPSTREAM_IMAGE $DOCKERFILE" - | if (!($?)) { echo "Failed to build the image" Exit 1 } +windows rust amd64 docker latest stable: + extends: '.windows rust docker build' + variables: + RUST_IMAGE: !reference [variables, "WINDOWS_RUST_LATEST_IMAGE"] + RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_LATEST_UPSTREAM_IMAGE"] + RUST_VERSION: !reference [variables, "RUST_LATEST_VERSION"] + +windows rust amd64 docker minimum supported version: + extends: '.windows rust docker build' + variables: + RUST_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_IMAGE"] + RUST_UPSTREAM_IMAGE: !reference [variables, "WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE"] + RUST_VERSION: !reference [variables, "RUST_MINIMUM_VERSION"] # # Job to create the manifest.xml to used for our builds From 72dcd55325513365a953cba9dbed91ec593606f8 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Wed, 8 Sep 2021 15:19:35 +0300 Subject: [PATCH 600/605] docker/windows: update the subprojects cache Avoid conflicting with gst-build subprojects when copying over from the cache. Part-of: --- docker/windows/prepare_gst_env.ps1 | 1 + gitlab/ci_template.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1 index 601e799a64..ebde9f6271 100644 --- a/docker/windows/prepare_gst_env.ps1 +++ b/docker/windows/prepare_gst_env.ps1 @@ -23,6 +23,7 @@ if (!$?) { Remove-Item -Force 'C:/gst-build/subprojects/*.wrap' Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-nasm' Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-flex-bison-binaries' +Remove-Item -Recurse -Force 'C:/gst-build/subprojects/macos-bison-binary' Move-Item C:\gst-build\subprojects C:\subprojects Remove-Item -Recurse -Force C:\gst-build diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index aaee920f8c..ffef5f6eda 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -28,7 +28,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-08-24.0" + WINDOWS_TAG: "2021-09-08.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' From 0d193db61895cffa4d2c10000fc291332f453429 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 26 Aug 2021 20:00:50 +0300 Subject: [PATCH 601/605] windows: use a single stage for building image With gitlab 14.2 now a job can depend on another from the same stage. Part-of: --- gitlab/ci_template.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ffef5f6eda..4e5c808f26 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -3,7 +3,6 @@ include: stages: - 'build docker' - - 'build docker stage 2' - 'preparation' - 'pre-build' - 'build' @@ -381,7 +380,7 @@ windows amd64 docker: } .windows rust docker build: - stage: 'build docker stage 2' + stage: 'build docker' needs: - job: 'windows amd64 docker' artifacts: false From d94c24b39004ee5b25d910b967a46681d7365d0f Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 9 Sep 2021 04:02:57 +0300 Subject: [PATCH 602/605] docker/windows: remove workaround to fetch github's cert looks like it might not be needed anymore Part-of: --- docker/windows/install_gst.ps1 | 5 ----- docker/windows/prepare_gst_env.ps1 | 5 ----- gitlab/ci_template.yml | 2 +- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/docker/windows/install_gst.ps1 b/docker/windows/install_gst.ps1 index f0002590f0..6105f3623b 100644 --- a/docker/windows/install_gst.ps1 +++ b/docker/windows/install_gst.ps1 @@ -1,10 +1,5 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; -# FIXME: Python fails to validate github.com SSL certificate, unless we first -# run a dummy download to force refreshing Windows' CA database. -# See: https://bugs.python.org/issue36137 -(New-Object System.Net.WebClient).DownloadString("https://github.com") >$null - # Download gst-build and all its subprojects # git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build # FIXME: need 1.19+ for cairo subproject :/ diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1 index ebde9f6271..652fe2b4f7 100644 --- a/docker/windows/prepare_gst_env.ps1 +++ b/docker/windows/prepare_gst_env.ps1 @@ -1,10 +1,5 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; -# FIXME: Python fails to validate github.com SSL certificate, unless we first -# run a dummy download to force refreshing Windows' CA database. -# See: https://bugs.python.org/issue36137 -(New-Object System.Net.WebClient).DownloadString("https://github.com") >$null - # Download gst-build and all its subprojects git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build if (!$?) { diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 4e5c808f26..ac0e28717d 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -27,7 +27,7 @@ variables: MANIFEST_TAG: '2020-10-22.0' TEST_MANIFEST_TAG: '2020-10-22.0' INDENT_TAG: '2020-10-22.0' - WINDOWS_TAG: "2021-09-08.0" + WINDOWS_TAG: "2021-09-09.0" GST_UPSTREAM_REPO: 'gstreamer/gst-ci' From ab610be59d1a8d9f364da1cd72bdbf5a15bcf289 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Thu, 9 Sep 2021 13:26:01 +0300 Subject: [PATCH 603/605] citemplate: post-merge use upstream urls for fetching scripts since gstreamer/gst-ci!414 landed Part-of: --- gitlab/ci_template.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index ac0e28717d..8f06c1a761 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -67,9 +67,7 @@ variables: # Can't use $CI_* variables since we use this template in all repos, not just gst-ci CERBERO_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/cerbero_setup.sh" - # FIXME: fix the file path url to point to the upstram once merged - # WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" - WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/alatiera/gst-ci/raw/alatiera/windows-rs-wip/docker/windows/container.ps1" + WINDOWS_CONTAINER_SCRIPT_URL: "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/docker/windows/container.ps1" GIT_STRATEGY: none MESON_BUILDTYPE_ARGS: --default-library=both From 2a93d4f347ad190b9b515e3231e0ceb32fa39591 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 10 Sep 2021 18:46:29 +0300 Subject: [PATCH 604/605] citemplate: Bump tag of latest rustc version Part-of: --- gitlab/ci_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 8f06c1a761..37bdfeac1c 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -51,7 +51,7 @@ variables: WINDOWS_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH" RUST_MINIMUM_VERSION: '1.54.0' - RUST_LATEST_VERSION: '1.54.0' + RUST_LATEST_VERSION: '1.55.0' WINDOWS_RUST_MINIMUM_IMAGE: "$CI_REGISTRY_IMAGE/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" WINDOWS_RUST_MINIMUM_UPSTREAM_IMAGE: "$CI_REGISTRY/$GST_UPSTREAM_REPO/$WINDOWS_RUST_AMD64_SUFFIX:$WINDOWS_TAG-$GST_UPSTREAM_BRANCH-rust-$RUST_MINIMUM_VERSION" From d68d52e1373c07a72bc2e15aee82b5483b9e7449 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Fri, 10 Sep 2021 18:48:08 +0300 Subject: [PATCH 605/605] citemplate: only build rust docker images in gst-ci repos They are only used by the gst-rs repos which aren't using this template anyway, so we don't need to have the jobs for the rest of the pipelines. Part-of: --- gitlab/ci_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index 37bdfeac1c..cb8ae521d3 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -382,6 +382,8 @@ windows amd64 docker: needs: - job: 'windows amd64 docker' artifacts: false + rules: + - if: '$CI_PROJECT_NAME == "gst-ci"' variables: GIT_STRATEGY: 'fetch'