From e7fc0400bc620e749111a34115080e7323561bb3 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 29 Aug 2023 16:10:49 +1000 Subject: [PATCH] ci: add macos (arm64) CI Part-of: --- .gitlab-ci.yml | 86 ++++++++++++++++++++------ ci/docker/fedora/prepare.sh | 2 +- ci/docker/windows/prepare_gst_env.ps1 | 2 +- ci/scripts/handle-subprojects-cache.py | 14 ++--- 4 files changed, 75 insertions(+), 29 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6d9320c2a..3ee4d79710 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -188,6 +188,10 @@ windows amd64 docker: ) - ci/docker/windows/container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_IMAGE $WINDOWS_UPSTREAM_IMAGE $DOCKERFILE +.macos image: + image: "registry.freedesktop.org/gstreamer/gstreamer/macos-arm64/13-ventura:2023-08-29.1" + tags: + - gst-mac-arm # ---- Preparation ----- # # @@ -205,9 +209,7 @@ gst indent: - ./scripts/check-format-c - ./scripts/format-csharp --check rules: - # Don't check indentation on post merge pipelines - - if: '$CI_PROJECT_NAMESPACE == "gstreamer" && $CI_COMMIT_BRANCH == $GST_UPSTREAM_BRANCH' - when: 'manual' + - !reference [.upstream-branch-rules, rules] - if: '$CI_PROJECT_NAMESPACE != "gstreamer" || $CI_COMMIT_BRANCH != $GST_UPSTREAM_BRANCH' when: 'always' @@ -254,29 +256,33 @@ commitlint: # corresponds to -Dgstreamer:gst_debug meson arg # GST_WERROR: make warning fatal or not # must be a string of a boolean, "true" or "false". Not yaml bool. -# +# SUBPROJECTS_CACHE_DIR: The location in the image of the subprojects cache +# HAVE_CCACHE: whether we have ccache available +# must be a string of a boolean, "true" or "false". Not yaml bool. .build_template: &build - export RUSTUP_HOME="/usr/local/rustup" - export CARGO_HOME="/usr/local/cargo" - export PATH="/usr/local/cargo/bin:$PATH" - date -R - - ci/scripts/handle-subprojects-cache.py subprojects/ + - ci/scripts/handle-subprojects-cache.py --cache-dir ${SUBPROJECTS_CACHE_DIR} subprojects/ - date -R - export ARGS="--default-library=${BUILD_TYPE:-both} -Dgstreamer:gst_debug=${BUILD_GST_DEBUG:-true} ${MESON_ARGS}" + - echo $GST_WERROR - |- - if [ $GST_WERROR == "true" ]; then + if [ "$GST_WERROR" = "true" ]; then export ARGS="$ARGS --native-file ./ci/meson/gst-werror.ini" fi - echo $ARGS - date -R - - meson build/ $ARGS + - meson setup build/ $(echo ${ARGS}) - date -R - ninja -C build/ - date -R - - ccache --show-stats + - test "x$HAVE_CCACHE" = "xtrue" && ccache --show-stats .build_ccache_vars: variables: + HAVE_CCACHE: 'true' CCACHE_COMPILERCHECK: 'content' CCACHE_COMPRESS: 'true' CCACHE_BASEDIR: '/cache/gstreamer/gstreamer' @@ -305,10 +311,15 @@ commitlint: - subprojects/gstreamer-sharp/**/* - subprojects/gst-rtsp-server/**/* +.upstream-branch-rules: + rules: + # If this matches, it means the pipeline is running against either the main + # or a stable branch, so make it manual + - if: '$CI_PROJECT_NAMESPACE == "gstreamer" && $CI_COMMIT_BRANCH == $GST_UPSTREAM_BRANCH' + when: manual + .build: stage: 'build' - extends: - - '.build_ccache_vars' needs: - "trigger" # Taking into account the slowest shared runner + time needed to upload the binaries to artifacts @@ -318,6 +329,7 @@ commitlint: BUILD_GST_DEBUG: "true" GST_WERROR: "true" MESON_ARGS: "${DEFAULT_MESON_ARGS}" + SUBPROJECTS_CACHE_DIR: "/subprojects" script: - *build - ./gst-env.py gst-inspect-1.0 --version @@ -330,10 +342,7 @@ commitlint: paths: - 'meson-logs/' rules: - # If this matches, it means the pipeline is running against either the main - # or a stable branch, so make it manual - - if: '$CI_PROJECT_NAMESPACE == "gstreamer" && $CI_COMMIT_BRANCH == $GST_UPSTREAM_BRANCH' - when: manual + - !reference [.upstream-branch-rules, rules] - changes: *modules_changes - changes: @@ -344,12 +353,14 @@ commitlint: - '.fedora image' - '.fdo.suffixed-image@fedora' - '.build' + - '.build_ccache_vars' needs: - "trigger" - "fedora amd64 docker" variables: GST_WERROR: "true" MESON_ARGS: "${SIMPLE_BUILD}" + SUBPROJECTS_CACHE_DIR: "/subprojects" script: - *build @@ -444,10 +455,7 @@ build fedora clang: -Dgst-plugins-good:soup-lookup-dep=true -Dgst-plugins-bad:d3d11-wgc=enabled rules: - # If this matches, it means the pipeline is running against either the main - # or a stable branch, so make it manual - - if: '$CI_PROJECT_NAMESPACE == "gstreamer" && $CI_COMMIT_BRANCH == $GST_UPSTREAM_BRANCH' - when: manual + - !reference [.upstream-branch-rules, rules] - changes: *modules_changes - changes: @@ -462,7 +470,7 @@ build fedora clang: # Renormalize line endings - git add --renormalize subprojects/packagefiles - $env:PYTHONUNBUFFERED = '1' - - ci/scripts/handle-subprojects-cache.py subprojects/ + - ci/scripts/handle-subprojects-cache.py --cache-dir /subprojects subprojects/ - $env:PYTHONUNBUFFERED = '' - echo $env:MESON_ARGS # We do not pick up any deps with cmake, so speed up dependency checks @@ -543,6 +551,40 @@ build msys2: # and inherit PATH - C:\msys64\msys2_shell.cmd -ucrt64 -defterm -no-start -here -use-full-path -lc "meson setup build $env:MESON_ARGS && meson compile -C build" +build macos: + extends: + - '.build' + - '.macos image' + variables: + BUILD_GST_DEBUG: "true" + # gst-libav/ffmpeg throws Wundef errors + GST_WERROR: "false" + MESON_ARGS: "${DEFAULT_MESON_ARGS}" + SUBPROJECTS_CACHE_DIR: "/Users/gst-ci/subprojects" + rules: + - !reference [.upstream-branch-rules, rules] + - changes: + *modules_changes + - changes: + - subprojects/macos-*/* + parallel: + matrix: + - BUILD_TYPE: ['both'] + BUILD_GST_DEBUG: ["true"] + # FIXME: We currently don't have pkg-config required by gobject-introspection/pygobject/sharp + # macos doesn't support vaapi + MESON_ARGS: >- + ${SIMPLE_BUILD} + -Dintrospection=disabled + -Dvaapi=disabled + -Dsharp=disabled + -Dpython=disabled + -Ddoc=disabled + -Dgst-plugins-base:pango=enabled + -Dgst-plugins-good:cairo=enabled + -Dgst-plugins-good:soup=enabled + -Dgst-plugins-good:soup-lookup-dep=true + # ---- Tests ----- # .test: @@ -554,6 +596,7 @@ build msys2: variables: GST_WERROR: "true" MESON_ARGS: "${SIMPLE_BUILD}" + SUBPROJECTS_CACHE_DIR: "/subprojects" # Disable colored output to avoid weird rendering issues GST_DEBUG_NO_COLOR: "true" @@ -680,6 +723,7 @@ gstreamer-full static build: BUILD_TYPE: "static" MESON_ARGS: >- -Ddoc=disabled + SUBPROJECTS_CACHE_DIR: "/subprojects" script: - *build - meson test -C build -v test-gst-full @@ -703,6 +747,7 @@ gstreamer-full-minimal static build: -Dgst-full-typefind-functions=typefindfunctions:wav,flv -Dgst-full-device-providers=alsa:alsadeviceprovider -Dgst-full-dynamic-types=pbtypes:video_multiview_flagset + SUBPROJECTS_CACHE_DIR: "/subprojects" script: - *build @@ -827,12 +872,13 @@ valgrind ges: -Dgst-docs:fatal_warnings=true # Disable werror for the docs build, we don't need it GST_WERROR: 'false' + SUBPROJECTS_CACHE_DIR: "/subprojects" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/" script: - export PATH=/usr/local/cargo/bin/:/usr/local/bin/:$PATH - export RUSTUP_HOME='/usr/local/rustup' - - ci/scripts/handle-subprojects-cache.py subprojects/ + - ci/scripts/handle-subprojects-cache.py --cache-dir /subprojects subprojects/ - echo $MESON_ARGS - meson build/ $MESON_ARGS - ccache --show-stats diff --git a/ci/docker/fedora/prepare.sh b/ci/docker/fedora/prepare.sh index a6111ef980..006f79b75c 100644 --- a/ci/docker/fedora/prepare.sh +++ b/ci/docker/fedora/prepare.sh @@ -247,7 +247,7 @@ rustc --version git clone -b ${GIT_BRANCH} ${GIT_URL} /gstreamer git -C /gstreamer submodule update --init --depth=1 meson subprojects download --sourcedir /gstreamer -/gstreamer/ci/scripts/handle-subprojects-cache.py --build /gstreamer/subprojects/ +/gstreamer/ci/scripts/handle-subprojects-cache.py --build --cache-dir /subprojects /gstreamer/subprojects/ # Run git gc to prune unwanted refs and reduce the size of the image for i in $(find /subprojects/ -mindepth 1 -maxdepth 1 -type d); diff --git a/ci/docker/windows/prepare_gst_env.ps1 b/ci/docker/windows/prepare_gst_env.ps1 index 51f6926554..93436d14b3 100644 --- a/ci/docker/windows/prepare_gst_env.ps1 +++ b/ci/docker/windows/prepare_gst_env.ps1 @@ -27,5 +27,5 @@ Write-Host "Downloading subprojects" meson subprojects download --sourcedir C:\gstreamer Write-Host "Caching subprojects into /subprojects/" -python C:/gstreamer/ci/scripts/handle-subprojects-cache.py --build C:/gstreamer/subprojects +python C:/gstreamer/ci/scripts/handle-subprojects-cache.py --build --cache-dir /subprojects C:/gstreamer/subprojects Remove-Item -Recurse -Force C:\gstreamer diff --git a/ci/scripts/handle-subprojects-cache.py b/ci/scripts/handle-subprojects-cache.py index 37fd222228..be6c83841f 100755 --- a/ci/scripts/handle-subprojects-cache.py +++ b/ci/scripts/handle-subprojects-cache.py @@ -10,14 +10,14 @@ import sys import argparse import subprocess -DEST = "/subprojects" PARSER = argparse.ArgumentParser() PARSER.add_argument('subprojects_dir') PARSER.add_argument('--build', action="store_true", default=False) +PARSER.add_argument('--cache-dir', default="/subprojects") def create_cache_in_image(options): - os.makedirs(DEST, exist_ok=True) + os.makedirs(options.cache_dir, exist_ok=True) print("Creating cache from %s" % options.subprojects_dir) for project_name in os.listdir(options.subprojects_dir): project_path = os.path.join(options.subprojects_dir, project_name) @@ -25,26 +25,26 @@ def create_cache_in_image(options): if project_name != "packagecache" and not os.path.exists(os.path.join(project_path, '.git')): continue - if os.path.exists(os.path.join(DEST, project_name)): + if os.path.exists(os.path.join(options.cache_dir, project_name)): continue print("Copying %s" % project_name) - shutil.copytree(project_path, os.path.join(DEST, project_name)) + shutil.copytree(project_path, os.path.join(options.cache_dir, project_name)) media_path = os.path.join(options.subprojects_dir, '..', '.git', 'modules', 'subprojects', 'gst-integration-testsuites', 'medias') - if os.path.exists(os.path.join(DEST, 'medias.git')): + if os.path.exists(os.path.join(options.cache_dir, 'medias.git')): return if os.path.exists(media_path): print("Creating media cache") - shutil.copytree(media_path, os.path.join(DEST, 'medias.git')) + shutil.copytree(media_path, os.path.join(options.cache_dir, 'medias.git')) else: print("Did not find medias in %s" % media_path) def copy_cache(options): - for path in [DEST, r"C:\gst-build\subprojects"]: + for path in [options.cache_dir, r"C:\gst-build\subprojects"]: if not os.path.exists(path): print("%s doesn't exist." % path) continue