gstreamer/gitlab/ci_template.yml

stages:
  - 'preparation'
  # Test just one basic build, if it succeeds proceed to test the rest
  - 'build'
  - 'test'
  # Run multiple builds and tests, multi-distro, multi-arch
  - 'full builds'
  - 'full tests'
  # build some apps to check that cross-platform binaries are usable
  - 'apps'
  - 'deploy'

variables:
  ANDROID_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/android:2019-03-26-196225'
  CERBERO_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/cerbero-fedora:7ed820aa1fcf95938cc161e397b1bf455f1f5b74'
  FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-03-26-196223'
  FEDORA30_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578'
  INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb'
  MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e'

  GIT_STRATEGY: none
  DEFAULT_MESON_ARGS: >
    --werror
    -Dpython=enabled
    -Dlibav=enabled
    -Dugly=enabled
    -Dbad=enabled
    -Ddevtools=enabled
    -Dges=enabled
    -Drtsp_server=enabled
    -Dvaapi=enabled
    -Dsharp=disabled

  DEFAULT_CERBERO_ARGS: >
    --variants werror
    --timestamps

manifest:
  image: $MANIFEST_IMAGE
  stage: 'preparation'
  script:
    - cd /gst-ci
    - gitlab/build_manifest.py --self-update
    - gitlab/build_manifest.py ${CI_PROJECT_DIR}/manifest.xml
    - cat ${CI_PROJECT_DIR}/manifest.xml
  artifacts:
    expire_in: "7 days"
    paths:
      - "manifest.xml"

gst indent:
  image: $INDENT_IMAGE
  stage: 'preparation'
  variables:
    GIT_STRATEGY: 'fetch'
  script:
    # man indent. grep RETURN VALUE, grab a beer on my behalf...
    - indent --version || true
    - curl -o gst-indent https://gitlab.freedesktop.org/gstreamer/gstreamer/raw/master/tools/gst-indent
    - chmod +x gst-indent
    - find . -name '*.c' -exec ./gst-indent {} +
    - |
      if git diff --quiet; then
          echo "Code is properly formatted"
      else
          git diff --color=always
          echo 'style diverges, please run gst-indent first'
          exit 1
      fi
  except:
    variables:
      # No point on trying to format C files in those repositories
      - $CI_PROJECT_NAME == "gstreamer-sharp"
      - $CI_PROJECT_NAME == "gst-integration-testsuites"
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-docs"

.build:
  stage: 'full builds'
  dependencies:
    - "manifest"
  variables:
    CC: "ccache gcc"
    CXX: "ccache g++"
    CCACHE_BASEDIR: "${CI_PROJECT_DIR}"
    CCACHE_DIR: "${CI_PROJECT_DIR}/ccache"
    MESON_ARGS: "${DEFAULT_MESON_ARGS}"

  script:
    - ccache -z
    # Not sure why, but permission errors else
    # https://gitlab.freedesktop.org/alatiera/gstreamer/-/jobs/41441
    - cp -r /gst-build/ . && cd gst-build
    - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml"
    - meson build/ $MESON_ARGS
    - ninja -C build/
    - ccache -s
  after_script:
    - cd gst-build/
    # Clean the artifacts packages to avoid copying "useless" build products.
    - test -d build && find build -name '*.[ao]' -delete
    # Clean the .git repos since we won't need them anymore
    - rm -rf subprojects/*/.git/
    - rm -rf build/subprojects/*/.git/
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - "${CCACHE_DIR}"
  artifacts:
    expire_in: '5 days'
    when: always
    paths:
      - "manifest.xml"
      - "gst-build/"
  except:
    variables:
      - $CI_PROJECT_NAME == "cerbero"

build fedora x86_64:
  extends: '.build'
  stage: 'build'
  image: $FEDORA_IMAGE
  variables:
    MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Domx=enabled -Dgst-omx:target=generic"
  except:
    variables:
      - $CI_PROJECT_NAME == "gst-docs"
      - $CI_PROJECT_NAME == "cerbero"

build nodebug fedora x86_64:
  extends: '.build'
  stage: 'build'
  image: $FEDORA30_IMAGE
  variables:
    MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic -Ddoc=enabled"

.test:
  stage: 'test'
  variables:
    # Disable colored output to avoid weird rendering issues
    GST_DEBUG_NO_COLOR: "true"
    CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/"
    GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true"
  script:
    - cd gst-build/
    - echo "-> Running ${TEST_SUITE}"
    - >
      ./gst-uninstalled.py
      gst-validate-launcher ${TEST_SUITE}
      --dump-on-failure
      --mute
      --shuffle
      --no-display
      --meson-no-rebuild
      --fail-on-testlist-change
      -l "${CI_PROJECT_DIR}/validate-logs/"
      --xunit-file "${CI_PROJECT_DIR}/validate-logs/xunit.xml"
      ${EXTRA_VALIDATE_ARGS}
  artifacts:
    expire_in: '14 days'
    when: always
    paths:
      - 'gst-build/build/meson-logs/'
      - 'validate-logs'
    reports:
      junit:
        - "validate-logs/*.xml"
  # We disable the .build above, which this job usually depends upon for cerbero
  except:
    variables:
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-examples"
      - $CI_PROJECT_NAME == "gst-docs"

.test fedora x86_64:
  image: $FEDORA_IMAGE
  extends: '.test'
  dependencies:
    - build fedora x86_64

check fedora:
  extends: '.test fedora x86_64'
  variables:
    TEST_SUITE: "check.gst*"
  except:
    variables:
      - $CI_PROJECT_NAME == "gst-integration-testsuites"
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-examples"
      - $CI_PROJECT_NAME == "gst-omx"
      - $CI_PROJECT_NAME == "gst-docs"

integration testsuites fedora:
  extends: '.test fedora x86_64'
  before_script:
    - rm -f gst-build/build/subprojects/gstreamer-vaapi/gst/vaapi/libgstvaapi.so
  variables:
    EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs"
    TEST_SUITE: "validate ges"
  except:
    variables:
      - $CI_PROJECT_NAME == "gst-omx"
      - $CI_PROJECT_NAME == "gstreamer-vaapi"
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-docs"

# Valgrind
.valgrind fedora x86_64:
  extends: '.test fedora x86_64'
  stage: 'full tests'
  variables:
    EXTRA_VALIDATE_ARGS: "--valgrind"
  # Some suppression files are missing a newline at the end which messes things
  # up when concatenating them. awk will add missing newlines (unlike cat)

valgrind core:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gstreamer\\..*"
  only:
    variables:
      - $CI_PROJECT_NAME =~ /^(gstreamer|gst-ci)$/

valgrind base:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-plugins-base\\..*"
  only:
    variables:
      - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-ci)$/

valgrind good:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-plugins-good\\..*"
  only:
    variables:
      - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci)$/

valgrind ugly:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-plugins-ugly\\..*"
  only:
    variables:
      - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci)$/

valgrind bad:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-plugins-bad\\..*"
  only:
    variables:
      - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci)$/

valgrind ges:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-editing-services\\..*"
  only:
    variables:
      - $CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci)$/

# Template for Cerbero GStreamer Build
#
# Parameters:
# CONFIG: The name of the configuration file to use
#
# Produces runtime and devel tarball packages.
.cerbero:
  stage: "full builds"
  image: $CERBERO_IMAGE
  dependencies:
    - "manifest"
  variables:
    CCACHE_BASEDIR: "${CI_PROJECT_DIR}"
    CCACHE_DIR: "${CI_PROJECT_DIR}/ccache"
    CCACHE_MAXSIZE: "1.7G"
    CERBERO_HOME: "${CI_PROJECT_DIR}/cerbero-build"
    CERBERO_SOURCES: "${CI_PROJECT_DIR}/cerbero-sources"
    CERBERO_DEPS: "cerbero-deps.tar.gz"
    CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS}"
    CERBERO: "./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml"
  before_script:
    # FIXME Wrong namespace
    # Workaround build-tools having hardcoded internal path
    - mkdir -p /builds/gstreamer
    - ln -sf ${CI_PROJECT_DIR} /builds/gstreamer/cerbero
    - rsync -aH /cerbero/ .
    - test -f ${CERBERO_DEPS} && tar -C ${CERBERO_HOME} -xf ${CERBERO_DEPS}
    - echo "home_dir = \"${CERBERO_HOME}\"" >> localconf.cbc
    - echo "local_sources = \"${CERBERO_SOURCES}\"" >> localconf.cbc
    - ./cerbero-uninstalled --self-update manifest.xml
    - ccache -z
  script:
    - $CERBERO $CERBERO_ARGS show-config
    - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
    - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
    - $CERBERO $CERBERO_ARGS fetch-cache
    - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only
    - $CERBERO $CERBERO_ARGS package --offline -t -o ${CI_PROJECT_DIR} gstreamer-1.0
    - ccache -s
  except:
    variables:
      - $CI_PROJECT_NAME == "gst-build"
      - $CI_PROJECT_NAME == "gst-docs"
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - "${CCACHE_DIR}"
      - "${CERBERO_SOURCES}"
  artifacts:
    name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}"
    expire_in: '5 days'
    when: 'always'
    paths:
      - "manifest.xml"
      - "cerbero-build/logs"
      - "*.tar.bz2"

# Template for Cerbero GStreamer Deps
#
# This template is used by cerbero/ project to pre-built the GStreamer
# depedencies. When available, the .cerbero jobs will download this artifact
# in order to speed up the build.
#
# Parameters:
# CONFIG: The name of the configuration file to use
# ARCH: The cerbero <os>_<cpu> (used in cache key)
#
# Produce an artifact with the dist/ and .cache along
# with the associated build-tools.
.cerbero deps:
  extends: .cerbero
  stage: "build"
  script:
    - $CERBERO $CERBERO_ARGS show-config
    - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
    - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
    - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only
    - $CERBERO $CERBERO_ARGS build-deps --offline
          gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0
          gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0
          gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice
    - $CERBERO $CERBERO_ARGS fetch-cache --skip-fetch --job-id=${CI_JOB_ID}
    - tar -C ${CERBERO_HOME} -czf $CERBERO_DEPS
              build-tools build-tools.cache
              dist/${ARCH} ${ARCH}.cache
    - ccache -s
  only:
    variables:
      - $CI_PROJECT_NAME == "cerbero"
  artifacts:
    name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}"
    expire_in: '10 days'
    when: 'always'
    paths:
      - "manifest.xml"
      - "cerbero-build/logs"
      - "cerbero-build/cerbero-deps.log"
      - "${CERBERO_DEPS}"

#
# Cerbero Linux X86_64 build
#
cerbero deps fedora x86_64:
  extends: '.cerbero deps'
  variables:
    CONFIG: "linux.config"
    ARCH: "linux_x86_64"

.cerbero fedora x86_64:
  extends: '.cerbero'
  variables:
    CONFIG: "linux.config"

cerbero fedora x86_64:
  extends: '.cerbero fedora x86_64'
  dependencies:
    - "cerbero deps fedora x86_64"
  only:
    variables:
      - $CI_PROJECT_NAME == "cerbero"

build cerbero fedora x86_64:
  extends: '.cerbero fedora x86_64'
  except:
    variables:
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-build"
      - $CI_PROJECT_NAME == "gst-omx"
      - $CI_PROJECT_NAME == "gstreamer-vaapi"
      - $CI_PROJECT_NAME == "gst-docs"

#
# Cerbero Android Universal build
#
cerbero deps android universal:
  extends: '.cerbero deps'
  variables:
    CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror"
    CONFIG: "cross-android-universal.cbc"
    ARCH: "android_universal"

.cerbero android universal:
  extends: '.cerbero'
  variables:
    CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v nowerror"
    CONFIG: "cross-android-universal.cbc"

cerbero android universal:
  extends: '.cerbero android universal'
  dependencies:
    - "cerbero deps android universal"
  only:
    variables:
      - $CI_PROJECT_NAME == "cerbero"

build cerbero android universal:
  extends: '.cerbero android universal'
  except:
    variables:
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-build"
      - $CI_PROJECT_NAME == "gst-omx"
      - $CI_PROJECT_NAME == "gstreamer-vaapi"
      - $CI_PROJECT_NAME == "gst-docs"

#
# Cerbero Cross Windows builds
#
cerbero deps windows x86:
  extends: '.cerbero deps'
  variables:
    CONFIG: "cross-win32.cbc"
    ARCH: "windows_x86"

.cerbero cross win32:
  extends: '.cerbero'
  variables:
    CONFIG: "cross-win32.cbc"

cerbero cross win32:
  extends: '.cerbero cross win32'
  dependencies:
    - "cerbero deps windows x86"
  only:
    variables:
      - $CI_PROJECT_NAME == "cerbero"

build cerbero cross win32:
  extends: '.cerbero cross win32'
  except:
    variables:
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-build"
      - $CI_PROJECT_NAME == "gst-omx"
      - $CI_PROJECT_NAME == "gstreamer-vaapi"
      - $CI_PROJECT_NAME == "gst-docs"

cerbero deps windows x86_64:
  extends: '.cerbero deps'
  variables:
    CONFIG: "cross-win64.cbc"
    ARCH: "windows_x86_64"

.cerbero cross win64:
  extends: '.cerbero'
  variables:
    CONFIG: "cross-win64.cbc"

cerbero cross win64:
  extends: '.cerbero cross win64'
  dependencies:
    - "cerbero deps windows x86_64"
  only:
    variables:
      - $CI_PROJECT_NAME == "cerbero"

build cerbero cross win64:
  extends: '.cerbero cross win64'
  except:
    variables:
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-build"
      - $CI_PROJECT_NAME == "gst-omx"
      - $CI_PROJECT_NAME == "gstreamer-vaapi"
      - $CI_PROJECT_NAME == "gst-docs"

#
# Build an Android App using the android binaries
#
.android universal examples:
  image: $ANDROID_IMAGE
  stage: 'apps'
  variables:
    EXAMPLES_HOME: ${CI_PROJECT_DIR}/examples
    GSTREAMER_ROOT_ANDROID: ${CI_PROJECT_DIR}/examples/cerbero-android-universal
  script:
    - mkdir -p ${EXAMPLES_HOME}/outputs
    - curl -o clone_manifest_ref.py https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/master/gitlab/clone_manifest_ref.py
    - chmod +x clone_manifest_ref.py
    - ./clone_manifest_ref.py --manifest manifest.xml --project gst-examples --destination ${EXAMPLES_HOME}/gst-examples
    - ./clone_manifest_ref.py --manifest manifest.xml --project gst-docs --destination ${EXAMPLES_HOME}/gst-docs
    - rm clone_manifest_ref.py

    # extract our binaries
    - rm -f gstreamer-1.0-android-universal-*-runtime.tar.bz2
    - mkdir ${GSTREAMER_ROOT_ANDROID}
    - tar -C ${GSTREAMER_ROOT_ANDROID} -xf gstreamer-1.0-android-universal-*.tar.bz2

    # gst-examples
    - chmod +x ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew
    - ${EXAMPLES_HOME}/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-examples/playback/player/android assembleDebug
    - cp ${EXAMPLES_HOME}/gst-examples/playback/player/android/app/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/

    # gst-docs android tutorials
    - chmod +x ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew
    - ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android assembleDebug
    - cp ${EXAMPLES_HOME}/gst-docs/examples/tutorials/android/android-tutorial-*/build/outputs/apk/debug/*.apk ${EXAMPLES_HOME}/outputs/
  after_script:
    - rm -rf ${GSTREAMER_ROOT_ANDROID}
    - rm -rf ${EXAMPLES_HOME}/gst-examples ${EXAMPLES_HOME}/gst-docs
  artifacts:
    name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}"
    expire_in: '5 days'
    when: 'always'
    paths:
      - "manifest.xml"
      - "${EXAMPLES_HOME}/outputs"

android universal examples:
  extends: ".android universal examples"
  dependencies:
    - "build cerbero android universal"
  except:
    variables:
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-build"
      - $CI_PROJECT_NAME == "gst-omx"
      - $CI_PROJECT_NAME == "gst-integration-testsuites"
      - $CI_PROJECT_NAME == "gstreamer-vaapi"
      - $CI_PROJECT_NAME == "gst-docs"

cerbero android universal examples:
  extends: ".android universal examples"
  dependencies:
    - "cerbero android universal"
  only:
    variables:
      - $CI_PROJECT_NAME == "cerbero"

documentation:
  image: $FEDORA30_IMAGE
  dependencies:
    - 'build nodebug fedora x86_64'
  stage: deploy

  script:
  - cd gst-build/
  - ./gst-uninstalled.py hotdoc run --conf-file=build/subprojects/gst-docs/GStreamer-doc.json --fatal-warnings
  - cd -
  - mv gst-build/build/subprojects/gst-docs/GStreamer-doc/html documentation/

  artifacts:
    paths:
    - documentation/
  except:
    variables:
      - $CI_PROJECT_NAME == "cerbero"
      - $CI_PROJECT_NAME == "gst-integration-testsuite"
      - $CI_PROJECT_NAME == "gst-python"
      - $CI_PROJECT_NAME == "gst-sharp"
      - $CI_PROJECT_NAME == "gst-build"