tests: Add virtme fluster tests

Set up a test suite that runs fluster in a virtual machine using virtme.
This test only runs when a kernel image path is set in the new
`virtme_kernel_image` meson option.

The kernel iimage must have support for visl.

The suite contains 4 tests, 1 for each supported codec in visl:
 - vp8
 - vp9
 - h.264
 - hevc

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5434>

.gitlab-ci.yml

@@ -761,6 +761,55 @@ gstreamer-full-minimal static build:
   - strip build/libgstreamer-full-1.0.so
   - ls -l build/libgstreamer-full-1.0.so
 
+# v4l2-stateless-decoders tests
+fluster v4l2-stateless on visl:
+  extends: 'gstreamer-full static build'
+  stage: 'test'
+  variables:
+    BUILD_GST_DEBUG: '-Dgstreamer:gst_debug=false'
+    GST_WERROR: "true"
+    BUILD_TYPE: "--default-library=static"
+    MESON_ARGS: >-
+      -Dvirtme_kernel_image=/opt/linux/bzImage
+      -Dauto_features=disabled
+      -Dgstreamer:check=enabled
+      -Ddoc=disabled
+      -Dtests=enabled
+      -Dtools=enabled
+      -Dbase=enabled
+      -Dbad=enabled
+      -Dugly=disabled
+      -Dlibav=disabled
+      -Drtsp_server=disabled
+      -Dges=disabled
+      -Dgst-plugins-bad:debugutils=enabled
+      -Dgst-plugins-bad:ivfparse=enabled
+      -Dgst-plugins-bad:v4l2codecs=enabled
+      -Dgst-plugins-bad:videoparsers=enabled
+      -Dgst-plugins-base:app=enabled
+      -Dgst-plugins-base:videoconvertscale=enabled
+      -Dgst-plugins-base:typefind=enabled
+      -Dgst-plugins-base:playback=enabled
+      -Dgst-plugins-good:matroska=enabled
+    SUBPROJECTS_CACHE_DIR: "/subprojects"
+  script:
+  - *build
+  - meson test -C build -v --suite v4l2-stateless-decoders
+  artifacts:
+    reports:
+      junit: build/fluster-results-*.xml
+  rules:
+    - changes:
+      - .gitlab-ci.yml
+      - .gitlab-image-tags.yml
+      - subprojects/gst-plugins-bad/sys/v4l2codecs/*
+      - subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/*
+      - subprojects/gst-plugins-bad/gst-libs/gst/codecs/*
+      - subprojects/gst-plugins-bad/gst/videoparsers/*
+      - tests/virtme/*
+      - ci/scripts/build-linux.sh
+      - ci/docker/fedora/*
+
 # Valgrind
 .valgrind fedora x86_64:
   extends: '.test fedora x86_64'

meson_options.txt

@@ -41,6 +41,7 @@ option('gst-full-target-type', type : 'combo', value : 'shared_library', choices
   description : '''The type of library of gstreamer-full-1.0.''')
 option('orc-source', type: 'combo', choices: ['system', 'subproject', 'auto'], value: 'subproject')
 option('build-tools-source', type: 'combo', choices: ['system', 'subproject'], value: 'subproject')
+option('virtme_kernel_image', type: 'string', value: '', description: 'Path to a Linux kernel image to be used for virtme testing')
 
 # License-related feature options
 option('gpl', type: 'feature', value: 'disabled',
@@ -72,4 +73,4 @@ option('glib-checks', type : 'feature', value : 'enabled', yield : true,
 option('package-name', type : 'string', yield : true,
        description : 'package name to use in plugins')
 option('gstreamer-full-license', type : 'string', value : 'unknown',
-       description : 'gstreamer-full license (default unknown)')
+       description : 'gstreamer-full license (default unknown)')

tests/meson.build

@@ -2,4 +2,5 @@ if get_option('tests').disabled()
 	subdir_done()
 endif
 subdir('static-plugins')
-subdir('python')
+subdir('python')
+subdir('virtme')

tests/virtme/README.md

@@ -0,0 +1,16 @@
+## High level description of the files in this directory.
+
+### virtme-run.sh
+
+A helper script that uses 'virtme' to launch a qemu virtual machine with the
+host filesystem exposed inside the virtual machine.
+
+### run-virt-test.sh
+
+Run the given command and retrieve the command status in the given status file.
+This is necessary because virtme doesn't return the exit code of the command.
+
+### meson.build
+
+Contains one rule for meson test cases that launches tests inside virtual
+machines.

tests/virtme/meson.build

@@ -0,0 +1,52 @@
+kernel_image_path = get_option('virtme_kernel_image')
+if kernel_image_path != ''
+  if not fs.is_absolute(kernel_image_path)
+    error('Kernel image path @0@ must be absolute'.format(kernel_image_path))
+  endif
+  if not fs.is_file(kernel_image_path)
+    error('Kernel image @0@ does not exist'.format(kernel_image_path))
+  endif
+
+  virtme_run = find_program('virtme-run.sh')
+
+  supported_decoders = {
+    'vp8': [],
+    'vp9': [],
+    'h.264': [
+      'FM1_BT_B',
+      'FM2_SVA_C'
+    ],
+    'h.265': [
+      'PICSIZE_A_Bossen_1',
+      'PICSIZE_B_Bossen_1',
+      'PICSIZE_C_Bossen_1',
+      'PICSIZE_D_Bossen_1',
+      'EXTPREC_HIGHTHROUGHPUT_444_16_INTRA_16BIT_RExt_Sony_1',
+      'EXTPREC_MAIN_444_16_INTRA_16BIT_RExt_Sony_1',
+      'GENERAL_12b_400_RExt_Sony_1',
+      'GENERAL_16b_400_RExt_Sony_1',
+      'GENERAL_16b_444_highThroughput_RExt_Sony_2',
+      'GENERAL_16b_444_RExt_Sony_2',
+      'GENERAL_8b_400_RExt_Sony_1',
+      'WAVETILES_RExt_Sony_2'
+    ],
+  }
+
+  foreach decoder, skipped : supported_decoders
+    test('v4l2-stateless-decoders-' + decoder, virtme_run,
+      workdir: meson.current_source_dir(),
+      suite: 'v4l2-stateless-decoders',
+      args: [
+        kernel_image_path,
+        meson.project_build_root(),
+        meson.source_root(),
+        decoder,
+        skipped,
+      ],
+      is_parallel: true,
+      timeout: 180,
+    )
+  endforeach
+else
+  warning('No kernel image path specified, skipping virtme tests')
+endif

tests/virtme/run-virt-test.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Run the given command in the meson dev environment.
+# The command return value will be stored in the given
+# status file.
+#
+# $1: The command to be run
+# $2: The status file
+
+set -e
+
+COMMAND="${1}"
+STATUS_FILE="${2}"
+
+echo Run ${COMMAND} in the devenv
+meson devenv -C ${MESON_BUILD_DIR} ${COMMAND}
+
+STATUS=$?
+echo $STATUS > ${STATUS_FILE}
+
+exit $STATUS

tests/virtme/virtme-run.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Based on the virtme-run.sh script from the Mutter project:
+# https://gitlab.gnome.org/GNOME/mutter/-/blob/main/src/tests/kvm/virtme-run.sh
+#
+# Run fluster tests in a virtual machine using virtme-ng.
+#
+# $1: A Linux kernel image
+# $2: The test build dir
+# $3: GStreamer source dir
+# $4: The decoder to be run in [vp8, vp9, h.264, h.265, av1]
+# ($@: The test vectors to be skipped)
+
+set -e
+
+DIRNAME="$(dirname "$0")"
+IMAGE="$1"
+MESON_BUILD_DIR="$2"
+MESON_SOURCE_DIR="$3"
+DECODER="${4}"
+
+shift
+shift
+shift
+shift
+
+SKIPPED="$@"
+
+if [ ! -z "${SKIPPED}" ]; then
+	sv="-sv ${SKIPPED}"
+fi
+
+TEST_RESULT_FILE=$(mktemp -p "$MESON_BUILD_DIR" -t test-result-XXXXXX)
+echo 1 > "$TEST_RESULT_FILE"
+
+VIRTME_ENV="\
+MESON_BUILD_DIR=${MESON_BUILD_DIR} \
+"
+
+TEST_SUITES_DIR="${MESON_SOURCE_DIR}/ci/fluster/visl_references"
+
+FLUSTER_PATH=/opt/fluster
+TEST_COMMAND="${FLUSTER_PATH}/fluster.py -tsd ${TEST_SUITES_DIR} run -d GStreamer-${DECODER}-V4L2SL-Gst1.0 -f junitxml -so $MESON_BUILD_DIR/fluster-results-${DECODER}.xml ${sv} -t 60"
+
+SCRIPT="\
+  env $VIRTME_ENV $DIRNAME/run-virt-test.sh \
+  \\\"$TEST_COMMAND\\\" \
+  \\\"$TEST_RESULT_FILE\\\" \
+"
+
+HALF_MEMORY="$(grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=0; 1+{}/1024^2/2" | bc)G"
+
+echo Running tests in virtual machine ...
+virtme-run \
+  --memory=${HALF_MEMORY} \
+  --rw \
+  --pwd \
+  --kimg "$IMAGE" \
+  --script-sh "sh -c \"$SCRIPT\"" \
+  -a visl.stable_output=true \
+  -a visl.codec_variability=true \
+  --show-boot-console --show-command \
+  --qemu-opts -cpu host,pdcm=off -smp 8
+VM_RESULT=$?
+if [ $VM_RESULT != 0 ]; then
+  echo Virtual machine exited with a failure: $VM_RESULT
+else
+  echo Virtual machine terminated.
+fi
+TEST_RESULT="$(cat "$TEST_RESULT_FILE")"
+
+echo Test result exit status: $TEST_RESULT
+
+rm "$TEST_RESULT_FILE"
+exit "$TEST_RESULT"