build/fuzzing: integrate fuzz targets into the build system

Currently disabled but may be enabled later.

Updates the existing fuzzing to use shared libraries as that's easier
for meson to deal with if there is a mix of static and shared libraries
on the system.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2123>

ci/fuzzing/README.txt

@@ -32,6 +32,12 @@ Fuzzing GStreamer
   function whose signature follows the LibFuzzer signature:
   https://llvm.org/docs/LibFuzzer.html
 
+* *.corpus
+
+  A file matching a test name that contains a list of files to use when
+  starting a fuzzing run.  Providing an initial set files can speed up
+  the fuzzing process significantly.
+
 * TODO
 
   * Add a standalone build script

ci/fuzzing/build-oss-fuzz.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -eu
+#!/bin/bash -eux
 
 # build-oss-fuzz.sh
 #
@@ -11,11 +11,8 @@
 # /!\ Do not override any CC, CXX, CFLAGS, ... variables
 #
 
-# This script is divided in two parts
+rm -rf $WORK/*
-#
+rm -rf $OUT/lib $OUT/*_seed_corpus.zip
-# 1) Build all the dependencies statically
-#
-# 2) Build the fuzzing targets
 
 # Prefix where we will temporarily install everything
 PREFIX=$WORK/prefix
@@ -30,176 +27,108 @@ export PATH=$PREFIX/bin:$PATH
 # Minimize gst-debug level/code
 export CFLAGS="$CFLAGS -DGST_LEVEL_MAX=2"
 
-#
 echo "CFLAGS : " $CFLAGS
 echo "CXXFLAGS : " $CXXFLAGS
-PLUGIN_DIR=$PREFIX/lib/gstreamer-1.0
-
-rm -rf $WORK/*
 
 # Switch to work directory
 cd $WORK
 
-# 1) BUILD GLIB AND GSTREAMER
+mkdir -p $OUT/lib/gstreamer-1.0
-# Note: we build glib ourselves so that we get proper malloc/free backtraces
+
-tar xvJf $SRC/glib-2.72.0.tar.xz
+# build ogg
-cd glib-2.72.0
+pushd $SRC/ogg
-# options taken from glib's oss-fuzz build definition
+./autogen.sh
-meson \
+./configure --prefix="$PREFIX" --libdir="$PREFIX/lib"
-    --prefix=$PREFIX \
+make clean
-    --libdir=lib \
+make -j$(nproc)
-    --default-library=static \
+make install
-    -Db_lundef=false \
+popd
-    -Doss_fuzz=enabled \
+
-    -Dlibmount=disabled \
+# build vorbis
-    _builddir
+pushd $SRC/vorbis
-ninja -C _builddir
+./autogen.sh
-ninja -C _builddir install
+./configure --prefix="$PREFIX" --libdir="$PREFIX/lib"
-cd ..
+make clean
+make -j$(nproc)
+make install
+popd
+
+# build theora
+pushd $SRC/theora
+./autogen.sh
+./configure --prefix="$PREFIX" --libdir="$PREFIX/lib"
+make clean
+make -j$(nproc)
+make install
+popd
 
 # Note: We don't use/build orc since it still seems to be problematic
 # with clang and the various sanitizers.
 
 # For now we only build core and base. Add other modules when/if needed
-for i in gstreamer;
+meson \
+    --prefix=$PREFIX \
+    --libdir=lib \
+    --default-library=shared \
+    --force-fallback-for=zlib \
+    -Db_lundef=false \
+    -Doss_fuzz=enabled \
+    -Dglib:oss_fuzz=enabled \
+    -Dglib:libmount=disabled \
+    -Dglib:tests=false \
+    -Ddoc=disabled \
+    -Dexamples=disabled \
+    -Dintrospection=disabled \
+    -Dgood=disabled \
+    -Dugly=disabled \
+    -Dbad=disabled \
+    -Dlibav=disabled \
+    -Dges=disabled \
+    -Domx=disabled \
+    -Dvaapi=disabled \
+    -Dsharp=disabled \
+    -Drs=disabled \
+    -Dpython=disabled \
+    -Dlibnice=disabled \
+    -Ddevtools=disabled \
+    -Drtsp_server=disabled \
+    -Dgst-examples=disabled \
+    -Dqt5=disabled \
+    -Dorc=disabled \
+    -Dgtk_doc=disabled \
+    -Dgstreamer:tracer_hooks=false \
+    -Dgst-plugins-base:opus=disabled \
+    -Dgst-plugins-base:pango=disabled \
+    _builddir \
+    $SRC/gstreamer
+ninja -C _builddir
+ninja -C _builddir install
+
+# copy out the fuzzing binaries
+for BINARY in $(find _builddir/ci/fuzzing -type f -executable -print)
 do
-    mkdir -p $i
+  BASENAME=${BINARY##*/}
-    cd $i
+  rm -rf "$OUT/$BASENAME*"
-    meson \
+  cp $BINARY $OUT/$BASENAME
-        --prefix=$PREFIX \
+  patchelf --set-rpath '$ORIGIN/lib' $OUT/$BASENAME
-        --libdir=lib \
-        --default-library=static \
-        -Db_lundef=false \
-        -Ddoc=disabled \
-        -Dexamples=disabled \
-        -Dintrospection=disabled \
-        -Dgood=disabled \
-        -Dugly=disabled \
-        -Dbad=disabled \
-        -Dlibav=disabled \
-        -Dges=disabled \
-        -Domx=disabled \
-        -Dvaapi=disabled \
-        -Dsharp=disabled \
-        -Drs=disabled \
-        -Dpython=disabled \
-        -Dlibnice=disabled \
-        -Ddevtools=disabled \
-        -Drtsp_server=disabled \
-        -Dgst-examples=disabled \
-        -Dqt5=disabled \
-        -Dorc=disabled \
-        -Dgtk_doc=disabled \
-        -Dgstreamer:tracer_hooks=false \
-        -Dgstreamer:registry=false \
-        -Dgst-plugins-base:opus=disabled \
-        -Dgst-plugins-base:pango=disabled \
-        _builddir \
-        $SRC/$i
-    ninja -C _builddir
-    ninja -C _builddir install
-    cd ..
 done
 
+# copy any relevant corpus
+for CORPUS in $(find "$SRC/gstreamer/ci/fuzzing" -type f -name "*.corpus"); do
+  BASENAME=${CORPUS##*/}
+  pushd "$SRC/gstreamer"
+  zip $OUT/${BASENAME%%.*}_seed_corpus.zip . -ws -r -i@$CORPUS
+  popd
+done
 
+# copy dependant libraries
+find "$PREFIX/lib" -maxdepth 1 -type f -name "*.so*" -exec cp -d "{}" $OUT/lib \; -print
+# add rpath that point to the correct place to all shared libraries
+find "$OUT/lib" -maxdepth 1 -type f -name "*.so*" -exec patchelf --debug --set-rpath '$ORIGIN' {} \;
+find "$PREFIX/lib" -maxdepth 1 -type l -name "*.so*" -exec cp -d "{}" $OUT/lib \; -print
 
-# 2) Build the target fuzzers
+find "$PREFIX/lib/gstreamer-1.0" -maxdepth 1 -type f -name "*.so" -exec cp -d "{}" $OUT/lib/gstreamer-1.0 \;
+find "$OUT/lib/gstreamer-1.0" -type f -name "*.so*" -exec patchelf --debug --set-rpath '$ORIGIN/..' {} \;
 
-# All targets will be linked in with $LIB_FUZZING_ENGINE which contains the
+# make it easier to spot dependency issues
-# actual fuzzing runner. Anything fuzzing engine can be used provided it calls
+find "$OUT/lib/gstreamer-1.0" -maxdepth 1 -type f -name "*.so" -print -exec ldd {} \;
-# the same function as libfuzzer.
-
-# Note: The fuzzer .o needs to be first compiled with CC and then linked with CXX
-
-# We want to statically link everything, except for shared libraries
-# that are present on the base image. Those need to be specified
-# beforehand and explicitely linked dynamically If any of the static
-# dependencies require a pre-installed shared library, you need to add
-# that library to the following list
-PREDEPS_LDFLAGS="-Wl,-Bdynamic -ldl -lm -pthread -lrt -lpthread"
-
-# These are the basic .pc dependencies required to build any of the fuzzing targets
-# That is : glib, gstreamer core and gst-app
-# The extra target-specific dependencies are to be specified later
-COMMON_DEPS="glib-2.0 gio-2.0 gstreamer-1.0 gstreamer-app-1.0"
-
-# For each target, defined the following:
-# TARGET_DEPS : Extra .pc dependencies for the target (in addition to $COMMON_DEPS)
-#               All dependencies (including sub-dependencies) must be speecified
-# PLUGINS : .a of the plugins to link
-#           They must match the static plugins declared/registered in the target
-
-#
-# TARGET : push-based ogg/theora/vorbis discoverer
-#
-# FIXME : Rename to discoverer_push_oggtheoravorbis
-
-TARGET_DEPS=" gstreamer-pbutils-1.0 \
-	      gstreamer-video-1.0 \
-	      gstreamer-audio-1.0 \
-	      gstreamer-riff-1.0 \
-	      gstreamer-tag-1.0 \
-	      zlib ogg vorbis vorbisenc \
-	      theoraenc theoradec theora cairo"
-
-PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \
-       $PLUGIN_DIR/libgsttypefindfunctions.a \
-       $PLUGIN_DIR/libgstplayback.a \
-       $PLUGIN_DIR/libgstapp.a \
-       $PLUGIN_DIR/libgstvorbis.a \
-       $PLUGIN_DIR/libgsttheora.a \
-       $PLUGIN_DIR/libgstogg.a"
-
-echo
-echo ">>>> BUILDING gst-discoverer"
-echo
-BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`"
-BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`"
-
-$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gstreamer/ci/fuzzing/gst-discoverer.c -o $SRC/gstreamer/ci/fuzzing/gst-discoverer.o
-$CXX $CXXFLAGS \
-      -o $OUT/gst-discoverer \
-      $PREDEPS_LDFLAGS \
-      $SRC/gstreamer/ci/fuzzing/gst-discoverer.o \
-      $PLUGINS \
-      $BUILD_LDFLAGS \
-      $LIB_FUZZING_ENGINE \
-      -Wl,-Bdynamic
-
-#
-# TARGET : push-based typefind
-#
-
-# typefindfunction depends on pbutils which depends on gst{audio|video|tag}
-TARGET_DEPS=" gstreamer-pbutils-1.0 \
-	      gstreamer-video-1.0 \
-	      gstreamer-audio-1.0 \
-	      gstreamer-tag-1.0"
-
-PLUGINS="$PLUGIN_DIR/libgstcoreelements.a \
-       $PLUGIN_DIR/libgsttypefindfunctions.a \
-       $PLUGIN_DIR/libgstapp.a"
-
-echo
-echo ">>>> BUILDING typefind"
-echo
-BUILD_CFLAGS="$CFLAGS `pkg-config --static --cflags $COMMON_DEPS $TARGET_DEPS`"
-BUILD_LDFLAGS="-Wl,-static `pkg-config --static --libs $COMMON_DEPS $TARGET_DEPS`"
-
-$CC $CFLAGS $BUILD_CFLAGS -c $SRC/gstreamer/ci/fuzzing/typefind.c -o $SRC/gstreamer/ci/fuzzing/typefind.o
-$CXX $CXXFLAGS \
-      -o $OUT/typefind \
-      $PREDEPS_LDFLAGS \
-      $SRC/gstreamer/ci/fuzzing/typefind.o \
-      $PLUGINS \
-      $BUILD_LDFLAGS \
-      $LIB_FUZZING_ENGINE \
-      -Wl,-Bdynamic
-
-echo
-echo ">>>> Installing seed corpus"
-echo
-# FIXME : Sadly we apparently need to have the corpus downloaded in the
-#         Dockerfile and not here.
-
-cp $SRC/*_seed_corpus.zip $OUT

ci/fuzzing/gst-discoverer.c

@@ -27,16 +27,6 @@
 #include <gst/gst.h>
 #include <gst/pbutils/pbutils.h>
 
-#ifndef LOCAL_FUZZ_BUILD
-GST_PLUGIN_STATIC_DECLARE (coreelements);
-GST_PLUGIN_STATIC_DECLARE (playback);
-GST_PLUGIN_STATIC_DECLARE (typefindfunctions);
-GST_PLUGIN_STATIC_DECLARE (app);
-GST_PLUGIN_STATIC_DECLARE (ogg);
-GST_PLUGIN_STATIC_DECLARE (theora);
-GST_PLUGIN_STATIC_DECLARE (vorbis);
-#endif
-
 /* push-based discoverer fuzzing target
  *
  * This application can be compiled with libFuzzer to simulate
@@ -99,17 +89,6 @@ LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
 
     /* Only initialize and register plugins once */
     gst_init (NULL, NULL);
-
-#ifndef LOCAL_FUZZ_BUILD
-    GST_PLUGIN_STATIC_REGISTER (coreelements);
-    GST_PLUGIN_STATIC_REGISTER (playback);
-    GST_PLUGIN_STATIC_REGISTER (typefindfunctions);
-    GST_PLUGIN_STATIC_REGISTER (app);
-    GST_PLUGIN_STATIC_REGISTER (ogg);
-    GST_PLUGIN_STATIC_REGISTER (theora);
-    GST_PLUGIN_STATIC_REGISTER (vorbis);
-#endif
-
     initialized = TRUE;
   }
 

ci/fuzzing/gst-discoverer.corpus

@@ -0,0 +1,4 @@
+subprojects/gst-integration-testsuites/medias/defaults/ogg/numerated_frames_blue.ogv
+subprojects/gst-integration-testsuites/medias/defaults/ogg/opus.1.ogg
+subprojects/gst-integration-testsuites/medias/defaults/ogg/vorbis_theora.0.ogg
+subprojects/gst-integration-testsuites/medias/defaults/ogg/vorbis_theora.1.ogg

ci/fuzzing/meson.build

@@ -0,0 +1,40 @@
+if get_option('oss_fuzz').disabled()
+  subdir_done()
+endif
+
+fuzz_targets = [
+  ['gst-discoverer.c', false, ['gstreamer-pbutils-1.0']],
+  ['typefind.c'],
+]
+
+extra_sources = []
+gst_dep = dependency('gstreamer-1.0')
+common_deps = [gst_dep]
+
+cxx = meson.get_compiler('cpp')
+fuzzing_engine = cxx.find_library('FuzzingEngine', required: false)
+if fuzzing_engine.found()
+  common_deps += fuzzing_engine
+else
+  extra_sources += ['localfuzzer.c']
+endif
+
+foreach target : fuzz_targets
+  file_name = target.get(0)
+  test_name = file_name.split('.').get(0)
+
+  extra_deps = []
+  if target.length() >= 3
+    extra_deps = dependency(target.get(2))
+  endif
+
+  skip_test = false
+  if target.length() >= 2
+    skip_test = target.get(1)
+  endif
+  if not skip_test
+    exe = executable(test_name, [extra_sources, file_name],
+      dependencies: common_deps + extra_deps,
+    )
+  endif
+endforeach

ci/fuzzing/typefind.c

@@ -26,12 +26,6 @@
 #include <glib.h>
 #include <gst/gst.h>
 
-#ifndef LOCAL_FUZZ_BUILD
-GST_PLUGIN_STATIC_DECLARE (coreelements);
-GST_PLUGIN_STATIC_DECLARE (typefindfunctions);
-GST_PLUGIN_STATIC_DECLARE (app);
-#endif
-
 /* push-based typefind fuzzing target
  *
  * This application can be compiled with libFuzzer to simulate
@@ -58,7 +52,6 @@ custom_logger (const gchar * log_domain,
 int
 LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
 {
-  GError *err = NULL;
   static gboolean initialized = FALSE;
   GstElement *pipeline, *source, *typefind, *fakesink;
   GstBuffer *buf;
@@ -73,12 +66,6 @@ LLVMFuzzerTestOneInput (const guint8 * data, size_t size)
     /* Only initialize and register plugins once */
     gst_init (NULL, NULL);
 
-#ifndef LOCAL_FUZZ_BUILD
-    GST_PLUGIN_STATIC_REGISTER (coreelements);
-    GST_PLUGIN_STATIC_REGISTER (typefindfunctions);
-    GST_PLUGIN_STATIC_REGISTER (app);
-#endif
-
     initialized = TRUE;
   }
 

meson.build

@@ -436,6 +436,8 @@ devenv_cmd = [setenv, '--builddir=@0@'.format(meson.global_build_root()),
               '--srcdir=@0@'.format(meson.global_source_root())]
 
 subdir('tests')
+subdir('ci/fuzzing')
+
 if meson.can_run_host_binaries() and build_machine.system() == 'linux' and host_machine.system() == 'windows'
   # FIXME: Ideally we could get the wrapper directly from meson
   devenv_cmd += ['--wine', host_machine.cpu_family() == 'x86_64' ? 'wine64' : 'wine32']

meson_options.txt

@@ -18,6 +18,10 @@ option('tls', type : 'feature', value : 'auto', description : 'TLS support using
 option('qt5', type : 'feature', value : 'auto', description : 'Qt5 Support')
 option('tools', type : 'feature', value : 'auto', yield : true, description : 'Build command line tools')
 
+# Build for fuzzing
+option('oss_fuzz', type : 'feature', value : 'disabled',
+       description: 'Use fuzzing build environment')
+
 # Other options
 option('custom_subprojects', type : 'string', value : '', description : 'Comma-separated project names')
 option('gst-full-libraries', type : 'array', value : [],