osxaudio: Automatically set up AVAudioSession on iOS

A correctly configured AVAudioSession is needed on iOS to:
- allow the application to capture microphone audio (in some cases)
- avoid playback being silenced in silent mode

Without this, initializing AudioUnit for capture can fail on iOS >=17 (from my testing).

Since AVAudioSession has a lot of settings, in most cases its setup should be handled by the user/app.
However, just to have a basic default scenario covered, let's configure the bare minimum ourselves,
and allow anyone to disable that behaviour by setting configure-session=false on src/sink.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7856>

subprojects/gst-plugins-good/meson.build

@@ -37,9 +37,22 @@ plugins_install_dir = join_paths(get_option('libdir'), 'gstreamer-1.0')
 plugins = []
 static_build = get_option('default_library') == 'static'
 
-cc = meson.get_compiler('c')
 host_system = host_machine.system()
 
+if host_system in ['ios', 'darwin']
+  have_objc = add_languages('objc', native: false)
+  have_objcpp = add_languages('objcpp', native: false)
+
+  if not have_objc
+    error('Building on MacOS/iOS/etc requires an ObjC compiler')
+  endif
+else
+  have_objc = false
+  have_objcpp = false
+endif
+
+cc = meson.get_compiler('c')
+
 if cc.get_id() == 'msvc'
   msvc_args = [
       # Ignore several spurious warnings for things gstreamer does very commonly
@@ -101,6 +114,10 @@ endif
 # Symbol visibility
 if cc.has_argument('-fvisibility=hidden')
   add_project_arguments('-fvisibility=hidden', language: 'c')
+
+  if have_objc
+    add_project_arguments('-fvisibility=hidden', language: 'objc')
+  endif
 endif
 
 # Disable strict aliasing

subprojects/gst-plugins-good/sys/osxaudio/gstiosaudiosession.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2024 Piotr Brzeziński <piotr@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _GST_IOS_AUDIO_SESSION_H
+#define _GST_IOS_AUDIO_SESSION_H
+
+#include <gst/gst.h>
+
+G_BEGIN_DECLS
+
+void gst_ios_audio_session_setup (gboolean is_src);
+
+G_END_DECLS
+
+#endif /* _GST_IOS_AUDIO_SESSION_H */

subprojects/gst-plugins-good/sys/osxaudio/gstiosaudiosession.m

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2024 Piotr Brzeziński <piotr@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include "gstiosaudiosession.h"
+#include <AVFAudio/AVFAudio.h>
+
+// Default according to: https://developer.apple.com/documentation/avfaudio/avaudiosessioncategorysoloambient
+#define DEFAULT_CAT AVAudioSessionCategorySoloAmbient
+
+void 
+gst_ios_audio_session_setup (gboolean is_src)
+{
+  AVAudioSession *session = [AVAudioSession sharedInstance];
+
+  /* This is just a quick best effort setup.
+   * In any serious applications, you should disable configure-session
+   * and handle AVAudioSession setup yourself. */
+
+  if (is_src) {
+    /* For mic capture, let's use PlayAndRecord as that allows simultaneous playback and recording
+     * We don't have to check if a sink (output) already set a category, as the behaviour will not
+     * change in PlayAndRecord. */
+    [session setCategory:AVAudioSessionCategoryPlayAndRecord error:NULL];
+  } else if ([session category] == DEFAULT_CAT) {
+    /* For output, let's just use Playback, but only if a src element didn't set things up first */
+    [session setCategory:AVAudioSessionCategoryPlayback error:NULL];
+  }
+
+  [session setActive:YES error:NULL];
+}

subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiosink.c

@@ -89,10 +89,12 @@ enum
   ARG_0,
   ARG_DEVICE,
   ARG_VOLUME,
-  ARG_UNIQUE_ID
+  ARG_UNIQUE_ID,
+  ARG_CONFIGURE_SESSION,
 };
 
 #define DEFAULT_VOLUME 1.0
+#define DEFAULT_CONFIGURE_SESSION TRUE
 
 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
     GST_PAD_SINK,
@@ -189,6 +191,29 @@ gst_osx_audio_sink_class_init (GstOsxAudioSinkClass * klass)
       g_param_spec_string ("unique-id", "Unique ID",
           "Unique persistent ID for the input device",
           NULL, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
+#else
+  /**
+   * GstOsxAudioSink:configure-session:
+   *
+   * Whether the app-wide AVAudioSession should be automatically set up for audio playback.
+   * This will set the category to AVAudioSessionCategoryPlayback and activate the session
+   * when the element goes to READY. No other settings will be changed.
+   *
+   * The category change will only occur if current category is the default one (SoloAmbient)
+   * to avoid setting the category 'lower' if an osxaudiosrc element is also running in the
+   * same process.
+   *
+   * If your application needs to configure anything more than the category, set this to FALSE
+   * for all osxaudiosink/src instances and handle the AVAudioSession setup yourself.
+   *
+   * Since: 1.26
+   */
+  g_object_class_install_property (gobject_class, ARG_CONFIGURE_SESSION,
+      g_param_spec_boolean ("configure-session",
+          "Enable automatic AVAudioSession setup",
+          "Whether the app-wide AVAudioSession should be automatically configured for audio playback",
+          DEFAULT_CONFIGURE_SESSION,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 #endif
 
   gstbasesink_class->query = GST_DEBUG_FUNCPTR (gst_osx_audio_sink_query);
@@ -219,6 +244,10 @@ gst_osx_audio_sink_init (GstOsxAudioSink * sink)
 
   sink->device_id = kAudioDeviceUnknown;
   sink->volume = DEFAULT_VOLUME;
+
+#ifdef HAVE_IOS
+  sink->configure_session = DEFAULT_CONFIGURE_SESSION;
+#endif
 }
 
 static void
@@ -232,6 +261,10 @@ gst_osx_audio_sink_set_property (GObject * object, guint prop_id,
     case ARG_DEVICE:
       sink->device_id = g_value_get_int (value);
       break;
+#else
+    case ARG_CONFIGURE_SESSION:
+      sink->configure_session = g_value_get_boolean (value);
+      break;
 #endif
     case ARG_VOLUME:
       sink->volume = g_value_get_double (value);
@@ -306,6 +339,10 @@ gst_osx_audio_sink_get_property (GObject * object, guint prop_id,
       g_value_set_string (value, sink->unique_id);
       GST_OBJECT_UNLOCK (sink);
       break;
+#else
+    case ARG_CONFIGURE_SESSION:
+      g_value_set_boolean (value, sink->configure_session);
+      break;
 #endif
     case ARG_VOLUME:
       g_value_set_double (value, sink->volume);
@@ -521,7 +558,11 @@ gst_osx_audio_sink_create_ringbuffer (GstAudioBaseSink * sink)
       (void *) gst_osx_audio_sink_io_proc);
 
   ringbuffer->core_audio = g_object_new (GST_TYPE_CORE_AUDIO,
-      "is-src", FALSE, "device", osxsink->device_id, NULL);
+      "is-src", FALSE, "device", osxsink->device_id,
+#ifdef HAVE_IOS
+      "configure-session", osxsink->configure_session,
+#endif
+      NULL);
   ringbuffer->core_audio->osxbuf = GST_OBJECT (ringbuffer);
   ringbuffer->core_audio->element =
       GST_OSX_AUDIO_ELEMENT_GET_INTERFACE (osxsink);

subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiosink.h

@@ -88,6 +88,10 @@ struct _GstOsxAudioSink
   double volume;
 
   guint channels;
+
+#ifdef HAVE_IOS
+  gboolean configure_session;
+#endif
 };
 
 struct _GstOsxAudioSinkClass

subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiosrc.c

@@ -78,8 +78,11 @@ enum
   ARG_0,
   ARG_DEVICE,
   ARG_UNIQUE_ID,
+  ARG_CONFIGURE_SESSION,
 };
 
+#define DEFAULT_CONFIGURE_SESSION TRUE
+
 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
@@ -164,6 +167,27 @@ gst_osx_audio_src_class_init (GstOsxAudioSrcClass * klass)
           "Unique persistent ID for the input device",
           NULL, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
 
+#ifdef HAVE_IOS
+  /**
+   * GstOsxAudioSrc:configure-session:
+   *
+   * Whether the app-wide AVAudioSession should be automatically set up for audio capture.
+   * This will set the category to AVAudioSessionCategoryPlayAndRecord and activate
+   * the session when the element goes to READY. No other settings will be changed.
+   *
+   * If your application needs to configure anything more than the category, set this to FALSE
+   * for all osxaudiosink/src instances and handle the AVAudioSession setup yourself.
+   *
+   * Since: 1.26
+   */
+  g_object_class_install_property (gobject_class, ARG_CONFIGURE_SESSION,
+      g_param_spec_boolean ("configure-session",
+          "Enable automatic AVAudioSession setup",
+          "Whether the app-wide AVAudioSession should be automatically configured for audio capture",
+          DEFAULT_CONFIGURE_SESSION,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+#endif
+
   gstaudiobasesrc_class->create_ringbuffer =
       GST_DEBUG_FUNCPTR (gst_osx_audio_src_create_ringbuffer);
 
@@ -182,6 +206,10 @@ gst_osx_audio_src_init (GstOsxAudioSrc * src)
 
   src->device_id = kAudioDeviceUnknown;
   src->unique_id = NULL;
+
+#ifdef HAVE_IOS
+  src->configure_session = DEFAULT_CONFIGURE_SESSION;
+#endif
 }
 
 static void
@@ -194,6 +222,11 @@ gst_osx_audio_src_set_property (GObject * object, guint prop_id,
     case ARG_DEVICE:
       src->device_id = g_value_get_int (value);
       break;
+#ifdef HAVE_IOS
+    case ARG_CONFIGURE_SESSION:
+      src->configure_session = g_value_get_boolean (value);
+      break;
+#endif
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -215,6 +248,11 @@ gst_osx_audio_src_get_property (GObject * object, guint prop_id,
       g_value_set_string (value, src->unique_id);
       GST_OBJECT_UNLOCK (src);
       break;
+#ifdef HAVE_IOS
+    case ARG_CONFIGURE_SESSION:
+      g_value_set_boolean (value, src->configure_session);
+      break;
+#endif
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -345,7 +383,11 @@ gst_osx_audio_src_create_ringbuffer (GstAudioBaseSrc * src)
       (void *) gst_osx_audio_src_io_proc);
 
   ringbuffer->core_audio = g_object_new (GST_TYPE_CORE_AUDIO,
-      "is-src", TRUE, "device", osxsrc->device_id, NULL);
+      "is-src", TRUE, "device", osxsrc->device_id,
+#ifdef HAVE_IOS
+      "configure-session", osxsrc->configure_session,
+#endif
+      NULL);
   ringbuffer->core_audio->osxbuf = GST_OBJECT (ringbuffer);
   ringbuffer->core_audio->element =
       GST_OSX_AUDIO_ELEMENT_GET_INTERFACE (osxsrc);

subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiosrc.h

@@ -74,6 +74,10 @@ struct _GstOsxAudioSrc
 
   AudioDeviceID device_id;
   const char *unique_id;
+
+#ifdef HAVE_IOS
+  gboolean configure_session;
+#endif
 };
 
 struct _GstOsxAudioSrcClass

subprojects/gst-plugins-good/sys/osxaudio/gstosxcoreaudio.c

@@ -41,6 +41,7 @@ enum
   PROP_0,
   PROP_DEVICE,
   PROP_IS_SRC,
+  PROP_CONFIGURE_SESSION,
 };
 
 static void gst_core_audio_set_property (GObject * object, guint prop_id,
@@ -76,6 +77,14 @@ gst_core_audio_class_init (GstCoreAudioClass * klass)
       g_param_spec_boolean ("is-src", "Is source", "Is a source device",
           FALSE,
           G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
+
+#ifdef HAVE_IOS
+  g_object_class_install_property (object_klass, PROP_CONFIGURE_SESSION,
+      g_param_spec_boolean ("configure-session",
+          "Enable automatic AVAudioSession setup",
+          "Auto-configure the AVAudioSession for audio playback/capture", FALSE,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT_ONLY));
+#endif
 }
 
 static void
@@ -91,6 +100,8 @@ gst_core_audio_init (GstCoreAudio * core_audio)
 #ifndef HAVE_IOS
   core_audio->hog_pid = -1;
   core_audio->disabled_mixing = FALSE;
+#else
+  core_audio->configure_session = FALSE;
 #endif
 
   mach_timebase_info (&core_audio->timebase);
@@ -110,6 +121,11 @@ gst_core_audio_set_property (GObject * object, guint prop_id,
     case PROP_DEVICE:
       self->device_id = g_value_get_int (value);
       break;
+#ifdef HAVE_IOS
+    case PROP_CONFIGURE_SESSION:
+      self->configure_session = g_value_get_boolean (value);
+      break;
+#endif
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -129,6 +145,11 @@ gst_core_audio_get_property (GObject * object, guint prop_id,
     case PROP_DEVICE:
       g_value_set_int (value, self->device_id);
       break;
+#ifdef HAVE_IOS
+    case PROP_CONFIGURE_SESSION:
+      g_value_set_boolean (value, self->configure_session);
+      break;
+#endif
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;

subprojects/gst-plugins-good/sys/osxaudio/gstosxcoreaudio.h

@@ -119,6 +119,10 @@ struct _GstCoreAudio
   uint64_t anchor_hosttime_ns;
   uint32_t anchor_pend_samples;
   float rate_scalar;
+
+#ifdef HAVE_IOS
+  gboolean configure_session;
+#endif
 };
 
 struct _GstCoreAudioClass

subprojects/gst-plugins-good/sys/osxaudio/gstosxcoreaudioremoteio.c

@@ -21,10 +21,19 @@
  */
 
 #import <AudioUnit/AUComponent.h>
+#import "gstiosaudiosession.h"
 
 static gboolean
 gst_core_audio_open_impl (GstCoreAudio * core_audio)
 {
+  /* On iOS, an AVAudioSession needs to be set up to 1) avoid playback being silenced
+   * by silent mode and 2) to allow audio to be captured from the microphone.
+   * However, AVAudioSession has a lot of settings and in more complicated scenarios,
+   * apps/users should handle that themselves. Disable auto-config through the
+   * configure-session property on osxaudiosrc/sink in those cases. */
+  if (core_audio->configure_session)
+    gst_ios_audio_session_setup (core_audio->is_src);
+
   return gst_core_audio_open_device (core_audio, kAudioUnitSubType_RemoteIO,
       "RemoteIO");
 }

subprojects/gst-plugins-good/sys/osxaudio/meson.build

@@ -22,18 +22,22 @@ if host_system == 'darwin'
   osxaudio_sources += ['gstosxaudiodeviceprovider.c']
 elif host_system == 'ios'
   have_osxaudio = cc.has_header('CoreAudio/CoreAudioTypes.h', required: osxaudio_option)
+  osxaudio_sources += ['gstiosaudiosession.m']
 endif
 
 if have_osxaudio
   modules = ['CoreAudio', 'AudioToolbox']
   if host_system == 'darwin'
     modules += ['AudioUnit', 'CoreServices']
+  elif host_system == 'ios'
+    modules += ['AVFAudio', 'Foundation']
   endif
   osxaudio_dep = dependency('appleframeworks', modules : modules)
 
   gstosxaudio = library('gstosxaudio',
     osxaudio_sources,
     c_args : gst_plugins_good_args,
+    objc_args : gst_plugins_good_args,
     include_directories : [configinc, libsinc],
     dependencies : [gstaudio_dep, gstpbutils_dep, osxaudio_dep],
     install : true,