From 6d94b77ae8e3deaad63d1c5b82a9180e3774e73d Mon Sep 17 00:00:00 2001
From: Guillaume Desmottes <guillaume.desmottes@onestream.live>
Date: Thu, 19 Oct 2023 08:34:56 +0200
Subject: [PATCH] libav: expose fake booleans properties as enums

Some ffmpeg options claims to be booleans but are actually 3-values enums
with -1 as default instead of 1 or 0. Handle those using a custom enum
so we keep the same defaults as ffmpeg and users can properly configure them
if they need to.

See https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3035
for an actual example of this problem. The GStreamer element was
automatically enabling a non-default option, leading to strange behavior
in the AAC encoder.

Fix #3035

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5507>
---
 .../gst-libav/docs/gst_plugins_cache.json     | 32 +++++++++--
 subprojects/gst-libav/ext/libav/gstavcfg.c    | 55 ++++++++++++++++++-
 2 files changed, 79 insertions(+), 8 deletions(-)

diff --git a/subprojects/gst-libav/docs/gst_plugins_cache.json b/subprojects/gst-libav/docs/gst_plugins_cache.json
index 48ed39b3c8..ed612680bd 100644
--- a/subprojects/gst-libav/docs/gst_plugins_cache.json
+++ b/subprojects/gst-libav/docs/gst_plugins_cache.json
@@ -30547,10 +30547,10 @@
                         "construct": false,
                         "construct-only": false,
                         "controllable": false,
-                        "default": "true",
+                        "default": "auto (-1)",
                         "mutable": "null",
                         "readable": true,
-                        "type": "gboolean",
+                        "type": "GstFFMpegTrilian",
                         "writable": true
                     },
                     "aac-pce": {
@@ -58461,10 +58461,10 @@
                         "construct": false,
                         "construct-only": false,
                         "controllable": false,
-                        "default": "true",
+                        "default": "auto (-1)",
                         "mutable": "null",
                         "readable": true,
-                        "type": "gboolean",
+                        "type": "GstFFMpegTrilian",
                         "writable": true
                     },
                     "slices": {
@@ -126853,10 +126853,10 @@
                         "construct": false,
                         "construct-only": false,
                         "controllable": false,
-                        "default": "true",
+                        "default": "auto (-1)",
                         "mutable": "null",
                         "readable": true,
-                        "type": "gboolean",
+                        "type": "GstFFMpegTrilian",
                         "writable": true
                     },
                     "max-pixels": {
@@ -140891,6 +140891,26 @@
                     }
                 ]
             },
+            "GstFFMpegTrilian": {
+                "kind": "enum",
+                "values": [
+                    {
+                        "desc": "Auto",
+                        "name": "auto",
+                        "value": "-1"
+                    },
+                    {
+                        "desc": "Off",
+                        "name": "off",
+                        "value": "0"
+                    },
+                    {
+                        "desc": "On",
+                        "name": "on",
+                        "value": "1"
+                    }
+                ]
+            },
             "GstFFMpegVidCmpMethod": {
                 "kind": "enum",
                 "values": [
diff --git a/subprojects/gst-libav/ext/libav/gstavcfg.c b/subprojects/gst-libav/ext/libav/gstavcfg.c
index d3deb12b7b..0278ae067d 100644
--- a/subprojects/gst-libav/ext/libav/gstavcfg.c
+++ b/subprojects/gst-libav/ext/libav/gstavcfg.c
@@ -258,6 +258,48 @@ done:
   return res;
 }
 
+/** GstFFMpegTrilian:
+ *
+ * Since: 1.24
+ */
+
+/** GstFFMpegTrilian::auto
+ *
+ * Since: 1.24
+ */
+
+/** GstFFMpegTrilian::on
+ *
+ * Since: 1.24
+ */
+
+/** GstFFMpegTrilian::off
+ *
+ * Since: 1.24
+ */
+
+#define GST_TYPE_FFMPEG_TRILIAN (gst_ffmpeg_trilian_get_type ())
+static GType
+gst_ffmpeg_trilian_get_type (void)
+{
+  static const GEnumValue types[] = {
+    {-1, "Auto", "auto"},
+    {0, "Off", "off"},
+    {1, "On", "on"},
+    {0, NULL, NULL},
+  };
+  static gsize id = 0;
+
+  if (g_once_init_enter (&id)) {
+    GType gtype = g_enum_register_static ("GstFFMpegTrilian", types);
+
+    gst_type_mark_as_plugin_api (gtype, 0);
+    g_once_init_leave (&id, gtype);
+  }
+
+  return (GType) id;
+}
+
 static guint
 install_opts (GObjectClass * gobject_class, const AVClass ** obj, guint prop_id,
     gint flags, const gchar * extra_help, GHashTable * overrides)
@@ -385,8 +427,17 @@ install_opts (GObjectClass * gobject_class, const AVClass ** obj, guint prop_id,
         g_object_class_install_property (gobject_class, prop_id++, pspec);
         break;
       case AV_OPT_TYPE_BOOL:
-        pspec = g_param_spec_boolean (name, name, help,
-            opt->default_val.i64 ? TRUE : FALSE, G_PARAM_READWRITE);
+        /* Some ffmpeg options claims to be booleans but are actually 3-values enums
+         * with -1 as default instead of 1 or 0. Handle those using a custom enum
+         * so we keep the same defaults as ffmpeg and users can properly configure them.
+         */
+        if (opt->default_val.i64 == -1) {
+          pspec = g_param_spec_enum (name, name, help,
+              GST_TYPE_FFMPEG_TRILIAN, opt->default_val.i64, G_PARAM_READWRITE);
+        } else {
+          pspec = g_param_spec_boolean (name, name, help,
+              opt->default_val.i64 ? TRUE : FALSE, G_PARAM_READWRITE);
+        }
         g_object_class_install_property (gobject_class, prop_id++, pspec);
         break;
         /* TODO: didn't find options for the video encoders with