applemedia: Disable 64RGBALE support on older macOS

The kCVPixelFormatType_64RGBALE enum is only available on macOS Big
Sur (11.3) and newer. We also cannot use that while configuring the
encoder or decoder on older macOS.

Define the symbol unconditionally, but only use it when we're running
on Big Sur with __builtin_available().

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1613>

subprojects/gst-plugins-bad/sys/applemedia/vtdec.c

@@ -132,7 +132,7 @@ const CFStringRef
 CFSTR ("RequireHardwareAcceleratedVideoDecoder");
 #endif
 
-#define VIDEO_SRC_CAPS_FORMATS "{ NV12, AYUV64, RGBA64_LE, ARGB64_BE }"
+#define VIDEO_SRC_CAPS_FORMATS "{ NV12, AYUV64, ARGB64_BE }"
 
 #define VIDEO_SRC_CAPS_NATIVE                                           \
     GST_VIDEO_CAPS_MAKE(VIDEO_SRC_CAPS_FORMATS) ";"                     \
@@ -161,9 +161,15 @@ gst_vtdec_class_init (GstVtdecClass * klass)
      base_class_init if you intend to subclass this class. */
   gst_element_class_add_static_pad_template (element_class,
       &gst_vtdec_sink_template);
-  gst_element_class_add_pad_template (element_class,
-      gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
-          gst_caps_from_string (VIDEO_SRC_CAPS)));
+
+  {
+    GstCaps *caps = gst_caps_from_string (VIDEO_SRC_CAPS);
+    /* RGBA64_LE is kCVPixelFormatType_64RGBALE, only available on macOS 11.3+ */
+    if (GST_VTUTIL_HAVE_64ARGBALE)
+      caps = gst_vtutil_caps_append_video_format (caps, "RGBA64_LE");
+    gst_element_class_add_pad_template (element_class,
+        gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps));
+  }
 
   gst_element_class_set_static_metadata (element_class,
       "Apple VideoToolbox decoder",
@@ -285,10 +291,18 @@ get_preferred_video_format (GstStructure * s, gboolean prores)
         break;
       case GST_VIDEO_FORMAT_AYUV64:
       case GST_VIDEO_FORMAT_ARGB64_BE:
-      case GST_VIDEO_FORMAT_RGBA64_LE:
         if (prores)
           return vfmt;
         break;
+      case GST_VIDEO_FORMAT_RGBA64_LE:
+        if (GST_VTUTIL_HAVE_64ARGBALE) {
+          if (prores)
+            return vfmt;
+        } else {
+          /* Codepath will never be hit on macOS older than Big Sur (11.3) */
+          g_warn_if_reached ();
+        }
+        break;
       default:
         break;
     }
@@ -676,7 +690,11 @@ gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format,
       cv_format = kCVPixelFormatType_64ARGB;
       break;
     case GST_VIDEO_FORMAT_RGBA64_LE:
-      cv_format = kCVPixelFormatType_64RGBALE;
+      if (GST_VTUTIL_HAVE_64ARGBALE)
+        cv_format = kCVPixelFormatType_64RGBALE;
+      else
+        /* Codepath will never be hit on macOS older than Big Sur (11.3) */
+        g_warn_if_reached ();
       break;
     default:
       g_warn_if_reached ();

subprojects/gst-plugins-bad/sys/applemedia/vtenc.c

@@ -203,9 +203,10 @@ GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ NV12, I420 }"));
 #else
 static GstStaticCaps sink_caps =
 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE
-    ("{ AYUV64, UYVY, NV12, I420, RGBA64_LE, ARGB64_BE }"));
+    ("{ AYUV64, UYVY, NV12, I420, ARGB64_BE }"));
 #endif
 
+
 static void
 gst_vtenc_base_init (GstVTEncClass * klass)
 {
@@ -229,9 +230,14 @@ gst_vtenc_base_init (GstVTEncClass * klass)
   g_free (longname);
   g_free (description);
 
-  gst_element_class_add_pad_template (element_class,
-      gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
-          gst_static_caps_get (&sink_caps));
+  {
+    GstCaps *caps = gst_static_caps_get (&sink_caps);
+    /* RGBA64_LE is kCVPixelFormatType_64RGBALE, only available on macOS 11.3+ */
+    if (GST_VTUTIL_HAVE_64ARGBALE)
+      caps = gst_vtutil_caps_append_video_format (caps, "RGBA64_LE");
+    gst_element_class_add_pad_template (element_class,
+        gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, caps));
+  }
 
 
   src_caps = gst_caps_new_simple (codec_details->mimetype,
@@ -1641,7 +1647,11 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
           pixel_format_type = kCVPixelFormatType_4444AYpCbCr16;
           break;
         case GST_VIDEO_FORMAT_RGBA64_LE:
-          pixel_format_type = kCVPixelFormatType_64RGBALE;
+          if (GST_VTUTIL_HAVE_64ARGBALE)
+            pixel_format_type = kCVPixelFormatType_64RGBALE;
+          else
+            /* Codepath will never be hit on macOS older than Big Sur (11.3) */
+            g_assert_not_reached ();
           break;
         case GST_VIDEO_FORMAT_I420:
           pixel_format_type = kCVPixelFormatType_420YpCbCr8Planar;

subprojects/gst-plugins-bad/sys/applemedia/vtutil.c

@@ -135,3 +135,27 @@ gst_vtutil_codec_type_to_prores_variant (CMVideoCodecType codec_type)
       g_assert_not_reached ();
   }
 }
+
+GstCaps *
+gst_vtutil_caps_append_video_format (GstCaps * caps, const char *vfmt)
+{
+  GstStructure *s;
+  GValueArray *arr;
+  GValue val = G_VALUE_INIT;
+
+  caps = gst_caps_make_writable (caps);
+  s = gst_caps_get_structure (caps, 0);
+  gst_structure_get_list (s, "format", &arr);
+
+  g_value_init (&val, G_TYPE_STRING);
+
+  g_value_set_string (&val, vfmt);
+  G_GNUC_BEGIN_IGNORE_DEPRECATIONS;
+  arr = g_value_array_append (arr, &val);
+  G_GNUC_END_IGNORE_DEPRECATIONS;
+
+  g_value_unset (&val);
+
+  gst_structure_set_list (s, "format", arr);
+  return caps;
+}

subprojects/gst-plugins-bad/sys/applemedia/vtutil.h

@@ -21,6 +21,7 @@
 #define __GST_VTUTIL_H__
 
 #include <glib.h>
+#include <gst/gst.h>
 #include <CoreFoundation/CoreFoundation.h>
 #include <CoreMedia/CoreMedia.h>
 
@@ -29,6 +30,13 @@
  * each variant, so we use a dummy type for details->format_id */
 #define GST_kCMVideoCodecType_Some_AppleProRes  1
 
+// kCVPixelFormatType_64RGBALE is only available for 11.3 +.
+// See https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers/kcvpixelformattype_64rgbale
+#if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 110300
+#define kCVPixelFormatType_64RGBALE 'l64r'
+#endif
+#define GST_VTUTIL_HAVE_64ARGBALE __builtin_available(macOS 11.3, *)
+
 G_BEGIN_DECLS
 
 gchar * gst_vtutil_object_to_string (CFTypeRef obj);
@@ -47,6 +55,9 @@ void gst_vtutil_dict_set_object (CFMutableDictionaryRef dict,
 CMVideoCodecType gst_vtutil_codec_type_from_prores_variant (const char * variant);
 const char * gst_vtutil_codec_type_to_prores_variant (CMVideoCodecType codec_type);
 
+GstCaps * gst_vtutil_caps_append_video_format (GstCaps * caps,
+                                               const char * vfmt);
+
 G_END_DECLS
 
 #endif /* __GST_VTUTIL_H__ */