applemedia: Consolidate GstVideoFormat <-> CVPixelFormat conversion

In the process we have changed the color value range from video-range
to full-range, which is probably what people want in the first place.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4135>

subprojects/gst-plugins-bad/sys/applemedia/helpers.c

@@ -68,6 +68,10 @@ gst_video_format_to_cvpixelformat (GstVideoFormat fmt)
       return kCVPixelFormatType_422YpCbCr8_yuvs;
       /* Alpha YUV */
     case GST_VIDEO_FORMAT_AYUV64:
+/* This is fine for now because Apple only ships LE devices */
+#if G_BYTE_ORDER != G_LITTLE_ENDIAN
+#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
+#endif
       return kCVPixelFormatType_4444AYpCbCr16;
       /* RGB formats */
     case GST_VIDEO_FORMAT_ARGB:

subprojects/gst-plugins-bad/sys/applemedia/helpers.h

@@ -24,6 +24,13 @@
 
 G_BEGIN_DECLS
 
+// kCVPixelFormatType_64RGBALE is only available for 11.3 +.
+// See https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers/kcvpixelformattype_64rgbale
+#if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 110300
+#define kCVPixelFormatType_64RGBALE 'l64r'
+#endif
+#define GST_APPLEMEDIA_HAVE_64RGBALE __builtin_available(macOS 11.3, *)
+
 #define GST_CVPIXELFORMAT_FOURCC_ARGS(fourcc) \
   __GST_PRINT_CHAR(((fourcc) >> 24) & 0xff),  \
   __GST_PRINT_CHAR(((fourcc) >> 16) & 0xff),  \

subprojects/gst-plugins-bad/sys/applemedia/vtdec.c

@@ -57,6 +57,7 @@
 #include <gst/gl/gstglcontext.h>
 #include "vtdec.h"
 #include "vtutil.h"
+#include "helpers.h"
 #include "corevideobuffer.h"
 #include "coremediabuffer.h"
 #include "videotexturecache-gl.h"
@@ -169,7 +170,7 @@ gst_vtdec_class_init (GstVtdecClass * klass)
   {
     GstCaps *caps = gst_caps_from_string (VIDEO_SRC_CAPS);
     /* RGBA64_LE is kCVPixelFormatType_64RGBALE, only available on macOS 11.3+ */
-    if (GST_VTUTIL_HAVE_64RGBALE)
+    if (GST_APPLEMEDIA_HAVE_64RGBALE)
       caps = gst_vtutil_caps_append_video_format (caps, "RGBA64_LE");
     gst_element_class_add_pad_template (element_class,
         gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps));
@@ -299,7 +300,7 @@ get_preferred_video_format (GstStructure * s, gboolean prores)
           return vfmt;
         break;
       case GST_VIDEO_FORMAT_RGBA64_LE:
-        if (GST_VTUTIL_HAVE_64RGBALE) {
+        if (GST_APPLEMEDIA_HAVE_64RGBALE) {
           if (prores)
             return vfmt;
         } else {
@@ -678,36 +679,10 @@ gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format,
   VTDecompressionOutputCallbackRecord callback;
   CFMutableDictionaryRef videoDecoderSpecification;
   OSStatus status;
-  guint32 cv_format = 0;
+  guint32 cv_format = gst_video_format_to_cvpixelformat (format);
 
   g_return_val_if_fail (vtdec->session == NULL, FALSE);
 
-  switch (format) {
-    case GST_VIDEO_FORMAT_NV12:
-      cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
-      break;
-    case GST_VIDEO_FORMAT_AYUV64:
-/* This is fine for now because Apple only ships LE devices */
-#if G_BYTE_ORDER != G_LITTLE_ENDIAN
-#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
-#endif
-      cv_format = kCVPixelFormatType_4444AYpCbCr16;
-      break;
-    case GST_VIDEO_FORMAT_ARGB64_BE:
-      cv_format = kCVPixelFormatType_64ARGB;
-      break;
-    case GST_VIDEO_FORMAT_RGBA64_LE:
-      if (GST_VTUTIL_HAVE_64RGBALE)
-        cv_format = kCVPixelFormatType_64RGBALE;
-      else
-        /* Codepath will never be hit on macOS older than Big Sur (11.3) */
-        g_warn_if_reached ();
-      break;
-    default:
-      g_warn_if_reached ();
-      break;
-  }
-
   videoDecoderSpecification =
       CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
       &kCFTypeDictionaryValueCallBacks);

subprojects/gst-plugins-bad/sys/applemedia/vtenc.c

@@ -69,6 +69,7 @@
 #include "coremediabuffer.h"
 #include "corevideobuffer.h"
 #include "vtutil.h"
+#include "helpers.h"
 #include <gst/pbutils/codec-utils.h>
 #include <sys/sysctl.h>
 
@@ -258,7 +259,7 @@ gst_vtenc_base_init (GstVTEncClass * klass)
     if (enable_argb) {
       caps = gst_vtutil_caps_append_video_format (caps, "ARGB64_BE");
       /* RGBA64_LE is kCVPixelFormatType_64RGBALE, only available on macOS 11.3+ */
-      if (GST_VTUTIL_HAVE_64RGBALE)
+      if (GST_APPLEMEDIA_HAVE_64RGBALE)
         caps = gst_vtutil_caps_append_video_format (caps, "RGBA64_LE");
     }
 #endif
@@ -1644,25 +1645,15 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
   if (pbuf == NULL) {
     GstVideoFrame inframe, outframe;
     GstBuffer *outbuf;
-    OSType pixel_format_type;
     CVReturn cv_ret;
+    OSType pixel_format_type =
+        gst_video_format_to_cvpixelformat (GST_VIDEO_INFO_FORMAT
+        (&self->video_info));
 
     /* FIXME: iOS has special stride requirements that we don't know yet.
      * Copy into a newly allocated pixelbuffer for now. Probably makes
      * sense to create a buffer pool around these at some point.
      */
-
-    switch (GST_VIDEO_INFO_FORMAT (&self->video_info)) {
-      case GST_VIDEO_FORMAT_I420:
-        pixel_format_type = kCVPixelFormatType_420YpCbCr8Planar;
-        break;
-      case GST_VIDEO_FORMAT_NV12:
-        pixel_format_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
-        break;
-      default:
-        g_assert_not_reached ();
-    }
-
     if (!gst_video_frame_map (&inframe, &self->video_info, frame->input_buffer,
             GST_MAP_READ)) {
       GST_ERROR_OBJECT (self, "failed to map input buffer");
@@ -1714,12 +1705,14 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
     }
 
     {
+      OSType pixel_format_type =
+          gst_video_format_to_cvpixelformat (GST_VIDEO_INFO_FORMAT
+          (&self->video_info));
       const size_t num_planes = GST_VIDEO_FRAME_N_PLANES (&vframe->videoframe);
       void *plane_base_addresses[GST_VIDEO_MAX_PLANES];
       size_t plane_widths[GST_VIDEO_MAX_PLANES];
       size_t plane_heights[GST_VIDEO_MAX_PLANES];
       size_t plane_bytes_per_row[GST_VIDEO_MAX_PLANES];
-      OSType pixel_format_type;
       size_t i;
 
       for (i = 0; i < num_planes; i++) {
@@ -1733,37 +1726,6 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
             GST_VIDEO_FRAME_COMP_STRIDE (&vframe->videoframe, i);
       }
 
-      switch (GST_VIDEO_INFO_FORMAT (&self->video_info)) {
-        case GST_VIDEO_FORMAT_ARGB64_BE:
-          pixel_format_type = kCVPixelFormatType_64ARGB;
-          break;
-        case GST_VIDEO_FORMAT_AYUV64:
-/* This is fine for now because Apple only ships LE devices */
-#if G_BYTE_ORDER != G_LITTLE_ENDIAN
-#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
-#endif
-          pixel_format_type = kCVPixelFormatType_4444AYpCbCr16;
-          break;
-        case GST_VIDEO_FORMAT_RGBA64_LE:
-          if (GST_VTUTIL_HAVE_64RGBALE)
-            pixel_format_type = kCVPixelFormatType_64RGBALE;
-          else
-            /* Codepath will never be hit on macOS older than Big Sur (11.3) */
-            g_assert_not_reached ();
-          break;
-        case GST_VIDEO_FORMAT_I420:
-          pixel_format_type = kCVPixelFormatType_420YpCbCr8Planar;
-          break;
-        case GST_VIDEO_FORMAT_NV12:
-          pixel_format_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
-          break;
-        case GST_VIDEO_FORMAT_UYVY:
-          pixel_format_type = kCVPixelFormatType_422YpCbCr8;
-          break;
-        default:
-          g_assert_not_reached ();
-      }
-
       cv_ret = CVPixelBufferCreateWithPlanarBytes (NULL,
           self->negotiated_width, self->negotiated_height,
           pixel_format_type,

subprojects/gst-plugins-bad/sys/applemedia/vtutil.h

@@ -30,13 +30,6 @@
  * each variant, so we use a dummy type for details->format_id */
 #define GST_kCMVideoCodecType_Some_AppleProRes  1
 
-// kCVPixelFormatType_64RGBALE is only available for 11.3 +.
-// See https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers/kcvpixelformattype_64rgbale
-#if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 110300
-#define kCVPixelFormatType_64RGBALE 'l64r'
-#endif
-#define GST_VTUTIL_HAVE_64RGBALE __builtin_available(macOS 11.3, *)
-
 G_BEGIN_DECLS
 
 gchar * gst_vtutil_object_to_string (CFTypeRef obj);