applemedia: Add ProRes support to vtenc and vtdec

For vtdec, we continue to prefer NV12; else we pick whatever
downstream wants. In the special case where we're decoding 10-bit or
12-bit ProRes formats, we will prefer AYUV64.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1214>

subprojects/gst-plugins-bad/sys/applemedia/coremediabuffer.h

@@ -24,7 +24,7 @@
 #include <gst/video/gstvideometa.h>
 #include "videotexturecache.h"
 
-#include "CoreMedia/CoreMedia.h"
+#include <CoreMedia/CoreMedia.h>
 
 G_BEGIN_DECLS
 

subprojects/gst-plugins-bad/sys/applemedia/vtdec.c

@@ -57,6 +57,7 @@ enum
   /* leave some headroom for new GstVideoCodecFrameFlags flags */
   VTDEC_FRAME_FLAG_SKIP = (1 << 10),
   VTDEC_FRAME_FLAG_DROP = (1 << 11),
+  VTDEC_FRAME_FLAG_ERROR = (1 << 12),
 };
 
 static void gst_vtdec_finalize (GObject * object);
@@ -101,7 +102,9 @@ static GstStaticPadTemplate gst_vtdec_sink_template =
     GST_STATIC_CAPS ("video/x-h264, stream-format=avc, alignment=au,"
         " width=(int)[1, MAX], height=(int)[1, MAX];"
         "video/mpeg, mpegversion=2, systemstream=false, parsed=true;"
-        "image/jpeg")
+        "image/jpeg;"
+        "video/x-prores, variant = { (string)standard, (string)hq, (string)lt,"
+        " (string)proxy, (string)4444, (string)4444xq };")
     );
 
 /* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */
@@ -114,20 +117,20 @@ const CFStringRef
 CFSTR ("RequireHardwareAcceleratedVideoDecoder");
 #endif
 
-#if defined(APPLEMEDIA_MOLTENVK)
+#define VIDEO_SRC_CAPS_FORMATS "{ NV12, AYUV64 }"
-#define VIDEO_SRC_CAPS \
+
-    GST_VIDEO_CAPS_MAKE("NV12") ";"                                     \
+#define VIDEO_SRC_CAPS_NATIVE                                           \
+    GST_VIDEO_CAPS_MAKE(VIDEO_SRC_CAPS_FORMATS) ";"                     \
     GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
-        "NV12") ", "                                                    \
+        VIDEO_SRC_CAPS_FORMATS) ", "                                    \
-    "texture-target = (string) rectangle ; "                            \
-    GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,\
-        "NV12")
-#else
-#define VIDEO_SRC_CAPS \
-    GST_VIDEO_CAPS_MAKE("NV12") ";"                                     \
-    GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
-        "NV12") ", "                                                    \
     "texture-target = (string) rectangle "
+
+#if defined(APPLEMEDIA_MOLTENVK)
+#define VIDEO_SRC_CAPS VIDEO_SRC_CAPS_NATIVE "; "                           \
+    GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, \
+        VIDEO_SRC_CAPS_FORMATS)
+#else
+#define VIDEO_SRC_CAPS VIDEO_SRC_CAPS_NATIVE
 #endif
 
 G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER);
@@ -234,24 +237,54 @@ gst_vtdec_stop (GstVideoDecoder * decoder)
 }
 
 static void
-setup_texture_cache (GstVtdec * vtdec)
+setup_texture_cache (GstVtdec * vtdec, GstVideoFormat format)
 {
   GstVideoCodecState *output_state;
 
+  GST_INFO_OBJECT (vtdec, "setting up texture cache");
   output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
-  gst_video_texture_cache_set_format (vtdec->texture_cache,
+  gst_video_texture_cache_set_format (vtdec->texture_cache, format,
-      GST_VIDEO_FORMAT_NV12, output_state->caps);
+      output_state->caps);
   gst_video_codec_state_unref (output_state);
 }
 
+/*
+ * Unconditionally output a high bit-depth + alpha format when decoding Apple
+ * ProRes video if downstream supports it.
+ * TODO: read src_pix_fmt to get the preferred output format
+ * https://wiki.multimedia.cx/index.php/Apple_ProRes#Frame_header
+ */
+static GstVideoFormat
+get_preferred_video_format (GstStructure * s, gboolean prores)
+{
+  const GValue *list = gst_structure_get_value (s, "format");
+  guint i, size = gst_value_list_get_size (list);
+  for (i = 0; i < size; i++) {
+    const GValue *value = gst_value_list_get_value (list, i);
+    const char *fmt = g_value_get_string (value);
+    GstVideoFormat vfmt = gst_video_format_from_string (fmt);
+    switch (vfmt) {
+      case GST_VIDEO_FORMAT_NV12:
+        if (!prores)
+          return vfmt;
+        break;
+      case GST_VIDEO_FORMAT_AYUV64:
+        if (prores)
+          return vfmt;
+        break;
+      default:
+        break;
+    }
+  }
+  return GST_VIDEO_FORMAT_UNKNOWN;
+}
+
 static gboolean
 gst_vtdec_negotiate (GstVideoDecoder * decoder)
 {
   GstVideoCodecState *output_state = NULL;
   GstCaps *peercaps = NULL, *caps = NULL, *templcaps = NULL, *prevcaps = NULL;
-  GstVideoFormat format;
+  GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN;
-  GstStructure *structure;
-  const gchar *s;
   GstVtdec *vtdec;
   OSStatus err = noErr;
   GstCapsFeatures *features = NULL;
@@ -290,9 +323,30 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
   gst_caps_unref (peercaps);
 
   caps = gst_caps_truncate (gst_caps_make_writable (caps));
-  structure = gst_caps_get_structure (caps, 0);
+
-  s = gst_structure_get_string (structure, "format");
+  /* Try to use whatever video format downstream prefers */
-  format = gst_video_format_from_string (s);
+  {
+    GstStructure *s = gst_caps_get_structure (caps, 0);
+
+    if (gst_structure_has_field_typed (s, "format", GST_TYPE_LIST)) {
+      GstStructure *is = gst_caps_get_structure (vtdec->input_state->caps, 0);
+      const char *name = gst_structure_get_name (is);
+      format = get_preferred_video_format (s,
+          g_strcmp0 (name, "video/x-prores") == 0);
+    }
+
+    if (format == GST_VIDEO_FORMAT_UNKNOWN) {
+      const char *fmt;
+      gst_structure_fixate_field (s, "format");
+      fmt = gst_structure_get_string (s, "format");
+      if (fmt)
+        format = gst_video_format_from_string (fmt);
+      else
+        /* If all fails, just use NV12 */
+        format = GST_VIDEO_FORMAT_NV12;
+    }
+  }
+
   features = gst_caps_get_features (caps, 0);
   if (features)
     features = gst_caps_features_copy (features);
@@ -383,7 +437,7 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
       if (!vtdec->texture_cache) {
         vtdec->texture_cache =
             gst_video_texture_cache_gl_new (vtdec->ctxh->context);
-        setup_texture_cache (vtdec);
+        setup_texture_cache (vtdec, format);
       }
     }
 #if defined(APPLEMEDIA_MOLTENVK)
@@ -420,7 +474,7 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
       if (!vtdec->texture_cache) {
         vtdec->texture_cache =
             gst_video_texture_cache_vulkan_new (vtdec->device);
-        setup_texture_cache (vtdec);
+        setup_texture_cache (vtdec, format);
       }
     }
 #endif
@@ -454,6 +508,16 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
     cm_format = kCMVideoCodecType_MPEG2Video;
   } else if (!strcmp (caps_name, "image/jpeg")) {
     cm_format = kCMVideoCodecType_JPEG;
+  } else if (!strcmp (caps_name, "video/x-prores")) {
+    const char *variant = gst_structure_get_string (structure, "variant");
+
+    if (variant)
+      cm_format = gst_vtutil_codec_type_from_prores_variant (variant);
+
+    if (cm_format == GST_kCMVideoCodecType_Some_AppleProRes) {
+      GST_ERROR_OBJECT (vtdec, "Invalid ProRes variant %s", variant);
+      return FALSE;
+    }
   }
 
   if (cm_format == kCMVideoCodecType_H264 && state->codec_data == NULL) {
@@ -584,6 +648,13 @@ gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format,
     case GST_VIDEO_FORMAT_NV12:
       cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
       break;
+    case GST_VIDEO_FORMAT_AYUV64:
+/* This is fine for now because Apple only ships LE devices */
+#if G_BYTE_ORDER != G_LITTLE_ENDIAN
+#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
+#endif
+      cv_format = kCVPixelFormatType_4444AYpCbCr16;
+      break;
     default:
       g_warn_if_reached ();
       break;
@@ -923,13 +994,17 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
      * example) or we're draining/flushing
      */
     if (frame) {
-      if (flush || frame->flags & VTDEC_FRAME_FLAG_SKIP)
+      if (frame->flags & VTDEC_FRAME_FLAG_ERROR) {
         gst_video_decoder_release_frame (decoder, frame);
-      else if (frame->flags & VTDEC_FRAME_FLAG_DROP)
+        ret = GST_FLOW_ERROR;
+      } else if (flush || frame->flags & VTDEC_FRAME_FLAG_SKIP) {
+        gst_video_decoder_release_frame (decoder, frame);
+      } else if (frame->flags & VTDEC_FRAME_FLAG_DROP) {
         gst_video_decoder_drop_frame (decoder, frame);
-      else
+      } else {
         ret = gst_video_decoder_finish_frame (decoder, frame);
       }
+    }
 
     if (!frame || ret != GST_FLOW_OK)
       break;

subprojects/gst-plugins-bad/sys/applemedia/vtenc.c

@@ -151,7 +151,7 @@ static GstStaticCaps sink_caps =
 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ NV12, I420 }"));
 #else
 static GstStaticCaps sink_caps =
-GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ UYVY, NV12, I420 }"));
+GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ AYUV64, UYVY, NV12, I420 }"));
 #endif
 
 static void
@@ -187,10 +187,41 @@ gst_vtenc_base_init (GstVTEncClass * klass)
       "height", GST_TYPE_INT_RANGE, min_height, max_height,
       "framerate", GST_TYPE_FRACTION_RANGE,
       min_fps_n, min_fps_d, max_fps_n, max_fps_d, NULL);
-  if (codec_details->format_id == kCMVideoCodecType_H264) {
+  switch (codec_details->format_id) {
+    case kCMVideoCodecType_H264:
       gst_structure_set (gst_caps_get_structure (src_caps, 0),
           "stream-format", G_TYPE_STRING, "avc",
           "alignment", G_TYPE_STRING, "au", NULL);
+      break;
+    case GST_kCMVideoCodecType_Some_AppleProRes:
+      if (g_strcmp0 (codec_details->mimetype, "video/x-prores") == 0) {
+        G_GNUC_BEGIN_IGNORE_DEPRECATIONS;
+        GValueArray *arr = g_value_array_new (6);
+        GValue val = G_VALUE_INIT;
+
+        g_value_init (&val, G_TYPE_STRING);
+        g_value_set_string (&val, "standard");
+        arr = g_value_array_append (arr, &val);
+        g_value_set_string (&val, "4444xq");
+        arr = g_value_array_append (arr, &val);
+        g_value_set_string (&val, "4444");
+        arr = g_value_array_append (arr, &val);
+        g_value_set_string (&val, "hq");
+        arr = g_value_array_append (arr, &val);
+        g_value_set_string (&val, "lt");
+        arr = g_value_array_append (arr, &val);
+        g_value_set_string (&val, "proxy");
+        arr = g_value_array_append (arr, &val);
+        gst_structure_set_list (gst_caps_get_structure (src_caps, 0),
+            "variant", arr);
+        g_value_array_free (arr);
+        g_value_unset (&val);
+        G_GNUC_END_IGNORE_DEPRECATIONS;
+        break;
+      }
+      /* fall through */
+    default:
+      g_assert_not_reached ();
   }
   src_template = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
       src_caps);
@@ -609,7 +640,8 @@ gst_vtenc_profile_level_key (GstVTEnc * self, const gchar * profile,
   } else if (!strcmp (profile, "main")) {
     profile = "Main";
   } else {
-    g_assert_not_reached ();
+    GST_ERROR_OBJECT (self, "invalid profile: %s", profile);
+    return ret;
   }
 
   if (strlen (level) == 1) {
@@ -631,13 +663,45 @@ gst_vtenc_profile_level_key (GstVTEnc * self, const gchar * profile,
 }
 
 static gboolean
-gst_vtenc_negotiate_profile_and_level (GstVideoEncoder * enc)
+gst_vtenc_negotiate_profile_and_level (GstVTEnc * self, GstStructure * s)
+{
+  const gchar *profile = gst_structure_get_string (s, "profile");
+  const gchar *level = gst_structure_get_string (s, "level");
+
+  if (self->profile_level)
+    CFRelease (self->profile_level);
+  self->profile_level = gst_vtenc_profile_level_key (self, profile, level);
+  if (self->profile_level == NULL) {
+    GST_ERROR_OBJECT (self, "unsupported h264 profile '%s' or level '%s'",
+        profile, level);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_vtenc_negotiate_prores_variant (GstVTEnc * self, GstStructure * s)
+{
+  const char *variant = gst_structure_get_string (s, "variant");
+  CMVideoCodecType codec_type =
+      gst_vtutil_codec_type_from_prores_variant (variant);
+
+  if (codec_type == GST_kCMVideoCodecType_Some_AppleProRes) {
+    GST_ERROR_OBJECT (self, "unsupported prores variant: %s", variant);
+    return FALSE;
+  }
+
+  self->specific_format_id = codec_type;
+  return TRUE;
+}
+
+static gboolean
+gst_vtenc_negotiate_specific_format_details (GstVideoEncoder * enc)
 {
   GstVTEnc *self = GST_VTENC_CAST (enc);
   GstCaps *allowed_caps = NULL;
   gboolean ret = TRUE;
-  const gchar *profile = NULL;
-  const gchar *level = NULL;
 
   allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SRC_PAD (enc));
   if (allowed_caps) {
@@ -651,17 +715,24 @@ gst_vtenc_negotiate_profile_and_level (GstVideoEncoder * enc)
     allowed_caps = gst_caps_make_writable (allowed_caps);
     allowed_caps = gst_caps_fixate (allowed_caps);
     s = gst_caps_get_structure (allowed_caps, 0);
-
+    switch (self->details->format_id) {
-    profile = gst_structure_get_string (s, "profile");
+      case kCMVideoCodecType_H264:
-    level = gst_structure_get_string (s, "level");
+        self->specific_format_id = kCMVideoCodecType_H264;
-  }
+        if (!gst_vtenc_negotiate_profile_and_level (self, s))
-
-  if (self->profile_level)
-    CFRelease (self->profile_level);
-  self->profile_level = gst_vtenc_profile_level_key (self, profile, level);
-  if (self->profile_level == NULL) {
-    GST_ERROR_OBJECT (enc, "invalid profile and level");
           goto fail;
+        break;
+      case GST_kCMVideoCodecType_Some_AppleProRes:
+        if (g_strcmp0 (self->details->mimetype, "video/x-prores") != 0) {
+          GST_ERROR_OBJECT (self, "format_id == %i mimetype must be Apple "
+              "ProRes", GST_kCMVideoCodecType_Some_AppleProRes);
+          goto fail;
+        }
+        if (!gst_vtenc_negotiate_prores_variant (self, s))
+          goto fail;
+        break;
+      default:
+        g_assert_not_reached ();
+    }
   }
 
 out:
@@ -695,7 +766,7 @@ gst_vtenc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state)
   gst_vtenc_destroy_session (self, &self->session);
   GST_OBJECT_UNLOCK (self);
 
-  gst_vtenc_negotiate_profile_and_level (enc);
+  gst_vtenc_negotiate_specific_format_details (enc);
 
   session = gst_vtenc_create_session (self);
   GST_OBJECT_LOCK (self);
@@ -735,7 +806,9 @@ gst_vtenc_negotiate_downstream (GstVTEnc * self, CMSampleBufferRef sbuf)
       "framerate", GST_TYPE_FRACTION,
       self->negotiated_fps_n, self->negotiated_fps_d, NULL);
 
-  if (self->details->format_id == kCMVideoCodecType_H264) {
+  switch (self->details->format_id) {
+    case kCMVideoCodecType_H264:
+    {
       CMFormatDescriptionRef fmt;
       CFDictionaryRef atoms;
       CFStringRef avccKey;
@@ -756,7 +829,8 @@ gst_vtenc_negotiate_downstream (GstVTEnc * self, CMSampleBufferRef sbuf)
       CFDataGetBytes (avcc, CFRangeMake (0, codec_data_size), codec_data);
       codec_data_buf = gst_buffer_new_wrapped (codec_data, codec_data_size);
 
-    gst_structure_set (s, "codec_data", GST_TYPE_BUFFER, codec_data_buf, NULL);
+      gst_structure_set (s, "codec_data", GST_TYPE_BUFFER, codec_data_buf,
+          NULL);
 
       sps[0] = codec_data[1];
       sps[1] = codec_data[2] & ~0xDF;
@@ -766,6 +840,15 @@ gst_vtenc_negotiate_downstream (GstVTEnc * self, CMSampleBufferRef sbuf)
 
       gst_buffer_unref (codec_data_buf);
     }
+      break;
+    case GST_kCMVideoCodecType_Some_AppleProRes:
+      gst_structure_set (s, "variant", G_TYPE_STRING,
+          gst_vtutil_codec_type_to_prores_variant (self->specific_format_id),
+          NULL);
+      break;
+    default:
+      g_assert_not_reached ();
+  }
 
   state =
       gst_video_encoder_set_output_state (GST_VIDEO_ENCODER_CAST (self), caps,
@@ -825,7 +908,7 @@ static VTCompressionSessionRef
 gst_vtenc_create_session (GstVTEnc * self)
 {
   VTCompressionSessionRef session = NULL;
-  CFMutableDictionaryRef encoder_spec = NULL, pb_attrs;
+  CFMutableDictionaryRef encoder_spec = NULL, pb_attrs = NULL;
   OSStatus status;
 
 #if !HAVE_IOS
@@ -843,16 +926,21 @@ gst_vtenc_create_session (GstVTEnc * self)
         TRUE);
 #endif
 
-  pb_attrs = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
+  if (self->profile_level) {
-      &kCFTypeDictionaryValueCallBacks);
+    pb_attrs = CFDictionaryCreateMutable (NULL, 0,
+        &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
     gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferWidthKey,
         self->negotiated_width);
     gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferHeightKey,
         self->negotiated_height);
+  }
+
+  /* This was set in gst_vtenc_negotiate_specific_format_details() */
+  g_assert_cmpint (self->specific_format_id, !=, 0);
 
   status = VTCompressionSessionCreate (NULL,
       self->negotiated_width, self->negotiated_height,
-      self->details->format_id, encoder_spec, pb_attrs, NULL,
+      self->specific_format_id, encoder_spec, pb_attrs, NULL,
       gst_vtenc_enqueue_buffer, self, &session);
   GST_INFO_OBJECT (self, "VTCompressionSessionCreate for %d x %d => %d",
       self->negotiated_width, self->negotiated_height, (int) status);
@@ -862,9 +950,13 @@ gst_vtenc_create_session (GstVTEnc * self)
     goto beach;
   }
 
+  if (self->profile_level) {
     gst_vtenc_session_configure_expected_framerate (self, session,
         (gdouble) self->negotiated_fps_n / (gdouble) self->negotiated_fps_d);
 
+    /*
+     * https://developer.apple.com/documentation/videotoolbox/vtcompressionsession/compression_properties/profile_and_level_constants
+     */
     status = VTSessionSetProperty (session,
         kVTCompressionPropertyKey_ProfileLevel, self->profile_level);
     GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_ProfileLevel => %d",
@@ -873,7 +965,8 @@ gst_vtenc_create_session (GstVTEnc * self)
     status = VTSessionSetProperty (session,
         kVTCompressionPropertyKey_AllowTemporalCompression, kCFBooleanTrue);
     GST_DEBUG_OBJECT (self,
-      "kVTCompressionPropertyKey_AllowTemporalCompression => %d", (int) status);
+        "kVTCompressionPropertyKey_AllowTemporalCompression => %d",
+        (int) status);
 
     gst_vtenc_session_configure_max_keyframe_interval (self, session,
         self->max_keyframe_interval);
@@ -882,6 +975,8 @@ gst_vtenc_create_session (GstVTEnc * self)
 
     gst_vtenc_session_configure_bitrate (self, session,
         gst_vtenc_get_bitrate (self));
+  }
+
   gst_vtenc_session_configure_realtime (self, session,
       gst_vtenc_get_realtime (self));
   gst_vtenc_session_configure_allow_frame_reordering (self, session,
@@ -906,6 +1001,7 @@ gst_vtenc_create_session (GstVTEnc * self)
 beach:
   if (encoder_spec)
     CFRelease (encoder_spec);
+  if (pb_attrs)
     CFRelease (pb_attrs);
 
   return session;
@@ -1231,6 +1327,13 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
       }
 
       switch (GST_VIDEO_INFO_FORMAT (&self->video_info)) {
+        case GST_VIDEO_FORMAT_AYUV64:
+/* This is fine for now because Apple only ships LE devices */
+#if G_BYTE_ORDER != G_LITTLE_ENDIAN
+#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
+#endif
+          pixel_format_type = kCVPixelFormatType_4444AYpCbCr16;
+          break;
         case GST_VIDEO_FORMAT_I420:
           pixel_format_type = kCVPixelFormatType_420YpCbCr8Planar;
           break;
@@ -1256,7 +1359,8 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
           plane_bytes_per_row, gst_pixel_buffer_release_cb, vframe, NULL,
           &pbuf);
       if (cv_ret != kCVReturnSuccess) {
-        GST_ERROR_OBJECT (self, "CVPixelBufferCreateWithPlanarBytes failed: %i", cv_ret);
+        GST_ERROR_OBJECT (self, "CVPixelBufferCreateWithPlanarBytes failed: %i",
+            cv_ret);
         gst_vtenc_frame_free (vframe);
         goto cv_error;
       }
@@ -1465,6 +1569,8 @@ static const GstVTEncoderDetails gst_vtenc_codecs[] = {
 #ifndef HAVE_IOS
   {"H.264 (HW only)", "h264_hw", "video/x-h264", kCMVideoCodecType_H264, TRUE},
 #endif
+  {"Apple ProRes", "prores", "video/x-prores",
+      GST_kCMVideoCodecType_Some_AppleProRes, FALSE},
 };
 
 void

subprojects/gst-plugins-bad/sys/applemedia/vtenc.h

@@ -58,6 +58,7 @@ struct _GstVTEnc
 
   const GstVTEncoderDetails * details;
 
+  CMVideoCodecType specific_format_id;
   CFStringRef profile_level;
   guint bitrate;
   gboolean allow_frame_reordering;

subprojects/gst-plugins-bad/sys/applemedia/vtutil.c

@@ -96,3 +96,42 @@ gst_vtutil_dict_set_object (CFMutableDictionaryRef dict, CFStringRef key,
   CFDictionarySetValue (dict, key, value);
   CFRelease (value);
 }
+
+CMVideoCodecType
+gst_vtutil_codec_type_from_prores_variant (const char *variant)
+{
+  if (g_strcmp0 (variant, "standard") == 0)
+    return kCMVideoCodecType_AppleProRes422;
+  else if (g_strcmp0 (variant, "4444xq") == 0)
+    return kCMVideoCodecType_AppleProRes4444XQ;
+  else if (g_strcmp0 (variant, "4444") == 0)
+    return kCMVideoCodecType_AppleProRes4444;
+  else if (g_strcmp0 (variant, "hq") == 0)
+    return kCMVideoCodecType_AppleProRes422HQ;
+  else if (g_strcmp0 (variant, "lt") == 0)
+    return kCMVideoCodecType_AppleProRes422LT;
+  else if (g_strcmp0 (variant, "proxy") == 0)
+    return kCMVideoCodecType_AppleProRes422Proxy;
+  return GST_kCMVideoCodecType_Some_AppleProRes;
+}
+
+const char *
+gst_vtutil_codec_type_to_prores_variant (CMVideoCodecType codec_type)
+{
+  switch (codec_type) {
+    case kCMVideoCodecType_AppleProRes422:
+      return "standard";
+    case kCMVideoCodecType_AppleProRes4444XQ:
+      return "4444xq";
+    case kCMVideoCodecType_AppleProRes4444:
+      return "4444";
+    case kCMVideoCodecType_AppleProRes422HQ:
+      return "hq";
+    case kCMVideoCodecType_AppleProRes422LT:
+      return "lt";
+    case kCMVideoCodecType_AppleProRes422Proxy:
+      return "proxy";
+    default:
+      g_assert_not_reached ();
+  }
+}

subprojects/gst-plugins-bad/sys/applemedia/vtutil.h

@@ -22,6 +22,12 @@
 
 #include <glib.h>
 #include <CoreFoundation/CoreFoundation.h>
+#include <CoreMedia/CoreMedia.h>
+
+/* Some formats such as Apple ProRes have separate codec type mappings for all
+ * variants / profiles, and we don't want to instantiate separate elements for
+ * each variant, so we use a dummy type for details->format_id */
+#define GST_kCMVideoCodecType_Some_AppleProRes  1
 
 G_BEGIN_DECLS
 
@@ -38,6 +44,9 @@ void gst_vtutil_dict_set_data (CFMutableDictionaryRef dict,
 void gst_vtutil_dict_set_object (CFMutableDictionaryRef dict,
     CFStringRef key, CFTypeRef * value);
 
+CMVideoCodecType gst_vtutil_codec_type_from_prores_variant (const char * variant);
+const char * gst_vtutil_codec_type_to_prores_variant (CMVideoCodecType codec_type);
+
 G_END_DECLS
 
 #endif /* __GST_VTUTIL_H__ */