vtdec: implement the GL texture upload meta

sys/applemedia/corevideotexturecache.m

@@ -26,12 +26,11 @@
 #endif
 #include "corevideotexturecache.h"
 #include "coremediabuffer.h"
+#include "corevideobuffer.h"
 
 GstCoreVideoTextureCache *
 gst_core_video_texture_cache_new (GstGLContext * ctx)
 {
-  CVReturn err;
-
   g_return_val_if_fail (ctx != NULL, NULL);
 
   GstCoreVideoTextureCache *cache = g_new0 (GstCoreVideoTextureCache, 1);
@@ -41,21 +40,21 @@ gst_core_video_texture_cache_new (GstGLContext * ctx)
   CGLPixelFormatAttribute attribs[1] = { 0 };
   int numPixelFormats;
   CGLPixelFormatObj pixelFormat;
-  CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5
+  CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats);       // 5
-  NSOpenGLContext *platform_ctx = (NSOpenGLContext *) gst_gl_context_get_gl_context (ctx);
+  NSOpenGLContext *platform_ctx =
-  err = CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL,
+      (NSOpenGLContext *) gst_gl_context_get_gl_context (ctx);
-      [platform_ctx CGLContextObj], pixelFormat,
+  CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL,
-      NULL, &cache->cache);
+      [platform_ctx CGLContextObj], pixelFormat, NULL, &cache->cache);
 #else
-  err = CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL,
+  CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL,
-      (CVEAGLContext) gst_gl_context_get_gl_context (ctx),
+      (CVEAGLContext) gst_gl_context_get_gl_context (ctx), NULL, &cache->cache);
-      NULL, &cache->cache);
 #endif
 
   return cache;
 }
 
-void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
+void
+gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
 {
   g_return_if_fail (cache != NULL);
 
@@ -69,11 +68,13 @@ void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
 }
 
 gboolean
-gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4])
+gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta,
+    guint texture_id[4])
 {
   g_return_val_if_fail (meta != NULL, FALSE);
 
-  GstCoreVideoTextureCache *cache = (GstCoreVideoTextureCache *) meta->user_data;
+  GstCoreVideoTextureCache *cache =
+      (GstCoreVideoTextureCache *) meta->user_data;
   const GstGLFuncs *gl = cache->ctx->gl_vtable;
 #if !HAVE_IOS
   CVOpenGLTextureRef texture = NULL;
@@ -81,16 +82,25 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint t
   CVOpenGLESTextureRef texture = NULL;
 #endif
   GstVideoMeta *video_meta = gst_buffer_get_video_meta (meta->buffer);
-  GstCoreMediaMeta *cv_meta = (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer,
+  GstCoreMediaMeta *cm_meta =
-          gst_core_media_meta_api_get_type ());
+      (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer,
+      gst_core_media_meta_api_get_type ());
+  GstCoreVideoMeta *cv_meta =
+      (GstCoreVideoMeta *) gst_buffer_get_meta (meta->buffer,
+      gst_core_video_meta_api_get_type ());
+  CVPixelBufferRef pixel_buf;
+  if (cm_meta)
+    pixel_buf = cm_meta->pixel_buf;
+  else
+    pixel_buf = cv_meta->pixbuf;
 #if !HAVE_IOS
   CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
-          cache->cache, cv_meta->pixel_buf, NULL, &texture);
+      cache->cache, pixel_buf, NULL, &texture);
 #else
   CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
-          cache->cache, cv_meta->image_buf, NULL, GL_TEXTURE_2D,
+      cache->cache, cm_meta->image_buf, NULL, GL_TEXTURE_2D,
-          GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE,
+      GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE,
-          0, &texture);
+      0, &texture);
 #endif
   GLuint fboId;
   gl->GenFramebuffers (1, &fboId);
@@ -98,13 +108,14 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint t
 
   gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
 #if !HAVE_IOS
-      CVOpenGLTextureGetTarget(texture), CVOpenGLTextureGetName(texture),
+      CVOpenGLTextureGetTarget (texture), CVOpenGLTextureGetName (texture),
 #else
-      CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture),
+      CVOpenGLESTextureGetTarget (texture), CVOpenGLESTextureGetName (texture),
 #endif
       0);
   gl->BindTexture (GL_TEXTURE_2D, texture_id[0]);
-  gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width, video_meta->height, 0);
+  gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width,
+      video_meta->height, 0);
   gl->BindTexture (GL_TEXTURE_2D, 0);
   gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
   gl->DeleteFramebuffers (1, &fboId);

sys/applemedia/vtdec.c

@@ -35,13 +35,15 @@
 #include "config.h"
 #endif
 
+#include <string.h>
 #include <gst/gst.h>
 #include <gst/video/video.h>
 #include <gst/video/gstvideodecoder.h>
+#include <gst/gl/gstglcontext.h>
 #include "vtdec.h"
 #include "vtutil.h"
 #include "corevideobuffer.h"
-#include <string.h>
+#include "coremediabuffer.h"
 
 GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category);
 #define GST_CAT_DEFAULT gst_vtdec_debug_category
@@ -50,6 +52,8 @@ static void gst_vtdec_finalize (GObject * object);
 
 static gboolean gst_vtdec_start (GstVideoDecoder * decoder);
 static gboolean gst_vtdec_stop (GstVideoDecoder * decoder);
+static gboolean gst_vtdec_decide_allocation (GstVideoDecoder * decoder,
+    GstQuery * query);
 static gboolean gst_vtdec_set_format (GstVideoDecoder * decoder,
     GstVideoCodecState * state);
 static gboolean gst_vtdec_flush (GstVideoDecoder * decoder);
@@ -57,7 +61,8 @@ static GstFlowReturn gst_vtdec_finish (GstVideoDecoder * decoder);
 static GstFlowReturn gst_vtdec_handle_frame (GstVideoDecoder * decoder,
     GstVideoCodecFrame * frame);
 
-static gboolean gst_vtdec_create_session (GstVtdec * vtdec);
+static gboolean gst_vtdec_create_session (GstVtdec * vtdec,
+    GstVideoFormat format);
 static void gst_vtdec_invalidate_session (GstVtdec * vtdec);
 static CMSampleBufferRef cm_sample_buffer_from_gst_buffer (GstVtdec * vtdec,
     GstBuffer * buf);
@@ -86,16 +91,6 @@ static GstStaticPadTemplate gst_vtdec_sink_template =
         "video/mpeg, mpegversion=2;" "image/jpeg")
     );
 
-#ifdef HAVE_IOS
-#define GST_VTDEC_VIDEO_FORMAT_STR "NV12"
-#define GST_VTDEC_VIDEO_FORMAT GST_VIDEO_FORMAT_NV12
-#define GST_VTDEC_CV_VIDEO_FORMAT kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
-#else
-#define GST_VTDEC_VIDEO_FORMAT_STR "UYVY"
-#define GST_VTDEC_VIDEO_FORMAT GST_VIDEO_FORMAT_UYVY
-#define GST_VTDEC_CV_VIDEO_FORMAT kCVPixelFormatType_422YpCbCr8
-#endif
-
 /* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */
 #if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 1090
 const CFStringRef
@@ -103,8 +98,17 @@ const CFStringRef
 CFSTR ("EnableHardwareAcceleratedVideoDecoder");
 #endif
 
+#ifdef HAVE_IOS
+#define GST_VTDEC_VIDEO_FORMAT_STR "NV12"
+#else
+#define GST_VTDEC_VIDEO_FORMAT_STR "UYVY"
+#endif
+
 #define VIDEO_SRC_CAPS \
-    GST_VIDEO_CAPS_MAKE("{" GST_VTDEC_VIDEO_FORMAT_STR "}")
+    GST_VIDEO_CAPS_MAKE(GST_VTDEC_VIDEO_FORMAT_STR) ";" \
+    GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
+    (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, \
+        "RGBA") ";"
 
 G_DEFINE_TYPE_WITH_CODE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER,
     GST_DEBUG_CATEGORY_INIT (gst_vtdec_debug_category, "vtdec", 0,
@@ -134,6 +138,8 @@ gst_vtdec_class_init (GstVtdecClass * klass)
   gobject_class->finalize = gst_vtdec_finalize;
   video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_vtdec_start);
   video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vtdec_stop);
+  video_decoder_class->decide_allocation =
+      GST_DEBUG_FUNCPTR (gst_vtdec_decide_allocation);
   video_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_vtdec_set_format);
   video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_vtdec_flush);
   video_decoder_class->finish = GST_DEBUG_FUNCPTR (gst_vtdec_finish);
@@ -156,6 +162,7 @@ gst_vtdec_finalize (GObject * object)
 
   g_async_queue_unref (vtdec->reorder_queue);
 
+
   G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object);
 }
 
@@ -177,11 +184,64 @@ gst_vtdec_stop (GstVideoDecoder * decoder)
   if (vtdec->session)
     gst_vtdec_invalidate_session (vtdec);
 
+  if (vtdec->texture_cache)
+    gst_core_video_texture_cache_free (vtdec->texture_cache);
+  vtdec->texture_cache = NULL;
+
   GST_DEBUG_OBJECT (vtdec, "stop");
 
   return TRUE;
 }
 
+static gboolean
+gst_vtdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
+{
+  gboolean ret;
+  GstVtdec *vtdec = GST_VTDEC (decoder);
+
+  ret =
+      GST_VIDEO_DECODER_CLASS (gst_vtdec_parent_class)->decide_allocation
+      (decoder, query);
+  if (ret) {
+    guint idx;
+
+    if (gst_query_find_allocation_meta (query,
+            GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) {
+      GstGLContext *context;
+      const GstStructure *upload_meta_params;
+
+      gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params);
+      if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext",
+              GST_GL_TYPE_CONTEXT, &context, NULL) && context) {
+        vtdec->texture_cache = gst_core_video_texture_cache_new (context);
+        gst_object_unref (context);
+      }
+    }
+  }
+
+  return ret;
+}
+
+static GstVideoFormat
+gst_vtdec_negotiate_output_format (GstVtdec * vtdec)
+{
+  GstCaps *caps = NULL;
+  GstVideoFormat format;
+  GstStructure *structure;
+  const gchar *s;
+
+  caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec));
+  if (!caps)
+    caps = gst_pad_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL);
+  caps = gst_caps_truncate (caps);
+  structure = gst_caps_get_structure (caps, 0);
+  s = gst_structure_get_string (structure, "format");
+  format = gst_video_format_from_string (s);
+  gst_caps_unref (caps);
+
+  return format;
+}
+
 static gboolean
 gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
 {
@@ -190,6 +250,8 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
   CMFormatDescriptionRef format_description = NULL;
   const char *caps_name;
   GstVtdec *vtdec = GST_VTDEC (decoder);
+  GstVideoFormat output_format;
+  GstVideoCodecState *output_state;
 
   GST_DEBUG_OBJECT (vtdec, "set_format");
 
@@ -229,12 +291,19 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
     CFRelease (vtdec->format_description);
   vtdec->format_description = format_description;
 
-  if (!gst_vtdec_create_session (vtdec))
+  output_format = gst_vtdec_negotiate_output_format (vtdec);
+
+  if (!gst_vtdec_create_session (vtdec, output_format))
     return FALSE;
 
-  gst_video_decoder_set_output_state (decoder,
+  output_state = gst_video_decoder_set_output_state (decoder,
-      GST_VTDEC_VIDEO_FORMAT, vtdec->video_info.width, vtdec->video_info.height,
+      output_format, vtdec->video_info.width, vtdec->video_info.height, state);
-      state);
+  output_state->caps = gst_video_info_to_caps (&output_state->info);
+  if (output_state->info.finfo->format == GST_VIDEO_FORMAT_RGBA) {
+    gst_caps_set_features (output_state->caps, 0,
+        gst_caps_features_new
+        (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL));
+  }
 
   return TRUE;
 }
@@ -312,12 +381,28 @@ error:
 }
 
 static gboolean
-gst_vtdec_create_session (GstVtdec * vtdec)
+gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format)
 {
   CFMutableDictionaryRef output_image_buffer_attrs;
   VTDecompressionOutputCallbackRecord callback;
   CFMutableDictionaryRef videoDecoderSpecification;
   OSStatus status;
+  guint32 cv_format;
+
+  switch (format) {
+    case GST_VIDEO_FORMAT_NV12:
+      cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
+      break;
+    case GST_VIDEO_FORMAT_UYVY:
+      cv_format = kCVPixelFormatType_422YpCbCr8;
+      break;
+    case GST_VIDEO_FORMAT_RGBA:
+      cv_format = kCVPixelFormatType_32BGRA;
+      break;
+    default:
+      g_warn_if_reached ();
+      break;
+  }
 
   videoDecoderSpecification =
       CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
@@ -333,7 +418,7 @@ gst_vtdec_create_session (GstVtdec * vtdec)
       CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
       &kCFTypeDictionaryValueCallBacks);
   gst_vtutil_dict_set_i32 (output_image_buffer_attrs,
-      kCVPixelBufferPixelFormatTypeKey, GST_VTDEC_CV_VIDEO_FORMAT);
+      kCVPixelBufferPixelFormatTypeKey, cv_format);
   gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferWidthKey,
       vtdec->video_info.width);
   gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferHeightKey,
@@ -557,13 +642,12 @@ gst_vtdec_session_output_callback (void *decompression_output_ref_con,
   }
   buf = gst_core_video_buffer_new (image_buffer, &state->info);
   gst_video_codec_state_unref (state);
-  frame->output_buffer = buf;
 
   gst_buffer_copy_into (buf, frame->input_buffer,
       GST_BUFFER_COPY_METADATA | GST_BUFFER_COPY_FLAGS, 0, -1);
   GST_BUFFER_PTS (buf) = pts.value;
   GST_BUFFER_DURATION (buf) = duration.value;
-
+  frame->output_buffer = buf;
   g_async_queue_push_sorted (vtdec->reorder_queue, frame,
       sort_frames_by_pts, NULL);
 
@@ -590,6 +674,11 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
   GstFlowReturn ret = GST_FLOW_OK;
   GstVideoDecoder *decoder = GST_VIDEO_DECODER (vtdec);
 
+  /* negotiate now so that we know whether we need to use the GL upload meta or
+   * not */
+  if (gst_pad_check_reconfigure (decoder->srcpad))
+    gst_video_decoder_negotiate (decoder);
+
   if (drain)
     VTDecompressionSessionWaitForAsynchronousFrames (vtdec->session);
 
@@ -599,6 +688,15 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
   while ((g_async_queue_length (vtdec->reorder_queue) >=
           vtdec->reorder_queue_length) || drain || flush) {
     frame = (GstVideoCodecFrame *) g_async_queue_try_pop (vtdec->reorder_queue);
+    if (vtdec->texture_cache != NULL) {
+      GstVideoGLTextureType texture_types[4] =
+          { GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 0 };
+      gst_buffer_add_video_gl_texture_upload_meta (frame->output_buffer,
+          GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL, 1, texture_types,
+          gst_core_video_texture_cache_upload, vtdec->texture_cache, NULL,
+          NULL);
+    }
+
     /* we need to check this in case reorder_queue_length=0 (jpeg for
      * example) or we're draining/flushing
      */

sys/applemedia/vtdec.h

@@ -25,6 +25,7 @@
 #include <gst/video/gstvideodecoder.h>
 #include <CoreMedia/CoreMedia.h>
 #include <VideoToolbox/VideoToolbox.h>
+#include "corevideotexturecache.h"
 
 G_BEGIN_DECLS
 
@@ -45,6 +46,7 @@ struct _GstVtdec
   VTDecompressionSessionRef session;
   GAsyncQueue *reorder_queue;
   gint reorder_queue_length;
+  GstCoreVideoTextureCache *texture_cache;
 };
 
 struct _GstVtdecClass