diff --git a/sys/applemedia/corevideotexturecache.m b/sys/applemedia/corevideotexturecache.m index 821aca5c96..e55ca69ba2 100644 --- a/sys/applemedia/corevideotexturecache.m +++ b/sys/applemedia/corevideotexturecache.m @@ -26,12 +26,11 @@ #endif #include "corevideotexturecache.h" #include "coremediabuffer.h" +#include "corevideobuffer.h" GstCoreVideoTextureCache * gst_core_video_texture_cache_new (GstGLContext * ctx) { - CVReturn err; - g_return_val_if_fail (ctx != NULL, NULL); GstCoreVideoTextureCache *cache = g_new0 (GstCoreVideoTextureCache, 1); @@ -41,21 +40,21 @@ gst_core_video_texture_cache_new (GstGLContext * ctx) CGLPixelFormatAttribute attribs[1] = { 0 }; int numPixelFormats; CGLPixelFormatObj pixelFormat; - CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5 - NSOpenGLContext *platform_ctx = (NSOpenGLContext *) gst_gl_context_get_gl_context (ctx); - err = CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL, - [platform_ctx CGLContextObj], pixelFormat, - NULL, &cache->cache); + CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5 + NSOpenGLContext *platform_ctx = + (NSOpenGLContext *) gst_gl_context_get_gl_context (ctx); + CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL, + [platform_ctx CGLContextObj], pixelFormat, NULL, &cache->cache); #else - err = CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL, - (CVEAGLContext) gst_gl_context_get_gl_context (ctx), - NULL, &cache->cache); + CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL, + (CVEAGLContext) gst_gl_context_get_gl_context (ctx), NULL, &cache->cache); #endif return cache; } -void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache) +void +gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache) { g_return_if_fail (cache != NULL); @@ -69,11 +68,13 @@ void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache) } gboolean -gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4]) +gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, + guint texture_id[4]) { g_return_val_if_fail (meta != NULL, FALSE); - GstCoreVideoTextureCache *cache = (GstCoreVideoTextureCache *) meta->user_data; + GstCoreVideoTextureCache *cache = + (GstCoreVideoTextureCache *) meta->user_data; const GstGLFuncs *gl = cache->ctx->gl_vtable; #if !HAVE_IOS CVOpenGLTextureRef texture = NULL; @@ -81,16 +82,25 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint t CVOpenGLESTextureRef texture = NULL; #endif GstVideoMeta *video_meta = gst_buffer_get_video_meta (meta->buffer); - GstCoreMediaMeta *cv_meta = (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer, - gst_core_media_meta_api_get_type ()); + GstCoreMediaMeta *cm_meta = + (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer, + gst_core_media_meta_api_get_type ()); + GstCoreVideoMeta *cv_meta = + (GstCoreVideoMeta *) gst_buffer_get_meta (meta->buffer, + gst_core_video_meta_api_get_type ()); + CVPixelBufferRef pixel_buf; + if (cm_meta) + pixel_buf = cm_meta->pixel_buf; + else + pixel_buf = cv_meta->pixbuf; #if !HAVE_IOS CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault, - cache->cache, cv_meta->pixel_buf, NULL, &texture); + cache->cache, pixel_buf, NULL, &texture); #else CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, - cache->cache, cv_meta->image_buf, NULL, GL_TEXTURE_2D, - GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE, - 0, &texture); + cache->cache, cm_meta->image_buf, NULL, GL_TEXTURE_2D, + GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE, + 0, &texture); #endif GLuint fboId; gl->GenFramebuffers (1, &fboId); @@ -98,13 +108,14 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint t gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, #if !HAVE_IOS - CVOpenGLTextureGetTarget(texture), CVOpenGLTextureGetName(texture), + CVOpenGLTextureGetTarget (texture), CVOpenGLTextureGetName (texture), #else - CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture), + CVOpenGLESTextureGetTarget (texture), CVOpenGLESTextureGetName (texture), #endif 0); gl->BindTexture (GL_TEXTURE_2D, texture_id[0]); - gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width, video_meta->height, 0); + gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width, + video_meta->height, 0); gl->BindTexture (GL_TEXTURE_2D, 0); gl->BindFramebuffer (GL_FRAMEBUFFER, 0); gl->DeleteFramebuffers (1, &fboId); diff --git a/sys/applemedia/vtdec.c b/sys/applemedia/vtdec.c index 82757b0af5..15385187e2 100644 --- a/sys/applemedia/vtdec.c +++ b/sys/applemedia/vtdec.c @@ -35,13 +35,15 @@ #include "config.h" #endif +#include #include #include #include +#include #include "vtdec.h" #include "vtutil.h" #include "corevideobuffer.h" -#include +#include "coremediabuffer.h" GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category); #define GST_CAT_DEFAULT gst_vtdec_debug_category @@ -50,6 +52,8 @@ static void gst_vtdec_finalize (GObject * object); static gboolean gst_vtdec_start (GstVideoDecoder * decoder); static gboolean gst_vtdec_stop (GstVideoDecoder * decoder); +static gboolean gst_vtdec_decide_allocation (GstVideoDecoder * decoder, + GstQuery * query); static gboolean gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state); static gboolean gst_vtdec_flush (GstVideoDecoder * decoder); @@ -57,7 +61,8 @@ static GstFlowReturn gst_vtdec_finish (GstVideoDecoder * decoder); static GstFlowReturn gst_vtdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame); -static gboolean gst_vtdec_create_session (GstVtdec * vtdec); +static gboolean gst_vtdec_create_session (GstVtdec * vtdec, + GstVideoFormat format); static void gst_vtdec_invalidate_session (GstVtdec * vtdec); static CMSampleBufferRef cm_sample_buffer_from_gst_buffer (GstVtdec * vtdec, GstBuffer * buf); @@ -86,16 +91,6 @@ static GstStaticPadTemplate gst_vtdec_sink_template = "video/mpeg, mpegversion=2;" "image/jpeg") ); -#ifdef HAVE_IOS -#define GST_VTDEC_VIDEO_FORMAT_STR "NV12" -#define GST_VTDEC_VIDEO_FORMAT GST_VIDEO_FORMAT_NV12 -#define GST_VTDEC_CV_VIDEO_FORMAT kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange -#else -#define GST_VTDEC_VIDEO_FORMAT_STR "UYVY" -#define GST_VTDEC_VIDEO_FORMAT GST_VIDEO_FORMAT_UYVY -#define GST_VTDEC_CV_VIDEO_FORMAT kCVPixelFormatType_422YpCbCr8 -#endif - /* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */ #if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 1090 const CFStringRef @@ -103,8 +98,17 @@ const CFStringRef CFSTR ("EnableHardwareAcceleratedVideoDecoder"); #endif +#ifdef HAVE_IOS +#define GST_VTDEC_VIDEO_FORMAT_STR "NV12" +#else +#define GST_VTDEC_VIDEO_FORMAT_STR "UYVY" +#endif + #define VIDEO_SRC_CAPS \ - GST_VIDEO_CAPS_MAKE("{" GST_VTDEC_VIDEO_FORMAT_STR "}") + GST_VIDEO_CAPS_MAKE(GST_VTDEC_VIDEO_FORMAT_STR) ";" \ + GST_VIDEO_CAPS_MAKE_WITH_FEATURES \ + (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, \ + "RGBA") ";" G_DEFINE_TYPE_WITH_CODE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER, GST_DEBUG_CATEGORY_INIT (gst_vtdec_debug_category, "vtdec", 0, @@ -134,6 +138,8 @@ gst_vtdec_class_init (GstVtdecClass * klass) gobject_class->finalize = gst_vtdec_finalize; video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_vtdec_start); video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vtdec_stop); + video_decoder_class->decide_allocation = + GST_DEBUG_FUNCPTR (gst_vtdec_decide_allocation); video_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_vtdec_set_format); video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_vtdec_flush); video_decoder_class->finish = GST_DEBUG_FUNCPTR (gst_vtdec_finish); @@ -156,6 +162,7 @@ gst_vtdec_finalize (GObject * object) g_async_queue_unref (vtdec->reorder_queue); + G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object); } @@ -177,11 +184,64 @@ gst_vtdec_stop (GstVideoDecoder * decoder) if (vtdec->session) gst_vtdec_invalidate_session (vtdec); + if (vtdec->texture_cache) + gst_core_video_texture_cache_free (vtdec->texture_cache); + vtdec->texture_cache = NULL; + GST_DEBUG_OBJECT (vtdec, "stop"); return TRUE; } +static gboolean +gst_vtdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) +{ + gboolean ret; + GstVtdec *vtdec = GST_VTDEC (decoder); + + ret = + GST_VIDEO_DECODER_CLASS (gst_vtdec_parent_class)->decide_allocation + (decoder, query); + if (ret) { + guint idx; + + if (gst_query_find_allocation_meta (query, + GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) { + GstGLContext *context; + const GstStructure *upload_meta_params; + + gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params); + if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext", + GST_GL_TYPE_CONTEXT, &context, NULL) && context) { + vtdec->texture_cache = gst_core_video_texture_cache_new (context); + gst_object_unref (context); + } + } + } + + return ret; +} + +static GstVideoFormat +gst_vtdec_negotiate_output_format (GstVtdec * vtdec) +{ + GstCaps *caps = NULL; + GstVideoFormat format; + GstStructure *structure; + const gchar *s; + + caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec)); + if (!caps) + caps = gst_pad_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL); + caps = gst_caps_truncate (caps); + structure = gst_caps_get_structure (caps, 0); + s = gst_structure_get_string (structure, "format"); + format = gst_video_format_from_string (s); + gst_caps_unref (caps); + + return format; +} + static gboolean gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) { @@ -190,6 +250,8 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) CMFormatDescriptionRef format_description = NULL; const char *caps_name; GstVtdec *vtdec = GST_VTDEC (decoder); + GstVideoFormat output_format; + GstVideoCodecState *output_state; GST_DEBUG_OBJECT (vtdec, "set_format"); @@ -229,12 +291,19 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) CFRelease (vtdec->format_description); vtdec->format_description = format_description; - if (!gst_vtdec_create_session (vtdec)) + output_format = gst_vtdec_negotiate_output_format (vtdec); + + if (!gst_vtdec_create_session (vtdec, output_format)) return FALSE; - gst_video_decoder_set_output_state (decoder, - GST_VTDEC_VIDEO_FORMAT, vtdec->video_info.width, vtdec->video_info.height, - state); + output_state = gst_video_decoder_set_output_state (decoder, + output_format, vtdec->video_info.width, vtdec->video_info.height, state); + output_state->caps = gst_video_info_to_caps (&output_state->info); + if (output_state->info.finfo->format == GST_VIDEO_FORMAT_RGBA) { + gst_caps_set_features (output_state->caps, 0, + gst_caps_features_new + (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL)); + } return TRUE; } @@ -312,12 +381,28 @@ error: } static gboolean -gst_vtdec_create_session (GstVtdec * vtdec) +gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format) { CFMutableDictionaryRef output_image_buffer_attrs; VTDecompressionOutputCallbackRecord callback; CFMutableDictionaryRef videoDecoderSpecification; OSStatus status; + guint32 cv_format; + + switch (format) { + case GST_VIDEO_FORMAT_NV12: + cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + break; + case GST_VIDEO_FORMAT_UYVY: + cv_format = kCVPixelFormatType_422YpCbCr8; + break; + case GST_VIDEO_FORMAT_RGBA: + cv_format = kCVPixelFormatType_32BGRA; + break; + default: + g_warn_if_reached (); + break; + } videoDecoderSpecification = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks, @@ -333,7 +418,7 @@ gst_vtdec_create_session (GstVtdec * vtdec) CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); gst_vtutil_dict_set_i32 (output_image_buffer_attrs, - kCVPixelBufferPixelFormatTypeKey, GST_VTDEC_CV_VIDEO_FORMAT); + kCVPixelBufferPixelFormatTypeKey, cv_format); gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferWidthKey, vtdec->video_info.width); gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferHeightKey, @@ -557,13 +642,12 @@ gst_vtdec_session_output_callback (void *decompression_output_ref_con, } buf = gst_core_video_buffer_new (image_buffer, &state->info); gst_video_codec_state_unref (state); - frame->output_buffer = buf; gst_buffer_copy_into (buf, frame->input_buffer, GST_BUFFER_COPY_METADATA | GST_BUFFER_COPY_FLAGS, 0, -1); GST_BUFFER_PTS (buf) = pts.value; GST_BUFFER_DURATION (buf) = duration.value; - + frame->output_buffer = buf; g_async_queue_push_sorted (vtdec->reorder_queue, frame, sort_frames_by_pts, NULL); @@ -590,6 +674,11 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain, GstFlowReturn ret = GST_FLOW_OK; GstVideoDecoder *decoder = GST_VIDEO_DECODER (vtdec); + /* negotiate now so that we know whether we need to use the GL upload meta or + * not */ + if (gst_pad_check_reconfigure (decoder->srcpad)) + gst_video_decoder_negotiate (decoder); + if (drain) VTDecompressionSessionWaitForAsynchronousFrames (vtdec->session); @@ -599,6 +688,15 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain, while ((g_async_queue_length (vtdec->reorder_queue) >= vtdec->reorder_queue_length) || drain || flush) { frame = (GstVideoCodecFrame *) g_async_queue_try_pop (vtdec->reorder_queue); + if (vtdec->texture_cache != NULL) { + GstVideoGLTextureType texture_types[4] = + { GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 0 }; + gst_buffer_add_video_gl_texture_upload_meta (frame->output_buffer, + GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL, 1, texture_types, + gst_core_video_texture_cache_upload, vtdec->texture_cache, NULL, + NULL); + } + /* we need to check this in case reorder_queue_length=0 (jpeg for * example) or we're draining/flushing */ diff --git a/sys/applemedia/vtdec.h b/sys/applemedia/vtdec.h index bc029bccf9..9603e1ada7 100644 --- a/sys/applemedia/vtdec.h +++ b/sys/applemedia/vtdec.h @@ -25,6 +25,7 @@ #include #include #include +#include "corevideotexturecache.h" G_BEGIN_DECLS @@ -45,6 +46,7 @@ struct _GstVtdec VTDecompressionSessionRef session; GAsyncQueue *reorder_queue; gint reorder_queue_length; + GstCoreVideoTextureCache *texture_cache; }; struct _GstVtdecClass