diff --git a/sys/applemedia/avfvideosrc.m b/sys/applemedia/avfvideosrc.m index 8d456a7263..695f4749a3 100644 --- a/sys/applemedia/avfvideosrc.m +++ b/sys/applemedia/avfvideosrc.m @@ -93,6 +93,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC); BOOL stopRequest; GstCaps *caps; + GstVideoFormat internalFormat; GstVideoFormat format; gint width, height; GstClockTime latency; @@ -625,6 +626,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer width = info.width; height = info.height; format = info.finfo->format; + internalFormat = GST_VIDEO_FORMAT_UNKNOWN; latency = gst_util_uint64_scale (GST_SECOND, info.fps_d, info.fps_n); dispatch_sync (mainQueue, ^{ @@ -664,22 +666,28 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer } } + internalFormat = format; switch (format) { case GST_VIDEO_FORMAT_NV12: newformat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; break; case GST_VIDEO_FORMAT_UYVY: - newformat = kCVPixelFormatType_422YpCbCr8; + newformat = kCVPixelFormatType_422YpCbCr8; break; case GST_VIDEO_FORMAT_YUY2: - newformat = kCVPixelFormatType_422YpCbCr8_yuvs; + newformat = kCVPixelFormatType_422YpCbCr8_yuvs; break; case GST_VIDEO_FORMAT_RGBA: - /* In order to do RGBA, we negotiate BGRA (since RGBA is not supported - * if not in textures) and then we get RGBA textures via - * CVOpenGL*TextureCacheCreateTextureFromImage. Computers. */ +#if !HAVE_IOS + newformat = kCVPixelFormatType_422YpCbCr8; + internalFormat = GST_VIDEO_FORMAT_UYVY; +#else + newformat = kCVPixelFormatType_32BGRA; + internalFormat = GST_VIDEO_FORMAT_BGRA; +#endif + break; case GST_VIDEO_FORMAT_BGRA: - newformat = kCVPixelFormatType_32BGRA; + newformat = kCVPixelFormatType_32BGRA; break; default: *successPtr = NO; @@ -688,10 +696,10 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer return; } - GST_DEBUG_OBJECT(element, - "Width: %d Height: %d Format: %" GST_FOURCC_FORMAT, - width, height, - GST_FOURCC_ARGS (gst_video_format_to_fourcc (format))); + GST_INFO_OBJECT(element, + "width: %d height: %d format: %s internalFormat: %s", width, height, + gst_video_format_to_string (format), + gst_video_format_to_string (internalFormat)); output.videoSettings = [NSDictionary dictionaryWithObject:[NSNumber numberWithInt:newformat] @@ -790,10 +798,10 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer gst_query_unref (query); if (glContext) { - GST_INFO_OBJECT (element, "pushing textures. GL context %p", glContext); + GST_INFO_OBJECT (element, "pushing textures. Internal format %s, context %p", + gst_video_format_to_string (internalFormat), glContext); textureCache = gst_core_video_texture_cache_new (glContext); - gst_core_video_texture_cache_set_format (textureCache, - "NV12", caps); + gst_core_video_texture_cache_set_format (textureCache, internalFormat, caps); gst_object_unref (glContext); } else { GST_WARNING_OBJECT (element, "got memory:GLMemory caps but not GL context from downstream element"); diff --git a/sys/applemedia/corevideotexturecache.h b/sys/applemedia/corevideotexturecache.h index b90cbf4ab9..d53ec41d25 100644 --- a/sys/applemedia/corevideotexturecache.h +++ b/sys/applemedia/corevideotexturecache.h @@ -42,7 +42,7 @@ typedef struct _GstCoreVideoTextureCache GstCoreVideoTextureCache *gst_core_video_texture_cache_new (GstGLContext * ctx); void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache); void gst_core_video_texture_cache_set_format (GstCoreVideoTextureCache * cache, - const gchar * input_format, GstCaps * out_caps); + GstVideoFormat in_format, GstCaps * out_caps); gboolean gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4]); GstBuffer * gst_core_video_texture_cache_get_gl_buffer (GstCoreVideoTextureCache * cache, GstBuffer * cv_buffer); diff --git a/sys/applemedia/corevideotexturecache.m b/sys/applemedia/corevideotexturecache.m index f935d38702..a35dcf65d4 100644 --- a/sys/applemedia/corevideotexturecache.m +++ b/sys/applemedia/corevideotexturecache.m @@ -84,7 +84,7 @@ gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache) void gst_core_video_texture_cache_set_format (GstCoreVideoTextureCache * cache, - const gchar * input_format, GstCaps * out_caps) + GstVideoFormat in_format, GstCaps * out_caps) { GstCaps *in_caps; GstCapsFeatures *features; @@ -97,7 +97,8 @@ gst_core_video_texture_cache_set_format (GstCoreVideoTextureCache * cache, gst_video_info_from_caps (&cache->output_info, out_caps); in_caps = gst_caps_copy (out_caps); - gst_caps_set_simple (in_caps, "format", G_TYPE_STRING, input_format, NULL); + gst_caps_set_simple (in_caps, "format", + G_TYPE_STRING, gst_video_format_to_string (in_format), NULL); features = gst_caps_get_features (in_caps, 0); gst_caps_features_add (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); gst_video_info_from_caps (&cache->input_info, in_caps); @@ -127,6 +128,7 @@ static gboolean gl_mem_from_buffer (GstCoreVideoTextureCache * cache, GstBuffer * buffer, GstMemory **mem1, GstMemory **mem2) { + gboolean ret = TRUE; #if !HAVE_IOS CVOpenGLTextureRef texture = NULL; #else @@ -139,49 +141,78 @@ gl_mem_from_buffer (GstCoreVideoTextureCache * cache, #if !HAVE_IOS CVOpenGLTextureCacheFlush (cache->cache, 0); - - if (CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault, - cache->cache, pixel_buf, NULL, &texture) != kCVReturnSuccess) - goto error; - - *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx, - CVOpenGLTextureGetName (texture), CVOpenGLTextureGetTarget (texture), - &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease); #else CVOpenGLESTextureCacheFlush (cache->cache, 0); - - if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, - cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_LUMINANCE, - GST_VIDEO_INFO_WIDTH (&cache->input_info), - GST_VIDEO_INFO_HEIGHT (&cache->input_info), - GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &texture) != kCVReturnSuccess) - goto error; - - *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx, - CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture), - &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease); - - if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, - cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA, - GST_VIDEO_INFO_WIDTH (&cache->input_info) / 2, - GST_VIDEO_INFO_HEIGHT (&cache->input_info) / 2, - GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &texture) != kCVReturnSuccess) - goto error; - - *mem2 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx, - CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture), - &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease); #endif - return TRUE; + switch (GST_VIDEO_INFO_FORMAT (&cache->input_info)) { +#if !HAVE_IOS + case GST_VIDEO_FORMAT_UYVY: + /* both avfvideosrc and vtdec on OSX when doing GLMemory negotiate UYVY + * under the hood, which means a single output texture. */ + if (CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault, + cache->cache, pixel_buf, NULL, &texture) != kCVReturnSuccess) + goto error; + + *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx, + CVOpenGLTextureGetName (texture), CVOpenGLTextureGetTarget (texture), + &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease); + break; +#else + case GST_VIDEO_FORMAT_BGRA: + /* avfvideosrc does BGRA on iOS when doing GLMemory */ + if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, + cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_RGBA, + GST_VIDEO_INFO_WIDTH (&cache->input_info), + GST_VIDEO_INFO_HEIGHT (&cache->input_info), + GL_RGBA, GL_UNSIGNED_BYTE, 0, &texture) != kCVReturnSuccess) + goto error; + + *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx, + CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture), + &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease); + break; + case GST_VIDEO_FORMAT_NV12: + /* vtdec does NV12 on iOS when doing GLMemory */ + if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, + cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_LUMINANCE, + GST_VIDEO_INFO_WIDTH (&cache->input_info), + GST_VIDEO_INFO_HEIGHT (&cache->input_info), + GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &texture) != kCVReturnSuccess) + goto error; + + *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx, + CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture), + &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease); + + if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, + cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA, + GST_VIDEO_INFO_WIDTH (&cache->input_info) / 2, + GST_VIDEO_INFO_HEIGHT (&cache->input_info) / 2, + GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &texture) != kCVReturnSuccess) + goto error; + + *mem2 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx, + CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture), + &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease); + break; +#endif + default: + g_warn_if_reached (); + ret = FALSE; + } + + return ret; error: + ret = FALSE; + if (*mem1) gst_memory_unref (*mem1); if (*mem2) gst_memory_unref (*mem2); - return FALSE; + return ret; } static void diff --git a/sys/applemedia/vtdec.c b/sys/applemedia/vtdec.c index 28319999a9..a696fcfcc1 100644 --- a/sys/applemedia/vtdec.c +++ b/sys/applemedia/vtdec.c @@ -313,7 +313,6 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) vtdec->format_description = format_description; output_format = gst_vtdec_negotiate_output_format (vtdec); - if (!gst_vtdec_create_session (vtdec, output_format)) return FALSE; @@ -710,10 +709,17 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain, if (gst_pad_check_reconfigure (decoder->srcpad)) { gst_video_decoder_negotiate (decoder); if (vtdec->texture_cache) { + GstVideoFormat internal_format; GstVideoCodecState *output_state = gst_video_decoder_get_output_state (decoder); + +#ifdef HAVE_IOS + internal_format = GST_VIDEO_FORMAT_NV12; +#else + internal_format = GST_VIDEO_FORMAT_UYVY; +#endif gst_core_video_texture_cache_set_format (vtdec->texture_cache, - GST_VTDEC_VIDEO_FORMAT_STR, output_state->caps); + internal_format, output_state->caps); gst_video_codec_state_unref (output_state); } }