mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2025-01-11 09:55:36 +00:00
vtdec: implement the GL texture upload meta
This commit is contained in:
Alessandro Decina
parent
cdfe90aaba
commit
d95a12e185
3 changed files with 154 additions and 43 deletions
|
|
@ -26,12 +26,11 @@
|
|||
#endif
|
||||
#include "corevideotexturecache.h"
|
||||
#include "coremediabuffer.h"
|
||||
#include "corevideobuffer.h"
|
||||
|
||||
GstCoreVideoTextureCache *
|
||||
gst_core_video_texture_cache_new (GstGLContext * ctx)
|
||||
{
|
||||
CVReturn err;
|
||||
|
||||
g_return_val_if_fail (ctx != NULL, NULL);
|
||||
|
||||
GstCoreVideoTextureCache *cache = g_new0 (GstCoreVideoTextureCache, 1);
|
||||
|
|
@ -41,21 +40,21 @@ gst_core_video_texture_cache_new (GstGLContext * ctx)
|
|||
CGLPixelFormatAttribute attribs[1] = { 0 };
|
||||
int numPixelFormats;
|
||||
CGLPixelFormatObj pixelFormat;
|
||||
CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5
|
||||
NSOpenGLContext *platform_ctx = (NSOpenGLContext *) gst_gl_context_get_gl_context (ctx);
|
||||
err = CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL,
|
||||
[platform_ctx CGLContextObj], pixelFormat,
|
||||
NULL, &cache->cache);
|
||||
CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5
|
||||
NSOpenGLContext *platform_ctx =
|
||||
(NSOpenGLContext *) gst_gl_context_get_gl_context (ctx);
|
||||
CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL,
|
||||
[platform_ctx CGLContextObj], pixelFormat, NULL, &cache->cache);
|
||||
#else
|
||||
err = CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL,
|
||||
(CVEAGLContext) gst_gl_context_get_gl_context (ctx),
|
||||
NULL, &cache->cache);
|
||||
CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL,
|
||||
(CVEAGLContext) gst_gl_context_get_gl_context (ctx), NULL, &cache->cache);
|
||||
#endif
|
||||
|
||||
return cache;
|
||||
}
|
||||
|
||||
void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
|
||||
void
|
||||
gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
|
||||
{
|
||||
g_return_if_fail (cache != NULL);
|
||||
|
||||
|
|
@ -69,11 +68,13 @@ void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
|
|||
}
|
||||
|
||||
gboolean
|
||||
gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4])
|
||||
gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta,
|
||||
guint texture_id[4])
|
||||
{
|
||||
g_return_val_if_fail (meta != NULL, FALSE);
|
||||
|
||||
GstCoreVideoTextureCache *cache = (GstCoreVideoTextureCache *) meta->user_data;
|
||||
GstCoreVideoTextureCache *cache =
|
||||
(GstCoreVideoTextureCache *) meta->user_data;
|
||||
const GstGLFuncs *gl = cache->ctx->gl_vtable;
|
||||
#if !HAVE_IOS
|
||||
CVOpenGLTextureRef texture = NULL;
|
||||
|
|
@ -81,16 +82,25 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint t
|
|||
CVOpenGLESTextureRef texture = NULL;
|
||||
#endif
|
||||
GstVideoMeta *video_meta = gst_buffer_get_video_meta (meta->buffer);
|
||||
GstCoreMediaMeta *cv_meta = (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer,
|
||||
gst_core_media_meta_api_get_type ());
|
||||
GstCoreMediaMeta *cm_meta =
|
||||
(GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer,
|
||||
gst_core_media_meta_api_get_type ());
|
||||
GstCoreVideoMeta *cv_meta =
|
||||
(GstCoreVideoMeta *) gst_buffer_get_meta (meta->buffer,
|
||||
gst_core_video_meta_api_get_type ());
|
||||
CVPixelBufferRef pixel_buf;
|
||||
if (cm_meta)
|
||||
pixel_buf = cm_meta->pixel_buf;
|
||||
else
|
||||
pixel_buf = cv_meta->pixbuf;
|
||||
#if !HAVE_IOS
|
||||
CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
|
||||
cache->cache, cv_meta->pixel_buf, NULL, &texture);
|
||||
cache->cache, pixel_buf, NULL, &texture);
|
||||
#else
|
||||
CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
|
||||
cache->cache, cv_meta->image_buf, NULL, GL_TEXTURE_2D,
|
||||
GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE,
|
||||
0, &texture);
|
||||
cache->cache, cm_meta->image_buf, NULL, GL_TEXTURE_2D,
|
||||
GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE,
|
||||
0, &texture);
|
||||
#endif
|
||||
GLuint fboId;
|
||||
gl->GenFramebuffers (1, &fboId);
|
||||
|
|
@ -98,13 +108,14 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint t
|
|||
|
||||
gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
|
||||
#if !HAVE_IOS
|
||||
CVOpenGLTextureGetTarget(texture), CVOpenGLTextureGetName(texture),
|
||||
CVOpenGLTextureGetTarget (texture), CVOpenGLTextureGetName (texture),
|
||||
#else
|
||||
CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture),
|
||||
CVOpenGLESTextureGetTarget (texture), CVOpenGLESTextureGetName (texture),
|
||||
#endif
|
||||
0);
|
||||
gl->BindTexture (GL_TEXTURE_2D, texture_id[0]);
|
||||
gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width, video_meta->height, 0);
|
||||
gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width,
|
||||
video_meta->height, 0);
|
||||
gl->BindTexture (GL_TEXTURE_2D, 0);
|
||||
gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
|
||||
gl->DeleteFramebuffers (1, &fboId);
|
||||
|
|
|
|||
|
|
@ -35,13 +35,15 @@
|
|||
#include "config.h"
|
||||
#endif
|
||||
|
||||
#include <string.h>
|
||||
#include <gst/gst.h>
|
||||
#include <gst/video/video.h>
|
||||
#include <gst/video/gstvideodecoder.h>
|
||||
#include <gst/gl/gstglcontext.h>
|
||||
#include "vtdec.h"
|
||||
#include "vtutil.h"
|
||||
#include "corevideobuffer.h"
|
||||
#include <string.h>
|
||||
#include "coremediabuffer.h"
|
||||
|
||||
GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category);
|
||||
#define GST_CAT_DEFAULT gst_vtdec_debug_category
|
||||
|
|
@ -50,6 +52,8 @@ static void gst_vtdec_finalize (GObject * object);
|
|||
|
||||
static gboolean gst_vtdec_start (GstVideoDecoder * decoder);
|
||||
static gboolean gst_vtdec_stop (GstVideoDecoder * decoder);
|
||||
static gboolean gst_vtdec_decide_allocation (GstVideoDecoder * decoder,
|
||||
GstQuery * query);
|
||||
static gboolean gst_vtdec_set_format (GstVideoDecoder * decoder,
|
||||
GstVideoCodecState * state);
|
||||
static gboolean gst_vtdec_flush (GstVideoDecoder * decoder);
|
||||
|
|
@ -57,7 +61,8 @@ static GstFlowReturn gst_vtdec_finish (GstVideoDecoder * decoder);
|
|||
static GstFlowReturn gst_vtdec_handle_frame (GstVideoDecoder * decoder,
|
||||
GstVideoCodecFrame * frame);
|
||||
|
||||
static gboolean gst_vtdec_create_session (GstVtdec * vtdec);
|
||||
static gboolean gst_vtdec_create_session (GstVtdec * vtdec,
|
||||
GstVideoFormat format);
|
||||
static void gst_vtdec_invalidate_session (GstVtdec * vtdec);
|
||||
static CMSampleBufferRef cm_sample_buffer_from_gst_buffer (GstVtdec * vtdec,
|
||||
GstBuffer * buf);
|
||||
|
|
@ -86,16 +91,6 @@ static GstStaticPadTemplate gst_vtdec_sink_template =
|
|||
"video/mpeg, mpegversion=2;" "image/jpeg")
|
||||
);
|
||||
|
||||
#ifdef HAVE_IOS
|
||||
#define GST_VTDEC_VIDEO_FORMAT_STR "NV12"
|
||||
#define GST_VTDEC_VIDEO_FORMAT GST_VIDEO_FORMAT_NV12
|
||||
#define GST_VTDEC_CV_VIDEO_FORMAT kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
|
||||
#else
|
||||
#define GST_VTDEC_VIDEO_FORMAT_STR "UYVY"
|
||||
#define GST_VTDEC_VIDEO_FORMAT GST_VIDEO_FORMAT_UYVY
|
||||
#define GST_VTDEC_CV_VIDEO_FORMAT kCVPixelFormatType_422YpCbCr8
|
||||
#endif
|
||||
|
||||
/* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */
|
||||
#if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 1090
|
||||
const CFStringRef
|
||||
|
|
@ -103,8 +98,17 @@ const CFStringRef
|
|||
CFSTR ("EnableHardwareAcceleratedVideoDecoder");
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_IOS
|
||||
#define GST_VTDEC_VIDEO_FORMAT_STR "NV12"
|
||||
#else
|
||||
#define GST_VTDEC_VIDEO_FORMAT_STR "UYVY"
|
||||
#endif
|
||||
|
||||
#define VIDEO_SRC_CAPS \
|
||||
GST_VIDEO_CAPS_MAKE("{" GST_VTDEC_VIDEO_FORMAT_STR "}")
|
||||
GST_VIDEO_CAPS_MAKE(GST_VTDEC_VIDEO_FORMAT_STR) ";" \
|
||||
GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
|
||||
(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, \
|
||||
"RGBA") ";"
|
||||
|
||||
G_DEFINE_TYPE_WITH_CODE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER,
|
||||
GST_DEBUG_CATEGORY_INIT (gst_vtdec_debug_category, "vtdec", 0,
|
||||
|
|
@ -134,6 +138,8 @@ gst_vtdec_class_init (GstVtdecClass * klass)
|
|||
gobject_class->finalize = gst_vtdec_finalize;
|
||||
video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_vtdec_start);
|
||||
video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vtdec_stop);
|
||||
video_decoder_class->decide_allocation =
|
||||
GST_DEBUG_FUNCPTR (gst_vtdec_decide_allocation);
|
||||
video_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_vtdec_set_format);
|
||||
video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_vtdec_flush);
|
||||
video_decoder_class->finish = GST_DEBUG_FUNCPTR (gst_vtdec_finish);
|
||||
|
|
@ -156,6 +162,7 @@ gst_vtdec_finalize (GObject * object)
|
|||
|
||||
g_async_queue_unref (vtdec->reorder_queue);
|
||||
|
||||
|
||||
G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object);
|
||||
}
|
||||
|
||||
|
|
@ -177,11 +184,64 @@ gst_vtdec_stop (GstVideoDecoder * decoder)
|
|||
if (vtdec->session)
|
||||
gst_vtdec_invalidate_session (vtdec);
|
||||
|
||||
if (vtdec->texture_cache)
|
||||
gst_core_video_texture_cache_free (vtdec->texture_cache);
|
||||
vtdec->texture_cache = NULL;
|
||||
|
||||
GST_DEBUG_OBJECT (vtdec, "stop");
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
gst_vtdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
|
||||
{
|
||||
gboolean ret;
|
||||
GstVtdec *vtdec = GST_VTDEC (decoder);
|
||||
|
||||
ret =
|
||||
GST_VIDEO_DECODER_CLASS (gst_vtdec_parent_class)->decide_allocation
|
||||
(decoder, query);
|
||||
if (ret) {
|
||||
guint idx;
|
||||
|
||||
if (gst_query_find_allocation_meta (query,
|
||||
GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) {
|
||||
GstGLContext *context;
|
||||
const GstStructure *upload_meta_params;
|
||||
|
||||
gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params);
|
||||
if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext",
|
||||
GST_GL_TYPE_CONTEXT, &context, NULL) && context) {
|
||||
vtdec->texture_cache = gst_core_video_texture_cache_new (context);
|
||||
gst_object_unref (context);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static GstVideoFormat
|
||||
gst_vtdec_negotiate_output_format (GstVtdec * vtdec)
|
||||
{
|
||||
GstCaps *caps = NULL;
|
||||
GstVideoFormat format;
|
||||
GstStructure *structure;
|
||||
const gchar *s;
|
||||
|
||||
caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec));
|
||||
if (!caps)
|
||||
caps = gst_pad_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL);
|
||||
caps = gst_caps_truncate (caps);
|
||||
structure = gst_caps_get_structure (caps, 0);
|
||||
s = gst_structure_get_string (structure, "format");
|
||||
format = gst_video_format_from_string (s);
|
||||
gst_caps_unref (caps);
|
||||
|
||||
return format;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
|
||||
{
|
||||
|
|
@ -190,6 +250,8 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
|
|||
CMFormatDescriptionRef format_description = NULL;
|
||||
const char *caps_name;
|
||||
GstVtdec *vtdec = GST_VTDEC (decoder);
|
||||
GstVideoFormat output_format;
|
||||
GstVideoCodecState *output_state;
|
||||
|
||||
GST_DEBUG_OBJECT (vtdec, "set_format");
|
||||
|
||||
|
|
@ -229,12 +291,19 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
|
|||
CFRelease (vtdec->format_description);
|
||||
vtdec->format_description = format_description;
|
||||
|
||||
if (!gst_vtdec_create_session (vtdec))
|
||||
output_format = gst_vtdec_negotiate_output_format (vtdec);
|
||||
|
||||
if (!gst_vtdec_create_session (vtdec, output_format))
|
||||
return FALSE;
|
||||
|
||||
gst_video_decoder_set_output_state (decoder,
|
||||
GST_VTDEC_VIDEO_FORMAT, vtdec->video_info.width, vtdec->video_info.height,
|
||||
state);
|
||||
output_state = gst_video_decoder_set_output_state (decoder,
|
||||
output_format, vtdec->video_info.width, vtdec->video_info.height, state);
|
||||
output_state->caps = gst_video_info_to_caps (&output_state->info);
|
||||
if (output_state->info.finfo->format == GST_VIDEO_FORMAT_RGBA) {
|
||||
gst_caps_set_features (output_state->caps, 0,
|
||||
gst_caps_features_new
|
||||
(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL));
|
||||
}
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
|
@ -312,12 +381,28 @@ error:
|
|||
}
|
||||
|
||||
static gboolean
|
||||
gst_vtdec_create_session (GstVtdec * vtdec)
|
||||
gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format)
|
||||
{
|
||||
CFMutableDictionaryRef output_image_buffer_attrs;
|
||||
VTDecompressionOutputCallbackRecord callback;
|
||||
CFMutableDictionaryRef videoDecoderSpecification;
|
||||
OSStatus status;
|
||||
guint32 cv_format;
|
||||
|
||||
switch (format) {
|
||||
case GST_VIDEO_FORMAT_NV12:
|
||||
cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
|
||||
break;
|
||||
case GST_VIDEO_FORMAT_UYVY:
|
||||
cv_format = kCVPixelFormatType_422YpCbCr8;
|
||||
break;
|
||||
case GST_VIDEO_FORMAT_RGBA:
|
||||
cv_format = kCVPixelFormatType_32BGRA;
|
||||
break;
|
||||
default:
|
||||
g_warn_if_reached ();
|
||||
break;
|
||||
}
|
||||
|
||||
videoDecoderSpecification =
|
||||
CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
|
||||
|
|
@ -333,7 +418,7 @@ gst_vtdec_create_session (GstVtdec * vtdec)
|
|||
CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
|
||||
&kCFTypeDictionaryValueCallBacks);
|
||||
gst_vtutil_dict_set_i32 (output_image_buffer_attrs,
|
||||
kCVPixelBufferPixelFormatTypeKey, GST_VTDEC_CV_VIDEO_FORMAT);
|
||||
kCVPixelBufferPixelFormatTypeKey, cv_format);
|
||||
gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferWidthKey,
|
||||
vtdec->video_info.width);
|
||||
gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferHeightKey,
|
||||
|
|
@ -557,13 +642,12 @@ gst_vtdec_session_output_callback (void *decompression_output_ref_con,
|
|||
}
|
||||
buf = gst_core_video_buffer_new (image_buffer, &state->info);
|
||||
gst_video_codec_state_unref (state);
|
||||
frame->output_buffer = buf;
|
||||
|
||||
gst_buffer_copy_into (buf, frame->input_buffer,
|
||||
GST_BUFFER_COPY_METADATA | GST_BUFFER_COPY_FLAGS, 0, -1);
|
||||
GST_BUFFER_PTS (buf) = pts.value;
|
||||
GST_BUFFER_DURATION (buf) = duration.value;
|
||||
|
||||
frame->output_buffer = buf;
|
||||
g_async_queue_push_sorted (vtdec->reorder_queue, frame,
|
||||
sort_frames_by_pts, NULL);
|
||||
|
||||
|
|
@ -590,6 +674,11 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
|
|||
GstFlowReturn ret = GST_FLOW_OK;
|
||||
GstVideoDecoder *decoder = GST_VIDEO_DECODER (vtdec);
|
||||
|
||||
/* negotiate now so that we know whether we need to use the GL upload meta or
|
||||
* not */
|
||||
if (gst_pad_check_reconfigure (decoder->srcpad))
|
||||
gst_video_decoder_negotiate (decoder);
|
||||
|
||||
if (drain)
|
||||
VTDecompressionSessionWaitForAsynchronousFrames (vtdec->session);
|
||||
|
||||
|
|
@ -599,6 +688,15 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
|
|||
while ((g_async_queue_length (vtdec->reorder_queue) >=
|
||||
vtdec->reorder_queue_length) || drain || flush) {
|
||||
frame = (GstVideoCodecFrame *) g_async_queue_try_pop (vtdec->reorder_queue);
|
||||
if (vtdec->texture_cache != NULL) {
|
||||
GstVideoGLTextureType texture_types[4] =
|
||||
{ GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 0 };
|
||||
gst_buffer_add_video_gl_texture_upload_meta (frame->output_buffer,
|
||||
GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL, 1, texture_types,
|
||||
gst_core_video_texture_cache_upload, vtdec->texture_cache, NULL,
|
||||
NULL);
|
||||
}
|
||||
|
||||
/* we need to check this in case reorder_queue_length=0 (jpeg for
|
||||
* example) or we're draining/flushing
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
#include <gst/video/gstvideodecoder.h>
|
||||
#include <CoreMedia/CoreMedia.h>
|
||||
#include <VideoToolbox/VideoToolbox.h>
|
||||
#include "corevideotexturecache.h"
|
||||
|
||||
G_BEGIN_DECLS
|
||||
|
||||
|
|
@ -45,6 +46,7 @@ struct _GstVtdec
|
|||
VTDecompressionSessionRef session;
|
||||
GAsyncQueue *reorder_queue;
|
||||
gint reorder_queue_length;
|
||||
GstCoreVideoTextureCache *texture_cache;
|
||||
};
|
||||
|
||||
struct _GstVtdecClass
|
||||
|
|
|
|||
Loading…
Reference in a new issue