mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-10-02 08:42:32 +00:00
Code Issues 1 Projects Releases Wiki Activity

nvcodec: nvsldec: Refactor graphics api resource handling

Browse source 
* Move GL context object to GstNVDecoder object, and remove
  duplicated handling of each codec decoder element
* Don't create GL context too early. We can create it only if
  we need to negotiate with downstream gl element.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2089>
This commit is contained in:
Seungha Yang 2021-03-17 14:30:09 +09:00 committed by GStreamer Marge Bot
parent be1f66a491
commit 791f1da7b8
6 changed files with 471 additions and 570 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

495
sys/nvcodec/gstnvdecoder.c
View file

@ -67,20 +67,39 @@ typedef struct _GstNvDecoderFrameInfo
gboolean available;
} GstNvDecoderFrameInfo;
typedef enum
{
GST_NV_DECODER_OUTPUT_TYPE_SYSTEM = 0,
GST_NV_DECODER_OUTPUT_TYPE_GL,
GST_NV_DECODER_OUTPUT_TYPE_CUDA,
/* FIXME: add support D3D11 memory */
} GstNvDecoderOutputType;
struct _GstNvDecoder
{
GstObject parent;
GstCudaContext *context;
CUstream cuda_stream;
CUvideodecoder decoder_handle;
GstNvDecoderFrameInfo *frame_pool;
guint pool_size;
GstVideoInfo info;
GstVideoInfo coded_info;
gboolean configured;
/* For OpenGL interop. */
GstObject *gl_display;
GstObject *gl_context;
GstObject *other_gl_context;
GstNvDecoderOutputType output_type;
};
static void gst_nv_decoder_dispose (GObject * object);
static void gst_nv_decoder_finalize (GObject * object);
static void gst_nv_decoder_reset (GstNvDecoder * self);
#define parent_class gst_nv_decoder_parent_class
G_DEFINE_TYPE (GstNvDecoder, gst_nv_decoder, GST_TYPE_OBJECT);
@ -91,7 +110,6 @@ gst_nv_decoder_class_init (GstNvDecoderClass * klass)
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
gobject_class->dispose = gst_nv_decoder_dispose;
gobject_class->finalize = gst_nv_decoder_finalize;
}
static void
@ -104,23 +122,24 @@ gst_nv_decoder_dispose (GObject * object)
{
GstNvDecoder *self = GST_NV_DECODER (object);
gst_nv_decoder_reset (self);
if (self->context && self->cuda_stream) {
if (gst_cuda_context_push (self->context)) {
gst_cuda_result (CuStreamDestroy (self->cuda_stream));
gst_cuda_context_pop (NULL);
self->cuda_stream = NULL;
}
}
gst_clear_object (&self->context);
gst_clear_object (&self->gl_display);
gst_clear_object (&self->gl_context);
gst_clear_object (&self->other_gl_context);
G_OBJECT_CLASS (parent_class)->dispose (object);
}
static void
gst_nv_decoder_finalize (GObject * object)
{
GstNvDecoder *self = GST_NV_DECODER (object);
g_free (self->frame_pool);
if (self->decoder_handle)
gst_cuda_result (CuvidDestroyDecoder (self->decoder_handle));
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static cudaVideoChromaFormat
chroma_format_from_video_format (GstVideoFormat format)
{
@ -206,27 +225,81 @@ gst_nv_decoder_prepare_frame_pool (GstNvDecoder * self, guint pool_size)
}
GstNvDecoder *
gst_nv_decoder_new (GstCudaContext * context, cudaVideoCodec codec,
GstVideoInfo * info, guint pool_size)
gst_nv_decoder_new (GstCudaContext * context)
{
GstNvDecoder *decoder;
CUVIDDECODECREATEINFO create_info = { 0, };
GstVideoFormat format;
GstNvDecoder *self;
g_return_val_if_fail (GST_IS_CUDA_CONTEXT (context), NULL);
g_return_val_if_fail (codec < cudaVideoCodec_NumCodecs, NULL);
g_return_val_if_fail (info != NULL, NULL);
g_return_val_if_fail (pool_size > 0, NULL);
decoder = g_object_new (GST_TYPE_NV_DECODER, NULL);
decoder->context = gst_object_ref (context);
gst_object_ref_sink (decoder);
self = g_object_new (GST_TYPE_NV_DECODER, NULL);
self->context = gst_object_ref (context);
gst_object_ref_sink (self);
if (gst_cuda_context_push (context)) {
CUresult cuda_ret;
cuda_ret = CuStreamCreate (&self->cuda_stream, CU_STREAM_DEFAULT);
if (!gst_cuda_result (cuda_ret)) {
GST_WARNING_OBJECT (self,
"Could not create CUDA stream, will use default stream");
self->cuda_stream = NULL;
}
gst_cuda_context_pop (NULL);
}
return self;
}
gboolean
gst_nv_decoder_is_configured (GstNvDecoder * decoder)
{
g_return_val_if_fail (GST_IS_NV_DECODER (decoder), FALSE);
return decoder->configured;
}
static void
gst_nv_decoder_reset (GstNvDecoder * self)
{
g_clear_pointer (&self->frame_pool, g_free);
if (self->decoder_handle) {
gst_cuda_context_push (self->context);
CuvidDestroyDecoder (self->decoder_handle);
gst_cuda_context_pop (NULL);
self->decoder_handle = NULL;
}
self->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
self->configured = FALSE;
}
gboolean
gst_nv_decoder_configure (GstNvDecoder * decoder, cudaVideoCodec codec,
GstVideoInfo * info, gint coded_width, gint coded_height, guint pool_size)
{
CUVIDDECODECREATEINFO create_info = { 0, };
GstVideoFormat format;
gboolean ret;
g_return_val_if_fail (GST_IS_NV_DECODER (decoder), FALSE);
g_return_val_if_fail (codec < cudaVideoCodec_NumCodecs, FALSE);
g_return_val_if_fail (info != NULL, FALSE);
g_return_val_if_fail (coded_width >= GST_VIDEO_INFO_WIDTH (info), FALSE);
g_return_val_if_fail (coded_height >= GST_VIDEO_INFO_HEIGHT (info), FALSE);
g_return_val_if_fail (pool_size > 0, FALSE);
gst_nv_decoder_reset (decoder);
decoder->info = *info;
gst_video_info_set_format (&decoder->coded_info, GST_VIDEO_INFO_FORMAT (info),
coded_width, coded_height);
format = GST_VIDEO_INFO_FORMAT (info);
/* FIXME: check aligned resolution or actaul coded resolution */
create_info.ulWidth = GST_VIDEO_INFO_WIDTH (info);;
create_info.ulHeight = GST_VIDEO_INFO_HEIGHT (info);;
/* FIXME: check aligned resolution or actual coded resolution */
create_info.ulWidth = GST_VIDEO_INFO_WIDTH (&decoder->coded_info);
create_info.ulHeight = GST_VIDEO_INFO_HEIGHT (&decoder->coded_info);
create_info.ulNumDecodeSurfaces = pool_size;
create_info.CodecType = codec;
create_info.ChromaFormat = chroma_format_from_video_format (format);
@ -241,44 +314,39 @@ gst_nv_decoder_new (GstCudaContext * context, cudaVideoCodec codec,
create_info.OutputFormat = output_format_from_video_format (format);
create_info.DeinterlaceMode = cudaVideoDeinterlaceMode_Weave;
create_info.ulTargetWidth = GST_VIDEO_INFO_WIDTH (info);;
create_info.ulTargetHeight = GST_VIDEO_INFO_HEIGHT (info);
create_info.ulTargetWidth = GST_VIDEO_INFO_WIDTH (&decoder->coded_info);
create_info.ulTargetHeight = GST_VIDEO_INFO_HEIGHT (&decoder->coded_info);
/* we always copy decoded picture to output buffer */
create_info.ulNumOutputSurfaces = 1;
create_info.target_rect.left = 0;
create_info.target_rect.top = 0;
create_info.target_rect.right = GST_VIDEO_INFO_WIDTH (info);
create_info.target_rect.bottom = GST_VIDEO_INFO_HEIGHT (info);
create_info.target_rect.right = GST_VIDEO_INFO_WIDTH (&decoder->coded_info);
create_info.target_rect.bottom = GST_VIDEO_INFO_HEIGHT (&decoder->coded_info);
if (!gst_cuda_context_push (context)) {
if (!gst_cuda_context_push (decoder->context)) {
GST_ERROR_OBJECT (decoder, "Failed to lock CUDA context");
goto error;
return FALSE;
}
if (!gst_cuda_result (CuvidCreateDecoder (&decoder->decoder_handle,
&create_info))) {
ret = gst_cuda_result (CuvidCreateDecoder (&decoder->decoder_handle,
&create_info));
gst_cuda_context_pop (NULL);
if (!ret) {
GST_ERROR_OBJECT (decoder, "Cannot create decoder instance");
goto error;
}
if (!gst_cuda_context_pop (NULL)) {
GST_ERROR_OBJECT (decoder, "Failed to unlock CUDA context");
goto error;
return FALSE;
}
if (!gst_nv_decoder_prepare_frame_pool (decoder, pool_size)) {
GST_ERROR_OBJECT (decoder, "Cannot prepare internal surface buffer pool");
goto error;
gst_nv_decoder_reset (decoder);
return FALSE;
}
decoder->info = *info;
decoder->configured = TRUE;
return decoder;
error:
gst_clear_object (&decoder);
return NULL;
return TRUE;
}
GstNvDecoderFrame *
@ -593,7 +661,7 @@ gst_nv_decoder_copy_frame_to_gl_internal (GstGLContext * context,
* GST_VIDEO_INFO_COMP_PSTRIDE (info, i);
copy_params.srcDevice = frame->devptr +
(i * frame->pitch * GST_VIDEO_INFO_HEIGHT (info));
(i * frame->pitch * GST_VIDEO_INFO_HEIGHT (&self->coded_info));
copy_params.dstDevice = dst_ptr;
copy_params.Height = GST_VIDEO_INFO_COMP_HEIGHT (info, i);
@ -662,18 +730,18 @@ gst_nv_decoder_copy_frame_to_system (GstNvDecoder * decoder,
for (i = 0; i < GST_VIDEO_FRAME_N_PLANES (&video_frame); i++) {
copy_params.srcDevice = frame->devptr +
(i * frame->pitch * GST_VIDEO_INFO_HEIGHT (&decoder->info));
(i * frame->pitch * GST_VIDEO_INFO_HEIGHT (&decoder->coded_info));
copy_params.dstHost = GST_VIDEO_FRAME_PLANE_DATA (&video_frame, i);
copy_params.dstPitch = GST_VIDEO_FRAME_PLANE_STRIDE (&video_frame, i);
copy_params.Height = GST_VIDEO_FRAME_COMP_HEIGHT (&video_frame, i);
if (!gst_cuda_result (CuMemcpy2DAsync (&copy_params, NULL))) {
if (!gst_cuda_result (CuMemcpy2DAsync (&copy_params, decoder->cuda_stream))) {
GST_ERROR_OBJECT (decoder, "failed to copy %dth plane", i);
goto done;
}
}
gst_cuda_result (CuStreamSynchronize (NULL));
gst_cuda_result (CuStreamSynchronize (decoder->cuda_stream));
ret = TRUE;
@ -737,13 +805,13 @@ gst_nv_decoder_copy_frame_to_cuda (GstNvDecoder * decoder,
* GST_VIDEO_INFO_COMP_PSTRIDE (&decoder->info, 0);
copy_params.Height = GST_VIDEO_INFO_COMP_HEIGHT (&decoder->info, i);
if (!gst_cuda_result (CuMemcpy2DAsync (&copy_params, NULL))) {
if (!gst_cuda_result (CuMemcpy2DAsync (&copy_params, decoder->cuda_stream))) {
GST_ERROR_OBJECT (decoder, "failed to copy %dth plane", i);
goto done;
}
}
gst_cuda_result (CuStreamSynchronize (NULL));
gst_cuda_result (CuStreamSynchronize (decoder->cuda_stream));
ret = TRUE;
@ -756,55 +824,84 @@ done:
}
gboolean
gst_nv_decoder_finish_frame (GstNvDecoder * decoder,
GstNvDecoderOutputType output_type, GstObject * graphics_context,
GstNvDecoderFrame * frame, GstBuffer * buffer)
gst_nv_decoder_finish_frame (GstNvDecoder * decoder, GstVideoDecoder * videodec,
GstNvDecoderFrame * frame, GstBuffer ** buffer)
{
GstBuffer *outbuf = NULL;
gboolean ret = FALSE;
g_return_val_if_fail (GST_IS_NV_DECODER (decoder), FALSE);
g_return_val_if_fail (frame != NULL, FALSE);
g_return_val_if_fail (GST_IS_BUFFER (buffer), FALSE);
g_return_val_if_fail (GST_IS_NV_DECODER (decoder), GST_FLOW_ERROR);
g_return_val_if_fail (GST_IS_VIDEO_DECODER (videodec), GST_FLOW_ERROR);
g_return_val_if_fail (frame != NULL, GST_FLOW_ERROR);
g_return_val_if_fail (buffer != NULL, GST_FLOW_ERROR);
#ifdef HAVE_NVCODEC_GST_GL
if (output_type == GST_NV_DECODER_OUTPUT_TYPE_GL && !graphics_context) {
if (!GST_IS_GL_CONTEXT (graphics_context)) {
GST_ERROR_OBJECT (decoder, "Invalid GL Context");
return FALSE;
}
outbuf = gst_video_decoder_allocate_output_buffer (videodec);
if (!outbuf) {
GST_ERROR_OBJECT (videodec, "Couldn't allocate output buffer");
return FALSE;
}
#endif
if (!gst_cuda_context_push (decoder->context)) {
GST_ERROR_OBJECT (decoder, "Failed to push CUDA context");
return FALSE;
goto error;
}
if (!gst_nv_decoder_frame_map (frame)) {
GST_ERROR_OBJECT (decoder, "Couldn't map frame");
gst_cuda_context_pop (NULL);
return FALSE;
goto error;
}
gst_cuda_context_pop (NULL);
switch (decoder->output_type) {
case GST_NV_DECODER_OUTPUT_TYPE_SYSTEM:
ret = gst_nv_decoder_copy_frame_to_system (decoder, frame, outbuf);
break;
#ifdef HAVE_NVCODEC_GST_GL
if (output_type == GST_NV_DECODER_OUTPUT_TYPE_GL) {
ret = gst_nv_decoder_copy_frame_to_gl (decoder,
GST_GL_CONTEXT (graphics_context), frame, buffer);
} else
case GST_NV_DECODER_OUTPUT_TYPE_GL:
g_assert (decoder->gl_context != NULL);
ret = gst_nv_decoder_copy_frame_to_gl (decoder,
GST_GL_CONTEXT (decoder->gl_context), frame, outbuf);
break;
#endif
if (output_type == GST_NV_DECODER_OUTPUT_TYPE_CUDA) {
ret = gst_nv_decoder_copy_frame_to_cuda (decoder, frame, buffer);
} else {
ret = gst_nv_decoder_copy_frame_to_system (decoder, frame, buffer);
case GST_NV_DECODER_OUTPUT_TYPE_CUDA:
ret = gst_nv_decoder_copy_frame_to_cuda (decoder, frame, outbuf);
break;
default:
g_assert_not_reached ();
goto error;
}
/* FIXME: This is the case where OpenGL context of downstream glbufferpool
* belongs to non-nvidia (or different device).
* There should be enhancement to ensure nvdec has compatible OpenGL context
*/
if (!ret && decoder->output_type == GST_NV_DECODER_OUTPUT_TYPE_GL) {
GST_WARNING_OBJECT (videodec,
"Couldn't copy frame to GL memory, fallback to system memory");
decoder->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
ret = gst_nv_decoder_copy_frame_to_system (decoder, frame, outbuf);
}
gst_cuda_context_push (decoder->context);
gst_nv_decoder_frame_unmap (frame);
gst_cuda_context_pop (NULL);
return ret;
if (!ret) {
GST_WARNING_OBJECT (videodec, "Failed to copy frame");
goto error;
}
*buffer = outbuf;
return TRUE;
error:
gst_clear_buffer (&outbuf);
return FALSE;
}
typedef enum
@ -1230,84 +1327,37 @@ gst_cuda_video_codec_to_string (cudaVideoCodec codec)
}
gboolean
gst_nv_decoder_ensure_element_data (GstElement * decoder, guint cuda_device_id,
GstCudaContext ** cuda_context, CUstream * cuda_stream,
GstObject ** gl_display, GstObject ** other_gl_context)
gst_nv_decoder_handle_set_context (GstNvDecoder * decoder,
GstElement * videodec, GstContext * context)
{
CUresult cuda_ret;
g_return_val_if_fail (GST_IS_NV_DECODER (decoder), FALSE);
g_return_val_if_fail (GST_IS_ELEMENT (videodec), FALSE);
g_return_val_if_fail (GST_IS_ELEMENT (decoder), FALSE);
g_return_val_if_fail (cuda_context, FALSE);
g_return_val_if_fail (cuda_stream, FALSE);
g_return_val_if_fail (gl_display, FALSE);
g_return_val_if_fail (other_gl_context, FALSE);
if (!gst_cuda_ensure_element_context (decoder, cuda_device_id, cuda_context)) {
GST_ERROR_OBJECT (decoder, "failed to create CUDA context");
return FALSE;
}
if (gst_cuda_context_push (*cuda_context)) {
CUstream stream;
cuda_ret = CuStreamCreate (&stream, CU_STREAM_DEFAULT);
if (!gst_cuda_result (cuda_ret)) {
GST_WARNING_OBJECT (decoder,
"Could not create CUDA stream, will use default stream");
*cuda_stream = NULL;
} else {
*cuda_stream = stream;
}
gst_cuda_context_pop (NULL);
}
#if HAVE_NVCODEC_GST_GL
gst_gl_ensure_element_data (decoder,
(GstGLDisplay **) gl_display, (GstGLContext **) other_gl_context);
if (*gl_display)
gst_gl_display_filter_gl_api (GST_GL_DISPLAY (*gl_display),
SUPPORTED_GL_APIS);
#endif
return TRUE;
}
void
gst_nv_decoder_set_context (GstElement * decoder, GstContext * context,
guint cuda_device_id, GstCudaContext ** cuda_context,
GstObject ** gl_display, GstObject ** other_gl_context)
{
g_return_if_fail (GST_IS_ELEMENT (decoder));
g_return_if_fail (GST_IS_CONTEXT (context));
g_return_if_fail (cuda_context != NULL);
g_return_if_fail (gl_display != NULL);
g_return_if_fail (other_gl_context != NULL);
if (gst_cuda_handle_set_context (decoder, context, cuda_device_id,
cuda_context)) {
return;
}
#ifdef HAVE_NVCODEC_GST_GL
gst_gl_handle_set_context (decoder, context,
(GstGLDisplay **) gl_display, (GstGLContext **) other_gl_context);
if (gst_gl_handle_set_context (videodec, context,
(GstGLDisplay **) & decoder->gl_display,
(GstGLContext **) & decoder->other_gl_context)) {
return TRUE;
}
#endif
return FALSE;
}
gboolean
gst_nv_decoder_handle_context_query (GstElement * decoder, GstQuery * query,
GstCudaContext * cuda_context, GstObject * gl_display,
GstObject * gl_context, GstObject * other_gl_context)
gst_nv_decoder_handle_context_query (GstNvDecoder * decoder,
GstVideoDecoder * videodec, GstQuery * query)
{
g_return_val_if_fail (GST_IS_ELEMENT (decoder), FALSE);
g_return_val_if_fail (GST_IS_NV_DECODER (decoder), FALSE);
g_return_val_if_fail (GST_IS_ELEMENT (videodec), FALSE);
if (gst_cuda_handle_context_query (decoder, query, cuda_context)) {
return TRUE;
}
#ifdef HAVE_NVCODEC_GST_GL
if (gst_gl_handle_context_query (GST_ELEMENT (decoder), query,
(GstGLDisplay *) gl_display,
(GstGLContext *) gl_context, (GstGLContext *) other_gl_context)) {
if (gl_display)
gst_gl_display_filter_gl_api (GST_GL_DISPLAY (gl_display),
if (gst_gl_handle_context_query (GST_ELEMENT (videodec), query,
(GstGLDisplay *) decoder->gl_display,
(GstGLContext *) decoder->gl_context,
(GstGLContext *) decoder->other_gl_context)) {
if (decoder->gl_display)
gst_gl_display_filter_gl_api (GST_GL_DISPLAY (decoder->gl_display),
SUPPORTED_GL_APIS);
return TRUE;
}
@ -1339,50 +1389,52 @@ gst_nv_decoder_check_cuda_device_from_context (GstGLContext * context,
}
static gboolean
gst_nv_decoder_ensure_gl_context (GstElement * decoder, GstObject * gl_display,
GstObject * other_gl_context, GstObject ** gl_context)
gst_nv_decoder_ensure_gl_context (GstNvDecoder * decoder, GstElement * videodec)
{
gboolean ret;
GstGLDisplay *display;
GstGLContext *context;
if (!gl_display) {
GST_DEBUG_OBJECT (decoder, "No available OpenGL display");
if (!gst_gl_ensure_element_data (videodec,
(GstGLDisplay **) & decoder->gl_display,
(GstGLContext **) & decoder->other_gl_context)) {
GST_DEBUG_OBJECT (videodec, "No available OpenGL display");
return FALSE;
}
display = GST_GL_DISPLAY (gl_display);
display = GST_GL_DISPLAY (decoder->gl_display);
if (!gst_gl_query_local_gl_context (decoder, GST_PAD_SRC,
(GstGLContext **) gl_context)) {
GST_INFO_OBJECT (decoder, "failed to query local OpenGL context");
if (!gst_gl_query_local_gl_context (videodec, GST_PAD_SRC,
(GstGLContext **) & decoder->gl_context)) {
GST_INFO_OBJECT (videodec, "failed to query local OpenGL context");
gst_clear_object (gl_context);
*gl_context =
gst_clear_object (&decoder->gl_context);
decoder->gl_context =
(GstObject *) gst_gl_display_get_gl_context_for_thread (display, NULL);
if (*gl_context == NULL
if (decoder->gl_context == NULL
|| !gst_gl_display_add_context (display,
GST_GL_CONTEXT (*gl_context))) {
gst_clear_object (gl_context);
GST_GL_CONTEXT (decoder->gl_context))) {
gst_clear_object (&decoder->gl_context);
if (!gst_gl_display_create_context (display,
(GstGLContext *) other_gl_context,
(GstGLContext **) gl_context, NULL)) {
GST_WARNING_OBJECT (decoder, "failed to create OpenGL context");
(GstGLContext *) decoder->other_gl_context,
(GstGLContext **) & decoder->gl_context, NULL)) {
GST_WARNING_OBJECT (videodec, "failed to create OpenGL context");
return FALSE;
}
if (!gst_gl_display_add_context (display, (GstGLContext *) * gl_context)) {
GST_WARNING_OBJECT (decoder,
if (!gst_gl_display_add_context (display,
(GstGLContext *) decoder->gl_context)) {
GST_WARNING_OBJECT (videodec,
"failed to add the OpenGL context to the display");
return FALSE;
}
}
}
context = GST_GL_CONTEXT (*gl_context);
context = GST_GL_CONTEXT (decoder->gl_context);
if (!gst_gl_context_check_gl_version (context, SUPPORTED_GL_APIS, 3, 0)) {
GST_WARNING_OBJECT (decoder,
GST_WARNING_OBJECT (videodec,
"OpenGL context could not support PBO download");
return FALSE;
}
@ -1392,7 +1444,7 @@ gst_nv_decoder_ensure_gl_context (GstElement * decoder, GstObject * gl_display,
&ret);
if (!ret) {
GST_WARNING_OBJECT (decoder,
GST_WARNING_OBJECT (videodec,
"Current OpenGL context is not CUDA-compatible");
return FALSE;
}
@ -1402,40 +1454,42 @@ gst_nv_decoder_ensure_gl_context (GstElement * decoder, GstObject * gl_display,
#endif
gboolean
gst_nv_decoder_negotiate (GstVideoDecoder * decoder,
GstVideoCodecState * input_state, GstVideoFormat format, guint width,
guint height, GstObject * gl_display, GstObject * other_gl_context,
GstObject ** gl_context, GstVideoCodecState ** output_state,
GstNvDecoderOutputType * output_type)
gst_nv_decoder_negotiate (GstNvDecoder * decoder,
GstVideoDecoder * videodec, GstVideoCodecState * input_state,
GstVideoCodecState ** output_state)
{
GstVideoCodecState *state;
GstVideoInfo *info;
g_return_val_if_fail (GST_IS_VIDEO_DECODER (decoder), FALSE);
g_return_val_if_fail (GST_IS_NV_DECODER (decoder), FALSE);
g_return_val_if_fail (GST_IS_VIDEO_DECODER (videodec), FALSE);
g_return_val_if_fail (input_state != NULL, FALSE);
g_return_val_if_fail (format != GST_VIDEO_FORMAT_UNKNOWN, FALSE);
g_return_val_if_fail (width > 0, FALSE);
g_return_val_if_fail (height > 0, FALSE);
g_return_val_if_fail (output_state != NULL, FALSE);
g_return_val_if_fail (gl_context != NULL, FALSE);
g_return_val_if_fail (output_type != NULL, FALSE);
state = gst_video_decoder_set_output_state (decoder,
format, width, height, input_state);
if (!decoder->configured) {
GST_ERROR_OBJECT (videodec, "Should configure decoder first");
return FALSE;
}
info = &decoder->info;
state = gst_video_decoder_set_output_state (videodec,
GST_VIDEO_INFO_FORMAT (info),
GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info), input_state);
state->caps = gst_video_info_to_caps (&state->info);
if (*output_state)
gst_video_codec_state_unref (*output_state);
*output_state = state;
*output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
decoder->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
{
GstCaps *caps;
caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (decoder));
GST_DEBUG_OBJECT (decoder, "Allowed caps %" GST_PTR_FORMAT, caps);
caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (videodec));
GST_DEBUG_OBJECT (videodec, "Allowed caps %" GST_PTR_FORMAT, caps);
if (!caps || gst_caps_is_any (caps)) {
GST_DEBUG_OBJECT (decoder,
GST_DEBUG_OBJECT (videodec,
"cannot determine output format, using system memory");
} else {
GstCapsFeatures *features;
@ -1448,47 +1502,45 @@ gst_nv_decoder_negotiate (GstVideoDecoder * decoder,
features = gst_caps_get_features (caps, i);
if (features && gst_caps_features_contains (features,
GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY)) {
GST_DEBUG_OBJECT (decoder, "found CUDA memory feature");
GST_DEBUG_OBJECT (videodec, "found CUDA memory feature");
have_cuda = TRUE;
break;
}
#ifdef HAVE_NVCODEC_GST_GL
if (gl_display &&
features && gst_caps_features_contains (features,
if (features && gst_caps_features_contains (features,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
GST_DEBUG_OBJECT (decoder, "found GL memory feature");
GST_DEBUG_OBJECT (videodec, "found GL memory feature");
have_gl = TRUE;
}
#endif
}
if (have_cuda)
*output_type = GST_NV_DECODER_OUTPUT_TYPE_CUDA;
decoder->output_type = GST_NV_DECODER_OUTPUT_TYPE_CUDA;
else if (have_gl)
*output_type = GST_NV_DECODER_OUTPUT_TYPE_GL;
decoder->output_type = GST_NV_DECODER_OUTPUT_TYPE_GL;
}
gst_clear_caps (&caps);
}
#ifdef HAVE_NVCODEC_GST_GL
if (*output_type == GST_NV_DECODER_OUTPUT_TYPE_GL &&
!gst_nv_decoder_ensure_gl_context (GST_ELEMENT (decoder),
gl_display, other_gl_context, gl_context)) {
GST_WARNING_OBJECT (decoder,
if (decoder->output_type == GST_NV_DECODER_OUTPUT_TYPE_GL &&
!gst_nv_decoder_ensure_gl_context (decoder, GST_ELEMENT (videodec))) {
GST_WARNING_OBJECT (videodec,
"OpenGL context is not CUDA-compatible, fallback to system memory");
*output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
decoder->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
}
#endif
switch (*output_type) {
switch (decoder->output_type) {
case GST_NV_DECODER_OUTPUT_TYPE_CUDA:
GST_DEBUG_OBJECT (decoder, "using CUDA memory");
GST_DEBUG_OBJECT (videodec, "using CUDA memory");
gst_caps_set_features (state->caps, 0,
gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, NULL));
break;
#ifdef HAVE_NVCODEC_GST_GL
case GST_NV_DECODER_OUTPUT_TYPE_GL:
GST_DEBUG_OBJECT (decoder, "using GL memory");
GST_DEBUG_OBJECT (videodec, "using GL memory");
gst_caps_set_features (state->caps, 0,
gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL));
gst_caps_set_simple (state->caps, "texture-target", G_TYPE_STRING,
@ -1496,7 +1548,7 @@ gst_nv_decoder_negotiate (GstVideoDecoder * decoder,
break;
#endif
default:
GST_DEBUG_OBJECT (decoder, "using system memory");
GST_DEBUG_OBJECT (videodec, "using system memory");
break;
}
@ -1547,17 +1599,24 @@ gst_nv_decoder_ensure_cuda_pool (GstNvDecoder * decoder, GstQuery * query)
#ifdef HAVE_NVCODEC_GST_GL
static gboolean
gst_nv_decoder_ensure_gl_pool (GstNvDecoder * decoder, GstQuery * query,
GstObject * gl_context)
gst_nv_decoder_ensure_gl_pool (GstNvDecoder * decoder, GstQuery * query)
{
GstCaps *outcaps;
GstBufferPool *pool = NULL;
guint n, size, min, max;
GstVideoInfo vinfo = { 0, };
GstStructure *config;
GstGLContext *gl_context;
GST_DEBUG_OBJECT (decoder, "decide allocation");
if (!decoder->gl_context) {
GST_ERROR_OBJECT (decoder, "GL context is not available");
return FALSE;
}
gl_context = GST_GL_CONTEXT (decoder->gl_context);
gst_query_parse_allocation (query, &outcaps, NULL);
n = gst_query_get_n_allocation_pools (query);
if (n > 0)
@ -1593,25 +1652,29 @@ gst_nv_decoder_ensure_gl_pool (GstNvDecoder * decoder, GstQuery * query,
#endif
gboolean
gst_nv_decoder_decide_allocation (GstNvDecoder * nvdec,
GstVideoDecoder * decoder, GstQuery * query, GstObject * gl_context,
GstNvDecoderOutputType output_type)
gst_nv_decoder_decide_allocation (GstNvDecoder * decoder,
GstVideoDecoder * videodec, GstQuery * query)
{
GST_DEBUG_OBJECT (decoder, "decide allocation");
gboolean ret = TRUE;
/* GstVideoDecoder will take care this case */
if (output_type == GST_NV_DECODER_OUTPUT_TYPE_SYSTEM)
return TRUE;
GST_DEBUG_OBJECT (videodec, "decide allocation");
switch (decoder->output_type) {
case GST_NV_DECODER_OUTPUT_TYPE_SYSTEM:
/* GstVideoDecoder will take care this case */
break;
#ifdef HAVE_NVCODEC_GST_GL
if (output_type == GST_NV_DECODER_OUTPUT_TYPE_GL) {
if (!gst_nv_decoder_ensure_gl_pool (nvdec, query, gl_context))
return FALSE;
} else
case GST_NV_DECODER_OUTPUT_TYPE_GL:
ret = gst_nv_decoder_ensure_gl_pool (decoder, query);
break;
#endif
if (!gst_nv_decoder_ensure_cuda_pool (nvdec, query)) {
return FALSE;
case GST_NV_DECODER_OUTPUT_TYPE_CUDA:
ret = gst_nv_decoder_ensure_cuda_pool (decoder, query);
break;
default:
g_assert_not_reached ();
return FALSE;
}
return TRUE;
return ret;
}

74
sys/nvcodec/gstnvdecoder.h
View file

@ -46,18 +46,16 @@ typedef struct _GstNvDecoderFrame
gint ref_count;
} GstNvDecoderFrame;
typedef enum
{
GST_NV_DECODER_OUTPUT_TYPE_SYSTEM = 0,
GST_NV_DECODER_OUTPUT_TYPE_GL,
GST_NV_DECODER_OUTPUT_TYPE_CUDA,
/* FIXME: add support D3D11 memory */
} GstNvDecoderOutputType;
GstNvDecoder * gst_nv_decoder_new (GstCudaContext * context);
GstNvDecoder * gst_nv_decoder_new (GstCudaContext * context,
cudaVideoCodec codec,
GstVideoInfo * info,
guint pool_size);
gboolean gst_nv_decoder_is_configured (GstNvDecoder * decoder);
gboolean gst_nv_decoder_configure (GstNvDecoder * decoder,
cudaVideoCodec codec,
GstVideoInfo * info,
gint coded_width,
gint coded_height,
guint pool_size);
GstNvDecoderFrame * gst_nv_decoder_new_frame (GstNvDecoder * decoder);
@ -68,11 +66,10 @@ void gst_nv_decoder_frame_unref (GstNvDecoderFrame * frame);
gboolean gst_nv_decoder_decode_picture (GstNvDecoder * decoder,
CUVIDPICPARAMS * params);
gboolean gst_nv_decoder_finish_frame (GstNvDecoder * decoder,
GstNvDecoderOutputType output_type,
GstObject * graphics_context,
GstNvDecoderFrame *frame,
GstBuffer *buffer);
gboolean gst_nv_decoder_finish_frame (GstNvDecoder * decoder,
GstVideoDecoder * videodec,
GstNvDecoderFrame *frame,
GstBuffer ** buffer);
/* utils for class registration */
gboolean gst_nv_decoder_check_device_caps (CUcontext cuda_ctx,
@ -83,43 +80,22 @@ gboolean gst_nv_decoder_check_device_caps (CUcontext cuda_ctx,
const gchar * gst_cuda_video_codec_to_string (cudaVideoCodec codec);
/* helper methods */
gboolean gst_nv_decoder_ensure_element_data (GstElement * decoder,
guint cuda_device_id,
GstCudaContext ** cuda_context,
CUstream * cuda_stream,
GstObject ** gl_display,
GstObject ** other_gl_context);
gboolean gst_nv_decoder_handle_set_context (GstNvDecoder * decoder,
GstElement * videodec,
GstContext * context);
void gst_nv_decoder_set_context (GstElement * decoder,
GstContext * context,
guint cuda_device_id,
GstCudaContext ** cuda_context,
GstObject ** gl_display,
GstObject ** other_gl_context);
gboolean gst_nv_decoder_handle_context_query (GstNvDecoder * decoder,
GstVideoDecoder * videodec,
GstQuery * query);
gboolean gst_nv_decoder_handle_context_query (GstElement * decoder,
GstQuery * query,
GstCudaContext * cuda_context,
GstObject * gl_display,
GstObject * gl_context,
GstObject * other_gl_context);
gboolean gst_nv_decoder_negotiate (GstVideoDecoder * decoder,
gboolean gst_nv_decoder_negotiate (GstNvDecoder * decoder,
GstVideoDecoder * videodec,
GstVideoCodecState * input_state,
GstVideoFormat format,
guint width,
guint height,
GstObject * gl_display,
GstObject * other_gl_context,
GstObject ** gl_context,
GstVideoCodecState ** output_state,
GstNvDecoderOutputType * output_type);
GstVideoCodecState ** output_state);
gboolean gst_nv_decoder_decide_allocation (GstNvDecoder * nvdec,
GstVideoDecoder * decoder,
GstQuery * query,
GstObject * gl_context,
GstNvDecoderOutputType output_type);
gboolean gst_nv_decoder_decide_allocation (GstNvDecoder * decoder,
GstVideoDecoder * videodec,
GstQuery * query);
G_END_DECLS

123
sys/nvcodec/gstnvh264dec.c
View file

@ -93,7 +93,6 @@ struct _GstNvH264Dec
const GstH264PPS *last_pps;
GstCudaContext *context;
CUstream cuda_stream;
GstNvDecoder *decoder;
CUVIDPICPARAMS params;
@ -113,14 +112,6 @@ struct _GstNvH264Dec
guint bitdepth;
guint chroma_format_idc;
gint max_dpb_size;
GstVideoFormat out_format;
/* For OpenGL interop. */
GstObject *gl_display;
GstObject *gl_context;
GstObject *other_gl_context;
GstNvDecoderOutputType output_type;
};
struct _GstNvH264DecClass
@ -232,9 +223,15 @@ gst_nv_h264_dec_set_context (GstElement * element, GstContext * context)
GST_DEBUG_OBJECT (self, "set context %s",
gst_context_get_context_type (context));
gst_nv_decoder_set_context (element, context, klass->cuda_device_id,
&self->context, &self->gl_display, &self->other_gl_context);
if (gst_cuda_handle_set_context (element, context, klass->cuda_device_id,
&self->context)) {
goto done;
}
if (self->decoder)
gst_nv_decoder_handle_set_context (self->decoder, element, context);
done:
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}
@ -248,7 +245,6 @@ gst_d3d11_h264_dec_reset (GstNvH264Dec * self)
self->coded_height = 0;
self->bitdepth = 0;
self->chroma_format_idc = 0;
self->out_format = GST_VIDEO_FORMAT_UNKNOWN;
self->max_dpb_size = 0;
}
@ -258,13 +254,20 @@ gst_nv_h264_dec_open (GstVideoDecoder * decoder)
GstNvH264Dec *self = GST_NV_H264_DEC (decoder);
GstNvH264DecClass *klass = GST_NV_H264_DEC_GET_CLASS (self);
if (!gst_nv_decoder_ensure_element_data (GST_ELEMENT (self),
klass->cuda_device_id, &self->context, &self->cuda_stream,
&self->gl_display, &self->other_gl_context)) {
if (!gst_cuda_ensure_element_context (GST_ELEMENT (self),
klass->cuda_device_id, &self->context)) {
GST_ERROR_OBJECT (self, "Required element data is unavailable");
return FALSE;
}
self->decoder = gst_nv_decoder_new (self->context);
if (!self->decoder) {
GST_ERROR_OBJECT (self, "Failed to create decoder object");
gst_clear_object (&self->context);
return FALSE;
}
gst_d3d11_h264_dec_reset (self);
return TRUE;
@ -277,19 +280,7 @@ gst_nv_h264_dec_close (GstVideoDecoder * decoder)
g_clear_pointer (&self->output_state, gst_video_codec_state_unref);
gst_clear_object (&self->decoder);
if (self->context && self->cuda_stream) {
if (gst_cuda_context_push (self->context)) {
gst_cuda_result (CuStreamDestroy (self->cuda_stream));
gst_cuda_context_pop (NULL);
}
}
gst_clear_object (&self->gl_context);
gst_clear_object (&self->other_gl_context);
gst_clear_object (&self->gl_display);
gst_clear_object (&self->context);
self->cuda_stream = NULL;
return TRUE;
}
@ -302,9 +293,8 @@ gst_nv_h264_dec_negotiate (GstVideoDecoder * decoder)
GST_DEBUG_OBJECT (self, "negotiate");
gst_nv_decoder_negotiate (decoder, h264dec->input_state, self->out_format,
self->width, self->height, self->gl_display, self->other_gl_context,
&self->gl_context, &self->output_state, &self->output_type);
gst_nv_decoder_negotiate (self->decoder, decoder, h264dec->input_state,
&self->output_state);
/* TODO: add support D3D11 memory */
@ -316,8 +306,10 @@ gst_nv_h264_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
{
GstNvH264Dec *self = GST_NV_H264_DEC (decoder);
gst_nv_decoder_decide_allocation (self->decoder, decoder, query,
self->gl_context, self->output_type);
if (!gst_nv_decoder_decide_allocation (self->decoder, decoder, query)) {
GST_WARNING_OBJECT (self, "Failed to handle decide allocation");
return FALSE;
}
return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
(decoder, query);
@ -330,9 +322,11 @@ gst_nv_h264_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:
if (gst_nv_decoder_handle_context_query (GST_ELEMENT (self), query,
self->context, self->gl_display, self->gl_context,
self->other_gl_context)) {
if (gst_cuda_handle_context_query (GST_ELEMENT (decoder), query,
self->context)) {
return TRUE;
} else if (self->decoder &&
gst_nv_decoder_handle_context_query (self->decoder, decoder, query)) {
return TRUE;
}
break;
@ -390,44 +384,38 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps,
modified = TRUE;
}
if (modified || !self->decoder) {
if (modified || !gst_nv_decoder_is_configured (self->decoder)) {
GstVideoInfo info;
self->out_format = GST_VIDEO_FORMAT_UNKNOWN;
GstVideoFormat out_format = GST_VIDEO_FORMAT_UNKNOWN;
if (self->bitdepth == 8) {
if (self->chroma_format_idc == 1)
self->out_format = GST_VIDEO_FORMAT_NV12;
out_format = GST_VIDEO_FORMAT_NV12;
else {
GST_FIXME_OBJECT (self, "Could not support 8bits non-4:2:0 format");
}
} else if (self->bitdepth == 10) {
if (self->chroma_format_idc == 1)
self->out_format = GST_VIDEO_FORMAT_P010_10LE;
out_format = GST_VIDEO_FORMAT_P010_10LE;
else {
GST_FIXME_OBJECT (self, "Could not support 10bits non-4:2:0 format");
}
}
if (self->out_format == GST_VIDEO_FORMAT_UNKNOWN) {
if (out_format == GST_VIDEO_FORMAT_UNKNOWN) {
GST_ERROR_OBJECT (self, "Could not support bitdepth/chroma format");
return FALSE;
}
gst_clear_object (&self->decoder);
gst_video_info_set_format (&info,
self->out_format, self->width, self->height);
gst_video_info_set_format (&info, out_format, self->width, self->height);
self->max_dpb_size = max_dpb_size;
/* FIXME: add support cudaVideoCodec_H264_SVC and cudaVideoCodec_H264_MVC */
self->decoder = gst_nv_decoder_new (self->context, cudaVideoCodec_H264,
&info,
/* Additional 2 buffers for margin */
max_dpb_size + 2);
if (!self->decoder) {
GST_ERROR_OBJECT (self, "Failed to create decoder");
if (!gst_nv_decoder_configure (self->decoder,
cudaVideoCodec_H264, &info, self->coded_width, self->coded_height,
/* Additional 2 buffers for margin */
max_dpb_size + 2)) {
GST_ERROR_OBJECT (self, "Failed to configure decoder");
return FALSE;
}
@ -473,7 +461,6 @@ gst_nv_h264_dec_output_picture (GstH264Decoder * decoder,
GstNvH264Dec *self = GST_NV_H264_DEC (decoder);
GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
GstNvDecoderFrame *decoder_frame;
gboolean ret G_GNUC_UNUSED = FALSE;
GST_LOG_OBJECT (self,
"Outputting picture %p (poc %d)", picture, picture->pic_order_cnt);
@ -485,36 +472,12 @@ gst_nv_h264_dec_output_picture (GstH264Decoder * decoder,
goto error;
}
frame->output_buffer = gst_video_decoder_allocate_output_buffer (vdec);
if (!frame->output_buffer) {
GST_ERROR_OBJECT (self, "Couldn't allocate output buffer");
if (!gst_nv_decoder_finish_frame (self->decoder, vdec, decoder_frame,
&frame->output_buffer)) {
GST_ERROR_OBJECT (self, "Failed to handle output picture");
goto error;
}
if (self->output_type == GST_NV_DECODER_OUTPUT_TYPE_GL) {
ret = gst_nv_decoder_finish_frame (self->decoder,
GST_NV_DECODER_OUTPUT_TYPE_GL, self->gl_context,
decoder_frame, frame->output_buffer);
/* FIXME: This is the case where OpenGL context of downstream glbufferpool
* belongs to non-nvidia (or different device).
* There should be enhancement to ensure nvdec has compatible OpenGL context
*/
if (!ret) {
GST_WARNING_OBJECT (self,
"Couldn't copy frame to GL memory, fallback to system memory");
self->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
}
}
if (!ret) {
if (!gst_nv_decoder_finish_frame (self->decoder,
self->output_type, NULL, decoder_frame, frame->output_buffer)) {
GST_ERROR_OBJECT (self, "Failed to finish frame");
goto error;
}
}
gst_h264_picture_unref (picture);
return gst_video_decoder_finish_frame (vdec, frame);

121
sys/nvcodec/gstnvh265dec.c
View file

@ -93,7 +93,6 @@ struct _GstNvH265Dec
const GstH265PPS *last_pps;
GstCudaContext *context;
CUstream cuda_stream;
GstNvDecoder *decoder;
CUVIDPICPARAMS params;
@ -112,14 +111,6 @@ struct _GstNvH265Dec
guint coded_width, coded_height;
guint bitdepth;
guint chroma_format_idc;
GstVideoFormat out_format;
/* For OpenGL interop. */
GstObject *gl_display;
GstObject *gl_context;
GstObject *other_gl_context;
GstNvDecoderOutputType output_type;
};
struct _GstNvH265DecClass
@ -226,9 +217,15 @@ gst_nv_h265_dec_set_context (GstElement * element, GstContext * context)
GST_DEBUG_OBJECT (self, "set context %s",
gst_context_get_context_type (context));
gst_nv_decoder_set_context (element, context, klass->cuda_device_id,
&self->context, &self->gl_display, &self->other_gl_context);
if (gst_cuda_handle_set_context (element, context, klass->cuda_device_id,
&self->context)) {
goto done;
}
if (self->decoder)
gst_nv_decoder_handle_set_context (self->decoder, element, context);
done:
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}
@ -238,13 +235,20 @@ gst_nv_h265_dec_open (GstVideoDecoder * decoder)
GstNvH265Dec *self = GST_NV_H265_DEC (decoder);
GstNvH265DecClass *klass = GST_NV_H265_DEC_GET_CLASS (self);
if (!gst_nv_decoder_ensure_element_data (GST_ELEMENT (self),
klass->cuda_device_id, &self->context, &self->cuda_stream,
&self->gl_display, &self->other_gl_context)) {
if (!gst_cuda_ensure_element_context (GST_ELEMENT (self),
klass->cuda_device_id, &self->context)) {
GST_ERROR_OBJECT (self, "Required element data is unavailable");
return FALSE;
}
self->decoder = gst_nv_decoder_new (self->context);
if (!self->decoder) {
GST_ERROR_OBJECT (self, "Failed to create decoder object");
gst_clear_object (&self->context);
return FALSE;
}
return TRUE;
}
@ -255,19 +259,7 @@ gst_nv_h265_dec_close (GstVideoDecoder * decoder)
g_clear_pointer (&self->output_state, gst_video_codec_state_unref);
gst_clear_object (&self->decoder);
if (self->context && self->cuda_stream) {
if (gst_cuda_context_push (self->context)) {
gst_cuda_result (CuStreamDestroy (self->cuda_stream));
gst_cuda_context_pop (NULL);
}
}
gst_clear_object (&self->gl_context);
gst_clear_object (&self->other_gl_context);
gst_clear_object (&self->gl_display);
gst_clear_object (&self->context);
self->cuda_stream = NULL;
return TRUE;
}
@ -280,9 +272,8 @@ gst_nv_h265_dec_negotiate (GstVideoDecoder * decoder)
GST_DEBUG_OBJECT (self, "negotiate");
gst_nv_decoder_negotiate (decoder, h265dec->input_state, self->out_format,
self->width, self->height, self->gl_display, self->other_gl_context,
&self->gl_context, &self->output_state, &self->output_type);
gst_nv_decoder_negotiate (self->decoder, decoder, h265dec->input_state,
&self->output_state);
/* TODO: add support D3D11 memory */
@ -294,8 +285,10 @@ gst_nv_h265_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
{
GstNvH265Dec *self = GST_NV_H265_DEC (decoder);
gst_nv_decoder_decide_allocation (self->decoder, decoder, query,
self->gl_context, self->output_type);
if (!gst_nv_decoder_decide_allocation (self->decoder, decoder, query)) {
GST_WARNING_OBJECT (self, "Failed to handle decide allocation");
return FALSE;
}
return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
(decoder, query);
@ -308,9 +301,11 @@ gst_nv_h265_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:
if (gst_nv_decoder_handle_context_query (GST_ELEMENT (self), query,
self->context, self->gl_display, self->gl_context,
self->other_gl_context)) {
if (gst_cuda_handle_context_query (GST_ELEMENT (decoder), query,
self->context)) {
return TRUE;
} else if (self->decoder &&
gst_nv_decoder_handle_context_query (self->decoder, decoder, query)) {
return TRUE;
}
break;
@ -362,42 +357,36 @@ gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
modified = TRUE;
}
if (modified || !self->decoder) {
if (modified || !gst_nv_decoder_is_configured (self->decoder)) {
GstVideoInfo info;
self->out_format = GST_VIDEO_FORMAT_UNKNOWN;
GstVideoFormat out_format = GST_VIDEO_FORMAT_UNKNOWN;
if (self->bitdepth == 8) {
if (self->chroma_format_idc == 1)
self->out_format = GST_VIDEO_FORMAT_NV12;
out_format = GST_VIDEO_FORMAT_NV12;
else {
GST_FIXME_OBJECT (self, "Could not support 8bits non-4:2:0 format");
}
} else if (self->bitdepth == 10) {
if (self->chroma_format_idc == 1)
self->out_format = GST_VIDEO_FORMAT_P010_10LE;
out_format = GST_VIDEO_FORMAT_P010_10LE;
else {
GST_FIXME_OBJECT (self, "Could not support 10bits non-4:2:0 format");
}
}
if (self->out_format == GST_VIDEO_FORMAT_UNKNOWN) {
if (out_format == GST_VIDEO_FORMAT_UNKNOWN) {
GST_ERROR_OBJECT (self, "Could not support bitdepth/chroma format");
return FALSE;
}
gst_clear_object (&self->decoder);
gst_video_info_set_format (&info, out_format, self->width, self->height);
gst_video_info_set_format (&info,
self->out_format, self->width, self->height);
self->decoder = gst_nv_decoder_new (self->context, cudaVideoCodec_HEVC,
&info,
/* Additional 2 buffers for margin */
max_dpb_size + 2);
if (!self->decoder) {
GST_ERROR_OBJECT (self, "Failed to create decoder");
if (!gst_nv_decoder_configure (self->decoder,
cudaVideoCodec_HEVC, &info, self->coded_width, self->coded_height,
/* Additional 2 buffers for margin */
max_dpb_size + 2)) {
GST_ERROR_OBJECT (self, "Failed to configure decoder");
return FALSE;
}
@ -442,7 +431,6 @@ gst_nv_h265_dec_output_picture (GstH265Decoder * decoder,
GstNvH265Dec *self = GST_NV_H265_DEC (decoder);
GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
GstNvDecoderFrame *decoder_frame;
gboolean ret G_GNUC_UNUSED = FALSE;
GST_LOG_OBJECT (self,
"Outputting picture %p (poc %d)", picture, picture->pic_order_cnt);
@ -454,31 +442,10 @@ gst_nv_h265_dec_output_picture (GstH265Decoder * decoder,
goto error;
}
frame->output_buffer =
gst_video_decoder_allocate_output_buffer (GST_VIDEO_DECODER (self));;
if (self->output_type == GST_NV_DECODER_OUTPUT_TYPE_GL) {
ret = gst_nv_decoder_finish_frame (self->decoder,
GST_NV_DECODER_OUTPUT_TYPE_GL, self->gl_context,
decoder_frame, frame->output_buffer);
/* FIXME: This is the case where OpenGL context of downstream glbufferpool
* belongs to non-nvidia (or different device).
* There should be enhancement to ensure nvdec has compatible OpenGL context
*/
if (!ret) {
GST_WARNING_OBJECT (self,
"Couldn't copy frame to GL memory, fallback to system memory");
self->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
}
}
if (!ret) {
if (!gst_nv_decoder_finish_frame (self->decoder,
self->output_type, NULL, decoder_frame, frame->output_buffer)) {
GST_ERROR_OBJECT (self, "Failed to finish frame");
goto error;
}
if (!gst_nv_decoder_finish_frame (self->decoder, vdec, decoder_frame,
&frame->output_buffer)) {
GST_ERROR_OBJECT (self, "Failed to handle output picture");
goto error;
}
gst_h265_picture_unref (picture);

106
sys/nvcodec/gstnvvp8dec.c
View file

@ -40,18 +40,10 @@ struct _GstNvVp8Dec
GstVideoCodecState *output_state;
GstCudaContext *context;
CUstream cuda_stream;
GstNvDecoder *decoder;
CUVIDPICPARAMS params;
guint width, height;
/* For OpenGL interop. */
GstObject *gl_display;
GstObject *gl_context;
GstObject *other_gl_context;
GstNvDecoderOutputType output_type;
};
struct _GstNvVp8DecClass
@ -134,9 +126,15 @@ gst_nv_vp8_dec_set_context (GstElement * element, GstContext * context)
GST_DEBUG_OBJECT (self, "set context %s",
gst_context_get_context_type (context));
gst_nv_decoder_set_context (element, context, klass->cuda_device_id,
&self->context, &self->gl_display, &self->other_gl_context);
if (gst_cuda_handle_set_context (element, context, klass->cuda_device_id,
&self->context)) {
goto done;
}
if (self->decoder)
gst_nv_decoder_handle_set_context (self->decoder, element, context);
done:
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}
@ -146,13 +144,20 @@ gst_nv_vp8_dec_open (GstVideoDecoder * decoder)
GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
GstNvVp8DecClass *klass = GST_NV_VP8_DEC_GET_CLASS (self);
if (!gst_nv_decoder_ensure_element_data (GST_ELEMENT (self),
klass->cuda_device_id, &self->context, &self->cuda_stream,
&self->gl_display, &self->other_gl_context)) {
if (!gst_cuda_ensure_element_context (GST_ELEMENT (self),
klass->cuda_device_id, &self->context)) {
GST_ERROR_OBJECT (self, "Required element data is unavailable");
return FALSE;
}
self->decoder = gst_nv_decoder_new (self->context);
if (!self->decoder) {
GST_ERROR_OBJECT (self, "Failed to create decoder object");
gst_clear_object (&self->context);
return FALSE;
}
return TRUE;
}
@ -163,19 +168,7 @@ gst_nv_vp8_dec_close (GstVideoDecoder * decoder)
g_clear_pointer (&self->output_state, gst_video_codec_state_unref);
gst_clear_object (&self->decoder);
if (self->context && self->cuda_stream) {
if (gst_cuda_context_push (self->context)) {
gst_cuda_result (CuStreamDestroy (self->cuda_stream));
gst_cuda_context_pop (NULL);
}
}
gst_clear_object (&self->gl_context);
gst_clear_object (&self->other_gl_context);
gst_clear_object (&self->gl_display);
gst_clear_object (&self->context);
self->cuda_stream = NULL;
return TRUE;
}
@ -188,9 +181,8 @@ gst_nv_vp8_dec_negotiate (GstVideoDecoder * decoder)
GST_DEBUG_OBJECT (self, "negotiate");
gst_nv_decoder_negotiate (decoder, vp8dec->input_state, GST_VIDEO_FORMAT_NV12,
self->width, self->height, self->gl_display, self->other_gl_context,
&self->gl_context, &self->output_state, &self->output_type);
gst_nv_decoder_negotiate (self->decoder, decoder, vp8dec->input_state,
&self->output_state);
/* TODO: add support D3D11 memory */
@ -202,8 +194,10 @@ gst_nv_vp8_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
{
GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
gst_nv_decoder_decide_allocation (self->decoder, decoder, query,
self->gl_context, self->output_type);
if (!gst_nv_decoder_decide_allocation (self->decoder, decoder, query)) {
GST_WARNING_OBJECT (self, "Failed to handle decide allocation");
return FALSE;
}
return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
(decoder, query);
@ -216,9 +210,11 @@ gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:
if (gst_nv_decoder_handle_context_query (GST_ELEMENT (self), query,
self->context, self->gl_display, self->gl_context,
self->other_gl_context)) {
if (gst_cuda_handle_context_query (GST_ELEMENT (decoder), query,
self->context)) {
return TRUE;
} else if (self->decoder &&
gst_nv_decoder_handle_context_query (self->decoder, decoder, query)) {
return TRUE;
}
break;
@ -250,19 +246,16 @@ gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
modified = TRUE;
}
if (modified || !self->decoder) {
if (modified || !gst_nv_decoder_is_configured (self->decoder)) {
GstVideoInfo info;
gst_clear_object (&self->decoder);
gst_video_info_set_format (&info,
GST_VIDEO_FORMAT_NV12, self->width, self->height);
self->decoder = gst_nv_decoder_new (self->context, cudaVideoCodec_VP8,
&info, NUM_OUTPUT_VIEW);
if (!self->decoder) {
GST_ERROR_OBJECT (self, "Failed to create decoder");
if (!gst_nv_decoder_configure (self->decoder,
cudaVideoCodec_VP8, &info, self->width, self->height,
NUM_OUTPUT_VIEW)) {
GST_ERROR_OBJECT (self, "Failed to configure decoder");
return FALSE;
}
@ -408,7 +401,6 @@ gst_nv_vp8_dec_output_picture (GstVp8Decoder * decoder,
GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
GstNvDecoderFrame *decoder_frame;
gboolean ret G_GNUC_UNUSED = FALSE;
GST_LOG_OBJECT (self, "Outputting picture %p", picture);
@ -418,36 +410,12 @@ gst_nv_vp8_dec_output_picture (GstVp8Decoder * decoder,
goto error;
}
frame->output_buffer = gst_video_decoder_allocate_output_buffer (vdec);
if (!frame->output_buffer) {
GST_ERROR_OBJECT (self, "Couldn't allocate output buffer");
if (!gst_nv_decoder_finish_frame (self->decoder, vdec, decoder_frame,
&frame->output_buffer)) {
GST_ERROR_OBJECT (self, "Failed to handle output picture");
goto error;
}
if (self->output_type == GST_NV_DECODER_OUTPUT_TYPE_GL) {
ret = gst_nv_decoder_finish_frame (self->decoder,
GST_NV_DECODER_OUTPUT_TYPE_GL, self->gl_context,
decoder_frame, frame->output_buffer);
/* FIXME: This is the case where OpenGL context of downstream glbufferpool
* belongs to non-nvidia (or different device).
* There should be enhancement to ensure nvdec has compatible OpenGL context
*/
if (!ret) {
GST_WARNING_OBJECT (self,
"Couldn't copy frame to GL memory, fallback to system memory");
self->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
}
}
if (!ret) {
if (!gst_nv_decoder_finish_frame (self->decoder,
self->output_type, NULL, decoder_frame, frame->output_buffer)) {
GST_ERROR_OBJECT (self, "Failed to finish frame");
goto error;
}
}
gst_vp8_picture_unref (picture);
return gst_video_decoder_finish_frame (vdec, frame);

122
sys/nvcodec/gstnvvp9dec.c
View file

@ -40,21 +40,11 @@ struct _GstNvVp9Dec
GstVideoCodecState *output_state;
GstCudaContext *context;
CUstream cuda_stream;
GstNvDecoder *decoder;
CUVIDPICPARAMS params;
guint width, height;
GstVP9Profile profile;
GstVideoFormat out_format;
/* For OpenGL interop. */
GstObject *gl_display;
GstObject *gl_context;
GstObject *other_gl_context;
GstNvDecoderOutputType output_type;
};
struct _GstNvVp9DecClass
@ -141,9 +131,15 @@ gst_nv_vp9_dec_set_context (GstElement * element, GstContext * context)
GST_DEBUG_OBJECT (self, "set context %s",
gst_context_get_context_type (context));
gst_nv_decoder_set_context (element, context, klass->cuda_device_id,
&self->context, &self->gl_display, &self->other_gl_context);
if (gst_cuda_handle_set_context (element, context, klass->cuda_device_id,
&self->context)) {
goto done;
}
if (self->decoder)
gst_nv_decoder_handle_set_context (self->decoder, element, context);
done:
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}
@ -153,13 +149,20 @@ gst_nv_vp9_dec_open (GstVideoDecoder * decoder)
GstNvVp9Dec *self = GST_NV_VP9_DEC (decoder);
GstNvVp9DecClass *klass = GST_NV_VP9_DEC_GET_CLASS (self);
if (!gst_nv_decoder_ensure_element_data (GST_ELEMENT (self),
klass->cuda_device_id, &self->context, &self->cuda_stream,
&self->gl_display, &self->other_gl_context)) {
if (!gst_cuda_ensure_element_context (GST_ELEMENT (self),
klass->cuda_device_id, &self->context)) {
GST_ERROR_OBJECT (self, "Required element data is unavailable");
return FALSE;
}
self->decoder = gst_nv_decoder_new (self->context);
if (!self->decoder) {
GST_ERROR_OBJECT (self, "Failed to create decoder object");
gst_clear_object (&self->context);
return FALSE;
}
return TRUE;
}
@ -170,19 +173,7 @@ gst_nv_vp9_dec_close (GstVideoDecoder * decoder)
g_clear_pointer (&self->output_state, gst_video_codec_state_unref);
gst_clear_object (&self->decoder);
if (self->context && self->cuda_stream) {
if (gst_cuda_context_push (self->context)) {
gst_cuda_result (CuStreamDestroy (self->cuda_stream));
gst_cuda_context_pop (NULL);
}
}
gst_clear_object (&self->gl_context);
gst_clear_object (&self->other_gl_context);
gst_clear_object (&self->gl_display);
gst_clear_object (&self->context);
self->cuda_stream = NULL;
return TRUE;
}
@ -195,9 +186,8 @@ gst_nv_vp9_dec_negotiate (GstVideoDecoder * decoder)
GST_DEBUG_OBJECT (self, "negotiate");
gst_nv_decoder_negotiate (decoder, vp9dec->input_state, self->out_format,
self->width, self->height, self->gl_display, self->other_gl_context,
&self->gl_context, &self->output_state, &self->output_type);
gst_nv_decoder_negotiate (self->decoder, decoder, vp9dec->input_state,
&self->output_state);
/* TODO: add support D3D11 memory */
@ -209,8 +199,10 @@ gst_nv_vp9_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
{
GstNvVp9Dec *self = GST_NV_VP9_DEC (decoder);
gst_nv_decoder_decide_allocation (self->decoder, decoder, query,
self->gl_context, self->output_type);
if (!gst_nv_decoder_decide_allocation (self->decoder, decoder, query)) {
GST_WARNING_OBJECT (self, "Failed to handle decide allocation");
return FALSE;
}
return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
(decoder, query);
@ -223,9 +215,11 @@ gst_nv_vp9_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:
if (gst_nv_decoder_handle_context_query (GST_ELEMENT (self), query,
self->context, self->gl_display, self->gl_context,
self->other_gl_context)) {
if (gst_cuda_handle_context_query (GST_ELEMENT (decoder), query,
self->context)) {
return TRUE;
} else if (self->decoder &&
gst_nv_decoder_handle_context_query (self->decoder, decoder, query)) {
return TRUE;
}
break;
@ -266,35 +260,30 @@ gst_nv_vp9_dec_new_sequence (GstVp9Decoder * decoder,
modified = TRUE;
}
if (modified || !self->decoder) {
if (modified || !gst_nv_decoder_is_configured (self->decoder)) {
GstVideoInfo info;
gst_clear_object (&self->decoder);
self->out_format = GST_VIDEO_FORMAT_UNKNOWN;
GstVideoFormat out_format = GST_VIDEO_FORMAT_UNKNOWN;
if (self->profile == GST_VP9_PROFILE_0) {
self->out_format = GST_VIDEO_FORMAT_NV12;
out_format = GST_VIDEO_FORMAT_NV12;
} else if (self->profile == GST_VP9_PROFILE_2) {
if (parser->bit_depth == 10)
self->out_format = GST_VIDEO_FORMAT_P010_10LE;
out_format = GST_VIDEO_FORMAT_P010_10LE;
else
self->out_format = GST_VIDEO_FORMAT_P016_LE;
out_format = GST_VIDEO_FORMAT_P016_LE;
}
if (self->out_format == GST_VIDEO_FORMAT_UNKNOWN) {
if (out_format == GST_VIDEO_FORMAT_UNKNOWN) {
GST_ERROR_OBJECT (self, "Could not support profile %d", self->profile);
return FALSE;
}
gst_video_info_set_format (&info,
self->out_format, self->width, self->height);
gst_video_info_set_format (&info, out_format, self->width, self->height);
self->decoder = gst_nv_decoder_new (self->context, cudaVideoCodec_VP9,
&info, NUM_OUTPUT_VIEW);
if (!self->decoder) {
GST_ERROR_OBJECT (self, "Failed to create decoder");
if (!gst_nv_decoder_configure (self->decoder,
cudaVideoCodec_VP9, &info, self->width, self->height,
NUM_OUTPUT_VIEW)) {
GST_ERROR_OBJECT (self, "Failed to configure decoder");
return FALSE;
}
@ -519,7 +508,6 @@ gst_nv_vp9_dec_output_picture (GstVp9Decoder * decoder,
GstNvVp9Dec *self = GST_NV_VP9_DEC (decoder);
GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
GstNvDecoderFrame *decoder_frame;
gboolean ret G_GNUC_UNUSED = FALSE;
GST_LOG_OBJECT (self, "Outputting picture %p", picture);
@ -529,36 +517,12 @@ gst_nv_vp9_dec_output_picture (GstVp9Decoder * decoder,
goto error;
}
frame->output_buffer = gst_video_decoder_allocate_output_buffer (vdec);
if (!frame->output_buffer) {
GST_ERROR_OBJECT (self, "Couldn't allocate output buffer");
if (!gst_nv_decoder_finish_frame (self->decoder, vdec, decoder_frame,
&frame->output_buffer)) {
GST_ERROR_OBJECT (self, "Failed to handle output picture");
goto error;
}
if (self->output_type == GST_NV_DECODER_OUTPUT_TYPE_GL) {
ret = gst_nv_decoder_finish_frame (self->decoder,
GST_NV_DECODER_OUTPUT_TYPE_GL, self->gl_context,
decoder_frame, frame->output_buffer);
/* FIXME: This is the case where OpenGL context of downstream glbufferpool
* belongs to non-nvidia (or different device).
* There should be enhancement to ensure nvdec has compatible OpenGL context
*/
if (!ret) {
GST_WARNING_OBJECT (self,
"Couldn't copy frame to GL memory, fallback to system memory");
self->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
}
}
if (!ret) {
if (!gst_nv_decoder_finish_frame (self->decoder,
self->output_type, NULL, decoder_frame, frame->output_buffer)) {
GST_ERROR_OBJECT (self, "Failed to finish frame");
goto error;
}
}
gst_vp9_picture_unref (picture);
return gst_video_decoder_finish_frame (vdec, frame);