From 82c00057930fa2b2b7dc7f06d91b2fd14aa9a434 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Wed, 7 Apr 2021 18:24:27 -0300 Subject: [PATCH] codecs: gstvp8decoder: add support for render delay Some decoding APIs support delayed output for performance reasons. One example would be to request decoding for multiple frames and then query for the oldest frame in the output queue. This also increases throughput for transcoding and improves seek performance when supported by the underlying backend. Introduce support in the vp8 base class, so that backends that support render delays can actually implement it. Part-of: --- gst-libs/gst/codecs/gstvp8decoder.c | 73 ++++++++++++++++++++++++++++- gst-libs/gst/codecs/gstvp8decoder.h | 15 ++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/codecs/gstvp8decoder.c b/gst-libs/gst/codecs/gstvp8decoder.c index 870cc9c253..1ae42a629e 100644 --- a/gst-libs/gst/codecs/gstvp8decoder.c +++ b/gst-libs/gst/codecs/gstvp8decoder.c @@ -29,6 +29,7 @@ #include #endif +#include #include "gstvp8decoder.h" GST_DEBUG_CATEGORY (gst_vp8_decoder_debug); @@ -42,8 +43,19 @@ struct _GstVp8DecoderPrivate gboolean had_sequence; GstVp8Parser parser; gboolean wait_keyframe; + guint preferred_output_delay; + /* for delayed output */ + GstQueueArray *output_queue; + gboolean is_live; }; +typedef struct +{ + GstVideoCodecFrame *frame; + GstVp8Picture *picture; + GstVp8Decoder *self; +} GstVp8DecoderOutputFrame; + #define parent_class gst_vp8_decoder_parent_class G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVp8Decoder, gst_vp8_decoder, GST_TYPE_VIDEO_DECODER, @@ -60,6 +72,11 @@ static gboolean gst_vp8_decoder_flush (GstVideoDecoder * decoder); static GstFlowReturn gst_vp8_decoder_drain (GstVideoDecoder * decoder); static GstFlowReturn gst_vp8_decoder_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame); +static void gst_vp8_decoder_clear_output_frame (GstVp8DecoderOutputFrame * + output_frame); +static void gst_vp8_decoder_drain_output_queue (GstVp8Decoder * self, + guint num); + static void gst_vp8_decoder_class_init (GstVp8DecoderClass * klass) @@ -93,6 +110,11 @@ gst_vp8_decoder_start (GstVideoDecoder * decoder) gst_vp8_parser_init (&priv->parser); priv->wait_keyframe = TRUE; + priv->output_queue = + gst_queue_array_new_for_struct (sizeof (GstVp8DecoderOutputFrame), 1); + gst_queue_array_set_clear_func (priv->output_queue, + (GDestroyNotify) gst_vp8_decoder_clear_output_frame); + return TRUE; } @@ -106,12 +128,14 @@ gst_vp8_decoder_reset (GstVp8Decoder * self) gst_vp8_picture_clear (&self->alt_ref_picture); priv->wait_keyframe = TRUE; + gst_queue_array_clear (priv->output_queue); } static gboolean gst_vp8_decoder_stop (GstVideoDecoder * decoder) { GstVp8Decoder *self = GST_VP8_DECODER (decoder); + GstVp8DecoderPrivate *priv = self->priv; if (self->input_state) { gst_video_codec_state_unref (self->input_state); @@ -119,6 +143,7 @@ gst_vp8_decoder_stop (GstVideoDecoder * decoder) } gst_vp8_decoder_reset (self); + gst_queue_array_free (priv->output_queue); return TRUE; } @@ -157,6 +182,7 @@ gst_vp8_decoder_set_format (GstVideoDecoder * decoder, { GstVp8Decoder *self = GST_VP8_DECODER (decoder); GstVp8DecoderPrivate *priv = self->priv; + GstQuery *query; GST_DEBUG_OBJECT (decoder, "Set format"); @@ -168,6 +194,11 @@ gst_vp8_decoder_set_format (GstVideoDecoder * decoder, priv->width = GST_VIDEO_INFO_WIDTH (&state->info); priv->height = GST_VIDEO_INFO_HEIGHT (&state->info); + query = gst_query_new_latency (); + if (gst_pad_peer_query (GST_VIDEO_DECODER_SINK_PAD (self), query)) + gst_query_parse_latency (query, &priv->is_live, NULL, NULL); + gst_query_unref (query); + return TRUE; } @@ -266,11 +297,27 @@ gst_vp8_decoder_drain (GstVideoDecoder * decoder) GST_DEBUG_OBJECT (self, "drain"); + gst_vp8_decoder_drain_output_queue (GST_VP8_DECODER (decoder), 0); gst_vp8_decoder_reset (self); return GST_FLOW_OK; } +static void +gst_vp8_decoder_clear_output_frame (GstVp8DecoderOutputFrame * output_frame) +{ + if (!output_frame) + return; + + if (output_frame->frame) { + gst_video_decoder_release_frame (GST_VIDEO_DECODER (output_frame->self), + output_frame->frame); + output_frame->frame = NULL; + } + + gst_vp8_picture_clear (&output_frame->picture); +} + static GstFlowReturn gst_vp8_decoder_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame) @@ -284,12 +331,17 @@ gst_vp8_decoder_handle_frame (GstVideoDecoder * decoder, GstVp8ParserResult pres; GstVp8Picture *picture = NULL; GstFlowReturn ret = GST_FLOW_OK; + GstVp8DecoderOutputFrame output_frame; GST_LOG_OBJECT (self, "handle frame, PTS: %" GST_TIME_FORMAT ", DTS: %" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_PTS (in_buf)), GST_TIME_ARGS (GST_BUFFER_DTS (in_buf))); + if (klass->get_preferred_output_delay) + priv->preferred_output_delay = + klass->get_preferred_output_delay (self, priv->is_live); + if (!gst_buffer_map (in_buf, &map, GST_MAP_READ)) { GST_ERROR_OBJECT (self, "Cannot map buffer"); @@ -372,10 +424,13 @@ gst_vp8_decoder_handle_frame (GstVideoDecoder * decoder, ret = gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame); } else { - g_assert (klass->output_picture); - ret = klass->output_picture (self, frame, picture); + output_frame.frame = frame; + output_frame.picture = picture; + output_frame.self = self; + gst_queue_array_push_tail_struct (priv->output_queue, &output_frame); } + gst_vp8_decoder_drain_output_queue (self, priv->preferred_output_delay); return ret; unmap_and_error: @@ -396,3 +451,17 @@ error: return ret; } } + +static void +gst_vp8_decoder_drain_output_queue (GstVp8Decoder * self, guint num) +{ + GstVp8DecoderPrivate *priv = self->priv; + GstVp8DecoderClass *klass = GST_VP8_DECODER_GET_CLASS (self); + g_assert (klass->output_picture); + + while (gst_queue_array_get_length (priv->output_queue) > num) { + GstVp8DecoderOutputFrame *output_frame = (GstVp8DecoderOutputFrame *) + gst_queue_array_pop_head_struct (priv->output_queue); + klass->output_picture (self, output_frame->frame, output_frame->picture); + } +} diff --git a/gst-libs/gst/codecs/gstvp8decoder.h b/gst-libs/gst/codecs/gstvp8decoder.h index 8adc86695b..9541088b91 100644 --- a/gst-libs/gst/codecs/gstvp8decoder.h +++ b/gst-libs/gst/codecs/gstvp8decoder.h @@ -121,6 +121,21 @@ struct _GstVp8DecoderClass GstVideoCodecFrame * frame, GstVp8Picture * picture); + /** + * GstVp8DecoderClass::get_preferred_output_delay: + * @decoder: a #GstVp8Decoder + * @is_live: whether upstream is live or not + * + * Optional. Called by baseclass to query whether delaying output is + * preferred by subclass or not. + * + * Returns: the number of perferred delayed output frame + * + * Since: 1.20 + */ + guint (*get_preferred_output_delay) (GstVp8Decoder * decoder, + gboolean is_live); + /*< private >*/ gpointer padding[GST_PADDING_LARGE]; };