From 724c443a65c03027ee892f0ae02f464d04b97e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Brzezi=C5=84ski?= Date: Mon, 5 Aug 2024 14:37:57 +0200 Subject: [PATCH] videoencoder: Expose release_frame() and drop_frame() as public API release_frame() can be useful for manually dropping frames without posting QoS messages like finish_frame() would. Matches the same kind of API on the decoder side of things. Modifies the behaviour of release_frame() to make sure events from released frames are stored as 'pending' and pushed before the next non-dropped frame. This is needed because now release_frame() can be called outside of finish_frame(), so we would potentially just lose events and bad things would happen. drop_frame() was also added to match the decoder API. It functions almost identically to finish_frame() without a buffer attached to the frame, except instead of immediately pushing the frame's events, it will store them as pending. Part-of: --- girs/GstVideo-1.0.gir | 46 ++++++ .../gst-libs/gst/video/gstvideoencoder.c | 131 +++++++++++++++--- .../gst-libs/gst/video/gstvideoencoder.h | 6 + 3 files changed, 167 insertions(+), 16 deletions(-) diff --git a/girs/GstVideo-1.0.gir b/girs/GstVideo-1.0.gir index f0ac89363c..af1762780a 100644 --- a/girs/GstVideo-1.0.gir +++ b/girs/GstVideo-1.0.gir @@ -8929,6 +8929,28 @@ keep references to the frame, not the buffer. + + Removes @frame from the list of pending frames, releases it +and posts a QoS message with the frame's details on the bus. +Similar to calling gst_video_encoder_finish_frame() without a buffer +attached to @frame, but this function additionally stores events +from @frame as pending, to be pushed out alongside the next frame +submitted via gst_video_encoder_finish_frame(). + + + + + + + a #GstVideoEncoder + + + + a #GstVideoCodecFrame + + + + @frame must have a valid encoded data buffer, whose metadata fields are then appropriately set according to frame data or no buffer at @@ -8936,6 +8958,10 @@ all if the frame should be dropped. It is subsequently pushed downstream or provided to @pre_push. In any case, the frame is considered finished and released. +If @frame does not have a buffer attached, it will be dropped, and +a QoS message will be posted on the bus. Events from @frame will be +pushed out immediately. + After calling this function the output buffer of the frame is to be considered read-only. This function will also change the metadata of the buffer. @@ -9216,6 +9242,26 @@ elements (e.g. muxers). + + Removes @frame from list of pending frames and releases it, similar +to calling gst_video_encoder_finish_frame() without a buffer attached +to the frame, but does not post a QoS message or do any additional +processing. Events from @frame are moved to the pending events list. + + + + + + + a #GstVideoEncoder + + + + a #GstVideoCodecFrame + + + + Set the codec headers to be sent downstream whenever requested. diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c index 2c269e1aa3..23ea21f2fc 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c @@ -137,8 +137,12 @@ struct _GstVideoEncoderPrivate /* Tracks whether the latency message was posted at least once */ gboolean posted_latency_msg; + /* events that should apply to the current frame */ /* FIXME 2.0: Use a GQueue or similar, see GstVideoCodecFrame::events */ GList *current_frame_events; + /* events that should be pushed before the next frame */ + /* FIXME 2.0: Use a GQueue or similar, see GstVideoCodecFrame::events */ + GList *pending_events; GList *headers; gboolean new_headers; /* Whether new headers were just set */ @@ -281,6 +285,11 @@ static gboolean gst_video_encoder_negotiate_default (GstVideoEncoder * encoder); static gboolean gst_video_encoder_negotiate_unlocked (GstVideoEncoder * encoder); +static void gst_video_encoder_post_qos_drop (GstVideoEncoder * enc, + GstVideoCodecFrame * frame); +static void gst_video_encoder_push_pending_unlocked (GstVideoEncoder * encoder, + GstVideoCodecFrame * frame, gboolean dropping); + static gboolean gst_video_encoder_sink_query_default (GstVideoEncoder * encoder, GstQuery * query); static gboolean gst_video_encoder_src_query_default (GstVideoEncoder * encoder, @@ -503,9 +512,11 @@ gst_video_encoder_reset (GstVideoEncoder * encoder, gboolean hard) priv->allocator = NULL; } - g_list_foreach (priv->current_frame_events, (GFunc) gst_event_unref, NULL); - g_list_free (priv->current_frame_events); + g_list_free_full (priv->current_frame_events, + (GDestroyNotify) gst_event_unref); priv->current_frame_events = NULL; + g_list_free_full (priv->pending_events, (GDestroyNotify) gst_event_unref); + priv->pending_events = NULL; GST_OBJECT_LOCK (encoder); priv->proportion = 0.5; @@ -2135,7 +2146,7 @@ gst_video_encoder_allocate_output_frame (GstVideoEncoder * } static void -gst_video_encoder_release_frame (GstVideoEncoder * enc, +gst_video_encoder_release_frame_unlocked (GstVideoEncoder * enc, GstVideoCodecFrame * frame) { GList *link; @@ -2146,10 +2157,73 @@ gst_video_encoder_release_frame (GstVideoEncoder * enc, gst_video_codec_frame_unref (frame); g_queue_delete_link (&enc->priv->frames, link); } + if (frame->events) { + enc->priv->pending_events = + g_list_concat (frame->events, enc->priv->pending_events); + frame->events = NULL; + } + /* unref because this function takes ownership */ gst_video_codec_frame_unref (frame); } +/** + * gst_video_encoder_release_frame: + * @encoder: a #GstVideoEncoder + * @frame: (transfer full): a #GstVideoCodecFrame + * + * Removes @frame from list of pending frames and releases it, similar + * to calling gst_video_encoder_finish_frame() without a buffer attached + * to the frame, but does not post a QoS message or do any additional + * processing. Events from @frame are moved to the pending events list. + * + * Since: 1.26 + */ +void +gst_video_encoder_release_frame (GstVideoEncoder * enc, + GstVideoCodecFrame * frame) +{ + g_return_if_fail (GST_IS_VIDEO_ENCODER (enc)); + g_return_if_fail (frame != NULL); + + GST_LOG_OBJECT (enc, "Releasing frame %p", frame); + + GST_VIDEO_ENCODER_STREAM_LOCK (enc); + gst_video_encoder_release_frame_unlocked (enc, frame); + GST_VIDEO_ENCODER_STREAM_UNLOCK (enc); +} + +/** + * gst_video_encoder_drop_frame: + * @encoder: a #GstVideoEncoder + * @frame: (transfer full): a #GstVideoCodecFrame + * + * Removes @frame from the list of pending frames, releases it + * and posts a QoS message with the frame's details on the bus. + * Similar to calling gst_video_encoder_finish_frame() without a buffer + * attached to @frame, but this function additionally stores events + * from @frame as pending, to be pushed out alongside the next frame + * submitted via gst_video_encoder_finish_frame(). + * + * Since: 1.26 + */ +void +gst_video_encoder_drop_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame) +{ + g_return_if_fail (GST_IS_VIDEO_ENCODER (enc)); + g_return_if_fail (frame != NULL); + + GST_VIDEO_ENCODER_STREAM_LOCK (enc); + + GST_LOG_OBJECT (enc, "Dropping frame %p", frame); + + gst_video_encoder_push_pending_unlocked (enc, frame, TRUE); + gst_video_encoder_post_qos_drop (enc, frame); + gst_video_encoder_release_frame_unlocked (enc, frame); + + GST_VIDEO_ENCODER_STREAM_UNLOCK (enc); +} + static gboolean gst_video_encoder_transform_meta_default (GstVideoEncoder * encoder, GstVideoCodecFrame * frame, GstMeta * meta) @@ -2217,8 +2291,10 @@ foreach_metadata (GstBuffer * inbuf, GstMeta ** meta, gpointer user_data) return TRUE; } +/* called with STREAM_LOCK */ static void -gst_video_encoder_drop_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame) +gst_video_encoder_post_qos_drop (GstVideoEncoder * enc, + GstVideoCodecFrame * frame) { GstVideoEncoderPrivate *priv = enc->priv; GstClockTime stream_time, jitter, earliest_time, qostime, timestamp; @@ -2282,23 +2358,32 @@ gst_video_encoder_can_push_unlocked (GstVideoEncoder * encoder) return GST_FLOW_OK; } +static void +gst_video_encoder_push_event_list (GstVideoEncoder * encoder, GList * events) +{ + GList *l; + + /* events are stored in reverse order */ + for (l = g_list_last (events); l; l = g_list_previous (l)) { + GST_LOG_OBJECT (encoder, "pushing %s event", GST_EVENT_TYPE_NAME (l->data)); + gst_video_encoder_push_event (encoder, l->data); + } + g_list_free (events); +} + static void gst_video_encoder_push_pending_unlocked (GstVideoEncoder * encoder, - GstVideoCodecFrame * frame) + GstVideoCodecFrame * frame, gboolean dropping) { GstVideoEncoderPrivate *priv = encoder->priv; - GList *l; + GList *l, *events = NULL; /* Push all pending events that arrived before this frame */ for (l = priv->frames.head; l; l = l->next) { GstVideoCodecFrame *tmp = l->data; if (tmp->events) { - GList *k; - - for (k = g_list_last (tmp->events); k; k = k->prev) - gst_video_encoder_push_event (encoder, k->data); - g_list_free (tmp->events); + events = g_list_concat (tmp->events, events); tmp->events = NULL; } @@ -2306,6 +2391,16 @@ gst_video_encoder_push_pending_unlocked (GstVideoEncoder * encoder, break; } + if (dropping) { + /* Push before the next frame that is not dropped */ + priv->pending_events = g_list_concat (events, priv->pending_events); + } else { + gst_video_encoder_push_event_list (encoder, priv->pending_events); + priv->pending_events = NULL; + + gst_video_encoder_push_event_list (encoder, events); + } + gst_video_encoder_check_and_push_tags (encoder); } @@ -2501,6 +2596,10 @@ gst_video_encoder_send_key_unit_unlocked (GstVideoEncoder * encoder, * It is subsequently pushed downstream or provided to @pre_push. * In any case, the frame is considered finished and released. * + * If @frame does not have a buffer attached, it will be dropped, and + * a QoS message will be posted on the bus. Events from @frame will be + * pushed out immediately. + * * After calling this function the output buffer of the frame is to be * considered read-only. This function will also change the metadata * of the buffer. @@ -2541,11 +2640,11 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, goto done; if (frame->abidata.ABI.num_subframes == 0) - gst_video_encoder_push_pending_unlocked (encoder, frame); + gst_video_encoder_push_pending_unlocked (encoder, frame, FALSE); /* no buffer data means this frame is skipped/dropped */ if (!frame->output_buffer) { - gst_video_encoder_drop_frame (encoder, frame); + gst_video_encoder_post_qos_drop (encoder, frame); goto done; } @@ -2626,7 +2725,7 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, * if possible, i.e. if the subclass does not hold additional references * to the frame */ - gst_video_encoder_release_frame (encoder, frame); + gst_video_encoder_release_frame_unlocked (encoder, frame); frame = NULL; if (ret == GST_FLOW_OK) { @@ -2638,7 +2737,7 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, done: /* handed out */ if (frame) - gst_video_encoder_release_frame (encoder, frame); + gst_video_encoder_release_frame_unlocked (encoder, frame); GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder); @@ -2704,7 +2803,7 @@ gst_video_encoder_finish_subframe (GstVideoEncoder * encoder, * Push new incoming events on finish_frame otherwise. */ if (frame->abidata.ABI.num_subframes == 0) - gst_video_encoder_push_pending_unlocked (encoder, frame); + gst_video_encoder_push_pending_unlocked (encoder, frame, FALSE); if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && frame->abidata.ABI.num_subframes == 0) { diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.h b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.h index 2a03fdda20..86372a1106 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.h @@ -387,6 +387,12 @@ void gst_video_encoder_set_min_force_key_unit_interval (GstVideo GST_VIDEO_API GstClockTime gst_video_encoder_get_min_force_key_unit_interval (GstVideoEncoder * encoder); +GST_VIDEO_API +void gst_video_encoder_release_frame (GstVideoEncoder *encoder, GstVideoCodecFrame *frame); + +GST_VIDEO_API +void gst_video_encoder_drop_frame (GstVideoEncoder *encoder, GstVideoCodecFrame *frame); + G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstVideoEncoder, gst_object_unref) G_END_DECLS