videoencoder: Expose release_frame() and drop_frame() as public API

release_frame() can be useful for manually dropping frames without posting QoS messages like finish_frame() would.
Matches the same kind of API on the decoder side of things.

Modifies the behaviour of release_frame() to make sure events from released frames are stored as 'pending'
and pushed before the next non-dropped frame. This is needed because now release_frame() can be called outside of
finish_frame(), so we would potentially just lose events and bad things would happen.

drop_frame() was also added to match the decoder API. It functions almost identically to finish_frame() without a buffer
attached to the frame, except instead of immediately pushing the frame's events, it will store them as pending.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7190>
This commit is contained in:
Piotr Brzeziński 2024-08-05 14:37:57 +02:00 committed by GStreamer Marge Bot
parent 9bbb7accb3
commit 724c443a65
3 changed files with 167 additions and 16 deletions

View file

@ -8929,6 +8929,28 @@ keep references to the frame, not the buffer.</doc>
</parameter>
</parameters>
</method>
<method name="drop_frame" c:identifier="gst_video_encoder_drop_frame" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c">Removes @frame from the list of pending frames, releases it
and posts a QoS message with the frame's details on the bus.
Similar to calling gst_video_encoder_finish_frame() without a buffer
attached to @frame, but this function additionally stores events
from @frame as pending, to be pushed out alongside the next frame
submitted via gst_video_encoder_finish_frame().</doc>
<source-position filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.h"/>
<return-value transfer-ownership="none">
<type name="none" c:type="void"/>
</return-value>
<parameters>
<instance-parameter name="encoder" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c">a #GstVideoEncoder</doc>
<type name="VideoEncoder" c:type="GstVideoEncoder*"/>
</instance-parameter>
<parameter name="frame" transfer-ownership="full">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c">a #GstVideoCodecFrame</doc>
<type name="VideoCodecFrame" c:type="GstVideoCodecFrame*"/>
</parameter>
</parameters>
</method>
<method name="finish_frame" c:identifier="gst_video_encoder_finish_frame">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c">@frame must have a valid encoded data buffer, whose metadata fields
are then appropriately set according to frame data or no buffer at
@ -8936,6 +8958,10 @@ all if the frame should be dropped.
It is subsequently pushed downstream or provided to @pre_push.
In any case, the frame is considered finished and released.
If @frame does not have a buffer attached, it will be dropped, and
a QoS message will be posted on the bus. Events from @frame will be
pushed out immediately.
After calling this function the output buffer of the frame is to be
considered read-only. This function will also change the metadata
of the buffer.</doc>
@ -9216,6 +9242,26 @@ elements (e.g. muxers).</doc>
</parameter>
</parameters>
</method>
<method name="release_frame" c:identifier="gst_video_encoder_release_frame" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c">Removes @frame from list of pending frames and releases it, similar
to calling gst_video_encoder_finish_frame() without a buffer attached
to the frame, but does not post a QoS message or do any additional
processing. Events from @frame are moved to the pending events list.</doc>
<source-position filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.h"/>
<return-value transfer-ownership="none">
<type name="none" c:type="void"/>
</return-value>
<parameters>
<instance-parameter name="encoder" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c">a #GstVideoEncoder</doc>
<type name="VideoEncoder" c:type="GstVideoEncoder*"/>
</instance-parameter>
<parameter name="frame" transfer-ownership="full">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c">a #GstVideoCodecFrame</doc>
<type name="VideoCodecFrame" c:type="GstVideoCodecFrame*"/>
</parameter>
</parameters>
</method>
<method name="set_headers" c:identifier="gst_video_encoder_set_headers">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c">Set the codec headers to be sent downstream whenever requested.</doc>
<source-position filename="../subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.h"/>

View file

@ -137,8 +137,12 @@ struct _GstVideoEncoderPrivate
/* Tracks whether the latency message was posted at least once */
gboolean posted_latency_msg;
/* events that should apply to the current frame */
/* FIXME 2.0: Use a GQueue or similar, see GstVideoCodecFrame::events */
GList *current_frame_events;
/* events that should be pushed before the next frame */
/* FIXME 2.0: Use a GQueue or similar, see GstVideoCodecFrame::events */
GList *pending_events;
GList *headers;
gboolean new_headers; /* Whether new headers were just set */
@ -281,6 +285,11 @@ static gboolean gst_video_encoder_negotiate_default (GstVideoEncoder * encoder);
static gboolean gst_video_encoder_negotiate_unlocked (GstVideoEncoder *
encoder);
static void gst_video_encoder_post_qos_drop (GstVideoEncoder * enc,
GstVideoCodecFrame * frame);
static void gst_video_encoder_push_pending_unlocked (GstVideoEncoder * encoder,
GstVideoCodecFrame * frame, gboolean dropping);
static gboolean gst_video_encoder_sink_query_default (GstVideoEncoder * encoder,
GstQuery * query);
static gboolean gst_video_encoder_src_query_default (GstVideoEncoder * encoder,
@ -503,9 +512,11 @@ gst_video_encoder_reset (GstVideoEncoder * encoder, gboolean hard)
priv->allocator = NULL;
}
g_list_foreach (priv->current_frame_events, (GFunc) gst_event_unref, NULL);
g_list_free (priv->current_frame_events);
g_list_free_full (priv->current_frame_events,
(GDestroyNotify) gst_event_unref);
priv->current_frame_events = NULL;
g_list_free_full (priv->pending_events, (GDestroyNotify) gst_event_unref);
priv->pending_events = NULL;
GST_OBJECT_LOCK (encoder);
priv->proportion = 0.5;
@ -2135,7 +2146,7 @@ gst_video_encoder_allocate_output_frame (GstVideoEncoder *
}
static void
gst_video_encoder_release_frame (GstVideoEncoder * enc,
gst_video_encoder_release_frame_unlocked (GstVideoEncoder * enc,
GstVideoCodecFrame * frame)
{
GList *link;
@ -2146,10 +2157,73 @@ gst_video_encoder_release_frame (GstVideoEncoder * enc,
gst_video_codec_frame_unref (frame);
g_queue_delete_link (&enc->priv->frames, link);
}
if (frame->events) {
enc->priv->pending_events =
g_list_concat (frame->events, enc->priv->pending_events);
frame->events = NULL;
}
/* unref because this function takes ownership */
gst_video_codec_frame_unref (frame);
}
/**
* gst_video_encoder_release_frame:
* @encoder: a #GstVideoEncoder
* @frame: (transfer full): a #GstVideoCodecFrame
*
* Removes @frame from list of pending frames and releases it, similar
* to calling gst_video_encoder_finish_frame() without a buffer attached
* to the frame, but does not post a QoS message or do any additional
* processing. Events from @frame are moved to the pending events list.
*
* Since: 1.26
*/
void
gst_video_encoder_release_frame (GstVideoEncoder * enc,
GstVideoCodecFrame * frame)
{
g_return_if_fail (GST_IS_VIDEO_ENCODER (enc));
g_return_if_fail (frame != NULL);
GST_LOG_OBJECT (enc, "Releasing frame %p", frame);
GST_VIDEO_ENCODER_STREAM_LOCK (enc);
gst_video_encoder_release_frame_unlocked (enc, frame);
GST_VIDEO_ENCODER_STREAM_UNLOCK (enc);
}
/**
* gst_video_encoder_drop_frame:
* @encoder: a #GstVideoEncoder
* @frame: (transfer full): a #GstVideoCodecFrame
*
* Removes @frame from the list of pending frames, releases it
* and posts a QoS message with the frame's details on the bus.
* Similar to calling gst_video_encoder_finish_frame() without a buffer
* attached to @frame, but this function additionally stores events
* from @frame as pending, to be pushed out alongside the next frame
* submitted via gst_video_encoder_finish_frame().
*
* Since: 1.26
*/
void
gst_video_encoder_drop_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame)
{
g_return_if_fail (GST_IS_VIDEO_ENCODER (enc));
g_return_if_fail (frame != NULL);
GST_VIDEO_ENCODER_STREAM_LOCK (enc);
GST_LOG_OBJECT (enc, "Dropping frame %p", frame);
gst_video_encoder_push_pending_unlocked (enc, frame, TRUE);
gst_video_encoder_post_qos_drop (enc, frame);
gst_video_encoder_release_frame_unlocked (enc, frame);
GST_VIDEO_ENCODER_STREAM_UNLOCK (enc);
}
static gboolean
gst_video_encoder_transform_meta_default (GstVideoEncoder *
encoder, GstVideoCodecFrame * frame, GstMeta * meta)
@ -2217,8 +2291,10 @@ foreach_metadata (GstBuffer * inbuf, GstMeta ** meta, gpointer user_data)
return TRUE;
}
/* called with STREAM_LOCK */
static void
gst_video_encoder_drop_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame)
gst_video_encoder_post_qos_drop (GstVideoEncoder * enc,
GstVideoCodecFrame * frame)
{
GstVideoEncoderPrivate *priv = enc->priv;
GstClockTime stream_time, jitter, earliest_time, qostime, timestamp;
@ -2282,23 +2358,32 @@ gst_video_encoder_can_push_unlocked (GstVideoEncoder * encoder)
return GST_FLOW_OK;
}
static void
gst_video_encoder_push_event_list (GstVideoEncoder * encoder, GList * events)
{
GList *l;
/* events are stored in reverse order */
for (l = g_list_last (events); l; l = g_list_previous (l)) {
GST_LOG_OBJECT (encoder, "pushing %s event", GST_EVENT_TYPE_NAME (l->data));
gst_video_encoder_push_event (encoder, l->data);
}
g_list_free (events);
}
static void
gst_video_encoder_push_pending_unlocked (GstVideoEncoder * encoder,
GstVideoCodecFrame * frame)
GstVideoCodecFrame * frame, gboolean dropping)
{
GstVideoEncoderPrivate *priv = encoder->priv;
GList *l;
GList *l, *events = NULL;
/* Push all pending events that arrived before this frame */
for (l = priv->frames.head; l; l = l->next) {
GstVideoCodecFrame *tmp = l->data;
if (tmp->events) {
GList *k;
for (k = g_list_last (tmp->events); k; k = k->prev)
gst_video_encoder_push_event (encoder, k->data);
g_list_free (tmp->events);
events = g_list_concat (tmp->events, events);
tmp->events = NULL;
}
@ -2306,6 +2391,16 @@ gst_video_encoder_push_pending_unlocked (GstVideoEncoder * encoder,
break;
}
if (dropping) {
/* Push before the next frame that is not dropped */
priv->pending_events = g_list_concat (events, priv->pending_events);
} else {
gst_video_encoder_push_event_list (encoder, priv->pending_events);
priv->pending_events = NULL;
gst_video_encoder_push_event_list (encoder, events);
}
gst_video_encoder_check_and_push_tags (encoder);
}
@ -2501,6 +2596,10 @@ gst_video_encoder_send_key_unit_unlocked (GstVideoEncoder * encoder,
* It is subsequently pushed downstream or provided to @pre_push.
* In any case, the frame is considered finished and released.
*
* If @frame does not have a buffer attached, it will be dropped, and
* a QoS message will be posted on the bus. Events from @frame will be
* pushed out immediately.
*
* After calling this function the output buffer of the frame is to be
* considered read-only. This function will also change the metadata
* of the buffer.
@ -2541,11 +2640,11 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder,
goto done;
if (frame->abidata.ABI.num_subframes == 0)
gst_video_encoder_push_pending_unlocked (encoder, frame);
gst_video_encoder_push_pending_unlocked (encoder, frame, FALSE);
/* no buffer data means this frame is skipped/dropped */
if (!frame->output_buffer) {
gst_video_encoder_drop_frame (encoder, frame);
gst_video_encoder_post_qos_drop (encoder, frame);
goto done;
}
@ -2626,7 +2725,7 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder,
* if possible, i.e. if the subclass does not hold additional references
* to the frame
*/
gst_video_encoder_release_frame (encoder, frame);
gst_video_encoder_release_frame_unlocked (encoder, frame);
frame = NULL;
if (ret == GST_FLOW_OK) {
@ -2638,7 +2737,7 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder,
done:
/* handed out */
if (frame)
gst_video_encoder_release_frame (encoder, frame);
gst_video_encoder_release_frame_unlocked (encoder, frame);
GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);
@ -2704,7 +2803,7 @@ gst_video_encoder_finish_subframe (GstVideoEncoder * encoder,
* Push new incoming events on finish_frame otherwise.
*/
if (frame->abidata.ABI.num_subframes == 0)
gst_video_encoder_push_pending_unlocked (encoder, frame);
gst_video_encoder_push_pending_unlocked (encoder, frame, FALSE);
if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)
&& frame->abidata.ABI.num_subframes == 0) {

View file

@ -387,6 +387,12 @@ void gst_video_encoder_set_min_force_key_unit_interval (GstVideo
GST_VIDEO_API
GstClockTime gst_video_encoder_get_min_force_key_unit_interval (GstVideoEncoder * encoder);
GST_VIDEO_API
void gst_video_encoder_release_frame (GstVideoEncoder *encoder, GstVideoCodecFrame *frame);
GST_VIDEO_API
void gst_video_encoder_drop_frame (GstVideoEncoder *encoder, GstVideoCodecFrame *frame);
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstVideoEncoder, gst_object_unref)
G_END_DECLS