mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-07-03 13:16:04 +00:00
videoencoder: implement QoS
It allows encoders to detect and drop input frames which are already late to increase the chance of the pipeline to catch up. The QoS logic and code is directly copied from gstvideodecoder.c. https://bugzilla.gnome.org/show_bug.cgi?id=582166
This commit is contained in:
Guillaume Desmottes
Sebastian Dröge
parent
9264a298bf
commit
bcca3b926c
|
|
@ -152,6 +152,14 @@ struct _GstVideoEncoderPrivate
|
|||
/* adjustment needed on pts, dts, segment start and stop to accomodate
|
||||
* min_pts */
|
||||
GstClockTime time_adjustment;
|
||||
|
||||
/* QoS properties */
|
||||
gdouble proportion; /* OBJECT_LOCK */
|
||||
GstClockTime earliest_time; /* OBJECT_LOCK */
|
||||
GstClockTime qos_frame_duration; /* OBJECT_LOCK */
|
||||
/* qos messages: frames dropped/processed */
|
||||
guint dropped;
|
||||
guint processed;
|
||||
};
|
||||
|
||||
typedef struct _ForcedKeyUnitEvent ForcedKeyUnitEvent;
|
||||
|
|
@ -374,6 +382,14 @@ gst_video_encoder_reset (GstVideoEncoder * encoder, gboolean hard)
|
|||
g_list_free (priv->current_frame_events);
|
||||
priv->current_frame_events = NULL;
|
||||
|
||||
GST_OBJECT_LOCK (encoder);
|
||||
priv->proportion = 0.5;
|
||||
priv->earliest_time = GST_CLOCK_TIME_NONE;
|
||||
priv->qos_frame_duration = 0;
|
||||
GST_OBJECT_UNLOCK (encoder);
|
||||
|
||||
priv->dropped = 0;
|
||||
priv->processed = 0;
|
||||
} else {
|
||||
GList *l;
|
||||
|
||||
|
|
@ -1144,6 +1160,7 @@ gst_video_encoder_src_event_default (GstVideoEncoder * encoder,
|
|||
GstEvent * event)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
GstVideoEncoderPrivate *priv = encoder->priv;
|
||||
|
||||
switch (GST_EVENT_TYPE (event)) {
|
||||
case GST_EVENT_CUSTOM_UPSTREAM:
|
||||
|
|
@ -1174,6 +1191,36 @@ gst_video_encoder_src_event_default (GstVideoEncoder * encoder,
|
|||
}
|
||||
break;
|
||||
}
|
||||
case GST_EVENT_QOS:
|
||||
{
|
||||
GstQOSType type;
|
||||
gdouble proportion;
|
||||
GstClockTimeDiff diff;
|
||||
GstClockTime timestamp;
|
||||
|
||||
gst_event_parse_qos (event, &type, &proportion, &diff, ×tamp);
|
||||
|
||||
GST_OBJECT_LOCK (encoder);
|
||||
priv->proportion = proportion;
|
||||
if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (timestamp))) {
|
||||
if (G_UNLIKELY (diff > 0)) {
|
||||
priv->earliest_time = timestamp + 2 * diff + priv->qos_frame_duration;
|
||||
} else {
|
||||
priv->earliest_time = timestamp + diff;
|
||||
}
|
||||
} else {
|
||||
priv->earliest_time = GST_CLOCK_TIME_NONE;
|
||||
}
|
||||
GST_OBJECT_UNLOCK (encoder);
|
||||
|
||||
GST_DEBUG_OBJECT (encoder,
|
||||
"got QoS %" GST_TIME_FORMAT ", %" GST_STIME_FORMAT ", %g",
|
||||
GST_TIME_ARGS (timestamp), GST_STIME_ARGS (diff), proportion);
|
||||
|
||||
ret = gst_pad_push_event (encoder->sinkpad, event);
|
||||
event = NULL;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -1435,6 +1482,10 @@ gst_video_encoder_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
|
|||
GST_LOG_OBJECT (encoder, "passing frame pfn %d to subclass",
|
||||
frame->presentation_frame_number);
|
||||
|
||||
frame->deadline =
|
||||
gst_segment_to_running_time (&encoder->input_segment, GST_FORMAT_TIME,
|
||||
frame->pts);
|
||||
|
||||
ret = klass->handle_frame (encoder, frame);
|
||||
|
||||
done:
|
||||
|
|
@ -1902,6 +1953,43 @@ foreach_metadata (GstBuffer * inbuf, GstMeta ** meta, gpointer user_data)
|
|||
return TRUE;
|
||||
}
|
||||
|
||||
static void
|
||||
gst_video_encoder_drop_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame)
|
||||
{
|
||||
GstVideoEncoderPrivate *priv = enc->priv;
|
||||
GstClockTime stream_time, jitter, earliest_time, qostime, timestamp;
|
||||
GstSegment *segment;
|
||||
GstMessage *qos_msg;
|
||||
gdouble proportion;
|
||||
|
||||
GST_DEBUG_OBJECT (enc, "dropping frame %" GST_TIME_FORMAT,
|
||||
GST_TIME_ARGS (frame->pts));
|
||||
|
||||
priv->dropped++;
|
||||
|
||||
/* post QoS message */
|
||||
GST_OBJECT_LOCK (enc);
|
||||
proportion = priv->proportion;
|
||||
earliest_time = priv->earliest_time;
|
||||
GST_OBJECT_UNLOCK (enc);
|
||||
|
||||
timestamp = frame->pts;
|
||||
segment = &enc->output_segment;
|
||||
if (G_UNLIKELY (segment->format == GST_FORMAT_UNDEFINED))
|
||||
segment = &enc->input_segment;
|
||||
stream_time =
|
||||
gst_segment_to_stream_time (segment, GST_FORMAT_TIME, timestamp);
|
||||
qostime = gst_segment_to_running_time (segment, GST_FORMAT_TIME, timestamp);
|
||||
jitter = GST_CLOCK_DIFF (qostime, earliest_time);
|
||||
qos_msg =
|
||||
gst_message_new_qos (GST_OBJECT_CAST (enc), FALSE, qostime, stream_time,
|
||||
timestamp, GST_CLOCK_TIME_NONE);
|
||||
gst_message_set_qos_values (qos_msg, jitter, proportion, 1000000);
|
||||
gst_message_set_qos_stats (qos_msg, GST_FORMAT_BUFFERS,
|
||||
priv->processed, priv->dropped);
|
||||
gst_element_post_message (GST_ELEMENT_CAST (enc), qos_msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* gst_video_encoder_finish_frame:
|
||||
* @encoder: a #GstVideoEncoder
|
||||
|
|
@ -1980,11 +2068,12 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder,
|
|||
|
||||
/* no buffer data means this frame is skipped/dropped */
|
||||
if (!frame->output_buffer) {
|
||||
GST_DEBUG_OBJECT (encoder, "skipping frame %" GST_TIME_FORMAT,
|
||||
GST_TIME_ARGS (frame->pts));
|
||||
gst_video_encoder_drop_frame (encoder, frame);
|
||||
goto done;
|
||||
}
|
||||
|
||||
priv->processed++;
|
||||
|
||||
if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && priv->force_key_unit) {
|
||||
GstClockTime stream_time, running_time;
|
||||
GstEvent *ev;
|
||||
|
|
@ -2264,6 +2353,14 @@ gst_video_encoder_set_output_state (GstVideoEncoder * encoder, GstCaps * caps,
|
|||
gst_video_codec_state_unref (priv->output_state);
|
||||
priv->output_state = gst_video_codec_state_ref (state);
|
||||
|
||||
if (priv->output_state != NULL && priv->output_state->info.fps_n > 0) {
|
||||
priv->qos_frame_duration =
|
||||
gst_util_uint64_scale (GST_SECOND, priv->output_state->info.fps_d,
|
||||
priv->output_state->info.fps_n);
|
||||
} else {
|
||||
priv->qos_frame_duration = 0;
|
||||
}
|
||||
|
||||
priv->output_state_changed = TRUE;
|
||||
GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);
|
||||
|
||||
|
|
@ -2479,3 +2576,41 @@ gst_video_encoder_set_min_pts (GstVideoEncoder * encoder, GstClockTime min_pts)
|
|||
encoder->priv->min_pts = min_pts;
|
||||
encoder->priv->time_adjustment = GST_CLOCK_TIME_NONE;
|
||||
}
|
||||
|
||||
/**
|
||||
* gst_video_encoder_get_max_encode_time:
|
||||
* @encoder: a #GstVideoEncoder
|
||||
* @frame: a #GstVideoCodecFrame
|
||||
*
|
||||
* Determines maximum possible encoding time for @frame that will
|
||||
* allow it to encode and arrive in time (as determined by QoS events).
|
||||
* In particular, a negative result means encoding in time is no longer possible
|
||||
* and should therefore occur as soon/skippy as possible.
|
||||
*
|
||||
* Returns: max decoding time.
|
||||
* Since: 1.14
|
||||
*/
|
||||
GstClockTimeDiff
|
||||
gst_video_encoder_get_max_encode_time (GstVideoEncoder *
|
||||
encoder, GstVideoCodecFrame * frame)
|
||||
{
|
||||
GstClockTimeDiff deadline;
|
||||
GstClockTime earliest_time;
|
||||
|
||||
GST_OBJECT_LOCK (encoder);
|
||||
earliest_time = encoder->priv->earliest_time;
|
||||
if (GST_CLOCK_TIME_IS_VALID (earliest_time)
|
||||
&& GST_CLOCK_TIME_IS_VALID (frame->deadline))
|
||||
deadline = GST_CLOCK_DIFF (earliest_time, frame->deadline);
|
||||
else
|
||||
deadline = G_MAXINT64;
|
||||
|
||||
GST_LOG_OBJECT (encoder, "earliest %" GST_TIME_FORMAT
|
||||
", frame deadline %" GST_TIME_FORMAT ", deadline %" GST_STIME_FORMAT,
|
||||
GST_TIME_ARGS (earliest_time), GST_TIME_ARGS (frame->deadline),
|
||||
GST_STIME_ARGS (deadline));
|
||||
|
||||
GST_OBJECT_UNLOCK (encoder);
|
||||
|
||||
return deadline;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -368,6 +368,9 @@ void gst_video_encoder_get_allocator (GstVideoEncoder *encoder,
|
|||
GST_EXPORT
|
||||
void gst_video_encoder_set_min_pts(GstVideoEncoder *encoder, GstClockTime min_pts);
|
||||
|
||||
GST_EXPORT
|
||||
GstClockTimeDiff gst_video_encoder_get_max_encode_time (GstVideoEncoder *encoder, GstVideoCodecFrame * frame);
|
||||
|
||||
#ifdef G_DEFINE_AUTOPTR_CLEANUP_FUNC
|
||||
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstVideoEncoder, gst_object_unref)
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -90,6 +90,13 @@ gst_video_encoder_tester_handle_frame (GstVideoEncoder * enc,
|
|||
guint8 *data;
|
||||
GstMapInfo map;
|
||||
guint64 input_num;
|
||||
GstClockTimeDiff deadline;
|
||||
|
||||
deadline = gst_video_encoder_get_max_encode_time (enc, frame);
|
||||
if (deadline < 0) {
|
||||
/* Calling finish_frame() with frame->output_buffer == NULL means to drop it */
|
||||
goto out;
|
||||
}
|
||||
|
||||
gst_buffer_map (frame->input_buffer, &map, GST_MAP_READ);
|
||||
input_num = *((guint64 *) map.data);
|
||||
|
|
@ -102,6 +109,7 @@ gst_video_encoder_tester_handle_frame (GstVideoEncoder * enc,
|
|||
frame->pts = GST_BUFFER_PTS (frame->input_buffer);
|
||||
frame->duration = GST_BUFFER_DURATION (frame->input_buffer);
|
||||
|
||||
out:
|
||||
return gst_video_encoder_finish_frame (enc, frame);
|
||||
}
|
||||
|
||||
|
|
@ -534,6 +542,58 @@ GST_START_TEST (videoencoder_pre_push_fails)
|
|||
|
||||
GST_END_TEST;
|
||||
|
||||
GST_START_TEST (videoencoder_qos)
|
||||
{
|
||||
GstSegment segment;
|
||||
GstBuffer *buffer;
|
||||
GstClockTime ts, rt;
|
||||
GstBus *bus;
|
||||
GstMessage *msg;
|
||||
|
||||
setup_videoencodertester ();
|
||||
|
||||
gst_pad_set_active (mysrcpad, TRUE);
|
||||
gst_element_set_state (enc, GST_STATE_PLAYING);
|
||||
gst_pad_set_active (mysinkpad, TRUE);
|
||||
|
||||
bus = gst_bus_new ();
|
||||
gst_element_set_bus (enc, bus);
|
||||
|
||||
send_startup_events ();
|
||||
|
||||
/* push a new segment */
|
||||
gst_segment_init (&segment, GST_FORMAT_TIME);
|
||||
fail_unless (gst_pad_push_event (mysrcpad, gst_event_new_segment (&segment)));
|
||||
|
||||
/* push the first buffer */
|
||||
buffer = create_test_buffer (0);
|
||||
ts = GST_BUFFER_PTS (buffer);
|
||||
fail_unless (gst_pad_push (mysrcpad, buffer) == GST_FLOW_OK);
|
||||
|
||||
/* pretend this buffer was late in the sink */
|
||||
rt = gst_segment_to_running_time (&segment, GST_FORMAT_TIME, ts);
|
||||
fail_unless (gst_pad_push_event (mysinkpad,
|
||||
gst_event_new_qos (GST_QOS_TYPE_UNDERFLOW, 1.5, 500 * GST_MSECOND,
|
||||
rt)));
|
||||
|
||||
/* push a second buffer which will be dropped as it's already late */
|
||||
buffer = create_test_buffer (1);
|
||||
fail_unless (gst_pad_push (mysrcpad, buffer) == GST_FLOW_OK);
|
||||
|
||||
/* A QoS message was sent by the encoder */
|
||||
msg = gst_bus_pop_filtered (bus, GST_MESSAGE_QOS);
|
||||
g_assert (msg != NULL);
|
||||
gst_message_unref (msg);
|
||||
|
||||
fail_unless (gst_pad_push_event (mysrcpad, gst_event_new_eos ()));
|
||||
|
||||
gst_bus_set_flushing (bus, TRUE);
|
||||
gst_object_unref (bus);
|
||||
cleanup_videoencodertest ();
|
||||
}
|
||||
|
||||
GST_END_TEST;
|
||||
|
||||
static Suite *
|
||||
gst_videoencoder_suite (void)
|
||||
{
|
||||
|
|
@ -547,6 +607,7 @@ gst_videoencoder_suite (void)
|
|||
tcase_add_test (tc, videoencoder_events_before_eos);
|
||||
tcase_add_test (tc, videoencoder_flush_events);
|
||||
tcase_add_test (tc, videoencoder_pre_push_fails);
|
||||
tcase_add_test (tc, videoencoder_qos);
|
||||
|
||||
return s;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue