From daa52708b340304e327d4d34d844b8a6b5a097c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 11 Jan 2010 18:59:43 +0100 Subject: [PATCH] videomixer: Implement basic QoS This drops frames if they're too late anyway before blending and all that starts but QoS events are not forwarded upstream. In the future the QoS events should be transformed somehow and forwarded upstream. --- gst/videomixer/videomixer.c | 154 +++++++++++++++++++++++++++++++----- gst/videomixer/videomixer.h | 8 +- 2 files changed, 141 insertions(+), 21 deletions(-) diff --git a/gst/videomixer/videomixer.c b/gst/videomixer/videomixer.c index f441614b7d..7c6c5564a5 100644 --- a/gst/videomixer/videomixer.c +++ b/gst/videomixer/videomixer.c @@ -198,6 +198,86 @@ gst_videomixer_pad_set_property (GObject * object, guint prop_id, gst_object_unref (mix); } +static void +gst_videomixer_update_qos (GstVideoMixer * mix, gdouble proportion, + GstClockTimeDiff diff, GstClockTime timestamp) +{ + GST_DEBUG_OBJECT (mix, + "Updating QoS: proportion %lf, diff %s%" GST_TIME_FORMAT ", timestamp %" + GST_TIME_FORMAT, proportion, (diff < 0) ? "-" : "", + GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp)); + + GST_OBJECT_LOCK (mix); + mix->proportion = proportion; + if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { + if (G_UNLIKELY (diff > 0)) + mix->earliest_time = + timestamp + 2 * diff + gst_util_uint64_scale_int (GST_SECOND, + mix->fps_d, mix->fps_n); + else + mix->earliest_time = timestamp + diff; + } else { + mix->earliest_time = GST_CLOCK_TIME_NONE; + } + GST_OBJECT_UNLOCK (mix); +} + +static void +gst_videomixer_reset_qos (GstVideoMixer * mix) +{ + gst_videomixer_update_qos (mix, 0.5, 0, GST_CLOCK_TIME_NONE); +} + +static void +gst_videomixer_read_qos (GstVideoMixer * mix, gdouble * proportion, + GstClockTime * time) +{ + GST_OBJECT_LOCK (mix); + *proportion = mix->proportion; + *time = mix->earliest_time; + GST_OBJECT_UNLOCK (mix); +} + +/* Perform qos calculations before processing the next frame. Returns TRUE if + * the frame should be processed, FALSE if the frame can be dropped entirely */ +static gboolean +gst_videomixer_do_qos (GstVideoMixer * mix, GstClockTime timestamp) +{ + GstClockTime qostime, earliest_time; + gdouble proportion; + + /* no timestamp, can't do QoS => process frame */ + if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (timestamp))) { + GST_LOG_OBJECT (mix, "invalid timestamp, can't do QoS, process frame"); + return TRUE; + } + + /* get latest QoS observation values */ + gst_videomixer_read_qos (mix, &proportion, &earliest_time); + + /* skip qos if we have no observation (yet) => process frame */ + if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (earliest_time))) { + GST_LOG_OBJECT (mix, "no observation yet, process frame"); + return TRUE; + } + + /* qos is done on running time */ + qostime = + gst_segment_to_running_time (&mix->segment, GST_FORMAT_TIME, timestamp); + + /* see how our next timestamp relates to the latest qos timestamp */ + GST_LOG_OBJECT (mix, "qostime %" GST_TIME_FORMAT ", earliest %" + GST_TIME_FORMAT, GST_TIME_ARGS (qostime), GST_TIME_ARGS (earliest_time)); + + if (qostime != GST_CLOCK_TIME_NONE && qostime <= earliest_time) { + GST_DEBUG_OBJECT (mix, "we are late, drop frame"); + return FALSE; + } + + GST_LOG_OBJECT (mix, "process frame"); + return TRUE; +} + static void gst_videomixer_set_master_geometry (GstVideoMixer * mix) { @@ -233,6 +313,7 @@ gst_videomixer_set_master_geometry (GstVideoMixer * mix) || mix->fps_d != fps_d) { mix->setcaps = TRUE; mix->sendseg = TRUE; + gst_videomixer_reset_qos (mix); mix->master = master; mix->in_width = width; mix->in_height = height; @@ -551,8 +632,11 @@ gst_videomixer_reset (GstVideoMixer * mix) mix->fps_n = mix->fps_d = 0; mix->setcaps = FALSE; mix->sendseg = FALSE; + mix->segment_position = 0; - mix->segment_rate = 1.0; + gst_segment_init (&mix->segment, GST_FORMAT_TIME); + + gst_videomixer_reset_qos (mix); mix->fmt = GST_VIDEO_FORMAT_UNKNOWN; @@ -1132,7 +1216,10 @@ gst_videomixer_fill_queues (GstVideoMixer * mix) else stop = -1; - event = gst_event_new_new_segment_full (FALSE, segment->rate, 1.0, + gst_segment_set_newsegment (&mix->segment, FALSE, segment->rate, + segment->format, start, stop, start + mix->segment_position); + event = + gst_event_new_new_segment_full (FALSE, segment->rate, 1.0, segment->format, start, stop, start + mix->segment_position); gst_pad_push_event (mix->srcpad, event); mix->sendseg = FALSE; @@ -1179,21 +1266,6 @@ gst_videomixer_blend_buffers (GstVideoMixer * mix, GstBuffer * outbuf) mix->blend (GST_BUFFER_DATA (mixcol->buffer), pad->xpos, pad->ypos, pad->in_width, pad->in_height, pad->alpha, GST_BUFFER_DATA (outbuf), mix->out_width, mix->out_height); - - if (pad == mix->master) { - gint64 running_time; - - running_time = - gst_segment_to_running_time (seg, GST_FORMAT_TIME, timestamp); - - /* outgoing buffers need the running_time */ - GST_BUFFER_TIMESTAMP (outbuf) = running_time; - GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (mixcol->buffer); - - mix->last_ts = running_time; - if (GST_BUFFER_DURATION_IS_VALID (outbuf)) - mix->last_ts += GST_BUFFER_DURATION (outbuf); - } } } } @@ -1253,6 +1325,8 @@ gst_videomixer_collected (GstCollectPads * pads, GstVideoMixer * mix) GstBuffer *outbuf = NULL; size_t outsize = 0; gboolean eos = FALSE; + GstClockTime timestamp = GST_CLOCK_TIME_NONE; + GstClockTime duration = GST_CLOCK_TIME_NONE; g_return_val_if_fail (GST_IS_VIDEO_MIXER (mix), GST_FLOW_ERROR); @@ -1293,6 +1367,30 @@ gst_videomixer_collected (GstCollectPads * pads, GstVideoMixer * mix) gst_caps_unref (newcaps); } + /* Get timestamp & duration */ + if (mix->master->mixcol->buffer != NULL) { + GstClockTime in_ts; + GstSegment *seg; + GstVideoMixerCollect *mixcol = mix->master->mixcol; + + seg = &mixcol->collect.segment; + in_ts = GST_BUFFER_TIMESTAMP (mixcol->buffer); + + timestamp = gst_segment_to_running_time (seg, GST_FORMAT_TIME, in_ts); + + mix->last_ts = timestamp; + duration = GST_BUFFER_DURATION (mixcol->buffer); + if (GST_CLOCK_TIME_IS_VALID (duration)) + mix->last_ts += duration; + } + + if (!gst_videomixer_do_qos (mix, timestamp)) { + gst_videomixer_update_queues (mix); + GST_VIDEO_MIXER_STATE_UNLOCK (mix); + ret = GST_FLOW_OK; + goto beach; + } + /* allocate an output buffer */ outsize = gst_video_format_get_size (mix->fmt, mix->out_width, mix->out_height); @@ -1307,6 +1405,9 @@ gst_videomixer_collected (GstCollectPads * pads, GstVideoMixer * mix) goto error; } + GST_BUFFER_TIMESTAMP (outbuf) = timestamp; + GST_BUFFER_DURATION (outbuf) = duration; + switch (mix->background) { case VIDEO_MIXER_BACKGROUND_CHECKER: mix->fill_checker (GST_BUFFER_DATA (outbuf), mix->out_width, @@ -1393,10 +1494,20 @@ gst_videomixer_src_event (GstPad * pad, GstEvent * event) gboolean result; switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_QOS: - /* QoS might be tricky */ - result = FALSE; + case GST_EVENT_QOS:{ + GstClockTimeDiff diff; + GstClockTime timestamp; + gdouble proportion; + + gst_event_parse_qos (event, &proportion, &diff, ×tamp); + + gst_videomixer_update_qos (mix, proportion, diff, timestamp); + gst_event_unref (event); + + /* TODO: The QoS event should be transformed and send upstream */ + result = TRUE; break; + } case GST_EVENT_SEEK: { GstSeekFlags flags; @@ -1426,6 +1537,7 @@ gst_videomixer_src_event (GstPad * pad, GstEvent * event) mix->segment_position = 0; mix->sendseg = TRUE; GST_OBJECT_UNLOCK (mix->collect); + gst_videomixer_reset_qos (mix); result = forward_event (mix, event); break; @@ -1463,6 +1575,7 @@ gst_videomixer_sink_event (GstPad * pad, GstEvent * event) * and downstream (using our source pad, the bastard!). */ videomixer->sendseg = TRUE; + gst_videomixer_reset_qos (videomixer); /* Reset pad state after FLUSH_STOP */ if (vpad->mixcol->buffer) @@ -1472,6 +1585,7 @@ gst_videomixer_sink_event (GstPad * pad, GstEvent * event) break; case GST_EVENT_NEWSEGMENT: videomixer->sendseg = TRUE; + gst_videomixer_reset_qos (videomixer); break; default: break; diff --git a/gst/videomixer/videomixer.h b/gst/videomixer/videomixer.h index 3390fecd0b..72b3a575b9 100644 --- a/gst/videomixer/videomixer.h +++ b/gst/videomixer/videomixer.h @@ -104,7 +104,13 @@ struct _GstVideoMixer /* sink event handling */ GstPadEventFunction collect_event; guint64 segment_position; - gdouble segment_rate; + + /* Current downstream segment */ + GstSegment segment; + + /* QoS stuff */ + gdouble proportion; + GstClockTime earliest_time; BlendFunction blend; FillCheckerFunction fill_checker;