query: add a new bitrate query

Allows determining from downstream what the expected bitrate of a stream
may be which is useful in queue2 for setting time based limits when
upstream does not provide timing information.

Implement bitrate query handling in queue2

https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/60
This commit is contained in:
Matthew Waters 2018-05-17 21:42:43 +10:00 committed by Edward Hervey
parent c4ccff7861
commit 4fc4ad87d5
9 changed files with 338 additions and 18 deletions

View file

@ -2658,6 +2658,10 @@ gst_query_new_context
gst_query_set_context
gst_query_parse_context
gst_query_parse_context_type
gst_query_new_bitrate
gst_query_set_bitrate
gst_query_parse_bitrate
<SUBSECTION Standard>
GstQueryClass
GST_QUERY

View file

@ -3415,6 +3415,10 @@ gst_pad_query_default (GstPad * pad, GstObject * parent, GstQuery * query)
ret = gst_pad_query_latency_default (pad, query);
forward = FALSE;
break;
case GST_QUERY_BITRATE:
/* FIXME: better default handling */
forward = TRUE;
break;
case GST_QUERY_POSITION:
case GST_QUERY_SEEKING:
case GST_QUERY_FORMATS:

View file

@ -75,7 +75,7 @@ static const gchar *_quark_strings[] = {
"GstMessageStreamCollection", "collection", "stream", "stream-collection",
"GstMessageStreamsSelected", "GstMessageRedirect", "redirect-entry-locations",
"redirect-entry-taglists", "redirect-entry-structures",
"GstEventStreamGroupDone"
"GstEventStreamGroupDone", "GstQueryBitrate", "nominal-bitrate"
};
GQuark _priv_gst_quark_table[GST_QUARK_MAX];

View file

@ -217,7 +217,9 @@ typedef enum _GstQuarkId
GST_QUARK_REDIRECT_ENTRY_TAGLISTS = 186,
GST_QUARK_REDIRECT_ENTRY_STRUCTURES = 187,
GST_QUARK_EVENT_STREAM_GROUP_DONE = 188,
GST_QUARK_MAX = 189
GST_QUARK_QUERY_BITRATE = 189,
GST_QUARK_NOMINAL_BITRATE = 190,
GST_QUARK_MAX = 191
} GstQuarkId;
extern GQuark _priv_gst_quark_table[GST_QUARK_MAX];

View file

@ -105,6 +105,7 @@ static GstQueryQuarks query_quarks[] = {
{GST_QUERY_CAPS, "caps", 0},
{GST_QUERY_DRAIN, "drain", 0},
{GST_QUERY_CONTEXT, "context", 0},
{GST_QUERY_BITRATE, "bitrate", 0},
{0, NULL, 0}
};
@ -2654,3 +2655,75 @@ gst_query_parse_context_type (GstQuery * query, const gchar ** context_type)
return TRUE;
}
/**
* gst_query_new_bitrate:
*
* Constructs a new query object for querying the bitrate.
*
* Free-function: gst_query_unref()
*
* Returns: (transfer full): a new #GstQuery
*
* Since: 1.16
*/
GstQuery *
gst_query_new_bitrate (void)
{
GstQuery *query;
GstStructure *structure;
structure = gst_structure_new_id_empty (GST_QUARK (QUERY_BITRATE));
query = gst_query_new_custom (GST_QUERY_BITRATE, structure);
return query;
}
/**
* gst_query_set_bitrate:
* @query: a GST_QUERY_BITRATE type #GstQuery
* @nominal_bitrate: the nominal bitrate in bits per second
*
* Set the results of a bitrate query. The nominal bitrate is the average
* bitrate expected over the length of the stream as advertised in file
* headers (or similar).
*
* Since: 1.16
*/
void
gst_query_set_bitrate (GstQuery * query, guint nominal_bitrate)
{
GstStructure *s;
g_return_if_fail (GST_QUERY_TYPE (query) == GST_QUERY_BITRATE);
s = GST_QUERY_STRUCTURE (query);
gst_structure_id_set (s,
GST_QUARK (NOMINAL_BITRATE), G_TYPE_UINT, nominal_bitrate, NULL);
}
/**
* gst_query_parse_bitrate:
* @query: a GST_QUERY_BITRATE type #GstQuery
* @nominal_bitrate: (out) (allow-none): The resulting bitrate in bits per second
*
* Get the results of a bitrate query. See also gst_query_set_bitrate().
*
* Since: 1.16
*/
void
gst_query_parse_bitrate (GstQuery * query, guint * nominal_bitrate)
{
GstStructure *structure;
const GValue *value;
g_return_if_fail (GST_QUERY_TYPE (query) == GST_QUERY_BITRATE);
structure = GST_QUERY_STRUCTURE (query);
if (nominal_bitrate) {
value = gst_structure_id_get_value (structure, GST_QUARK (NOMINAL_BITRATE));
*nominal_bitrate = g_value_get_uint (value);
}
}

View file

@ -99,6 +99,7 @@ typedef enum {
* @GST_QUERY_DRAIN: wait till all serialized data is consumed downstream
* @GST_QUERY_CONTEXT: query the pipeline-local context from
* downstream or upstream (since 1.2)
* @GST_QUERY_BITRATE: the bitrate query (since 1.16)
*
* Standard predefined Query types
*/
@ -123,7 +124,8 @@ typedef enum {
GST_QUERY_ACCEPT_CAPS = GST_QUERY_MAKE_TYPE (160, FLAG(BOTH)),
GST_QUERY_CAPS = GST_QUERY_MAKE_TYPE (170, FLAG(BOTH)),
GST_QUERY_DRAIN = GST_QUERY_MAKE_TYPE (180, FLAG(DOWNSTREAM) | FLAG(SERIALIZED)),
GST_QUERY_CONTEXT = GST_QUERY_MAKE_TYPE (190, FLAG(BOTH))
GST_QUERY_CONTEXT = GST_QUERY_MAKE_TYPE (190, FLAG(BOTH)),
GST_QUERY_BITRATE = GST_QUERY_MAKE_TYPE (200, FLAG(DOWNSTREAM)),
} GstQueryType;
#undef FLAG
@ -686,6 +688,17 @@ void gst_query_set_context (GstQuery *query, GstContext
GST_API
void gst_query_parse_context (GstQuery *query, GstContext **context);
/* bitrate query */
GST_API
GstQuery * gst_query_new_bitrate (void) G_GNUC_MALLOC;
GST_API
void gst_query_set_bitrate (GstQuery * query, guint nominal_bitrate);
GST_API
void gst_query_parse_bitrate (GstQuery * query, guint * nominal_bitrate);
#ifdef G_DEFINE_AUTOPTR_CLEANUP_FUNC
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstQuery, gst_query_unref)
#endif

View file

@ -118,6 +118,7 @@ enum
#define DEFAULT_HIGH_WATERMARK 0.99
#define DEFAULT_TEMP_REMOVE TRUE
#define DEFAULT_RING_BUFFER_MAX_SIZE 0
#define DEFAULT_USE_BITRATE_QUERY TRUE
enum
{
@ -140,6 +141,8 @@ enum
PROP_TEMP_REMOVE,
PROP_RING_BUFFER_MAX_SIZE,
PROP_AVG_IN_RATE,
PROP_USE_BITRATE_QUERY,
PROP_BITRATE,
PROP_LAST
};
@ -413,6 +416,13 @@ gst_queue2_class_init (GstQueue2Class * klass)
"Location to store temporary files in (Only read this property, "
"use temp-template to configure the name template)",
NULL, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_USE_BITRATE_QUERY,
g_param_spec_boolean ("use-bitrate-query",
"Use bitrate from downstream query",
"Use a bitrate from a downstream query to estimate buffer duration if not provided",
DEFAULT_USE_BITRATE_QUERY,
G_PARAM_READWRITE | GST_PARAM_MUTABLE_PLAYING |
G_PARAM_STATIC_STRINGS));
/**
* GstQueue2:temp-remove
@ -447,6 +457,18 @@ gst_queue2_class_init (GstQueue2Class * klass)
"Average input data rate (bytes/s)",
0, G_MAXINT64, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
/**
* GstQueue2:bitrate
*
* The value used to convert between byte and time values for limiting
* the size of the queue. Values are taken from either the upstream tags
* or from the downstream bitrate query.
*/
g_object_class_install_property (gobject_class, PROP_BITRATE,
g_param_spec_uint64 ("bitrate", "Bitrate (bits/s)",
"Conversion value between data size and time",
0, G_MAXUINT64, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
/* set several parent class virtual functions */
gobject_class->finalize = gst_queue2_finalize;
@ -541,6 +563,8 @@ gst_queue2_init (GstQueue2 * queue)
queue->ring_buffer = NULL;
queue->ring_buffer_max_size = DEFAULT_RING_BUFFER_MAX_SIZE;
queue->use_bitrate_query = DEFAULT_USE_BITRATE_QUERY;
GST_DEBUG_OBJECT (queue,
"initialized queue's not_empty & not_full conditions");
}
@ -807,6 +831,29 @@ apply_gap (GstQueue2 * queue, GstEvent * event,
}
}
static void
query_downstream_bitrate (GstQueue2 * queue)
{
GstQuery *query = gst_query_new_bitrate ();
guint downstream_bitrate = 0;
if (gst_pad_peer_query (queue->srcpad, query)) {
gst_query_parse_bitrate (query, &downstream_bitrate);
GST_DEBUG_OBJECT (queue, "Got bitrate of %u from downstream",
downstream_bitrate);
} else {
GST_DEBUG_OBJECT (queue, "Failed to query bitrate from downstream");
}
gst_query_unref (query);
GST_QUEUE2_MUTEX_LOCK (queue);
queue->downstream_bitrate = downstream_bitrate;
GST_QUEUE2_MUTEX_UNLOCK (queue);
g_object_notify (G_OBJECT (queue), "bitrate");
}
/* take a buffer and update segment, updating the time level of the queue. */
static void
apply_buffer (GstQueue2 * queue, GstBuffer * buffer, GstSegment * segment,
@ -818,11 +865,24 @@ apply_buffer (GstQueue2 * queue, GstBuffer * buffer, GstSegment * segment,
duration = GST_BUFFER_DURATION (buffer);
/* If we have no duration, pick one from the bitrate if we can */
if (duration == GST_CLOCK_TIME_NONE && queue->use_tags_bitrate) {
guint bitrate =
is_sink ? queue->sink_tags_bitrate : queue->src_tags_bitrate;
if (bitrate)
duration = gst_util_uint64_scale (size, 8 * GST_SECOND, bitrate);
if (duration == GST_CLOCK_TIME_NONE) {
if (queue->use_tags_bitrate) {
guint bitrate =
is_sink ? queue->sink_tags_bitrate : queue->src_tags_bitrate;
if (bitrate)
duration = gst_util_uint64_scale (size, 8 * GST_SECOND, bitrate);
}
if (duration == GST_CLOCK_TIME_NONE && !is_sink && queue->use_bitrate_query) {
if (queue->downstream_bitrate > 0) {
duration =
gst_util_uint64_scale (size, 8 * GST_SECOND,
queue->downstream_bitrate);
GST_LOG_OBJECT (queue, "got bitrate %u resulting in estimated "
"duration %" GST_TIME_FORMAT, queue->downstream_bitrate,
GST_TIME_ARGS (duration));
}
}
}
/* if no timestamp is set, assume it's continuous with the previous
@ -895,13 +955,16 @@ apply_buffer_list (GstQueue2 * queue, GstBufferList * buffer_list,
/* if no timestamp is set, assume it's continuous with the previous time */
bld.timestamp = segment->position;
bld.bitrate = 0;
if (queue->use_tags_bitrate) {
if (is_sink)
bld.bitrate = queue->sink_tags_bitrate;
else
bld.bitrate = queue->src_tags_bitrate;
} else
bld.bitrate = 0;
}
if (!is_sink && bld.bitrate == 0 && queue->use_bitrate_query) {
bld.bitrate = queue->downstream_bitrate;
}
gst_buffer_list_foreach (buffer_list, buffer_list_apply_time, &bld);
@ -960,9 +1023,10 @@ get_buffering_level (GstQueue2 * queue, gboolean * is_buffering,
GST_LOG_OBJECT (queue, "we are %s", queue->is_eos ? "EOS" : "NOT_LINKED");
} else {
GST_LOG_OBJECT (queue,
"Cur level bytes/time/buffers %u/%" GST_TIME_FORMAT "/%u",
queue->cur_level.bytes, GST_TIME_ARGS (queue->cur_level.time),
queue->cur_level.buffers);
"Cur level bytes/time/rate-time/buffers %u/%" GST_TIME_FORMAT "/%"
GST_TIME_FORMAT "/%u", queue->cur_level.bytes,
GST_TIME_ARGS (queue->cur_level.time),
GST_TIME_ARGS (queue->cur_level.rate_time), queue->cur_level.buffers);
/* figure out the buffering level we are filled, we take the max of all formats. */
if (!QUEUE_IS_USING_RING_BUFFER (queue)) {
@ -1201,7 +1265,15 @@ update_in_rates (GstQueue2 * queue, gboolean force)
queue->bytes_in = 0;
}
if (queue->byte_in_rate > 0.0) {
if (queue->use_bitrate_query && queue->downstream_bitrate > 0) {
queue->cur_level.rate_time =
gst_util_uint64_scale (8 * queue->cur_level.bytes, GST_SECOND,
queue->downstream_bitrate);
GST_LOG_OBJECT (queue,
"got bitrate %u with byte level %u resulting in time %"
GST_TIME_FORMAT, queue->downstream_bitrate, queue->cur_level.bytes,
GST_TIME_ARGS (queue->cur_level.rate_time));
} else if (queue->byte_in_rate > 0.0) {
queue->cur_level.rate_time =
queue->cur_level.bytes / queue->byte_in_rate * GST_SECOND;
}
@ -2531,6 +2603,7 @@ gst_queue2_handle_sink_event (GstPad * pad, GstObject * parent,
gst_event_unref (event);
}
g_object_notify (G_OBJECT (queue), "bitrate");
break;
}
case GST_EVENT_TAG:{
@ -2545,12 +2618,14 @@ gst_queue2_handle_sink_event (GstPad * pad, GstObject * parent,
queue->sink_tags_bitrate = bitrate;
GST_QUEUE2_MUTEX_UNLOCK (queue);
GST_LOG_OBJECT (queue, "Sink pad bitrate from tags now %u", bitrate);
g_object_notify (G_OBJECT (queue), "bitrate");
}
}
/* Fall-through */
}
default:
if (GST_EVENT_IS_SERIALIZED (event)) {
gboolean bitrate_changed = TRUE;
/* serialized events go in the queue */
/* STREAM_START and SEGMENT reset the EOS status of a
@ -2600,6 +2675,7 @@ gst_queue2_handle_sink_event (GstPad * pad, GstObject * parent,
queue->seeking = FALSE;
queue->src_tags_bitrate = queue->sink_tags_bitrate = 0;
}
bitrate_changed = TRUE;
break;
default:
@ -2610,6 +2686,8 @@ gst_queue2_handle_sink_event (GstPad * pad, GstObject * parent,
gst_queue2_locked_enqueue (queue, event, GST_QUEUE2_ITEM_TYPE_EVENT);
GST_QUEUE2_MUTEX_UNLOCK (queue);
gst_queue2_post_buffering (queue);
if (bitrate_changed)
g_object_notify (G_OBJECT (queue), "bitrate");
} else {
/* non-serialized events are passed downstream. */
ret = gst_pad_push_event (queue->srcpad, event);
@ -2974,6 +3052,7 @@ next:
queue->src_tags_bitrate = bitrate;
GST_QUEUE2_MUTEX_UNLOCK (queue);
GST_LOG_OBJECT (queue, "src pad bitrate from tags now %u", bitrate);
g_object_notify (G_OBJECT (queue), "bitrate");
}
}
}
@ -3167,9 +3246,13 @@ gst_queue2_handle_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
gst_pad_start_task (pad, (GstTaskFunction) gst_queue2_loop, pad,
NULL);
}
}
GST_QUEUE2_MUTEX_UNLOCK (queue);
/* force a new bitrate query to be performed */
query_downstream_bitrate (queue);
res = gst_pad_push_event (queue->sinkpad, event);
break;
default:
@ -3670,6 +3753,7 @@ gst_queue2_change_state (GstElement * element, GstStateChange transition)
queue->starting_segment = NULL;
gst_event_replace (&queue->stream_start_event, NULL);
GST_QUEUE2_MUTEX_UNLOCK (queue);
query_downstream_bitrate (queue);
break;
case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
break;
@ -3832,6 +3916,9 @@ gst_queue2_set_property (GObject * object,
case PROP_RING_BUFFER_MAX_SIZE:
queue->ring_buffer_max_size = g_value_get_uint64 (value);
break;
case PROP_USE_BITRATE_QUERY:
queue->use_bitrate_query = g_value_get_boolean (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@ -3916,6 +4003,23 @@ gst_queue2_get_property (GObject * object,
g_value_set_int64 (value, (gint64) in_rate);
break;
}
case PROP_USE_BITRATE_QUERY:
g_value_set_boolean (value, queue->use_bitrate_query);
break;
case PROP_BITRATE:{
guint64 bitrate = 0;
if (bitrate == 0 && queue->use_tags_bitrate) {
if (queue->sink_tags_bitrate > 0)
bitrate = queue->sink_tags_bitrate;
else if (queue->src_tags_bitrate)
bitrate = queue->src_tags_bitrate;
}
if (bitrate == 0 && queue->use_bitrate_query) {
bitrate = queue->downstream_bitrate;
}
g_value_set_uint64 (value, (guint64) bitrate);
break;
}
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;

View file

@ -109,8 +109,10 @@ struct _GstQueue2
GstQueue2Size max_level; /* max. amount of data allowed in the queue */
gboolean use_buffering;
gboolean use_tags_bitrate;
gboolean use_bitrate_query;
gboolean use_rate_estimate;
GstClockTime buffering_interval;
guint downstream_bitrate; /* the bitrate reported by downstream */
/* low/high watermarks for buffering */
gint low_watermark;

View file

@ -246,9 +246,16 @@ pad_push_datablock_thread (gpointer data)
}
static GstPadProbeReturn
block_probe (GstPad * pad, GstPadProbeInfo * info, gpointer user_data)
block_without_queries_probe (GstPad * pad, GstPadProbeInfo * info,
gpointer user_data)
{
return GST_PAD_PROBE_OK;
GstPadProbeReturn ret = GST_PAD_PROBE_OK;
/* allows queries to pass through */
if ((GST_PAD_PROBE_INFO_TYPE (info) & GST_PAD_PROBE_TYPE_QUERY_BOTH) != 0)
ret = GST_PAD_PROBE_PASS;
return ret;
}
#define CHECK_FOR_BUFFERING_MSG(PIPELINE, EXPECTED_PERC) \
@ -298,8 +305,8 @@ GST_START_TEST (test_watermark_and_fill_level)
/* Block fakesink sinkpad flow to ensure the queue isn't emptied
* by the prerolling sink */
sinkpad = gst_element_get_static_pad (fakesink, "sink");
gst_pad_add_probe (sinkpad, GST_PAD_PROBE_TYPE_BLOCK, block_probe, NULL,
NULL);
gst_pad_add_probe (sinkpad, GST_PAD_PROBE_TYPE_BLOCK,
block_without_queries_probe, NULL, NULL);
gst_object_unref (sinkpad);
g_object_set (queue2,
@ -544,6 +551,116 @@ GST_START_TEST (test_small_ring_buffer)
GST_END_TEST;
#define DOWNSTREAM_BITRATE (8 * 100 * 1000)
static GstPadProbeReturn
bitrate_query_probe (GstPad * pad, GstPadProbeInfo * info, gpointer user_data)
{
GstPadProbeReturn ret = GST_PAD_PROBE_OK;
/* allows queries to pass through */
if ((GST_PAD_PROBE_INFO_TYPE (info) & GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM) !=
0) {
GstQuery *query = GST_PAD_PROBE_INFO_QUERY (info);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_BITRATE:{
gst_query_set_bitrate (query, DOWNSTREAM_BITRATE);
ret = GST_PAD_PROBE_HANDLED;
break;
}
default:
break;
}
}
return ret;
}
GST_START_TEST (test_bitrate_query)
{
/* This test checks the behavior of the bitrate query usage with the
* fill levels and buffering messages */
GstElement *pipe;
GstElement *queue2, *fakesink;
GstPad *inputpad;
GstPad *queue2_sinkpad;
GstPad *sinkpad;
GstSegment segment;
GThread *thread;
/* Setup test pipeline with one queue2 and one fakesink */
pipe = gst_pipeline_new ("pipeline");
queue2 = gst_element_factory_make ("queue2", NULL);
fail_unless (queue2 != NULL);
gst_bin_add (GST_BIN (pipe), queue2);
fakesink = gst_element_factory_make ("fakesink", NULL);
fail_unless (fakesink != NULL);
gst_bin_add (GST_BIN (pipe), fakesink);
/* Block fakesink sinkpad flow to ensure the queue isn't emptied
* by the prerolling sink */
sinkpad = gst_element_get_static_pad (fakesink, "sink");
gst_pad_add_probe (sinkpad, GST_PAD_PROBE_TYPE_BLOCK,
block_without_queries_probe, NULL, NULL);
gst_pad_add_probe (sinkpad, GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM,
bitrate_query_probe, NULL, NULL);
gst_object_unref (sinkpad);
g_object_set (queue2,
"use-buffering", (gboolean) TRUE,
"use-bitrate-query", (gboolean) TRUE,
"max-size-bytes", (guint) 0,
"max-size-buffers", (guint) 0,
"max-size-time", (guint64) 1 * GST_SECOND, NULL);
gst_segment_init (&segment, GST_FORMAT_TIME);
inputpad = gst_pad_new ("dummysrc", GST_PAD_SRC);
gst_pad_set_query_function (inputpad, queue2_dummypad_query);
queue2_sinkpad = gst_element_get_static_pad (queue2, "sink");
fail_unless (queue2_sinkpad != NULL);
fail_unless (gst_pad_link (inputpad, queue2_sinkpad) == GST_PAD_LINK_OK);
gst_pad_set_active (inputpad, TRUE);
gst_pad_push_event (inputpad, gst_event_new_stream_start ("test"));
gst_pad_push_event (inputpad, gst_event_new_segment (&segment));
gst_object_unref (queue2_sinkpad);
fail_unless (gst_element_link (queue2, fakesink));
/* Start pipeline in paused state to ensure the sink remains
* in preroll mode and blocks */
gst_element_set_state (pipe, GST_STATE_PAUSED);
/* When the use-buffering property is set to TRUE, a buffering
* message is posted. Since the queue is empty at that point,
* the buffering message contains a value of 0%. */
CHECK_FOR_BUFFERING_MSG (pipe, 0);
/* Feed data. queue will be filled to 80% (80000 bytes is pushed and
* with a bitrate of 100 * 1000, 80000 bytes is 80% of 1 second of data as
* set in the max-size-time limit) */
thread = g_thread_new ("push1", pad_push_datablock_thread, inputpad);
g_thread_join (thread);
/* Check for the buffering message; it should indicate 80% fill level
* (Note that the percentage from the message is normalized) */
CHECK_FOR_BUFFERING_MSG (pipe, 80);
gst_element_set_state (pipe, GST_STATE_NULL);
gst_object_unref (pipe);
gst_object_unref (inputpad);
}
GST_END_TEST;
static Suite *
queue2_suite (void)
{
@ -560,6 +677,7 @@ queue2_suite (void)
tcase_add_test (tc_chain, test_filled_read);
tcase_add_test (tc_chain, test_percent_overflow);
tcase_add_test (tc_chain, test_small_ring_buffer);
tcase_add_test (tc_chain, test_bitrate_query);
return s;
}