gstreamer/subprojects/gst-plugins-bad/tests/examples/playout.c

1110 lines
38 KiB
C

/* Copyright (C) 2015 Centricular Ltd
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <gst/gst.h>
#include <gst/video/gstvideosink.h>
#define STR_HELPER(x) #x
#define STR(x) STR_HELPER(x)
/* Change this to set the output resolution */
#define OUTPUT_VIDEO_WIDTH 1280
#define OUTPUT_VIDEO_HEIGHT 720
/* Video and audio caps outputted by the mixers */
#define RAW_AUDIO_CAPS_STR "audio/x-raw, format=(string)S16LE, " \
"layout=(string)interleaved, rate=(int)44100, channels=(int)2, " \
"channel-mask=(bitmask)0x03"
#define RAW_VIDEO_CAPS_STR "video/x-raw, width=(int)" STR(OUTPUT_VIDEO_WIDTH) \
", height=(int)" STR(OUTPUT_VIDEO_HEIGHT) ", framerate=(fraction)25/1, " \
"format=I420, pixel-aspect-ratio=(fraction)1/1, " \
"interlace-mode=(string)progressive"
GST_DEBUG_CATEGORY_STATIC (playout);
#define GST_CAT_DEFAULT playout
typedef enum
{
PLAYOUT_APP_STATE_READY, /* Newly created */
PLAYOUT_APP_STATE_PLAYING, /* Playing an item */
PLAYOUT_APP_STATE_EOS /* Finished playing, all items EOS */
} PlayoutAppState;
typedef struct
{
/* Application state */
PlayoutAppState state;
/* An array of PlayoutItems that will be played in sequence */
GPtrArray *play_queue;
/* Index of the currently-playing item */
gint play_queue_current;
/* Lock access to the play queue */
GMutex play_queue_lock;
GMainLoop *main_loop;
/* Pipeline */
GstElement *pipeline;
/* Output */
GstElement *video_mixer;
GstElement *video_sink;
GstVideoRectangle video_orect; /* w/h/x/y of the output */
GstElement *audio_mixer;
GstElement *audio_sink;
/* The duration of all items that have been played in ns.
* Only updates when a new item is activated. */
guint64 elapsed_duration;
} PlayoutApp;
typedef enum
{
PLAYOUT_ITEM_STATE_NEW, /* Newly created */
PLAYOUT_ITEM_STATE_PREPARED, /* Prepared and ready to activate */
PLAYOUT_ITEM_STATE_ACTIVATED, /* Activated */
PLAYOUT_ITEM_STATE_FIRST_VBUFFER, /* First video buffer has gone through */
PLAYOUT_ITEM_STATE_AGGREGATING, /* Audio & video buffers are aggregating */
PLAYOUT_ITEM_STATE_EOS /* At least one pad is EOS */
} PlayoutItemState;
typedef struct
{
PlayoutApp *app;
PlayoutItemState state;
gchar *fn;
GstElement *decoder; /* bin with uridecodebin + converters */
/* We just use the first audio stream and ignore the rest (if there is audio) */
GstPad *audio_pad; /* decoder bin audio src ghostpad */
GstPad *video_pad; /* decoder bin video src ghostpad */
GstVideoRectangle video_irect; /* input w/h/x/y of the item */
GstVideoRectangle video_orect; /* output w/h/x/y of the item */
/* When this item has finished preparing and all pads have been connected to
* mixers, the pads will be blocked till it's this item's turn to be played */
gulong audio_pad_probe_block_id;
gulong video_pad_probe_block_id;
/* The current running time of this item; updated with every audio buffer if
* this item has audio; otherwise it's updated with very video buffer */
guint64 running_time;
} PlayoutItem;
static PlayoutApp *playout_app_new (void);
static void playout_app_free (PlayoutApp * app);
static PlayoutItem *playout_item_new (PlayoutApp * app, const gchar * fn);
static void playout_item_free (PlayoutItem * item);
static void playout_app_add_item (PlayoutApp * app, const gchar * fn);
static gboolean playout_app_prepare_item (PlayoutItem * item);
static gboolean playout_app_activate_item (PlayoutItem * item);
static gboolean playout_app_activate_next_item (PlayoutApp * app);
static gboolean playout_app_activate_next_item_early (PlayoutApp * app);
static PlayoutItem *playout_app_get_current_item (PlayoutApp * app);
static gboolean playout_app_remove_item (PlayoutItem * item);
static void
playout_app_add_audio_sink (PlayoutApp * app)
{
GstElement *audio_resample, *audio_conv, *queue;
/* audiomixer doesn't do conversion yet, so we don't need an output capsfilter
* for this branch. The output format is decided by the sink pads, which all
* have to have the same format. */
app->audio_mixer = gst_element_factory_make ("audiomixer", "audio_mixer");
audio_conv = gst_element_factory_make ("audioconvert", "mixer_audioconvert");
audio_resample = gst_element_factory_make ("audioresample",
"audio_mixer_audioresample");
queue = gst_element_factory_make ("queue", "asink_queue");
app->audio_sink = gst_element_factory_make ("autoaudiosink", NULL);
g_object_set (app->audio_sink, "async-handling", TRUE, NULL);
gst_bin_add_many (GST_BIN (app->pipeline), app->audio_mixer, audio_conv,
audio_resample, queue, app->audio_sink, NULL);
gst_element_link_many (app->audio_mixer, audio_conv, audio_resample,
queue, app->audio_sink, NULL);
if (!gst_element_sync_state_with_parent (app->audio_mixer) ||
!gst_element_sync_state_with_parent (audio_conv) ||
!gst_element_sync_state_with_parent (audio_resample) ||
!gst_element_sync_state_with_parent (queue) ||
!gst_element_sync_state_with_parent (app->audio_sink))
GST_ERROR ("app: unable to sync audio mixer + sink state with pipeline");
}
static PlayoutApp *
playout_app_new (void)
{
GstElement *video_capsfilter, *queue;
GstCaps *caps;
PlayoutApp *app;
app = g_new0 (PlayoutApp, 1);
app->state = PLAYOUT_APP_STATE_READY;
app->play_queue =
g_ptr_array_new_with_free_func ((GDestroyNotify) playout_item_free);
app->play_queue_current = -1;
g_mutex_init (&app->play_queue_lock);
app->main_loop = g_main_loop_new (NULL, FALSE);
app->pipeline = gst_pipeline_new ("pipeline");
/* It's best to set a caps filter for the video output format */
app->video_orect.w = OUTPUT_VIDEO_WIDTH;
app->video_orect.h = OUTPUT_VIDEO_HEIGHT;
app->video_orect.x = 0;
app->video_orect.y = 0;
app->video_mixer = gst_element_factory_make ("compositor", "video_mixer");
/* Set the background as black; faster while compositing, and allows us to
* rescale videos with a different aspect ratio than the output in a way that
* adds black borders automatically */
g_object_set (app->video_mixer, "background", 1, NULL);
queue = gst_element_factory_make ("queue", "vsink_queue");
app->video_sink = gst_element_factory_make ("autovideosink", NULL);
g_object_set (app->video_sink, "async-handling", TRUE, NULL);
video_capsfilter = gst_element_factory_make ("capsfilter",
"video_mixer_capsfilter");
caps = gst_caps_from_string (RAW_VIDEO_CAPS_STR);
g_object_set (video_capsfilter, "caps", caps, NULL);
gst_caps_unref (caps);
gst_bin_add_many (GST_BIN (app->pipeline), app->video_mixer, video_capsfilter,
queue, app->video_sink, NULL);
gst_element_link_many (app->video_mixer, video_capsfilter, queue,
app->video_sink, NULL);
return app;
}
static void
playout_app_free (PlayoutApp * app)
{
GST_DEBUG ("Freeing app");
g_ptr_array_unref (app->play_queue);
g_main_loop_unref (app->main_loop);
gst_element_set_state (app->pipeline, GST_STATE_NULL);
gst_object_unref (app->pipeline);
g_free (app);
}
static void
playout_app_eos (GstBus * bus, GstMessage * msg, PlayoutApp * app)
{
g_print ("All streams EOS, exiting...\n");
g_main_loop_quit (app->main_loop);
}
static PlayoutItem *
playout_item_new (PlayoutApp * app, const gchar * fn)
{
PlayoutItem *item = g_new0 (PlayoutItem, 1);
item->app = app;
item->state = PLAYOUT_ITEM_STATE_NEW;
item->fn = g_strdup (fn);
return item;
}
/* Unlink and release the pad */
static gboolean
playout_remove_pad (GstPad * srcpad)
{
GstPad *sinkpad;
GstElement *mixer;
sinkpad = gst_pad_get_peer (srcpad);
if (!sinkpad)
return FALSE;
if (!gst_pad_unlink (srcpad, sinkpad))
return FALSE;
mixer = gst_pad_get_parent_element (sinkpad);
gst_element_release_request_pad (mixer, sinkpad);
GST_DEBUG ("Released some pad");
gst_object_unref (sinkpad);
gst_object_unref (mixer);
return FALSE;
}
static GstPadProbeReturn
playout_item_pad_probe_blocked (GstPad * srcpad, GstPadProbeInfo * info,
PlayoutItem * item)
{
if (srcpad == item->audio_pad) {
item->audio_pad_probe_block_id = GST_PAD_PROBE_INFO_ID (info);
} else if (srcpad == item->video_pad) {
item->video_pad_probe_block_id = GST_PAD_PROBE_INFO_ID (info);
} else {
g_assert_not_reached ();
}
return GST_PAD_PROBE_OK;
}
static GstPadProbeReturn
playout_item_pad_probe_pad_running_time (GstPad * srcpad,
GstPadProbeInfo * info, PlayoutItem * item)
{
GstEvent *event;
GstBuffer *buffer;
guint64 running_time;
const GstSegment *segment;
buffer = GST_PAD_PROBE_INFO_BUFFER (info);
event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0);
GST_TRACE ("%s: pad sticky event: %" GST_PTR_FORMAT, item->fn, event);
if (event) {
gst_event_parse_segment (event, &segment);
gst_event_unref (event);
running_time = gst_segment_to_running_time (segment, GST_FORMAT_TIME,
GST_BUFFER_PTS (buffer));
} else {
GST_WARNING ("%s: unable to parse event for segment; falling back to pts. "
"Output will probably have glitches.", item->fn);
running_time = GST_BUFFER_PTS (buffer);
}
item->running_time = running_time + GST_BUFFER_DURATION (buffer);
GST_TRACE ("%s: running time is %" G_GUINT64_FORMAT ", duration is %"
G_GUINT64_FORMAT, item->fn, item->running_time,
GST_BUFFER_DURATION (buffer));
return GST_PAD_PROBE_PASS;
}
static GstPadProbeReturn
playout_item_pad_probe_video_pad_eos_on_buffer (GstPad * srcpad,
GstPadProbeInfo * info, PlayoutItem * prev_item)
{
PlayoutItem *current_item;
current_item = playout_app_get_current_item (prev_item->app);
if (!current_item)
return GST_PAD_PROBE_REMOVE;
/* Step through the item's states as buffers pass through. The first buffer
* will be taken by the video_mixer, and kept till the audio running time
* matches the video buffer running time. When the second buffer gets through,
* we know that this pad has begun aggregating. */
switch (current_item->state) {
case PLAYOUT_ITEM_STATE_NEW:
case PLAYOUT_ITEM_STATE_PREPARED:
GST_DEBUG ("%s: new/prepared", current_item->fn);
break;
case PLAYOUT_ITEM_STATE_ACTIVATED:
GST_DEBUG ("%s: activated -> first vbuffer", current_item->fn);
current_item->state = PLAYOUT_ITEM_STATE_FIRST_VBUFFER;
break;
case PLAYOUT_ITEM_STATE_FIRST_VBUFFER:
GST_DEBUG ("%s: first vbuffer -> aggregating", current_item->fn);
current_item->state = PLAYOUT_ITEM_STATE_AGGREGATING;
gst_pad_remove_probe (srcpad, GST_PAD_PROBE_INFO_ID (info));
/* Item is aggregating, release the previous item's video pad */
goto release;
break;
case PLAYOUT_ITEM_STATE_EOS:
return GST_PAD_PROBE_REMOVE;
default:
g_assert_not_reached ();
}
return GST_PAD_PROBE_PASS;
release:
{
playout_remove_pad (prev_item->video_pad);
GST_DEBUG ("%s: released video pad", prev_item->fn);
prev_item->video_pad = NULL;
/* If there's no audio pad, or if the audio pad is already EOS, we can
* remove this item from the queue which will free it. Need to free the
* item from the main thread because it causes the item's decoder bin
* to be removed from the pipeline, which cannot be done in the
* streaming thread */
if (prev_item->audio_pad == NULL) {
GST_DEBUG ("%s: queued item removal (last pad is video)", prev_item->fn);
g_main_context_invoke (NULL, (GSourceFunc) playout_app_remove_item,
prev_item);
}
/* Pad probe has already been removed above */
return GST_PAD_PROBE_PASS;
}
}
/* This is called on EOS for both item->audio_pad and item->video_pad
*
* FIXME: Add locking. Both pads could go EOS at the exact same time. */
static GstPadProbeReturn
playout_item_pad_probe_event (GstPad * srcpad, GstPadProbeInfo * info,
PlayoutItem * item)
{
GstEventType type;
gboolean ret = TRUE;
GstPadProbeReturn probe_ret = GST_PAD_PROBE_DROP;
type = GST_EVENT_TYPE (GST_PAD_PROBE_INFO_DATA (info));
if (type != GST_EVENT_EOS)
return GST_PAD_PROBE_PASS;
/* We might get two EOSes on this pad if we send an artificial EOS. Remove
* the probe so this is only called once for each pad */
gst_pad_remove_probe (srcpad, GST_PAD_PROBE_INFO_ID (info));
GST_DEBUG ("%s: recvd some EOS", item->fn);
if (item->state != PLAYOUT_ITEM_STATE_EOS) {
/* We have more than one pad per item (video + audio item), and this is the
* first pad to go EOS or we have only one pad per item, and that pad has
* gone EOS. For the first case, the other pad might still have some buffers
* to output before going EOS, but we need to activate the next item and
* start outputting buffers from that immediately. */
/* Update the total elapsed duration from the item's current running time */
item->app->elapsed_duration += item->running_time;
GST_DEBUG ("%s: activating next item", item->fn);
/* Activate the next item if and only if this is the first pad to go EOS */
ret = playout_app_activate_next_item (item->app);
if (!ret) {
GST_DEBUG ("%s: App is going EOS", item->fn);
item->state = PLAYOUT_ITEM_STATE_EOS;
item->app->state = PLAYOUT_APP_STATE_EOS;
/* If we couldn't activate the next item, pass the EOS event onward,
* ending the stream */
probe_ret = GST_PAD_PROBE_PASS;
}
}
g_assert (srcpad != NULL);
if (srcpad == item->audio_pad) {
GST_DEBUG ("%s: audio pad was EOS", item->fn);
if (item->app->state != PLAYOUT_APP_STATE_EOS) {
/* While activating the next item, we ensure that there's no offset mismatch
* which would cause audiomixer to output silence, so we can release the
* previous item's audio request pad here. We also unlink the audio pad;
* nothing else is needed from it */
playout_remove_pad (srcpad);
GST_DEBUG ("%s: released audio pad", item->fn);
/* If there's no video pad, or if the video pad is already EOS, we can
* remove this item from the queue which will free it. Need to free the
* item from the main thread because it causes the item's decoder bin
* to be removed from the pipeline, which cannot be done in the
* streaming thread */
if (item->video_pad == NULL) {
GST_DEBUG ("%s: queued item removal (last pad is audio)", item->fn);
g_main_context_invoke (NULL, (GSourceFunc) playout_app_remove_item,
item);
}
} else {
/* If this is the last pad on audio_mixer, let the EOS go through
* before unlinking/releasing the pad. This should happen within 500ms. */
g_timeout_add (500, (GSourceFunc) playout_remove_pad, srcpad);
GST_DEBUG ("%s: queued audio pad release", item->fn);
if (item->video_pad == NULL) {
/* Unlike above, we need to wait till the pad is removed before removing
* the item from the app, so we queue it for 100ms afterwards */
GST_DEBUG ("%s: queued last item removal (last pad is audio)",
item->fn);
g_timeout_add (600, (GSourceFunc) playout_app_remove_item, item);
}
}
item->audio_pad = NULL;
} else if (srcpad == item->video_pad) {
GST_DEBUG ("%s: video pad was EOS", item->fn);
if (item->audio_pad != NULL)
GST_WARNING ("%s: video pad went EOS before audio pad! "
"There will be audio/video glitches while switching.", item->fn);
if (item->app->state != PLAYOUT_APP_STATE_EOS) {
PlayoutItem *next_item;
next_item = playout_app_get_current_item (item->app);
GST_DEBUG ("%s: next item is %s, %i/%i", item->fn, next_item->fn,
next_item->state, PLAYOUT_ITEM_STATE_ACTIVATED);
g_assert (next_item != NULL);
/* If there's another item being activated, release this video pad only
* when the next item's video pad starts being aggregated; that happens
* when this probe receives its 2nd buffer from the next item */
gst_pad_add_probe (next_item->video_pad, GST_PAD_PROBE_TYPE_BUFFER,
(GstPadProbeCallback) playout_item_pad_probe_video_pad_eos_on_buffer,
item, NULL);
} else {
/* If this is the last pad on video_mixer, let the EOS go through
* before unlinking/releasing the pad. This should happen within 500ms. */
g_timeout_add (500, (GSourceFunc) playout_remove_pad, srcpad);
GST_DEBUG ("%s: queued video pad release", item->fn);
item->video_pad = NULL;
}
probe_ret = GST_PAD_PROBE_PASS;
} else {
g_assert_not_reached ();
}
item->state = PLAYOUT_ITEM_STATE_EOS;
/* NOTE: If the srcpad has been unlinked, the return value is useless */
return probe_ret;
}
/* On the "pad-added" signal of uridecodebin, add converters and connect to
* audio/video mixers */
static void
playout_item_new_pad (GstElement * uridecodebin, GstPad * pad,
PlayoutItem * item)
{
GstStructure *s;
GstCaps *caps;
GstPad *sinkpad, *srcpad;
GstElement *queue;
GstPadProbeType block_probe_type;
caps = gst_pad_get_current_caps (pad);
s = gst_caps_get_structure (caps, 0);
GST_DEBUG ("%s: new pad: %p, caps: %s", item->fn, pad,
gst_structure_get_name (s));
if (gst_structure_has_name (s, "audio/x-raw")) {
if (item->audio_pad != NULL)
/* Ignore all audio pads after the first one */
goto out;
goto audio;
} else if (gst_structure_has_name (s, "video/x-raw")) {
if (item->video_pad != NULL)
/* Ignore all video pads after the first one */
goto out;
goto video;
} else {
goto out;
}
audio:
{
GstCaps *wanted_caps;
GstElement *audioconvert, *audioresample, *capsfilter;
/* Audio pad found; add audio mixer and audio sink to the pipeline.
* NOTE: If any items after this do not have an audio pad, the pipeline will
* mess up because the audio sink will not receive any data. */
if (item->app->audio_sink == NULL)
playout_app_add_audio_sink (item->app);
wanted_caps = gst_caps_from_string (RAW_AUDIO_CAPS_STR);
if (!gst_caps_is_equal (caps, wanted_caps)) {
GST_DEBUG ("%s: converting audio caps", item->fn);
/* We need to convert the audio to the wanted format because
* audiomixer doesn't do format conversion */
audioresample = gst_element_factory_make ("audioresample", NULL);
audioconvert = gst_element_factory_make ("audioconvert", NULL);
capsfilter = gst_element_factory_make ("capsfilter", NULL);
g_object_set (capsfilter, "caps", wanted_caps, NULL);
queue = gst_element_factory_make ("queue", NULL);
gst_bin_add_many (GST_BIN (item->decoder), audioresample, audioconvert,
capsfilter, queue, NULL);
sinkpad = gst_element_get_static_pad (audioresample, "sink");
gst_pad_link (pad, sinkpad);
gst_object_unref (sinkpad);
gst_element_link_many (audioresample, audioconvert, capsfilter, queue,
NULL);
srcpad = gst_element_get_static_pad (queue, "src");
if (!gst_element_sync_state_with_parent (audioresample) ||
!gst_element_sync_state_with_parent (audioconvert) ||
!gst_element_sync_state_with_parent (capsfilter) ||
!gst_element_sync_state_with_parent (queue)) {
GST_ERROR ("%s: unable to sync audio converter state with decoder",
item->fn);
goto out;
}
} else {
queue = gst_element_factory_make ("queue", NULL);
gst_bin_add (GST_BIN (item->decoder), queue);
sinkpad = gst_element_get_static_pad (queue, "sink");
gst_pad_link (pad, sinkpad);
gst_object_unref (sinkpad);
srcpad = gst_element_get_static_pad (queue, "src");
if (!gst_element_sync_state_with_parent (queue)) {
GST_ERROR ("%s: unable to sync audio queue state with decoder",
item->fn);
goto out;
}
}
gst_caps_unref (wanted_caps);
/* Convert the audioconvert src pad to a ghostpad on the bin */
item->audio_pad = gst_ghost_pad_new (NULL, srcpad);
gst_pad_set_active (item->audio_pad, TRUE);
gst_element_add_pad (item->decoder, item->audio_pad);
gst_object_unref (srcpad);
srcpad = item->audio_pad;
GST_DEBUG ("%s: created audio pad", item->fn);
goto done;
}
video:
{
if (!gst_structure_get_int (s, "width", &item->video_irect.w) ||
!gst_structure_get_int (s, "height", &item->video_irect.h))
GST_WARNING ("%s: unable to set width/height from caps", item->fn);
item->video_irect.x = item->video_irect.y = 0;
queue = gst_element_factory_make ("queue", "video-decoder-queue-%u");
gst_bin_add (GST_BIN (item->decoder), queue);
if (!gst_element_sync_state_with_parent (queue)) {
GST_ERROR ("%s: unable to sync video queue state with decoder", item->fn);
goto out;
}
sinkpad = gst_element_get_static_pad (queue, "sink");
gst_pad_link (pad, sinkpad);
gst_object_unref (sinkpad);
/* Convert the queue src pad to a ghostpad on the bin */
srcpad = gst_element_get_static_pad (queue, "src");
item->video_pad = gst_ghost_pad_new (NULL, srcpad);
gst_pad_set_active (item->video_pad, TRUE);
gst_element_add_pad (item->decoder, item->video_pad);
gst_object_unref (srcpad);
srcpad = item->video_pad;
GST_DEBUG ("%s: created video pad", item->fn);
goto done;
}
done:
/* We let events and queries through */
block_probe_type = GST_PAD_PROBE_TYPE_BLOCK |
GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST;
/* If the app is already playing an item, block everything except queries
* till we need to play this item */
if (item->app->state != PLAYOUT_APP_STATE_READY)
gst_pad_add_probe (srcpad, block_probe_type,
(GstPadProbeCallback) playout_item_pad_probe_blocked, item, NULL);
/* Probe events for EOS */
gst_pad_add_probe (srcpad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM,
(GstPadProbeCallback) playout_item_pad_probe_event, item, NULL);
out:
gst_caps_unref (caps);
}
/* All pads on uridecodebin have finished being populated; the item has been
* prepared and is ready to be activated */
static void
playout_item_no_more_pads (GstElement * uridecodebin, PlayoutItem * item)
{
/* Set a buffer pad probe that constantly updates the item's
* elapsed_duration using the duration of each audio buffer */
if (item->audio_pad) {
gst_pad_add_probe (item->audio_pad, GST_PAD_PROBE_TYPE_BUFFER,
(GstPadProbeCallback) playout_item_pad_probe_pad_running_time,
item, NULL);
} else if (item->video_pad) {
gst_pad_add_probe (item->video_pad, GST_PAD_PROBE_TYPE_BUFFER,
(GstPadProbeCallback) playout_item_pad_probe_pad_running_time,
item, NULL);
} else {
GST_ERROR ("%s: no pads were generated! Can't continue playing!", item->fn);
return;
}
item->state = PLAYOUT_ITEM_STATE_PREPARED;
GST_DEBUG ("%s: prepared", item->fn);
if (item->app->state != PLAYOUT_APP_STATE_READY)
/* This item will be activated when the previous one is EOS */
return;
GST_DEBUG ("Application isn't already playing; activate the item and prepare"
" the next one");
playout_app_activate_item (item);
item->state = PLAYOUT_ITEM_STATE_ACTIVATED;
item->app->state = PLAYOUT_APP_STATE_PLAYING;
if (item->app->play_queue->len > 1)
playout_app_prepare_item (g_ptr_array_index (item->app->play_queue, 1));
}
static GstElement *
playout_item_create_decoder (PlayoutItem * item)
{
GstElement *bin, *dec;
GError *err = NULL;
gchar *uri;
uri = gst_filename_to_uri (item->fn, &err);
if (err != NULL) {
GST_WARNING ("Could not convert '%s' to uri: %s", item->fn, err->message);
g_clear_error (&err);
return NULL;
}
bin = gst_bin_new (NULL);
dec = gst_element_factory_make ("uridecodebin", NULL);
g_object_set (dec, "uri", uri, NULL);
g_free (uri);
gst_bin_add (GST_BIN (bin), dec);
g_signal_connect (dec, "pad-added", G_CALLBACK (playout_item_new_pad), item);
g_signal_connect (dec, "no-more-pads", G_CALLBACK (playout_item_no_more_pads),
item);
return bin;
}
static void
playout_item_free (PlayoutItem * item)
{
GST_DEBUG ("Entering free");
switch (gst_element_set_state (item->decoder, GST_STATE_NULL)) {
case GST_STATE_CHANGE_FAILURE:
GST_ERROR ("%s: Unable to change state to NULL", item->fn);
break;
case GST_STATE_CHANGE_SUCCESS:
GST_DEBUG ("%s: State change success", item->fn);
break;
default:
GST_DEBUG ("%s: Some async/no-preroll", item->fn);
}
gst_bin_remove (GST_BIN (item->app->pipeline), item->decoder);
GST_DEBUG ("%s: bin removed", item->fn);
g_free (item->fn);
g_free (item);
GST_DEBUG ("item freed");
}
static guint64
playout_item_pad_get_segment_time (GstPad * srcpad)
{
GstEvent *event;
const GstSegment *segment;
event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0);
if (!event)
return 0;
gst_event_parse_segment (event, &segment);
gst_event_unref (event);
return segment->time;
}
static void
playout_app_add_item (PlayoutApp * app, const gchar * fn)
{
PlayoutItem *item;
item = playout_item_new (app, fn);
g_mutex_lock (&app->play_queue_lock);
g_ptr_array_add (app->play_queue, item);
g_mutex_unlock (&app->play_queue_lock);
}
static gboolean
playout_app_remove_item (PlayoutItem * item)
{
PlayoutApp *app;
GST_DEBUG ("%s: removing and freeing", item->fn);
app = item->app;
g_mutex_lock (&app->play_queue_lock);
g_ptr_array_remove (app->play_queue, item);
if (item->state >= PLAYOUT_ITEM_STATE_ACTIVATED)
/* Removed item was playing; decrement the current-play-queue index */
app->play_queue_current--;
g_mutex_unlock (&app->play_queue_lock);
/* Don't call this again */
return FALSE;
}
static PlayoutItem *
playout_app_get_current_item (PlayoutApp * app)
{
if (app->play_queue_current < 0 ||
app->play_queue->len < (app->play_queue_current + 1))
return NULL;
return g_ptr_array_index (app->play_queue, app->play_queue_current);
}
static gboolean
playout_app_prepare_item (PlayoutItem * item)
{
PlayoutApp *app = item->app;
if (item->decoder != NULL)
return TRUE;
item->decoder = playout_item_create_decoder (item);
if (item->decoder == NULL)
return FALSE;
gst_bin_add (GST_BIN (app->pipeline), item->decoder);
if (!gst_element_sync_state_with_parent (item->decoder)) {
GST_ERROR ("%s: unable to sync state with parent", item->fn);
return FALSE;
}
GST_DEBUG ("%s: preparing", item->fn);
/* All further processing is done in the "no-more-pads" callback of
* uridecodebin */
return TRUE;
}
/* Called exactly once for each item */
static gboolean
playout_app_activate_item (PlayoutItem * item)
{
GstPad *sinkpad;
guint64 segment_time;
PlayoutApp *app = item->app;
if (item->state != PLAYOUT_ITEM_STATE_PREPARED) {
GST_ERROR ("Item %s is not ready to be activated!", item->fn);
return FALSE;
}
if (!item->audio_pad && !item->video_pad) {
GST_ERROR ("Item %s has no pads! Can't activate it!", item->fn);
return FALSE;
}
/* Hook up to mixers and remove the probes blocking the pads */
if (item->audio_pad) {
GST_DEBUG ("%s: hooking up audio pad to mixer", item->fn);
sinkpad = gst_element_request_pad_simple (app->audio_mixer, "sink_%u");
gst_pad_link (item->audio_pad, sinkpad);
segment_time = playout_item_pad_get_segment_time (item->audio_pad);
if (segment_time > 0) {
/* If the segment time is > 0, the new pad wants audiomixer to output audio
* silence for that duration. This will cause audio glitches, so we move
* the pad offset back by that amount and tell audiomixer to start mixing
* our buffers immediately. */
GST_DEBUG ("%s: subtracting segment time %" G_GUINT64_FORMAT " from "
"elapsed duration before setting it as the pad offset", item->fn,
segment_time);
if (app->elapsed_duration > segment_time)
app->elapsed_duration -= segment_time;
else
app->elapsed_duration = 0;
}
if (app->elapsed_duration > 0) {
GST_DEBUG ("%s: set audio pad offset to %" G_GUINT64_FORMAT "ms",
item->fn, app->elapsed_duration / GST_MSECOND);
gst_pad_set_offset (item->audio_pad, app->elapsed_duration);
}
if (item->audio_pad_probe_block_id > 0) {
GST_DEBUG ("%s: removing audio pad block probe", item->fn);
gst_pad_remove_probe (item->audio_pad, item->audio_pad_probe_block_id);
}
gst_object_unref (sinkpad);
}
if (item->video_pad) {
GST_DEBUG ("%s: hooking up video pad to mixer", item->fn);
sinkpad = gst_element_request_pad_simple (app->video_mixer, "sink_%u");
/* Get new height/width/xpos/ypos such that the video scales up or down to
* fit within the output video size without any cropping */
gst_video_sink_center_rect (item->video_irect, item->app->video_orect,
&item->video_orect, TRUE);
GST_DEBUG ("%s: w: %i, h: %i, x: %i, y: %i", item->fn,
item->video_orect.w, item->video_orect.h, item->video_orect.x,
item->video_orect.y);
g_object_set (sinkpad, "width", item->video_orect.w, "height",
item->video_orect.h, "xpos", item->video_orect.x, "ypos",
item->video_orect.y, NULL);
/* If this is not the last item, on EOS, continue to aggregate using the
* last buffer till the pad is released */
if (item->app->play_queue->len != (item->app->play_queue_current + 2))
g_object_set (sinkpad, "repeat-after-eos", TRUE, NULL);
else
GST_DEBUG ("%s: last item, not setting repeat-after-eos", item->fn);
gst_pad_link (item->video_pad, sinkpad);
if (app->elapsed_duration > 0) {
GST_DEBUG ("%s: set video pad offset to %" G_GUINT64_FORMAT "ms",
item->fn, app->elapsed_duration / GST_MSECOND);
gst_pad_set_offset (item->video_pad, app->elapsed_duration);
}
if (item->video_pad_probe_block_id > 0) {
GST_DEBUG ("%s: removing video pad block probe", item->fn);
gst_pad_remove_probe (item->video_pad, item->video_pad_probe_block_id);
}
gst_object_unref (sinkpad);
}
item->state = PLAYOUT_ITEM_STATE_ACTIVATED;
g_mutex_lock (&item->app->play_queue_lock);
item->app->play_queue_current++;
g_mutex_unlock (&item->app->play_queue_lock);
GST_DEBUG ("%s: activated", item->fn);
return TRUE;
}
/* Activate the next item, and prepare the one after that for later activation */
static gboolean
playout_app_activate_next_item (PlayoutApp * app)
{
PlayoutItem *item;
gboolean ret;
if (app->play_queue->len < (app->play_queue_current + 2)) {
g_print ("No more items to play\n");
return FALSE;
}
item = g_ptr_array_index (app->play_queue, app->play_queue_current + 1);
ret = playout_app_activate_item (item);
if (!ret) {
/* Tell caller, who can then decide whether to skip or error out */
GST_ERROR ("%s: unable to activate", item->fn);
return FALSE;
}
if (app->play_queue->len > (app->play_queue_current + 1)) {
item = g_ptr_array_index (app->play_queue, app->play_queue_current + 1);
/* FIXME: What if this fails? Prepare the next one in the queue? */
ret = playout_app_prepare_item (item);
if (!ret)
GST_ERROR ("%s: unable to prepare", item->fn);
}
return ret;
}
static GstPadProbeReturn
playout_item_pad_probe_video_pad_running_time (GstPad * srcpad,
GstPadProbeInfo * info, PlayoutItem * item)
{
GstEvent *event;
GstBuffer *buffer;
guint64 running_time;
const GstSegment *segment;
buffer = GST_PAD_PROBE_INFO_BUFFER (info);
event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0);
GST_TRACE ("%s: video sticky event: %" GST_PTR_FORMAT, item->fn, event);
if (event) {
gst_event_parse_segment (event, &segment);
gst_event_unref (event);
running_time = gst_segment_to_running_time (segment, GST_FORMAT_TIME,
GST_BUFFER_PTS (buffer));
} else {
GST_WARNING ("%s: unable to parse video event for segment; falling back to "
"pts", item->fn);
running_time = GST_BUFFER_PTS (buffer);
}
if (running_time >= item->running_time) {
/* The video buffer passing through video_mixer now matches the audio buffer
* that passed through audio_mixer when the early switch was requested, so
* this is the time to send an EOS to video_pad, which will complete the
* switch */
GST_DEBUG ("Sending video EOS to %s", item->fn);
gst_pad_push_event (item->video_pad, gst_event_new_eos ());
return GST_PAD_PROBE_DROP;
} else {
return GST_PAD_PROBE_PASS;
}
}
static gboolean
playout_app_activate_next_item_early (PlayoutApp * app)
{
PlayoutItem *item;
item = playout_app_get_current_item (app);
if (!item) {
GST_WARNING ("Unable to switch early, no current item");
return FALSE;
}
if (item->audio_pad) {
/* If we have an audio pad, EOS audio first, always */
GST_DEBUG ("Sending audio EOS to %s", item->fn);
gst_pad_push_event (item->audio_pad, gst_event_new_eos ());
/* We can't send the EOS to the video_pad yet because the running times for
* both mixers are different due to buffering at the audio sink. So we wait
* till the running time of the video_pad matches that of the audio_pad at
* the time the audio EOS was sent, and then EOS video as well. */
gst_pad_add_probe (item->video_pad, GST_PAD_PROBE_TYPE_BUFFER,
(GstPadProbeCallback) playout_item_pad_probe_video_pad_running_time,
item, NULL);
} else if (item->video_pad) {
/* If we have a video pad, EOS audio first, always */
GST_DEBUG ("Sending video EOS to %s", item->fn);
gst_pad_push_event (item->video_pad, gst_event_new_eos ());
} else {
g_assert_not_reached ();
}
/* Return FALSE so this function is called only once */
return FALSE;
}
static gboolean
playout_app_play (PlayoutApp * app)
{
PlayoutItem *item;
item = app->play_queue->len ? g_ptr_array_index (app->play_queue, 0) : NULL;
if (!item) {
g_printerr ("Nothing to play\n");
return FALSE;
}
playout_app_prepare_item (item);
return TRUE;
}
/*
* playout: An example application to sequentially and seamlessly play a list of
* audio-video or video-only files.
*
* This example application uses the compositor and audiomixer elements combined
* with pad probes to stitch together a list of A/V or V-only files in such
* a way that audio and video glitching is minimised. Mixing A/V and V-only
* files is not supported because it complicates the architecture quite a bit.
*
* Due to the fundamental difference in the representation of audio and video
* data, unless constructed specifically for the purpose of being stitched back,
* the audio and video tracks of files will rarely end at the same PTS. There is
* usually a sync difference of a few frames. This application tries to stitch
* together the audio tracks as perfectly as possible, and duplicates/drops
* video frames if there is an underrun/overrun. Even when audio samples are
* played back-to-back, there might be glitches due to quirks in the decoder.
*
* The list of PlayoutItems can be edited and added to dynamically; except the
* currently-playing item and the next one (which has been prepared already).
*/
int
main (int argc, char **argv)
{
GstBus *bus;
gint switch_after_ms = 0;
gchar **f, **filenames = NULL;
GOptionEntry options[] = {
{"switch-after", 's', 0, G_OPTION_ARG_INT, &switch_after_ms, "Time after "
"which the next file will be forcibly activated", "MILLISECONDS"},
{G_OPTION_REMAINING, 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &filenames, NULL,
"FILENAME1 [FILENAME2] [FILENAME3] ..."},
{NULL}
};
GOptionContext *ctx;
PlayoutApp *app;
GError *err = NULL;
ctx = g_option_context_new (NULL);
g_option_context_set_summary (ctx, "An example application to sequentially "
"and seamlessly play a list of audio-video or video-only files.");
g_option_context_add_main_entries (ctx, options, NULL);
g_option_context_add_group (ctx, gst_init_get_option_group ());
if (!g_option_context_parse (ctx, &argc, &argv, &err)) {
if (err)
g_printerr ("Error initializing: %s\n", err->message);
else
g_printerr ("Error initializing: Unknown error!\n");
g_option_context_free (ctx);
g_clear_error (&err);
return 1;
}
if (filenames == NULL || *filenames == NULL) {
g_printerr ("%s", g_option_context_get_help (ctx, TRUE, NULL));
return 1;
}
g_option_context_free (ctx);
GST_DEBUG_CATEGORY_INIT (playout, "playout", 0, "Playout example app");
app = playout_app_new ();
for (f = filenames; f != NULL && *f != NULL; ++f)
playout_app_add_item (app, *f);
g_strfreev (filenames);
if (!playout_app_play (app))
return 1;
GST_DEBUG ("Setting pipeline to PLAYING");
bus = gst_pipeline_get_bus (GST_PIPELINE (app->pipeline));
gst_bus_add_signal_watch (bus);
g_signal_connect (bus, "message::eos", G_CALLBACK (playout_app_eos), app);
gst_object_unref (bus);
gst_element_set_state (app->pipeline, GST_STATE_PLAYING);
if (switch_after_ms)
g_timeout_add (switch_after_ms,
(GSourceFunc) playout_app_activate_next_item_early, app);
GST_DEBUG ("Running mainloop");
g_main_loop_run (app->main_loop);
playout_app_free (app);
return 0;
}