mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-12-04 23:46:43 +00:00
790 lines
27 KiB
C
790 lines
27 KiB
C
/*
|
|
* GStreamer
|
|
* Copyright (C) 2015 Vivia Nikolaidou <vivia@toolsonair.com>
|
|
*
|
|
* Based on gstlevel.c:
|
|
* Copyright (C) 2000,2001,2002,2003,2005
|
|
* Thomas Vander Stichele <thomas at apestaart dot org>
|
|
*
|
|
* This library is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Library General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2 of the License, or (at your option) any later version.
|
|
*
|
|
* This library is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Library General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Library General Public
|
|
* License along with this library; if not, write to the
|
|
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
|
* Boston, MA 02110-1301, USA.
|
|
*/
|
|
|
|
/**
|
|
* SECTION:element-videoframe-audiolevel
|
|
* @title: videoframe-audiolevel
|
|
*
|
|
* This element acts like a synchronized audio/video "level". It gathers
|
|
* all audio buffers sent between two video frames, and then sends a message
|
|
* that contains the RMS value of all samples for these buffers.
|
|
*
|
|
* ## Example launch line
|
|
* |[
|
|
* gst-launch-1.0 -m filesrc location="file.mkv" ! decodebin name=d ! "audio/x-raw" ! videoframe-audiolevel name=l ! autoaudiosink d. ! "video/x-raw" ! l. l. ! queue ! autovideosink ]|
|
|
*
|
|
*/
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
#include "config.h"
|
|
#endif
|
|
|
|
/* FIXME 2.0: suppress warnings for deprecated API such as GValueArray
|
|
* with newer GLib versions (>= 2.31.0) */
|
|
#define GLIB_DISABLE_DEPRECATION_WARNINGS
|
|
|
|
#include "gstvideoframe-audiolevel.h"
|
|
#include <math.h>
|
|
|
|
#define GST_CAT_DEFAULT gst_videoframe_audiolevel_debug
|
|
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
|
|
# define FORMATS "{ S8, S16LE, S32LE, F32LE, F64LE }"
|
|
#else
|
|
# define FORMATS "{ S8, S16BE, S32BE, F32BE, F64BE }"
|
|
#endif
|
|
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
|
|
|
|
static GstStaticPadTemplate audio_sink_template =
|
|
GST_STATIC_PAD_TEMPLATE ("asink",
|
|
GST_PAD_SINK,
|
|
GST_PAD_ALWAYS,
|
|
GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (FORMATS))
|
|
);
|
|
|
|
static GstStaticPadTemplate audio_src_template =
|
|
GST_STATIC_PAD_TEMPLATE ("asrc",
|
|
GST_PAD_SRC,
|
|
GST_PAD_ALWAYS,
|
|
GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (FORMATS))
|
|
);
|
|
|
|
static GstStaticPadTemplate video_sink_template =
|
|
GST_STATIC_PAD_TEMPLATE ("vsink",
|
|
GST_PAD_SINK,
|
|
GST_PAD_ALWAYS,
|
|
GST_STATIC_CAPS ("video/x-raw")
|
|
);
|
|
|
|
static GstStaticPadTemplate video_src_template =
|
|
GST_STATIC_PAD_TEMPLATE ("vsrc",
|
|
GST_PAD_SRC,
|
|
GST_PAD_ALWAYS,
|
|
GST_STATIC_CAPS ("video/x-raw")
|
|
);
|
|
|
|
#define parent_class gst_videoframe_audiolevel_parent_class
|
|
G_DEFINE_TYPE (GstVideoFrameAudioLevel, gst_videoframe_audiolevel,
|
|
GST_TYPE_ELEMENT);
|
|
GST_ELEMENT_REGISTER_DEFINE (videoframe_audiolevel, "videoframe-audiolevel",
|
|
GST_RANK_NONE, GST_TYPE_VIDEOFRAME_AUDIOLEVEL);
|
|
|
|
static GstFlowReturn gst_videoframe_audiolevel_asink_chain (GstPad * pad,
|
|
GstObject * parent, GstBuffer * inbuf);
|
|
static GstFlowReturn gst_videoframe_audiolevel_vsink_chain (GstPad * pad,
|
|
GstObject * parent, GstBuffer * inbuf);
|
|
static gboolean gst_videoframe_audiolevel_asink_event (GstPad * pad,
|
|
GstObject * parent, GstEvent * event);
|
|
static gboolean gst_videoframe_audiolevel_vsink_event (GstPad * pad,
|
|
GstObject * parent, GstEvent * event);
|
|
static GstIterator *gst_videoframe_audiolevel_iterate_internal_links (GstPad *
|
|
pad, GstObject * parent);
|
|
|
|
static void gst_videoframe_audiolevel_finalize (GObject * gobject);
|
|
|
|
static GstStateChangeReturn gst_videoframe_audiolevel_change_state (GstElement *
|
|
element, GstStateChange transition);
|
|
|
|
static void
|
|
gst_videoframe_audiolevel_class_init (GstVideoFrameAudioLevelClass * klass)
|
|
{
|
|
GstElementClass *gstelement_class;
|
|
GObjectClass *gobject_class = (GObjectClass *) klass;
|
|
|
|
GST_DEBUG_CATEGORY_INIT (gst_videoframe_audiolevel_debug,
|
|
"videoframe-audiolevel", 0, "Synchronized audio/video level");
|
|
|
|
gstelement_class = (GstElementClass *) klass;
|
|
|
|
gst_element_class_set_static_metadata (gstelement_class,
|
|
"Video-frame audio level", "Filter/Analyzer/Audio",
|
|
"Synchronized audio/video RMS Level messenger for audio/raw",
|
|
"Vivia Nikolaidou <vivia@toolsonair.com>");
|
|
|
|
gobject_class->finalize = gst_videoframe_audiolevel_finalize;
|
|
gstelement_class->change_state = gst_videoframe_audiolevel_change_state;
|
|
|
|
gst_element_class_add_static_pad_template (gstelement_class,
|
|
&audio_src_template);
|
|
gst_element_class_add_static_pad_template (gstelement_class,
|
|
&audio_sink_template);
|
|
|
|
gst_element_class_add_static_pad_template (gstelement_class,
|
|
&video_src_template);
|
|
gst_element_class_add_static_pad_template (gstelement_class,
|
|
&video_sink_template);
|
|
}
|
|
|
|
static void
|
|
gst_videoframe_audiolevel_init (GstVideoFrameAudioLevel * self)
|
|
{
|
|
self->asinkpad =
|
|
gst_pad_new_from_static_template (&audio_sink_template, "asink");
|
|
gst_pad_set_chain_function (self->asinkpad,
|
|
GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_asink_chain));
|
|
gst_pad_set_event_function (self->asinkpad,
|
|
GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_asink_event));
|
|
gst_pad_set_iterate_internal_links_function (self->asinkpad,
|
|
GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
|
|
gst_element_add_pad (GST_ELEMENT (self), self->asinkpad);
|
|
|
|
self->vsinkpad =
|
|
gst_pad_new_from_static_template (&video_sink_template, "vsink");
|
|
gst_pad_set_chain_function (self->vsinkpad,
|
|
GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_vsink_chain));
|
|
gst_pad_set_event_function (self->vsinkpad,
|
|
GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_vsink_event));
|
|
gst_pad_set_iterate_internal_links_function (self->vsinkpad,
|
|
GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
|
|
gst_element_add_pad (GST_ELEMENT (self), self->vsinkpad);
|
|
|
|
self->asrcpad =
|
|
gst_pad_new_from_static_template (&audio_src_template, "asrc");
|
|
gst_pad_set_iterate_internal_links_function (self->asrcpad,
|
|
GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
|
|
gst_element_add_pad (GST_ELEMENT (self), self->asrcpad);
|
|
|
|
self->vsrcpad =
|
|
gst_pad_new_from_static_template (&video_src_template, "vsrc");
|
|
gst_pad_set_iterate_internal_links_function (self->vsrcpad,
|
|
GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
|
|
gst_element_add_pad (GST_ELEMENT (self), self->vsrcpad);
|
|
|
|
GST_PAD_SET_PROXY_CAPS (self->asinkpad);
|
|
GST_PAD_SET_PROXY_ALLOCATION (self->asinkpad);
|
|
|
|
GST_PAD_SET_PROXY_CAPS (self->asrcpad);
|
|
GST_PAD_SET_PROXY_SCHEDULING (self->asrcpad);
|
|
|
|
GST_PAD_SET_PROXY_CAPS (self->vsinkpad);
|
|
GST_PAD_SET_PROXY_ALLOCATION (self->vsinkpad);
|
|
|
|
GST_PAD_SET_PROXY_CAPS (self->vsrcpad);
|
|
GST_PAD_SET_PROXY_SCHEDULING (self->vsrcpad);
|
|
|
|
self->adapter = gst_adapter_new ();
|
|
|
|
g_queue_init (&self->vtimeq);
|
|
self->first_time = GST_CLOCK_TIME_NONE;
|
|
self->total_frames = 0;
|
|
/* alignment_threshold and discont_wait should become properties if needed */
|
|
self->alignment_threshold = 40 * GST_MSECOND;
|
|
self->discont_time = GST_CLOCK_TIME_NONE;
|
|
self->next_offset = -1;
|
|
self->discont_wait = 1 * GST_SECOND;
|
|
|
|
self->video_eos_flag = FALSE;
|
|
self->audio_flush_flag = FALSE;
|
|
self->shutdown_flag = FALSE;
|
|
|
|
g_mutex_init (&self->mutex);
|
|
g_cond_init (&self->cond);
|
|
}
|
|
|
|
static GstStateChangeReturn
|
|
gst_videoframe_audiolevel_change_state (GstElement * element,
|
|
GstStateChange transition)
|
|
{
|
|
GstStateChangeReturn ret;
|
|
GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (element);
|
|
|
|
switch (transition) {
|
|
case GST_STATE_CHANGE_PAUSED_TO_READY:
|
|
g_mutex_lock (&self->mutex);
|
|
self->shutdown_flag = TRUE;
|
|
g_cond_signal (&self->cond);
|
|
g_mutex_unlock (&self->mutex);
|
|
break;
|
|
case GST_STATE_CHANGE_READY_TO_PAUSED:
|
|
g_mutex_lock (&self->mutex);
|
|
self->shutdown_flag = FALSE;
|
|
self->video_eos_flag = FALSE;
|
|
self->audio_flush_flag = FALSE;
|
|
g_mutex_unlock (&self->mutex);
|
|
default:
|
|
break;
|
|
}
|
|
|
|
ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
|
|
|
|
switch (transition) {
|
|
case GST_STATE_CHANGE_PAUSED_TO_READY:
|
|
g_mutex_lock (&self->mutex);
|
|
self->first_time = GST_CLOCK_TIME_NONE;
|
|
self->total_frames = 0;
|
|
gst_segment_init (&self->asegment, GST_FORMAT_UNDEFINED);
|
|
gst_segment_init (&self->vsegment, GST_FORMAT_UNDEFINED);
|
|
self->vsegment.position = GST_CLOCK_TIME_NONE;
|
|
gst_adapter_clear (self->adapter);
|
|
g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
|
|
g_queue_clear (&self->vtimeq);
|
|
if (self->CS) {
|
|
g_free (self->CS);
|
|
self->CS = NULL;
|
|
}
|
|
g_mutex_unlock (&self->mutex);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void
|
|
gst_videoframe_audiolevel_finalize (GObject * object)
|
|
{
|
|
GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (object);
|
|
|
|
if (self->adapter) {
|
|
g_object_unref (self->adapter);
|
|
self->adapter = NULL;
|
|
}
|
|
g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
|
|
g_queue_clear (&self->vtimeq);
|
|
self->first_time = GST_CLOCK_TIME_NONE;
|
|
self->total_frames = 0;
|
|
if (self->CS) {
|
|
g_free (self->CS);
|
|
self->CS = NULL;
|
|
}
|
|
|
|
g_mutex_clear (&self->mutex);
|
|
g_cond_clear (&self->cond);
|
|
|
|
G_OBJECT_CLASS (parent_class)->finalize (object);
|
|
}
|
|
|
|
#define DEFINE_INT_LEVEL_CALCULATOR(TYPE, RESOLUTION) \
|
|
static void inline \
|
|
gst_videoframe_audiolevel_calculate_##TYPE (gpointer data, guint num, guint channels, \
|
|
gdouble *NCS) \
|
|
{ \
|
|
TYPE * in = (TYPE *)data; \
|
|
register guint j; \
|
|
gdouble squaresum = 0.0; /* square sum of the input samples */ \
|
|
register gdouble square = 0.0; /* Square */ \
|
|
gdouble normalizer; /* divisor to get a [-1.0, 1.0] range */ \
|
|
\
|
|
/* *NCS = 0.0; Normalized Cumulative Square */ \
|
|
\
|
|
for (j = 0; j < num; j += channels) { \
|
|
square = ((gdouble) in[j]) * in[j]; \
|
|
squaresum += square; \
|
|
} \
|
|
\
|
|
normalizer = (gdouble) (G_GINT64_CONSTANT(1) << (RESOLUTION * 2)); \
|
|
*NCS = squaresum / normalizer; \
|
|
}
|
|
|
|
DEFINE_INT_LEVEL_CALCULATOR (gint32, 31);
|
|
DEFINE_INT_LEVEL_CALCULATOR (gint16, 15);
|
|
DEFINE_INT_LEVEL_CALCULATOR (gint8, 7);
|
|
|
|
#define DEFINE_FLOAT_LEVEL_CALCULATOR(TYPE) \
|
|
static void inline \
|
|
gst_videoframe_audiolevel_calculate_##TYPE (gpointer data, guint num, guint channels, \
|
|
gdouble *NCS) \
|
|
{ \
|
|
TYPE * in = (TYPE *)data; \
|
|
register guint j; \
|
|
gdouble squaresum = 0.0; /* square sum of the input samples */ \
|
|
register gdouble square = 0.0; /* Square */ \
|
|
\
|
|
/* *NCS = 0.0; Normalized Cumulative Square */ \
|
|
\
|
|
for (j = 0; j < num; j += channels) { \
|
|
square = ((gdouble) in[j]) * in[j]; \
|
|
squaresum += square; \
|
|
} \
|
|
\
|
|
*NCS = squaresum; \
|
|
}
|
|
|
|
DEFINE_FLOAT_LEVEL_CALCULATOR (gfloat);
|
|
DEFINE_FLOAT_LEVEL_CALCULATOR (gdouble);
|
|
|
|
static gboolean
|
|
gst_videoframe_audiolevel_vsink_event (GstPad * pad, GstObject * parent,
|
|
GstEvent * event)
|
|
{
|
|
GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
|
|
GST_LOG_OBJECT (pad, "Got %s event", GST_EVENT_TYPE_NAME (event));
|
|
|
|
switch (GST_EVENT_TYPE (event)) {
|
|
case GST_EVENT_SEGMENT:
|
|
g_mutex_lock (&self->mutex);
|
|
g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
|
|
g_queue_clear (&self->vtimeq);
|
|
g_mutex_unlock (&self->mutex);
|
|
gst_event_copy_segment (event, &self->vsegment);
|
|
if (self->vsegment.format != GST_FORMAT_TIME)
|
|
return FALSE;
|
|
self->vsegment.position = GST_CLOCK_TIME_NONE;
|
|
break;
|
|
case GST_EVENT_GAP:
|
|
return TRUE;
|
|
case GST_EVENT_EOS:
|
|
g_mutex_lock (&self->mutex);
|
|
self->video_eos_flag = TRUE;
|
|
g_cond_signal (&self->cond);
|
|
g_mutex_unlock (&self->mutex);
|
|
break;
|
|
case GST_EVENT_FLUSH_STOP:
|
|
g_mutex_lock (&self->mutex);
|
|
g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
|
|
g_queue_clear (&self->vtimeq);
|
|
gst_segment_init (&self->vsegment, GST_FORMAT_UNDEFINED);
|
|
g_cond_signal (&self->cond);
|
|
g_mutex_unlock (&self->mutex);
|
|
self->vsegment.position = GST_CLOCK_TIME_NONE;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
return gst_pad_event_default (pad, parent, event);
|
|
}
|
|
|
|
static gboolean
|
|
gst_videoframe_audiolevel_asink_event (GstPad * pad, GstObject * parent,
|
|
GstEvent * event)
|
|
{
|
|
GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
|
|
GST_LOG_OBJECT (pad, "Got %s event", GST_EVENT_TYPE_NAME (event));
|
|
|
|
switch (GST_EVENT_TYPE (event)) {
|
|
case GST_EVENT_SEGMENT:
|
|
self->first_time = GST_CLOCK_TIME_NONE;
|
|
self->total_frames = 0;
|
|
gst_adapter_clear (self->adapter);
|
|
gst_event_copy_segment (event, &self->asegment);
|
|
if (self->asegment.format != GST_FORMAT_TIME)
|
|
return FALSE;
|
|
break;
|
|
case GST_EVENT_FLUSH_START:
|
|
g_mutex_lock (&self->mutex);
|
|
self->audio_flush_flag = TRUE;
|
|
g_cond_signal (&self->cond);
|
|
g_mutex_unlock (&self->mutex);
|
|
break;
|
|
case GST_EVENT_FLUSH_STOP:
|
|
self->audio_flush_flag = FALSE;
|
|
self->total_frames = 0;
|
|
self->first_time = GST_CLOCK_TIME_NONE;
|
|
gst_adapter_clear (self->adapter);
|
|
gst_segment_init (&self->asegment, GST_FORMAT_UNDEFINED);
|
|
break;
|
|
case GST_EVENT_CAPS:{
|
|
GstCaps *caps;
|
|
gint channels;
|
|
gst_event_parse_caps (event, &caps);
|
|
GST_DEBUG_OBJECT (self, "Got caps %" GST_PTR_FORMAT, caps);
|
|
if (!gst_audio_info_from_caps (&self->ainfo, caps))
|
|
return FALSE;
|
|
switch (GST_AUDIO_INFO_FORMAT (&self->ainfo)) {
|
|
case GST_AUDIO_FORMAT_S8:
|
|
self->process = gst_videoframe_audiolevel_calculate_gint8;
|
|
break;
|
|
case GST_AUDIO_FORMAT_S16:
|
|
self->process = gst_videoframe_audiolevel_calculate_gint16;
|
|
break;
|
|
case GST_AUDIO_FORMAT_S32:
|
|
self->process = gst_videoframe_audiolevel_calculate_gint32;
|
|
break;
|
|
case GST_AUDIO_FORMAT_F32:
|
|
self->process = gst_videoframe_audiolevel_calculate_gfloat;
|
|
break;
|
|
case GST_AUDIO_FORMAT_F64:
|
|
self->process = gst_videoframe_audiolevel_calculate_gdouble;
|
|
break;
|
|
default:
|
|
self->process = NULL;
|
|
break;
|
|
}
|
|
gst_adapter_clear (self->adapter);
|
|
channels = GST_AUDIO_INFO_CHANNELS (&self->ainfo);
|
|
self->first_time = GST_CLOCK_TIME_NONE;
|
|
self->total_frames = 0;
|
|
if (self->CS)
|
|
g_free (self->CS);
|
|
self->CS = g_new0 (gdouble, channels);
|
|
break;
|
|
}
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return gst_pad_event_default (pad, parent, event);
|
|
}
|
|
|
|
static GstMessage *
|
|
update_rms_from_buffer (GstVideoFrameAudioLevel * self, GstBuffer * inbuf)
|
|
{
|
|
GstMapInfo map;
|
|
guint8 *in_data;
|
|
gsize in_size;
|
|
gdouble CS;
|
|
guint i;
|
|
guint num_frames, frames;
|
|
guint num_int_samples = 0; /* number of interleaved samples
|
|
* ie. total count for all channels combined */
|
|
gint channels, rate, bps;
|
|
GValue v = G_VALUE_INIT;
|
|
GValue va = G_VALUE_INIT;
|
|
GValueArray *a;
|
|
GstStructure *s;
|
|
GstMessage *msg;
|
|
GstClockTime duration, running_time;
|
|
|
|
channels = GST_AUDIO_INFO_CHANNELS (&self->ainfo);
|
|
bps = GST_AUDIO_INFO_BPS (&self->ainfo);
|
|
rate = GST_AUDIO_INFO_RATE (&self->ainfo);
|
|
|
|
gst_buffer_map (inbuf, &map, GST_MAP_READ);
|
|
in_data = map.data;
|
|
in_size = map.size;
|
|
|
|
num_int_samples = in_size / bps;
|
|
|
|
GST_LOG_OBJECT (self, "analyzing %u sample frames at ts %" GST_TIME_FORMAT,
|
|
num_int_samples, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (inbuf)));
|
|
|
|
g_return_val_if_fail (num_int_samples % channels == 0, NULL);
|
|
|
|
num_frames = num_int_samples / channels;
|
|
frames = num_frames;
|
|
duration = GST_FRAMES_TO_CLOCK_TIME (frames, rate);
|
|
if (num_frames > 0) {
|
|
for (i = 0; i < channels; ++i) {
|
|
self->process (in_data + (bps * i), num_int_samples, channels, &CS);
|
|
GST_LOG_OBJECT (self,
|
|
"[%d]: cumulative squares %lf, over %d samples/%d channels",
|
|
i, CS, num_int_samples, channels);
|
|
self->CS[i] += CS;
|
|
}
|
|
in_data += num_frames * bps;
|
|
|
|
self->total_frames += num_frames;
|
|
}
|
|
running_time =
|
|
self->first_time + gst_util_uint64_scale (self->total_frames, GST_SECOND,
|
|
rate);
|
|
|
|
a = g_value_array_new (channels);
|
|
s = gst_structure_new ("videoframe-audiolevel", "running-time", G_TYPE_UINT64,
|
|
running_time, "duration", G_TYPE_UINT64, duration, NULL);
|
|
|
|
g_value_init (&v, G_TYPE_DOUBLE);
|
|
g_value_init (&va, G_TYPE_VALUE_ARRAY);
|
|
for (i = 0; i < channels; i++) {
|
|
gdouble rms;
|
|
if (frames == 0 || self->CS[i] == 0) {
|
|
rms = 0; /* empty buffer */
|
|
} else {
|
|
rms = sqrt (self->CS[i] / frames);
|
|
}
|
|
self->CS[i] = 0.0;
|
|
g_value_set_double (&v, rms);
|
|
g_value_array_append (a, &v);
|
|
}
|
|
g_value_take_boxed (&va, a);
|
|
gst_structure_take_value (s, "rms", &va);
|
|
msg = gst_message_new_element (GST_OBJECT (self), s);
|
|
|
|
gst_buffer_unmap (inbuf, &map);
|
|
|
|
return msg;
|
|
}
|
|
|
|
static GstFlowReturn
|
|
gst_videoframe_audiolevel_vsink_chain (GstPad * pad, GstObject * parent,
|
|
GstBuffer * inbuf)
|
|
{
|
|
GstClockTime timestamp;
|
|
GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
|
|
GstClockTime duration;
|
|
GstClockTime *ptrtime = g_new (GstClockTime, 1);
|
|
|
|
timestamp = GST_BUFFER_TIMESTAMP (inbuf);
|
|
*ptrtime =
|
|
gst_segment_to_running_time (&self->vsegment, GST_FORMAT_TIME, timestamp);
|
|
g_mutex_lock (&self->mutex);
|
|
self->vsegment.position = timestamp;
|
|
duration = GST_BUFFER_DURATION (inbuf);
|
|
if (duration != GST_CLOCK_TIME_NONE)
|
|
self->vsegment.position += duration;
|
|
g_queue_push_tail (&self->vtimeq, ptrtime);
|
|
g_cond_signal (&self->cond);
|
|
GST_DEBUG_OBJECT (pad, "Pushed a frame");
|
|
g_mutex_unlock (&self->mutex);
|
|
return gst_pad_push (self->vsrcpad, inbuf);
|
|
}
|
|
|
|
static GstFlowReturn
|
|
gst_videoframe_audiolevel_asink_chain (GstPad * pad, GstObject * parent,
|
|
GstBuffer * inbuf)
|
|
{
|
|
GstClockTime timestamp, cur_time;
|
|
GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
|
|
GstBuffer *buf;
|
|
gsize inbuf_size;
|
|
guint64 start_offset, end_offset;
|
|
GstClockTime running_time;
|
|
gint rate, bpf;
|
|
gboolean discont = FALSE;
|
|
|
|
timestamp = GST_BUFFER_TIMESTAMP (inbuf);
|
|
running_time =
|
|
gst_segment_to_running_time (&self->asegment, GST_FORMAT_TIME, timestamp);
|
|
|
|
rate = GST_AUDIO_INFO_RATE (&self->ainfo);
|
|
bpf = GST_AUDIO_INFO_BPF (&self->ainfo);
|
|
start_offset = gst_util_uint64_scale (timestamp, rate, GST_SECOND);
|
|
inbuf_size = gst_buffer_get_size (inbuf);
|
|
end_offset = start_offset + inbuf_size / bpf;
|
|
|
|
g_mutex_lock (&self->mutex);
|
|
|
|
if (GST_BUFFER_IS_DISCONT (inbuf)
|
|
|| GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_RESYNC)
|
|
|| self->first_time == GST_CLOCK_TIME_NONE) {
|
|
discont = TRUE;
|
|
} else {
|
|
guint64 diff, max_sample_diff;
|
|
|
|
/* Check discont, based on audiobasesink */
|
|
if (start_offset <= self->next_offset)
|
|
diff = self->next_offset - start_offset;
|
|
else
|
|
diff = start_offset - self->next_offset;
|
|
|
|
max_sample_diff =
|
|
gst_util_uint64_scale_int (self->alignment_threshold, rate, GST_SECOND);
|
|
|
|
/* Discont! */
|
|
if (G_UNLIKELY (diff >= max_sample_diff)) {
|
|
if (self->discont_wait > 0) {
|
|
if (self->discont_time == GST_CLOCK_TIME_NONE) {
|
|
self->discont_time = timestamp;
|
|
} else if (timestamp - self->discont_time >= self->discont_wait) {
|
|
discont = TRUE;
|
|
self->discont_time = GST_CLOCK_TIME_NONE;
|
|
}
|
|
} else {
|
|
discont = TRUE;
|
|
}
|
|
} else if (G_UNLIKELY (self->discont_time != GST_CLOCK_TIME_NONE)) {
|
|
/* we have had a discont, but are now back on track! */
|
|
self->discont_time = GST_CLOCK_TIME_NONE;
|
|
}
|
|
}
|
|
|
|
if (discont) {
|
|
/* Have discont, need resync */
|
|
if (self->next_offset != -1)
|
|
GST_INFO_OBJECT (pad, "Have discont. Expected %"
|
|
G_GUINT64_FORMAT ", got %" G_GUINT64_FORMAT,
|
|
self->next_offset, start_offset);
|
|
self->total_frames = 0;
|
|
self->first_time = running_time;
|
|
self->next_offset = end_offset;
|
|
} else {
|
|
self->next_offset += inbuf_size / bpf;
|
|
}
|
|
|
|
gst_adapter_push (self->adapter, gst_buffer_ref (inbuf));
|
|
|
|
GST_DEBUG_OBJECT (self, "Queue length %i",
|
|
g_queue_get_length (&self->vtimeq));
|
|
|
|
while (TRUE) {
|
|
GstClockTime *vt0, *vt1;
|
|
GstClockTime vtemp;
|
|
GstMessage *msg;
|
|
gsize bytes, available_bytes;
|
|
|
|
vtemp = GST_CLOCK_TIME_NONE;
|
|
|
|
while (!(g_queue_get_length (&self->vtimeq) >= 2 || self->video_eos_flag
|
|
|| self->audio_flush_flag || self->shutdown_flag))
|
|
g_cond_wait (&self->cond, &self->mutex);
|
|
|
|
if (self->audio_flush_flag || self->shutdown_flag) {
|
|
g_mutex_unlock (&self->mutex);
|
|
gst_buffer_unref (inbuf);
|
|
return GST_FLOW_FLUSHING;
|
|
} else if (self->video_eos_flag) {
|
|
GST_DEBUG_OBJECT (self, "Video EOS flag alert");
|
|
/* nothing to do here if queue is empty */
|
|
if (g_queue_get_length (&self->vtimeq) == 0)
|
|
break;
|
|
|
|
if (g_queue_get_length (&self->vtimeq) < 2) {
|
|
vtemp = self->vsegment.position;
|
|
} else if (self->vsegment.position == GST_CLOCK_TIME_NONE) {
|
|
/* g_queue_get_length is surely >= 2 at this point
|
|
* so the adapter isn't empty */
|
|
buf =
|
|
gst_adapter_take_buffer (self->adapter,
|
|
gst_adapter_available (self->adapter));
|
|
if (buf != NULL) {
|
|
GstMessage *msg;
|
|
msg = update_rms_from_buffer (self, buf);
|
|
g_mutex_unlock (&self->mutex);
|
|
gst_element_post_message (GST_ELEMENT (self), msg);
|
|
gst_buffer_unref (buf);
|
|
g_mutex_lock (&self->mutex); /* we unlock again later */
|
|
}
|
|
break;
|
|
}
|
|
} else if (g_queue_get_length (&self->vtimeq) < 2) {
|
|
continue;
|
|
}
|
|
|
|
vt0 = g_queue_pop_head (&self->vtimeq);
|
|
if (vtemp == GST_CLOCK_TIME_NONE)
|
|
vt1 = g_queue_peek_head (&self->vtimeq);
|
|
else
|
|
vt1 = &vtemp;
|
|
|
|
cur_time =
|
|
self->first_time + gst_util_uint64_scale (self->total_frames,
|
|
GST_SECOND, rate);
|
|
GST_DEBUG_OBJECT (self,
|
|
"Processing: current time is %" GST_TIME_FORMAT,
|
|
GST_TIME_ARGS (cur_time));
|
|
GST_DEBUG_OBJECT (self, "Total frames is %i with a rate of %d",
|
|
self->total_frames, rate);
|
|
GST_DEBUG_OBJECT (self, "Start time is %" GST_TIME_FORMAT,
|
|
GST_TIME_ARGS (self->first_time));
|
|
GST_DEBUG_OBJECT (self, "Time on top is %" GST_TIME_FORMAT,
|
|
GST_TIME_ARGS (*vt0));
|
|
|
|
if (cur_time < *vt0) {
|
|
guint num_frames =
|
|
gst_util_uint64_scale (*vt0 - cur_time, rate, GST_SECOND);
|
|
bytes = num_frames * GST_AUDIO_INFO_BPF (&self->ainfo);
|
|
available_bytes = gst_adapter_available (self->adapter);
|
|
if (available_bytes == 0) {
|
|
g_queue_push_head (&self->vtimeq, vt0);
|
|
break;
|
|
}
|
|
if (bytes == 0) {
|
|
cur_time = *vt0;
|
|
} else {
|
|
GST_DEBUG_OBJECT (self,
|
|
"Flushed %" G_GSIZE_FORMAT " out of %" G_GSIZE_FORMAT " bytes",
|
|
bytes, available_bytes);
|
|
gst_adapter_flush (self->adapter, MIN (bytes, available_bytes));
|
|
self->total_frames += num_frames;
|
|
if (available_bytes <= bytes) {
|
|
g_queue_push_head (&self->vtimeq, vt0);
|
|
break;
|
|
}
|
|
cur_time =
|
|
self->first_time + gst_util_uint64_scale (self->total_frames,
|
|
GST_SECOND, rate);
|
|
}
|
|
}
|
|
if (*vt1 > cur_time) {
|
|
bytes =
|
|
GST_AUDIO_INFO_BPF (&self->ainfo) * gst_util_uint64_scale (*vt1 -
|
|
cur_time, rate, GST_SECOND);
|
|
} else {
|
|
bytes = 0; /* We just need to discard vt0 */
|
|
}
|
|
available_bytes = gst_adapter_available (self->adapter);
|
|
GST_DEBUG_OBJECT (self,
|
|
"Adapter contains %" G_GSIZE_FORMAT " out of %" G_GSIZE_FORMAT " bytes",
|
|
available_bytes, bytes);
|
|
|
|
if (available_bytes < bytes) {
|
|
g_queue_push_head (&self->vtimeq, vt0);
|
|
goto done;
|
|
}
|
|
|
|
if (bytes > 0) {
|
|
buf = gst_adapter_take_buffer (self->adapter, bytes);
|
|
g_assert (buf != NULL);
|
|
} else {
|
|
/* Just an empty buffer */
|
|
buf = gst_buffer_new ();
|
|
}
|
|
msg = update_rms_from_buffer (self, buf);
|
|
g_mutex_unlock (&self->mutex);
|
|
gst_element_post_message (GST_ELEMENT (self), msg);
|
|
g_mutex_lock (&self->mutex);
|
|
|
|
gst_buffer_unref (buf);
|
|
g_free (vt0);
|
|
if (available_bytes == bytes)
|
|
break;
|
|
}
|
|
done:
|
|
g_mutex_unlock (&self->mutex);
|
|
return gst_pad_push (self->asrcpad, inbuf);
|
|
}
|
|
|
|
static GstIterator *
|
|
gst_videoframe_audiolevel_iterate_internal_links (GstPad * pad,
|
|
GstObject * parent)
|
|
{
|
|
GstIterator *it = NULL;
|
|
GstPad *opad;
|
|
GValue val = { 0, };
|
|
GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
|
|
|
|
if (self->asinkpad == pad)
|
|
opad = gst_object_ref (self->asrcpad);
|
|
else if (self->asrcpad == pad)
|
|
opad = gst_object_ref (self->asinkpad);
|
|
else if (self->vsinkpad == pad)
|
|
opad = gst_object_ref (self->vsrcpad);
|
|
else if (self->vsrcpad == pad)
|
|
opad = gst_object_ref (self->vsinkpad);
|
|
else
|
|
goto out;
|
|
|
|
g_value_init (&val, GST_TYPE_PAD);
|
|
g_value_set_object (&val, opad);
|
|
it = gst_iterator_new_single (GST_TYPE_PAD, &val);
|
|
g_value_unset (&val);
|
|
|
|
gst_object_unref (opad);
|
|
|
|
out:
|
|
return it;
|
|
}
|
|
|
|
static gboolean
|
|
gst_videoframe_audiolevel_plugin_init (GstPlugin * plugin)
|
|
{
|
|
return GST_ELEMENT_REGISTER (videoframe_audiolevel, plugin);
|
|
}
|
|
|
|
GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
|
|
GST_VERSION_MINOR,
|
|
videoframe_audiolevel,
|
|
"Video frame-synchronized audio level",
|
|
gst_videoframe_audiolevel_plugin_init, VERSION, GST_LICENSE,
|
|
GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN);
|