webrtcdsp: add support for using F32/non-interleaved buffers

This is the native format that is in use by the webrtc audio processing
library internally, so this avoids internal {de,}interleaving and
format conversion (S16->F32 and back)

https://bugzilla.gnome.org/show_bug.cgi?id=793605
This commit is contained in:
George Kiagiadakis 2018-02-19 18:30:13 +02:00
parent 0591bc934a
commit d299c27892
6 changed files with 248 additions and 58 deletions

View file

@ -6,9 +6,12 @@ libgstwebrtcdsp_la_CXXFLAGS = \
$(GST_CXXFLAGS) \ $(GST_CXXFLAGS) \
$(GST_BASE_CFLAGS) \ $(GST_BASE_CFLAGS) \
$(GST_PLUGINS_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \
-I$(top_srcdir)/gst-libs \
-I$(top_builddir)/gst-libs \
$(WEBRTCDSP_CFLAGS) $(WEBRTCDSP_CFLAGS)
libgstwebrtcdsp_la_LIBADD = \ libgstwebrtcdsp_la_LIBADD = \
-lgstaudio-$(GST_API_VERSION) \ -lgstaudio-$(GST_API_VERSION) \
$(top_builddir)/gst-libs/gst/audio/libgstbadaudio-$(GST_API_VERSION).la \
$(GST_LIBS) $(GST_BASE_LIBS) \ $(GST_LIBS) $(GST_BASE_LIBS) \
$(GST_PLUGINS_BASE_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \
$(WEBRTCDSP_LIBS) $(WEBRTCDSP_LIBS)

View file

@ -95,6 +95,11 @@ GST_STATIC_PAD_TEMPLATE ("sink",
"format = (string) " GST_AUDIO_NE (S16) ", " "format = (string) " GST_AUDIO_NE (S16) ", "
"layout = (string) interleaved, " "layout = (string) interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, " "rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX];"
"audio/x-raw, "
"format = (string) " GST_AUDIO_NE (F32) ", "
"layout = (string) non-interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX]") "channels = (int) [1, MAX]")
); );
@ -106,6 +111,11 @@ GST_STATIC_PAD_TEMPLATE ("src",
"format = (string) " GST_AUDIO_NE (S16) ", " "format = (string) " GST_AUDIO_NE (S16) ", "
"layout = (string) interleaved, " "layout = (string) interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, " "rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX];"
"audio/x-raw, "
"format = (string) " GST_AUDIO_NE (F32) ", "
"layout = (string) non-interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX]") "channels = (int) [1, MAX]")
); );
@ -230,11 +240,14 @@ struct _GstWebrtcDsp
/* Protected by the object lock */ /* Protected by the object lock */
GstAudioInfo info; GstAudioInfo info;
gboolean interleaved;
guint period_size; guint period_size;
guint period_samples;
gboolean stream_has_voice; gboolean stream_has_voice;
/* Protected by the stream lock */ /* Protected by the stream lock */
GstAdapter *adapter; GstAdapter *adapter;
GstPlanarAudioAdapter *padapter;
webrtc::AudioProcessing * apm; webrtc::AudioProcessing * apm;
/* Protected by the object lock */ /* Protected by the object lock */
@ -321,20 +334,35 @@ gst_webrtc_dsp_take_buffer (GstWebrtcDsp * self)
GstBuffer *buffer; GstBuffer *buffer;
GstClockTime timestamp; GstClockTime timestamp;
guint64 distance; guint64 distance;
gboolean at_discont;
if (self->interleaved) {
timestamp = gst_adapter_prev_pts (self->adapter, &distance); timestamp = gst_adapter_prev_pts (self->adapter, &distance);
timestamp += gst_util_uint64_scale_int (distance / self->info.bpf, distance /= self->info.bpf;
GST_SECOND, self->info.rate); } else {
timestamp = gst_planar_audio_adapter_prev_pts (self->padapter, &distance);
}
timestamp += gst_util_uint64_scale_int (distance, GST_SECOND, self->info.rate);
if (self->interleaved) {
buffer = gst_adapter_take_buffer (self->adapter, self->period_size); buffer = gst_adapter_take_buffer (self->adapter, self->period_size);
at_discont = (gst_adapter_pts_at_discont (self->adapter) == timestamp);
} else {
buffer = gst_planar_audio_adapter_take_buffer (self->padapter,
self->period_samples, GST_MAP_READWRITE);
at_discont =
(gst_planar_audio_adapter_pts_at_discont (self->padapter) == timestamp);
}
GST_BUFFER_PTS (buffer) = timestamp; GST_BUFFER_PTS (buffer) = timestamp;
GST_BUFFER_DURATION (buffer) = 10 * GST_MSECOND; GST_BUFFER_DURATION (buffer) = 10 * GST_MSECOND;
if (gst_adapter_pts_at_discont (self->adapter) == timestamp && distance == 0) { if (at_discont && distance == 0) {
GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT); GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT);
} else } else {
GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DISCONT); GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DISCONT);
}
return buffer; return buffer;
} }
@ -346,6 +374,7 @@ gst_webrtc_dsp_analyze_reverse_stream (GstWebrtcDsp * self,
GstWebrtcEchoProbe *probe = NULL; GstWebrtcEchoProbe *probe = NULL;
webrtc::AudioProcessing * apm; webrtc::AudioProcessing * apm;
webrtc::AudioFrame frame; webrtc::AudioFrame frame;
GstBuffer *buf = NULL;
GstFlowReturn ret = GST_FLOW_OK; GstFlowReturn ret = GST_FLOW_OK;
gint err, delay; gint err, delay;
@ -364,7 +393,7 @@ gst_webrtc_dsp_analyze_reverse_stream (GstWebrtcDsp * self,
rec_time = GST_CLOCK_TIME_NONE; rec_time = GST_CLOCK_TIME_NONE;
again: again:
delay = gst_webrtc_echo_probe_read (probe, rec_time, (gpointer) &frame); delay = gst_webrtc_echo_probe_read (probe, rec_time, (gpointer) &frame, &buf);
apm->set_stream_delay_ms (delay); apm->set_stream_delay_ms (delay);
if (delay < 0) if (delay < 0)
@ -379,15 +408,31 @@ again:
goto done; goto done;
} }
if (buf) {
webrtc::StreamConfig config (frame.sample_rate_hz_, frame.num_channels_,
false);
GstAudioBuffer abuf;
float * const * data;
gst_audio_buffer_map (&abuf, &self->info, buf, GST_MAP_READWRITE);
data = (float * const *) abuf.planes;
if ((err = apm->ProcessReverseStream (data, config, config, data)) < 0)
GST_WARNING_OBJECT (self, "Reverse stream analyses failed: %s.",
webrtc_error_to_string (err));
gst_audio_buffer_unmap (&abuf);
gst_buffer_replace (&buf, NULL);
} else {
if ((err = apm->AnalyzeReverseStream (&frame)) < 0) if ((err = apm->AnalyzeReverseStream (&frame)) < 0)
GST_WARNING_OBJECT (self, "Reverse stream analyses failed: %s.", GST_WARNING_OBJECT (self, "Reverse stream analyses failed: %s.",
webrtc_error_to_string (err)); webrtc_error_to_string (err));
}
if (self->delay_agnostic) if (self->delay_agnostic)
goto again; goto again;
done: done:
gst_object_unref (probe); gst_object_unref (probe);
gst_buffer_replace (&buf, NULL);
return ret; return ret;
} }
@ -418,23 +463,34 @@ static GstFlowReturn
gst_webrtc_dsp_process_stream (GstWebrtcDsp * self, gst_webrtc_dsp_process_stream (GstWebrtcDsp * self,
GstBuffer * buffer) GstBuffer * buffer)
{ {
GstMapInfo info; GstAudioBuffer abuf;
webrtc::AudioProcessing * apm = self->apm; webrtc::AudioProcessing * apm = self->apm;
webrtc::AudioFrame frame;
gint err; gint err;
frame.num_channels_ = self->info.channels; if (!gst_audio_buffer_map (&abuf, &self->info, buffer,
frame.sample_rate_hz_ = self->info.rate; (GstMapFlags) GST_MAP_READWRITE)) {
frame.samples_per_channel_ = self->period_size / self->info.bpf;
if (!gst_buffer_map (buffer, &info, (GstMapFlags) GST_MAP_READWRITE)) {
gst_buffer_unref (buffer); gst_buffer_unref (buffer);
return GST_FLOW_ERROR; return GST_FLOW_ERROR;
} }
memcpy (frame.data_, info.data, self->period_size); if (self->interleaved) {
webrtc::AudioFrame frame;
frame.num_channels_ = self->info.channels;
frame.sample_rate_hz_ = self->info.rate;
frame.samples_per_channel_ = self->period_samples;
if ((err = apm->ProcessStream (&frame)) < 0) { memcpy (frame.data_, abuf.planes[0], self->period_size);
err = apm->ProcessStream (&frame);
if (err >= 0)
memcpy (abuf.planes[0], frame.data_, self->period_size);
} else {
float * const * data = (float * const *) abuf.planes;
webrtc::StreamConfig config (self->info.rate, self->info.channels, false);
err = apm->ProcessStream (data, config, config, data);
}
if (err < 0) {
GST_WARNING_OBJECT (self, "Failed to filter the audio: %s.", GST_WARNING_OBJECT (self, "Failed to filter the audio: %s.",
webrtc_error_to_string (err)); webrtc_error_to_string (err));
} else { } else {
@ -446,10 +502,9 @@ gst_webrtc_dsp_process_stream (GstWebrtcDsp * self,
self->stream_has_voice = stream_has_voice; self->stream_has_voice = stream_has_voice;
} }
memcpy (info.data, frame.data_, self->period_size);
} }
gst_buffer_unmap (buffer, &info); gst_audio_buffer_unmap (&abuf);
return GST_FLOW_OK; return GST_FLOW_OK;
} }
@ -467,10 +522,16 @@ gst_webrtc_dsp_submit_input_buffer (GstBaseTransform * btrans,
if (is_discont) { if (is_discont) {
GST_DEBUG_OBJECT (self, GST_DEBUG_OBJECT (self,
"Received discont, clearing adapter."); "Received discont, clearing adapter.");
if (self->interleaved)
gst_adapter_clear (self->adapter); gst_adapter_clear (self->adapter);
else
gst_planar_audio_adapter_clear (self->padapter);
} }
if (self->interleaved)
gst_adapter_push (self->adapter, buffer); gst_adapter_push (self->adapter, buffer);
else
gst_planar_audio_adapter_push (self->padapter, buffer);
return GST_FLOW_OK; return GST_FLOW_OK;
} }
@ -480,8 +541,15 @@ gst_webrtc_dsp_generate_output (GstBaseTransform * btrans, GstBuffer ** outbuf)
{ {
GstWebrtcDsp *self = GST_WEBRTC_DSP (btrans); GstWebrtcDsp *self = GST_WEBRTC_DSP (btrans);
GstFlowReturn ret; GstFlowReturn ret;
gboolean not_enough;
if (gst_adapter_available (self->adapter) < self->period_size) { if (self->interleaved)
not_enough = gst_adapter_available (self->adapter) < self->period_size;
else
not_enough = gst_planar_audio_adapter_available (self->padapter) <
self->period_samples;
if (not_enough) {
*outbuf = NULL; *outbuf = NULL;
return GST_FLOW_OK; return GST_FLOW_OK;
} }
@ -545,13 +613,21 @@ gst_webrtc_dsp_setup (GstAudioFilter * filter, const GstAudioInfo * info)
GST_OBJECT_LOCK (self); GST_OBJECT_LOCK (self);
gst_adapter_clear (self->adapter); gst_adapter_clear (self->adapter);
gst_planar_audio_adapter_clear (self->padapter);
self->info = *info; self->info = *info;
self->interleaved = (info->layout == GST_AUDIO_LAYOUT_INTERLEAVED);
apm = self->apm; apm = self->apm;
/* WebRTC library works with 10ms buffers, compute once this size */ if (!self->interleaved)
self->period_size = info->bpf * info->rate / 100; gst_planar_audio_adapter_configure (self->padapter, info);
if ((webrtc::AudioFrame::kMaxDataSizeSamples * 2) < self->period_size) /* WebRTC library works with 10ms buffers, compute once this size */
self->period_samples = info->rate / 100;
self->period_size = self->period_samples * info->bpf;
if (self->interleaved &&
(webrtc::AudioFrame::kMaxDataSizeSamples * 2) < self->period_size)
goto period_too_big; goto period_too_big;
if (self->probe) { if (self->probe) {
@ -676,6 +752,7 @@ gst_webrtc_dsp_stop (GstBaseTransform * btrans)
GST_OBJECT_LOCK (self); GST_OBJECT_LOCK (self);
gst_adapter_clear (self->adapter); gst_adapter_clear (self->adapter);
gst_planar_audio_adapter_clear (self->padapter);
if (self->probe) { if (self->probe) {
gst_webrtc_release_echo_probe (self->probe); gst_webrtc_release_echo_probe (self->probe);
@ -840,6 +917,7 @@ gst_webrtc_dsp_finalize (GObject * object)
GstWebrtcDsp *self = GST_WEBRTC_DSP (object); GstWebrtcDsp *self = GST_WEBRTC_DSP (object);
gst_object_unref (self->adapter); gst_object_unref (self->adapter);
gst_object_unref (self->padapter);
g_free (self->probe_name); g_free (self->probe_name);
G_OBJECT_CLASS (gst_webrtc_dsp_parent_class)->finalize (object); G_OBJECT_CLASS (gst_webrtc_dsp_parent_class)->finalize (object);
@ -849,6 +927,7 @@ static void
gst_webrtc_dsp_init (GstWebrtcDsp * self) gst_webrtc_dsp_init (GstWebrtcDsp * self)
{ {
self->adapter = gst_adapter_new (); self->adapter = gst_adapter_new ();
self->padapter = gst_planar_audio_adapter_new ();
gst_audio_info_init (&self->info); gst_audio_info_init (&self->info);
} }

View file

@ -28,6 +28,9 @@
#include <gst/base/gstbasetransform.h> #include <gst/base/gstbasetransform.h>
#include <gst/audio/audio.h> #include <gst/audio/audio.h>
#define GST_USE_UNSTABLE_API
#include <gst/audio/gstplanaraudioadapter.h>
G_BEGIN_DECLS G_BEGIN_DECLS
#define GST_TYPE_WEBRTC_DSP (gst_webrtc_dsp_get_type()) #define GST_TYPE_WEBRTC_DSP (gst_webrtc_dsp_get_type())

View file

@ -49,6 +49,11 @@ GST_STATIC_PAD_TEMPLATE ("sink",
"format = (string) " GST_AUDIO_NE (S16) ", " "format = (string) " GST_AUDIO_NE (S16) ", "
"layout = (string) interleaved, " "layout = (string) interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, " "rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX];"
"audio/x-raw, "
"format = (string) " GST_AUDIO_NE (F32) ", "
"layout = (string) non-interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX]") "channels = (int) [1, MAX]")
); );
@ -60,6 +65,11 @@ GST_STATIC_PAD_TEMPLATE ("src",
"format = (string) " GST_AUDIO_NE (S16) ", " "format = (string) " GST_AUDIO_NE (S16) ", "
"layout = (string) interleaved, " "layout = (string) interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, " "rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX];"
"audio/x-raw, "
"format = (string) " GST_AUDIO_NE (F32) ", "
"layout = (string) non-interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX]") "channels = (int) [1, MAX]")
); );
@ -80,11 +90,17 @@ gst_webrtc_echo_probe_setup (GstAudioFilter * filter, const GstAudioInfo * info)
GST_WEBRTC_ECHO_PROBE_LOCK (self); GST_WEBRTC_ECHO_PROBE_LOCK (self);
self->info = *info; self->info = *info;
self->interleaved = (info->layout == GST_AUDIO_LAYOUT_INTERLEAVED);
if (!self->interleaved)
gst_planar_audio_adapter_configure (self->padapter, info);
/* WebRTC library works with 10ms buffers, compute once this size */ /* WebRTC library works with 10ms buffers, compute once this size */
self->period_size = info->bpf * info->rate / 100; self->period_samples = info->rate / 100;
self->period_size = self->period_samples * info->bpf;
if ((webrtc::AudioFrame::kMaxDataSizeSamples * 2) < self->period_size) if (self->interleaved &&
(webrtc::AudioFrame::kMaxDataSizeSamples * 2) < self->period_size)
goto period_too_big; goto period_too_big;
GST_WEBRTC_ECHO_PROBE_UNLOCK (self); GST_WEBRTC_ECHO_PROBE_UNLOCK (self);
@ -107,6 +123,7 @@ gst_webrtc_echo_probe_stop (GstBaseTransform * btrans)
GST_WEBRTC_ECHO_PROBE_LOCK (self); GST_WEBRTC_ECHO_PROBE_LOCK (self);
gst_adapter_clear (self->adapter); gst_adapter_clear (self->adapter);
gst_planar_audio_adapter_clear (self->padapter);
GST_WEBRTC_ECHO_PROBE_UNLOCK (self); GST_WEBRTC_ECHO_PROBE_UNLOCK (self);
return TRUE; return TRUE;
@ -163,11 +180,24 @@ gst_webrtc_echo_probe_transform_ip (GstBaseTransform * btrans,
/* Moves the buffer timestamp to be in Running time */ /* Moves the buffer timestamp to be in Running time */
GST_BUFFER_PTS (newbuf) = gst_segment_to_running_time (&btrans->segment, GST_BUFFER_PTS (newbuf) = gst_segment_to_running_time (&btrans->segment,
GST_FORMAT_TIME, GST_BUFFER_PTS (buffer)); GST_FORMAT_TIME, GST_BUFFER_PTS (buffer));
if (self->interleaved) {
gst_adapter_push (self->adapter, newbuf); gst_adapter_push (self->adapter, newbuf);
if (gst_adapter_available (self->adapter) > MAX_ADAPTER_SIZE) if (gst_adapter_available (self->adapter) > MAX_ADAPTER_SIZE)
gst_adapter_flush (self->adapter, gst_adapter_flush (self->adapter,
gst_adapter_available (self->adapter) - MAX_ADAPTER_SIZE); gst_adapter_available (self->adapter) - MAX_ADAPTER_SIZE);
} else {
gsize available;
gst_planar_audio_adapter_push (self->padapter, newbuf);
available =
gst_planar_audio_adapter_available (self->padapter) * self->info.bpf;
if (available > MAX_ADAPTER_SIZE)
gst_planar_audio_adapter_flush (self->padapter,
(available - MAX_ADAPTER_SIZE) / self->info.bpf);
}
GST_WEBRTC_ECHO_PROBE_UNLOCK (self); GST_WEBRTC_ECHO_PROBE_UNLOCK (self);
return GST_FLOW_OK; return GST_FLOW_OK;
@ -183,7 +213,9 @@ gst_webrtc_echo_probe_finalize (GObject * object)
G_UNLOCK (gst_aec_probes); G_UNLOCK (gst_aec_probes);
gst_object_unref (self->adapter); gst_object_unref (self->adapter);
gst_object_unref (self->padapter);
self->adapter = NULL; self->adapter = NULL;
self->padapter = NULL;
G_OBJECT_CLASS (gst_webrtc_echo_probe_parent_class)->finalize (object); G_OBJECT_CLASS (gst_webrtc_echo_probe_parent_class)->finalize (object);
} }
@ -192,6 +224,7 @@ static void
gst_webrtc_echo_probe_init (GstWebrtcEchoProbe * self) gst_webrtc_echo_probe_init (GstWebrtcEchoProbe * self)
{ {
self->adapter = gst_adapter_new (); self->adapter = gst_adapter_new ();
self->padapter = gst_planar_audio_adapter_new ();
gst_audio_info_init (&self->info); gst_audio_info_init (&self->info);
g_mutex_init (&self->lock); g_mutex_init (&self->lock);
@ -268,7 +301,7 @@ gst_webrtc_release_echo_probe (GstWebrtcEchoProbe * probe)
gint gint
gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self, GstClockTime rec_time, gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self, GstClockTime rec_time,
gpointer _frame) gpointer _frame, GstBuffer ** buf)
{ {
webrtc::AudioFrame * frame = (webrtc::AudioFrame *) _frame; webrtc::AudioFrame * frame = (webrtc::AudioFrame *) _frame;
GstClockTimeDiff diff; GstClockTimeDiff diff;
@ -281,31 +314,39 @@ gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self, GstClockTime rec_time,
!GST_AUDIO_INFO_IS_VALID (&self->info)) !GST_AUDIO_INFO_IS_VALID (&self->info))
goto done; goto done;
if (self->interleaved)
avail = gst_adapter_available (self->adapter) / self->info.bpf;
else
avail = gst_planar_audio_adapter_available (self->padapter);
/* In delay agnostic mode, just return 10ms of data */ /* In delay agnostic mode, just return 10ms of data */
if (!GST_CLOCK_TIME_IS_VALID (rec_time)) { if (!GST_CLOCK_TIME_IS_VALID (rec_time)) {
avail = gst_adapter_available (self->adapter); if (avail < self->period_samples)
if (avail < self->period_size)
goto done; goto done;
size = self->period_size; size = self->period_samples;
skip = 0; skip = 0;
offset = 0; offset = 0;
goto copy; goto copy;
} }
if (gst_adapter_available (self->adapter) == 0) { if (avail == 0) {
diff = G_MAXINT64; diff = G_MAXINT64;
} else { } else {
GstClockTime play_time; GstClockTime play_time;
guint64 distance; guint64 distance;
if (self->interleaved) {
play_time = gst_adapter_prev_pts (self->adapter, &distance); play_time = gst_adapter_prev_pts (self->adapter, &distance);
distance /= self->info.bpf;
} else {
play_time = gst_planar_audio_adapter_prev_pts (self->padapter, &distance);
}
if (GST_CLOCK_TIME_IS_VALID (play_time)) { if (GST_CLOCK_TIME_IS_VALID (play_time)) {
play_time += gst_util_uint64_scale_int (distance / self->info.bpf, play_time += gst_util_uint64_scale_int (distance, GST_SECOND,
GST_SECOND, self->info.rate); self->info.rate);
play_time += self->latency; play_time += self->latency;
diff = GST_CLOCK_DIFF (rec_time, play_time) / GST_MSECOND; diff = GST_CLOCK_DIFF (rec_time, play_time) / GST_MSECOND;
@ -315,33 +356,91 @@ gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self, GstClockTime rec_time,
} }
} }
avail = gst_adapter_available (self->adapter);
if (diff > self->delay) { if (diff > self->delay) {
skip = (diff - self->delay) * self->info.rate / 1000 * self->info.bpf; skip = (diff - self->delay) * self->info.rate / 1000;
skip = MIN (self->period_size, skip); skip = MIN (self->period_samples, skip);
offset = 0; offset = 0;
} else { } else {
skip = 0; skip = 0;
offset = (self->delay - diff) * self->info.rate / 1000 * self->info.bpf; offset = (self->delay - diff) * self->info.rate / 1000;
offset = MIN (avail, offset); offset = MIN (avail, offset);
} }
size = MIN (avail - offset, self->period_size - skip); size = MIN (avail - offset, self->period_samples - skip);
copy:
if (self->interleaved) {
skip *= self->info.bpf;
offset *= self->info.bpf;
size *= self->info.bpf;
if (size < self->period_size) if (size < self->period_size)
memset (frame->data_, 0, self->period_size); memset (frame->data_, 0, self->period_size);
copy:
if (size) { if (size) {
gst_adapter_copy (self->adapter, (guint8 *) frame->data_ + skip, gst_adapter_copy (self->adapter, (guint8 *) frame->data_ + skip,
offset, size); offset, size);
gst_adapter_flush (self->adapter, offset + size); gst_adapter_flush (self->adapter, offset + size);
} }
} else {
GstBuffer *ret, *taken, *tmp;
if (size) {
gst_planar_audio_adapter_flush (self->padapter, offset);
/* we need to fill silence at the beginning and/or the end of each
* channel plane in order to have exactly period_samples in the buffer */
if (size < self->period_samples) {
GstAudioMeta *meta;
gint bps = self->info.finfo->width / 8;
gsize padding = self->period_samples - (skip + size);
gint c;
taken = gst_planar_audio_adapter_take_buffer (self->padapter, size,
GST_MAP_READ);
meta = gst_buffer_get_audio_meta (taken);
ret = gst_buffer_new ();
for (c = 0; c < meta->info.channels; c++) {
/* need some silence at the beginning */
if (skip) {
tmp = gst_buffer_new_allocate (NULL, skip * bps, NULL);
gst_buffer_memset (tmp, 0, 0, skip * bps);
ret = gst_buffer_append (ret, tmp);
}
tmp = gst_buffer_copy_region (taken, GST_BUFFER_COPY_MEMORY,
meta->offsets[c], size * bps);
ret = gst_buffer_append (ret, tmp);
/* need some silence at the end */
if (padding) {
tmp = gst_buffer_new_allocate (NULL, padding * bps, NULL);
gst_buffer_memset (tmp, 0, 0, padding * bps);
ret = gst_buffer_append (ret, tmp);
}
}
gst_buffer_unref (taken);
gst_buffer_add_audio_meta (ret, &self->info, self->period_samples,
NULL);
} else {
ret = gst_planar_audio_adapter_take_buffer (self->padapter, size,
GST_MAP_READWRITE);
}
} else {
ret = gst_buffer_new_allocate (NULL, self->period_size, NULL);
gst_buffer_memset (ret, 0, 0, self->period_size);
gst_buffer_add_audio_meta (ret, &self->info, self->period_samples,
NULL);
}
*buf = ret;
}
frame->num_channels_ = self->info.channels; frame->num_channels_ = self->info.channels;
frame->sample_rate_hz_ = self->info.rate; frame->sample_rate_hz_ = self->info.rate;
frame->samples_per_channel_ = self->period_size / self->info.bpf; frame->samples_per_channel_ = self->period_samples;
delay = self->delay; delay = self->delay;

View file

@ -28,6 +28,9 @@
#include <gst/base/gstbasetransform.h> #include <gst/base/gstbasetransform.h>
#include <gst/audio/audio.h> #include <gst/audio/audio.h>
#define GST_USE_UNSTABLE_API
#include <gst/audio/gstplanaraudioadapter.h>
G_BEGIN_DECLS G_BEGIN_DECLS
#define GST_TYPE_WEBRTC_ECHO_PROBE (gst_webrtc_echo_probe_get_type()) #define GST_TYPE_WEBRTC_ECHO_PROBE (gst_webrtc_echo_probe_get_type())
@ -62,11 +65,14 @@ struct _GstWebrtcEchoProbe
/* Protected by the lock */ /* Protected by the lock */
GstAudioInfo info; GstAudioInfo info;
guint period_size; guint period_size;
guint period_samples;
GstClockTime latency; GstClockTime latency;
gint delay; gint delay;
gboolean interleaved;
GstSegment segment; GstSegment segment;
GstAdapter *adapter; GstAdapter *adapter;
GstPlanarAudioAdapter *padapter;
/* Private */ /* Private */
gboolean acquired; gboolean acquired;
@ -82,7 +88,7 @@ GType gst_webrtc_echo_probe_get_type (void);
GstWebrtcEchoProbe *gst_webrtc_acquire_echo_probe (const gchar * name); GstWebrtcEchoProbe *gst_webrtc_acquire_echo_probe (const gchar * name);
void gst_webrtc_release_echo_probe (GstWebrtcEchoProbe * probe); void gst_webrtc_release_echo_probe (GstWebrtcEchoProbe * probe);
gint gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self, gint gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self,
GstClockTime rec_time, gpointer frame); GstClockTime rec_time, gpointer frame, GstBuffer ** buf);
G_END_DECLS G_END_DECLS
#endif /* __GST_WEBRTC_ECHO_PROBE_H__ */ #endif /* __GST_WEBRTC_ECHO_PROBE_H__ */

View file

@ -12,7 +12,7 @@ if webrtc_dep.found()
cpp_args : gst_plugins_bad_args, cpp_args : gst_plugins_bad_args,
link_args : noseh_link_args, link_args : noseh_link_args,
include_directories : [configinc], include_directories : [configinc],
dependencies : [gstbase_dep, gstaudio_dep, webrtc_dep], dependencies : [gstbase_dep, gstaudio_dep, gstbadaudio_dep, webrtc_dep],
install : true, install : true,
install_dir : plugins_install_dir, install_dir : plugins_install_dir,
) )