mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-27 12:11:13 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/ext/webrtcdsp/gstwebrtcdsp.cpp
Philippe Normand 8b1051cdea webrtcdsp: Propagate VAD to audio level meta 
Whenever the voice activity changed on the stream, update or create an
AudioLevelMeta and associate it to the corresponding buffer.

Fixes #1073

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2170>
2021-04-19 15:51:32 +00:00

1137 lines
38 KiB
C++
Raw Blame History

/*
* WebRTC Audio Processing Elements
*
* Copyright 2016 Collabora Ltd
* @author: Nicolas Dufresne <nicolas.dufresne@collabora.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
/**
* SECTION:element-webrtcdsp
* @short_description: Audio Filter using WebRTC Audio Processing library
*
* A voice enhancement filter based on WebRTC Audio Processing library. This
* library provides a whide variety of enhancement algorithms. This element
* tries to enable as much as possible. The currently enabled enhancements are
* High Pass Filter, Echo Canceller, Noise Suppression, Automatic Gain Control,
* and some extended filters.
*
* While webrtcdsp element can be used alone, there is an exception for the
* echo canceller. The audio canceller need to be aware of the far end streams
* that are played to loud speakers. For this, you must place a webrtcechoprobe
* element at that far end. Note that the sample rate must match between
* webrtcdsp and the webrtechoprobe. Though, the number of channels can differ.
* The probe is found by the DSP element using it's object name. By default,
* webrtcdsp looks for webrtcechoprobe0, which means it just work if you have
* a single probe and DSP.
*
* The probe can only be used within the same top level GstPipeline.
* Additionally, to simplify the code, the probe element must be created
* before the DSP sink pad is activated. It does not need to be in any
* particular state and does not even need to be added to the pipeline yet.
*
* # Example launch line
*
* As a convenience, the echo canceller can be tested using an echo loop. In
* this configuration, one would expect a single echo to be heard.
*
* |[
* gst-launch-1.0 pulsesrc ! webrtcdsp ! webrtcechoprobe ! pulsesink
* ]|
*
* In real environment, you'll place the probe before the playback, but only
* process the far end streams. The DSP should be placed as close as possible
* to the audio capture. The following pipeline is astracted and does not
* represent a real pipeline.
*
* |[
* gst-launch-1.0 far-end-src ! audio/x-raw,rate=48000 ! webrtcechoprobe ! pulsesink \
* pulsesrc ! audio/x-raw,rate=48000 ! webrtcdsp ! far-end-sink
* ]|
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstwebrtcdsp.h"
#include "gstwebrtcechoprobe.h"
#include <webrtc/modules/audio_processing/include/audio_processing.h>
#include <webrtc/modules/interface/module_common_types.h>
#include <webrtc/system_wrappers/include/trace.h>
GST_DEBUG_CATEGORY (webrtc_dsp_debug);
#define GST_CAT_DEFAULT (webrtc_dsp_debug)
#define DEFAULT_TARGET_LEVEL_DBFS 3
#define DEFAULT_COMPRESSION_GAIN_DB 9
#define DEFAULT_STARTUP_MIN_VOLUME 12
#define DEFAULT_LIMITER TRUE
#define DEFAULT_GAIN_CONTROL_MODE webrtc::GainControl::kAdaptiveDigital
#define DEFAULT_VOICE_DETECTION FALSE
#define DEFAULT_VOICE_DETECTION_FRAME_SIZE_MS 10
#define DEFAULT_VOICE_DETECTION_LIKELIHOOD webrtc::VoiceDetection::kLowLikelihood
static GstStaticPadTemplate gst_webrtc_dsp_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("audio/x-raw, "
"format = (string) " GST_AUDIO_NE (S16) ", "
"layout = (string) interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX];"
"audio/x-raw, "
"format = (string) " GST_AUDIO_NE (F32) ", "
"layout = (string) non-interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX]")
);
static GstStaticPadTemplate gst_webrtc_dsp_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("audio/x-raw, "
"format = (string) " GST_AUDIO_NE (S16) ", "
"layout = (string) interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX];"
"audio/x-raw, "
"format = (string) " GST_AUDIO_NE (F32) ", "
"layout = (string) non-interleaved, "
"rate = (int) { 48000, 32000, 16000, 8000 }, "
"channels = (int) [1, MAX]")
);
typedef webrtc::EchoCancellation::SuppressionLevel GstWebrtcEchoSuppressionLevel;
#define GST_TYPE_WEBRTC_ECHO_SUPPRESSION_LEVEL \
(gst_webrtc_echo_suppression_level_get_type ())
static GType
gst_webrtc_echo_suppression_level_get_type (void)
{
static GType suppression_level_type = 0;
static const GEnumValue level_types[] = {
{webrtc::EchoCancellation::kLowSuppression, "Low Suppression", "low"},
{webrtc::EchoCancellation::kModerateSuppression,
"Moderate Suppression", "moderate"},
{webrtc::EchoCancellation::kHighSuppression, "high Suppression", "high"},
{0, NULL, NULL}
};
if (!suppression_level_type) {
suppression_level_type =
g_enum_register_static ("GstWebrtcEchoSuppressionLevel", level_types);
}
return suppression_level_type;
}
typedef webrtc::NoiseSuppression::Level GstWebrtcNoiseSuppressionLevel;
#define GST_TYPE_WEBRTC_NOISE_SUPPRESSION_LEVEL \
(gst_webrtc_noise_suppression_level_get_type ())
static GType
gst_webrtc_noise_suppression_level_get_type (void)
{
static GType suppression_level_type = 0;
static const GEnumValue level_types[] = {
{webrtc::NoiseSuppression::kLow, "Low Suppression", "low"},
{webrtc::NoiseSuppression::kModerate, "Moderate Suppression", "moderate"},
{webrtc::NoiseSuppression::kHigh, "High Suppression", "high"},
{webrtc::NoiseSuppression::kVeryHigh, "Very High Suppression",
"very-high"},
{0, NULL, NULL}
};
if (!suppression_level_type) {
suppression_level_type =
g_enum_register_static ("GstWebrtcNoiseSuppressionLevel", level_types);
}
return suppression_level_type;
}
typedef webrtc::GainControl::Mode GstWebrtcGainControlMode;
#define GST_TYPE_WEBRTC_GAIN_CONTROL_MODE \
(gst_webrtc_gain_control_mode_get_type ())
static GType
gst_webrtc_gain_control_mode_get_type (void)
{
static GType gain_control_mode_type = 0;
static const GEnumValue mode_types[] = {
{webrtc::GainControl::kAdaptiveDigital, "Adaptive Digital", "adaptive-digital"},
{webrtc::GainControl::kFixedDigital, "Fixed Digital", "fixed-digital"},
{0, NULL, NULL}
};
if (!gain_control_mode_type) {
gain_control_mode_type =
g_enum_register_static ("GstWebrtcGainControlMode", mode_types);
}
return gain_control_mode_type;
}
typedef webrtc::VoiceDetection::Likelihood GstWebrtcVoiceDetectionLikelihood;
#define GST_TYPE_WEBRTC_VOICE_DETECTION_LIKELIHOOD \
(gst_webrtc_voice_detection_likelihood_get_type ())
static GType
gst_webrtc_voice_detection_likelihood_get_type (void)
{
static GType likelihood_type = 0;
static const GEnumValue likelihood_types[] = {
{webrtc::VoiceDetection::kVeryLowLikelihood, "Very Low Likelihood", "very-low"},
{webrtc::VoiceDetection::kLowLikelihood, "Low Likelihood", "low"},
{webrtc::VoiceDetection::kModerateLikelihood, "Moderate Likelihood", "moderate"},
{webrtc::VoiceDetection::kHighLikelihood, "High Likelihood", "high"},
{0, NULL, NULL}
};
if (!likelihood_type) {
likelihood_type =
g_enum_register_static ("GstWebrtcVoiceDetectionLikelihood", likelihood_types);
}
return likelihood_type;
}
enum
{
PROP_0,
PROP_PROBE,
PROP_HIGH_PASS_FILTER,
PROP_ECHO_CANCEL,
PROP_ECHO_SUPPRESSION_LEVEL,
PROP_NOISE_SUPPRESSION,
PROP_NOISE_SUPPRESSION_LEVEL,
PROP_GAIN_CONTROL,
PROP_EXPERIMENTAL_AGC,
PROP_EXTENDED_FILTER,
PROP_DELAY_AGNOSTIC,
PROP_TARGET_LEVEL_DBFS,
PROP_COMPRESSION_GAIN_DB,
PROP_STARTUP_MIN_VOLUME,
PROP_LIMITER,
PROP_GAIN_CONTROL_MODE,
PROP_VOICE_DETECTION,
PROP_VOICE_DETECTION_FRAME_SIZE_MS,
PROP_VOICE_DETECTION_LIKELIHOOD,
};
/**
* GstWebrtcDSP:
*
* The adder object structure.
*/
struct _GstWebrtcDsp
{
GstAudioFilter element;
/* Protected by the object lock */
GstAudioInfo info;
gboolean interleaved;
guint period_size;
guint period_samples;
gboolean stream_has_voice;
/* Protected by the stream lock */
GstAdapter *adapter;
GstPlanarAudioAdapter *padapter;
webrtc::AudioProcessing * apm;
/* Protected by the object lock */
gchar *probe_name;
GstWebrtcEchoProbe *probe;
/* Properties */
gboolean high_pass_filter;
gboolean echo_cancel;
webrtc::EchoCancellation::SuppressionLevel echo_suppression_level;
gboolean noise_suppression;
webrtc::NoiseSuppression::Level noise_suppression_level;
gboolean gain_control;
gboolean experimental_agc;
gboolean extended_filter;
gboolean delay_agnostic;
gint target_level_dbfs;
gint compression_gain_db;
gint startup_min_volume;
gboolean limiter;
webrtc::GainControl::Mode gain_control_mode;
gboolean voice_detection;
gint voice_detection_frame_size_ms;
webrtc::VoiceDetection::Likelihood voice_detection_likelihood;
};
G_DEFINE_TYPE_WITH_CODE (GstWebrtcDsp, gst_webrtc_dsp, GST_TYPE_AUDIO_FILTER,
GST_DEBUG_CATEGORY_INIT (webrtc_dsp_debug, "webrtcdsp", 0,
"libwebrtcdsp wrapping elements"););
GST_ELEMENT_REGISTER_DEFINE (webrtcdsp, "webrtcdsp", GST_RANK_NONE,
GST_TYPE_WEBRTC_DSP);
static const gchar *
webrtc_error_to_string (gint err)
{
const gchar *str = "unknown error";
switch (err) {
case webrtc::AudioProcessing::kNoError:
str = "success";
break;
case webrtc::AudioProcessing::kUnspecifiedError:
str = "unspecified error";
break;
case webrtc::AudioProcessing::kCreationFailedError:
str = "creating failed";
break;
case webrtc::AudioProcessing::kUnsupportedComponentError:
str = "unsupported component";
break;
case webrtc::AudioProcessing::kUnsupportedFunctionError:
str = "unsupported function";
break;
case webrtc::AudioProcessing::kNullPointerError:
str = "null pointer";
break;
case webrtc::AudioProcessing::kBadParameterError:
str = "bad parameter";
break;
case webrtc::AudioProcessing::kBadSampleRateError:
str = "bad sample rate";
break;
case webrtc::AudioProcessing::kBadDataLengthError:
str = "bad data length";
break;
case webrtc::AudioProcessing::kBadNumberChannelsError:
str = "bad number of channels";
break;
case webrtc::AudioProcessing::kFileError:
str = "file IO error";
break;
case webrtc::AudioProcessing::kStreamParameterNotSetError:
str = "stream parameter not set";
break;
case webrtc::AudioProcessing::kNotEnabledError:
str = "not enabled";
break;
default:
break;
}
return str;
}
static GstBuffer *
gst_webrtc_dsp_take_buffer (GstWebrtcDsp * self)
{
GstBuffer *buffer;
GstClockTime timestamp;
guint64 distance;
gboolean at_discont;
if (self->interleaved) {
timestamp = gst_adapter_prev_pts (self->adapter, &distance);
distance /= self->info.bpf;
} else {
timestamp = gst_planar_audio_adapter_prev_pts (self->padapter, &distance);
}
timestamp += gst_util_uint64_scale_int (distance, GST_SECOND, self->info.rate);
if (self->interleaved) {
buffer = gst_adapter_take_buffer (self->adapter, self->period_size);
at_discont = (gst_adapter_pts_at_discont (self->adapter) == timestamp);
} else {
buffer = gst_planar_audio_adapter_take_buffer (self->padapter,
self->period_samples, GST_MAP_READWRITE);
at_discont =
(gst_planar_audio_adapter_pts_at_discont (self->padapter) == timestamp);
}
GST_BUFFER_PTS (buffer) = timestamp;
GST_BUFFER_DURATION (buffer) = 10 * GST_MSECOND;
if (at_discont && distance == 0) {
GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT);
} else {
GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DISCONT);
}
return buffer;
}
static GstFlowReturn
gst_webrtc_dsp_analyze_reverse_stream (GstWebrtcDsp * self,
GstClockTime rec_time)
{
GstWebrtcEchoProbe *probe = NULL;
webrtc::AudioProcessing * apm;
webrtc::AudioFrame frame;
GstBuffer *buf = NULL;
GstFlowReturn ret = GST_FLOW_OK;
gint err, delay;
GST_OBJECT_LOCK (self);
if (self->echo_cancel)
probe = GST_WEBRTC_ECHO_PROBE (g_object_ref (self->probe));
GST_OBJECT_UNLOCK (self);
/* If echo cancellation is disabled */
if (!probe)
return GST_FLOW_OK;
apm = self->apm;
if (self->delay_agnostic)
rec_time = GST_CLOCK_TIME_NONE;
again:
delay = gst_webrtc_echo_probe_read (probe, rec_time, (gpointer) &frame, &buf);
apm->set_stream_delay_ms (delay);
if (delay < 0)
goto done;
if (frame.sample_rate_hz_ != self->info.rate) {
GST_ELEMENT_ERROR (self, STREAM, FORMAT,
("Echo Probe has rate %i , while the DSP is running at rate %i,"
" use a caps filter to ensure those are the same.",
frame.sample_rate_hz_, self->info.rate), (NULL));
ret = GST_FLOW_ERROR;
goto done;
}
if (buf) {
webrtc::StreamConfig config (frame.sample_rate_hz_, frame.num_channels_,
false);
GstAudioBuffer abuf;
float * const * data;
gst_audio_buffer_map (&abuf, &self->info, buf, GST_MAP_READWRITE);
data = (float * const *) abuf.planes;
if ((err = apm->ProcessReverseStream (data, config, config, data)) < 0)
GST_WARNING_OBJECT (self, "Reverse stream analyses failed: %s.",
webrtc_error_to_string (err));
gst_audio_buffer_unmap (&abuf);
gst_buffer_replace (&buf, NULL);
} else {
if ((err = apm->AnalyzeReverseStream (&frame)) < 0)
GST_WARNING_OBJECT (self, "Reverse stream analyses failed: %s.",
webrtc_error_to_string (err));
}
if (self->delay_agnostic)
goto again;
done:
gst_object_unref (probe);
gst_buffer_replace (&buf, NULL);
return ret;
}
static void
gst_webrtc_vad_post_activity (GstWebrtcDsp *self, GstBuffer *buffer,
gboolean stream_has_voice)
{
GstClockTime timestamp = GST_BUFFER_PTS (buffer);
GstBaseTransform *trans = GST_BASE_TRANSFORM_CAST (self);
GstStructure *s;
GstClockTime stream_time;
GstAudioLevelMeta *meta;
guint8 level;
level = self->apm->level_estimator ()->RMS ();
meta = gst_buffer_get_audio_level_meta (buffer);
if (meta) {
meta->voice_activity = stream_has_voice;
meta->level = level;
} else {
gst_buffer_add_audio_level_meta (buffer, level, stream_has_voice);
}
stream_time = gst_segment_to_stream_time (&trans->segment, GST_FORMAT_TIME,
timestamp);
s = gst_structure_new ("voice-activity",
"stream-time", G_TYPE_UINT64, stream_time,
"stream-has-voice", G_TYPE_BOOLEAN, stream_has_voice, NULL);
GST_LOG_OBJECT (self, "Posting voice activity message, stream %s voice",
stream_has_voice ? "now has" : "no longer has");
gst_element_post_message (GST_ELEMENT (self),
gst_message_new_element (GST_OBJECT (self), s));
}
static GstFlowReturn
gst_webrtc_dsp_process_stream (GstWebrtcDsp * self,
GstBuffer * buffer)
{
GstAudioBuffer abuf;
webrtc::AudioProcessing * apm = self->apm;
gint err;
if (!gst_audio_buffer_map (&abuf, &self->info, buffer,
(GstMapFlags) GST_MAP_READWRITE)) {
gst_buffer_unref (buffer);
return GST_FLOW_ERROR;
}
if (self->interleaved) {
webrtc::AudioFrame frame;
frame.num_channels_ = self->info.channels;
frame.sample_rate_hz_ = self->info.rate;
frame.samples_per_channel_ = self->period_samples;
memcpy (frame.data_, abuf.planes[0], self->period_size);
err = apm->ProcessStream (&frame);
if (err >= 0)
memcpy (abuf.planes[0], frame.data_, self->period_size);
} else {
float * const * data = (float * const *) abuf.planes;
webrtc::StreamConfig config (self->info.rate, self->info.channels, false);
err = apm->ProcessStream (data, config, config, data);
}
if (err < 0) {
GST_WARNING_OBJECT (self, "Failed to filter the audio: %s.",
webrtc_error_to_string (err));
} else {
if (self->voice_detection) {
gboolean stream_has_voice = apm->voice_detection ()->stream_has_voice ();
if (stream_has_voice != self->stream_has_voice)
gst_webrtc_vad_post_activity (self, buffer, stream_has_voice);
self->stream_has_voice = stream_has_voice;
}
}
gst_audio_buffer_unmap (&abuf);
return GST_FLOW_OK;
}
static GstFlowReturn
gst_webrtc_dsp_submit_input_buffer (GstBaseTransform * btrans,
gboolean is_discont, GstBuffer * buffer)
{
GstWebrtcDsp *self = GST_WEBRTC_DSP (btrans);
buffer = gst_buffer_make_writable (buffer);
GST_BUFFER_PTS (buffer) = gst_segment_to_running_time (&btrans->segment,
GST_FORMAT_TIME, GST_BUFFER_PTS (buffer));
if (is_discont) {
GST_DEBUG_OBJECT (self,
"Received discont, clearing adapter.");
if (self->interleaved)
gst_adapter_clear (self->adapter);
else
gst_planar_audio_adapter_clear (self->padapter);
}
if (self->interleaved)
gst_adapter_push (self->adapter, buffer);
else
gst_planar_audio_adapter_push (self->padapter, buffer);
return GST_FLOW_OK;
}
static GstFlowReturn
gst_webrtc_dsp_generate_output (GstBaseTransform * btrans, GstBuffer ** outbuf)
{
GstWebrtcDsp *self = GST_WEBRTC_DSP (btrans);
GstFlowReturn ret;
gboolean not_enough;
if (self->interleaved)
not_enough = gst_adapter_available (self->adapter) < self->period_size;
else
not_enough = gst_planar_audio_adapter_available (self->padapter) <
self->period_samples;
if (not_enough) {
*outbuf = NULL;
return GST_FLOW_OK;
}
*outbuf = gst_webrtc_dsp_take_buffer (self);
ret = gst_webrtc_dsp_analyze_reverse_stream (self, GST_BUFFER_PTS (*outbuf));
if (ret == GST_FLOW_OK)
ret = gst_webrtc_dsp_process_stream (self, *outbuf);
return ret;
}
static gboolean
gst_webrtc_dsp_start (GstBaseTransform * btrans)
{
GstWebrtcDsp *self = GST_WEBRTC_DSP (btrans);
webrtc::Config config;
GST_OBJECT_LOCK (self);
config.Set < webrtc::ExtendedFilter >
(new webrtc::ExtendedFilter (self->extended_filter));
config.Set < webrtc::ExperimentalAgc >
(new webrtc::ExperimentalAgc (self->experimental_agc, self->startup_min_volume));
config.Set < webrtc::DelayAgnostic >
(new webrtc::DelayAgnostic (self->delay_agnostic));
/* TODO Intelligibility enhancer, Beamforming, etc. */
self->apm = webrtc::AudioProcessing::Create (config);
if (self->echo_cancel) {
self->probe = gst_webrtc_acquire_echo_probe (self->probe_name);
if (self->probe == NULL) {
GST_OBJECT_UNLOCK (self);
GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND,
("No echo probe with name %s found.", self->probe_name), (NULL));
return FALSE;
}
}
GST_OBJECT_UNLOCK (self);
return TRUE;
}
static gboolean
gst_webrtc_dsp_setup (GstAudioFilter * filter, const GstAudioInfo * info)
{
GstWebrtcDsp *self = GST_WEBRTC_DSP (filter);
webrtc::AudioProcessing * apm;
webrtc::ProcessingConfig pconfig;
GstAudioInfo probe_info = *info;
gint err = 0;
GST_LOG_OBJECT (self, "setting format to %s with %i Hz and %i channels",
info->finfo->description, info->rate, info->channels);
GST_OBJECT_LOCK (self);
gst_adapter_clear (self->adapter);
gst_planar_audio_adapter_clear (self->padapter);
self->info = *info;
self->interleaved = (info->layout == GST_AUDIO_LAYOUT_INTERLEAVED);
apm = self->apm;
if (!self->interleaved)
gst_planar_audio_adapter_configure (self->padapter, info);
/* WebRTC library works with 10ms buffers, compute once this size */
self->period_samples = info->rate / 100;
self->period_size = self->period_samples * info->bpf;
if (self->interleaved &&
(webrtc::AudioFrame::kMaxDataSizeSamples * 2) < self->period_size)
goto period_too_big;
if (self->probe) {
GST_WEBRTC_ECHO_PROBE_LOCK (self->probe);
if (self->probe->info.rate != 0) {
if (self->probe->info.rate != info->rate)
goto probe_has_wrong_rate;
probe_info = self->probe->info;
}
GST_WEBRTC_ECHO_PROBE_UNLOCK (self->probe);
}
/* input stream */
pconfig.streams[webrtc::ProcessingConfig::kInputStream] =
webrtc::StreamConfig (info->rate, info->channels, false);
/* output stream */
pconfig.streams[webrtc::ProcessingConfig::kOutputStream] =
webrtc::StreamConfig (info->rate, info->channels, false);
/* reverse input stream */
pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] =
webrtc::StreamConfig (probe_info.rate, probe_info.channels, false);
/* reverse output stream */
pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] =
webrtc::StreamConfig (probe_info.rate, probe_info.channels, false);
if ((err = apm->Initialize (pconfig)) < 0)
goto initialize_failed;
/* Setup Filters */
if (self->high_pass_filter) {
GST_DEBUG_OBJECT (self, "Enabling High Pass filter");
apm->high_pass_filter ()->Enable (true);
}
if (self->echo_cancel) {
GST_DEBUG_OBJECT (self, "Enabling Echo Cancellation");
apm->echo_cancellation ()->enable_drift_compensation (false);
apm->echo_cancellation ()
->set_suppression_level (self->echo_suppression_level);
apm->echo_cancellation ()->Enable (true);
}
if (self->noise_suppression) {
GST_DEBUG_OBJECT (self, "Enabling Noise Suppression");
apm->noise_suppression ()->set_level (self->noise_suppression_level);
apm->noise_suppression ()->Enable (true);
}
if (self->gain_control) {
GEnumClass *mode_class = (GEnumClass *)
g_type_class_ref (GST_TYPE_WEBRTC_GAIN_CONTROL_MODE);
GST_DEBUG_OBJECT (self, "Enabling Digital Gain Control, target level "
"dBFS %d, compression gain dB %d, limiter %senabled, mode: %s",
self->target_level_dbfs, self->compression_gain_db,
self->limiter ? "" : "NOT ",
g_enum_get_value (mode_class, self->gain_control_mode)->value_name);
g_type_class_unref (mode_class);
apm->gain_control ()->set_mode (self->gain_control_mode);
apm->gain_control ()->set_target_level_dbfs (self->target_level_dbfs);
apm->gain_control ()->set_compression_gain_db (self->compression_gain_db);
apm->gain_control ()->enable_limiter (self->limiter);
apm->gain_control ()->Enable (true);
}
if (self->voice_detection) {
GEnumClass *likelihood_class = (GEnumClass *)
g_type_class_ref (GST_TYPE_WEBRTC_VOICE_DETECTION_LIKELIHOOD);
GST_DEBUG_OBJECT (self, "Enabling Voice Activity Detection, frame size "
"%d milliseconds, likelihood: %s", self->voice_detection_frame_size_ms,
g_enum_get_value (likelihood_class,
self->voice_detection_likelihood)->value_name);
g_type_class_unref (likelihood_class);
self->stream_has_voice = FALSE;
apm->voice_detection ()->Enable (true);
apm->voice_detection ()->set_likelihood (self->voice_detection_likelihood);
apm->voice_detection ()->set_frame_size_ms (
self->voice_detection_frame_size_ms);
apm->level_estimator ()->Enable (true);
}
GST_OBJECT_UNLOCK (self);
return TRUE;
period_too_big:
GST_OBJECT_UNLOCK (self);
GST_WARNING_OBJECT (self, "webrtcdsp format produce too big period "
"(maximum is %" G_GSIZE_FORMAT " samples and we have %u samples), "
"reduce the number of channels or the rate.",
webrtc::AudioFrame::kMaxDataSizeSamples, self->period_size / 2);
return FALSE;
probe_has_wrong_rate:
GST_WEBRTC_ECHO_PROBE_UNLOCK (self->probe);
GST_OBJECT_UNLOCK (self);
GST_ELEMENT_ERROR (self, STREAM, FORMAT,
("Echo Probe has rate %i , while the DSP is running at rate %i,"
" use a caps filter to ensure those are the same.",
probe_info.rate, info->rate), (NULL));
return FALSE;
initialize_failed:
GST_OBJECT_UNLOCK (self);
GST_ELEMENT_ERROR (self, LIBRARY, INIT,
("Failed to initialize WebRTC Audio Processing library"),
("webrtc::AudioProcessing::Initialize() failed: %s",
webrtc_error_to_string (err)));
return FALSE;
}
static gboolean
gst_webrtc_dsp_stop (GstBaseTransform * btrans)
{
GstWebrtcDsp *self = GST_WEBRTC_DSP (btrans);
GST_OBJECT_LOCK (self);
gst_adapter_clear (self->adapter);
gst_planar_audio_adapter_clear (self->padapter);
if (self->probe) {
gst_webrtc_release_echo_probe (self->probe);
self->probe = NULL;
}
delete self->apm;
self->apm = NULL;
GST_OBJECT_UNLOCK (self);
return TRUE;
}
static void
gst_webrtc_dsp_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec)
{
GstWebrtcDsp *self = GST_WEBRTC_DSP (object);
GST_OBJECT_LOCK (self);
switch (prop_id) {
case PROP_PROBE:
g_free (self->probe_name);
self->probe_name = g_value_dup_string (value);
break;
case PROP_HIGH_PASS_FILTER:
self->high_pass_filter = g_value_get_boolean (value);
break;
case PROP_ECHO_CANCEL:
self->echo_cancel = g_value_get_boolean (value);
break;
case PROP_ECHO_SUPPRESSION_LEVEL:
self->echo_suppression_level =
(GstWebrtcEchoSuppressionLevel) g_value_get_enum (value);
break;
case PROP_NOISE_SUPPRESSION:
self->noise_suppression = g_value_get_boolean (value);
break;
case PROP_NOISE_SUPPRESSION_LEVEL:
self->noise_suppression_level =
(GstWebrtcNoiseSuppressionLevel) g_value_get_enum (value);
break;
case PROP_GAIN_CONTROL:
self->gain_control = g_value_get_boolean (value);
break;
case PROP_EXPERIMENTAL_AGC:
self->experimental_agc = g_value_get_boolean (value);
break;
case PROP_EXTENDED_FILTER:
self->extended_filter = g_value_get_boolean (value);
break;
case PROP_DELAY_AGNOSTIC:
self->delay_agnostic = g_value_get_boolean (value);
break;
case PROP_TARGET_LEVEL_DBFS:
self->target_level_dbfs = g_value_get_int (value);
break;
case PROP_COMPRESSION_GAIN_DB:
self->compression_gain_db = g_value_get_int (value);
break;
case PROP_STARTUP_MIN_VOLUME:
self->startup_min_volume = g_value_get_int (value);
break;
case PROP_LIMITER:
self->limiter = g_value_get_boolean (value);
break;
case PROP_GAIN_CONTROL_MODE:
self->gain_control_mode =
(GstWebrtcGainControlMode) g_value_get_enum (value);
break;
case PROP_VOICE_DETECTION:
self->voice_detection = g_value_get_boolean (value);
break;
case PROP_VOICE_DETECTION_FRAME_SIZE_MS:
self->voice_detection_frame_size_ms = g_value_get_int (value);
break;
case PROP_VOICE_DETECTION_LIKELIHOOD:
self->voice_detection_likelihood =
(GstWebrtcVoiceDetectionLikelihood) g_value_get_enum (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
GST_OBJECT_UNLOCK (self);
}
static void
gst_webrtc_dsp_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec)
{
GstWebrtcDsp *self = GST_WEBRTC_DSP (object);
GST_OBJECT_LOCK (self);
switch (prop_id) {
case PROP_PROBE:
g_value_set_string (value, self->probe_name);
break;
case PROP_HIGH_PASS_FILTER:
g_value_set_boolean (value, self->high_pass_filter);
break;
case PROP_ECHO_CANCEL:
g_value_set_boolean (value, self->echo_cancel);
break;
case PROP_ECHO_SUPPRESSION_LEVEL:
g_value_set_enum (value, self->echo_suppression_level);
break;
case PROP_NOISE_SUPPRESSION:
g_value_set_boolean (value, self->noise_suppression);
break;
case PROP_NOISE_SUPPRESSION_LEVEL:
g_value_set_enum (value, self->noise_suppression_level);
break;
case PROP_GAIN_CONTROL:
g_value_set_boolean (value, self->gain_control);
break;
case PROP_EXPERIMENTAL_AGC:
g_value_set_boolean (value, self->experimental_agc);
break;
case PROP_EXTENDED_FILTER:
g_value_set_boolean (value, self->extended_filter);
break;
case PROP_DELAY_AGNOSTIC:
g_value_set_boolean (value, self->delay_agnostic);
break;
case PROP_TARGET_LEVEL_DBFS:
g_value_set_int (value, self->target_level_dbfs);
break;
case PROP_COMPRESSION_GAIN_DB:
g_value_set_int (value, self->compression_gain_db);
break;
case PROP_STARTUP_MIN_VOLUME:
g_value_set_int (value, self->startup_min_volume);
break;
case PROP_LIMITER:
g_value_set_boolean (value, self->limiter);
break;
case PROP_GAIN_CONTROL_MODE:
g_value_set_enum (value, self->gain_control_mode);
break;
case PROP_VOICE_DETECTION:
g_value_set_boolean (value, self->voice_detection);
break;
case PROP_VOICE_DETECTION_FRAME_SIZE_MS:
g_value_set_int (value, self->voice_detection_frame_size_ms);
break;
case PROP_VOICE_DETECTION_LIKELIHOOD:
g_value_set_enum (value, self->voice_detection_likelihood);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
GST_OBJECT_UNLOCK (self);
}
static void
gst_webrtc_dsp_finalize (GObject * object)
{
GstWebrtcDsp *self = GST_WEBRTC_DSP (object);
gst_object_unref (self->adapter);
gst_object_unref (self->padapter);
g_free (self->probe_name);
G_OBJECT_CLASS (gst_webrtc_dsp_parent_class)->finalize (object);
}
static void
gst_webrtc_dsp_init (GstWebrtcDsp * self)
{
self->adapter = gst_adapter_new ();
self->padapter = gst_planar_audio_adapter_new ();
gst_audio_info_init (&self->info);
}
static void
gst_webrtc_dsp_class_init (GstWebrtcDspClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstBaseTransformClass *btrans_class = GST_BASE_TRANSFORM_CLASS (klass);
GstAudioFilterClass *audiofilter_class = GST_AUDIO_FILTER_CLASS (klass);
gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_webrtc_dsp_finalize);
gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_webrtc_dsp_set_property);
gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_webrtc_dsp_get_property);
btrans_class->passthrough_on_same_caps = FALSE;
btrans_class->start = GST_DEBUG_FUNCPTR (gst_webrtc_dsp_start);
btrans_class->stop = GST_DEBUG_FUNCPTR (gst_webrtc_dsp_stop);
btrans_class->submit_input_buffer =
GST_DEBUG_FUNCPTR (gst_webrtc_dsp_submit_input_buffer);
btrans_class->generate_output =
GST_DEBUG_FUNCPTR (gst_webrtc_dsp_generate_output);
audiofilter_class->setup = GST_DEBUG_FUNCPTR (gst_webrtc_dsp_setup);
gst_element_class_add_static_pad_template (element_class,
&gst_webrtc_dsp_src_template);
gst_element_class_add_static_pad_template (element_class,
&gst_webrtc_dsp_sink_template);
gst_element_class_set_static_metadata (element_class,
"Voice Processor (AGC, AEC, filters, etc.)",
"Generic/Audio",
"Pre-processes voice with WebRTC Audio Processing Library",
"Nicolas Dufresne <nicolas.dufresne@collabora.com>");
g_object_class_install_property (gobject_class,
PROP_PROBE,
g_param_spec_string ("probe", "Echo Probe",
"The name of the webrtcechoprobe element that record the audio being "
"played through loud speakers. Must be set before PAUSED state.",
"webrtcechoprobe0",
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_HIGH_PASS_FILTER,
g_param_spec_boolean ("high-pass-filter", "High Pass Filter",
"Enable or disable high pass filtering", TRUE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_ECHO_CANCEL,
g_param_spec_boolean ("echo-cancel", "Echo Cancel",
"Enable or disable echo canceller", TRUE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_ECHO_SUPPRESSION_LEVEL,
g_param_spec_enum ("echo-suppression-level", "Echo Suppression Level",
"Controls the aggressiveness of the suppressor. A higher level "
"trades off double-talk performance for increased echo suppression.",
GST_TYPE_WEBRTC_ECHO_SUPPRESSION_LEVEL,
webrtc::EchoCancellation::kModerateSuppression,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_NOISE_SUPPRESSION,
g_param_spec_boolean ("noise-suppression", "Noise Suppression",
"Enable or disable noise suppression", TRUE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_NOISE_SUPPRESSION_LEVEL,
g_param_spec_enum ("noise-suppression-level", "Noise Suppression Level",
"Controls the aggressiveness of the suppression. Increasing the "
"level will reduce the noise level at the expense of a higher "
"speech distortion.", GST_TYPE_WEBRTC_NOISE_SUPPRESSION_LEVEL,
webrtc::EchoCancellation::kModerateSuppression,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_GAIN_CONTROL,
g_param_spec_boolean ("gain-control", "Gain Control",
"Enable or disable automatic digital gain control",
TRUE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_EXPERIMENTAL_AGC,
g_param_spec_boolean ("experimental-agc", "Experimental AGC",
"Enable or disable experimental automatic gain control.",
FALSE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_EXTENDED_FILTER,
g_param_spec_boolean ("extended-filter", "Extended Filter",
"Enable or disable the extended filter.",
TRUE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_DELAY_AGNOSTIC,
g_param_spec_boolean ("delay-agnostic", "Delay Agnostic",
"Enable or disable the delay agnostic mode.",
FALSE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_TARGET_LEVEL_DBFS,
g_param_spec_int ("target-level-dbfs", "Target Level dBFS",
"Sets the target peak |level| (or envelope) of the gain control in "
"dBFS (decibels from digital full-scale).",
0, 31, DEFAULT_TARGET_LEVEL_DBFS, (GParamFlags) (G_PARAM_READWRITE |
G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_COMPRESSION_GAIN_DB,
g_param_spec_int ("compression-gain-db", "Compression Gain dB",
"Sets the maximum |gain| the digital compression stage may apply, "
"in dB.",
0, 90, DEFAULT_COMPRESSION_GAIN_DB, (GParamFlags) (G_PARAM_READWRITE |
G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_STARTUP_MIN_VOLUME,
g_param_spec_int ("startup-min-volume", "Startup Minimum Volume",
"At startup the experimental AGC moves the microphone volume up to "
"|startup_min_volume| if the current microphone volume is set too "
"low. No effect if experimental-agc isn't enabled.",
12, 255, DEFAULT_STARTUP_MIN_VOLUME, (GParamFlags) (G_PARAM_READWRITE |
G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_LIMITER,
g_param_spec_boolean ("limiter", "Limiter",
"When enabled, the compression stage will hard limit the signal to "
"the target level. Otherwise, the signal will be compressed but not "
"limited above the target level.",
DEFAULT_LIMITER, (GParamFlags) (G_PARAM_READWRITE |
G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_GAIN_CONTROL_MODE,
g_param_spec_enum ("gain-control-mode", "Gain Control Mode",
"Controls the mode of the compression stage",
GST_TYPE_WEBRTC_GAIN_CONTROL_MODE,
DEFAULT_GAIN_CONTROL_MODE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_VOICE_DETECTION,
g_param_spec_boolean ("voice-detection", "Voice Detection",
"Enable or disable the voice activity detector",
DEFAULT_VOICE_DETECTION, (GParamFlags) (G_PARAM_READWRITE |
G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_VOICE_DETECTION_FRAME_SIZE_MS,
g_param_spec_int ("voice-detection-frame-size-ms",
"Voice Detection Frame Size Milliseconds",
"Sets the |size| of the frames in ms on which the VAD will operate. "
"Larger frames will improve detection accuracy, but reduce the "
"frequency of updates",
10, 30, DEFAULT_VOICE_DETECTION_FRAME_SIZE_MS,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_VOICE_DETECTION_LIKELIHOOD,
g_param_spec_enum ("voice-detection-likelihood",
"Voice Detection Likelihood",
"Specifies the likelihood that a frame will be declared to contain "
"voice.",
GST_TYPE_WEBRTC_VOICE_DETECTION_LIKELIHOOD,
DEFAULT_VOICE_DETECTION_LIKELIHOOD,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
gst_type_mark_as_plugin_api (GST_TYPE_WEBRTC_GAIN_CONTROL_MODE, (GstPluginAPIFlags) 0);
gst_type_mark_as_plugin_api (GST_TYPE_WEBRTC_NOISE_SUPPRESSION_LEVEL, (GstPluginAPIFlags) 0);
gst_type_mark_as_plugin_api (GST_TYPE_WEBRTC_ECHO_SUPPRESSION_LEVEL, (GstPluginAPIFlags) 0);
gst_type_mark_as_plugin_api (GST_TYPE_WEBRTC_VOICE_DETECTION_LIKELIHOOD, (GstPluginAPIFlags) 0);
}
Reference in a new issue View git blame Copy permalink