webrtcdsp: Add delay-agnostic property

In this mode, we let WebRTC Audio Processing figure-out the delay. This
is useful when the latency reported by the stack cannot be trusted. Note
that in this mode, the leaking of echo during packet lost is much worst.
It is recommanded to use PLC (e.g. spanplc, or opus built-in plc).

In this mode, we don't do any synchronization. Instead, we simply process all
the available reverse stream data as it comes.
This commit is contained in:
Nicolas Dufresne 2016-07-13 23:00:51 -04:00
parent 064f46e9e1
commit 74c0d5fdd2
2 changed files with 41 additions and 3 deletions

View file

@ -156,7 +156,8 @@ enum
PROP_NOISE_SUPPRESSION_LEVEL, PROP_NOISE_SUPPRESSION_LEVEL,
PROP_GAIN_CONTROL, PROP_GAIN_CONTROL,
PROP_EXPERIMENTAL_AGC, PROP_EXPERIMENTAL_AGC,
PROP_EXTENDED_FILTER PROP_EXTENDED_FILTER,
PROP_DELAY_AGNOSTIC
}; };
/** /**
@ -189,6 +190,7 @@ struct _GstWebrtcDsp
gboolean gain_control; gboolean gain_control;
gboolean experimental_agc; gboolean experimental_agc;
gboolean extended_filter; gboolean extended_filter;
gboolean delay_agnostic;
}; };
G_DEFINE_TYPE (GstWebrtcDsp, gst_webrtc_dsp, GST_TYPE_AUDIO_FILTER); G_DEFINE_TYPE (GstWebrtcDsp, gst_webrtc_dsp, GST_TYPE_AUDIO_FILTER);
@ -290,8 +292,11 @@ gst_webrtc_dsp_analyze_reverse_stream (GstWebrtcDsp * self,
apm = self->apm; apm = self->apm;
delay = gst_webrtc_echo_probe_read (probe, rec_time, (gpointer) &frame); if (self->delay_agnostic)
rec_time = GST_CLOCK_TIME_NONE;
again:
delay = gst_webrtc_echo_probe_read (probe, rec_time, (gpointer) &frame);
apm->set_stream_delay_ms (delay); apm->set_stream_delay_ms (delay);
if (delay < 0) if (delay < 0)
@ -310,6 +315,9 @@ gst_webrtc_dsp_analyze_reverse_stream (GstWebrtcDsp * self,
GST_WARNING_OBJECT (self, "Reverse stream analyses failed: %s.", GST_WARNING_OBJECT (self, "Reverse stream analyses failed: %s.",
webrtc_error_to_string (err)); webrtc_error_to_string (err));
if (self->delay_agnostic)
goto again;
done: done:
gst_object_unref (probe); gst_object_unref (probe);
@ -401,6 +409,8 @@ gst_webrtc_dsp_start (GstBaseTransform * btrans)
(new webrtc::ExtendedFilter (self->extended_filter)); (new webrtc::ExtendedFilter (self->extended_filter));
config.Set < webrtc::ExperimentalAgc > config.Set < webrtc::ExperimentalAgc >
(new webrtc::ExperimentalAgc (self->experimental_agc)); (new webrtc::ExperimentalAgc (self->experimental_agc));
config.Set < webrtc::DelayAgnostic >
(new webrtc::DelayAgnostic (self->delay_agnostic));
/* TODO Intelligibility enhancer, Beamforming, etc. */ /* TODO Intelligibility enhancer, Beamforming, etc. */
@ -590,6 +600,9 @@ gst_webrtc_dsp_set_property (GObject * object,
case PROP_EXTENDED_FILTER: case PROP_EXTENDED_FILTER:
self->extended_filter = g_value_get_boolean (value); self->extended_filter = g_value_get_boolean (value);
break; break;
case PROP_DELAY_AGNOSTIC:
self->delay_agnostic = g_value_get_boolean (value);
break;
default: default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break; break;
@ -632,6 +645,9 @@ gst_webrtc_dsp_get_property (GObject * object,
case PROP_EXTENDED_FILTER: case PROP_EXTENDED_FILTER:
g_value_set_boolean (value, self->extended_filter); g_value_set_boolean (value, self->extended_filter);
break; break;
case PROP_DELAY_AGNOSTIC:
g_value_set_boolean (value, self->delay_agnostic);
break;
default: default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break; break;
@ -761,6 +777,13 @@ gst_webrtc_dsp_class_init (GstWebrtcDspClass * klass)
"Enable or disable the extended filter.", "Enable or disable the extended filter.",
TRUE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | TRUE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT))); G_PARAM_CONSTRUCT)));
g_object_class_install_property (gobject_class,
PROP_DELAY_AGNOSTIC,
g_param_spec_boolean ("delay-agnostic", "Delay Agnostic",
"Enable or disable the delay agnostic mode.",
FALSE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
G_PARAM_CONSTRUCT)));
} }
static gboolean static gboolean

View file

@ -280,6 +280,20 @@ gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self, GstClockTime rec_time,
if (!GST_CLOCK_TIME_IS_VALID (self->latency)) if (!GST_CLOCK_TIME_IS_VALID (self->latency))
goto done; goto done;
/* In delay agnostic mode, just return 10ms of data */
if (!GST_CLOCK_TIME_IS_VALID (rec_time)) {
avail = gst_adapter_available (self->adapter);
if (avail < self->period_size)
goto done;
size = self->period_size;
skip = 0;
offset = 0;
goto copy;
}
if (gst_adapter_available (self->adapter) == 0) { if (gst_adapter_available (self->adapter) == 0) {
diff = G_MAXINT64; diff = G_MAXINT64;
} else { } else {
@ -317,6 +331,7 @@ gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self, GstClockTime rec_time,
if (size < self->period_size) if (size < self->period_size)
memset (frame->data_, 0, self->period_size); memset (frame->data_, 0, self->period_size);
copy:
if (size) { if (size) {
gst_adapter_copy (self->adapter, (guint8 *) frame->data_ + skip, gst_adapter_copy (self->adapter, (guint8 *) frame->data_ + skip,
offset, size); offset, size);