webrtcbin: implement support for FEC and RTX

https://bugzilla.gnome.org/show_bug.cgi?id=795044

ext/webrtc/gstwebrtcbin.h

@@ -57,6 +57,9 @@ struct _GstWebRTCBinPad
   guint                 mlineindex;
 
   GstWebRTCRTPTransceiver *trans;
+  gulong                block_id;
+
+  GstCaps              *received_caps;
 };
 
 struct _GstWebRTCBinPadClass
@@ -125,6 +128,7 @@ struct _GstWebRTCBinPrivate
   gboolean async_pending;
 
   GList *pending_pads;
+  GList *pending_sink_transceivers;
 
   /* count of the number of media streams we've offered for uniqueness */
   /* FIXME: overflow? */

ext/webrtc/webrtctransceiver.c

@@ -29,10 +29,17 @@
 G_DEFINE_TYPE (WebRTCTransceiver, webrtc_transceiver,
     GST_TYPE_WEBRTC_RTP_TRANSCEIVER);
 
+#define DEFAULT_FEC_TYPE GST_WEBRTC_FEC_TYPE_NONE
+#define DEFAULT_DO_NACK FALSE
+#define DEFAULT_FEC_PERCENTAGE 100
+
 enum
 {
   PROP_0,
   PROP_WEBRTC,
+  PROP_FEC_TYPE,
+  PROP_FEC_PERCENTAGE,
+  PROP_DO_NACK,
 };
 
 void
@@ -78,6 +85,15 @@ webrtc_transceiver_set_property (GObject * object, guint prop_id,
   switch (prop_id) {
     case PROP_WEBRTC:
       break;
+    case PROP_FEC_TYPE:
+      trans->fec_type = g_value_get_enum (value);
+      break;
+    case PROP_DO_NACK:
+      trans->do_nack = g_value_get_boolean (value);
+      break;
+    case PROP_FEC_PERCENTAGE:
+      trans->fec_percentage = g_value_get_uint (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -93,6 +109,15 @@ webrtc_transceiver_get_property (GObject * object, guint prop_id,
 
   GST_OBJECT_LOCK (trans);
   switch (prop_id) {
+    case PROP_FEC_TYPE:
+      g_value_set_enum (value, trans->fec_type);
+      break;
+    case PROP_DO_NACK:
+      g_value_set_boolean (value, trans->do_nack);
+      break;
+    case PROP_FEC_PERCENTAGE:
+      g_value_set_uint (value, trans->fec_percentage);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -109,6 +134,10 @@ webrtc_transceiver_finalize (GObject * object)
     gst_object_unref (trans->stream);
   trans->stream = NULL;
 
+  if (trans->local_rtx_ssrc_map)
+    gst_structure_free (trans->local_rtx_ssrc_map);
+  trans->local_rtx_ssrc_map = NULL;
+
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
@@ -129,6 +158,28 @@ webrtc_transceiver_class_init (WebRTCTransceiverClass * klass)
           "Parent webrtcbin",
           GST_TYPE_WEBRTC_BIN,
           G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class,
+      PROP_FEC_TYPE,
+      g_param_spec_enum ("fec-type", "FEC type",
+          "The type of Forward Error Correction to use",
+          GST_TYPE_WEBRTC_FEC_TYPE,
+          DEFAULT_FEC_TYPE,
+          G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class,
+      PROP_DO_NACK,
+      g_param_spec_boolean ("do-nack", "Do nack",
+          "Whether to send negative acknowledgements for feedback",
+          DEFAULT_DO_NACK,
+          G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class,
+      PROP_FEC_PERCENTAGE,
+      g_param_spec_uint ("fec-percentage", "FEC percentage",
+          "The amount of Forward Error Correction to apply",
+          0, 100, DEFAULT_FEC_PERCENTAGE,
+          G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 }
 
 static void

ext/webrtc/webrtctransceiver.h

@@ -38,6 +38,12 @@ struct _WebRTCTransceiver
   GstWebRTCRTPTransceiver   parent;
 
   TransportStream          *stream;
+  GstStructure             *local_rtx_ssrc_map;
+
+  /* Properties */
+  GstWebRTCFECType         fec_type;
+  guint                    fec_percentage;
+  gboolean                 do_nack;
 };
 
 struct _WebRTCTransceiverClass

gst-libs/gst/webrtc/webrtc_fwd.h

@@ -253,4 +253,15 @@ typedef enum /*< underscore_name=gst_webrtc_stats_type >*/
   GST_WEBRTC_STATS_CERTIFICATE,
 } GstWebRTCStatsType;
 
+/**
+ * GstWebRTCFECType:
+ * GST_WEBRTC_FEC_TYPE_NONE: none
+ * GST_WEBRTC_FEC_TYPE_ULP_RED: ulpfec + red
+ */
+typedef enum /*< underscore_name=gst_webrtc_fec_type >*/
+{
+  GST_WEBRTC_FEC_TYPE_NONE,
+  GST_WEBRTC_FEC_TYPE_ULP_RED,
+} GstWebRTCFECType;
+
 #endif /* __GST_WEBRTC_FWD_H__ */

tests/check/elements/webrtcbin.c

@@ -821,6 +821,67 @@ GST_START_TEST (test_media_direction)
 
 GST_END_TEST;
 
+static void
+on_sdp_media_payload_types (struct test_webrtc *t, GstElement * element,
+    GstWebRTCSessionDescription * desc, gpointer user_data)
+{
+  const GstSDPMedia *vmedia;
+  guint j;
+
+  fail_unless_equals_int (gst_sdp_message_medias_len (desc->sdp), 2);
+
+  vmedia = gst_sdp_message_get_media (desc->sdp, 1);
+
+  for (j = 0; j < gst_sdp_media_attributes_len (vmedia); j++) {
+    const GstSDPAttribute *attr = gst_sdp_media_get_attribute (vmedia, j);
+
+    if (!g_strcmp0 (attr->key, "rtpmap")) {
+      if (g_str_has_prefix (attr->value, "97")) {
+        fail_unless_equals_string (attr->value, "97 VP8/90000");
+      } else if (g_str_has_prefix (attr->value, "96")) {
+        fail_unless_equals_string (attr->value, "96 red/90000");
+      } else if (g_str_has_prefix (attr->value, "98")) {
+        fail_unless_equals_string (attr->value, "98 ulpfec/90000");
+      } else if (g_str_has_prefix (attr->value, "99")) {
+        fail_unless_equals_string (attr->value, "99 rtx/90000");
+      } else if (g_str_has_prefix (attr->value, "100")) {
+        fail_unless_equals_string (attr->value, "100 rtx/90000");
+      }
+    }
+  }
+}
+
+/* In this test we verify that webrtcbin will pick available payload
+ * types when it needs to, in that example for RTX and FEC */
+GST_START_TEST (test_payload_types)
+{
+  struct test_webrtc *t = create_audio_video_test ();
+  struct validate_sdp offer = { on_sdp_media_payload_types, NULL };
+  GstWebRTCRTPTransceiver *trans;
+  GArray *transceivers;
+
+  t->offer_data = &offer;
+  t->on_offer_created = validate_sdp;
+  t->on_ice_candidate = NULL;
+  /* We don't really care about the answer here */
+  t->on_answer_created = NULL;
+
+  g_signal_emit_by_name (t->webrtc1, "get-transceivers", &transceivers);
+  fail_unless_equals_int (transceivers->len, 2);
+  trans = g_array_index (transceivers, GstWebRTCRTPTransceiver *, 1);
+  g_object_set (trans, "fec-type", GST_WEBRTC_FEC_TYPE_ULP_RED, "do-nack", TRUE,
+      NULL);
+  g_array_unref (transceivers);
+
+  test_webrtc_create_offer (t, t->webrtc1);
+
+  test_webrtc_wait_for_answer_error_eos (t);
+  fail_unless_equals_int (STATE_ANSWER_CREATED, t->state);
+  test_webrtc_free (t);
+}
+
+GST_END_TEST;
+
 static void
 on_sdp_media_setup (struct test_webrtc *t, GstElement * element,
     GstWebRTCSessionDescription * desc, gpointer user_data)
@@ -1367,6 +1428,7 @@ webrtcbin_suite (void)
     tcase_add_test (tc, test_get_transceivers);
     tcase_add_test (tc, test_add_recvonly_transceiver);
     tcase_add_test (tc, test_recvonly_sendonly);
+    tcase_add_test (tc, test_payload_types);
   }
 
   if (nicesrc)

tests/examples/webrtc/Makefile.am

@@ -1,5 +1,5 @@
 
-noinst_PROGRAMS = webrtc webrtcbidirectional webrtcswap
+noinst_PROGRAMS = webrtc webrtcbidirectional webrtcswap webrtctransceiver
 
 webrtc_SOURCES = webrtc.c
 webrtc_CFLAGS=\
@@ -39,3 +39,16 @@ webrtcswap_LDADD=\
 	$(GST_LIBS) \
 	$(GST_SDP_LIBS) \
 	$(top_builddir)/gst-libs/gst/webrtc/libgstwebrtc-@GST_API_VERSION@.la
+
+webrtctransceiver_SOURCES = webrtctransceiver.c
+webrtctransceiver_CFLAGS=\
+	-I$(top_srcdir)/gst-libs \
+	-I$(top_builddir)/gst-libs \
+	$(GST_PLUGINS_BASE_CFLAGS) \
+	$(GST_CFLAGS) \
+	$(GST_SDP_CFLAGS)
+webrtctransceiver_LDADD=\
+	$(GST_PLUGINS_BASE_LIBS) \
+	$(GST_LIBS) \
+	$(GST_SDP_LIBS) \
+	$(top_builddir)/gst-libs/gst/webrtc/libgstwebrtc-@GST_API_VERSION@.la

tests/examples/webrtc/meson.build

@@ -1,4 +1,4 @@
-examples = ['webrtc', 'webrtcbidirectional', 'webrtcswap']
+examples = ['webrtc', 'webrtcbidirectional', 'webrtcswap', 'webrtctransceiver']
 
 foreach example : examples
   exe_name = example

tests/examples/webrtc/webrtctransceiver.c

@@ -0,0 +1,218 @@
+#include <gst/gst.h>
+#include <gst/sdp/sdp.h>
+#include <gst/webrtc/webrtc.h>
+
+#include <string.h>
+
+static GMainLoop *loop;
+static GstElement *pipe1, *webrtc1, *webrtc2;
+static GstBus *bus1;
+
+static gboolean
+_bus_watch (GstBus * bus, GstMessage * msg, GstElement * pipe)
+{
+  switch (GST_MESSAGE_TYPE (msg)) {
+    case GST_MESSAGE_STATE_CHANGED:
+      if (GST_ELEMENT (msg->src) == pipe) {
+        GstState old, new, pending;
+
+        gst_message_parse_state_changed (msg, &old, &new, &pending);
+
+        {
+          gchar *dump_name = g_strconcat ("state_changed-",
+              gst_element_state_get_name (old), "_",
+              gst_element_state_get_name (new), NULL);
+          GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (msg->src),
+              GST_DEBUG_GRAPH_SHOW_ALL, dump_name);
+          g_free (dump_name);
+        }
+      }
+      break;
+    case GST_MESSAGE_ERROR:{
+      GError *err = NULL;
+      gchar *dbg_info = NULL;
+
+      GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (pipe),
+          GST_DEBUG_GRAPH_SHOW_ALL, "error");
+
+      gst_message_parse_error (msg, &err, &dbg_info);
+      g_printerr ("ERROR from element %s: %s\n",
+          GST_OBJECT_NAME (msg->src), err->message);
+      g_printerr ("Debugging info: %s\n", (dbg_info) ? dbg_info : "none");
+      g_error_free (err);
+      g_free (dbg_info);
+      g_main_loop_quit (loop);
+      break;
+    }
+    case GST_MESSAGE_EOS:{
+      GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (pipe),
+          GST_DEBUG_GRAPH_SHOW_ALL, "eos");
+      g_print ("EOS received\n");
+      g_main_loop_quit (loop);
+      break;
+    }
+    default:
+      break;
+  }
+
+  return TRUE;
+}
+
+static void
+_webrtc_pad_added (GstElement * webrtc, GstPad * new_pad, GstElement * pipe)
+{
+  GstElement *out;
+  GstPad *sink;
+
+  if (GST_PAD_DIRECTION (new_pad) != GST_PAD_SRC)
+    return;
+
+  out = gst_parse_bin_from_description ("rtpvp8depay ! vp8dec ! "
+      "videoconvert ! queue ! xvimagesink", TRUE, NULL);
+  gst_bin_add (GST_BIN (pipe), out);
+  gst_element_sync_state_with_parent (out);
+
+  sink = out->sinkpads->data;
+
+  gst_pad_link (new_pad, sink);
+}
+
+static void
+_on_answer_received (GstPromise * promise, gpointer user_data)
+{
+  GstWebRTCSessionDescription *answer = NULL;
+  const GstStructure *reply;
+  gchar *desc;
+
+  g_assert (gst_promise_wait (promise) == GST_PROMISE_RESULT_REPLIED);
+  reply = gst_promise_get_reply (promise);
+  gst_structure_get (reply, "answer",
+      GST_TYPE_WEBRTC_SESSION_DESCRIPTION, &answer, NULL);
+  gst_promise_unref (promise);
+  desc = gst_sdp_message_as_text (answer->sdp);
+  g_print ("Created answer:\n%s\n", desc);
+  g_free (desc);
+
+  g_signal_emit_by_name (webrtc1, "set-remote-description", answer, NULL);
+  g_signal_emit_by_name (webrtc2, "set-local-description", answer, NULL);
+
+  gst_webrtc_session_description_free (answer);
+}
+
+static void
+_on_offer_received (GstPromise * promise, gpointer user_data)
+{
+  GstWebRTCSessionDescription *offer = NULL;
+  const GstStructure *reply;
+  gchar *desc;
+
+  g_assert (gst_promise_wait (promise) == GST_PROMISE_RESULT_REPLIED);
+  reply = gst_promise_get_reply (promise);
+  gst_structure_get (reply, "offer",
+      GST_TYPE_WEBRTC_SESSION_DESCRIPTION, &offer, NULL);
+  gst_promise_unref (promise);
+  desc = gst_sdp_message_as_text (offer->sdp);
+  g_print ("Created offer:\n%s\n", desc);
+  g_free (desc);
+
+  g_signal_emit_by_name (webrtc1, "set-local-description", offer, NULL);
+  g_signal_emit_by_name (webrtc2, "set-remote-description", offer, NULL);
+
+  promise = gst_promise_new_with_change_func (_on_answer_received, user_data,
+      NULL);
+  g_signal_emit_by_name (webrtc2, "create-answer", NULL, promise);
+
+  gst_webrtc_session_description_free (offer);
+}
+
+static void
+_on_negotiation_needed (GstElement * element, gpointer user_data)
+{
+  GstPromise *promise;
+
+  promise = gst_promise_new_with_change_func (_on_offer_received, user_data,
+      NULL);
+  g_signal_emit_by_name (webrtc1, "create-offer", NULL, promise);
+}
+
+static void
+_on_ice_candidate (GstElement * webrtc, guint mlineindex, gchar * candidate,
+    GstElement * other)
+{
+  g_signal_emit_by_name (other, "add-ice-candidate", mlineindex, candidate);
+}
+
+static void
+_on_new_transceiver (GstElement * webrtc, GstWebRTCRTPTransceiver * trans)
+{
+  /* If we expected more than one transceiver, we would take a look at
+   * trans->mline, and compare it with webrtcbin's local description */
+  g_object_set (trans, "fec-type", GST_WEBRTC_FEC_TYPE_ULP_RED, NULL);
+}
+
+static void
+add_fec_to_offer (GstElement * webrtc)
+{
+  GstWebRTCRTPTransceiver *trans;
+  GArray *transceivers;
+
+  /* A transceiver has already been created when a sink pad was
+   * requested on the sending webrtcbin */
+
+  g_signal_emit_by_name (webrtc, "get-transceivers", &transceivers);
+
+  trans = g_array_index (transceivers, GstWebRTCRTPTransceiver *, 0);
+
+  g_object_set (trans, "fec-type", GST_WEBRTC_FEC_TYPE_ULP_RED,
+      "fec-percentage", 100, NULL);
+
+  g_array_unref (transceivers);
+}
+
+int
+main (int argc, char *argv[])
+{
+  gst_init (&argc, &argv);
+
+  loop = g_main_loop_new (NULL, FALSE);
+  pipe1 =
+      gst_parse_launch
+      ("videotestsrc pattern=ball ! video/x-raw ! queue ! vp8enc ! rtpvp8pay ! queue ! "
+      "application/x-rtp,media=video,payload=96,encoding-name=VP8 ! "
+      "webrtcbin name=send webrtcbin name=recv", NULL);
+  bus1 = gst_pipeline_get_bus (GST_PIPELINE (pipe1));
+  gst_bus_add_watch (bus1, (GstBusFunc) _bus_watch, pipe1);
+
+  webrtc1 = gst_bin_get_by_name (GST_BIN (pipe1), "send");
+  g_signal_connect (webrtc1, "on-negotiation-needed",
+      G_CALLBACK (_on_negotiation_needed), NULL);
+  add_fec_to_offer (webrtc1);
+
+  webrtc2 = gst_bin_get_by_name (GST_BIN (pipe1), "recv");
+  g_signal_connect (webrtc2, "pad-added", G_CALLBACK (_webrtc_pad_added),
+      pipe1);
+  g_signal_connect (webrtc1, "on-ice-candidate",
+      G_CALLBACK (_on_ice_candidate), webrtc2);
+  g_signal_connect (webrtc2, "on-ice-candidate",
+      G_CALLBACK (_on_ice_candidate), webrtc1);
+  g_signal_connect (webrtc2, "on-new-transceiver",
+      G_CALLBACK (_on_new_transceiver), NULL);
+
+  g_print ("Starting pipeline\n");
+  gst_element_set_state (GST_ELEMENT (pipe1), GST_STATE_PLAYING);
+
+  g_main_loop_run (loop);
+
+  gst_element_set_state (GST_ELEMENT (pipe1), GST_STATE_NULL);
+  g_print ("Pipeline stopped\n");
+
+  gst_object_unref (webrtc1);
+  gst_object_unref (webrtc2);
+  gst_bus_remove_watch (bus1);
+  gst_object_unref (bus1);
+  gst_object_unref (pipe1);
+
+  gst_deinit ();
+
+  return 0;
+}