webrtc-sendrecv: Fix create-answer caps negotiation

We need to parse the payload type map provided by the offer SDP and
set those values on the payloader, otherwise webrtcbin will create
a recvonly answer SDP and we won't send anything to the browser.

Fixed it for both C and Python sendrecv examples.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1864>

subprojects/gst-examples/webrtc/sendrecv/gst/webrtc-sendrecv.c

@@ -301,10 +301,6 @@ on_negotiation_needed (GstElement * element, gpointer user_data)
   }
 }
 
-#define STUN_SERVER " stun-server=stun://stun.l.google.com:19302 "
-#define RTP_CAPS_OPUS "application/x-rtp,media=audio,encoding-name=OPUS,payload="
-#define RTP_CAPS_VP8 "application/x-rtp,media=video,encoding-name=VP8,payload="
-
 static void
 data_channel_on_error (GObject * dc, gpointer user_data)
 {
@@ -421,16 +417,23 @@ webrtcbin_get_stats (GstElement * webrtcbin)
   return G_SOURCE_REMOVE;
 }
 
+
+#define STUN_SERVER " stun-server=stun://stun.l.google.com:19302 "
 #define RTP_TWCC_URI "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"
+#define RTP_CAPS_OPUS "application/x-rtp,media=audio,encoding-name=OPUS"
+#define RTP_OPUS_DEFAULT_PT 97
+#define RTP_CAPS_VP8 "application/x-rtp,media=video,encoding-name=VP8"
+#define RTP_VP8_DEFAULT_PT 96
 
 static gboolean
-start_pipeline (gboolean create_offer)
+start_pipeline (gboolean create_offer, guint opus_pt, guint vp8_pt)
 {
+  char *pipeline;
   GstStateChangeReturn ret;
   GError *error = NULL;
 
-  pipe1 =
+  pipeline =
-      gst_parse_launch ("webrtcbin bundle-policy=max-bundle name=sendrecv "
+      g_strdup_printf ("webrtcbin bundle-policy=max-bundle name=sendrecv "
       STUN_SERVER
       "videotestsrc is-live=true pattern=ball ! videoconvert ! queue ! "
       /* increase the default keyframe distance, browsers have really long
@@ -441,10 +444,13 @@ start_pipeline (gboolean create_offer)
       /* picture-id-mode=15-bit seems to make TWCC stats behave better, and
        * fixes stuttery video playback in Chrome */
       "rtpvp8pay name=videopay picture-id-mode=15-bit ! "
-      "queue ! " RTP_CAPS_VP8 "96 ! sendrecv. "
+      "queue ! %s,payload=%u ! sendrecv. "
       "audiotestsrc is-live=true wave=red-noise ! audioconvert ! audioresample ! queue ! opusenc ! rtpopuspay name=audiopay ! "
-      "queue ! " RTP_CAPS_OPUS "97 ! sendrecv. ", &error);
+      "queue ! %s,payload=%u ! sendrecv. ", RTP_CAPS_VP8, vp8_pt,
+      RTP_CAPS_OPUS, opus_pt);
 
+  pipe1 = gst_parse_launch (pipeline, &error);
+  g_free (pipeline);
   if (error) {
     gst_printerr ("Failed to parse launch: %s\n", error->message);
     g_error_free (error);
@@ -454,7 +460,7 @@ start_pipeline (gboolean create_offer)
   webrtc1 = gst_bin_get_by_name (GST_BIN (pipe1), "sendrecv");
   g_assert_nonnull (webrtc1);
 
-  if (remote_is_offerer) {
+  if (!create_offer) {
     /* XXX: this will fail when the remote offers twcc as the extension id
      * cannot currently be negotiated when receiving an offer.
      */
@@ -630,6 +636,50 @@ on_offer_received (GstSDPMessage * sdp)
   GstWebRTCSessionDescription *offer = NULL;
   GstPromise *promise;
 
+  /* If we got an offer and we have no webrtcbin, we need to parse the SDP,
+   * get the payload types, then start the pipeline */
+  if (!webrtc1 && our_id) {
+    guint medias_len, formats_len;
+    guint opus_pt = 0, vp8_pt = 0;
+
+    gst_println ("Parsing offer to find payload types");
+
+    medias_len = gst_sdp_message_medias_len (sdp);
+    for (int i = 0; i < medias_len; i++) {
+      const GstSDPMedia *media = gst_sdp_message_get_media (sdp, i);
+      formats_len = gst_sdp_media_formats_len (media);
+      for (int j = 0; j < formats_len; j++) {
+        guint pt;
+        GstCaps *caps;
+        GstStructure *s;
+        const char *fmt, *encoding_name;
+
+        fmt = gst_sdp_media_get_format (media, j);
+        if (g_strcmp0 (fmt, "webrtc-datachannel") == 0)
+          continue;
+        pt = atoi (fmt);
+        caps = gst_sdp_media_get_caps_from_media (media, pt);
+        s = gst_caps_get_structure (caps, 0);
+        encoding_name = gst_structure_get_string (s, "encoding-name");
+        if (vp8_pt == 0 && g_strcmp0 (encoding_name, "VP8") == 0)
+          vp8_pt = pt;
+        if (opus_pt == 0 && g_strcmp0 (encoding_name, "OPUS") == 0)
+          opus_pt = pt;
+      }
+    }
+
+    g_assert_cmpint (opus_pt, !=, 0);
+    g_assert_cmpint (vp8_pt, !=, 0);
+
+    gst_println ("Starting pipeline with opus pt: %u vp8 pt: %u", opus_pt,
+        vp8_pt);
+
+    if (!start_pipeline (FALSE, opus_pt, vp8_pt)) {
+      cleanup_and_quit_loop ("ERROR: failed to start pipeline",
+          PEER_CALL_ERROR);
+    }
+  }
+
   offer = gst_webrtc_session_description_new (GST_WEBRTC_SDP_TYPE_OFFER, sdp);
   g_assert_nonnull (offer);
 
@@ -692,7 +742,7 @@ on_server_message (SoupWebsocketConnection * conn, SoupWebsocketDataType type,
 
     app_state = PEER_CONNECTED;
     /* Start negotiation (exchange SDP and ICE candidates) */
-    if (!start_pipeline (TRUE))
+    if (!start_pipeline (TRUE, RTP_OPUS_DEFAULT_PT, RTP_VP8_DEFAULT_PT))
       cleanup_and_quit_loop ("ERROR: failed to start pipeline",
           PEER_CALL_ERROR);
   } else if (g_strcmp0 (text, "OFFER_REQUEST") == 0) {
@@ -702,7 +752,7 @@ on_server_message (SoupWebsocketConnection * conn, SoupWebsocketDataType type,
     }
     gst_print ("Received OFFER_REQUEST, sending offer\n");
     /* Peer wants us to start negotiation (exchange SDP and ICE candidates) */
-    if (!start_pipeline (TRUE))
+    if (!start_pipeline (TRUE, RTP_OPUS_DEFAULT_PT, RTP_VP8_DEFAULT_PT))
       cleanup_and_quit_loop ("ERROR: failed to start pipeline",
           PEER_CALL_ERROR);
   } else if (g_str_has_prefix (text, "ERROR")) {
@@ -743,17 +793,6 @@ on_server_message (SoupWebsocketConnection * conn, SoupWebsocketDataType type,
       goto out;
     }
 
-    /* If peer connection wasn't made yet and we are expecting peer will
-     * connect to us, launch pipeline at this moment */
-    if (!webrtc1 && our_id) {
-      if (!start_pipeline (FALSE)) {
-        cleanup_and_quit_loop ("ERROR: failed to start pipeline",
-            PEER_CALL_ERROR);
-      }
-
-      app_state = PEER_CALL_NEGOTIATING;
-    }
-
     object = json_node_get_object (root);
     /* Check type of JSON message */
     if (json_object_has_member (object, "sdp")) {
@@ -762,7 +801,7 @@ on_server_message (SoupWebsocketConnection * conn, SoupWebsocketDataType type,
       const gchar *text, *sdptype;
       GstWebRTCSessionDescription *answer;
 
-      g_assert_cmphex (app_state, ==, PEER_CALL_NEGOTIATING);
+      app_state = PEER_CALL_NEGOTIATING;
 
       child = json_object_get_object_member (object, "sdp");
 

subprojects/gst-examples/webrtc/sendrecv/gst/webrtc_sendrecv.py

@@ -35,9 +35,9 @@ PIPELINE_DESC = '''
 webrtcbin name=sendrecv bundle-policy=max-bundle stun-server=stun://stun.l.google.com:19302
  videotestsrc is-live=true pattern=ball ! videoconvert ! queue ! \
   vp8enc deadline=1 keyframe-max-dist=2000 ! rtpvp8pay picture-id-mode=15-bit !
-  queue ! application/x-rtp,media=video,encoding-name=VP8,payload=97 ! sendrecv.
+  queue ! application/x-rtp,media=video,encoding-name=VP8,payload={vp8_pt} ! sendrecv.
  audiotestsrc is-live=true wave=red-noise ! audioconvert ! audioresample ! queue ! opusenc ! rtpopuspay !
-  queue ! application/x-rtp,media=audio,encoding-name=OPUS,payload=96 ! sendrecv.
+  queue ! application/x-rtp,media=audio,encoding-name=OPUS,payload={opus_pt} ! sendrecv.
 '''
 
 from websockets.version import version as wsv
@@ -51,6 +51,33 @@ def print_error(msg):
     print(f'!!! {msg}', file=sys.stderr)
 
 
+def get_payload_types(sdpmsg, video_encoding, audio_encoding):
+    '''
+    Find the payload types for the specified video and audio encoding.
+
+    Very simplistically finds the first payload type matching the encoding
+    name. More complex applications will want to match caps on
+    profile-level-id, packetization-mode, etc.
+    '''
+    video_pt = None
+    audio_pt = None
+    for i in range(0, sdpmsg.medias_len()):
+        media = sdpmsg.get_media(i)
+        for j in range(0, media.formats_len()):
+            fmt = media.get_format(j)
+            if fmt == 'webrtc-datachannel':
+                continue
+            pt = int(fmt)
+            caps = media.get_caps_from_media(pt)
+            s = caps.get_structure(0)
+            encoding_name = s['encoding-name']
+            if video_pt is None and encoding_name == video_encoding:
+                video_pt = pt
+            elif audio_pt is None and encoding_name == audio_encoding:
+                audio_pt = pt
+    return {video_encoding: video_pt, audio_encoding: audio_pt}
+
+
 class WebRTCClient:
     def __init__(self, loop, our_id, peer_id, server, remote_is_offerer):
         self.conn = None
@@ -114,10 +141,6 @@ class WebRTCClient:
             print_status('Call was connected: creating offer')
             promise = Gst.Promise.new_with_change_func(self.on_offer_created, None, None)
             self.webrtc.emit('create-offer', None, promise)
-        elif self.remote_is_offerer:
-            # We are initiating the call, but we want the remote peer to create the offer
-            print_status('Call was connected: requesting remote peer for offer')
-            self.send_soon('OFFER_REQUEST')
 
     def send_ice_candidate_message(self, _, mlineindex, candidate):
         icemsg = json.dumps({'ice': {'candidate': candidate, 'sdpMLineIndex': mlineindex}})
@@ -167,9 +190,9 @@ class WebRTCClient:
         decodebin.sync_state_with_parent()
         self.webrtc.link(decodebin)
 
-    def start_pipeline(self, create_offer=True):
+    def start_pipeline(self, create_offer=True, opus_pt=96, vp8_pt=97):
         print_status(f'Creating pipeline, create_offer: {create_offer}')
-        self.pipe = Gst.parse_launch(PIPELINE_DESC)
+        self.pipe = Gst.parse_launch(PIPELINE_DESC.format(vp8_pt=vp8_pt, opus_pt=opus_pt))
         self.webrtc = self.pipe.get_by_name('sendrecv')
         self.webrtc.connect('on-negotiation-needed', self.on_negotiation_needed, create_offer)
         self.webrtc.connect('on-ice-candidate', self.send_ice_candidate_message)
@@ -192,7 +215,6 @@ class WebRTCClient:
         self.webrtc.emit('create-answer', None, promise)
 
     def handle_json(self, message):
-        assert (self.webrtc)
         try:
             msg = json.loads(message)
         except json.decoder.JSONDecoderError:
@@ -212,10 +234,21 @@ class WebRTCClient:
                 print_status('Received offer:\n%s' % sdp)
                 res, sdpmsg = GstSdp.SDPMessage.new()
                 GstSdp.sdp_message_parse_buffer(bytes(sdp.encode()), sdpmsg)
+
+                if not self.webrtc:
+                    print_status('Incoming call: received an offer, creating pipeline')
+                    pts = get_payload_types(sdpmsg, video_encoding='VP8', audio_encoding='OPUS')
+                    assert('VP8' in pts)
+                    assert('OPUS' in pts)
+                    self.start_pipeline(create_offer=False, vp8_pt=pts['VP8'], opus_pt=pts['OPUS'])
+
+                assert(self.webrtc)
+
                 offer = GstWebRTC.WebRTCSessionDescription.new(GstWebRTC.WebRTCSDPType.OFFER, sdpmsg)
                 promise = Gst.Promise.new_with_change_func(self.on_offer_set, None, None)
                 self.webrtc.emit('set-remote-description', offer, promise)
         elif 'ice' in msg:
+            assert(self.webrtc)
             ice = msg['ice']
             candidate = ice['candidate']
             sdpmlineindex = ice['sdpMLineIndex']
@@ -229,13 +262,6 @@ class WebRTCClient:
             self.pipe = None
         self.webrtc = None
 
-    def is_incoming_offer(self, msg):
-        if self.webrtc:
-            return False
-        if self.remote_is_offerer:
-            return True
-        return True
-
     async def loop(self):
         assert self.conn
         async for message in self.conn:
@@ -254,7 +280,9 @@ class WebRTCClient:
                     await self.setup_call()
             elif message == 'SESSION_OK':
                 if self.remote_is_offerer:
-                    self.start_pipeline(create_offer=False)
+                    # We are initiating the call, but we want the remote peer to create the offer
+                    print_status('Call was connected: requesting remote peer for offer')
+                    await self.send('OFFER_REQUEST')
                 else:
                     self.start_pipeline()
             elif message == 'OFFER_REQUEST':
@@ -265,9 +293,6 @@ class WebRTCClient:
                 self.close_pipeline()
                 return 1
             else:
-                if self.is_incoming_offer(message):
-                    print_status('Incoming call: received an offer, creating pipeline')
-                    self.start_pipeline(create_offer=False)
                 self.handle_json(message)
         self.close_pipeline()
         return 0