From 0007fa38e034e1acc401213320f4e4086db38de9 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 15 Mar 2022 16:31:56 +0530 Subject: [PATCH] webrtc-sendrecv: Fix create-answer caps negotiation We need to parse the payload type map provided by the offer SDP and set those values on the payloader, otherwise webrtcbin will create a recvonly answer SDP and we won't send anything to the browser. Fixed it for both C and Python sendrecv examples. Part-of: --- .../webrtc/sendrecv/gst/webrtc-sendrecv.c | 87 ++++++++++++++----- .../webrtc/sendrecv/gst/webrtc_sendrecv.py | 65 +++++++++----- 2 files changed, 108 insertions(+), 44 deletions(-) diff --git a/subprojects/gst-examples/webrtc/sendrecv/gst/webrtc-sendrecv.c b/subprojects/gst-examples/webrtc/sendrecv/gst/webrtc-sendrecv.c index b51a345361..edbaafac2a 100644 --- a/subprojects/gst-examples/webrtc/sendrecv/gst/webrtc-sendrecv.c +++ b/subprojects/gst-examples/webrtc/sendrecv/gst/webrtc-sendrecv.c @@ -301,10 +301,6 @@ on_negotiation_needed (GstElement * element, gpointer user_data) } } -#define STUN_SERVER " stun-server=stun://stun.l.google.com:19302 " -#define RTP_CAPS_OPUS "application/x-rtp,media=audio,encoding-name=OPUS,payload=" -#define RTP_CAPS_VP8 "application/x-rtp,media=video,encoding-name=VP8,payload=" - static void data_channel_on_error (GObject * dc, gpointer user_data) { @@ -421,16 +417,23 @@ webrtcbin_get_stats (GstElement * webrtcbin) return G_SOURCE_REMOVE; } + +#define STUN_SERVER " stun-server=stun://stun.l.google.com:19302 " #define RTP_TWCC_URI "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" +#define RTP_CAPS_OPUS "application/x-rtp,media=audio,encoding-name=OPUS" +#define RTP_OPUS_DEFAULT_PT 97 +#define RTP_CAPS_VP8 "application/x-rtp,media=video,encoding-name=VP8" +#define RTP_VP8_DEFAULT_PT 96 static gboolean -start_pipeline (gboolean create_offer) +start_pipeline (gboolean create_offer, guint opus_pt, guint vp8_pt) { + char *pipeline; GstStateChangeReturn ret; GError *error = NULL; - pipe1 = - gst_parse_launch ("webrtcbin bundle-policy=max-bundle name=sendrecv " + pipeline = + g_strdup_printf ("webrtcbin bundle-policy=max-bundle name=sendrecv " STUN_SERVER "videotestsrc is-live=true pattern=ball ! videoconvert ! queue ! " /* increase the default keyframe distance, browsers have really long @@ -441,10 +444,13 @@ start_pipeline (gboolean create_offer) /* picture-id-mode=15-bit seems to make TWCC stats behave better, and * fixes stuttery video playback in Chrome */ "rtpvp8pay name=videopay picture-id-mode=15-bit ! " - "queue ! " RTP_CAPS_VP8 "96 ! sendrecv. " + "queue ! %s,payload=%u ! sendrecv. " "audiotestsrc is-live=true wave=red-noise ! audioconvert ! audioresample ! queue ! opusenc ! rtpopuspay name=audiopay ! " - "queue ! " RTP_CAPS_OPUS "97 ! sendrecv. ", &error); + "queue ! %s,payload=%u ! sendrecv. ", RTP_CAPS_VP8, vp8_pt, + RTP_CAPS_OPUS, opus_pt); + pipe1 = gst_parse_launch (pipeline, &error); + g_free (pipeline); if (error) { gst_printerr ("Failed to parse launch: %s\n", error->message); g_error_free (error); @@ -454,7 +460,7 @@ start_pipeline (gboolean create_offer) webrtc1 = gst_bin_get_by_name (GST_BIN (pipe1), "sendrecv"); g_assert_nonnull (webrtc1); - if (remote_is_offerer) { + if (!create_offer) { /* XXX: this will fail when the remote offers twcc as the extension id * cannot currently be negotiated when receiving an offer. */ @@ -630,6 +636,50 @@ on_offer_received (GstSDPMessage * sdp) GstWebRTCSessionDescription *offer = NULL; GstPromise *promise; + /* If we got an offer and we have no webrtcbin, we need to parse the SDP, + * get the payload types, then start the pipeline */ + if (!webrtc1 && our_id) { + guint medias_len, formats_len; + guint opus_pt = 0, vp8_pt = 0; + + gst_println ("Parsing offer to find payload types"); + + medias_len = gst_sdp_message_medias_len (sdp); + for (int i = 0; i < medias_len; i++) { + const GstSDPMedia *media = gst_sdp_message_get_media (sdp, i); + formats_len = gst_sdp_media_formats_len (media); + for (int j = 0; j < formats_len; j++) { + guint pt; + GstCaps *caps; + GstStructure *s; + const char *fmt, *encoding_name; + + fmt = gst_sdp_media_get_format (media, j); + if (g_strcmp0 (fmt, "webrtc-datachannel") == 0) + continue; + pt = atoi (fmt); + caps = gst_sdp_media_get_caps_from_media (media, pt); + s = gst_caps_get_structure (caps, 0); + encoding_name = gst_structure_get_string (s, "encoding-name"); + if (vp8_pt == 0 && g_strcmp0 (encoding_name, "VP8") == 0) + vp8_pt = pt; + if (opus_pt == 0 && g_strcmp0 (encoding_name, "OPUS") == 0) + opus_pt = pt; + } + } + + g_assert_cmpint (opus_pt, !=, 0); + g_assert_cmpint (vp8_pt, !=, 0); + + gst_println ("Starting pipeline with opus pt: %u vp8 pt: %u", opus_pt, + vp8_pt); + + if (!start_pipeline (FALSE, opus_pt, vp8_pt)) { + cleanup_and_quit_loop ("ERROR: failed to start pipeline", + PEER_CALL_ERROR); + } + } + offer = gst_webrtc_session_description_new (GST_WEBRTC_SDP_TYPE_OFFER, sdp); g_assert_nonnull (offer); @@ -692,7 +742,7 @@ on_server_message (SoupWebsocketConnection * conn, SoupWebsocketDataType type, app_state = PEER_CONNECTED; /* Start negotiation (exchange SDP and ICE candidates) */ - if (!start_pipeline (TRUE)) + if (!start_pipeline (TRUE, RTP_OPUS_DEFAULT_PT, RTP_VP8_DEFAULT_PT)) cleanup_and_quit_loop ("ERROR: failed to start pipeline", PEER_CALL_ERROR); } else if (g_strcmp0 (text, "OFFER_REQUEST") == 0) { @@ -702,7 +752,7 @@ on_server_message (SoupWebsocketConnection * conn, SoupWebsocketDataType type, } gst_print ("Received OFFER_REQUEST, sending offer\n"); /* Peer wants us to start negotiation (exchange SDP and ICE candidates) */ - if (!start_pipeline (TRUE)) + if (!start_pipeline (TRUE, RTP_OPUS_DEFAULT_PT, RTP_VP8_DEFAULT_PT)) cleanup_and_quit_loop ("ERROR: failed to start pipeline", PEER_CALL_ERROR); } else if (g_str_has_prefix (text, "ERROR")) { @@ -743,17 +793,6 @@ on_server_message (SoupWebsocketConnection * conn, SoupWebsocketDataType type, goto out; } - /* If peer connection wasn't made yet and we are expecting peer will - * connect to us, launch pipeline at this moment */ - if (!webrtc1 && our_id) { - if (!start_pipeline (FALSE)) { - cleanup_and_quit_loop ("ERROR: failed to start pipeline", - PEER_CALL_ERROR); - } - - app_state = PEER_CALL_NEGOTIATING; - } - object = json_node_get_object (root); /* Check type of JSON message */ if (json_object_has_member (object, "sdp")) { @@ -762,7 +801,7 @@ on_server_message (SoupWebsocketConnection * conn, SoupWebsocketDataType type, const gchar *text, *sdptype; GstWebRTCSessionDescription *answer; - g_assert_cmphex (app_state, ==, PEER_CALL_NEGOTIATING); + app_state = PEER_CALL_NEGOTIATING; child = json_object_get_object_member (object, "sdp"); diff --git a/subprojects/gst-examples/webrtc/sendrecv/gst/webrtc_sendrecv.py b/subprojects/gst-examples/webrtc/sendrecv/gst/webrtc_sendrecv.py index b7c182f18c..900365db93 100755 --- a/subprojects/gst-examples/webrtc/sendrecv/gst/webrtc_sendrecv.py +++ b/subprojects/gst-examples/webrtc/sendrecv/gst/webrtc_sendrecv.py @@ -35,9 +35,9 @@ PIPELINE_DESC = ''' webrtcbin name=sendrecv bundle-policy=max-bundle stun-server=stun://stun.l.google.com:19302 videotestsrc is-live=true pattern=ball ! videoconvert ! queue ! \ vp8enc deadline=1 keyframe-max-dist=2000 ! rtpvp8pay picture-id-mode=15-bit ! - queue ! application/x-rtp,media=video,encoding-name=VP8,payload=97 ! sendrecv. + queue ! application/x-rtp,media=video,encoding-name=VP8,payload={vp8_pt} ! sendrecv. audiotestsrc is-live=true wave=red-noise ! audioconvert ! audioresample ! queue ! opusenc ! rtpopuspay ! - queue ! application/x-rtp,media=audio,encoding-name=OPUS,payload=96 ! sendrecv. + queue ! application/x-rtp,media=audio,encoding-name=OPUS,payload={opus_pt} ! sendrecv. ''' from websockets.version import version as wsv @@ -51,6 +51,33 @@ def print_error(msg): print(f'!!! {msg}', file=sys.stderr) +def get_payload_types(sdpmsg, video_encoding, audio_encoding): + ''' + Find the payload types for the specified video and audio encoding. + + Very simplistically finds the first payload type matching the encoding + name. More complex applications will want to match caps on + profile-level-id, packetization-mode, etc. + ''' + video_pt = None + audio_pt = None + for i in range(0, sdpmsg.medias_len()): + media = sdpmsg.get_media(i) + for j in range(0, media.formats_len()): + fmt = media.get_format(j) + if fmt == 'webrtc-datachannel': + continue + pt = int(fmt) + caps = media.get_caps_from_media(pt) + s = caps.get_structure(0) + encoding_name = s['encoding-name'] + if video_pt is None and encoding_name == video_encoding: + video_pt = pt + elif audio_pt is None and encoding_name == audio_encoding: + audio_pt = pt + return {video_encoding: video_pt, audio_encoding: audio_pt} + + class WebRTCClient: def __init__(self, loop, our_id, peer_id, server, remote_is_offerer): self.conn = None @@ -114,10 +141,6 @@ class WebRTCClient: print_status('Call was connected: creating offer') promise = Gst.Promise.new_with_change_func(self.on_offer_created, None, None) self.webrtc.emit('create-offer', None, promise) - elif self.remote_is_offerer: - # We are initiating the call, but we want the remote peer to create the offer - print_status('Call was connected: requesting remote peer for offer') - self.send_soon('OFFER_REQUEST') def send_ice_candidate_message(self, _, mlineindex, candidate): icemsg = json.dumps({'ice': {'candidate': candidate, 'sdpMLineIndex': mlineindex}}) @@ -167,9 +190,9 @@ class WebRTCClient: decodebin.sync_state_with_parent() self.webrtc.link(decodebin) - def start_pipeline(self, create_offer=True): + def start_pipeline(self, create_offer=True, opus_pt=96, vp8_pt=97): print_status(f'Creating pipeline, create_offer: {create_offer}') - self.pipe = Gst.parse_launch(PIPELINE_DESC) + self.pipe = Gst.parse_launch(PIPELINE_DESC.format(vp8_pt=vp8_pt, opus_pt=opus_pt)) self.webrtc = self.pipe.get_by_name('sendrecv') self.webrtc.connect('on-negotiation-needed', self.on_negotiation_needed, create_offer) self.webrtc.connect('on-ice-candidate', self.send_ice_candidate_message) @@ -192,7 +215,6 @@ class WebRTCClient: self.webrtc.emit('create-answer', None, promise) def handle_json(self, message): - assert (self.webrtc) try: msg = json.loads(message) except json.decoder.JSONDecoderError: @@ -212,10 +234,21 @@ class WebRTCClient: print_status('Received offer:\n%s' % sdp) res, sdpmsg = GstSdp.SDPMessage.new() GstSdp.sdp_message_parse_buffer(bytes(sdp.encode()), sdpmsg) + + if not self.webrtc: + print_status('Incoming call: received an offer, creating pipeline') + pts = get_payload_types(sdpmsg, video_encoding='VP8', audio_encoding='OPUS') + assert('VP8' in pts) + assert('OPUS' in pts) + self.start_pipeline(create_offer=False, vp8_pt=pts['VP8'], opus_pt=pts['OPUS']) + + assert(self.webrtc) + offer = GstWebRTC.WebRTCSessionDescription.new(GstWebRTC.WebRTCSDPType.OFFER, sdpmsg) promise = Gst.Promise.new_with_change_func(self.on_offer_set, None, None) self.webrtc.emit('set-remote-description', offer, promise) elif 'ice' in msg: + assert(self.webrtc) ice = msg['ice'] candidate = ice['candidate'] sdpmlineindex = ice['sdpMLineIndex'] @@ -229,13 +262,6 @@ class WebRTCClient: self.pipe = None self.webrtc = None - def is_incoming_offer(self, msg): - if self.webrtc: - return False - if self.remote_is_offerer: - return True - return True - async def loop(self): assert self.conn async for message in self.conn: @@ -254,7 +280,9 @@ class WebRTCClient: await self.setup_call() elif message == 'SESSION_OK': if self.remote_is_offerer: - self.start_pipeline(create_offer=False) + # We are initiating the call, but we want the remote peer to create the offer + print_status('Call was connected: requesting remote peer for offer') + await self.send('OFFER_REQUEST') else: self.start_pipeline() elif message == 'OFFER_REQUEST': @@ -265,9 +293,6 @@ class WebRTCClient: self.close_pipeline() return 1 else: - if self.is_incoming_offer(message): - print_status('Incoming call: received an offer, creating pipeline') - self.start_pipeline(create_offer=False) self.handle_json(message) self.close_pipeline() return 0