webrtc/js: Support renegotiation during a call correctly

When a video track is muted, hide the video element to differentiate
it from a track that is stuck because we stopped receiving RTP data.
Show it again when it is unmuted.

When a video track is removed, remove the video element. It will be
re-added on renegotiation.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5045>

subprojects/gst-examples/webrtc/sendrecv/js/webrtc.js

@@ -18,11 +18,16 @@ var rtc_configuration = {iceServers: [{urls: "stun:stun.l.google.com:19302"}]};
 var default_constraints = {video: true, audio: true};
 
 var connect_attempts = 0;
-var peer_connection;
+var peer_connection = new RTCPeerConnection(rtc_configuration);
 var send_channel;
 var ws_conn;
-// Promise for local stream after constraints are approved by the user
+// Local stream after constraints are approved by the user
-var local_stream_promise;
+var local_stream = null;
+
+// keep track of some negotiation state to prevent races and errors
+var callCreateTriggered = false;
+var makingOffer = false;
+var isSettingRemoteAnswerPending = false;
 
 function setConnectButtonState(value) {
     document.getElementById("peer-connect-button").value = value;
@@ -98,44 +103,68 @@ function setError(text) {
 
 function resetVideo() {
     // Release the webcam and mic
-    if (local_stream_promise)
+    if (local_stream) {
-        local_stream_promise.then(stream => {
+        local_stream.then(stream => {
             if (stream) {
                 stream.getTracks().forEach(function (track) { track.stop(); });
             }
         });
+        local_stream = null;
+    }
 
     // Remove all video players
     document.getElementById("video").innerHTML = "";
 }
 
-// SDP offer received from peer, set remote description and create an answer
 function onIncomingSDP(sdp) {
+    try {
+        // An offer may come in while we are busy processing SRD(answer).
+        // In this case, we will be in "stable" by the time the offer is processed
+        // so it is safe to chain it on our Operations Chain now.
+        const readyForOffer =
+            !makingOffer &&
+            (peer_connection.signalingState == "stable" || isSettingRemoteAnswerPending);
+        const offerCollision = sdp.type == "offer" && !readyForOffer;
+
+        if (offerCollision) {
+            return;
+        }
+        isSettingRemoteAnswerPending = sdp.type == "answer";
         peer_connection.setRemoteDescription(sdp).then(() => {
             setStatus("Remote SDP set");
-        if (sdp.type != "offer")
+            isSettingRemoteAnswerPending = false;
-            return;
+            if (sdp.type == "offer") {
-        setStatus("Got SDP offer");
+                setStatus("Got SDP offer, waiting for getUserMedia to complete");
-        local_stream_promise.then((stream) => {
+                local_stream.then((stream) => {
-            setStatus("Got local stream, creating answer");
+                    setStatus("getUserMedia to completed, setting local description");
-            peer_connection.createAnswer()
+                    peer_connection.setLocalDescription().then(() => {
-            .then(onLocalDescription).catch(setError);
+                        let desc = peer_connection.localDescription;
-        }).catch(setError);
-    }).catch(setError);
-}
-
-// Local description was set, send it to peer
-function onLocalDescription(desc) {
                         console.log("Got local description: " + JSON.stringify(desc));
-    peer_connection.setLocalDescription(desc).then(function() {
                         setStatus("Sending SDP " + desc.type);
-        sdp = {'sdp': peer_connection.localDescription}
+                        ws_conn.send(JSON.stringify({'sdp': desc}));
-        ws_conn.send(JSON.stringify(sdp));
+                        if (peer_connection.iceConnectionState == "connected") {
+                            setStatus("SDP " + desc.type + " sent, ICE connected, all looks OK");
+                        }
+                    });
                 });
             }
+        });
+    } catch (err) {
+        handleIncomingError(err);
+    }
+}
 
-function generateOffer() {
+// Local description was set by incoming SDP offer, send answer to peer
-    peer_connection.createOffer().then(onLocalDescription).catch(setError);
+function onLocalDescription(desc) {
+    if (desc.type != "answer") {
+        console.warn("Expected SDP answer, received: " + desc.type);
+    }
+    console.log("Got local description: " + JSON.stringify(desc));
+    peer_connection.setLocalDescription(desc).then(() => {
+        var dsc = peer_connection.localDescription;
+        setStatus("Sending SDP " + desc.type);
+        ws_conn.send(JSON.stringify({'sdp': desc}));
+    });
 }
 
 // ICE candidate received from peer, add it to the peer connection
@@ -157,13 +186,15 @@ function onServerMessage(event) {
                 setStatus("Sent OFFER_REQUEST, waiting for offer");
                 return;
             }
-            if (!peer_connection)
+            if (!callCreateTriggered) {
-                createCall(null).then (generateOffer);
+                createCall();
+                setStatus("Created peer connection for call, waiting for SDP");
+            }
             return;
         case "OFFER_REQUEST":
             // The peer wants us to set up and then send an offer
-            if (!peer_connection)
+            if (!callCreateTriggered)
-                createCall(null).then (generateOffer);
+                createCall();
             return;
         default:
             if (event.data.startsWith("ERROR")) {
@@ -183,7 +214,7 @@ function onServerMessage(event) {
             }
 
             // Incoming JSON signals the beginning of a call
-            if (!peer_connection)
+            if (!callCreateTriggered)
                 createCall(msg);
 
             if (msg.sdp != null) {
@@ -202,8 +233,9 @@ function onServerClose(event) {
 
     if (peer_connection) {
         peer_connection.close();
-        peer_connection = null;
+        peer_connection = new RTCPeerConnection(rtc_configuration);
     }
+    callCreateTriggered = false;
 
     // Reset after a second
     window.setTimeout(websocketServerConnect, 1000);
@@ -268,19 +300,14 @@ function websocketServerConnect() {
         ws_conn.send('HELLO ' + peer_id);
         setStatus("Registering with server");
         setConnectButtonState("Connect");
+        // Reset connection attempts because we connected successfully
+        connect_attempts = 0;
     });
     ws_conn.addEventListener('error', onServerError);
     ws_conn.addEventListener('message', onServerMessage);
     ws_conn.addEventListener('close', onServerClose);
 }
 
-function onRemoteTrack(event) {
-    var videoElem = getVideoElement();
-    if (event.track.kind === 'audio')
-        videoElem.style = 'display: none;';
-    videoElem.srcObject = new MediaStream([event.track]);
-}
-
 function errorUserMediaHandler() {
     setError("Browser doesn't support getUserMedia!");
 }
@@ -320,30 +347,36 @@ function onDataChannel(event) {
     receiveChannel.onclose = handleDataChannelClose;
 }
 
-function createCall(msg) {
+function createCall() {
-    // Reset connection attempts because we connected successfully
+    callCreateTriggered = true;
-    connect_attempts = 0;
+    console.log('Configuring RTCPeerConnection');
-
-    console.log('Creating RTCPeerConnection');
-
-    peer_connection = new RTCPeerConnection(rtc_configuration);
     send_channel = peer_connection.createDataChannel('label', null);
     send_channel.onopen = handleDataChannelOpen;
     send_channel.onmessage = handleDataChannelMessageReceived;
     send_channel.onerror = handleDataChannelError;
     send_channel.onclose = handleDataChannelClose;
     peer_connection.ondatachannel = onDataChannel;
-    peer_connection.ontrack = onRemoteTrack;
-    /* Send our video/audio to the other peer */
-    local_stream_promise = getLocalStream().then((stream) => {
-        console.log('Adding local stream');
-        peer_connection.addStream(stream);
-        return stream;
-    }).catch(setError);
 
-    if (msg != null && !msg.sdp) {
+    peer_connection.ontrack = ({track, streams}) => {
-        console.log("WARNING: First message wasn't an SDP message!?");
+        console.log("ontrack triggered");
-    }
+        var videoElem = getVideoElement();
+        if (event.track.kind === 'audio')
+            videoElem.style.display = 'none';
+
+        videoElem.srcObject = streams[0];
+        videoElem.srcObject.addEventListener('mute', (e) => {
+            console.log("track muted, hiding video element");
+            videoElem.style.display = 'none';
+        });
+        videoElem.srcObject.addEventListener('unmute', (e) => {
+            console.log("track unmuted, showing video element");
+            videoElem.style.display = 'block';
+        });
+        videoElem.srcObject.addEventListener('removetrack', (e) => {
+            console.log("track removed, removing video element");
+            videoElem.remove();
+        });
+    };
 
     peer_connection.onicecandidate = (event) => {
         // We have a candidate, send it to the remote party with the
@@ -354,10 +387,38 @@ function createCall(msg) {
         }
         ws_conn.send(JSON.stringify({'ice': event.candidate}));
     };
+    peer_connection.oniceconnectionstatechange = (event) => {
+        if (peer_connection.iceConnectionState == "connected") {
+            setStatus("ICE gathering complete");
+        }
+    };
 
-    if (msg != null)
+    // let the "negotiationneeded" event trigger offer generation
-        setStatus("Created peer connection for call, waiting for SDP");
+    peer_connection.onnegotiationneeded = async () => {
+        setStatus("Negotiation needed");
+        if (wantRemoteOfferer())
+            return;
+        try {
+            makingOffer = true;
+            await peer_connection.setLocalDescription();
+            let desc = peer_connection.localDescription;
+            setStatus("Sending SDP " + desc.type);
+            ws_conn.send(JSON.stringify({'sdp': desc}));
+        } catch (err) {
+            handleIncomingError(err);
+        } finally {
+            makingOffer = false;
+        }
+    };
+
+    /* Send our video/audio to the other peer */
+    local_stream = getLocalStream().then((stream) => {
+        console.log('Adding local stream');
+        for (const track of stream.getTracks()) {
+            peer_connection.addTrack(track, stream);
+        }
+        return stream;
+    }).catch(setError);
 
     setConnectButtonState("Disconnect");
-    return local_stream_promise;
 }