gstwebrtc-api example: add support for requesting mix matrix

This is one example of how a consumer might send over custom upstream
event requests to the producer.

As webrtcsink will deserialize numbers in priority as integers, we need
a custom stringifying function to ensure members of the matrix array are
indeed serialized with the floating point.

An optional stringifier parameter is thus added to the
sendControlRequest API.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1711>

docs/plugins/gst_plugins_cache.json

@@ -10335,6 +10335,18 @@
                         "type": "gboolean",
                         "writable": true
                     },
+                    "enable-control-data-channel": {
+                        "blurb": "Enable receiving arbitrary events through data channel",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "false",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "gboolean",
+                        "writable": true
+                    },
                     "enable-data-channel-navigation": {
                         "blurb": "Enable navigation events through a dedicated WebRTCDataChannel",
                         "conditionally-available": false,
@@ -10685,6 +10697,18 @@
                         "type": "gboolean",
                         "writable": true
                     },
+                    "enable-control-data-channel": {
+                        "blurb": "Enable sending control requests through a dedicated WebRTCDataChannel",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "false",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "gboolean",
+                        "writable": true
+                    },
                     "enable-data-channel-navigation": {
                         "blurb": "Enable navigation events through a dedicated WebRTCDataChannel",
                         "conditionally-available": false,

net/webrtc/gstwebrtc-api/index.html

@@ -82,7 +82,7 @@
       outline: none;
     }
 
-    div.video, div.offer-options {
+    div.video, div.offer-options, div.mix-matrix {
       position: relative;
       margin: 1em;
     }
@@ -299,6 +299,52 @@
       });
     }
 
+    const beginFloat = "~begin~float~";
+    const endFloat = "~end~float~";
+
+    // Count the number of nested elements in array
+    function deepCount(arr = []) {
+       return arr
+       .reduce((acc, val) => {
+          return acc + (Array.isArray(val) ? deepCount(val) : 0);
+       }, arr.length);
+    };
+
+    // Adapted from https://www.npmjs.com/package/stringify-with-floats in order to
+    // target the mix matrix and make sure the JSON output contains floating point
+    // numbers no matter what
+    function stringifyMatrix(inputValue) {
+      let elementsInMatrix = 0;
+      const jsonReplacer = (key, val) => {
+        let value;
+        let inMatrix = false;
+
+        value = val;
+        if (key == "matrix") {
+          elementsInMatrix = deepCount(val);
+        } else if (elementsInMatrix > 0) {
+          elementsInMatrix--;
+          inMatrix = true;
+        }
+        const forceFloat =
+          inMatrix == true &&
+          (value || value === 0) &&
+          typeof value === "number" &&
+          !value.toString().toLowerCase().includes("e");
+        return forceFloat ? `${beginFloat}${value}${endFloat}` : value;
+      };
+      const json = JSON.stringify(inputValue, jsonReplacer, 2);
+      const regexReplacer = (match, num) => {
+        return num.includes(".") || Number.isNaN(num)
+          ? Number.isNaN(num)
+            ? num
+            : Number(num).toFixed(1)
+          : `${num}.${"0".repeat(1)}`;
+      };
+      const re = new RegExp(`"${beginFloat}(.+?)${endFloat}"`, "g");
+      return json.replace(re, regexReplacer);
+    };
+
     function initRemoteStreams(api) {
       const remoteStreamsElement = document.getElementById("remote-streams");
 
@@ -313,6 +359,10 @@
                                 <div class="offer-options">
                                   <textarea rows="5" cols="50" placeholder="offer options, empty to answer. For example:\n{\n  &quot;offerToReceiveAudio&quot;: 1\n  &quot;offerToReceiveVideo&quot;: 1\n}\n"></textarea>
                                 </div>
+                                <div class="mix-matrix">
+                                  <textarea rows="4" cols="50" placeholder="Mix matrix, if any, for example [[1.0, 0.0], [0.0, 1.0]]"></textarea>
+                                  <button disabled="disabled">Submit mix matrix</button>
+                                </div>
                                 <div class="video">
                                     <div class="spinner">
                                         <div></div>
@@ -328,7 +378,25 @@
 
             const entryElement = document.getElementById(producerId);
             const videoElement = entryElement.getElementsByTagName("video")[0];
-            const textareaElement = entryElement.getElementsByTagName("textarea")[0];
+            const offerTextareaElement = entryElement.getElementsByTagName("textarea")[0];
+            const mixmatrixTextAreaElement = entryElement.getElementsByTagName("textarea")[1];
+            const submitMixmatrixButtonElement = entryElement.getElementsByTagName("button")[0];
+
+            submitMixmatrixButtonElement.addEventListener("click", (event) => {
+              try {
+                let matrix = JSON.parse(mixmatrixTextAreaElement.value);
+                let id = entryElement._consumerSession.remoteController.sendControlRequest({
+                  type: "customUpstreamEvent",
+                  structureName: "GstRequestMixMatrix",
+                  structure: {
+                    matrix: matrix
+                  }
+                }, stringifyMatrix);
+              } catch (ex) {
+                console.error("Failed to parse mix matrix:", ex);
+                return;
+              }
+            });
 
             videoElement.addEventListener("playing", () => {
               if (entryElement.classList.contains("has-session")) {
@@ -346,12 +414,11 @@
                 entryElement._consumerSession.close();
               } else {
                 let session = null;
-                if (textareaElement.value == '') {
+                if (offerTextareaElement.value == '') {
                   session = api.createConsumerSession(producerId);
                 } else {
                   try {
-                    let offerOptions = JSON.parse(textareaElement.value);
+                    let offerOptions = JSON.parse(offerTextareaElement.value);
-                    console.log("Offer options: ", offerOptions);
                     session = api.createConsumerSessionWithOfferOptions(producerId, offerOptions);
                   } catch (ex) {
                     console.error("Failed to parse offer options:", ex);
@@ -391,9 +458,18 @@
                       const remoteController = session.remoteController;
                       if (remoteController) {
                         entryElement.classList.add("has-remote-control");
+                        submitMixmatrixButtonElement.disabled = false;
                         remoteController.attachVideoElement(videoElement);
+                        let id = remoteController.sendControlRequest({
+                          type: "customUpstreamEvent",
+                          structureName: "GstRequestMixMatrix",
+                          structure: {
+                            matrix: [[1.0, 0.0], [0.0, 1.0]]
+                          }
+                        });
                       } else {
                         entryElement.classList.remove("has-remote-control");
+                        submitMixmatrixButtonElement.disabled = true;
                       }
                     }
                   });

net/webrtc/gstwebrtc-api/src/index.js

@@ -48,6 +48,10 @@ import GstWebRTCAPI from "./gstwebrtc-api.js";
  * @external HTMLVideoElement
  * @see https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement
  */
+/**
+ * @external JSON.stringify
+ * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify
+ */
 
 if (!window.GstWebRTCAPI) {
   window.GstWebRTCAPI = GstWebRTCAPI;

net/webrtc/gstwebrtc-api/src/remote-controller.js

@@ -203,10 +203,18 @@ export default class RemoteController extends EventTarget {
      * @fires {@link GstWebRTCAPI#event:ErrorEvent}
      * @param {object} request - The request to stringify and send over the channel
      * @param {string} request.type - The type of the request
+     * @param {function} stringifier - An optional callback for stringifying,
+     * {@link external:JSON.stringify} will be used otherwise.
      * @returns {number} The identifier attributed to the request, or -1 if an exception occurred
      */
-  sendControlRequest(request) {
+  sendControlRequest(request, stringifier) {
     try {
+      if (stringifier && (typeof(stringifier) !== "function")) {
+        throw new Error("invalid stringifier");
+      } else if (!stringifier) {
+        stringifier = JSON.stringify;
+      }
+
       if (!request || (typeof (request) !== "object")) {
         throw new Error("invalid request");
       }
@@ -220,7 +228,7 @@ export default class RemoteController extends EventTarget {
         request: request
       };
 
-      this._rtcDataChannel.send(JSON.stringify(message));
+      this._rtcDataChannel.send(stringifier(message));
 
       return message.id;
     } catch (ex) {

net/webrtc/src/webrtcsink/imp.rs

@@ -553,9 +553,7 @@ fn create_navigation_event(sink: &super::BaseWebRTCSink, msg: &str) {
 
 fn deserialize_serde_value(val: &serde_json::Value) -> Result<gst::glib::SendValue, Error> {
     match val {
-        serde_json::Value::Null => {
+        serde_json::Value::Null => Err(anyhow!("Untyped null values are not handled")),
-            return Err(anyhow!("Untyped null values are not handled"));
-        }
         serde_json::Value::Bool(v) => Ok(v.to_send_value()),
         serde_json::Value::Number(v) => {
             if let Some(v) = v.as_i64() {
@@ -573,7 +571,7 @@ fn deserialize_serde_value(val: &serde_json::Value) -> Result<gst::glib::SendVal
             let mut gst_array = gst::Array::default();
 
             for val in a {
-                gst_array.append_value(deserialize_serde_value(&val)?);
+                gst_array.append_value(deserialize_serde_value(val)?);
             }
 
             Ok(gst_array.to_send_value())