From 1c54c77840eabd2edb43d7f190e318671c276d73 Mon Sep 17 00:00:00 2001 From: Arun Raghavan Date: Wed, 3 Apr 2024 11:31:55 -0400 Subject: [PATCH] webrtcsink: Add a mechanism for SDP munging Unfortunately, server implementations might have odd SDP-related quirks, so let's allow clients a way to work around these oddities themselves. For now, this means that a client can fix up the H.264 profile-level-id as required by Twitch (whose media pipeline is more permissive than the WHIP implementation). Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/516 Part-of: --- docs/plugins/gst_plugins_cache.json | 14 +++++ net/webrtc/src/aws_kvs_signaller/imp.rs | 7 +++ net/webrtc/src/janusvr_signaller/imp.rs | 1 + net/webrtc/src/livekit_signaller/imp.rs | 7 +++ net/webrtc/src/signaller/iface.rs | 70 +++++++++++++++++++++++++ net/webrtc/src/signaller/imp.rs | 7 +++ net/webrtc/src/webrtcsink/imp.rs | 26 ++++++++- net/webrtc/src/whip_signaller/imp.rs | 21 ++++++-- 8 files changed, 148 insertions(+), 5 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 32f21b3d..8ccf400a 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -10015,6 +10015,20 @@ "return-type": "void", "when": "last" }, + "munge-session-description": { + "args": [ + { + "name": "arg0", + "type": "gchararray" + }, + { + "name": "arg1", + "type": "GstWebRTCSessionDescription" + } + ], + "return-type": "GstWebRTCSessionDescription", + "when": "last" + }, "producer-added": { "args": [ { diff --git a/net/webrtc/src/aws_kvs_signaller/imp.rs b/net/webrtc/src/aws_kvs_signaller/imp.rs index 4f42fa8c..1c476cf8 100644 --- a/net/webrtc/src/aws_kvs_signaller/imp.rs +++ b/net/webrtc/src/aws_kvs_signaller/imp.rs @@ -651,6 +651,12 @@ impl ObjectImpl for Signaller { fn properties() -> &'static [glib::ParamSpec] { static PROPERTIES: Lazy> = Lazy::new(|| { vec![ + glib::ParamSpecBoolean::builder("manual-sdp-munging") + .nick("Manual SDP munging") + .blurb("Whether the signaller manages SDP munging itself") + .default_value(false) + .read_only() + .build(), glib::ParamSpecString::builder("address") .nick("Address") .blurb("Address of the signalling server") @@ -721,6 +727,7 @@ impl ObjectImpl for Signaller { fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { match pspec.name() { + "manual-sdp-munging" => false.to_value(), "address" => self.settings.lock().unwrap().address.to_value(), "cafile" => { let settings = self.settings.lock().unwrap(); diff --git a/net/webrtc/src/janusvr_signaller/imp.rs b/net/webrtc/src/janusvr_signaller/imp.rs index c9afcad1..e18713e7 100644 --- a/net/webrtc/src/janusvr_signaller/imp.rs +++ b/net/webrtc/src/janusvr_signaller/imp.rs @@ -295,6 +295,7 @@ impl Default for Settings { #[properties(wrapper_type = super::JanusVRSignaller)] pub struct Signaller { state: Mutex, + #[property(name="manual-sdp-munging", default = false, get = |_| false, type = bool, blurb = "Whether the signaller manages SDP munging itself")] #[property(name="janus-endpoint", get, set, type = String, member = janus_endpoint, blurb = "The Janus server endpoint to POST SDP offer to")] #[property(name="display-name", get, set, type = String, member = display_name, blurb = "The name of the publisher in the Janus Video Room")] #[property(name="secret-key", get, set, type = String, member = secret_key, blurb = "The secret API key to communicate with Janus server")] diff --git a/net/webrtc/src/livekit_signaller/imp.rs b/net/webrtc/src/livekit_signaller/imp.rs index dd565eea..9946c4ce 100644 --- a/net/webrtc/src/livekit_signaller/imp.rs +++ b/net/webrtc/src/livekit_signaller/imp.rs @@ -777,6 +777,12 @@ impl ObjectImpl for Signaller { fn properties() -> &'static [glib::ParamSpec] { static PROPERTIES: Lazy> = Lazy::new(|| { vec![ + glib::ParamSpecBoolean::builder("manual-sdp-munging") + .nick("Manual SDP munging") + .blurb("Whether the signaller manages SDP munging itself") + .default_value(false) + .read_only() + .build(), glib::ParamSpecString::builder("ws-url") .nick("WebSocket URL") .blurb("The URL of the websocket of the LiveKit server") @@ -897,6 +903,7 @@ impl ObjectImpl for Signaller { fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { let settings = self.settings.lock().unwrap(); match pspec.name() { + "manual-sdp-munging" => false.to_value(), "ws-url" => settings.wsurl.to_value(), "api-key" => settings.api_key.to_value(), "secret-key" => settings.secret_key.to_value(), diff --git a/net/webrtc/src/signaller/iface.rs b/net/webrtc/src/signaller/iface.rs index 8a9cb1d5..a2cd9507 100644 --- a/net/webrtc/src/signaller/iface.rs +++ b/net/webrtc/src/signaller/iface.rs @@ -58,6 +58,19 @@ impl prelude::ObjectInterface for Signallable { iface.end_session = Signallable::end_session; } + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![glib::ParamSpecBoolean::builder("manual-sdp-munging") + .nick("Manual SDP munging") + .blurb("Whether the signaller manages SDP munging itself") + .default_value(false) + .read_only() + .build()] + }); + + PROPERTIES.as_ref() + } + fn signals() -> &'static [Signal] { static SIGNALS: Lazy> = Lazy::new(|| { vec![ @@ -309,6 +322,47 @@ impl prelude::ObjectInterface for Signallable { Signal::builder("consumer-removed") .param_types([String::static_type(), gst::Element::static_type()]) .build(), + /** + * GstRSWebRTCSignallableIface::munge-session-description: + * @self: The object implementing #GstRSWebRTCSignallableIface + * @session_id: Id of the session being described + * @description: The WebRTC session description to modify + * + * For special-case handling, a callback can be registered to modify the session + * description before the signaller sends it to the peer. Only the first connected + * handler has any effect. + * + * Return: A modified session description + */ + Signal::builder("munge-session-description") + .run_last() + .param_types([ + str::static_type(), + gst_webrtc::WebRTCSessionDescription::static_type(), + ]) + .return_type::() + .class_handler(|_tokens, args| { + let _ = args[0usize] + .get::<&super::Signallable>() + .unwrap_or_else(|e| { + panic!("Wrong type for argument {}: {:?}", 0usize, e) + }); + let _ = args[1usize].get::<&str>().unwrap_or_else(|e| { + panic!("Wrong type for argument {}: {:?}", 1usize, e) + }); + let desc = args[2usize] + .get::<&gst_webrtc::WebRTCSessionDescription>() + .unwrap_or_else(|e| { + panic!("Wrong type for argument {}: {:?}", 2usize, e) + }); + + Some(desc.clone().into()) + }) + .accumulator(move |_hint, output, input| { + *output = input.clone(); + false + }) + .build(), /** * GstRSWebRTCSignallableIface::send-session-description: * @self: The object implementing #GstRSWebRTCSignallableIface @@ -499,6 +553,11 @@ pub trait SignallableImpl: object::ObjectImpl + Send + Sync + 'static { pub trait SignallableExt: 'static { fn start(&self); fn stop(&self); + fn munge_sdp( + &self, + session_id: &str, + sdp: &gst_webrtc::WebRTCSessionDescription, + ) -> gst_webrtc::WebRTCSessionDescription; fn send_sdp(&self, session_id: &str, sdp: &gst_webrtc::WebRTCSessionDescription); fn add_ice( &self, @@ -519,6 +578,17 @@ impl> SignallableExt for Obj { self.emit_by_name::("stop", &[]); } + fn munge_sdp( + &self, + session_id: &str, + sdp: &gst_webrtc::WebRTCSessionDescription, + ) -> gst_webrtc::WebRTCSessionDescription { + self.emit_by_name::( + "munge-session-description", + &[&session_id, sdp], + ) + } + fn send_sdp(&self, session_id: &str, sdp: &gst_webrtc::WebRTCSessionDescription) { self.emit_by_name::("send-session-description", &[&session_id, sdp]); } diff --git a/net/webrtc/src/signaller/imp.rs b/net/webrtc/src/signaller/imp.rs index 541bfb84..87636c19 100644 --- a/net/webrtc/src/signaller/imp.rs +++ b/net/webrtc/src/signaller/imp.rs @@ -509,6 +509,12 @@ impl ObjectImpl for Signaller { fn properties() -> &'static [glib::ParamSpec] { static PROPS: Lazy> = Lazy::new(|| { vec![ + glib::ParamSpecBoolean::builder("manual-sdp-munging") + .nick("Manual SDP munging") + .blurb("Whether the signaller manages SDP munging itself") + .default_value(false) + .read_only() + .build(), glib::ParamSpecString::builder("uri") .nick("Signaller URI") .blurb("URI for connecting to the signaller server") @@ -604,6 +610,7 @@ impl ObjectImpl for Signaller { fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { let settings = self.settings.lock().unwrap(); match pspec.name() { + "manual-sdp-munging" => false.to_value(), "uri" => settings.uri.to_string().to_value(), "producer-peer-id" => { if !matches!(settings.role, WebRTCSignallerRole::Consumer) { diff --git a/net/webrtc/src/webrtcsink/imp.rs b/net/webrtc/src/webrtcsink/imp.rs index ad2f102c..2b8baa09 100644 --- a/net/webrtc/src/webrtcsink/imp.rs +++ b/net/webrtc/src/webrtcsink/imp.rs @@ -2169,7 +2169,17 @@ impl BaseWebRTCSink { .emit_by_name::<()>("set-local-description", &[&offer, &None::]); drop(state); - signaller.send_sdp(session_id, &offer); + let maybe_munged_offer = if signaller + .has_property("manual-sdp-munging", Some(bool::static_type())) + && signaller.property("manual-sdp-munging") + { + // Don't munge, signaller will manage this + offer + } else { + // Use the default munging mechanism (signal registered by user) + signaller.munge_sdp(session_id, &offer) + }; + signaller.send_sdp(session_id, &maybe_munged_offer); } } @@ -2213,7 +2223,19 @@ impl BaseWebRTCSink { } drop(state); - signaller.send_sdp(&session_id, &answer); + + let maybe_munged_answer = if signaller + .has_property("manual-sdp-munging", Some(bool::static_type())) + && signaller.property("manual-sdp-munging") + { + // Don't munge, signaller will manage this + answer + } else { + // Use the default munging mechanism (signal registered by user) + signaller.munge_sdp(session_id.as_str(), &answer) + }; + + signaller.send_sdp(&session_id, &maybe_munged_answer); self.on_remote_description_set(element, session_id) } diff --git a/net/webrtc/src/whip_signaller/imp.rs b/net/webrtc/src/whip_signaller/imp.rs index f5317f47..1ab9b53e 100644 --- a/net/webrtc/src/whip_signaller/imp.rs +++ b/net/webrtc/src/whip_signaller/imp.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MPL-2.0 -use crate::signaller::{Signallable, SignallableImpl}; +use crate::signaller::{Signallable, SignallableExt, SignallableImpl}; use crate::utils::{ build_link_header, build_reqwest_client, parse_redirect_location, set_ice_servers, wait, wait_async, WaitError, @@ -118,7 +118,7 @@ impl WhipClient { self.raise_error("Local description is not set".to_string()); return; } - Some(offer) => offer, + Some(offer) => self.obj().munge_sdp("unique", &offer), }; gst::debug!( @@ -533,7 +533,14 @@ impl ObjectSubclass for WhipClient { impl ObjectImpl for WhipClient { fn properties() -> &'static [glib::ParamSpec] { static PROPERTIES: Lazy> = Lazy::new(|| { - vec![glib::ParamSpecString::builder("whip-endpoint") + vec![ + glib::ParamSpecBoolean::builder("manual-sdp-munging") + .nick("Manual SDP munging") + .blurb("Whether the signaller manages SDP munging itself") + .default_value(false) + .read_only() + .build(), + glib::ParamSpecString::builder("whip-endpoint") .nick("WHIP Endpoint") .blurb("The WHIP server endpoint to POST SDP offer to. e.g.: https://example.com/whip/endpoint/room1234") @@ -590,6 +597,7 @@ impl ObjectImpl for WhipClient { fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { match pspec.name() { + "manual-sdp-munging" => true.to_value(), "whip-endpoint" => { let settings = self.settings.lock().unwrap(); settings.whip_endpoint.to_value() @@ -1079,6 +1087,12 @@ impl ObjectImpl for WhipServer { fn properties() -> &'static [glib::ParamSpec] { static PROPERTIES: Lazy> = Lazy::new(|| { vec![ + glib::ParamSpecBoolean::builder("manual-sdp-munging") + .nick("Manual SDP munging") + .blurb("Whether the signaller manages SDP munging itself") + .default_value(false) + .read_only() + .build(), glib::ParamSpecString::builder("host-addr") .nick("Host address") .blurb("The the host address of the WHIP endpoint e.g., http://127.0.0.1:8080") @@ -1141,6 +1155,7 @@ impl ObjectImpl for WhipServer { fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { let settings = self.settings.lock().unwrap(); match pspec.name() { + "manual-sdp-munging" => false.to_value(), "host-addr" => settings.host_addr.to_string().to_value(), "stun-server" => settings.stun_server.to_value(), "turn-servers" => settings.turn_servers.to_value(),