webrtcsink: add signal to configure mitigation modes

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/2279>

docs/plugins/gst_plugins_cache.json

@@ -13886,6 +13886,18 @@
                         "type": "gboolean",
                         "writable": true
                     },
+                    "enable-mitigation-modes": {
+                        "blurb": "Flags for whether the element should dynamically scale the source resolution and framerate based on the bitrate",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "downsampled+downscaled",
+                        "mutable": "playing",
+                        "readable": true,
+                        "type": "GstWebRTCSinkMitigationMode",
+                        "writable": true
+                    },
                     "forward-metas": {
                         "blurb": "Comma-separated list of buffer meta names to forward over the control data channel. Currently supported names are: timecode",
                         "conditionally-available": false,
@@ -14095,6 +14107,28 @@
                     }
                 },
                 "signals": {
+                    "configure-mitigation-caps": {
+                        "args": [
+                            {
+                                "name": "arg0",
+                                "type": "gchararray"
+                            },
+                            {
+                                "name": "arg1",
+                                "type": "GstVideoInfo"
+                            },
+                            {
+                                "name": "arg2",
+                                "type": "gint"
+                            },
+                            {
+                                "name": "arg3",
+                                "type": "GstWebRTCSinkMitigationMode"
+                            }
+                        ],
+                        "return-type": "GstCaps",
+                        "when": "last"
+                    },
                     "consumer-added": {
                         "args": [
                             {
@@ -14742,6 +14776,26 @@
                     }
                 ]
             },
+            "GstWebRTCSinkMitigationMode": {
+                "kind": "flags",
+                "values": [
+                    {
+                        "desc": "No mitigation applied",
+                        "name": "none",
+                        "value": "0x00000000"
+                    },
+                    {
+                        "desc": "Lowered resolution",
+                        "name": "downscaled",
+                        "value": "0x00000001"
+                    },
+                    {
+                        "desc": "Lowered framerate",
+                        "name": "downsampled",
+                        "value": "0x00000002"
+                    }
+                ]
+            },
             "GstWebRTCSinkPad": {
                 "hierarchy": [
                     "GstWebRTCSinkPad",

net/webrtc/src/webrtcsink/homegrown_cc.rs

@@ -1,13 +1,13 @@
 // SPDX-License-Identifier: MPL-2.0
 
+use super::imp::VideoEncoder;
+use crate::webrtcsink::WebRTCSinkMitigationMode;
 use gst::{
     glib::{self, value::FromValue},
     prelude::*,
 };
 use std::sync::LazyLock;
 
-use super::imp::VideoEncoder;
-
 static CAT: LazyLock<gst::DebugCategory> = LazyLock::new(|| {
     gst::DebugCategory::new(
         "webrtcsink-homegrowncc",
@@ -413,7 +413,10 @@ impl CongestionController {
         let fec_percentage = (fec_ratio * 50f64) as u32;
 
         for encoder in encoders.iter_mut() {
-            if encoder.set_bitrate(element, target_bitrate).is_ok() {
+            if encoder
+                .set_bitrate(element, target_bitrate, WebRTCSinkMitigationMode::all())
+                .is_ok()
+            {
                 encoder
                     .transceiver
                     .set_property("fec-percentage", fec_percentage);

net/webrtc/src/webrtcsink/imp.rs

@@ -12,7 +12,7 @@ use gst_plugin_webrtc_signalling::server::{Server, ServerError};
 use gst_rtp::prelude::*;
 use gst_utils::StreamProducer;
 use gst_video::subclass::prelude::*;
-use gst_video::VideoMultiviewMode;
+use gst_video::{VideoInfo, VideoMultiviewMode};
 use gst_webrtc::{WebRTCDataChannel, WebRTCICETransportPolicy};
 use tokio::io::AsyncReadExt;
 use tokio::net::TcpListener;
@@ -85,6 +85,7 @@ const DEFAULT_WEB_SERVER_DIRECTORY: &str = "gstwebrtc-api/dist";
 #[cfg(feature = "web_server")]
 const DEFAULT_WEB_SERVER_HOST_ADDR: &str = "http://127.0.0.1:8080";
 const DEFAULT_FORWARD_METAS: &str = "";
+const DEFAULT_ENABLE_MITIGATION_MODES: WebRTCSinkMitigationMode = WebRTCSinkMitigationMode::all();
 /* Start adding some FEC when the bitrate > 2Mbps as we found experimentally
  * that it is not worth it below that threshold */
 const DO_FEC_THRESHOLD: u32 = 2000000;
@@ -126,9 +127,11 @@ struct Settings {
     #[cfg(feature = "web_server")]
     web_server_host_addr: url::Url,
     forward_metas: HashSet<String>,
+    enabled_mitigation_modes: WebRTCSinkMitigationMode,
 }
 
 use std::sync::atomic::{AtomicU32, Ordering};
+
 static BD_SEQ: AtomicU32 = AtomicU32::new(0);
 fn get_bdseq() -> u32 {
     BD_SEQ.fetch_and(1, Ordering::SeqCst) + 1
@@ -329,6 +332,7 @@ struct SessionInner {
 
     navigation_handler: Option<NavigationEventHandler>,
     control_events_handler: Option<ControlRequestHandler>,
+    enabled_mitigation_modes: WebRTCSinkMitigationMode,
 }
 
 #[derive(Clone)]
@@ -552,6 +556,7 @@ impl Default for Settings {
             #[cfg(feature = "web_server")]
             web_server_host_addr: url::Url::parse(DEFAULT_WEB_SERVER_HOST_ADDR).unwrap(),
             forward_metas: HashSet::new(),
+            enabled_mitigation_modes: DEFAULT_ENABLE_MITIGATION_MODES,
         }
     }
 }
@@ -1007,6 +1012,43 @@ impl PayloadChainBuilder {
     }
 }
 
+fn default_configure_mitigation_mode(
+    video_info: &VideoInfo,
+    bitrate: i32,
+    enabled_mitigation_modes: WebRTCSinkMitigationMode,
+) -> gst::Caps {
+    let mut s = gst::Structure::new_empty("video/x-raw");
+    if enabled_mitigation_modes.is_empty() {
+        return gst::Caps::builder_full_with_any_features()
+            .structure(s)
+            .build();
+    }
+
+    if bitrate < 500000 && enabled_mitigation_modes.contains(WebRTCSinkMitigationMode::DOWNSAMPLED)
+    {
+        let scaled_framerate = video_info.fps().mul(gst::Fraction::new(1, 2));
+        if scaled_framerate.numer() != 0 {
+            s.set("framerate", scaled_framerate);
+        }
+    }
+
+    if enabled_mitigation_modes.contains(WebRTCSinkMitigationMode::DOWNSCALED) {
+        let height = match bitrate {
+            b if b < 1_000_000 => 360,
+            b if b < 2_000_000 => 720,
+            _ => video_info.height() as i32,
+        };
+        let width = VideoEncoder::scale_height_round_2(video_info, height);
+
+        s.set("height", height);
+        s.set("width", width);
+    }
+
+    gst::Caps::builder_full_with_any_features()
+        .structure(s)
+        .build()
+}
+
 impl VideoEncoder {
     fn new(
         encoding_elements: &EncodingChain,
@@ -1070,11 +1112,11 @@ impl VideoEncoder {
         Ok(bitrate)
     }
 
-    fn scale_height_round_2(&self, height: i32) -> i32 {
+    fn scale_height_round_2(video_info: &VideoInfo, height: i32) -> i32 {
         let ratio = gst_video::calculate_display_ratio(
-            self.video_info.width(),
+            video_info.width(),
-            self.video_info.height(),
+            video_info.height(),
-            self.video_info.par(),
+            video_info.par(),
             gst::Fraction::new(1, 1),
         )
         .unwrap();
@@ -1088,6 +1130,7 @@ impl VideoEncoder {
         &mut self,
         element: &super::BaseWebRTCSink,
         bitrate: i32,
+        enabled_mitigation_modes: WebRTCSinkMitigationMode,
     ) -> Result<(), WebRTCSinkError> {
         match self.factory_name.as_str() {
             "vp8enc" | "vp9enc" => self.element.set_property("target-bitrate", bitrate),
@@ -1107,65 +1150,43 @@ impl VideoEncoder {
         }
 
         let current_caps = self.filter.property::<gst::Caps>("caps");
-        let mut s = current_caps.structure(0).unwrap().to_owned();
 
-        // Hardcoded thresholds, may be tuned further in the future, and
+        let mitigation_mode_caps = element.emit_by_name::<gst::Caps>(
-        // adapted according to the codec in use
+            "configure-mitigation-caps",
-        if bitrate < 500000 {
+            &[
-            let height = 360i32.min(self.video_info.height() as i32);
+                &self.stream_name,
-            let width = self.scale_height_round_2(height);
+                &self.video_info,
+                &bitrate,
+                &enabled_mitigation_modes,
+            ],
+        );
+        let mitigation_mode_caps_structure = mitigation_mode_caps.structure(0).unwrap().to_owned();
 
-            s.set("height", height);
+        self.mitigation_mode = WebRTCSinkMitigationMode::NONE;
-            s.set("width", width);
-
-            if self.halved_framerate.numer() != 0 {
-                s.set("framerate", self.halved_framerate);
-            }
-
-            self.mitigation_mode =
-                WebRTCSinkMitigationMode::DOWNSAMPLED | WebRTCSinkMitigationMode::DOWNSCALED;
-        } else if bitrate < 1000000 {
-            let height = 360i32.min(self.video_info.height() as i32);
-            let width = self.scale_height_round_2(height);
-
-            s.set("height", height);
-            s.set("width", width);
-            s.remove_field("framerate");
 
+        if mitigation_mode_caps_structure.get::<i32>("height").is_ok() {
             self.mitigation_mode = WebRTCSinkMitigationMode::DOWNSCALED;
-        } else if bitrate < 2000000 {
-            let height = 720i32.min(self.video_info.height() as i32);
-            let width = self.scale_height_round_2(height);
-
-            s.set("height", height);
-            s.set("width", width);
-            s.remove_field("framerate");
-
-            self.mitigation_mode = WebRTCSinkMitigationMode::DOWNSCALED;
-        } else {
-            s.remove_field("height");
-            s.remove_field("width");
-            s.remove_field("framerate");
-
-            self.mitigation_mode = WebRTCSinkMitigationMode::NONE;
         }
 
-        let caps = gst::Caps::builder_full_with_any_features()
+        if mitigation_mode_caps_structure
-            .structure(s)
+            .get::<gst::Fraction>("framerate")
-            .build();
+            .is_ok()
+        {
+            self.mitigation_mode |= WebRTCSinkMitigationMode::DOWNSAMPLED;
+        }
 
-        if !caps.is_strictly_equal(&current_caps) {
+        if !mitigation_mode_caps.is_strictly_equal(&current_caps) {
             gst::log!(
                 CAT,
                 obj = element,
                 "session {}: setting bitrate {} and caps {} on encoder {:?}",
                 self.session_id,
                 bitrate,
-                caps,
+                mitigation_mode_caps,
                 self.element
             );
 
-            self.filter.set_property("caps", caps);
+            self.filter.set_property("caps", mitigation_mode_caps);
         }
 
         Ok(())
@@ -1278,6 +1299,7 @@ impl State {
 }
 
 impl SessionInner {
+    #[allow(clippy::too_many_arguments)]
     fn new(
         id: String,
         pipeline: gst::Pipeline,
@@ -1286,6 +1308,7 @@ impl SessionInner {
         congestion_controller: Option<CongestionController>,
         rtpgccbwe: Option<gst::Element>,
         cc_info: CCInfo,
+        enabled_mitigation_modes: WebRTCSinkMitigationMode,
     ) -> Self {
         Self {
             id,
@@ -1306,6 +1329,7 @@ impl SessionInner {
             stats_collection_handle: None,
             navigation_handler: None,
             control_events_handler: None,
+            enabled_mitigation_modes,
         }
     }
 
@@ -1406,7 +1430,11 @@ impl SessionInner {
                     WebRTCSinkCongestionControl::Disabled => {
                         // If congestion control is disabled, we simply use the highest
                         // known "safe" value for the bitrate.
-                        let _ = enc.set_bitrate(element, self.cc_info.max_bitrate as i32);
+                        let _ = enc.set_bitrate(
+                            element,
+                            self.cc_info.max_bitrate as i32,
+                            self.enabled_mitigation_modes,
+                        );
                         enc.transceiver.set_property("fec-percentage", 50u32);
                     }
                     WebRTCSinkCongestionControl::Homegrown => {
@@ -1420,7 +1448,11 @@ impl SessionInner {
                         } else {
                             /* If congestion control is disabled, we simply use the highest
                              * known "safe" value for the bitrate. */
-                            let _ = enc.set_bitrate(element, self.cc_info.max_bitrate as i32);
+                            let _ = enc.set_bitrate(
+                                element,
+                                self.cc_info.max_bitrate as i32,
+                                self.enabled_mitigation_modes,
+                            );
                             enc.transceiver.set_property("fec-percentage", 50u32);
                         }
                     }
@@ -3113,6 +3145,7 @@ impl BaseWebRTCSink {
             },
             rtpgccbwe,
             settings.cc_info,
+            settings.enabled_mitigation_modes,
         );
 
         let rtpbin = webrtcbin
@@ -3538,7 +3571,11 @@ impl BaseWebRTCSink {
                     };
 
                 if encoder
-                    .set_bitrate(&self.obj(), defined_encoder_bitrate)
+                    .set_bitrate(
+                        &self.obj(),
+                        defined_encoder_bitrate,
+                        settings.enabled_mitigation_modes,
+                    )
                     .is_ok()
                 {
                     encoder
@@ -4714,6 +4751,21 @@ impl ObjectImpl for BaseWebRTCSink {
                     .default_value(DEFAULT_FORWARD_METAS)
                     .mutable_playing()
                     .build(),
+                /**
+                 * GstBaseWebRTCSink:enable-mitigation-modes:
+                 *
+                 * Whether the element should dynamically scale the source resolution
+                 * based on the bitrate.
+                 *
+                 * Since: plugins-rs-0.14.0
+                 */
+                glib::ParamSpecFlags::builder("enable-mitigation-modes")
+                    .nick("Enable dynamic scaling / sampling of the source resolution")
+                    .blurb("Flags for whether the element should dynamically scale the source resolution and framerate based on the bitrate")
+                    .default_value(DEFAULT_ENABLE_MITIGATION_MODES)
+                    .mutable_playing()
+                    .build(),
+
             ]
         });
 
@@ -4855,6 +4907,12 @@ impl ObjectImpl for BaseWebRTCSink {
                     .map(String::from)
                     .collect();
             }
+            "enable-mitigation-modes" => {
+                let mut settings = self.settings.lock().unwrap();
+                settings.enabled_mitigation_modes = value
+                    .get::<WebRTCSinkMitigationMode>()
+                    .expect("type checked upstream");
+            }
             _ => unimplemented!(),
         }
     }
@@ -4957,6 +5015,10 @@ impl ObjectImpl for BaseWebRTCSink {
                 let settings = self.settings.lock().unwrap();
                 settings.forward_metas.iter().join(",").to_value()
             }
+            "enable-mitigation-modes" => {
+                let settings = self.settings.lock().unwrap();
+                settings.enabled_mitigation_modes.to_value()
+            }
             _ => unimplemented!(),
         }
     }
@@ -5169,6 +5231,35 @@ impl ObjectImpl for BaseWebRTCSink {
                         std::ops::ControlFlow::Break(value.clone())
                     })
                     .build(),
+                /**
+                 * GstBaseWebRTCSink::configure-mitigation-caps:
+                 * @stream_name: name of the sink pad feeding the encoder
+                 * @current_bitrate: The current bitrate being applied to the encoder
+                 *
+                 *
+                 * Returns: the mitigation mode to apply.
+                 * Since: plugins-rs-0.14.0
+                 */
+                glib::subclass::Signal::builder("configure-mitigation-caps")
+                    .param_types([
+                        String::static_type(),
+                        VideoInfo::static_type(),
+                        i32::static_type(),
+                        WebRTCSinkMitigationMode::static_type(),
+                    ])
+                    .return_type::<gst::Caps>()
+                    .run_last()
+                    .class_handler(|args| {
+                        let video_info = args[2].get::<VideoInfo>().expect("No video info");
+                        let current_bitrate = args[3].get::<i32>().expect("No bitrate");
+                        let enabled_mitigation_modes = args[4].get::<WebRTCSinkMitigationMode>().expect("No enabled mitigation modes");
+
+                        Some(default_configure_mitigation_mode(&video_info, current_bitrate, enabled_mitigation_modes).to_value())
+                    })
+                    .accumulator(move |_hint, _acc, value| {
+                        std::ops::ControlFlow::Break(value.clone())
+                    })
+                    .build(),
             ]
         });
 

net/webrtc/src/webrtcsink/mod.rs

@@ -123,7 +123,7 @@ pub enum WebRTCSinkCongestionControl {
 }
 
 #[glib::flags(name = "GstWebRTCSinkMitigationMode")]
-enum WebRTCSinkMitigationMode {
+pub enum WebRTCSinkMitigationMode {
     #[flags_value(name = "No mitigation applied", nick = "none")]
     NONE = 0b00000000,
     #[flags_value(name = "Lowered resolution", nick = "downscaled")]
@@ -156,6 +156,7 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
     WebRTCSinkPad::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
     BaseWebRTCSink::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
     WebRTCSinkCongestionControl::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
+    WebRTCSinkMitigationMode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
     gst::Element::register(
         Some(plugin),
         "webrtcsink",