mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2025-09-02 17:53:48 +00:00
Code Issues Projects Releases Wiki Activity

webrtcsink: add signal to configure mitigation modes

Browse source 
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/2279>
This commit is contained in:
Jerome Colle 2025-06-03 13:52:32 +02:00
parent 0be4cd8b32
commit 388b442891
4 changed files with 204 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
docs/plugins/gst_plugins_cache.json
View file

@ -13886,6 +13886,18 @@
"type": "gboolean",
"writable": true
},
"enable-mitigation-modes": {
"blurb": "Flags for whether the element should dynamically scale the source resolution and framerate based on the bitrate",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "downsampled+downscaled",
"mutable": "playing",
"readable": true,
"type": "GstWebRTCSinkMitigationMode",
"writable": true
},
"forward-metas": {
"blurb": "Comma-separated list of buffer meta names to forward over the control data channel. Currently supported names are: timecode",
"conditionally-available": false,
@ -14095,6 +14107,28 @@
}
},
"signals": {
"configure-mitigation-caps": {
"args": [
{
"name": "arg0",
"type": "gchararray"
},
{
"name": "arg1",
"type": "GstVideoInfo"
},
{
"name": "arg2",
"type": "gint"
},
{
"name": "arg3",
"type": "GstWebRTCSinkMitigationMode"
}
],
"return-type": "GstCaps",
"when": "last"
},
"consumer-added": {
"args": [
{
@ -14742,6 +14776,26 @@
}
]
},
"GstWebRTCSinkMitigationMode": {
"kind": "flags",
"values": [
{
"desc": "No mitigation applied",
"name": "none",
"value": "0x00000000"
},
{
"desc": "Lowered resolution",
"name": "downscaled",
"value": "0x00000001"
},
{
"desc": "Lowered framerate",
"name": "downsampled",
"value": "0x00000002"
}
]
},
"GstWebRTCSinkPad": {
"hierarchy": [
"GstWebRTCSinkPad",

9
net/webrtc/src/webrtcsink/homegrown_cc.rs
View file

@ -1,13 +1,13 @@
// SPDX-License-Identifier: MPL-2.0
use super::imp::VideoEncoder;
use crate::webrtcsink::WebRTCSinkMitigationMode;
use gst::{
glib::{self, value::FromValue},
prelude::*,
};
use std::sync::LazyLock;
use super::imp::VideoEncoder;
static CAT: LazyLock<gst::DebugCategory> = LazyLock::new(|| {
gst::DebugCategory::new(
"webrtcsink-homegrowncc",
@ -413,7 +413,10 @@ impl CongestionController {
let fec_percentage = (fec_ratio * 50f64) as u32;
for encoder in encoders.iter_mut() {
if encoder.set_bitrate(element, target_bitrate).is_ok() {
if encoder
.set_bitrate(element, target_bitrate, WebRTCSinkMitigationMode::all())
.is_ok()
{
encoder
.transceiver
.set_property("fec-percentage", fec_percentage);

193
net/webrtc/src/webrtcsink/imp.rs
View file

@ -12,7 +12,7 @@ use gst_plugin_webrtc_signalling::server::{Server, ServerError};
use gst_rtp::prelude::*;
use gst_utils::StreamProducer;
use gst_video::subclass::prelude::*;
use gst_video::VideoMultiviewMode;
use gst_video::{VideoInfo, VideoMultiviewMode};
use gst_webrtc::{WebRTCDataChannel, WebRTCICETransportPolicy};
use tokio::io::AsyncReadExt;
use tokio::net::TcpListener;
@ -85,6 +85,7 @@ const DEFAULT_WEB_SERVER_DIRECTORY: &str = "gstwebrtc-api/dist";
#[cfg(feature = "web_server")]
const DEFAULT_WEB_SERVER_HOST_ADDR: &str = "http://127.0.0.1:8080";
const DEFAULT_FORWARD_METAS: &str = "";
const DEFAULT_ENABLE_MITIGATION_MODES: WebRTCSinkMitigationMode = WebRTCSinkMitigationMode::all();
/* Start adding some FEC when the bitrate > 2Mbps as we found experimentally
* that it is not worth it below that threshold */
const DO_FEC_THRESHOLD: u32 = 2000000;
@ -126,9 +127,11 @@ struct Settings {
#[cfg(feature = "web_server")]
web_server_host_addr: url::Url,
forward_metas: HashSet<String>,
enabled_mitigation_modes: WebRTCSinkMitigationMode,
}
use std::sync::atomic::{AtomicU32, Ordering};
static BD_SEQ: AtomicU32 = AtomicU32::new(0);
fn get_bdseq() -> u32 {
BD_SEQ.fetch_and(1, Ordering::SeqCst) + 1
@ -329,6 +332,7 @@ struct SessionInner {
navigation_handler: Option<NavigationEventHandler>,
control_events_handler: Option<ControlRequestHandler>,
enabled_mitigation_modes: WebRTCSinkMitigationMode,
}
#[derive(Clone)]
@ -552,6 +556,7 @@ impl Default for Settings {
#[cfg(feature = "web_server")]
web_server_host_addr: url::Url::parse(DEFAULT_WEB_SERVER_HOST_ADDR).unwrap(),
forward_metas: HashSet::new(),
enabled_mitigation_modes: DEFAULT_ENABLE_MITIGATION_MODES,
}
}
}
@ -1007,6 +1012,43 @@ impl PayloadChainBuilder {
}
}
fn default_configure_mitigation_mode(
video_info: &VideoInfo,
bitrate: i32,
enabled_mitigation_modes: WebRTCSinkMitigationMode,
) -> gst::Caps {
let mut s = gst::Structure::new_empty("video/x-raw");
if enabled_mitigation_modes.is_empty() {
return gst::Caps::builder_full_with_any_features()
.structure(s)
.build();
}
if bitrate < 500000 && enabled_mitigation_modes.contains(WebRTCSinkMitigationMode::DOWNSAMPLED)
{
let scaled_framerate = video_info.fps().mul(gst::Fraction::new(1, 2));
if scaled_framerate.numer() != 0 {
s.set("framerate", scaled_framerate);
}
}
if enabled_mitigation_modes.contains(WebRTCSinkMitigationMode::DOWNSCALED) {
let height = match bitrate {
b if b < 1_000_000 => 360,
b if b < 2_000_000 => 720,
_ => video_info.height() as i32,
};
let width = VideoEncoder::scale_height_round_2(video_info, height);
s.set("height", height);
s.set("width", width);
}
gst::Caps::builder_full_with_any_features()
.structure(s)
.build()
}
impl VideoEncoder {
fn new(
encoding_elements: &EncodingChain,
@ -1070,11 +1112,11 @@ impl VideoEncoder {
Ok(bitrate)
}
fn scale_height_round_2(&self, height: i32) -> i32 {
fn scale_height_round_2(video_info: &VideoInfo, height: i32) -> i32 {
let ratio = gst_video::calculate_display_ratio(
self.video_info.width(),
self.video_info.height(),
self.video_info.par(),
video_info.width(),
video_info.height(),
video_info.par(),
gst::Fraction::new(1, 1),
)
.unwrap();
@ -1088,6 +1130,7 @@ impl VideoEncoder {
&mut self,
element: &super::BaseWebRTCSink,
bitrate: i32,
enabled_mitigation_modes: WebRTCSinkMitigationMode,
) -> Result<(), WebRTCSinkError> {
match self.factory_name.as_str() {
"vp8enc" | "vp9enc" => self.element.set_property("target-bitrate", bitrate),
@ -1107,65 +1150,43 @@ impl VideoEncoder {
}
let current_caps = self.filter.property::<gst::Caps>("caps");
let mut s = current_caps.structure(0).unwrap().to_owned();
// Hardcoded thresholds, may be tuned further in the future, and
// adapted according to the codec in use
if bitrate < 500000 {
let height = 360i32.min(self.video_info.height() as i32);
let width = self.scale_height_round_2(height);
let mitigation_mode_caps = element.emit_by_name::<gst::Caps>(
"configure-mitigation-caps",
&[
&self.stream_name,
&self.video_info,
&bitrate,
&enabled_mitigation_modes,
],
);
let mitigation_mode_caps_structure = mitigation_mode_caps.structure(0).unwrap().to_owned();
s.set("height", height);
s.set("width", width);
if self.halved_framerate.numer() != 0 {
s.set("framerate", self.halved_framerate);
}
self.mitigation_mode =
WebRTCSinkMitigationMode::DOWNSAMPLED | WebRTCSinkMitigationMode::DOWNSCALED;
} else if bitrate < 1000000 {
let height = 360i32.min(self.video_info.height() as i32);
let width = self.scale_height_round_2(height);
s.set("height", height);
s.set("width", width);
s.remove_field("framerate");
self.mitigation_mode = WebRTCSinkMitigationMode::NONE;
if mitigation_mode_caps_structure.get::<i32>("height").is_ok() {
self.mitigation_mode = WebRTCSinkMitigationMode::DOWNSCALED;
} else if bitrate < 2000000 {
let height = 720i32.min(self.video_info.height() as i32);
let width = self.scale_height_round_2(height);
s.set("height", height);
s.set("width", width);
s.remove_field("framerate");
self.mitigation_mode = WebRTCSinkMitigationMode::DOWNSCALED;
} else {
s.remove_field("height");
s.remove_field("width");
s.remove_field("framerate");
self.mitigation_mode = WebRTCSinkMitigationMode::NONE;
}
let caps = gst::Caps::builder_full_with_any_features()
.structure(s)
.build();
if mitigation_mode_caps_structure
.get::<gst::Fraction>("framerate")
.is_ok()
{
self.mitigation_mode |= WebRTCSinkMitigationMode::DOWNSAMPLED;
}
if !caps.is_strictly_equal(&current_caps) {
if !mitigation_mode_caps.is_strictly_equal(&current_caps) {
gst::log!(
CAT,
obj = element,
"session {}: setting bitrate {} and caps {} on encoder {:?}",
self.session_id,
bitrate,
caps,
mitigation_mode_caps,
self.element
);
self.filter.set_property("caps", caps);
self.filter.set_property("caps", mitigation_mode_caps);
}
Ok(())
@ -1278,6 +1299,7 @@ impl State {
}
impl SessionInner {
#[allow(clippy::too_many_arguments)]
fn new(
id: String,
pipeline: gst::Pipeline,
@ -1286,6 +1308,7 @@ impl SessionInner {
congestion_controller: Option<CongestionController>,
rtpgccbwe: Option<gst::Element>,
cc_info: CCInfo,
enabled_mitigation_modes: WebRTCSinkMitigationMode,
) -> Self {
Self {
id,
@ -1306,6 +1329,7 @@ impl SessionInner {
stats_collection_handle: None,
navigation_handler: None,
control_events_handler: None,
enabled_mitigation_modes,
}
}
@ -1406,7 +1430,11 @@ impl SessionInner {
WebRTCSinkCongestionControl::Disabled => {
// If congestion control is disabled, we simply use the highest
// known "safe" value for the bitrate.
let _ = enc.set_bitrate(element, self.cc_info.max_bitrate as i32);
let _ = enc.set_bitrate(
element,
self.cc_info.max_bitrate as i32,
self.enabled_mitigation_modes,
);
enc.transceiver.set_property("fec-percentage", 50u32);
}
WebRTCSinkCongestionControl::Homegrown => {
@ -1420,7 +1448,11 @@ impl SessionInner {
} else {
/* If congestion control is disabled, we simply use the highest
* known "safe" value for the bitrate. */
let _ = enc.set_bitrate(element, self.cc_info.max_bitrate as i32);
let _ = enc.set_bitrate(
element,
self.cc_info.max_bitrate as i32,
self.enabled_mitigation_modes,
);
enc.transceiver.set_property("fec-percentage", 50u32);
}
}
@ -3113,6 +3145,7 @@ impl BaseWebRTCSink {
},
rtpgccbwe,
settings.cc_info,
settings.enabled_mitigation_modes,
);
let rtpbin = webrtcbin
@ -3538,7 +3571,11 @@ impl BaseWebRTCSink {
};
if encoder
.set_bitrate(&self.obj(), defined_encoder_bitrate)
.set_bitrate(
&self.obj(),
defined_encoder_bitrate,
settings.enabled_mitigation_modes,
)
.is_ok()
{
encoder
@ -4714,6 +4751,21 @@ impl ObjectImpl for BaseWebRTCSink {
.default_value(DEFAULT_FORWARD_METAS)
.mutable_playing()
.build(),
/**
* GstBaseWebRTCSink:enable-mitigation-modes:
*
* Whether the element should dynamically scale the source resolution
* based on the bitrate.
*
* Since: plugins-rs-0.14.0
*/
glib::ParamSpecFlags::builder("enable-mitigation-modes")
.nick("Enable dynamic scaling / sampling of the source resolution")
.blurb("Flags for whether the element should dynamically scale the source resolution and framerate based on the bitrate")
.default_value(DEFAULT_ENABLE_MITIGATION_MODES)
.mutable_playing()
.build(),
]
});
@ -4855,6 +4907,12 @@ impl ObjectImpl for BaseWebRTCSink {
.map(String::from)
.collect();
}
"enable-mitigation-modes" => {
let mut settings = self.settings.lock().unwrap();
settings.enabled_mitigation_modes = value
.get::<WebRTCSinkMitigationMode>()
.expect("type checked upstream");
}
_ => unimplemented!(),
}
}
@ -4957,6 +5015,10 @@ impl ObjectImpl for BaseWebRTCSink {
let settings = self.settings.lock().unwrap();
settings.forward_metas.iter().join(",").to_value()
}
"enable-mitigation-modes" => {
let settings = self.settings.lock().unwrap();
settings.enabled_mitigation_modes.to_value()
}
_ => unimplemented!(),
}
}
@ -5169,6 +5231,35 @@ impl ObjectImpl for BaseWebRTCSink {
std::ops::ControlFlow::Break(value.clone())
})
.build(),
/**
* GstBaseWebRTCSink::configure-mitigation-caps:
* @stream_name: name of the sink pad feeding the encoder
* @current_bitrate: The current bitrate being applied to the encoder
*
*
* Returns: the mitigation mode to apply.
* Since: plugins-rs-0.14.0
*/
glib::subclass::Signal::builder("configure-mitigation-caps")
.param_types([
String::static_type(),
VideoInfo::static_type(),
i32::static_type(),
WebRTCSinkMitigationMode::static_type(),
])
.return_type::<gst::Caps>()
.run_last()
.class_handler(|args| {
let video_info = args[2].get::<VideoInfo>().expect("No video info");
let current_bitrate = args[3].get::<i32>().expect("No bitrate");
let enabled_mitigation_modes = args[4].get::<WebRTCSinkMitigationMode>().expect("No enabled mitigation modes");
Some(default_configure_mitigation_mode(&video_info, current_bitrate, enabled_mitigation_modes).to_value())
})
.accumulator(move |_hint, _acc, value| {
std::ops::ControlFlow::Break(value.clone())
})
.build(),
]
});

3
net/webrtc/src/webrtcsink/mod.rs
View file

@ -123,7 +123,7 @@ pub enum WebRTCSinkCongestionControl {
}
#[glib::flags(name = "GstWebRTCSinkMitigationMode")]
enum WebRTCSinkMitigationMode {
pub enum WebRTCSinkMitigationMode {
#[flags_value(name = "No mitigation applied", nick = "none")]
NONE = 0b00000000,
#[flags_value(name = "Lowered resolution", nick = "downscaled")]
@ -156,6 +156,7 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
WebRTCSinkPad::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
BaseWebRTCSink::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
WebRTCSinkCongestionControl::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
WebRTCSinkMitigationMode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
gst::Element::register(
Some(plugin),
"webrtcsink",