From 60f7f4a664144452586463b0c3e6e0329aae3c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laignel?= Date: Tue, 16 Apr 2024 18:49:01 +0200 Subject: [PATCH] webrtc: add raw payload support This commit adds support for raw payloads such as L24 audio to `webrtcsink` & `webrtcsrc`. Most changes take place within the `Codec` helper structure: * A `Codec` can now advertise a depayloader. This also ensures that a format not only can be decoded when necessary, but it can also be depayloaded in the first place. * It is possible to declare raw `Codec`s, meaning that their caps are compatible with a payloader and a depayloader without the need for an encoder and decoder. * Previous accessor `has_decoder` was renamed as `can_be_received` to account for codecs which can be handled by an available depayloader with or without the need for a decoder. * New codecs were added for the following formats: * L24, L16, L8 audio. * RAW video. The `webrtc-precise-sync` examples were updated to demonstrate streaming of raw audio or video. --- docs/plugins/gst_plugins_cache.json | 8 +- net/webrtc/examples/README.md | 30 +++ .../examples/webrtc-precise-sync-recv.rs | 20 ++ .../examples/webrtc-precise-sync-send.rs | 75 ++++-- net/webrtc/src/utils.rs | 232 +++++++++++++++--- net/webrtc/src/webrtcsink/imp.rs | 69 +++--- net/webrtc/src/webrtcsrc/imp.rs | 18 +- 7 files changed, 363 insertions(+), 89 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 3d4a22b6..aa5d03ff 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -8045,7 +8045,7 @@ "construct": false, "construct-only": false, "controllable": false, - "default": "audio/x-opus", + "default": "audio/x-opus; audio/x-raw, format=(string)S24BE, layout=(GstAudioLayout)interleaved; audio/x-raw, format=(string)S16BE, layout=(GstAudioLayout)interleaved; audio/x-raw, format=(string)U8, layout=(GstAudioLayout)interleaved", "mutable": "ready", "readable": true, "type": "GstCaps", @@ -8228,7 +8228,7 @@ "construct": false, "construct-only": false, "controllable": false, - "default": "video/x-vp8; video/x-h264; video/x-vp9; video/x-h265", + "default": "video/x-vp8; video/x-h264; video/x-vp9; video/x-h265; video/x-raw, format=(string){ RGB, RGBA, BGR, BGRA, AYUV, UYVY, I420, Y41B, UYVP }", "mutable": "ready", "readable": true, "type": "GstCaps", @@ -8355,7 +8355,7 @@ "kind": "object", "properties": { "audio-codecs": { - "blurb": "Names of audio codecs to be be used during the SDP negotiation. Valid values: [OPUS]", + "blurb": "Names of audio codecs to be be used during the SDP negotiation. Valid values: [OPUS, L24, L16, L8]", "conditionally-available": false, "construct": false, "construct-only": false, @@ -8435,7 +8435,7 @@ "writable": true }, "video-codecs": { - "blurb": "Names of video codecs to be be used during the SDP negotiation. Valid values: [VP8, H264, VP9, H265]", + "blurb": "Names of video codecs to be be used during the SDP negotiation. Valid values: [VP8, H264, VP9, H265, RAW]", "conditionally-available": false, "construct": false, "construct-only": false, diff --git a/net/webrtc/examples/README.md b/net/webrtc/examples/README.md index 4088e8f6..7a585f90 100644 --- a/net/webrtc/examples/README.md +++ b/net/webrtc/examples/README.md @@ -34,6 +34,8 @@ mode and an example based on RTSP instead of WebRTC. The examples can also be used for [RFC 7273] NTP or PTP clock signalling and synchronization. +Finally, raw payloads (e.g. L24 audio) can be negotiated. + [RFC 6051]: https://datatracker.ietf.org/doc/html/rfc6051 [RFC 7273]: https://datatracker.ietf.org/doc/html/rfc7273 [Instantaneous RTP synchronization...]: https://coaxion.net/blog/2022/05/instantaneous-rtp-synchronization-retrieval-of-absolute-sender-clock-times-with-gstreamer/ @@ -129,3 +131,31 @@ cargo r --example webrtc-precise-sync-recv -- --expect-clock-signalling cargo r --example webrtc-precise-sync-send -- --clock ntp --do-clock-signalling \ --video-streams 0 --audio-streams 2 ``` + +### Raw payload + +The sender can be instructed to send raw payloads. + +This command will stream two stereo L24 streams: + +```shell +cargo r --example webrtc-precise-sync-send -- \ + --video-streams 0 --audio-streams 2 \ + --audio-caps 'audio/x-raw,format=S24BE,rate=48000,channels=2' +``` + +Launch the receiver with: + +```shell +cargo r --example webrtc-precise-sync-recv +``` + +This can be used to stream multiple RAW video streams forcing width and +allowing fallback to VP8 & OPUS if remote doesn't support raw payloads: + +```shell +cargo r --example webrtc-precise-sync-send -- \ + --video-streams 2 --audio-streams 1 \ + --video-caps 'video/x-raw,format=I420,width=400;video/x-vp8' \ + --audio-caps 'audio/x-raw,format=S24BE,rate=48000,channels=2;video/x-opus' +``` diff --git a/net/webrtc/examples/webrtc-precise-sync-recv.rs b/net/webrtc/examples/webrtc-precise-sync-recv.rs index d787dbfe..7b08201c 100644 --- a/net/webrtc/examples/webrtc-precise-sync-recv.rs +++ b/net/webrtc/examples/webrtc-precise-sync-recv.rs @@ -43,6 +43,18 @@ struct Args { #[clap(long, help = "RTP jitterbuffer latency (ms)", default_value = "40")] pub rtp_latency: u32, + #[clap( + long, + help = "Force accepted audio codecs. See 'webrtcsrc' 'audio-codecs' property (ex. 'OPUS'). Accepts several occurrences." + )] + pub audio_codecs: Vec, + + #[clap( + long, + help = "Force accepted video codecs. See 'webrtcsrc' 'video-codecs' property (ex. 'VP8'). Accepts several occurrences." + )] + pub video_codecs: Vec, + #[clap(long, help = "Signalling server host", default_value = "localhost")] pub server: String, @@ -123,6 +135,14 @@ fn spawn_consumer( webrtcsrc.set_property("do-retransmission", false); } + if !args.audio_codecs.is_empty() { + webrtcsrc.set_property("audio-codecs", gst::Array::new(&args.audio_codecs)); + } + + if !args.video_codecs.is_empty() { + webrtcsrc.set_property("video-codecs", gst::Array::new(&args.video_codecs)); + } + bin.add(&webrtcsrc).context("Adding webrtcsrc")?; let signaller = webrtcsrc.property::("signaller"); diff --git a/net/webrtc/examples/webrtc-precise-sync-send.rs b/net/webrtc/examples/webrtc-precise-sync-send.rs index 863980ab..225e7665 100644 --- a/net/webrtc/examples/webrtc-precise-sync-send.rs +++ b/net/webrtc/examples/webrtc-precise-sync-send.rs @@ -45,6 +45,12 @@ struct Args { )] pub video_streams: usize, + #[clap( + long, + help = "Force audio caps (ex. for L24 'audio/x-raw,format=S24BE,rate=48000,channels=2')" + )] + pub audio_caps: Option, + #[clap(long, help = "Force video caps (ex. 'video/x-h264')")] pub video_caps: Option, @@ -135,6 +141,8 @@ impl App { } async fn prepare(&mut self) -> anyhow::Result<()> { + use std::str::FromStr; + debug!("Preparing"); self.pipeline = Some(gst::Pipeline::new()); @@ -217,6 +225,19 @@ impl App { }); } + let raw_audio_caps = if let Some(ref audio_caps) = self.args.audio_caps { + let caps = gst::Caps::from_str(audio_caps).context("Parsing audio caps")?; + webrtcsink.set_property("audio-caps", &caps); + + // Reuse the first user defined caps for the raw caps + let mut s = caps.structure(0).expect("parsed above").to_owned(); + s.set_name("audio/x-raw"); + + Some(gst::Caps::from(s)) + } else { + None + }; + for idx in 0..self.args.audio_streams { let audiosrc = gst::ElementFactory::make("audiotestsrc") .property("is-live", true) @@ -226,11 +247,47 @@ impl App { .context("Creating audiotestsrc")?; self.pipeline().add(&audiosrc).context("Adding audiosrc")?; - audiosrc - .link_pads(None, &webrtcsink, Some("audio_%u")) - .context("Linking audiosrc")?; + if let Some(ref raw_caps) = raw_audio_caps { + audiosrc + .link_pads_filtered(None, &webrtcsink, Some("audio_%u"), raw_caps) + .context("Linking audiosrc")?; + } else { + audiosrc + .link_pads(None, &webrtcsink, Some("audio_%u")) + .context("Linking audiosrc")?; + } } + let raw_video_caps = { + let mut raw_video_caps = if let Some(ref video_caps) = self.args.video_caps { + let caps = gst::Caps::from_str(video_caps).context("Parsing video caps")?; + webrtcsink.set_property("video-caps", &caps); + + // Reuse the first user defined caps for the raw caps + let mut s = caps.structure(0).expect("parsed above").to_owned(); + s.set_name("video/x-raw"); + + gst::Caps::from(s) + } else { + gst::Caps::new_empty_simple("video/x-raw") + }; + + // If no width / height are specified, set something big enough + let caps_mut = raw_video_caps.make_mut(); + let s = caps_mut.structure_mut(0).expect("created above"); + match (s.get::("width").ok(), s.get::("height").ok()) { + (None, None) => { + s.set("width", 800i32); + s.set("height", 600i32); + } + (Some(width), None) => s.set("height", 3 * width / 4), + (None, Some(height)) => s.set("width", 4 * height / 3), + _ => (), + } + + raw_video_caps + }; + for idx in 0..self.args.video_streams { let videosrc = gst::ElementFactory::make("videotestsrc") .property("is-live", true) @@ -247,13 +304,7 @@ impl App { .expect("adding video elements"); videosrc - .link_filtered( - &video_overlay, - &gst::Caps::builder("video/x-raw") - .field("width", 800i32) - .field("height", 600i32) - .build(), - ) + .link_filtered(&video_overlay, &raw_video_caps) .context("Linking videosrc to timeoverlay")?; video_overlay @@ -261,10 +312,6 @@ impl App { .context("Linking video overlay")?; } - if let Some(ref video_caps) = self.args.video_caps { - webrtcsink.set_property("video-caps", &gst::Caps::builder(video_caps).build()); - } - Ok(()) } diff --git a/net/webrtc/src/utils.rs b/net/webrtc/src/utils.rs index 1c806c91..3c114577 100644 --- a/net/webrtc/src/utils.rs +++ b/net/webrtc/src/utils.rs @@ -427,7 +427,7 @@ impl Clone for DecodingInfo { #[derive(Clone, Debug)] struct EncodingInfo { - encoder: gst::ElementFactory, + encoder: Option, payloader: gst::ElementFactory, output_filter: Option, } @@ -449,16 +449,27 @@ impl Codec { stream_type: gst::StreamType, caps: &gst::Caps, decoders: &glib::List, + depayloaders: &glib::List, encoders: &glib::List, payloaders: &glib::List, ) -> Self { let has_decoder = Self::has_decoder_for_caps(caps, decoders); + let has_depayloader = Self::has_depayloader_for_codec(name, depayloaders); + + let decoding_info = if has_depayloader && has_decoder { + Some(DecodingInfo { + has_decoder: AtomicBool::new(has_decoder), + }) + } else { + None + }; + let encoder = Self::get_encoder_for_caps(caps, encoders); let payloader = Self::get_payloader_for_codec(name, payloaders); let encoding_info = if let (Some(encoder), Some(payloader)) = (encoder, payloader) { Some(EncodingInfo { - encoder, + encoder: Some(encoder), payloader, output_filter: None, }) @@ -471,11 +482,39 @@ impl Codec { stream_type, name: name.into(), payload_type: None, + decoding_info, + encoding_info, + } + } - decoding_info: Some(DecodingInfo { - has_decoder: AtomicBool::new(has_decoder), - }), + pub fn new_raw( + name: &str, + stream_type: gst::StreamType, + caps: &gst::Caps, + depayloaders: &glib::List, + payloaders: &glib::List, + ) -> Self { + let decoding_info = if Self::has_depayloader_for_codec(name, depayloaders) { + Some(DecodingInfo { + has_decoder: AtomicBool::new(false), + }) + } else { + None + }; + let payloader = Self::get_payloader_for_codec(name, payloaders); + let encoding_info = payloader.map(|payloader| EncodingInfo { + encoder: None, + payloader, + output_filter: None, + }); + + Self { + caps: caps.clone(), + stream_type, + name: name.into(), + payload_type: None, + decoding_info, encoding_info, } } @@ -488,34 +527,16 @@ impl Codec { self.payload_type = Some(pt); } - pub fn new_decoding( - name: &str, - stream_type: gst::StreamType, - caps: &gst::Caps, - decoders: &glib::List, - ) -> Self { - let has_decoder = Self::has_decoder_for_caps(caps, decoders); - - Self { - caps: caps.clone(), - stream_type, - name: name.into(), - payload_type: None, - - decoding_info: Some(DecodingInfo { - has_decoder: AtomicBool::new(has_decoder), - }), - - encoding_info: None, - } - } - - pub fn has_decoder(&self) -> bool { + pub fn can_be_received(&self) -> bool { if self.decoding_info.is_none() { return false; } let decoder_info = self.decoding_info.as_ref().unwrap(); + if is_raw_caps(&self.caps) { + return true; + } + if decoder_info.has_decoder.load(Ordering::SeqCst) { true } else if Self::has_decoder_for_caps( @@ -599,6 +620,40 @@ impl Codec { }) } + fn has_depayloader_for_codec( + codec: &str, + depayloaders: &glib::List, + ) -> bool { + depayloaders.iter().any(|factory| { + factory.static_pad_templates().iter().any(|template| { + let template_caps = template.caps(); + + if template.direction() != gst::PadDirection::Sink || template_caps.is_any() { + return false; + } + + template_caps.iter().any(|s| { + s.has_field("encoding-name") + && s.get::("encoding-name").map_or_else( + |_| { + if let Ok(encoding_name) = s.get::<&str>("encoding-name") { + encoding_name == codec + } else { + false + } + }, + |encoding_names| { + encoding_names.iter().any(|v| { + v.get::<&str>() + .map_or(false, |encoding_name| encoding_name == codec) + }) + }, + ) + }) + }) + }) + } + pub fn is_video(&self) -> bool { matches!(self.stream_type, gst::StreamType::VIDEO) } @@ -608,11 +663,13 @@ impl Codec { } pub fn build_encoder(&self) -> Option> { - self.encoding_info.as_ref().map(|info| { - info.encoder - .create() - .build() - .with_context(|| format!("Creating encoder {}", info.encoder.name())) + self.encoding_info.as_ref().and_then(|info| { + info.encoder.as_ref().map(|encoder| { + encoder + .create() + .build() + .with_context(|| format!("Creating encoder {}", encoder.name())) + }) }) } @@ -655,13 +712,17 @@ impl Codec { } pub fn encoder_factory(&self) -> Option { - self.encoding_info.as_ref().map(|info| info.encoder.clone()) + self.encoding_info + .as_ref() + .and_then(|info| info.encoder.clone()) } pub fn encoder_name(&self) -> Option { - self.encoding_info - .as_ref() - .map(|info| info.encoder.name().to_string()) + self.encoding_info.as_ref().and_then(|info| { + info.encoder + .as_ref() + .map(|encoder| encoder.name().to_string()) + }) } pub fn set_output_filter(&mut self, caps: gst::Caps) { @@ -713,6 +774,36 @@ impl Codec { pub static AUDIO_CAPS: Lazy = Lazy::new(|| gst::Caps::new_empty_simple("audio/x-raw")); pub static OPUS_CAPS: Lazy = Lazy::new(|| gst::Caps::new_empty_simple("audio/x-opus")); +pub static L24_CAPS: Lazy = Lazy::new(|| { + gst::Caps::builder_full() + .structure( + gst::Structure::builder("audio/x-raw") + .field("format", gst_audio::AudioFormat::S24be.to_str()) + .field("layout", glib::gstr!("interleaved")) + .build(), + ) + .build() +}); +pub static L16_CAPS: Lazy = Lazy::new(|| { + gst::Caps::builder_full() + .structure( + gst::Structure::builder("audio/x-raw") + .field("format", gst_audio::AudioFormat::S16be.to_str()) + .field("layout", glib::gstr!("interleaved")) + .build(), + ) + .build() +}); +pub static L8_CAPS: Lazy = Lazy::new(|| { + gst::Caps::builder_full() + .structure( + gst::Structure::builder("audio/x-raw") + .field("format", gst_audio::AudioFormat::U8.to_str()) + .field("layout", glib::gstr!("interleaved")) + .build(), + ) + .build() +}); pub static VIDEO_CAPS: Lazy = Lazy::new(|| { gst::Caps::builder_full_with_any_features() @@ -723,6 +814,32 @@ pub static VP8_CAPS: Lazy = Lazy::new(|| gst::Caps::new_empty_simple( pub static VP9_CAPS: Lazy = Lazy::new(|| gst::Caps::new_empty_simple("video/x-vp9")); pub static H264_CAPS: Lazy = Lazy::new(|| gst::Caps::new_empty_simple("video/x-h264")); pub static H265_CAPS: Lazy = Lazy::new(|| gst::Caps::new_empty_simple("video/x-h265")); +pub static VRAW_CAPS: Lazy = Lazy::new(|| { + gst::Caps::builder_full() + .structure( + gst::Structure::builder("video/x-raw") + .field( + "format", + gst::List::new( + [ + gst_video::VideoFormat::Rgb, + gst_video::VideoFormat::Rgba, + gst_video::VideoFormat::Bgr, + gst_video::VideoFormat::Bgra, + gst_video::VideoFormat::Ayuv, + gst_video::VideoFormat::Uyvy, + gst_video::VideoFormat::I420, + gst_video::VideoFormat::Y41b, + gst_video::VideoFormat::Uyvp, + ] + .into_iter() + .map(|f| f.to_str()), + ), + ) + .build(), + ) + .build() +}); pub static RTP_CAPS: Lazy = Lazy::new(|| gst::Caps::new_empty_simple("application/x-rtp")); @@ -750,7 +867,7 @@ impl Codecs { Self(codecs.values().cloned().collect()) } - pub fn find_for_encoded_caps(&self, caps: &gst::Caps) -> Option { + pub fn find_for_payloadable_caps(&self, caps: &gst::Caps) -> Option { self.iter() .find(|codec| codec.caps.can_intersect(caps) && codec.encoding_info.is_some()) .cloned() @@ -763,6 +880,11 @@ static CODECS: Lazy = Lazy::new(|| { gst::Rank::MARGINAL, ); + let depayloaders = gst::ElementFactory::factories_with_type( + gst::ElementFactoryType::DEPAYLOADER, + gst::Rank::MARGINAL, + ); + let encoders = gst::ElementFactory::factories_with_type( gst::ElementFactoryType::ENCODER, gst::Rank::MARGINAL, @@ -779,14 +901,37 @@ static CODECS: Lazy = Lazy::new(|| { gst::StreamType::AUDIO, &OPUS_CAPS, &decoders, + &depayloaders, &encoders, &payloaders, ), + Codec::new_raw( + "L24", + gst::StreamType::AUDIO, + &L24_CAPS, + &depayloaders, + &payloaders, + ), + Codec::new_raw( + "L16", + gst::StreamType::AUDIO, + &L16_CAPS, + &depayloaders, + &payloaders, + ), + Codec::new_raw( + "L8", + gst::StreamType::AUDIO, + &L8_CAPS, + &depayloaders, + &payloaders, + ), Codec::new( "VP8", gst::StreamType::VIDEO, &VP8_CAPS, &decoders, + &depayloaders, &encoders, &payloaders, ), @@ -795,6 +940,7 @@ static CODECS: Lazy = Lazy::new(|| { gst::StreamType::VIDEO, &H264_CAPS, &decoders, + &depayloaders, &encoders, &payloaders, ), @@ -803,6 +949,7 @@ static CODECS: Lazy = Lazy::new(|| { gst::StreamType::VIDEO, &VP9_CAPS, &decoders, + &depayloaders, &encoders, &payloaders, ), @@ -811,9 +958,17 @@ static CODECS: Lazy = Lazy::new(|| { gst::StreamType::VIDEO, &H265_CAPS, &decoders, + &depayloaders, &encoders, &payloaders, ), + Codec::new_raw( + "RAW", + gst::StreamType::VIDEO, + &VRAW_CAPS, + &depayloaders, + &payloaders, + ), ]) }); @@ -899,7 +1054,6 @@ impl Codecs { } pub fn is_raw_caps(caps: &gst::Caps) -> bool { - assert!(caps.is_fixed()); ["video/x-raw", "audio/x-raw"].contains(&caps.structure(0).unwrap().name().as_str()) } diff --git a/net/webrtc/src/webrtcsink/imp.rs b/net/webrtc/src/webrtcsink/imp.rs index 847352e9..cfe60656 100644 --- a/net/webrtc/src/webrtcsink/imp.rs +++ b/net/webrtc/src/webrtcsink/imp.rs @@ -845,7 +845,7 @@ impl PayloadChainBuilder { codec = self.codec, ); - let needs_encoding = is_raw_caps(&self.input_caps); + let needs_encoding = !is_raw_caps(&self.codec.caps) && is_raw_caps(&self.input_caps); let mut elements: Vec = Vec::new(); let (raw_filter, encoder) = if needs_encoding { @@ -3503,13 +3503,40 @@ impl BaseWebRTCSink { output_caps: gst::Caps, codecs: &Codecs, ) -> Result<(), Error> { - let futs = if let Some(codec) = codecs.find_for_encoded_caps(&discovery_info.caps) { + let futs = if is_raw_caps(&discovery_info.caps) { + let sink_caps = discovery_info.caps.clone(); + + let is_video = match sink_caps.structure(0).unwrap().name().as_str() { + "video/x-raw" => true, + "audio/x-raw" => false, + _ => anyhow::bail!("expected audio or video raw caps: {sink_caps}"), + }; + + codecs + .iter() + .filter(|codec| { + codec.is_video() == is_video + && (!is_raw_caps(&codec.caps) || codec.caps.can_intersect(&sink_caps)) + }) + .map(|codec| { + BaseWebRTCSink::run_discovery_pipeline( + element, + &name, + &discovery_info, + codec.clone(), + sink_caps.clone(), + &output_caps, + ExtensionConfigurationType::Auto, + ) + }) + .collect() + } else if let Some(codec) = codecs.find_for_payloadable_caps(&discovery_info.caps) { let mut caps = discovery_info.caps.clone(); gst::info!( CAT, obj: element, - "Stream is already encoded with codec {}, still need to payload it", + "Stream already conforms to {}, still need to payload it", codec.name ); @@ -3525,29 +3552,7 @@ impl BaseWebRTCSink { ExtensionConfigurationType::Auto, )] } else { - let sink_caps = discovery_info.caps.clone(); - - let is_video = match sink_caps.structure(0).unwrap().name().as_str() { - "video/x-raw" => true, - "audio/x-raw" => false, - _ => anyhow::bail!("Unsupported caps: {}", discovery_info.caps), - }; - - codecs - .iter() - .filter(|codec| codec.is_video() == is_video) - .map(|codec| { - BaseWebRTCSink::run_discovery_pipeline( - element, - &name, - &discovery_info, - codec.clone(), - sink_caps.clone(), - &output_caps, - ExtensionConfigurationType::Auto, - ) - }) - .collect() + anyhow::bail!("Unsupported caps: {}", discovery_info.caps); }; let mut payloader_caps = gst::Caps::new_empty(); @@ -4258,6 +4263,8 @@ impl GstObjectImpl for BaseWebRTCSink {} impl ElementImpl for BaseWebRTCSink { fn pad_templates() -> &'static [gst::PadTemplate] { static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + // Ignore specific raw caps from Codecs: they are covered by video/x-raw & audio/x-raw + let mut caps_builder = gst::Caps::builder_full() .structure(gst::Structure::builder("video/x-raw").build()) .structure_with_features( @@ -4277,7 +4284,10 @@ impl ElementImpl for BaseWebRTCSink { gst::CapsFeatures::new([D3D11_MEMORY_FEATURE]), ); - for codec in Codecs::video_codecs() { + for codec in Codecs::video_codecs() + .iter() + .filter(|codec| !is_raw_caps(&codec.caps)) + { caps_builder = caps_builder.structure(codec.caps.structure(0).unwrap().to_owned()); } @@ -4292,7 +4302,10 @@ impl ElementImpl for BaseWebRTCSink { let mut caps_builder = gst::Caps::builder_full().structure(gst::Structure::builder("audio/x-raw").build()); - for codec in Codecs::audio_codecs() { + for codec in Codecs::audio_codecs() + .iter() + .filter(|codec| !is_raw_caps(&codec.caps)) + { caps_builder = caps_builder.structure(codec.caps.structure(0).unwrap().to_owned()); } let audio_pad_template = gst::PadTemplate::with_gtype( diff --git a/net/webrtc/src/webrtcsrc/imp.rs b/net/webrtc/src/webrtcsrc/imp.rs index 8a9edf3b..9ef17305 100644 --- a/net/webrtc/src/webrtcsrc/imp.rs +++ b/net/webrtc/src/webrtcsrc/imp.rs @@ -274,11 +274,11 @@ impl Default for Settings { meta: Default::default(), audio_codecs: Codecs::audio_codecs() .into_iter() - .filter(|codec| codec.has_decoder()) + .filter(|codec| codec.can_be_received()) .collect(), video_codecs: Codecs::video_codecs() .into_iter() - .filter(|codec| codec.has_decoder()) + .filter(|codec| codec.can_be_received()) .collect(), enable_data_channel_navigation: DEFAULT_ENABLE_DATA_CHANNEL_NAVIGATION, do_retransmission: DEFAULT_DO_RETRANSMISSION, @@ -1027,11 +1027,18 @@ impl BaseWebRTCSrc { impl ElementImpl for BaseWebRTCSrc { fn pad_templates() -> &'static [gst::PadTemplate] { static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + use crate::utils::is_raw_caps; + + // Ignore specific raw caps from Codecs: they are covered by VIDEO_CAPS & AUDIO_CAPS + let mut video_caps_builder = gst::Caps::builder_full() .structure_with_any_features(VIDEO_CAPS.structure(0).unwrap().to_owned()) .structure(RTP_CAPS.structure(0).unwrap().to_owned()); - for codec in Codecs::video_codecs() { + for codec in Codecs::video_codecs() + .iter() + .filter(|codec| !is_raw_caps(&codec.caps)) + { video_caps_builder = video_caps_builder.structure(codec.caps.structure(0).unwrap().to_owned()); } @@ -1040,7 +1047,10 @@ impl ElementImpl for BaseWebRTCSrc { .structure_with_any_features(AUDIO_CAPS.structure(0).unwrap().to_owned()) .structure(RTP_CAPS.structure(0).unwrap().to_owned()); - for codec in Codecs::audio_codecs() { + for codec in Codecs::audio_codecs() + .iter() + .filter(|codec| !is_raw_caps(&codec.caps)) + { audio_caps_builder = audio_caps_builder.structure(codec.caps.structure(0).unwrap().to_owned()); }