Add support for cuda and GL memory

This way we don't need to download/upload when unnecessary

plugins/src/webrtcsink/imp.rs

@@ -27,6 +27,9 @@ static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
     )
 });
 
+const CUDA_MEMORY_FEATURE: &str = "memory:CUDAMemory";
+const GL_MEMORY_FEATURE: &str = "memory:GLMemory";
+
 const RTP_TWCC_URI: &str =
     "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01";
 
@@ -240,22 +243,45 @@ impl Default for State {
     }
 }
 
+fn make_converter_for_video_caps(caps: &gst::Caps) -> Result<gst::Element, Error> {
+    assert!(caps.is_fixed());
+
+    for feature in caps.features(0) {
+        if feature.contains(CUDA_MEMORY_FEATURE) {
+            return Ok(gst::parse_bin_from_description(
+                "cudaupload ! cudaconvert ! cudascale ! videorate drop-only=true",
+                true,
+            )?
+            .upcast());
+        } else if feature.contains(GL_MEMORY_FEATURE) {
+            return Ok(gst::parse_bin_from_description(
+                "glupload ! glcolorconvert ! glcolorscale ! videorate drop-only=true skip-to-first=true",
+                true,
+            )?
+            .upcast());
+        }
+    }
+
+    Ok(gst::parse_bin_from_description(
+        "videoconvert ! videoscale ! videorate drop-only=true skip-to-first=true",
+        true,
+    )?
+    .upcast())
+}
+
 /// Bit of an awkward function, but the goal here is to keep
 /// most of the encoding code for consumers in line with
 /// the codec discovery code, and this gets the job done.
 fn setup_encoding(
     pipeline: &gst::Pipeline,
     src: &gst::Element,
+    input_caps: &gst::Caps,
     codec: &Codec,
     ssrc: Option<u32>,
     twcc: bool,
 ) -> Result<(gst::Element, gst::Element, gst::Element), Error> {
     let conv = match codec.is_video {
-        true => gst::parse_bin_from_description(
+        true => make_converter_for_video_caps(input_caps)?.upcast(),
-            "videoconvert ! videoscale ! videorate drop-only=true",
-            true,
-        )?
-        .upcast(),
         false => gst::parse_bin_from_description("audioresample ! audioconvert", true)?.upcast(),
     };
 
@@ -300,19 +326,21 @@ fn setup_encoding(
             .with_context(|| "Linking encoding elements")?;
     }
 
-    // Quirk: nvh264enc can perform conversion from RGB formats, but
+    let conv_caps = if codec.is_video {
-    // doesn't advertise / negotiate colorimetry correctly, leading
+        let mut structure_builder = gst::Structure::builder("video/x-raw")
-    // to incorrect color display in Chrome (but interestingly not in
+            .field("pixel-aspect-ratio", gst::Fraction::new(1, 1));
-    // Firefox). In any case, restrict to exclude RGB formats altogether,
+
-    // and let videoconvert do the conversion properly if needed.
+        if codec.encoder.name() == "nvh264enc" {
-    let conv_caps = if codec.encoder.name() == "nvh264enc" {
+            // Quirk: nvh264enc can perform conversion from RGB formats, but
-        gst::Caps::builder("video/x-raw")
+            // doesn't advertise / negotiate colorimetry correctly, leading
-            .field("format", &gst::List::new(&[&"NV12", &"YV12", &"I420"]))
+            // to incorrect color display in Chrome (but interestingly not in
-            .field("pixel-aspect-ratio", gst::Fraction::new(1, 1))
+            // Firefox). In any case, restrict to exclude RGB formats altogether,
-            .build()
+            // and let videoconvert do the conversion properly if needed.
-    } else if codec.is_video {
+            structure_builder = structure_builder.field("format", &gst::List::new(&[&"NV12", &"YV12", &"I420"]));
-        gst::Caps::builder("video/x-raw")
+        }
-            .field("pixel-aspect-ratio", gst::Fraction::new(1, 1))
+
+        gst::Caps::builder_full_with_any_features()
+            .structure(structure_builder.build())
             .build()
     } else {
         gst::Caps::builder("audio/x-raw").build()
@@ -516,7 +544,9 @@ impl VideoEncoder {
             self.mitigation_mode = WebRTCSinkMitigationMode::NONE;
         }
 
-        let caps = gst::Caps::builder_full().structure(s).build();
+        let caps = gst::Caps::builder_full_with_any_features()
+            .structure(s)
+            .build();
 
         gst_log!(
             CAT,
@@ -978,8 +1008,14 @@ impl Consumer {
         let pay_filter = make_element("capsfilter", None)?;
         self.pipeline.add(&pay_filter).unwrap();
 
-        let (enc, raw_filter, pay) =
+        let (enc, raw_filter, pay) = setup_encoding(
-            setup_encoding(&self.pipeline, &appsrc, codec, Some(webrtc_pad.ssrc), false)?;
+            &self.pipeline,
+            &appsrc,
+            &webrtc_pad.in_caps,
+            codec,
+            Some(webrtc_pad.ssrc),
+            false,
+        )?;
 
         // At this point, the peer has provided its answer, and we want to
         // let the payloader / encoder perform negotiation according to that.
@@ -1798,7 +1834,17 @@ impl WebRTCSink {
                  * very well equipped to deal with this at the moment */
                 if let Some(media) = sdp.media(media_idx) {
                     if media.attribute_val("inactive").is_some() {
-                        gst_warning!(CAT, "consumer {} refused media {}", peer_id, media_idx);
+                        let media_str = sdp
+                            .media(webrtc_pad.media_idx)
+                            .and_then(|media| media.as_text().ok());
+
+                        gst_warning!(
+                            CAT,
+                            "consumer {} refused media {}: {:?}",
+                            peer_id,
+                            media_idx,
+                            media_str
+                        );
                         state.remove_consumer(element, peer_id, true);
 
                         return Err(WebRTCSinkError::ConsumerRefusedMedia {
@@ -1860,17 +1906,24 @@ impl WebRTCSink {
     ) -> Result<gst::Structure, Error> {
         let pipe = PipelineWrapper(gst::Pipeline::new(None));
 
-        let src = match codec.is_video {
+        let src = if codec.is_video {
-            true => make_element("videotestsrc", None)?,
+            make_element("videotestsrc", None)?
-            false => make_element("audiotestsrc", None)?,
+        } else {
+            make_element("audiotestsrc", None)?
         };
-        let capsfilter = make_element("capsfilter", None)?;
+        let mut elements = Vec::new();
+        elements.push(src.clone());
 
-        pipe.0.add_many(&[&src, &capsfilter]).unwrap();
+        elements.push(make_converter_for_video_caps(caps)?);
-        src.link(&capsfilter)
+
+        let capsfilter = make_element("capsfilter", None)?;
+        elements.push(capsfilter.clone());
+        let elements_slice = &elements.iter().collect::<Vec<_>>();
+        pipe.0.add_many(elements_slice).unwrap();
+        gst::Element::link_many(elements_slice)
             .with_context(|| format!("Running discovery pipeline for caps {}", caps))?;
 
-        let (_, _, pay) = setup_encoding(&pipe.0, &capsfilter, codec, None, true)?;
+        let (_, _, pay) = setup_encoding(&pipe.0, &capsfilter, &caps, codec, None, true)?;
 
         let sink = make_element("fakesink", None)?;
 
@@ -2368,7 +2421,17 @@ impl ElementImpl for WebRTCSink {
 
     fn pad_templates() -> &'static [gst::PadTemplate] {
         static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
-            let caps = gst::Caps::builder("video/x-raw").build();
+            let caps = gst::Caps::builder_full()
+                .structure(gst::Structure::builder("video/x-raw").build())
+                .structure_with_features(
+                    gst::Structure::builder("video/x-raw").build(),
+                    gst::CapsFeatures::new(&[CUDA_MEMORY_FEATURE]),
+                )
+                .structure_with_features(
+                    gst::Structure::builder("video/x-raw").build(),
+                    gst::CapsFeatures::new(&[GL_MEMORY_FEATURE]),
+                )
+                .build();
             let video_pad_template = gst::PadTemplate::new(
                 "video_%u",
                 gst::PadDirection::Sink,