From 23e7226c94dd4b2c86ae1844f2fb9390d7ae2317 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= <sebastian@centricular.com>
Date: Tue, 18 Jul 2023 15:18:08 +0300
Subject: [PATCH] webrtcsink: NVIDIA V4L2 encoders always require NVMM memory

And if the input is not like that then a corresponding converter must be
inserted.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1330>
---
 net/webrtc/src/webrtcsink/imp.rs | 46 +++++++++++++++++++-------------
 1 file changed, 28 insertions(+), 18 deletions(-)

diff --git a/net/webrtc/src/webrtcsink/imp.rs b/net/webrtc/src/webrtcsink/imp.rs
index 839affbc..99dceb5e 100644
--- a/net/webrtc/src/webrtcsink/imp.rs
+++ b/net/webrtc/src/webrtcsink/imp.rs
@@ -92,6 +92,10 @@ impl Codec {
         self.encoder
             .has_type(gst::ElementFactoryType::VIDEO_ENCODER)
     }
+
+    fn encoder_factory(&self) -> &gst::ElementFactory {
+        &self.encoder
+    }
 }
 
 /// Wrapper around our sink pads
@@ -314,7 +318,7 @@ impl Default for State {
     }
 }
 
-fn make_converter_for_video_caps(caps: &gst::Caps) -> Result<gst::Element, Error> {
+fn make_converter_for_video_caps(caps: &gst::Caps, codec: &Codec) -> Result<gst::Element, Error> {
     assert!(caps.is_fixed());
 
     let video_info = gst_video::VideoInfo::from_caps(caps)?;
@@ -323,7 +327,27 @@ fn make_converter_for_video_caps(caps: &gst::Caps) -> Result<gst::Element, Error
 
     let (head, mut tail) = {
         if let Some(feature) = caps.features(0) {
-            if feature.contains(CUDA_MEMORY_FEATURE) {
+            if feature.contains(NVMM_MEMORY_FEATURE)
+                // NVIDIA V4L2 encoders require NVMM memory as input and that requires using the
+                // corresponding converter
+                || codec
+                    .encoder_factory()
+                    .name().starts_with("nvv4l2")
+            {
+                let queue = make_element("queue", None)?;
+                let nvconvert = if let Ok(nvconvert) = make_element("nvvideoconvert", None) {
+                    nvconvert.set_property_from_str("compute-hw", "Default");
+                    nvconvert.set_property_from_str("nvbuf-memory-type", "nvbuf-mem-default");
+                    nvconvert
+                } else {
+                    make_element("nvvidconv", None)?
+                };
+
+                ret.add_many(&[&queue, &nvconvert])?;
+                gst::Element::link_many(&[&queue, &nvconvert])?;
+
+                (queue, nvconvert)
+            } else if feature.contains(CUDA_MEMORY_FEATURE) {
                 if let Some(convert_factory) = gst::ElementFactory::find("cudaconvert") {
                     let cudaupload = make_element("cudaupload", None)?;
                     let cudaconvert = convert_factory.create().build()?;
@@ -357,20 +381,6 @@ fn make_converter_for_video_caps(caps: &gst::Caps) -> Result<gst::Element, Error
                 gst::Element::link_many(&[&glupload, &glconvert, &glscale])?;
 
                 (glupload, glscale)
-            } else if feature.contains(NVMM_MEMORY_FEATURE) {
-                let queue = make_element("queue", None)?;
-                let nvconvert = if let Ok(nvconvert) = make_element("nvvideoconvert", None) {
-                    nvconvert.set_property_from_str("compute-hw", "Default");
-                    nvconvert.set_property_from_str("nvbuf-memory-type", "nvbuf-mem-default");
-                    nvconvert
-                } else {
-                    make_element("nvvidconv", None)?
-                };
-
-                ret.add_many(&[&queue, &nvconvert])?;
-                gst::Element::link_many(&[&queue, &nvconvert])?;
-
-                (queue, nvconvert)
             } else {
                 let convert = make_element("videoconvert", None)?;
                 let scale = make_element("videoscale", None)?;
@@ -524,7 +534,7 @@ fn setup_encoding(
     twcc: bool,
 ) -> Result<(gst::Element, gst::Element, gst::Element), Error> {
     let conv = match codec.is_video() {
-        true => make_converter_for_video_caps(input_caps)?.upcast(),
+        true => make_converter_for_video_caps(input_caps, codec)?.upcast(),
         false => gst::parse_bin_from_description("audioresample ! audioconvert", true)?.upcast(),
     };
 
@@ -2178,7 +2188,7 @@ impl WebRTCSink {
         let mut elements = vec![src.clone()];
 
         if codec.is_video() {
-            elements.push(make_converter_for_video_caps(caps)?);
+            elements.push(make_converter_for_video_caps(caps, &codec)?);
         }
 
         gst::debug!(