Merge branch 'livesync-handle-variable-framerate' into 'main'

livesync: add support for variable framerate

See merge request gstreamer/gst-plugins-rs!1516

audio/audiofx/src/hrtfrender/imp.rs

@@ -649,7 +649,7 @@ impl BaseTransformImpl for HrtfRender {
 
                 if direction == gst::PadDirection::Sink {
                     s.set("channels", 2);
-                    s.set("channel-mask", 0x3);
+                    s.set("channel-mask", gst::Bitmask(0x3));
                 } else {
                     let settings = self.settings.lock().unwrap();
                     if let Some(objs) = &settings.spatial_objects {

utils/livesync/src/livesync/imp.rs

@@ -39,7 +39,12 @@ fn audio_info_from_caps(
 fn duration_from_caps(caps: &gst::CapsRef) -> Option<gst::ClockTime> {
     caps.structure(0)
         .filter(|s| s.name().starts_with("video/") || s.name().starts_with("image/"))
-        .and_then(|s| s.get::<gst::Fraction>("framerate").ok())
+        .and_then(|s| {
+            s.get::<gst::Fraction>("framerate")
+                .ok()
+                .filter(|f| f.denom() != 1 || f.numer() != 0)
+                .or_else(|| s.get::<gst::Fraction>("max-framerate").ok())
+        })
         .filter(|framerate| framerate.denom() > 0 && framerate.numer() > 0)
         .and_then(|framerate| {
             gst::ClockTime::SECOND.mul_div_round(framerate.denom() as u64, framerate.numer() as u64)
@@ -748,6 +753,118 @@ impl LiveSync {
         }
     }
 
+    fn transform_caps(
+        &self,
+        direction: gst::PadDirection,
+        caps: &gst::Caps,
+        filter: Option<&gst::Caps>,
+    ) -> Option<gst::Caps> {
+        let other_caps = if direction == gst::PadDirection::Src {
+            let mut caps = caps.clone();
+            let mut x = gst::Caps::new_empty();
+
+            for s in caps.make_mut().iter_mut() {
+                if let Ok(Some(framerate)) = s.get_optional::<gst::Fraction>("framerate") {
+                    if framerate.numer() != 0 {
+                        let mut f = s.to_owned();
+                        f.set("framerate", gst::Fraction::new(0, 1));
+                        f.set("max-framerate", framerate);
+                        x.merge_structure(f);
+                    }
+                }
+            }
+
+            caps.merge(x);
+            caps
+        } else {
+            let mut caps = caps.clone();
+            let mut x = gst::Caps::new_empty();
+
+            for s in caps.make_mut().iter_mut() {
+                if let Ok(Some(framerate)) = s.get_optional::<gst::Fraction>("framerate") {
+                    if framerate.numer() == 0 && framerate.denom() == 1 {
+                        let mut f = s.to_owned();
+                        if let Ok(Some(r)) = f.get_optional::<gst::Fraction>("max-framerate") {
+                            f.set("framerate", r);
+                            f.remove_field("max-framerate");
+                        } else {
+                            f.remove_field("framerate");
+                        }
+                        x.merge_structure(f);
+                    }
+                }
+            }
+
+            caps.merge(x);
+            caps
+        };
+
+        gst::debug!(
+            CAT,
+            imp: self,
+            "Transformed caps from {} to {} in direction {:?}",
+            caps,
+            other_caps,
+            direction
+        );
+
+        if let Some(filter) = filter {
+            Some(filter.intersect_with_mode(&other_caps, gst::CapsIntersectMode::First))
+        } else {
+            Some(other_caps)
+        }
+    }
+
+    fn find_transform(&self, pad: &gst::Pad, caps: &gst::Caps) -> Option<gst::Caps> {
+        let otherpad = if *pad == self.srcpad {
+            &self.sinkpad
+        } else {
+            &self.srcpad
+        };
+
+        let othercaps = self.transform_caps(pad.direction(), caps, None);
+
+        let peercaps = otherpad.peer_query_caps(othercaps.as_ref());
+
+        let mut othercaps = if let Some(c) = othercaps {
+            peercaps.intersect_with_mode(&c, gst::CapsIntersectMode::First)
+        } else {
+            peercaps
+        };
+        othercaps.fixate();
+
+        gst::debug!(CAT, imp: self, "Input caps were {:?}, and got final caps {:?}", caps, othercaps);
+
+        Some(othercaps)
+    }
+
+    fn accept_caps(&self, direction: gst::PadDirection, caps: &gst::Caps) -> bool {
+        let other_caps = self.transform_caps(direction, caps, None);
+        if let Some(o) = other_caps {
+            !o.is_empty()
+        } else {
+            true
+        }
+    }
+
+    fn query_caps(&self, pad: &gst::Pad, filter: Option<&gst::Caps>) -> Option<gst::Caps> {
+        let otherpad = if *pad == self.srcpad {
+            &self.sinkpad
+        } else {
+            &self.srcpad
+        };
+
+        let peerfilter = if let Some(f) = filter {
+            self.transform_caps(pad.direction(), f, None)
+        } else {
+            None
+        };
+
+        let peercaps = otherpad.peer_query_caps(peerfilter.as_ref());
+
+        self.transform_caps(otherpad.direction(), &peercaps, filter)
+    }
+
     fn sink_query(&self, pad: &gst::Pad, query: &mut gst::QueryRef) -> bool {
         if query.is_serialized() {
             let (sender, receiver) = mpsc::sync_channel(1);
@@ -767,7 +884,21 @@ impl LiveSync {
             // If the sender gets dropped, we will also unblock
             receiver.recv().unwrap_or(false)
         } else {
-            gst::Pad::query_default(pad, Some(&*self.obj()), query)
+            match query.view_mut() {
+                gst::QueryViewMut::Caps(q) => {
+                    let caps = self.query_caps(pad, q.filter_owned().as_ref());
+                    q.set_result(&caps);
+                    true
+                }
+                gst::QueryViewMut::AcceptCaps(q) => {
+                    let ret = self.accept_caps(gst::PadDirection::Src, &q.caps_owned());
+
+                    q.set_result(ret);
+
+                    true
+                }
+                _ => gst::Pad::query_default(pad, Some(&*self.obj()), query),
+            }
         }
     }
 
@@ -793,6 +924,12 @@ impl LiveSync {
                 true
             }
 
+            gst::QueryViewMut::Caps(q) => {
+                let caps = self.query_caps(pad, q.filter_owned().as_ref());
+                q.set_result(&caps);
+                true
+            }
+
             _ => gst::Pad::query_default(pad, Some(&*self.obj()), query),
         }
     }
@@ -1069,7 +1206,7 @@ impl LiveSync {
                     }
 
                     gst::EventView::Caps(e) => {
-                        state.pending_caps = Some(e.caps_owned());
+                        state.pending_caps = self.find_transform(&self.sinkpad, &e.caps_owned());
                         push = false;
                     }
 

utils/livesync/tests/livesync.rs

@@ -95,6 +95,88 @@ fn test_audio_nonsinglesegment() {
     test_audio(false);
 }
 
+#[test]
+fn test_video_negotiate_framerate_fixed() {
+    init();
+
+    let mut h = gst_check::Harness::new("livesync");
+
+    let src_caps = gst::Caps::builder("video/x-raw")
+        .field("framerate", gst::Fraction::new(25, 1))
+        .build();
+    h.set_src_caps(src_caps.clone());
+
+    h.play();
+
+    let buffer = gst::Buffer::new();
+    assert!(h.push_and_pull(buffer).is_ok());
+
+    let current_caps = h
+        .sinkpad()
+        .expect("harness has no sinkpad")
+        .current_caps()
+        .expect("current caps missing");
+
+    assert_eq!(current_caps, src_caps);
+}
+
+#[test]
+fn test_video_negotiate_framerate_variable() {
+    init();
+
+    let mut h = gst_check::Harness::new("livesync");
+
+    let src_caps = gst::Caps::builder("video/x-raw")
+        .field("framerate", gst::Fraction::new(0, 1))
+        .field("max-framerate", gst::Fraction::new(30, 1))
+        .build();
+    h.set_src_caps(src_caps.clone());
+
+    h.play();
+
+    let buffer = gst::Buffer::new();
+    assert!(h.push_and_pull(buffer).is_ok());
+
+    let current_caps = h
+        .sinkpad()
+        .expect("harness has no sinkpad")
+        .current_caps()
+        .expect("current caps missing");
+
+    assert_eq!(current_caps, src_caps);
+}
+
+#[test]
+fn test_video_negotiate_framerate_downstream() {
+    init();
+
+    let mut h = gst_check::Harness::new("livesync");
+
+    let sink_caps = gst::Caps::builder("video/x-raw")
+        .field("framerate", gst::Fraction::new(60, 1))
+        .build();
+    h.set_sink_caps(sink_caps.clone());
+
+    let src_caps = gst::Caps::builder("video/x-raw")
+        .field("framerate", gst::Fraction::new(0, 1))
+        .field("max-framerate", gst::Fraction::new(60, 1))
+        .build();
+    h.set_src_caps(src_caps.clone());
+
+    h.play();
+
+    let buffer = gst::Buffer::new();
+    assert!(h.push_and_pull(buffer).is_ok());
+
+    let current_caps = h
+        .sinkpad()
+        .expect("harness has no sinkpad")
+        .current_caps()
+        .expect("current caps missing");
+
+    assert_eq!(current_caps, sink_caps);
+}
+
 fn test_video(singlesegment: bool) {
     init();
 

video/dav1d/src/dav1ddec/imp.rs

@@ -134,7 +134,7 @@ impl Dav1dDec {
         let matrix = match pic.matrix_coefficients() {
             pixel::MatrixCoefficients::Identity => gst_video::VideoColorMatrix::Rgb,
             pixel::MatrixCoefficients::BT709 => gst_video::VideoColorMatrix::Bt709,
-            pixel::MatrixCoefficients::Unspecified => gst_video::VideoColorMatrix::Unknown,
+            pixel::MatrixCoefficients::Unspecified => gst_video::VideoColorMatrix::Bt709,
             pixel::MatrixCoefficients::BT470M => gst_video::VideoColorMatrix::Fcc,
             pixel::MatrixCoefficients::BT470BG => gst_video::VideoColorMatrix::Bt601,
             pixel::MatrixCoefficients::ST240M => gst_video::VideoColorMatrix::Smpte240m,
@@ -149,7 +149,7 @@ impl Dav1dDec {
 
         let transfer = match pic.transfer_characteristic() {
             pixel::TransferCharacteristic::BT1886 => gst_video::VideoTransferFunction::Bt709,
-            pixel::TransferCharacteristic::Unspecified => gst_video::VideoTransferFunction::Unknown,
+            pixel::TransferCharacteristic::Unspecified => gst_video::VideoTransferFunction::Bt709,
             pixel::TransferCharacteristic::BT470M => gst_video::VideoTransferFunction::Bt709,
             pixel::TransferCharacteristic::BT470BG => gst_video::VideoTransferFunction::Gamma28,
             pixel::TransferCharacteristic::ST170M => gst_video::VideoTransferFunction::Bt601,
@@ -180,7 +180,7 @@ impl Dav1dDec {
 
         let primaries = match pic.color_primaries() {
             pixel::ColorPrimaries::BT709 => gst_video::VideoColorPrimaries::Bt709,
-            pixel::ColorPrimaries::Unspecified => gst_video::VideoColorPrimaries::Unknown,
+            pixel::ColorPrimaries::Unspecified => gst_video::VideoColorPrimaries::Bt709,
             pixel::ColorPrimaries::BT470M => gst_video::VideoColorPrimaries::Bt470m,
             pixel::ColorPrimaries::BT470BG => gst_video::VideoColorPrimaries::Bt470bg,
             pixel::ColorPrimaries::ST240M => gst_video::VideoColorPrimaries::Smpte240m,