video: Add a new ffv1 decoder plugin

Cargo.toml

@@ -17,6 +17,7 @@ members = [
     "video/cdg",
     "video/closedcaption",
     "video/dav1d",
+    "video/ffv1",
     "video/flavors",
     "video/gif",
     "video/rav1e",

deny.toml

@@ -136,5 +136,6 @@ allow-git = [
   "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs",
   "https://github.com/gtk-rs/gtk-rs-core",
   "https://github.com/fengalin/tokio",
+  "https://github.com/rust-av/ffv1",
   "https://github.com/rust-av/flavors",
 ]

meson.build

@@ -36,6 +36,7 @@ plugins_rep = {
   'video/cdg': 'libgstcdg',
   'audio/claxon': 'libgstclaxon',
   'utils/fallbackswitch': 'libgstfallbackswitch',
+  'video/ffv1': 'libgstffv1',
   'generic/file': 'libgstrsfile',
   'video/flavors': 'libgstrsflv',
   'video/gif': 'libgstgif',

video/ffv1/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "gst-plugin-ffv1"
+version = "0.6.0"
+authors = ["Arun Raghavan <arun@asymptotic.io>"]
+repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs"
+license = "MIT/Apache-2.0"
+description = "FFV1 Decoder Plugin"
+edition = "2018"
+
+[dependencies]
+ffv1 = { git = "https://github.com/rust-av/ffv1.git", rev = "2afb025a327173ce891954c052e804d0f880368a" }
+gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_18"] }
+gst-video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_18"] }
+once_cell = "1.0"
+
+[dev-dependencies]
+gst-check = { package = "gstreamer-check", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_18"] }
+[lib]
+name = "gstffv1"
+crate-type = ["cdylib", "rlib"]
+path = "src/lib.rs"
+
+[build-dependencies]
+gst-plugin-version-helper = { path="../../version-helper" }
+
+[features]
+static = ["gst/v1_18"]
+capi = []
+
+[package.metadata.capi]
+min_version = "0.8.0"
+
+[package.metadata.capi.header]
+enabled = false
+
+[package.metadata.capi.library]
+install_subdir = "gstreamer-1.0"
+versioning = false
+
+[package.metadata.capi.pkg_config]
+requires_private = "gstreamer-1.0, gstreamer-base-1.0, gstreamer-video-1.0, gobject-2.0, glib-2.0"

video/ffv1/LICENSE-APACHE

@@ -0,0 +1 @@
+../../LICENSE-APACHE

video/ffv1/LICENSE-MIT

@@ -0,0 +1 @@
+../../LICENSE-MIT

video/ffv1/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    gst_plugin_version_helper::info()
+}

video/ffv1/src/ffv1dec/imp.rs

@@ -0,0 +1,367 @@
+// Copyright (C) 2020 Arun Raghavan <arun@asymptotic.io>
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use ffv1::constants::{RGB, YCBCR};
+use ffv1::decoder::{Decoder, Frame};
+use ffv1::record::ConfigRecord;
+
+use gst::glib;
+use gst::prelude::*;
+use gst::subclass::prelude::*;
+use gst_video::prelude::*;
+use gst_video::subclass::prelude::*;
+use gst_video::VideoFormat;
+
+use once_cell::sync::Lazy;
+use std::sync::Mutex;
+
+enum DecoderState {
+    Stopped,
+    Started {
+        output_info: Option<gst_video::VideoInfo>,
+        decoder: Box<Decoder>,
+    },
+}
+
+impl Default for DecoderState {
+    fn default() -> Self {
+        DecoderState::Stopped
+    }
+}
+
+#[derive(Default)]
+pub struct Ffv1Dec {
+    state: Mutex<DecoderState>,
+}
+
+fn get_all_video_formats() -> Vec<glib::SendValue> {
+    let values = [
+        VideoFormat::Gray8,
+        // VideoFormat::Gray16Le,
+        // VideoFormat::Gray16Be,
+        VideoFormat::Y444,
+        // VideoFormat::Y44410le,
+        // VideoFormat::Y44410be,
+        // VideoFormat::A44410le,
+        // VideoFormat::A44410be,
+        // VideoFormat::Y44412le,
+        // VideoFormat::Y44412be,
+        // VideoFormat::Y44416le,
+        // VideoFormat::Y44416be,
+        VideoFormat::A420,
+        VideoFormat::Y42b,
+        // VideoFormat::I42210le,
+        // VideoFormat::I42210be,
+        // VideoFormat::A42210le,
+        // VideoFormat::A42210be,
+        // VideoFormat::I42212le,
+        // VideoFormat::I42212be,
+        VideoFormat::I420,
+        // VideoFormat::I42010le,
+        // VideoFormat::I42010be,
+        // VideoFormat::I42012le,
+        // VideoFormat::I42012be,
+        VideoFormat::Gbra,
+        VideoFormat::Gbr,
+        // VideoFormat::Gbr10le,
+        // VideoFormat::Gbr10be,
+        // VideoFormat::Gbra10le,
+        // VideoFormat::Gbra10be,
+        // VideoFormat::Gbr12le,
+        // VideoFormat::Gbr12be,
+        // VideoFormat::Gbra12le,
+        // VideoFormat::Gbra12be,
+    ];
+
+    values.iter().map(|i| i.to_str().to_send_value()).collect()
+}
+
+fn get_output_format(record: &ConfigRecord) -> Option<VideoFormat> {
+    const IS_LITTLE_ENDIAN: bool = cfg!(target_endian = "little");
+
+    match record.colorspace_type as usize {
+        YCBCR => match (
+            record.chroma_planes,
+            record.log2_v_chroma_subsample,
+            record.log2_h_chroma_subsample,
+            record.bits_per_raw_sample,
+            record.extra_plane,
+            IS_LITTLE_ENDIAN,
+        ) {
+            // Interpret luma-only as grayscale
+            (false, _, _, 8, false, _) => Some(VideoFormat::Gray8),
+            // (false, _, _, 16, false, true) => Some(VideoFormat::Gray16Le),
+            // (false, _, _, 16, false, false) => Some(VideoFormat::Gray16Be),
+            // 4:4:4
+            (true, 4, 4, 8, false, _) => Some(VideoFormat::Y444),
+            // (true, 4, 4, 10, false, true) => Some(VideoFormat::Y44410le),
+            // (true, 4, 4, 10, false, false) => Some(VideoFormat::Y44410be),
+            // (true, 4, 4, 10, true, true) => Some(VideoFormat::A44410le),
+            // (true, 4, 4, 10, true, false) => Some(VideoFormat::A44410be),
+            // (true, 4, 4, 12, false, true) => Some(VideoFormat::Y44412le),
+            // (true, 4, 4, 12, false, false) => Some(VideoFormat::Y44412be),
+            // (true, 4, 4, 16, false, true) => Some(VideoFormat::Y44416le),
+            // (true, 4, 4, 16, false, false) => Some(VideoFormat::Y44416be),
+            // 4:2:2
+            (true, 2, 2, 8, false, _) => Some(VideoFormat::Y42b),
+            // (true, 2, 2, 10, false, true) => Some(VideoFormat::I42210le),
+            // (true, 2, 2, 10, false, false) => Some(VideoFormat::I42210be),
+            // (true, 2, 2, 10, true, true) => Some(VideoFormat::A42210le),
+            // (true, 2, 2, 10, true, false) => Some(VideoFormat::A42210be),
+            // (true, 2, 2, 12, false, true) => Some(VideoFormat::I42212le),
+            // (true, 2, 2, 12, false, false) => Some(VideoFormat::I42212be),
+            // 4:2:0
+            (true, 1, 1, 8, false, _) => Some(VideoFormat::I420),
+            (true, 1, 1, 8, true, _) => Some(VideoFormat::A420),
+            // (true, 1, 1, 10, false, true) => Some(VideoFormat::I42010le),
+            // (true, 1, 1, 10, false, false) => Some(VideoFormat::I42010be),
+            // (true, 1, 1, 12, false, true) => Some(VideoFormat::I42012le),
+            // (true, 1, 1, 12, false, false) => Some(VideoFormat::I42012be),
+            // Nothing matched
+            (_, _, _, _, _, _) => None,
+        },
+        RGB => match (
+            record.bits_per_raw_sample,
+            record.extra_plane,
+            IS_LITTLE_ENDIAN,
+        ) {
+            (8, true, _) => Some(VideoFormat::Gbra),
+            (8, false, _) => Some(VideoFormat::Gbr),
+            // (10, false, true) => Some(VideoFormat::Gbr10le),
+            // (10, false, false) => Some(VideoFormat::Gbr10be),
+            // (10, true, true) => Some(VideoFormat::Gbra10le),
+            // (10, true, false) => Some(VideoFormat::Gbra10be),
+            // (12, false, true) => Some(VideoFormat::Gbr12le),
+            // (12, false, false) => Some(VideoFormat::Gbr12be),
+            // (12, true, true) => Some(VideoFormat::Gbra12le),
+            // (12, true, false) => Some(VideoFormat::Gbra12be),
+            (_, _, _) => None,
+        },
+        _ => panic!("Unknown color_space type"),
+    }
+}
+
+impl Ffv1Dec {
+    // FIXME: Implement other pixel depths
+    pub fn get_decoded_frame(
+        &self,
+        mut decoded_frame: Frame,
+        output_info: &gst_video::VideoInfo,
+    ) -> gst::Buffer {
+        let mut buf = gst::Buffer::new();
+        let mut_buf = buf.make_mut();
+        let format_info = output_info.format_info();
+
+        // Greater depths are not yet supported
+        assert_eq!(decoded_frame.bit_depth, 8);
+
+        for (plane, decoded_plane) in decoded_frame.buf.drain(..).enumerate() {
+            let component = format_info
+                .plane()
+                .iter()
+                .position(|&p| p == plane as u32)
+                .unwrap() as u8;
+
+            let comp_height = format_info.scale_height(component, output_info.height()) as usize;
+            let src_stride = decoded_plane.len() / comp_height;
+            let dest_stride = output_info.stride()[plane] as usize;
+
+            // FIXME: we can also do this if we have video meta support and differing strides
+            let mem = if src_stride == dest_stride {
+                // Just wrap the decoded frame vecs and push them out
+                gst::Memory::from_slice(decoded_plane)
+            } else {
+                // Mismatched stride, let's copy
+                let out_plane = gst::Memory::with_size(dest_stride * comp_height);
+                let mut out_plane_mut = out_plane.into_mapped_memory_writable().unwrap();
+
+                for (in_line, out_line) in decoded_plane
+                    .as_slice()
+                    .chunks_exact(src_stride)
+                    .zip(out_plane_mut.as_mut_slice().chunks_exact_mut(dest_stride))
+                {
+                    out_line[..src_stride].copy_from_slice(in_line);
+                }
+
+                out_plane_mut.into_memory()
+            };
+
+            mut_buf.append_memory(mem);
+        }
+
+        // FIXME: attach video meta if supported
+
+        buf
+    }
+}
+
+static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
+    gst::DebugCategory::new(
+        "ffv1dec",
+        gst::DebugColorFlags::empty(),
+        Some("FFV1 decoder"),
+    )
+});
+
+#[glib::object_subclass]
+impl ObjectSubclass for Ffv1Dec {
+    const NAME: &'static str = "Ffv1Dec";
+    type Type = super::Ffv1Dec;
+    type ParentType = gst_video::VideoDecoder;
+}
+
+impl ObjectImpl for Ffv1Dec {}
+
+impl ElementImpl for Ffv1Dec {
+    fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
+        static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
+            gst::subclass::ElementMetadata::new(
+                "FFV1 Decoder",
+                "Codec/Decoder/Video",
+                "Decode FFV1 video streams",
+                "Arun Raghavan <arun@asymptotic.io>",
+            )
+        });
+
+        Some(&*ELEMENT_METADATA)
+    }
+
+    fn pad_templates() -> &'static [gst::PadTemplate] {
+        static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
+            let sink_caps = gst::Caps::builder("video/x-ffv")
+                .field("ffvversion", &1)
+                .field("width", &gst::IntRange::<i32>::new(1, i32::MAX))
+                .field("height", &gst::IntRange::<i32>::new(1, i32::MAX))
+                .field(
+                    "framerate",
+                    &gst::FractionRange::new(
+                        gst::Fraction::new(0, 1),
+                        gst::Fraction::new(i32::MAX, 1),
+                    ),
+                )
+                .build();
+            let sink_pad_template = gst::PadTemplate::new(
+                "sink",
+                gst::PadDirection::Sink,
+                gst::PadPresence::Always,
+                &sink_caps,
+            )
+            .unwrap();
+
+            let src_caps = gst::Caps::builder("video/x-raw")
+                .field("format", &gst::List::from_owned(get_all_video_formats()))
+                .field("width", &gst::IntRange::<i32>::new(1, i32::MAX))
+                .field("height", &gst::IntRange::<i32>::new(1, i32::MAX))
+                .field(
+                    "framerate",
+                    &gst::FractionRange::new(
+                        gst::Fraction::new(0, 1),
+                        gst::Fraction::new(i32::MAX, 1),
+                    ),
+                )
+                .build();
+            let src_pad_template = gst::PadTemplate::new(
+                "src",
+                gst::PadDirection::Src,
+                gst::PadPresence::Always,
+                &src_caps,
+            )
+            .unwrap();
+
+            vec![sink_pad_template, src_pad_template]
+        });
+
+        &*PAD_TEMPLATES
+    }
+}
+
+impl VideoDecoderImpl for Ffv1Dec {
+    /* We allocate the decoder here rather than start() because we need the sink caps */
+    fn set_format(
+        &self,
+        element: &super::Ffv1Dec,
+        state: &gst_video::VideoCodecState<'static, gst_video::video_codec_state::Readable>,
+    ) -> Result<(), gst::LoggableError> {
+        let info = state.info();
+        let codec_data = state
+            .codec_data()
+            .ok_or_else(|| gst::loggable_error!(CAT, "Missing codec_data"))?
+            .map_readable()?;
+        let decoder = Decoder::new(codec_data.as_slice(), info.width(), info.height())
+            .map_err(|err| gst::loggable_error!(CAT, "Could not instantiate decoder: {}", err))?;
+
+        let format = get_output_format(decoder.config_record())
+            .ok_or_else(|| gst::loggable_error!(CAT, "Unsupported format"))?;
+
+        let output_state = element
+            .set_output_state(format, info.width(), info.height(), Some(state))
+            .map_err(|err| gst::loggable_error!(CAT, "Failed to set output params: {}", err))?;
+
+        let output_info = Some(output_state.info());
+
+        element
+            .negotiate(output_state)
+            .map_err(|err| gst::loggable_error!(CAT, "Negotiation failed: {}", err))?;
+
+        let mut decoder_state = self.state.lock().unwrap();
+        *decoder_state = DecoderState::Started {
+            output_info,
+            decoder: Box::new(decoder),
+        };
+
+        self.parent_set_format(element, state)
+    }
+
+    fn stop(&self, element: &super::Ffv1Dec) -> Result<(), gst::ErrorMessage> {
+        let mut decoder_state = self.state.lock().unwrap();
+        *decoder_state = DecoderState::Stopped;
+
+        self.parent_stop(element)
+    }
+
+    fn handle_frame(
+        &self,
+        element: &super::Ffv1Dec,
+        mut frame: gst_video::VideoCodecFrame,
+    ) -> Result<gst::FlowSuccess, gst::FlowError> {
+        let mut state = self.state.lock().unwrap();
+
+        let (output_info, decoder) = match *state {
+            DecoderState::Stopped => Err(gst::FlowError::Error),
+            DecoderState::Started {
+                ref mut output_info,
+                ref mut decoder,
+            } => Ok((output_info, decoder)),
+        }?;
+
+        let input_buffer = frame
+            .input_buffer()
+            .expect("Frame must have input buffer")
+            .map_readable()
+            .expect("Could not map input buffer for read");
+
+        let decoded_frame = decoder.decode_frame(input_buffer.as_slice()).map_err(|e| {
+            gst::gst_error!(CAT, "Decoding failed: {}", e);
+            gst::FlowError::Error
+        })?;
+
+        // Drop so we can mutably borrow frame later
+        drop(input_buffer);
+
+        //  * Make sure the decoder and output plane orders match for all cases
+        let buf = self.get_decoded_frame(decoded_frame, output_info.as_ref().unwrap());
+
+        // We no longer need the state lock
+        drop(state);
+
+        frame.set_output_buffer(buf);
+        element.finish_frame(frame)?;
+
+        Ok(gst::FlowSuccess::Ok)
+    }
+}

video/ffv1/src/ffv1dec/mod.rs

@@ -0,0 +1,30 @@
+// Copyright (C) 2021 Arun Raghavan <arun@asymptotic.io>
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use gst::glib;
+use gst::prelude::*;
+
+mod imp;
+
+glib::wrapper! {
+    pub struct Ffv1Dec(ObjectSubclass<imp::Ffv1Dec>) @extends gst_video::VideoDecoder, gst::Element, gst::Object;
+}
+
+// GStreamer elements need to be thread-safe. For the private implementation this is automatically
+// enforced but for the public wrapper type we need to specify this manually.
+unsafe impl Send for Ffv1Dec {}
+unsafe impl Sync for Ffv1Dec {}
+
+pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
+    gst::Element::register(
+        Some(plugin),
+        "ffv1dec",
+        gst::Rank::Primary + 1,
+        Ffv1Dec::static_type(),
+    )
+}

video/ffv1/src/lib.rs

@@ -0,0 +1,25 @@
+// Copyright (C) 2019 Arun Raghavan <arun@asymptotic.io>
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+mod ffv1dec;
+
+fn plugin_init(plugin: &gst::Plugin) -> Result<(), gst::glib::BoolError> {
+    ffv1dec::register(plugin)
+}
+
+gst::plugin_define!(
+    ffv1,
+    env!("CARGO_PKG_DESCRIPTION"),
+    plugin_init,
+    concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")),
+    "MIT/X11",
+    env!("CARGO_PKG_NAME"),
+    env!("CARGO_PKG_NAME"),
+    env!("CARGO_PKG_REPOSITORY"),
+    env!("BUILD_REL_DATE")
+);

video/ffv1/tests/ffv1dec.rs

@@ -0,0 +1,62 @@
+// Copyright (C) 2019 Sebastian Dröge <sebastian@centricular.com>
+// Copyright (C) 2021 Arun Raghavan <arun@asymptotic.io>
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use gst::glib;
+use gst::prelude::*;
+
+use std::fs;
+use std::path::PathBuf;
+
+fn init() {
+    use std::sync::Once;
+    static INIT: Once = Once::new();
+
+    INIT.call_once(|| {
+        gst::init().unwrap();
+        gstffv1::plugin_register_static().expect("ffv1 test");
+    });
+}
+
+#[test]
+fn test_decode_yuv420p() {
+    init();
+    test_decode("yuv420p");
+}
+
+fn test_decode(name: &str) {
+    let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    path.push(format!("tests/ffv1_v3_{}.mkv", name));
+
+    let bin = gst::parse_bin_from_description(
+        &format!(
+            "filesrc location={:?} ! matroskademux name=m m.video_0 ! ffv1dec name=ffv1dec",
+            path
+        ),
+        false,
+    )
+    .unwrap();
+
+    let srcpad = bin.by_name("ffv1dec").unwrap().static_pad("src").unwrap();
+    let _ = bin.add_pad(&gst::GhostPad::with_target(Some("src"), &srcpad).unwrap());
+
+    let mut h = gst_check::Harness::with_element(&bin, None, Some("src"));
+
+    h.play();
+
+    let buf = h.pull().unwrap();
+    let frame = buf.into_mapped_buffer_readable().unwrap();
+
+    let mut refpath = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    refpath.push(format!("tests/ffv1_v3_{}.ref", name));
+
+    let ref_frame = fs::read(refpath).unwrap();
+
+    assert_eq!(frame.len(), ref_frame.len());
+    assert_eq!(frame.as_slice(), glib::Bytes::from_owned(ref_frame));
+}