mux/mp4: support ISO/IEC 23001-17 uncompressed encoding

This adds support for direct encoding of common formats into ISO base media file
format.

There are unit tests for formats that are not completely supported, to
check that those functions work correctly, and to ease future extension.

End-to-end testing currently requires use of gpac to validate files.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1990>

Cargo.lock

@@ -2788,6 +2788,7 @@ dependencies = [
  "gstreamer-base",
  "gstreamer-pbutils",
  "gstreamer-video",
+ "num-integer",
  "tempfile",
  "url",
 ]

docs/plugins/gst_plugins_cache.json

@@ -4571,7 +4571,7 @@
                 "klass": "Codec/Muxer",
                 "pad-templates": {
                     "sink_%%u": {
-                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp8:\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp9:\n        profile: { (string)0, (string)1, (string)2, (string)3 }\n  chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-av1:\n  stream-format: obu-stream\n      alignment: tu\n        profile: { (string)main, (string)high, (string)professional }\n  chroma-format: { (string)4:0:0, (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n       channels: [ 1, 8 ]\n           rate: [ 1, 2147483647 ]\naudio/x-flac:\n         framed: true\n       channels: [ 1, 8 ]\n           rate: [ 1, 655350 ]\n",
+                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp8:\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp9:\n        profile: { (string)0, (string)1, (string)2, (string)3 }\n  chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-av1:\n  stream-format: obu-stream\n      alignment: tu\n        profile: { (string)main, (string)high, (string)professional }\n  chroma-format: { (string)4:0:0, (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-raw:\n         format: { IYU2, RGB, BGR, NV12, NV21, RGBA, ARGB, ABGR, BGRA, RGBx, BGRx, Y444, AYUV, GRAY8, GRAY16_BE, GBR, RGBP, BGRP, v308, r210 }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\nvideo/x-raw:\n         format: { Y41B, NV16, NV61, Y42B }\n          width: [ 4, 2147483644, 4 ]\n         height: [ 1, 2147483647 ]\nvideo/x-raw:\n         format: { I420, YV12, YUY2, YVYU, UYVY, VYUY }\n          width: [ 4, 2147483644, 4 ]\n         height: [ 2, 2147483646, 2 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n       channels: [ 1, 8 ]\n           rate: [ 1, 2147483647 ]\naudio/x-flac:\n         framed: true\n       channels: [ 1, 8 ]\n           rate: [ 1, 655350 ]\n",
                         "direction": "sink",
                         "presence": "request",
                         "type": "GstRsMP4MuxPad"

mux/mp4/Cargo.toml

@@ -13,9 +13,10 @@ anyhow = "1"
 gst = { workspace = true,  features = ["v1_18"] }
 gst-base = { workspace = true, features = ["v1_18"] }
 gst-audio = { workspace = true, features = ["v1_18"] }
-gst-video = { workspace = true, features = ["v1_18"] }
+gst-video = { workspace = true, features = ["v1_20"] }
 gst-pbutils = { workspace = true, features = ["v1_18"] }
 bitstream-io = "2.3"
+num-integer = { version = "0.1", default-features = false, features = [] }
 
 [lib]
 name = "gstmp4"

mux/mp4/src/mp4mux/imp.rs

@@ -13,6 +13,7 @@ use gst::subclass::prelude::*;
 use gst_base::prelude::*;
 use gst_base::subclass::prelude::*;
 
+use num_integer::Integer;
 use std::collections::VecDeque;
 use std::sync::Mutex;
 
@@ -1166,6 +1167,7 @@ impl MP4Mux {
                     delta_frames = super::DeltaFrames::PredictiveOnly;
                 }
                 "image/jpeg" => (),
+                "video/x-raw" => (),
                 "audio/mpeg" => {
                     if !s.has_field_with_type("codec_data", gst::Buffer::static_type()) {
                         gst::error!(CAT, obj = pad, "Received caps without codec_data");
@@ -1875,6 +1877,67 @@ impl ElementImpl for ISOMP4Mux {
                         .field("width", gst::IntRange::new(1, u16::MAX as i32))
                         .field("height", gst::IntRange::new(1, u16::MAX as i32))
                         .build(),
+                    gst::Structure::builder("video/x-raw")
+                        // TODO: this could be extended to handle gst_video::VideoMeta for non-default stride and plane offsets
+                        .field(
+                            "format",
+                            // formats that do not use subsampling
+                            // Plus NV12 and NV21 because that works OK with the interleaved planes
+                            gst::List::new([
+                                "IYU2",
+                                "RGB",
+                                "BGR",
+                                "NV12",
+                                "NV21",
+                                "RGBA",
+                                "ARGB",
+                                "ABGR",
+                                "BGRA",
+                                "RGBx",
+                                "BGRx",
+                                "Y444",
+                                "AYUV",
+                                "GRAY8",
+                                "GRAY16_BE",
+                                "GBR",
+                                "RGBP",
+                                "BGRP",
+                                "v308",
+                                "r210",
+                            ]),
+                        )
+                        .field("width", gst::IntRange::new(1, i32::MAX))
+                        .field("height", gst::IntRange::new(1, i32::MAX))
+                        .build(),
+                    gst::Structure::builder("video/x-raw")
+                        // TODO: this could be extended to handle gst_video::VideoMeta for non-default stride and plane offsets
+                        .field(
+                            "format",
+                            // Formats that use horizontal subsampling, but not vertical subsampling (4:2:2 and 4:1:1)
+                            gst::List::new(["Y41B", "NV16", "NV61", "Y42B"]),
+                        )
+                        .field(
+                            "width",
+                            gst::IntRange::with_step(4, i32::MAX.prev_multiple_of(&4), 4),
+                        )
+                        .field("height", gst::IntRange::new(1, i32::MAX))
+                        .build(),
+                    gst::Structure::builder("video/x-raw")
+                        // TODO: this could be extended to handle gst_video::VideoMeta for non-default stride and plane offsets
+                        .field(
+                            "format",
+                            // Formats that use both horizontal and vertical subsampling (4:2:0)
+                            gst::List::new(["I420", "YV12", "YUY2", "YVYU", "UYVY", "VYUY"]),
+                        )
+                        .field(
+                            "width",
+                            gst::IntRange::with_step(4, i32::MAX.prev_multiple_of(&4), 4),
+                        )
+                        .field(
+                            "height",
+                            gst::IntRange::with_step(2, i32::MAX.prev_multiple_of(&2), 2),
+                        )
+                        .build(),
                     gst::Structure::builder("audio/mpeg")
                         .field("mpegversion", 4i32)
                         .field("stream-format", "raw")

mux/mp4/tests/tests.rs

@@ -172,3 +172,221 @@ fn test_roundtrip_av1_aac() {
         pipeline.into_completion();
     })
 }
+
+fn test_encode_uncompressed(video_format: &str, width: u32, height: u32) {
+    let pipeline_text = format!("videotestsrc num-buffers=250 ! video/x-raw,format={format},width={width},height={height} ! isomp4mux ! filesink location={format}_{width}x{height}.mp4", format = video_format);
+    let Ok(pipeline) = gst::parse::launch(&pipeline_text) else {
+        panic!("could not build encoding pipeline")
+    };
+    pipeline
+        .set_state(gst::State::Playing)
+        .expect("Unable to set the pipeline to the `Playing` state");
+    for msg in pipeline.bus().unwrap().iter_timed(gst::ClockTime::NONE) {
+        use gst::MessageView;
+
+        match msg.view() {
+            MessageView::Eos(..) => break,
+            MessageView::Error(err) => {
+                panic!(
+                    "Error from {:?}: {} ({:?})",
+                    err.src().map(|s| s.path_string()),
+                    err.error(),
+                    err.debug()
+                );
+            }
+            _ => (),
+        }
+    }
+    pipeline
+        .set_state(gst::State::Null)
+        .expect("Unable to set the pipeline to the `Null` state");
+}
+
+#[test]
+fn encode_uncompressed_iyu2() {
+    init();
+    test_encode_uncompressed("IYU2", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_rgb() {
+    init();
+    test_encode_uncompressed("RGB", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_rgb_no_pad() {
+    init();
+    test_encode_uncompressed("RGB", 1280, 720);
+}
+
+#[test]
+fn encode_uncompressed_bgr() {
+    init();
+    test_encode_uncompressed("BGR", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_nv12() {
+    init();
+    test_encode_uncompressed("NV12", 1275, 714);
+}
+
+#[test]
+fn encode_uncompressed_nv21() {
+    init();
+    test_encode_uncompressed("NV21", 1275, 714);
+}
+
+#[test]
+fn encode_uncompressed_rgba() {
+    init();
+    test_encode_uncompressed("RGBA", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_argb() {
+    init();
+    test_encode_uncompressed("ARGB", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_abgr() {
+    init();
+    test_encode_uncompressed("ABGR", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_bgra() {
+    init();
+    test_encode_uncompressed("BGRA", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_rgbx() {
+    init();
+    test_encode_uncompressed("RGBx", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_bgrx() {
+    init();
+    test_encode_uncompressed("BGRx", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_y444() {
+    init();
+    test_encode_uncompressed("Y444", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_i420() {
+    init();
+    test_encode_uncompressed("I420", 1280, 720);
+}
+
+#[test]
+fn encode_uncompressed_yv12() {
+    init();
+    test_encode_uncompressed("YV12", 1280, 720);
+}
+
+#[test]
+fn encode_uncompressed_yuy2() {
+    init();
+    test_encode_uncompressed("YUY2", 320, 120);
+}
+
+#[test]
+fn encode_uncompressed_yvyu() {
+    init();
+    test_encode_uncompressed("YVYU", 320, 120);
+}
+
+#[test]
+fn encode_uncompressed_vyuy() {
+    init();
+    test_encode_uncompressed("VYUY", 320, 120);
+}
+
+#[test]
+fn encode_uncompressed_uyvy() {
+    init();
+    test_encode_uncompressed("UYVY", 320, 120);
+}
+
+/*
+TODO: report YA4p unknown pixel format to GPAC
+*/
+#[test]
+fn encode_uncompressed_ayuv() {
+    init();
+    test_encode_uncompressed("AYUV", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_y41b() {
+    init();
+    test_encode_uncompressed("Y41B", 1280, 713);
+}
+
+#[test]
+fn encode_uncompressed_y42b() {
+    init();
+    test_encode_uncompressed("Y42B", 1280, 713);
+}
+
+#[test]
+fn encode_uncompressed_v308() {
+    init();
+    test_encode_uncompressed("v308", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_gray8() {
+    init();
+    test_encode_uncompressed("GRAY8", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_gray16_be() {
+    init();
+    test_encode_uncompressed("GRAY16_BE", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_r210() {
+    init();
+    test_encode_uncompressed("r210", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_nv16() {
+    init();
+    test_encode_uncompressed("NV16", 1280, 713);
+}
+
+#[test]
+fn encode_uncompressed_nv61() {
+    init();
+    test_encode_uncompressed("NV61", 1280, 713);
+}
+
+#[test]
+fn encode_uncompressed_gbr() {
+    init();
+    test_encode_uncompressed("GBR", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_rgbp() {
+    init();
+    test_encode_uncompressed("RGBP", 1275, 713);
+}
+
+#[test]
+fn encode_uncompressed_bgrp() {
+    init();
+    test_encode_uncompressed("BGRP", 1275, 713);
+}