gst-plugin-mp4: Add new MP4 plugin with a non-fragmented MP4 muxer

Cargo.toml

@@ -16,6 +16,7 @@ members = [
 
     "mux/flavors",
     "mux/fmp4",
+    "mux/mp4",
 
     "net/aws",
     "net/hlssink3",
@@ -63,6 +64,7 @@ default-members = [
     "generic/threadshare",
 
     "mux/fmp4",
+    "mux/mp4",
 
     "net/aws",
     "net/hlssink3",

docs/plugins/gst_plugins_cache.json

@@ -2497,6 +2497,129 @@
         "tracers": {},
         "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs"
     },
+    "mp4": {
+        "description": "GStreamer Rust MP4 Plugin",
+        "elements": {
+            "isomp4mux": {
+                "author": "Sebastian Dröge <sebastian@centricular.com>",
+                "description": "ISO MP4 muxer",
+                "hierarchy": [
+                    "GstISOMP4Mux",
+                    "GstRsMP4Mux",
+                    "GstAggregator",
+                    "GstElement",
+                    "GstObject",
+                    "GInitiallyUnowned",
+                    "GObject"
+                ],
+                "klass": "Codec/Muxer",
+                "pad-templates": {
+                    "sink_%%u": {
+                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp9:\n        profile: { (string)0, (string)1, (string)2, (string)3 }\n  chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n       channels: [ 1, 8 ]\n           rate: [ 1, 2147483647 ]\n",
+                        "direction": "sink",
+                        "presence": "request",
+                        "type": "GstRsMP4MuxPad"
+                    },
+                    "src": {
+                        "caps": "video/quicktime:\n        variant: iso\n",
+                        "direction": "src",
+                        "presence": "always"
+                    }
+                },
+                "rank": "marginal"
+            }
+        },
+        "filename": "gstmp4",
+        "license": "MPL",
+        "other-types": {
+            "GstRsMP4Mux": {
+                "hierarchy": [
+                    "GstRsMP4Mux",
+                    "GstAggregator",
+                    "GstElement",
+                    "GstObject",
+                    "GInitiallyUnowned",
+                    "GObject"
+                ],
+                "kind": "object",
+                "properties": {
+                    "interleave-bytes": {
+                        "blurb": "Interleave between streams in bytes",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "0",
+                        "max": "18446744073709551615",
+                        "min": "0",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "guint64",
+                        "writable": true
+                    },
+                    "interleave-time": {
+                        "blurb": "Interleave between streams in nanoseconds",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "500000000",
+                        "max": "18446744073709551615",
+                        "min": "0",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "guint64",
+                        "writable": true
+                    },
+                    "movie-timescale": {
+                        "blurb": "Timescale to use for the movie (units per second, 0 is automatic)",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "0",
+                        "max": "-1",
+                        "min": "0",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "guint",
+                        "writable": true
+                    }
+                }
+            },
+            "GstRsMP4MuxPad": {
+                "hierarchy": [
+                    "GstRsMP4MuxPad",
+                    "GstAggregatorPad",
+                    "GstPad",
+                    "GstObject",
+                    "GInitiallyUnowned",
+                    "GObject"
+                ],
+                "kind": "object",
+                "properties": {
+                    "trak-timescale": {
+                        "blurb": "Timescale to use for the track (units per second, 0 is automatic)",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "0",
+                        "max": "-1",
+                        "min": "0",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "guint",
+                        "writable": true
+                    }
+                }
+            }
+        },
+        "package": "gst-plugin-mp4",
+        "source": "gst-plugin-mp4",
+        "tracers": {},
+        "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs"
+    },
     "ndi": {
         "description": "GStreamer NewTek NDI Plugin",
         "elements": {

meson.build

@@ -49,6 +49,7 @@ plugins = {
   # sodium has an external dependency, see below
   'gst-plugin-threadshare': 'libgstthreadshare',
 
+  'gst-plugin-mp4': 'libgstmp4',
   'gst-plugin-fmp4': 'libgstfmp4',
 
   'gst-plugin-aws': 'libgstaws',

mux/mp4/Cargo.toml

@@ -0,0 +1,50 @@
+[package]
+name = "gst-plugin-mp4"
+version = "0.10.0-alpha.1"
+authors = ["Sebastian Dröge <sebastian@centricular.com>"]
+license = "MPL-2.0"
+description = "GStreamer Rust MP4 Plugin"
+repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs"
+edition = "2021"
+rust-version = "1.63"
+
+[dependencies]
+anyhow = "1"
+gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
+gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
+gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
+gst-video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
+gst-pbutils = { package = "gstreamer-pbutils", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
+once_cell = "1.0"
+
+[lib]
+name = "gstmp4"
+crate-type = ["cdylib", "rlib"]
+path = "src/lib.rs"
+
+[dev-dependencies]
+tempfile = "3"
+url = "1"
+
+[build-dependencies]
+gst-plugin-version-helper = { path="../../version-helper" }
+
+[features]
+default = ["v1_18"]
+static = []
+capi = []
+v1_18 = ["gst-video/v1_18"]
+doc = ["gst/v1_18"]
+
+[package.metadata.capi]
+min_version = "0.8.0"
+
+[package.metadata.capi.header]
+enabled = false
+
+[package.metadata.capi.library]
+install_subdir = "gstreamer-1.0"
+versioning = false
+
+[package.metadata.capi.pkg_config]
+requires_private = "gstreamer-1.0, gstreamer-base-1.0, gstreamer-audio-1.0, gstreamer-video-1.0, gobject-2.0, glib-2.0, gmodule-2.0"

mux/mp4/LICENSE

@@ -0,0 +1 @@
+../../LICENSE-MPL-2.0

mux/mp4/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    gst_plugin_version_helper::info()
+}

mux/mp4/src/lib.rs

@@ -0,0 +1,34 @@
+// Copyright (C) 2022 Sebastian Dröge <sebastian@centricular.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
+// If a copy of the MPL was not distributed with this file, You can obtain one at
+// <https://mozilla.org/MPL/2.0/>.
+//
+// SPDX-License-Identifier: MPL-2.0
+#![allow(clippy::non_send_fields_in_send_ty, unused_doc_comments)]
+
+/**
+ * plugin-mp4:
+ *
+ * Since: plugins-rs-0.10.0
+ */
+use gst::glib;
+
+mod mp4mux;
+
+fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
+    mp4mux::register(plugin)
+}
+
+gst::plugin_define!(
+    mp4,
+    env!("CARGO_PKG_DESCRIPTION"),
+    plugin_init,
+    concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")),
+    // FIXME: MPL-2.0 is only allowed since 1.18.3 (as unknown) and 1.20 (as known)
+    "MPL",
+    env!("CARGO_PKG_NAME"),
+    env!("CARGO_PKG_NAME"),
+    env!("CARGO_PKG_REPOSITORY"),
+    env!("BUILD_REL_DATE")
+);

mux/mp4/src/mp4mux/mod.rs

@@ -0,0 +1,134 @@
+// Copyright (C) 2022 Sebastian Dröge <sebastian@centricular.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
+// If a copy of the MPL was not distributed with this file, You can obtain one at
+// <https://mozilla.org/MPL/2.0/>.
+//
+// SPDX-License-Identifier: MPL-2.0
+
+use gst::glib;
+use gst::prelude::*;
+
+mod boxes;
+mod imp;
+
+glib::wrapper! {
+    pub(crate) struct MP4MuxPad(ObjectSubclass<imp::MP4MuxPad>) @extends gst_base::AggregatorPad, gst::Pad, gst::Object;
+}
+
+glib::wrapper! {
+    pub(crate) struct MP4Mux(ObjectSubclass<imp::MP4Mux>) @extends gst_base::Aggregator, gst::Element, gst::Object;
+}
+
+glib::wrapper! {
+    pub(crate) struct ISOMP4Mux(ObjectSubclass<imp::ISOMP4Mux>) @extends MP4Mux, gst_base::Aggregator, gst::Element, gst::Object;
+}
+
+pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
+    #[cfg(feature = "doc")]
+    {
+        MP4Mux::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
+        MP4MuxPad::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
+    }
+    gst::Element::register(
+        Some(plugin),
+        "isomp4mux",
+        gst::Rank::Marginal,
+        ISOMP4Mux::static_type(),
+    )?;
+
+    Ok(())
+}
+
+#[derive(Debug, Copy, Clone)]
+pub(crate) enum DeltaFrames {
+    /// Only single completely decodable frames
+    IntraOnly,
+    /// Frames may depend on past frames
+    PredictiveOnly,
+    /// Frames may depend on past or future frames
+    Bidirectional,
+}
+
+impl DeltaFrames {
+    /// Whether dts is required to order samples differently from presentation order
+    pub(crate) fn requires_dts(&self) -> bool {
+        matches!(self, Self::Bidirectional)
+    }
+    /// Whether this coding structure does not allow delta flags on samples
+    pub(crate) fn intra_only(&self) -> bool {
+        matches!(self, Self::IntraOnly)
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct Sample {
+    /// Sync point
+    sync_point: bool,
+
+    /// Sample duration
+    duration: gst::ClockTime,
+
+    /// Composition time offset
+    ///
+    /// This is `None` for streams that have no concept of DTS.
+    composition_time_offset: Option<i64>,
+
+    /// Size
+    size: u32,
+}
+
+#[derive(Debug)]
+pub(crate) struct Chunk {
+    /// Chunk start offset
+    offset: u64,
+
+    /// Samples of this stream that are part of this chunk
+    samples: Vec<Sample>,
+}
+
+#[derive(Debug)]
+pub(crate) struct Stream {
+    /// Caps of this stream
+    caps: gst::Caps,
+
+    /// If this stream has delta frames, and if so if it can have B frames.
+    delta_frames: DeltaFrames,
+
+    /// Pre-defined trak timescale if not 0.
+    trak_timescale: u32,
+
+    /// Start DTS
+    ///
+    /// If this is negative then an edit list entry is needed to
+    /// make all sample times positive.
+    ///
+    /// This is `None` for streams that have no concept of DTS.
+    start_dts: Option<gst::Signed<gst::ClockTime>>,
+
+    /// Earliest PTS
+    ///
+    /// If this is >0 then an edit list entry is needed to shift
+    earliest_pts: gst::ClockTime,
+
+    /// End PTS
+    end_pts: gst::ClockTime,
+
+    /// All the chunks stored for this stream
+    chunks: Vec<Chunk>,
+}
+
+#[derive(Debug)]
+pub(crate) struct Header {
+    #[allow(dead_code)]
+    variant: Variant,
+    /// Pre-defined movie timescale if not 0.
+    movie_timescale: u32,
+    streams: Vec<Stream>,
+}
+
+#[allow(clippy::upper_case_acronyms)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum Variant {
+    ISO,
+}

mux/mp4/tests/tests.rs

@@ -0,0 +1,132 @@
+// Copyright (C) 2022 Sebastian Dröge <sebastian@centricular.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
+// If a copy of the MPL was not distributed with this file, You can obtain one at
+// <https://mozilla.org/MPL/2.0/>.
+//
+// SPDX-License-Identifier: MPL-2.0
+//
+
+use gst::prelude::*;
+use gst_pbutils::prelude::*;
+
+fn init() {
+    use std::sync::Once;
+    static INIT: Once = Once::new();
+
+    INIT.call_once(|| {
+        gst::init().unwrap();
+        gstmp4::plugin_register_static().unwrap();
+    });
+}
+
+#[test]
+fn test_basic() {
+    init();
+
+    struct Pipeline(gst::Pipeline);
+    impl std::ops::Deref for Pipeline {
+        type Target = gst::Pipeline;
+
+        fn deref(&self) -> &Self::Target {
+            &self.0
+        }
+    }
+    impl Drop for Pipeline {
+        fn drop(&mut self) {
+            let _ = self.0.set_state(gst::State::Null);
+        }
+    }
+
+    let pipeline = match gst::parse_launch(
+        "videotestsrc num-buffers=99 ! x264enc ! mux. \
+         audiotestsrc num-buffers=140 ! fdkaacenc ! mux. \
+         isomp4mux name=mux ! filesink name=sink \
+    ",
+    ) {
+        Ok(pipeline) => Pipeline(pipeline.downcast::<gst::Pipeline>().unwrap()),
+        Err(_) => return,
+    };
+
+    let dir = tempfile::TempDir::new().unwrap();
+    let mut location = dir.path().to_owned();
+    location.push("test.mp4");
+
+    let sink = pipeline.by_name("sink").unwrap();
+    sink.set_property("location", location.to_str().expect("Non-UTF8 filename"));
+
+    pipeline
+        .set_state(gst::State::Playing)
+        .expect("Unable to set the pipeline to the `Playing` state");
+
+    for msg in pipeline.bus().unwrap().iter_timed(gst::ClockTime::NONE) {
+        use gst::MessageView;
+
+        match msg.view() {
+            MessageView::Eos(..) => break,
+            MessageView::Error(err) => {
+                panic!(
+                    "Error from {:?}: {} ({:?})",
+                    err.src().map(|s| s.path_string()),
+                    err.error(),
+                    err.debug()
+                );
+            }
+            _ => (),
+        }
+    }
+
+    pipeline
+        .set_state(gst::State::Null)
+        .expect("Unable to set the pipeline to the `Null` state");
+
+    drop(pipeline);
+
+    let discoverer = gst_pbutils::Discoverer::new(gst::ClockTime::from_seconds(5))
+        .expect("Failed to create discoverer");
+    let info = discoverer
+        .discover_uri(
+            url::Url::from_file_path(&location)
+                .expect("Failed to convert filename to URL")
+                .as_str(),
+        )
+        .expect("Failed to discover MP4 file");
+
+    assert_eq!(info.duration(), Some(gst::ClockTime::from_mseconds(3_300)));
+
+    let audio_streams = info.audio_streams();
+    assert_eq!(audio_streams.len(), 1);
+    let audio_stream = audio_streams[0]
+        .downcast_ref::<gst_pbutils::DiscovererAudioInfo>()
+        .unwrap();
+    assert_eq!(audio_stream.channels(), 1);
+    assert_eq!(audio_stream.sample_rate(), 44_100);
+    let caps = audio_stream.caps().unwrap();
+    assert!(
+        caps.can_intersect(
+            &gst::Caps::builder("audio/mpeg")
+                .any_features()
+                .field("mpegversion", 4i32)
+                .build()
+        ),
+        "Unexpected audio caps {:?}",
+        caps
+    );
+
+    let video_streams = info.video_streams();
+    assert_eq!(video_streams.len(), 1);
+    let video_stream = video_streams[0]
+        .downcast_ref::<gst_pbutils::DiscovererVideoInfo>()
+        .unwrap();
+    assert_eq!(video_stream.width(), 320);
+    assert_eq!(video_stream.height(), 240);
+    assert_eq!(video_stream.framerate(), gst::Fraction::new(30, 1));
+    assert_eq!(video_stream.par(), gst::Fraction::new(1, 1));
+    assert!(!video_stream.is_interlaced());
+    let caps = video_stream.caps().unwrap();
+    assert!(
+        caps.can_intersect(&gst::Caps::builder("video/x-h264").any_features().build()),
+        "Unexpected video caps {:?}",
+        caps
+    );
+}