diff --git a/Cargo.lock b/Cargo.lock index 05f482dff..a87a171c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -341,7 +341,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9dd2e03ee80ca2822dd6ea431163d2ef259f2066a4d6ccaca6d9dcb386aa43" dependencies = [ - "bindgen", + "bindgen 0.69.5", "cc", "cmake", "dunce", @@ -904,6 +904,26 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.99", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -3294,6 +3314,18 @@ dependencies = [ "rgb", ] +[[package]] +name = "gst-plugin-vvdec" +version = "0.14.0-alpha.1" +dependencies = [ + "gst-plugin-version-helper", + "gstreamer", + "gstreamer-audio", + "gstreamer-check", + "gstreamer-video", + "vvdec", +] + [[package]] name = "gst-plugin-webp" version = "0.14.0-alpha.1" @@ -8155,6 +8187,27 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "vvdec" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4dd0b089dddc930150dbce1f3222020d3c8f6d3c6d81ffea3492d913a1a195" +dependencies = [ + "thiserror 2.0.12", + "vvdec-sys", +] + +[[package]] +name = "vvdec-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f825818f7e4770c5807f26b6230d2ec369f8e8a35f00c0d081de519e7917db" +dependencies = [ + "bindgen 0.70.1", + "cmake", + "system-deps 7.0.3", +] + [[package]] name = "waker-fn" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index e0034a90d..8928bfa24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ members = [ "video/png", "video/rav1e", "video/videofx", + "video/vvdec", "video/webp", ] diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 7d37daed6..84e01d182 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -16751,6 +16751,74 @@ "tracers": {}, "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" }, + "vvdec": { + "description": "GStreamer VVdeC VVC/H.266 decoder plugin", + "elements": { + "vvdec": { + "author": "Carlos Bentzen ", + "description": "Decode VVC/H.266 video streams with VVdeC", + "hierarchy": [ + "GstVVdeC", + "GstVideoDecoder", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Decoder/Video", + "pad-templates": { + "sink": { + "caps": "video/x-h266:\n stream-format: byte-stream\n alignment: au\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "video/x-raw:\n format: { GRAY8, I420, Y42B, Y444, GRAY10_LE16, I420_10LE, I422_10LE, Y444_10LE }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "direction": "src", + "presence": "always" + } + }, + "properties": { + "n-parser-threads": { + "blurb": "Number of parser threads to use while decoding (set to -1 to let libvvdec choose the number of parser threads, set to 0 to parse on the element streaming thread)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "2147483647", + "min": "-1", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + }, + "n-threads": { + "blurb": "Number of threads to use while decoding (set to -1 to use the number of logical cores, set to 0 to decode in a single thread)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "2147483647", + "min": "-1", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + } + }, + "rank": "secondary" + } + }, + "filename": "gstvvdec", + "license": "MPL-2.0", + "other-types": {}, + "package": "gst-plugin-vvdec", + "source": "gst-plugin-vvdec", + "tracers": {}, + "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" + }, "webrtchttp": { "description": "GStreamer WebRTC Plugin for WebRTC HTTP protocols (WHIP/WHEP)", "elements": { diff --git a/meson.build b/meson.build index 83e92ab7d..f4e69ca1d 100644 --- a/meson.build +++ b/meson.build @@ -222,6 +222,10 @@ plugins = { ], }, 'speechmatics': {'library': 'libgstspeechmatics'}, + 'vvdec': { + 'library': 'libgstvvdec', + 'extra-deps': {'libvvdec': ['>= 3.0']} + } } # Won't build on platforms where it bundles the sources because of: diff --git a/meson_options.txt b/meson_options.txt index 448ad1dec..d75770ee4 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -67,6 +67,7 @@ option('hsv', type: 'feature', value: 'auto', description: 'Build hsv plugin') option('png', type: 'feature', value: 'auto', description: 'Build png plugin') option('rav1e', type: 'feature', value: 'auto', description: 'Build rav1e plugin') option('videofx', type: 'feature', value: 'auto', description: 'Build videofx plugin') +option('vvdec', type: 'feature', value: 'auto', description: 'Build vvdec plugin') option('webp', type: 'feature', value: 'auto', description: 'Build webp plugin') # Common options diff --git a/video/vvdec/Cargo.toml b/video/vvdec/Cargo.toml new file mode 100644 index 000000000..88397d562 --- /dev/null +++ b/video/vvdec/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "gst-plugin-vvdec" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors = ["Carlos Bentzen "] +repository.workspace = true +license = "MPL-2.0" +description = "GStreamer VVdeC VVC/H.266 decoder plugin" + +[dependencies] +gst.workspace = true +gst-audio.workspace = true +gst-video = { workspace = true, features = ["v1_26"] } +vvdec = { version = "0.6" } + +[dev-dependencies] +gst-check.workspace = true + +[lib] +name = "gstvvdec" +crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" + +[build-dependencies] +gst-plugin-version-helper.workspace = true + +[features] +capi = [] + +[package.metadata.capi] +min_version = "0.9.21" + +[package.metadata.capi.header] +enabled = false + +[package.metadata.capi.library] +install_subdir = "gstreamer-1.0" +versioning = false +import_library = false + +[package.metadata.capi.pkg_config] +requires_private = "gstreamer-1.0, gstreamer-base-1.0, gstreamer-video-1.0, gobject-2.0, glib-2.0, gmodule-2.0, libvvdec >= 3.0" diff --git a/video/vvdec/LICENSE-MPL-2.0 b/video/vvdec/LICENSE-MPL-2.0 new file mode 100644 index 000000000..14e2f777f --- /dev/null +++ b/video/vvdec/LICENSE-MPL-2.0 @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/video/vvdec/build.rs b/video/vvdec/build.rs new file mode 100644 index 000000000..cda12e57e --- /dev/null +++ b/video/vvdec/build.rs @@ -0,0 +1,3 @@ +fn main() { + gst_plugin_version_helper::info() +} diff --git a/video/vvdec/src/dec/imp.rs b/video/vvdec/src/dec/imp.rs new file mode 100644 index 000000000..551cb17d8 --- /dev/null +++ b/video/vvdec/src/dec/imp.rs @@ -0,0 +1,694 @@ +// Copyright (C) 2025 Carlos Bentzen +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +/** + * SECTION:element-vvdec + * + * #vvdec is a decoder element for VVC/H.266 video streams using the VVdeC decoder. + * + * ## Example pipeline + * + * |[ + * gst-launch filesrc location=vvc.mp4 ! qtdemux ! h266parse ! vvdec ! videoconvert ! autovideosink + * ]| + * + * Since: plugins-rs-0.14.0 + */ +use std::sync::{Mutex, MutexGuard}; + +use gst::glib::{self, Properties}; +use gst::subclass::prelude::*; +use gst_video::prelude::*; +use gst_video::subclass::prelude::*; +use std::sync::LazyLock; +use vvdec::AccessUnit; + +const DEFAULT_N_THREADS: i32 = -1; +const DEFAULT_N_PARSER_THREADS: i32 = -1; + +static CAT: LazyLock = LazyLock::new(|| { + gst::DebugCategory::new( + "vvdec", + gst::DebugColorFlags::empty(), + Some("VVdeC VVC/H.266 decoder"), + ) +}); + +struct State { + decoder: vvdec::Decoder, + video_meta_supported: bool, + output_info: Option, + input_state: gst_video::VideoCodecState<'static, gst_video::video_codec_state::Readable>, +} + +#[derive(Debug)] +struct Settings { + n_threads: i32, + parse_delay: i32, +} + +impl Default for Settings { + fn default() -> Self { + Self { + n_threads: DEFAULT_N_THREADS, + parse_delay: DEFAULT_N_PARSER_THREADS, + } + } +} + +impl Settings { + fn create_decoder(&self) -> Result { + vvdec::Decoder::builder() + .num_threads(self.n_threads) + .parse_delay(self.parse_delay) + .build() + } +} + +#[derive(Default, Properties)] +#[properties(wrapper_type = super::VVdeC)] +pub struct VVdeC { + state: Mutex>, + #[property(name = "n-threads", get, set, type = i32, member = n_threads, minimum = -1, default = DEFAULT_N_THREADS, blurb = "Number of threads to use while decoding (set to -1 to use the number of logical cores, set to 0 to decode in a single thread)")] + #[property(name = "n-parser-threads", get, set, type = i32, member = parse_delay, minimum = -1, default = DEFAULT_N_PARSER_THREADS, blurb = "Number of parser threads to use while decoding (set to -1 to let libvvdec choose the number of parser threads, set to 0 to parse on the element streaming thread)")] + settings: Mutex, +} + +type StateGuard<'s> = MutexGuard<'s, Option>; + +impl VVdeC { + fn create_decoder(&self) -> Result { + self.settings.lock().unwrap().create_decoder() + } + + fn decode<'s>( + &'s self, + mut state_guard: StateGuard<'s>, + input_buffer: gst::Buffer, + frame: gst_video::VideoCodecFrame, + ) -> Result<(), gst::FlowError> { + let state = state_guard.as_mut().ok_or(gst::FlowError::Flushing)?; + + // VVdeC doesn't have a mechanism for passing opaque data per-frame. + // But we can store system_frame_number via the cts field, since it's not + // used in the decoder but the value is returned in the matching decoded frames. + let cts = frame.system_frame_number() as u64; + let dts = input_buffer.dts().map(|ts| *ts); + let is_random_access_point = frame + .flags() + .contains(gst_video::VideoCodecFrameFlags::SYNC_POINT); + let payload = input_buffer + .into_mapped_buffer_readable() + .map_err(|_| gst::FlowError::Error)?; + + let access_unit = AccessUnit { + payload, + cts: Some(cts), + dts, + is_random_access_point, + }; + + match state.decoder.decode(access_unit) { + Ok(Some(frame)) => { + drop(self.handle_decoded_frame(state_guard, &frame)?); + } + Ok(None) => (), + Err(vvdec::Error::TryAgain) => { + gst::trace!(CAT, imp = self, "Decoder returned TryAgain"); + } + Err(vvdec::Error::DecInput) => { + gst_video::video_decoder_error!( + &*self.obj(), + 1, + gst::StreamError::Decode, + ["Discarding frame {} because it's undecodable", cts] + )?; + drop(state_guard); + self.obj().release_frame(frame); + } + Err(vvdec::Error::RestartRequired) => { + gst::warning!(CAT, imp = self, "decoder requires restart"); + state_guard = self.forward_pending_frames(state_guard)?; + let state = state_guard.as_mut().ok_or(gst::FlowError::Flushing)?; + state.decoder = self.create_decoder().map_err(|_| { + gst::error!(CAT, imp = self, "Failed to create new decoder instance"); + gst::FlowError::Error + })?; + } + Err(err) => { + gst::error!(CAT, imp = self, "decoder returned error: {err}"); + return Err(gst::FlowError::Error); + } + } + + Ok(()) + } + + fn handle_decoded_frame<'s>( + &'s self, + state_guard: StateGuard<'s>, + decoded_frame: &vvdec::Frame, + ) -> Result, gst::FlowError> { + let system_frame_number = decoded_frame.cts().expect("frames were pushed with cts"); + gst::trace!( + CAT, + imp = self, + "Handling decoded frame {system_frame_number}" + ); + + let state_guard = self.handle_resolution_changes(state_guard, decoded_frame)?; + let instance = self.obj(); + + let frame = instance.frame(system_frame_number.try_into().unwrap()); + if let Some(mut frame) = frame { + self.set_decoded_frame_as_output_buffer(state_guard, decoded_frame, &mut frame)?; + instance.finish_frame(frame)?; + gst::trace!(CAT, imp = self, "Finished frame {system_frame_number}"); + Ok(self.state.lock().unwrap()) + } else { + gst::warning!( + CAT, + imp = self, + "No frame found for system frame number {system_frame_number}" + ); + Ok(state_guard) + } + } + + fn handle_resolution_changes<'s>( + &'s self, + mut state_guard: StateGuard<'s>, + frame: &vvdec::Frame, + ) -> Result, gst::FlowError> { + let format = self.gst_video_format_from_vvdec_frame(frame); + if format == gst_video::VideoFormat::Unknown { + return Err(gst::FlowError::NotNegotiated); + } + + let state = state_guard.as_mut().ok_or(gst::FlowError::Flushing)?; + let need_negotiate = { + match state.output_info { + Some(ref i) => { + (i.width() != frame.width()) + || (i.height() != frame.height() || (i.format() != format)) + } + None => true, + } + }; + if !need_negotiate { + return Ok(state_guard); + } + + gst::info!( + CAT, + imp = self, + "Negotiating format {} frame dimensions {}x{}", + format, + frame.width(), + frame.height() + ); + + let input_state = state.input_state.clone(); + drop(state_guard); + + let instance = self.obj(); + + let interlace_mode = match frame.frame_format() { + vvdec::FrameFormat::TopBottom | vvdec::FrameFormat::BottomTop => { + gst_video::VideoInterlaceMode::Interleaved + } + vvdec::FrameFormat::TopField | vvdec::FrameFormat::BottomField => { + gst_video::VideoInterlaceMode::Alternate + } + vvdec::FrameFormat::Progressive => gst_video::VideoInterlaceMode::Progressive, + _ => { + gst::error!( + CAT, + imp = self, + "Unsupported VVdeC frame format {:?}", + frame.frame_format() + ); + return Err(gst::FlowError::NotNegotiated); + } + }; + + let mut output_state_height = frame.height(); + if interlace_mode == gst_video::VideoInterlaceMode::Alternate { + output_state_height *= 2; + } + + let output_state = instance.set_interlaced_output_state( + format, + interlace_mode, + frame.width(), + output_state_height, + Some(&input_state), + )?; + instance.negotiate(output_state)?; + let out_state = instance.output_state().unwrap(); + + let mut state_guard = self.state.lock().unwrap(); + let state = state_guard.as_mut().ok_or(gst::FlowError::Flushing)?; + state.output_info = Some(out_state.info()); + + gst::trace!(CAT, imp = self, "Negotiated format"); + + Ok(state_guard) + } + + fn gst_video_format_from_vvdec_frame(&self, frame: &vvdec::Frame) -> gst_video::VideoFormat { + let color_format = frame.color_format(); + let bit_depth = frame.bit_depth(); + + match (&color_format, bit_depth) { + (vvdec::ColorFormat::Yuv400Planar, 8) => gst_video::VideoFormat::Gray8, + (vvdec::ColorFormat::Yuv420Planar, 8) => gst_video::VideoFormat::I420, + (vvdec::ColorFormat::Yuv422Planar, 8) => gst_video::VideoFormat::Y42b, + (vvdec::ColorFormat::Yuv444Planar, 8) => gst_video::VideoFormat::Y444, + #[cfg(target_endian = "little")] + (vvdec::ColorFormat::Yuv400Planar, 10) => gst_video::VideoFormat::Gray10Le16, + #[cfg(target_endian = "little")] + (vvdec::ColorFormat::Yuv420Planar, 10) => gst_video::VideoFormat::I42010le, + #[cfg(target_endian = "little")] + (vvdec::ColorFormat::Yuv422Planar, 10) => gst_video::VideoFormat::I42210le, + #[cfg(target_endian = "little")] + (vvdec::ColorFormat::Yuv444Planar, 10) => gst_video::VideoFormat::Y44410le, + #[cfg(target_endian = "big")] + (vvdec::ColorFormat::Yuv420Planar, 10) => gst_video::VideoFormat::I42010be, + #[cfg(target_endian = "big")] + (vvdec::ColorFormat::Yuv422Planar, 10) => gst_video::VideoFormat::I42210be, + #[cfg(target_endian = "big")] + (vvdec::ColorFormat::Yuv444Planar, 10) => gst_video::VideoFormat::Y44410be, + _ => { + gst::warning!( + CAT, + imp = self, + "Unsupported VVdeC format {:?}/{:?}", + color_format, + bit_depth + ); + gst_video::VideoFormat::Unknown + } + } + } + + fn forward_pending_frames<'s>( + &'s self, + mut state_guard: StateGuard<'s>, + ) -> Result, gst::FlowError> { + loop { + let state = state_guard.as_mut().ok_or(gst::FlowError::Flushing)?; + match state.decoder.flush() { + Ok(Some(frame)) => { + gst::trace!(CAT, imp = self, "Forwarding pending frame."); + state_guard = self.handle_decoded_frame(state_guard, &frame)?; + } + Ok(None) | Err(vvdec::Error::RestartRequired) => break, + Err(err) => { + gst::error!( + CAT, + imp = self, + "Decoder returned error while flushing: {err}" + ); + return Err(gst::FlowError::Error); + } + }; + } + Ok(state_guard) + } + + fn flush_decoder(&self, state: &mut State) -> bool { + loop { + match state.decoder.flush() { + Ok(Some(_)) => continue, + Ok(None) | Err(vvdec::Error::RestartRequired) | Err(vvdec::Error::Eof) => break, + Err(err) => { + gst::error!( + CAT, + imp = self, + "Decoder returned error while flushing: {err}" + ); + return false; + } + } + } + true + } + + fn set_decoded_frame_as_output_buffer<'s>( + &'s self, + mut state_guard: StateGuard<'s>, + frame: &vvdec::Frame, + codec_frame: &mut gst_video::VideoCodecFrame, + ) -> Result<(), gst::FlowError> { + let state = state_guard.as_mut().expect("state is set"); + let video_meta_supported = state.video_meta_supported; + drop(state_guard); + + let mut out_buffer = video_meta_supported.then(gst::Buffer::new); + let mut_buffer = if video_meta_supported { + out_buffer.as_mut().unwrap().get_mut().unwrap() + } else { + self.obj().allocate_output_frame(codec_frame, None)?; + codec_frame + .output_buffer_mut() + .expect("output_buffer is set") + }; + + state_guard = self.state.lock().unwrap(); + let state = state_guard.as_mut().expect("state is set"); + let info = state.output_info.as_ref().expect("output_info is set"); + + let color_format = frame.color_format(); + if color_format == vvdec::ColorFormat::Invalid { + gst::error!(CAT, imp = self, "Invalid color format"); + return Err(gst::FlowError::Error); + } + + let frame_format = frame.frame_format(); + let video_flags = match frame_format { + vvdec::FrameFormat::Progressive => None, + vvdec::FrameFormat::TopField => Some( + gst_video::VideoBufferFlags::INTERLACED | gst_video::VideoBufferFlags::TOP_FIELD, + ), + vvdec::FrameFormat::BottomField => Some( + gst_video::VideoBufferFlags::INTERLACED | gst_video::VideoBufferFlags::BOTTOM_FIELD, + ), + vvdec::FrameFormat::TopBottom => { + Some(gst_video::VideoBufferFlags::INTERLACED | gst_video::VideoBufferFlags::TFF) + } + vvdec::FrameFormat::BottomTop => Some(gst_video::VideoBufferFlags::INTERLACED), + _ => { + gst::error!( + CAT, + imp = self, + "Unsupported VVdeC frame format {:?}", + frame_format + ); + return Err(gst::FlowError::Error); + } + }; + if let Some(video_flags) = video_flags { + mut_buffer.set_video_flags(video_flags); + } + + let components = if color_format == vvdec::ColorFormat::Yuv400Planar { + const GRAY_COMPONENTS: [vvdec::PlaneComponent; 1] = [vvdec::PlaneComponent::Y]; + &GRAY_COMPONENTS[..] + } else { + const YUV_COMPONENTS: [vvdec::PlaneComponent; 3] = [ + vvdec::PlaneComponent::Y, + vvdec::PlaneComponent::U, + vvdec::PlaneComponent::V, + ]; + &YUV_COMPONENTS[..] + }; + + if video_meta_supported { + let mut offsets = vec![]; + let mut strides = vec![]; + let mut acc_offset: usize = 0; + + assert!(mut_buffer.size() == 0); + + for component in components { + let plane = frame + .plane(*component) + .expect("frame contains the requested plane"); + let src_stride = plane.stride(); + let mem = gst::Memory::from_slice(plane); + let mem_size = mem.size(); + mut_buffer.append_memory(mem); + + strides.push(src_stride as i32); + offsets.push(acc_offset); + acc_offset += mem_size; + } + + let frame_flags = match frame_format { + vvdec::FrameFormat::Progressive => gst_video::VideoFrameFlags::empty(), + vvdec::FrameFormat::TopField => { + gst_video::VideoFrameFlags::INTERLACED | gst_video::VideoFrameFlags::TOP_FIELD + } + vvdec::FrameFormat::BottomField => { + gst_video::VideoFrameFlags::INTERLACED + | gst_video::VideoFrameFlags::BOTTOM_FIELD + } + vvdec::FrameFormat::TopBottom => { + gst_video::VideoFrameFlags::INTERLACED | gst_video::VideoFrameFlags::TFF + } + vvdec::FrameFormat::BottomTop => gst_video::VideoFrameFlags::INTERLACED, + _ => unreachable!("frame_format is checked above"), + }; + gst_video::VideoMeta::add_full( + mut_buffer, + frame_flags, + info.format(), + info.width(), + info.height(), + &offsets, + &strides[..], + ) + .unwrap(); + + assert!(codec_frame.output_buffer().is_none()); + codec_frame.set_output_buffer(out_buffer.expect("out_buffer is set")); + } else { + assert!(mut_buffer.size() > 0); + let mut vframe = gst_video::VideoFrameRef::from_buffer_ref_writable(mut_buffer, info) + .expect("can map writable frame"); + for component in components { + let dest_stride: u32 = vframe.plane_stride()[*component as usize] as u32; + let dest_height: u32 = vframe.plane_height(*component as u32); + let dest_plane_data = vframe + .plane_data_mut(*component as u32) + .expect("can get plane data"); + let plane = frame + .plane(*component) + .expect("frame contains the requested plane"); + let src_stride = plane.stride(); + let src_slice = plane.as_ref(); + let src_height = plane.height(); + let chunk_len = std::cmp::min(src_stride, dest_stride) as usize; + + if src_stride == dest_stride && src_height == dest_height { + dest_plane_data.copy_from_slice(src_slice); + } else { + for (out_line, in_line) in dest_plane_data + .chunks_exact_mut(dest_stride.try_into().unwrap()) + .zip(src_slice.chunks_exact(src_stride.try_into().unwrap())) + { + out_line.copy_from_slice(&in_line[..chunk_len]); + } + } + } + } + + Ok(()) + } +} + +fn video_output_formats() -> impl IntoIterator { + [ + gst_video::VideoFormat::Gray8, + gst_video::VideoFormat::I420, + gst_video::VideoFormat::Y42b, + gst_video::VideoFormat::Y444, + #[cfg(target_endian = "little")] + gst_video::VideoFormat::Gray10Le16, + #[cfg(target_endian = "little")] + gst_video::VideoFormat::I42010le, + #[cfg(target_endian = "little")] + gst_video::VideoFormat::I42210le, + #[cfg(target_endian = "little")] + gst_video::VideoFormat::Y44410le, + #[cfg(target_endian = "big")] + gst_video::VideoFormat::I42010be, + #[cfg(target_endian = "big")] + gst_video::VideoFormat::I42210be, + #[cfg(target_endian = "big")] + gst_video::VideoFormat::Y44410be, + ] +} + +#[glib::object_subclass] +impl ObjectSubclass for VVdeC { + const NAME: &'static str = "GstVVdeC"; + type Type = super::VVdeC; + type ParentType = gst_video::VideoDecoder; +} + +#[glib::derived_properties] +impl ObjectImpl for VVdeC {} + +impl GstObjectImpl for VVdeC {} + +impl ElementImpl for VVdeC { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: LazyLock = LazyLock::new(|| { + gst::subclass::ElementMetadata::new( + "VVdeC VVC/H.266 Decoder", + "Codec/Decoder/Video", + "Decode VVC/H.266 video streams with VVdeC", + "Carlos Bentzen ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: LazyLock> = LazyLock::new(|| { + let sink_caps = gst::Caps::builder("video/x-h266") + .field("stream-format", "byte-stream") + .field("alignment", "au") + .build(); + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &sink_caps, + ) + .unwrap(); + + let src_caps = gst_video::VideoCapsBuilder::new() + .format_list(video_output_formats()) + .build(); + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &src_caps, + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl VideoDecoderImpl for VVdeC { + fn set_format( + &self, + input_state: &gst_video::VideoCodecState<'static, gst_video::video_codec_state::Readable>, + ) -> Result<(), gst::LoggableError> { + let mut state_guard = self.state.lock().unwrap(); + let settings = self.settings.lock().unwrap(); + + if state_guard.is_some() { + state_guard = self + .forward_pending_frames(state_guard) + .map_err(|_| gst::loggable_error!(CAT, "Failed to forward pending frames"))?; + } + + gst::trace!(CAT, imp = self, "Creating decoder with {settings:?}"); + let decoder = settings + .create_decoder() + .map_err(|_| gst::loggable_error!(CAT, "Failed to create decoder instance"))?; + + *state_guard = Some(State { + decoder, + video_meta_supported: false, + output_info: None, + input_state: input_state.clone(), + }); + + self.parent_set_format(input_state) + } + + fn handle_frame( + &self, + frame: gst_video::VideoCodecFrame, + ) -> Result { + let system_frame_number = frame.system_frame_number(); + gst::trace!(CAT, imp = self, "Decoding frame {}", system_frame_number); + + let input_buffer = frame.input_buffer_owned().expect("frame has input buffer"); + { + let state_guard = self.state.lock().unwrap(); + self.decode(state_guard, input_buffer, frame)?; + } + + Ok(gst::FlowSuccess::Ok) + } + + fn stop(&self) -> Result<(), gst::ErrorMessage> { + gst::info!(CAT, imp = self, "Stopping"); + + { + let mut state_guard = self.state.lock().unwrap(); + *state_guard = None; + } + + self.parent_stop() + } + + fn drain(&self) -> Result { + gst::info!(CAT, imp = self, "Draining"); + + { + let mut state_guard = self.state.lock().unwrap(); + if state_guard.as_mut().is_some() { + drop(self.forward_pending_frames(state_guard)?); + } + } + + self.parent_drain() + } + + fn finish(&self) -> Result { + gst::info!(CAT, imp = self, "Finishing"); + + { + let mut state_guard = self.state.lock().unwrap(); + if state_guard.as_mut().is_some() { + drop(self.forward_pending_frames(state_guard)?); + } + } + + self.parent_finish() + } + + fn flush(&self) -> bool { + gst::info!(CAT, imp = self, "Flushing"); + + { + let mut state_guard = self.state.lock().unwrap(); + if let Some(state) = state_guard.as_mut() { + return self.flush_decoder(state); + } + } + + true + } + + fn decide_allocation( + &self, + query: &mut gst::query::Allocation, + ) -> Result<(), gst::LoggableError> { + gst::trace!(CAT, imp = self, "Deciding allocation"); + self.parent_decide_allocation(query)?; + + let mut state_guard = self.state.lock().unwrap(); + if let Some(state) = state_guard.as_mut() { + state.video_meta_supported = query + .find_allocation_meta::() + .is_some(); + gst::info!( + CAT, + imp = self, + "Video meta support: {}", + state.video_meta_supported + ); + }; + + Ok(()) + } +} diff --git a/video/vvdec/src/dec/mod.rs b/video/vvdec/src/dec/mod.rs new file mode 100644 index 000000000..27deb637a --- /dev/null +++ b/video/vvdec/src/dec/mod.rs @@ -0,0 +1,25 @@ +// Copyright (C) 2025 Carlos Bentzen +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct VVdeC(ObjectSubclass) @extends gst_video::VideoDecoder, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "vvdec", + gst::Rank::SECONDARY, + VVdeC::static_type(), + ) +} diff --git a/video/vvdec/src/lib.rs b/video/vvdec/src/lib.rs new file mode 100644 index 000000000..fc133f1b7 --- /dev/null +++ b/video/vvdec/src/lib.rs @@ -0,0 +1,33 @@ +// Copyright (C) 2025 Carlos Bentzen +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +/** + * plugin-vvdec: + * + * Since: plugins-rs-0.14.0 + */ +use gst::glib; + +mod dec; + +fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + dec::register(plugin)?; + Ok(()) +} + +gst::plugin_define!( + vvdec, + env!("CARGO_PKG_DESCRIPTION"), + plugin_init, + concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")), + "MPL-2.0", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_REPOSITORY"), + env!("BUILD_REL_DATE") +); diff --git a/video/vvdec/tests/vvc_yuv420p10le.mkv b/video/vvdec/tests/vvc_yuv420p10le.mkv new file mode 100644 index 000000000..0297f6f76 Binary files /dev/null and b/video/vvdec/tests/vvc_yuv420p10le.mkv differ diff --git a/video/vvdec/tests/vvc_yuv420p10le.ref b/video/vvdec/tests/vvc_yuv420p10le.ref new file mode 100644 index 000000000..c3ff1e8db Binary files /dev/null and b/video/vvdec/tests/vvc_yuv420p10le.ref differ diff --git a/video/vvdec/tests/vvdec.rs b/video/vvdec/tests/vvdec.rs new file mode 100644 index 000000000..146182851 --- /dev/null +++ b/video/vvdec/tests/vvdec.rs @@ -0,0 +1,58 @@ +// Copyright (C) 2025 Carlos Bentzen +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +use std::fs; +use std::path::PathBuf; + +fn init() { + use std::sync::Once; + static INIT: Once = Once::new(); + + INIT.call_once(|| { + gst::init().unwrap(); + gstvvdec::plugin_register_static().expect("vvdec test"); + }); +} + +#[test] +fn test_decode_yuv420p10le() { + init(); + test_decode("yuv420p10le"); +} + +fn test_decode(name: &str) { + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path.push(format!("tests/vvc_{name}.mkv")); + + let bin = gst::parse::bin_from_description( + &format!("filesrc location={path:?} ! matroskademux ! h266parse ! vvdec name=vvdec"), + false, + ) + .unwrap(); + + let srcpad = bin.by_name("vvdec").unwrap().static_pad("src").unwrap(); + let _ = bin.add_pad(&gst::GhostPad::with_target(&srcpad).unwrap()); + + let mut h = gst_check::Harness::with_element(&bin, None, Some("src")); + + h.play(); + + let buf = h.pull().unwrap(); + let frame = buf.into_mapped_buffer_readable().unwrap(); + + let mut refpath = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + refpath.push(format!("tests/vvc_{name}.ref")); + + let ref_frame = fs::read(refpath).unwrap(); + + assert_eq!(frame.len(), ref_frame.len()); + assert_eq!(frame.as_slice(), glib::Bytes::from_owned(ref_frame)); +}