add textahead plugin

Plugin displaying upcoming text buffers ahead.

Cargo.toml

@@ -30,6 +30,7 @@ members = [
     "video/rspng",
     "video/hsv",
     "video/webp",
+    "text/ahead",
     "text/wrap",
     "text/json",
     "text/regex",
@@ -56,6 +57,7 @@ default-members = [
     "video/rav1e",
     "video/rspng",
     "video/hsv",
+    "text/ahead",
     "text/wrap",
     "text/json",
     "text/regex",

README.md

@@ -85,6 +85,8 @@ You will find the following plugins in this repository:
 
     - `wrap`: A plugin to perform text wrapping with hyphenation.
 
+    - `ahead`: A plugin to display upcoming text buffers ahead.
+
   * `utils`
     - `fallbackswitch`: Aggregator element that allows falling back to a
       different sink pad after a timeout.

ci/utils.py

@@ -2,8 +2,9 @@ import os
 
 DIRS = ['audio', 'generic', 'net', 'text', 'utils', 'video']
 # Plugins whose name is prefixed by 'rs'
-RS_PREFIXED = ['audiofx', 'closedcaption', 'dav1d', 'file', 'json', 'regex', 'webp']
+RS_PREFIXED = ['audiofx', 'closedcaption',
-OVERRIDE = {'wrap': 'rstextwrap', 'flavors': 'rsflv'}
+               'dav1d', 'file', 'json', 'regex', 'webp']
+OVERRIDE = {'wrap': 'rstextwrap', 'flavors': 'rsflv', 'ahead': 'textahead'}
 
 
 def iterate_plugins():

meson.build

@@ -60,6 +60,7 @@ plugins = {
   'gst-plugin-videofx': 'libgstvideofx',
   'gst-plugin-uriplaylistbin': 'libgsturiplaylistbin',
   'gst-plugin-spotify': 'libgstspotify',
+  'gst-plugin-textahead': 'libgsttextahead',
 }
 
 extra_env = {}

text/ahead/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "gst-plugin-textahead"
+version = "0.8.0"
+authors = ["Guillaume Desmottes <guillaume@desmottes.be>"]
+repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs"
+license = "MPL-2.0"
+description = "GStreamer Plugin displaying upcoming text buffers ahead"
+edition = "2021"
+rust-version = "1.56"
+
+[dependencies]
+gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
+once_cell = "1.0"
+
+[lib]
+name = "gsttextahead"
+crate-type = ["cdylib", "rlib"]
+path = "src/lib.rs"
+
+[build-dependencies]
+gst-plugin-version-helper = { path="../../version-helper" }
+
+[features]
+# GStreamer 1.14 is required for static linking
+static = ["gst/v1_14"]
+capi = []
+
+[package.metadata.capi]
+min_version = "0.8.0"
+
+[package.metadata.capi.header]
+enabled = false
+
+[package.metadata.capi.library]
+install_subdir = "gstreamer-1.0"
+versioning = false
+
+[package.metadata.capi.pkg_config]
+requires_private = "gstreamer-1.0, gstreamer-base-1.0, gobject-2.0, glib-2.0, gmodule-2.0"

text/ahead/LICENSE-MPL-2.0

@@ -0,0 +1 @@
+../../LICENSE-MPL-2.0

text/ahead/README.md

@@ -0,0 +1,9 @@
+# gst-plugins-textahead
+
+This is [GStreamer](https://gstreamer.freedesktop.org/) plugin displays upcoming
+text buffers ahead with the current one. This is mainly useful for Karaoke
+applications where singers need to know beforehand the next lines of the song.
+
+```
+gst-launch-1.0 videotestsrc pattern=black ! video/x-raw,width=1920,height=1080 ! textoverlay name=txt ! autovideosink filesrc location=subtitles.srt ! subparse ! textahead n-ahead=2 ! txt.
+```

text/ahead/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    gst_plugin_version_helper::info()
+}

text/ahead/src/lib.rs

@@ -0,0 +1,29 @@
+// Copyright (C) 2021 Guillaume Desmottes <guillaume@desmottes.be>
+//
+// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
+// If a copy of the MPL was not distributed with this file, You can obtain one at
+// <https://mozilla.org/MPL/2.0/>.
+//
+// SPDX-License-Identifier: MPL-2.0
+
+use gst::glib;
+
+mod textahead;
+
+fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
+    textahead::register(plugin)?;
+    Ok(())
+}
+
+gst::plugin_define!(
+    textahead,
+    env!("CARGO_PKG_DESCRIPTION"),
+    plugin_init,
+    concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")),
+    // FIXME: MPL-2.0 is only allowed since 1.18.3 (as unknown) and 1.20 (as known)
+    "MPL",
+    env!("CARGO_PKG_NAME"),
+    env!("CARGO_PKG_NAME"),
+    env!("CARGO_PKG_REPOSITORY"),
+    env!("BUILD_REL_DATE")
+);

text/ahead/src/textahead/imp.rs

@@ -0,0 +1,369 @@
+// Copyright (C) 2021 Guillaume Desmottes <guillaume@desmottes.be>
+//
+// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
+// If a copy of the MPL was not distributed with this file, You can obtain one at
+// <https://mozilla.org/MPL/2.0/>.
+//
+// SPDX-License-Identifier: MPL-2.0
+
+use std::sync::{Mutex, MutexGuard};
+
+use once_cell::sync::Lazy;
+
+use gst::glib;
+use gst::prelude::*;
+use gst::subclass::prelude::*;
+use gst::{gst_debug, gst_log};
+
+static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
+    gst::DebugCategory::new(
+        "textahead",
+        gst::DebugColorFlags::empty(),
+        Some("textahead debug category"),
+    )
+});
+
+struct Settings {
+    n_ahead: u32,
+    separator: String,
+    current_attributes: String,
+    ahead_attributes: String,
+}
+
+impl Default for Settings {
+    fn default() -> Self {
+        Self {
+            n_ahead: 1,
+            separator: "\n".to_string(),
+            current_attributes: "size=\"larger\"".to_string(),
+            ahead_attributes: "size=\"smaller\"".to_string(),
+        }
+    }
+}
+
+struct Input {
+    text: String,
+    pts: Option<gst::ClockTime>,
+    duration: Option<gst::ClockTime>,
+}
+
+#[derive(Default)]
+struct State {
+    pending: Vec<Input>,
+    done: bool,
+}
+
+pub struct TextAhead {
+    sink_pad: gst::Pad,
+    src_pad: gst::Pad,
+
+    state: Mutex<State>,
+    settings: Mutex<Settings>,
+}
+
+#[glib::object_subclass]
+impl ObjectSubclass for TextAhead {
+    const NAME: &'static str = "GstTextAhead";
+    type Type = super::TextAhead;
+    type ParentType = gst::Element;
+
+    fn with_class(klass: &Self::Class) -> Self {
+        let templ = klass.pad_template("sink").unwrap();
+        let sink_pad = gst::Pad::builder_with_template(&templ, Some("sink"))
+            .chain_function(|pad, parent, buffer| {
+                TextAhead::catch_panic_pad_function(
+                    parent,
+                    || Err(gst::FlowError::Error),
+                    |self_, element| self_.sink_chain(pad, element, buffer),
+                )
+            })
+            .event_function(|pad, parent, event| {
+                TextAhead::catch_panic_pad_function(
+                    parent,
+                    || false,
+                    |self_, element| self_.sink_event(pad, element, event),
+                )
+            })
+            .build();
+
+        let templ = klass.pad_template("src").unwrap();
+        let src_pad = gst::Pad::builder_with_template(&templ, Some("src")).build();
+
+        Self {
+            sink_pad,
+            src_pad,
+            state: Mutex::new(State::default()),
+            settings: Mutex::new(Settings::default()),
+        }
+    }
+}
+
+impl ObjectImpl for TextAhead {
+    fn properties() -> &'static [glib::ParamSpec] {
+        static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
+            let default = Settings::default();
+
+            vec![
+                glib::ParamSpecUInt::new(
+                    "n-ahead",
+                    "n-ahead",
+                    "The number of ahead text buffers to display along with the current one",
+                    0,
+                    u32::MAX,
+                    default.n_ahead,
+                    glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
+                ),
+                glib::ParamSpecString::new(
+                    "separator",
+                    "Separator",
+                    "Text inserted between each text buffers",
+                    Some(&default.separator),
+                    glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
+                ),
+                // See https://developer.gimp.org/api/2.0/pango/PangoMarkupFormat.html for pango attributes
+                glib::ParamSpecString::new(
+                    "current-attributes",
+                    "Current attributes",
+                    "Pango span attributes to set on the text from the current buffer",
+                    Some(&default.current_attributes),
+                    glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
+                ),
+                glib::ParamSpecString::new(
+                    "ahead-attributes",
+                    "Ahead attributes",
+                    "Pango span attributes to set on the ahead text",
+                    Some(&default.ahead_attributes),
+                    glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
+                ),
+            ]
+        });
+
+        PROPERTIES.as_ref()
+    }
+
+    fn set_property(
+        &self,
+        _obj: &Self::Type,
+        _id: usize,
+        value: &glib::Value,
+        pspec: &glib::ParamSpec,
+    ) {
+        let mut settings = self.settings.lock().unwrap();
+
+        match pspec.name() {
+            "n-ahead" => {
+                settings.n_ahead = value.get().expect("type checked upstream");
+            }
+            "separator" => {
+                settings.separator = value.get().expect("type checked upstream");
+            }
+            "current-attributes" => {
+                settings.current_attributes = value.get().expect("type checked upstream");
+            }
+            "ahead-attributes" => {
+                settings.ahead_attributes = value.get().expect("type checked upstream");
+            }
+            _ => unimplemented!(),
+        }
+    }
+
+    fn property(&self, _obj: &Self::Type, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
+        let settings = self.settings.lock().unwrap();
+
+        match pspec.name() {
+            "n-ahead" => settings.n_ahead.to_value(),
+            "separator" => settings.separator.to_value(),
+            "current-attributes" => settings.current_attributes.to_value(),
+            "ahead-attributes" => settings.ahead_attributes.to_value(),
+            _ => unimplemented!(),
+        }
+    }
+
+    fn constructed(&self, obj: &Self::Type) {
+        self.parent_constructed(obj);
+
+        obj.add_pad(&self.sink_pad).unwrap();
+        obj.add_pad(&self.src_pad).unwrap();
+    }
+}
+
+impl GstObjectImpl for TextAhead {}
+
+impl ElementImpl for TextAhead {
+    fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
+        static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
+            gst::subclass::ElementMetadata::new(
+                "Text Ahead",
+                "Text/Filter",
+                "Display upcoming text buffers ahead",
+                "Guillaume Desmottes <guillaume@desmottes.be>",
+            )
+        });
+
+        Some(&*ELEMENT_METADATA)
+    }
+
+    fn pad_templates() -> &'static [gst::PadTemplate] {
+        static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
+            let sink_caps = gst::Caps::builder("text/x-raw")
+                .field("format", gst::List::new(["utf8", "pango-markup"]))
+                .build();
+            let sink_pad_template = gst::PadTemplate::new(
+                "sink",
+                gst::PadDirection::Sink,
+                gst::PadPresence::Always,
+                &sink_caps,
+            )
+            .unwrap();
+
+            let src_caps = gst::Caps::builder("text/x-raw")
+                .field("format", "pango-markup")
+                .build();
+            let src_pad_template = gst::PadTemplate::new(
+                "src",
+                gst::PadDirection::Src,
+                gst::PadPresence::Always,
+                &src_caps,
+            )
+            .unwrap();
+
+            vec![sink_pad_template, src_pad_template]
+        });
+
+        PAD_TEMPLATES.as_ref()
+    }
+
+    fn change_state(
+        &self,
+        element: &Self::Type,
+        transition: gst::StateChange,
+    ) -> Result<gst::StateChangeSuccess, gst::StateChangeError> {
+        let res = self.parent_change_state(element, transition);
+
+        match transition {
+            gst::StateChange::ReadyToPaused => *self.state.lock().unwrap() = State::default(),
+            gst::StateChange::PausedToReady => {
+                let mut state = self.state.lock().unwrap();
+                state.done = true;
+            }
+            _ => {}
+        }
+
+        res
+    }
+}
+
+impl TextAhead {
+    fn sink_chain(
+        &self,
+        _pad: &gst::Pad,
+        element: &super::TextAhead,
+        buffer: gst::Buffer,
+    ) -> Result<gst::FlowSuccess, gst::FlowError> {
+        let pts = buffer.pts();
+        let duration = buffer.duration();
+
+        let buffer = buffer
+            .into_mapped_buffer_readable()
+            .map_err(|_| gst::FlowError::Error)?;
+        let text =
+            String::from_utf8(Vec::from(buffer.as_slice())).map_err(|_| gst::FlowError::Error)?;
+
+        // queue buffer
+        let mut state = self.state.lock().unwrap();
+
+        gst_log!(CAT, obj: element, "input {:?}: {}", pts, text);
+
+        state.pending.push(Input {
+            text,
+            pts,
+            duration,
+        });
+
+        let n_ahead = {
+            let settings = self.settings.lock().unwrap();
+            settings.n_ahead as usize
+        };
+
+        // then check if we can output
+        // FIXME: this won't work on live pipelines as we can't really report latency
+        if state.pending.len() > n_ahead {
+            self.push_pending(element, &mut state)
+        } else {
+            Ok(gst::FlowSuccess::Ok)
+        }
+    }
+
+    fn sink_event(&self, pad: &gst::Pad, element: &super::TextAhead, event: gst::Event) -> bool {
+        match event.view() {
+            gst::EventView::Eos(_) => {
+                let mut state = self.state.lock().unwrap();
+
+                gst_debug!(CAT, obj: element, "eos");
+
+                while !state.pending.is_empty() {
+                    let _ = self.push_pending(element, &mut state);
+                }
+                pad.event_default(Some(element), event)
+            }
+            gst::EventView::Caps(_caps) => {
+                // set caps on src pad
+                let templ = element.class().pad_template("src").unwrap();
+                let _ = self
+                    .src_pad
+                    .push_event(gst::event::Caps::new(&templ.caps()));
+                true
+            }
+            _ => pad.event_default(Some(element), event),
+        }
+    }
+
+    /// push first pending buffer as current and all the other ones as ahead text
+    fn push_pending(
+        &self,
+        element: &super::TextAhead,
+        state: &mut MutexGuard<State>,
+    ) -> Result<gst::FlowSuccess, gst::FlowError> {
+        if state.done {
+            return Err(gst::FlowError::Flushing);
+        }
+        let settings = self.settings.lock().unwrap();
+
+        let first = state.pending.remove(0);
+        let mut text = if settings.current_attributes.is_empty() {
+            first.text
+        } else {
+            format!(
+                "<span {}>{}</span>",
+                settings.current_attributes, first.text
+            )
+        };
+
+        for input in state.pending.iter() {
+            if !settings.separator.is_empty() {
+                text.push_str(&settings.separator);
+            }
+
+            if settings.ahead_attributes.is_empty() {
+                text.push_str(&input.text);
+            } else {
+                text.push_str(&format!(
+                    "<span {}>{}</span>",
+                    settings.ahead_attributes, input.text
+                ));
+            }
+        }
+
+        gst_log!(CAT, obj: element, "output {:?}: {}", first.pts, text);
+
+        let mut output = gst::Buffer::from_mut_slice(text.into_bytes());
+        {
+            let output = output.get_mut().unwrap();
+
+            output.set_pts(first.pts);
+            output.set_duration(first.duration);
+        }
+
+        self.src_pad.push(output)
+    }
+}

text/ahead/src/textahead/mod.rs

@@ -0,0 +1,30 @@
+// Copyright (C) 2021 Guillaume Desmottes <guillaume@desmottes.be>
+//
+// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
+// If a copy of the MPL was not distributed with this file, You can obtain one at
+// <https://mozilla.org/MPL/2.0/>.
+//
+// SPDX-License-Identifier: MPL-2.0
+
+use gst::glib;
+use gst::prelude::*;
+
+mod imp;
+
+glib::wrapper! {
+    pub struct TextAhead(ObjectSubclass<imp::TextAhead>) @extends gst::Element, gst::Object;
+}
+
+// GStreamer elements need to be thread-safe. For the private implementation this is automatically
+// enforced but for the public wrapper type we need to specify this manually.
+unsafe impl Send for TextAhead {}
+unsafe impl Sync for TextAhead {}
+
+pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
+    gst::Element::register(
+        Some(plugin),
+        "textahead",
+        gst::Rank::Primary,
+        TextAhead::static_type(),
+    )
+}