add textahead plugin

Plugin displaying upcoming text buffers ahead.
This commit is contained in:
Guillaume Desmottes 2022-01-12 08:47:12 +01:00 committed by Guillaume Desmottes
parent 64dd588734
commit 4576bea10c
11 changed files with 488 additions and 2 deletions

View file

@ -30,6 +30,7 @@ members = [
"video/rspng", "video/rspng",
"video/hsv", "video/hsv",
"video/webp", "video/webp",
"text/ahead",
"text/wrap", "text/wrap",
"text/json", "text/json",
"text/regex", "text/regex",
@ -56,6 +57,7 @@ default-members = [
"video/rav1e", "video/rav1e",
"video/rspng", "video/rspng",
"video/hsv", "video/hsv",
"text/ahead",
"text/wrap", "text/wrap",
"text/json", "text/json",
"text/regex", "text/regex",

View file

@ -85,6 +85,8 @@ You will find the following plugins in this repository:
- `wrap`: A plugin to perform text wrapping with hyphenation. - `wrap`: A plugin to perform text wrapping with hyphenation.
- `ahead`: A plugin to display upcoming text buffers ahead.
* `utils` * `utils`
- `fallbackswitch`: Aggregator element that allows falling back to a - `fallbackswitch`: Aggregator element that allows falling back to a
different sink pad after a timeout. different sink pad after a timeout.

View file

@ -2,8 +2,9 @@ import os
DIRS = ['audio', 'generic', 'net', 'text', 'utils', 'video'] DIRS = ['audio', 'generic', 'net', 'text', 'utils', 'video']
# Plugins whose name is prefixed by 'rs' # Plugins whose name is prefixed by 'rs'
RS_PREFIXED = ['audiofx', 'closedcaption', 'dav1d', 'file', 'json', 'regex', 'webp'] RS_PREFIXED = ['audiofx', 'closedcaption',
OVERRIDE = {'wrap': 'rstextwrap', 'flavors': 'rsflv'} 'dav1d', 'file', 'json', 'regex', 'webp']
OVERRIDE = {'wrap': 'rstextwrap', 'flavors': 'rsflv', 'ahead': 'textahead'}
def iterate_plugins(): def iterate_plugins():

View file

@ -60,6 +60,7 @@ plugins = {
'gst-plugin-videofx': 'libgstvideofx', 'gst-plugin-videofx': 'libgstvideofx',
'gst-plugin-uriplaylistbin': 'libgsturiplaylistbin', 'gst-plugin-uriplaylistbin': 'libgsturiplaylistbin',
'gst-plugin-spotify': 'libgstspotify', 'gst-plugin-spotify': 'libgstspotify',
'gst-plugin-textahead': 'libgsttextahead',
} }
extra_env = {} extra_env = {}

39
text/ahead/Cargo.toml Normal file
View file

@ -0,0 +1,39 @@
[package]
name = "gst-plugin-textahead"
version = "0.8.0"
authors = ["Guillaume Desmottes <guillaume@desmottes.be>"]
repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs"
license = "MPL-2.0"
description = "GStreamer Plugin displaying upcoming text buffers ahead"
edition = "2021"
rust-version = "1.56"
[dependencies]
gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
once_cell = "1.0"
[lib]
name = "gsttextahead"
crate-type = ["cdylib", "rlib"]
path = "src/lib.rs"
[build-dependencies]
gst-plugin-version-helper = { path="../../version-helper" }
[features]
# GStreamer 1.14 is required for static linking
static = ["gst/v1_14"]
capi = []
[package.metadata.capi]
min_version = "0.8.0"
[package.metadata.capi.header]
enabled = false
[package.metadata.capi.library]
install_subdir = "gstreamer-1.0"
versioning = false
[package.metadata.capi.pkg_config]
requires_private = "gstreamer-1.0, gstreamer-base-1.0, gobject-2.0, glib-2.0, gmodule-2.0"

1
text/ahead/LICENSE-MPL-2.0 Symbolic link
View file

@ -0,0 +1 @@
../../LICENSE-MPL-2.0

9
text/ahead/README.md Normal file
View file

@ -0,0 +1,9 @@
# gst-plugins-textahead
This is [GStreamer](https://gstreamer.freedesktop.org/) plugin displays upcoming
text buffers ahead with the current one. This is mainly useful for Karaoke
applications where singers need to know beforehand the next lines of the song.
```
gst-launch-1.0 videotestsrc pattern=black ! video/x-raw,width=1920,height=1080 ! textoverlay name=txt ! autovideosink filesrc location=subtitles.srt ! subparse ! textahead n-ahead=2 ! txt.
```

3
text/ahead/build.rs Normal file
View file

@ -0,0 +1,3 @@
fn main() {
gst_plugin_version_helper::info()
}

29
text/ahead/src/lib.rs Normal file
View file

@ -0,0 +1,29 @@
// Copyright (C) 2021 Guillaume Desmottes <guillaume@desmottes.be>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
use gst::glib;
mod textahead;
fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
textahead::register(plugin)?;
Ok(())
}
gst::plugin_define!(
textahead,
env!("CARGO_PKG_DESCRIPTION"),
plugin_init,
concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")),
// FIXME: MPL-2.0 is only allowed since 1.18.3 (as unknown) and 1.20 (as known)
"MPL",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_REPOSITORY"),
env!("BUILD_REL_DATE")
);

View file

@ -0,0 +1,369 @@
// Copyright (C) 2021 Guillaume Desmottes <guillaume@desmottes.be>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
use std::sync::{Mutex, MutexGuard};
use once_cell::sync::Lazy;
use gst::glib;
use gst::prelude::*;
use gst::subclass::prelude::*;
use gst::{gst_debug, gst_log};
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
gst::DebugCategory::new(
"textahead",
gst::DebugColorFlags::empty(),
Some("textahead debug category"),
)
});
struct Settings {
n_ahead: u32,
separator: String,
current_attributes: String,
ahead_attributes: String,
}
impl Default for Settings {
fn default() -> Self {
Self {
n_ahead: 1,
separator: "\n".to_string(),
current_attributes: "size=\"larger\"".to_string(),
ahead_attributes: "size=\"smaller\"".to_string(),
}
}
}
struct Input {
text: String,
pts: Option<gst::ClockTime>,
duration: Option<gst::ClockTime>,
}
#[derive(Default)]
struct State {
pending: Vec<Input>,
done: bool,
}
pub struct TextAhead {
sink_pad: gst::Pad,
src_pad: gst::Pad,
state: Mutex<State>,
settings: Mutex<Settings>,
}
#[glib::object_subclass]
impl ObjectSubclass for TextAhead {
const NAME: &'static str = "GstTextAhead";
type Type = super::TextAhead;
type ParentType = gst::Element;
fn with_class(klass: &Self::Class) -> Self {
let templ = klass.pad_template("sink").unwrap();
let sink_pad = gst::Pad::builder_with_template(&templ, Some("sink"))
.chain_function(|pad, parent, buffer| {
TextAhead::catch_panic_pad_function(
parent,
|| Err(gst::FlowError::Error),
|self_, element| self_.sink_chain(pad, element, buffer),
)
})
.event_function(|pad, parent, event| {
TextAhead::catch_panic_pad_function(
parent,
|| false,
|self_, element| self_.sink_event(pad, element, event),
)
})
.build();
let templ = klass.pad_template("src").unwrap();
let src_pad = gst::Pad::builder_with_template(&templ, Some("src")).build();
Self {
sink_pad,
src_pad,
state: Mutex::new(State::default()),
settings: Mutex::new(Settings::default()),
}
}
}
impl ObjectImpl for TextAhead {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
let default = Settings::default();
vec![
glib::ParamSpecUInt::new(
"n-ahead",
"n-ahead",
"The number of ahead text buffers to display along with the current one",
0,
u32::MAX,
default.n_ahead,
glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
),
glib::ParamSpecString::new(
"separator",
"Separator",
"Text inserted between each text buffers",
Some(&default.separator),
glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
),
// See https://developer.gimp.org/api/2.0/pango/PangoMarkupFormat.html for pango attributes
glib::ParamSpecString::new(
"current-attributes",
"Current attributes",
"Pango span attributes to set on the text from the current buffer",
Some(&default.current_attributes),
glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
),
glib::ParamSpecString::new(
"ahead-attributes",
"Ahead attributes",
"Pango span attributes to set on the ahead text",
Some(&default.ahead_attributes),
glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
),
]
});
PROPERTIES.as_ref()
}
fn set_property(
&self,
_obj: &Self::Type,
_id: usize,
value: &glib::Value,
pspec: &glib::ParamSpec,
) {
let mut settings = self.settings.lock().unwrap();
match pspec.name() {
"n-ahead" => {
settings.n_ahead = value.get().expect("type checked upstream");
}
"separator" => {
settings.separator = value.get().expect("type checked upstream");
}
"current-attributes" => {
settings.current_attributes = value.get().expect("type checked upstream");
}
"ahead-attributes" => {
settings.ahead_attributes = value.get().expect("type checked upstream");
}
_ => unimplemented!(),
}
}
fn property(&self, _obj: &Self::Type, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
let settings = self.settings.lock().unwrap();
match pspec.name() {
"n-ahead" => settings.n_ahead.to_value(),
"separator" => settings.separator.to_value(),
"current-attributes" => settings.current_attributes.to_value(),
"ahead-attributes" => settings.ahead_attributes.to_value(),
_ => unimplemented!(),
}
}
fn constructed(&self, obj: &Self::Type) {
self.parent_constructed(obj);
obj.add_pad(&self.sink_pad).unwrap();
obj.add_pad(&self.src_pad).unwrap();
}
}
impl GstObjectImpl for TextAhead {}
impl ElementImpl for TextAhead {
fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
gst::subclass::ElementMetadata::new(
"Text Ahead",
"Text/Filter",
"Display upcoming text buffers ahead",
"Guillaume Desmottes <guillaume@desmottes.be>",
)
});
Some(&*ELEMENT_METADATA)
}
fn pad_templates() -> &'static [gst::PadTemplate] {
static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
let sink_caps = gst::Caps::builder("text/x-raw")
.field("format", gst::List::new(["utf8", "pango-markup"]))
.build();
let sink_pad_template = gst::PadTemplate::new(
"sink",
gst::PadDirection::Sink,
gst::PadPresence::Always,
&sink_caps,
)
.unwrap();
let src_caps = gst::Caps::builder("text/x-raw")
.field("format", "pango-markup")
.build();
let src_pad_template = gst::PadTemplate::new(
"src",
gst::PadDirection::Src,
gst::PadPresence::Always,
&src_caps,
)
.unwrap();
vec![sink_pad_template, src_pad_template]
});
PAD_TEMPLATES.as_ref()
}
fn change_state(
&self,
element: &Self::Type,
transition: gst::StateChange,
) -> Result<gst::StateChangeSuccess, gst::StateChangeError> {
let res = self.parent_change_state(element, transition);
match transition {
gst::StateChange::ReadyToPaused => *self.state.lock().unwrap() = State::default(),
gst::StateChange::PausedToReady => {
let mut state = self.state.lock().unwrap();
state.done = true;
}
_ => {}
}
res
}
}
impl TextAhead {
fn sink_chain(
&self,
_pad: &gst::Pad,
element: &super::TextAhead,
buffer: gst::Buffer,
) -> Result<gst::FlowSuccess, gst::FlowError> {
let pts = buffer.pts();
let duration = buffer.duration();
let buffer = buffer
.into_mapped_buffer_readable()
.map_err(|_| gst::FlowError::Error)?;
let text =
String::from_utf8(Vec::from(buffer.as_slice())).map_err(|_| gst::FlowError::Error)?;
// queue buffer
let mut state = self.state.lock().unwrap();
gst_log!(CAT, obj: element, "input {:?}: {}", pts, text);
state.pending.push(Input {
text,
pts,
duration,
});
let n_ahead = {
let settings = self.settings.lock().unwrap();
settings.n_ahead as usize
};
// then check if we can output
// FIXME: this won't work on live pipelines as we can't really report latency
if state.pending.len() > n_ahead {
self.push_pending(element, &mut state)
} else {
Ok(gst::FlowSuccess::Ok)
}
}
fn sink_event(&self, pad: &gst::Pad, element: &super::TextAhead, event: gst::Event) -> bool {
match event.view() {
gst::EventView::Eos(_) => {
let mut state = self.state.lock().unwrap();
gst_debug!(CAT, obj: element, "eos");
while !state.pending.is_empty() {
let _ = self.push_pending(element, &mut state);
}
pad.event_default(Some(element), event)
}
gst::EventView::Caps(_caps) => {
// set caps on src pad
let templ = element.class().pad_template("src").unwrap();
let _ = self
.src_pad
.push_event(gst::event::Caps::new(&templ.caps()));
true
}
_ => pad.event_default(Some(element), event),
}
}
/// push first pending buffer as current and all the other ones as ahead text
fn push_pending(
&self,
element: &super::TextAhead,
state: &mut MutexGuard<State>,
) -> Result<gst::FlowSuccess, gst::FlowError> {
if state.done {
return Err(gst::FlowError::Flushing);
}
let settings = self.settings.lock().unwrap();
let first = state.pending.remove(0);
let mut text = if settings.current_attributes.is_empty() {
first.text
} else {
format!(
"<span {}>{}</span>",
settings.current_attributes, first.text
)
};
for input in state.pending.iter() {
if !settings.separator.is_empty() {
text.push_str(&settings.separator);
}
if settings.ahead_attributes.is_empty() {
text.push_str(&input.text);
} else {
text.push_str(&format!(
"<span {}>{}</span>",
settings.ahead_attributes, input.text
));
}
}
gst_log!(CAT, obj: element, "output {:?}: {}", first.pts, text);
let mut output = gst::Buffer::from_mut_slice(text.into_bytes());
{
let output = output.get_mut().unwrap();
output.set_pts(first.pts);
output.set_duration(first.duration);
}
self.src_pad.push(output)
}
}

View file

@ -0,0 +1,30 @@
// Copyright (C) 2021 Guillaume Desmottes <guillaume@desmottes.be>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
use gst::glib;
use gst::prelude::*;
mod imp;
glib::wrapper! {
pub struct TextAhead(ObjectSubclass<imp::TextAhead>) @extends gst::Element, gst::Object;
}
// GStreamer elements need to be thread-safe. For the private implementation this is automatically
// enforced but for the public wrapper type we need to specify this manually.
unsafe impl Send for TextAhead {}
unsafe impl Sync for TextAhead {}
pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
gst::Element::register(
Some(plugin),
"textahead",
gst::Rank::Primary,
TextAhead::static_type(),
)
}