diff --git a/Cargo.lock b/Cargo.lock index 920ea8c4..7fc78698 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,9 +1020,9 @@ dependencies = [ [[package]] name = "cea708-types" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7f33493cb6f19aa19c6e688708f66bf792bc2c75137da2a03c7ebbdf7a44f9" +checksum = "82b825228dce83e7156c7cd189bcfe5ef8014320deca7dd619787fe594946426" dependencies = [ "env_logger 0.10.2", "log", diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 326b67ac..e4bd62ab 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -5311,6 +5311,75 @@ }, "rank": "none" }, + "cea708overlay": { + "author": "Matthew Waters ", + "description": "Renders CEA 708 closed caption meta over raw video frames", + "hierarchy": [ + "GstCea708Overlay", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Video/Overlay/Subtitle", + "pad-templates": { + "sink": { + "caps": "video/x-raw:\n format: { A444_16LE, A444_16BE, AYUV64, RGBA64_LE, ARGB64, ARGB64_LE, BGRA64_LE, ABGR64_LE, RGBA64_BE, ARGB64_BE, BGRA64_BE, ABGR64_BE, A422_16LE, A422_16BE, A420_16LE, A420_16BE, A444_12LE, GBRA_12LE, A444_12BE, GBRA_12BE, Y412_LE, Y412_BE, A422_12LE, A422_12BE, A420_12LE, A420_12BE, A444_10LE, GBRA_10LE, A444_10BE, GBRA_10BE, A422_10LE, A422_10BE, A420_10LE, A420_10BE, BGR10A2_LE, RGB10A2_LE, Y410, A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, A420, AV12, Y444_16LE, GBR_16LE, Y444_16BE, GBR_16BE, v216, P016_LE, P016_BE, Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y212_LE, Y212_BE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, Y444_10BE, GBR_10BE, r210, I422_10LE, I422_10BE, NV16_10LE32, Y210, UYVP, v210, I420_10LE, I420_10BE, P010_10LE, NV12_10LE40, NV12_10LE32, P010_10BE, MT2110R, MT2110T, NV12_10BE_8L128, NV12_10LE40_4L4, Y444, BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, BGR, Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "video/x-raw:\n format: { A444_16LE, A444_16BE, AYUV64, RGBA64_LE, ARGB64, ARGB64_LE, BGRA64_LE, ABGR64_LE, RGBA64_BE, ARGB64_BE, BGRA64_BE, ABGR64_BE, A422_16LE, A422_16BE, A420_16LE, A420_16BE, A444_12LE, GBRA_12LE, A444_12BE, GBRA_12BE, Y412_LE, Y412_BE, A422_12LE, A422_12BE, A420_12LE, A420_12BE, A444_10LE, GBRA_10LE, A444_10BE, GBRA_10BE, A422_10LE, A422_10BE, A420_10LE, A420_10BE, BGR10A2_LE, RGB10A2_LE, Y410, A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, A420, AV12, Y444_16LE, GBR_16LE, Y444_16BE, GBR_16BE, v216, P016_LE, P016_BE, Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y212_LE, Y212_BE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, Y444_10BE, GBR_10BE, r210, I422_10LE, I422_10BE, NV16_10LE32, Y210, UYVP, v210, I420_10LE, I420_10BE, P010_10LE, NV12_10LE40, NV12_10LE32, P010_10BE, MT2110R, MT2110T, NV12_10BE_8L128, NV12_10LE40_4L4, Y444, BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, BGR, Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "direction": "src", + "presence": "always" + } + }, + "properties": { + "cea608-channel": { + "blurb": "The cea608 channel (CC1-4) to render the caption for when available, (-1=automatic, 0=disabled)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "4", + "min": "-1", + "mutable": "playing", + "readable": true, + "type": "gint", + "writable": true + }, + "service": { + "blurb": "The service to render the caption for when available, (-1=automatic, 0=disabled)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "1", + "max": "31", + "min": "-1", + "mutable": "playing", + "readable": true, + "type": "gint", + "writable": true + }, + "timeout": { + "blurb": "Duration after which to erase overlay when no cc data has arrived for the selected service/channel", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "18446744073709551615", + "max": "18446744073709551615", + "min": "16000000000", + "mutable": "playing", + "readable": true, + "type": "guint64", + "writable": true + } + }, + "rank": "primary" + }, "jsontovtt": { "author": "Jan Schmidt ", "description": "Converts JSON to WebVTT", diff --git a/video/closedcaption/Cargo.toml b/video/closedcaption/Cargo.toml index 72b3af4f..7cf70f0d 100644 --- a/video/closedcaption/Cargo.toml +++ b/video/closedcaption/Cargo.toml @@ -20,7 +20,7 @@ pangocairo.workspace = true byteorder = "1" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["raw_value"] } -cea708-types = "0.3.1" +cea708-types = "0.3.2" cea608-types = "0.1.1" once_cell.workspace = true gst = { workspace = true, features = ["v1_16"]} diff --git a/video/closedcaption/src/ccutils.rs b/video/closedcaption/src/ccutils.rs index 32e74327..14597d4d 100644 --- a/video/closedcaption/src/ccutils.rs +++ b/video/closedcaption/src/ccutils.rs @@ -155,13 +155,12 @@ pub(crate) fn recalculate_pango_layout( layout: &pango::Layout, video_width: u32, video_height: u32, -) -> i32 { +) -> (i32, i32) { let mut font_desc = pango::FontDescription::from_string("monospace"); let video_width = video_width * 80 / 100; let video_height = video_height * 80 / 100; let mut font_size = 1; - let mut left_alignment = 0; loop { font_desc.set_size(font_size * pango::SCALE); layout.set_font_description(Some(&font_desc)); @@ -175,10 +174,12 @@ pub(crate) fn recalculate_pango_layout( layout.set_font_description(Some(&font_desc)); break; } - left_alignment = (video_width as i32 - logical_rect.width() / pango::SCALE) / 2 - + video_width as i32 / 10; font_size += 1; } - left_alignment + let (_ink_rect, logical_rect) = layout.extents(); + ( + logical_rect.width() / pango::SCALE, + logical_rect.height() / pango::SCALE, + ) } diff --git a/video/closedcaption/src/cea608utils.rs b/video/closedcaption/src/cea608utils.rs index 72208f65..a930e548 100644 --- a/video/closedcaption/src/cea608utils.rs +++ b/video/closedcaption/src/cea608utils.rs @@ -501,7 +501,7 @@ impl Cea608Renderer { context.set_base_dir(pango::Direction::Ltr); let layout = pango::Layout::new(&context); layout.set_alignment(pango::Alignment::Left); - let left_alignment = recalculate_pango_layout(&layout, video_width, video_height); + recalculate_pango_layout(&layout, video_width, video_height); Self { frame: Cea608Frame::new(), state: Cea608State::default(), @@ -510,7 +510,7 @@ impl Cea608Renderer { rectangle: None, video_width, video_height, - left_alignment, + left_alignment: 0, black_background: false, } } @@ -533,13 +533,19 @@ impl Cea608Renderer { } } + pub fn channel(&self) -> Option { + self.frame.selected_channel + } + pub fn set_video_size(&mut self, width: u32, height: u32) { if width != self.video_width || height != self.video_height { self.video_width = width; self.video_height = height; self.layout = pango::Layout::new(&self.context); self.layout.set_alignment(pango::Alignment::Left); - self.left_alignment = recalculate_pango_layout(&self.layout, width, height); + let (max_layout_width, _max_layout_height) = + recalculate_pango_layout(&self.layout, width, height); + self.left_alignment = (width as i32 - max_layout_width) / 2 + width as i32 / 10; self.rectangle.take(); } } diff --git a/video/closedcaption/src/cea708overlay/imp.rs b/video/closedcaption/src/cea708overlay/imp.rs new file mode 100644 index 00000000..5ff81527 --- /dev/null +++ b/video/closedcaption/src/cea708overlay/imp.rs @@ -0,0 +1,633 @@ +// Copyright (C) 2024 Matthew Waters +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; +use gst::subclass::prelude::*; +use gst_video::prelude::*; + +use once_cell::sync::Lazy; + +use std::sync::Mutex; + +use crate::ccutils::extract_cdp; +use crate::cea708utils::{Cea708Renderer, ServiceOrChannel}; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "cea708overlay", + gst::DebugColorFlags::empty(), + Some("CEA 708 overlay element"), + ) +}); + +const DEFAULT_CEA608_CHANNEL: i32 = -1; +const DEFAULT_SERVICE: i32 = 1; + +#[derive(Debug, Clone)] +struct Settings { + changed: bool, + cea608_channel: i32, + service: i32, + timeout: Option, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + changed: true, + cea608_channel: DEFAULT_CEA608_CHANNEL, + service: DEFAULT_SERVICE, + timeout: gst::ClockTime::NONE, + } + } +} + +struct State { + selected: Option, + enabled_608: bool, + enabled_708: bool, + + upstream_caps: Option, + video_info: Option, + cc_data_parser: cea708_types::CCDataParser, + cea708_renderer: Cea708Renderer, + attach: bool, + last_cc_pts: Option, +} + +impl Default for State { + fn default() -> Self { + let mut cc_data_parser = cea708_types::CCDataParser::default(); + cc_data_parser.handle_cea608(); + Self { + selected: None, + enabled_608: true, + enabled_708: true, + upstream_caps: None, + video_info: None, + cc_data_parser, + cea708_renderer: Cea708Renderer::new(), + attach: false, + last_cc_pts: gst::ClockTime::NONE, + } + } +} + +pub struct Cea708Overlay { + srcpad: gst::Pad, + sinkpad: gst::Pad, + state: Mutex, + settings: Mutex, +} + +impl Cea708Overlay { + fn render(&self, state: &mut State) -> Option { + state.cea708_renderer.generate_composition() + } + + fn check_service_channel(&self, state: &mut State) { + let mut settings = self.settings.lock().unwrap(); + if !settings.changed { + return; + } + state.selected = match settings.service { + -1 => state.selected, + 0 => { + if matches!(state.selected, Some(ServiceOrChannel::Service(_))) { + None + } else { + state.selected + } + } + val => Some(ServiceOrChannel::Service(val as u8)), + }; + if state.selected.is_none() || settings.cea608_channel == 0 { + state.selected = match settings.cea608_channel { + -1 => state.selected, + 0 => { + if matches!(state.selected, Some(ServiceOrChannel::Cea608Channel(_))) { + None + } else { + state.selected + } + } + val => Some(ServiceOrChannel::Cea608Channel( + cea608_types::Id::from_value(val as i8), + )), + }; + } + state.enabled_608 = settings.cea608_channel != 0; + state.enabled_708 = settings.service != 0; + gst::info!( + CAT, + "set service channel {:?}, from settings: {settings:?}", + state.selected + ); + + state.cea708_renderer.set_service_channel(state.selected); + settings.changed = false; + } + + fn negotiate(&self) { + let mut state = self.state.lock().unwrap(); + + let Some(caps) = state.upstream_caps.as_ref() else { + gst::element_imp_error!( + self, + gst::CoreError::Negotiation, + ["Element hasn't received valid video caps at negotiation time"] + ); + self.srcpad.mark_reconfigure(); + return; + }; + + let Some(video_info) = state.video_info.clone() else { + gst::element_imp_error!( + self, + gst::CoreError::Negotiation, + ["Element hasn't received valid video caps at negotiation time"] + ); + self.srcpad.mark_reconfigure(); + return; + }; + + let mut downstream_accepts_meta = false; + let mut caps = caps.clone(); + + let upstream_has_meta = state + .upstream_caps + .as_ref() + .and_then(|caps| { + caps.features(0) + .map(|f| f.contains(gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) + }) + .unwrap_or(false); + + if !upstream_has_meta { + let mut caps_clone = caps.clone(); + let overlay_caps = caps_clone.make_mut(); + + if let Some(features) = overlay_caps.features_mut(0) { + features.add(gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + drop(state); + let peercaps = self.srcpad.peer_query_caps(Some(&caps_clone)); + downstream_accepts_meta = !peercaps.is_empty(); + if downstream_accepts_meta { + caps = caps_clone; + } + state = self.state.lock().unwrap(); + } + } + + state.attach = upstream_has_meta || downstream_accepts_meta; + state + .cea708_renderer + .set_video_size(video_info.width(), video_info.height()); + + if !self.srcpad.push_event(gst::event::Caps::new(&caps)) { + self.srcpad.mark_reconfigure(); + } + } + + fn have_cea608( + &self, + state: &mut State, + field: cea608_types::tables::Field, + cea608: [u8; 2], + pts: gst::ClockTime, + ) { + gst::trace!(CAT, imp: self, "Handling CEA-608 for field {field:?} (selected: {:?}) data: {cea608:x?}", state.selected); + + match state.cea708_renderer.push_cea608(field, cea608) { + Err(e) => gst::warning!(CAT, imp: self, "Failed to parse CEA-608 data: {e:?}"), + Ok(true) => self.reset_timeout(state, pts), + _ => (), + } + } + + fn handle_cc_data(&self, state: &mut State, pts: gst::ClockTime) { + let cea608 = state.cc_data_parser.cea608().map(|c| c.to_vec()); + + while let Some(packet) = state.cc_data_parser.pop_packet() { + if !state.enabled_708 + || !matches!(state.selected, None | Some(ServiceOrChannel::Service(_))) + { + continue; + } + + for service in packet.services() { + if state.selected.is_none() { + gst::info!(CAT, imp: self, "Automatic selection chose CEA-708 service {}", service.number()); + state.selected = Some(ServiceOrChannel::Service(service.number())); + } + if Some(ServiceOrChannel::Service(service.number())) != state.selected { + continue; + } + + state.cea708_renderer.push_service(service); + } + } + + let Some(cea608) = cea608 else { + gst::log!(CAT, imp: self, "No CEA-608"); + return; + }; + + if !state.enabled_608 + || !matches!( + state.selected, + None | Some(ServiceOrChannel::Cea608Channel(_)) + ) + { + gst::log!(CAT, imp: self, "CEA-608 not to be used (enabled {}, selected {:?})", state.enabled_608, state.selected); + return; + } + + for pair in cea608 { + let (field, pair) = match pair { + cea708_types::Cea608::Field1(byte0, byte1) => { + (cea608_types::tables::Field::ONE, [byte0, byte1]) + } + cea708_types::Cea608::Field2(byte0, byte1) => { + (cea608_types::tables::Field::TWO, [byte0, byte1]) + } + }; + + self.have_cea608(state, field, pair, pts); + } + } + + fn decode_s334_1a(&self, state: &mut State, data: &[u8], pts: gst::ClockTime) { + if data.len() % 3 != 0 { + gst::warning!(CAT, "cc_data length is not a multiple of 3, truncating"); + } + + for triple in data.chunks_exact(3) { + let field = if triple[0] & 0x01 == 0x0 { + cea608_types::tables::Field::ONE + } else { + cea608_types::tables::Field::TWO + }; + + self.have_cea608(state, field, [triple[1], triple[2]], pts); + } + } + + fn reset_timeout(&self, state: &mut State, pts: gst::ClockTime) { + state.last_cc_pts = Some(pts); + } + + fn sink_chain( + &self, + pad: &gst::Pad, + mut buffer: gst::Buffer, + ) -> Result { + gst::log!(CAT, obj: pad, "Handling buffer {:?}", buffer); + + let pts = buffer.pts().ok_or_else(|| { + gst::error!(CAT, obj: pad, "Require timestamped buffers"); + gst::FlowError::Error + })?; + + let settings = self.settings.lock().unwrap(); + let caption_timeout = settings.timeout; + drop(settings); + + if self.srcpad.check_reconfigure() { + self.negotiate(); + } + + let mut state = self.state.lock().unwrap(); + self.check_service_channel(&mut state); + + for meta in buffer.iter_meta::() { + gst::log!(CAT, imp: self, "Have caption meta of type {:?}", meta.caption_type()); + + if meta.caption_type() == gst_video::VideoCaptionType::Cea708Cdp { + match extract_cdp(meta.data()) { + Ok(data) => { + let mut cc_data = vec![0x80 | 0x40 | ((data.len() / 3) & 0x1f) as u8, 0xFF]; + cc_data.extend(data); + match state.cc_data_parser.push(&cc_data) { + Ok(_) => self.handle_cc_data(&mut state, pts), + Err(e) => { + gst::warning!(CAT, "Failed to parse incoming data: {e}"); + gst::element_imp_warning!( + self, + gst::StreamError::Decode, + ["Failed to parse incoming data {e}"] + ); + state.cc_data_parser.flush(); + } + } + } + Err(e) => { + gst::warning!(CAT, "{e}"); + gst::element_imp_warning!(self, gst::StreamError::Decode, ["{e}"]); + } + } + } else if meta.caption_type() == gst_video::VideoCaptionType::Cea708Raw { + let mut cc_data = vec![0; 2]; + // reserved | process_cc_data | length + cc_data[0] = 0x80 | 0x40 | ((meta.data().len() / 3) & 0x1f) as u8; + cc_data[1] = 0xFF; + cc_data.extend(meta.data()); + match state.cc_data_parser.push(&cc_data) { + Ok(_) => self.handle_cc_data(&mut state, pts), + Err(e) => { + gst::warning!(CAT, "Failed to parse incoming data: {e}"); + gst::element_imp_warning!( + self, + gst::StreamError::Decode, + ["Failed to parse incoming data: {e}"] + ); + state.cc_data_parser.flush(); + } + } + } else if meta.caption_type() == gst_video::VideoCaptionType::Cea608S3341a { + self.decode_s334_1a(&mut state, meta.data(), pts); + } else if meta.caption_type() == gst_video::VideoCaptionType::Cea608Raw { + let data = meta.data(); + assert!(data.len() % 2 == 0); + for pair in data.chunks_exact(2) { + self.have_cea608( + &mut state, + cea608_types::tables::Field::ONE, + [pair[0], pair[1]], + pts, + ); + } + } + } + + let composition = self.render(&mut state); + + if let Some(timeout) = caption_timeout { + if let Some(interval) = pts.opt_saturating_sub(state.last_cc_pts) { + if interval > timeout { + gst::info!(CAT, imp: self, "Reached timeout, clearing overlay"); + state.cea708_renderer.clear_composition(); + state.last_cc_pts.take(); + } + } + } + + if let Some(composition) = &composition { + let buffer = buffer.make_mut(); + if state.attach { + gst_video::VideoOverlayCompositionMeta::add(buffer, composition); + } else { + let mut frame = gst_video::VideoFrameRef::from_buffer_ref_writable( + buffer, + state.video_info.as_ref().unwrap(), + ) + .unwrap(); + + if composition.blend(&mut frame).is_err() { + gst::error!(CAT, obj: pad, "Failed to blend composition"); + } + } + } + drop(state); + + self.srcpad.push(buffer) + } + + fn sink_event(&self, pad: &gst::Pad, event: gst::Event) -> bool { + use gst::EventView; + + gst::log!(CAT, obj: pad, "Handling event {:?}", event); + match event.view() { + EventView::Caps(c) => { + let mut state = self.state.lock().unwrap(); + state.upstream_caps = Some(c.caps_owned()); + state.video_info = gst_video::VideoInfo::from_caps(c.caps()).ok(); + drop(state); + self.srcpad.check_reconfigure(); + self.negotiate(); + true + } + EventView::FlushStop(..) => { + let mut state = self.state.lock().unwrap(); + state.cea708_renderer = Cea708Renderer::new(); + //state.cea608_renderer.set_black_background(settings.black_background); + drop(state); + + gst::Pad::event_default(pad, Some(&*self.obj()), event) + } + _ => gst::Pad::event_default(pad, Some(&*self.obj()), event), + } + } +} + +#[glib::object_subclass] +impl ObjectSubclass for Cea708Overlay { + const NAME: &'static str = "GstCea708Overlay"; + type Type = super::Cea708Overlay; + type ParentType = gst::Element; + + fn with_class(klass: &Self::Class) -> Self { + let templ = klass.pad_template("sink").unwrap(); + let sinkpad = gst::Pad::builder_from_template(&templ) + .chain_function(|pad, parent, buffer| { + Cea708Overlay::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |overlay| overlay.sink_chain(pad, buffer), + ) + }) + .event_function(|pad, parent, event| { + Cea708Overlay::catch_panic_pad_function( + parent, + || false, + |overlay| overlay.sink_event(pad, event), + ) + }) + .flags(gst::PadFlags::PROXY_CAPS) + .flags(gst::PadFlags::PROXY_ALLOCATION) + .build(); + + let templ = klass.pad_template("src").unwrap(); + let srcpad = gst::Pad::builder_from_template(&templ) + .flags(gst::PadFlags::PROXY_CAPS) + .flags(gst::PadFlags::PROXY_ALLOCATION) + .build(); + + Self { + srcpad, + sinkpad, + state: Mutex::new(State::default()), + settings: Mutex::new(Settings::default()), + } + } +} + +impl ObjectImpl for Cea708Overlay { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![ + glib::ParamSpecInt::builder("cea608-channel") + .nick("CEA-608 Channel") + .blurb("The cea608 channel (CC1-4) to render the caption for when available, (-1=automatic, 0=disabled)") + .minimum(-1) + .maximum(4) + .default_value(DEFAULT_CEA608_CHANNEL) + .mutable_playing() + .build(), + glib::ParamSpecInt::builder("service") + .nick("Service") + .blurb("The service to render the caption for when available, (-1=automatic, 0=disabled)") + .minimum(-1) + .maximum(31) + .default_value(DEFAULT_SERVICE) + .mutable_playing() + .build(), + glib::ParamSpecUInt64::builder("timeout") + .nick("Timeout") + .blurb("Duration after which to erase overlay when no cc data has arrived for the selected service/channel") + .minimum(16.seconds().nseconds()) + .default_value(u64::MAX) + .mutable_playing() + .build(), + ] + }); + + PROPERTIES.as_ref() + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + match pspec.name() { + "cea608-channel" => { + let mut settings = self.settings.lock().unwrap(); + + let new = value.get().expect("type checked upstream"); + if new != settings.cea608_channel { + settings.cea608_channel = new; + settings.changed = true; + } + } + "service" => { + let mut settings = self.settings.lock().unwrap(); + + let new = value.get().expect("type checked upstream"); + if new != settings.service { + settings.service = new; + settings.changed = true; + } + } + "timeout" => { + let mut settings = self.settings.lock().unwrap(); + + let timeout = value.get().expect("type checked upstream"); + + settings.timeout = match timeout { + u64::MAX => gst::ClockTime::NONE, + _ => Some(timeout.nseconds()), + }; + } + _ => unimplemented!(), + } + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "cea608-channel" => { + let settings = self.settings.lock().unwrap(); + settings.cea608_channel.to_value() + } + "service" => { + let settings = self.settings.lock().unwrap(); + settings.service.to_value() + } + "timeout" => { + let settings = self.settings.lock().unwrap(); + if let Some(timeout) = settings.timeout { + timeout.nseconds().to_value() + } else { + u64::MAX.to_value() + } + } + _ => unimplemented!(), + } + } + fn constructed(&self) { + self.parent_constructed(); + + let obj = self.obj(); + obj.add_pad(&self.sinkpad).unwrap(); + obj.add_pad(&self.srcpad).unwrap(); + } +} + +impl GstObjectImpl for Cea708Overlay {} + +impl ElementImpl for Cea708Overlay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "CEA 708 overlay", + "Video/Overlay/Subtitle", + "Renders CEA 708 closed caption meta over raw video frames", + "Matthew Waters ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let caps = gst_video::VideoFormat::iter_raw() + .into_video_caps() + .unwrap() + .build(); + + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } + + fn change_state( + &self, + transition: gst::StateChange, + ) -> Result { + gst::trace!(CAT, imp: self, "Changing state {:?}", transition); + + match transition { + gst::StateChange::ReadyToPaused | gst::StateChange::PausedToReady => { + // Reset the whole state + let mut state = self.state.lock().unwrap(); + *state = State::default(); + drop(state); + let mut settings = self.settings.lock().unwrap(); + settings.changed = true; + } + _ => (), + } + + self.parent_change_state(transition) + } +} diff --git a/video/closedcaption/src/cea708overlay/mod.rs b/video/closedcaption/src/cea708overlay/mod.rs new file mode 100644 index 00000000..e4048f4b --- /dev/null +++ b/video/closedcaption/src/cea708overlay/mod.rs @@ -0,0 +1,31 @@ +// Copyright (C) 2024 Matthew Waters +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +// Example command-line: +// +// gst-launch-1.0 cccombiner name=ccc ! cea708overlay ! autovideosink \ +// videotestsrc ! video/x-raw, width=1280, height=720 ! queue ! ccc.sink \ +// filesrc location=input.srt ! subparse ! tttocea708 ! queue ! ccc.caption + +use gst::glib; +use gst::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct Cea708Overlay(ObjectSubclass) @extends gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "cea708overlay", + gst::Rank::PRIMARY, + Cea708Overlay::static_type(), + ) +} diff --git a/video/closedcaption/src/cea708utils.rs b/video/closedcaption/src/cea708utils.rs index 4c5e5762..179648f2 100644 --- a/video/closedcaption/src/cea708utils.rs +++ b/video/closedcaption/src/cea708utils.rs @@ -8,11 +8,17 @@ use cea708_types::{tables::*, Service}; +use std::collections::VecDeque; + use gst::glib; +use gst::prelude::MulDiv; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; -use crate::cea608utils::TextStyle; +use pango::prelude::*; + +use crate::ccutils::recalculate_pango_layout; +use crate::cea608utils::{Cea608Renderer, TextStyle}; static CAT: Lazy = Lazy::new(|| { gst::DebugCategory::new( @@ -164,9 +170,9 @@ impl Cea708ServiceWriter { window, 0, Anchor::BottomMiddle, - false, - 70, - 105, + true, + 100, + 50, 14, 31, true, @@ -213,9 +219,9 @@ impl Cea708ServiceWriter { window, 0, Anchor::BottomMiddle, - false, - 70, - 105, + true, + 100, + 50, 14, 31, true, @@ -289,3 +295,1075 @@ impl Cea708ServiceWriter { self.push_codes(&[Code::SetPenColor(args)]) } } + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ServiceOrChannel { + Service(u8), + Cea608Channel(cea608_types::Id), +} + +pub struct Cea708Renderer { + selected: Option, + cea608: Cea608Renderer, + service: Option, + video_width: u32, + video_height: u32, + composition: Option, +} + +impl Cea708Renderer { + pub fn new() -> Self { + Self { + selected: None, + cea608: Cea608Renderer::new(), + service: None, + video_width: 0, + video_height: 0, + composition: None, + } + } + + pub fn set_video_size(&mut self, width: u32, height: u32) { + if width != self.video_width || height != self.video_height { + self.video_width = width; + self.video_height = height; + self.cea608.set_video_size(width, height); + if let Some(service) = self.service.as_mut() { + service.set_video_size(width, height); + } + self.composition.take(); + } + } + + pub fn set_service_channel(&mut self, service_channel: Option) { + if self.selected != service_channel { + self.selected = service_channel; + match service_channel { + Some(ServiceOrChannel::Cea608Channel(id)) => self.cea608.set_channel(id.channel()), + None => self.cea608.set_channel(cea608_types::tables::Channel::ONE), + _ => (), + } + } + } + + pub fn push_service(&mut self, service: &Service) { + for code in service.codes() { + let overlay_service = self.service.get_or_insert_with(|| { + let mut service = ServiceState::new(); + service.set_video_size(self.video_width, self.video_height); + service + }); + overlay_service.handle_code(code); + } + } + + pub fn push_cea608( + &mut self, + field: cea608_types::tables::Field, + pair: [u8; 2], + ) -> Result { + match self.selected { + Some(ServiceOrChannel::Cea608Channel(channel)) => { + if channel.field() != field { + // data is not for the configured field, ignore + return Ok(false); + } else { + channel + } + } + None => cea608_types::Id::from_caption_field_channel( + field, + cea608_types::tables::Channel::ONE, + ), + // data is not for the configured service, ignore + _ => return Ok(false), + }; + + let ret = self.cea608.push_pair(pair); + if let Ok(changed) = ret { + if self.selected.is_none() { + if let Some(chan) = self.cea608.channel() { + self.selected = Some(ServiceOrChannel::Cea608Channel( + cea608_types::Id::from_caption_field_channel(field, chan), + )); + } + } + if changed { + self.composition.take(); + } + } + ret + } + + pub fn clear_composition(&mut self) { + self.composition.take(); + } + + pub fn generate_composition(&mut self) -> Option { + let Some(selected) = self.selected else { + self.composition.take(); + return None; + }; + + if matches!(selected, ServiceOrChannel::Service(_)) { + let service = self.service.as_mut()?; + + let mut composition: Option = None; + for window in service.windows.iter_mut() { + if let Some(rectangle) = window.generate_rectangle() { + if let Some(composition) = composition.as_mut() { + composition.get_mut().unwrap().add_rectangle(&rectangle); + } else { + composition = + gst_video::VideoOverlayComposition::new(Some(&rectangle)).ok(); + } + } + } + + self.composition = composition; + } else if let Some(rectangle) = self.cea608.generate_rectangle() { + self.composition = gst_video::VideoOverlayComposition::new(Some(&rectangle)).ok(); + } + self.composition.clone() + } +} + +// SAFETY: Required because `pango::Layout` / `pango::Context` are not `Send` but the whole +// `ServiceState` needs to be. +// We ensure that no additional references to the layout are ever created, which makes it safe +// to send it to other threads as long as only a single thread uses it concurrently. +unsafe impl Send for ServiceState {} + +struct ServiceState { + windows: VecDeque, + current_window: usize, + pango_context: pango::Context, + video_width: u32, + video_height: u32, +} + +impl ServiceState { + fn new() -> Self { + let fontmap = pangocairo::FontMap::new(); + let context = fontmap.create_context(); + // XXX: may need a different language sometimes + context.set_language(Some(&pango::Language::from_string("en_US"))); + // XXX: May need a different direction + context.set_base_dir(pango::Direction::Ltr); + Self { + windows: VecDeque::new(), + current_window: usize::MAX, + pango_context: context, + video_width: 0, + video_height: 0, + } + } + + fn window_mut(&mut self, id: usize) -> Option<&mut Window> { + self.windows + .iter_mut() + .find(|window| window.define.window_id as usize == id) + } + + fn define_window(&mut self, args: &DefineWindowArgs) { + if let Some(window) = self.window_mut(args.window_id as usize) { + if &window.define != args { + // we only change these if they are different from the previous define_window + // command + window.attrs = args.window_attributes(); + window.pen_attrs = args.pen_attributes(); + window.pen_color = args.pen_color(); + } + window.define = *args; + window.recalculate_window_position(); + } else { + let layout = pango::Layout::new(&self.pango_context); + // XXX: May need a different alignment + layout.set_alignment(pango::Alignment::Left); + let mut window = Window { + visible: args.visible, + attrs: args.window_attributes(), + pen_attrs: args.pen_attributes(), + pen_color: args.pen_color(), + define: *args, + pen_location: SetPenLocationArgs::default(), + lines: VecDeque::new(), + rectangle: None, + layout, + video_dims: Dimensions::default(), + window_position: Dimensions::default(), + window_dims: Dimensions::default(), + max_layout_dims: Dimensions::default(), + }; + window.set_video_size(self.video_width, self.video_height); + self.windows.push_back(window); + }; + self.current_window = args.window_id as usize; + } + + fn set_current_window(&mut self, window_id: u8) { + self.current_window = window_id as usize; + } + + fn clear_windows(&mut self, args: &WindowBits) { + for window in self.windows.iter_mut() { + if (WindowBits::from_window_id(window.define.window_id) & *args) != WindowBits::NONE { + window.pen_location = SetPenLocationArgs::default(); + window.lines.clear(); + window.rectangle = None; + } + } + } + + fn delete_windows(&mut self, args: &WindowBits) { + self.windows.retain(|window| { + (WindowBits::from_window_id(window.define.window_id) & *args) == WindowBits::NONE + }); + } + + fn display_windows(&mut self, args: &WindowBits) { + for window in self.windows.iter_mut() { + if (WindowBits::from_window_id(window.define.window_id) & *args) != WindowBits::NONE { + window.visible = true; + } + } + } + + fn hide_windows(&mut self, args: &WindowBits) { + for window in self.windows.iter_mut() { + if (WindowBits::from_window_id(window.define.window_id) & *args) != WindowBits::NONE { + window.visible = false; + } + } + } + + fn toggle_windows(&mut self, args: &WindowBits) { + for window in self.windows.iter_mut() { + if (WindowBits::from_window_id(window.define.window_id) & *args) != WindowBits::NONE { + window.visible = !window.visible; + } + } + } + + fn set_window_attributes(&mut self, attrs: &SetWindowAttributesArgs) { + let Some(window) = self.window_mut(self.current_window) else { + return; + }; + if &window.attrs != attrs { + window.lines.clear(); + window.attrs = *attrs; + window.rectangle = None; + } + } + + fn set_pen_attributes(&mut self, attrs: &SetPenAttributesArgs) { + let Some(window) = self.window_mut(self.current_window) else { + return; + }; + window.pen_attrs = *attrs; + } + + fn set_pen_color(&mut self, color: &SetPenColorArgs) { + let Some(window) = self.window_mut(self.current_window) else { + return; + }; + window.pen_color = *color; + } + + fn set_pen_location(&mut self, location: &SetPenLocationArgs) { + let Some(window) = self.window_mut(self.current_window) else { + return; + }; + window.pen_location = *location; + } + + fn reset(&mut self) { + *self = Self::new(); + } + + fn handle_code(&mut self, code: &Code) { + match code { + Code::DefineWindow(args) => self.define_window(args), + Code::SetCurrentWindow0 => self.set_current_window(0), + Code::SetCurrentWindow1 => self.set_current_window(1), + Code::SetCurrentWindow2 => self.set_current_window(2), + Code::SetCurrentWindow3 => self.set_current_window(3), + Code::SetCurrentWindow4 => self.set_current_window(4), + Code::SetCurrentWindow5 => self.set_current_window(5), + Code::SetCurrentWindow6 => self.set_current_window(6), + Code::SetCurrentWindow7 => self.set_current_window(7), + Code::ClearWindows(args) => self.clear_windows(args), + Code::DeleteWindows(args) => self.delete_windows(args), + Code::DisplayWindows(args) => self.display_windows(args), + Code::HideWindows(args) => self.hide_windows(args), + Code::ToggleWindows(args) => self.toggle_windows(args), + Code::SetWindowAttributes(args) => self.set_window_attributes(args), + Code::SetPenAttributes(args) => self.set_pen_attributes(args), + Code::SetPenColor(args) => self.set_pen_color(args), + Code::SetPenLocation(args) => self.set_pen_location(args), + Code::BS => self.backspace(), + Code::CR => self.carriage_return(), + Code::FF => { + self.clear_windows(&WindowBits::from_window_id(self.current_window as u8)); + self.set_pen_location(&SetPenLocationArgs { row: 0, column: 0 }); + } + Code::ETX => (), + Code::HCR => self.horizontal_carriage_return(), + Code::Reset => self.reset(), + _ => { + if let Some(ch) = code.char() { + self.push_char(ch); + } + } + } + } + + fn push_char(&mut self, ch: char) { + let Some(window) = self.window_mut(self.current_window) else { + return; + }; + window.push_char(ch); + } + + fn backspace(&mut self) { + let Some(window) = self.window_mut(self.current_window) else { + return; + }; + window.backspace(); + } + + fn carriage_return(&mut self) { + let Some(window) = self.window_mut(self.current_window) else { + return; + }; + window.carriage_return(); + } + + fn horizontal_carriage_return(&mut self) { + let Some(window) = self.window_mut(self.current_window) else { + return; + }; + window.horizontal_carriage_return(); + } + + fn set_video_size(&mut self, video_width: u32, video_height: u32) { + for window in self.windows.iter_mut() { + window.set_video_size(video_width, video_height); + } + self.video_width = video_width; + self.video_height = video_height; + } +} + +fn color_value_as_u16(val: ColorValue) -> u16 { + match val { + ColorValue::None => 0, + ColorValue::OneThird => u16::MAX / 3, + ColorValue::TwoThirds => u16::MAX / 3 * 2, + ColorValue::Full => u16::MAX, + } +} + +fn opacity_as_u16(val: Opacity) -> u16 { + match val { + Opacity::Transparent => 0, + Opacity::Translucent => u16::MAX / 3, + // FIXME + Opacity::Flash => u16::MAX / 3 * 2, + Opacity::Solid => u16::MAX, + } +} + +fn pango_foreground_color_from_708(args: &SetPenColorArgs) -> pango::AttrColor { + pango::AttrColor::new_foreground( + color_value_as_u16(args.foreground_color.r), + color_value_as_u16(args.foreground_color.g), + color_value_as_u16(args.foreground_color.b), + ) +} + +fn pango_foreground_opacity_from_708(args: &SetPenColorArgs) -> pango::AttrInt { + pango::AttrInt::new_foreground_alpha(opacity_as_u16(args.foreground_opacity)) +} + +fn pango_background_color_from_708(args: &SetPenColorArgs) -> pango::AttrColor { + pango::AttrColor::new_foreground( + color_value_as_u16(args.background_color.r), + color_value_as_u16(args.background_color.g), + color_value_as_u16(args.background_color.b), + ) +} + +fn pango_background_opacity_from_708(args: &SetPenColorArgs) -> pango::AttrInt { + pango::AttrInt::new_background_alpha(opacity_as_u16(args.background_opacity)) +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct Dimensions { + w: u32, + h: u32, +} + +pub enum Align { + First, + Center, + Last, +} + +impl Align { + fn horizontal_from_anchor(anchor: Anchor) -> Self { + match anchor { + Anchor::TopLeft | Anchor::CenterLeft | Anchor::BottomLeft => Self::First, + Anchor::TopMiddle | Anchor::CenterMiddle | Anchor::BottomMiddle => Self::Center, + Anchor::TopRight | Anchor::CenterRight | Anchor::BottomRight => Self::Last, + _ => Self::Center, + } + } + + fn vertical_from_anchor(anchor: Anchor) -> Self { + match anchor { + Anchor::TopLeft | Anchor::TopMiddle | Anchor::TopRight => Self::First, + Anchor::CenterLeft | Anchor::CenterMiddle | Anchor::CenterRight => Self::Center, + Anchor::BottomLeft | Anchor::BottomMiddle | Anchor::BottomRight => Self::Last, + _ => Self::Last, + } + } +} + +struct Window { + visible: bool, + define: DefineWindowArgs, + attrs: SetWindowAttributesArgs, + pen_attrs: SetPenAttributesArgs, + pen_color: SetPenColorArgs, + pen_location: SetPenLocationArgs, + lines: VecDeque, + + window_position: Dimensions, + video_dims: Dimensions, + window_dims: Dimensions, + max_layout_dims: Dimensions, + rectangle: Option, + layout: pango::Layout, +} + +impl Window { + fn dump(&self) { + for line in self.lines.iter() { + let mut string = line.no.to_string(); + string.push(' '); + for cell in line.line.iter() { + string.push(cell.character.unwrap_or(' ')); + } + string.push('|'); + gst::trace!(CAT, "dump: {string}"); + } + } + + fn ensure_cell(&mut self, row: usize, column: usize) { + let line = if let Some(line) = self.lines.iter_mut().find(|line| line.no == row) { + line + } else { + self.lines.push_back(WindowLine { + no: row, + line: VecDeque::new(), + }); + self.lines.back_mut().unwrap() + }; + while line.line.len() <= column { + line.line + .push_back(Cell::new_empty(self.pen_attrs, self.pen_color)); + } + } + + fn cell_mut(&mut self, row: usize, column: usize) -> Option<&mut Cell> { + self.lines + .iter_mut() + .find(|line| line.no == row) + .and_then(|line| line.line.get_mut(column)) + } + + fn backspace(&mut self) { + match self.attrs.print_direction { + Direction::LeftToRight => { + self.pen_location.column = self.pen_location.column.max(1) - 1; + } + Direction::RightToLeft => { + self.pen_location.column = (self.pen_location.column + 1).min(self.column_count()); + } + Direction::TopToBottom => { + self.pen_location.row = self.pen_location.row.max(1) - 1; + } + Direction::BottomToTop => { + self.pen_location.row = (self.pen_location.row + 1).min(self.row_count()); + } + } + self.ensure_cell( + self.pen_location.row as usize, + self.pen_location.column as usize, + ); + let cell = self + .cell_mut( + self.pen_location.row as usize, + self.pen_location.column as usize, + ) + .unwrap(); + if cell.character.take().is_some() { + self.rectangle.take(); + } + } + + fn row_count(&self) -> u8 { + self.define.row_count + 1 + } + + fn column_count(&self) -> u8 { + self.define.column_count + 1 + } + + fn move_to_line_beginning(&mut self) { + match self.attrs.print_direction { + Direction::LeftToRight => { + self.pen_location.column = 0; + } + Direction::RightToLeft => { + self.pen_location.column = self.define.column_count; + } + Direction::TopToBottom => { + self.pen_location.row = 0; + } + Direction::BottomToTop => { + self.pen_location.row = self.row_count(); + } + } + } + + fn scroll_top_to_bottom(&mut self) { + if self.pen_location.row == 0 { + let row_count = self.row_count() as usize; + self.lines + .retain(|line| (0..=row_count - 1).contains(&line.no)); + for line in self.lines.iter_mut() { + line.no += 1; + } + } else { + self.pen_location.row -= 1; + } + } + + fn scroll_bottom_to_top(&mut self) { + if self.pen_location.row >= self.define.row_count { + let row_count = self.row_count() as usize; + self.lines.retain(|line| (1..=row_count).contains(&line.no)); + for line in self.lines.iter_mut() { + line.no -= 1 + } + } else { + self.pen_location.row += 1; + } + } + + fn scroll_left_to_right(&mut self) { + if self.pen_location.column == 0 { + gst::warning!(CAT, "Unsupported scroll direction left-to-right"); + } else { + self.pen_location.column -= 1; + } + } + + fn scroll_right_to_left(&mut self) { + if self.pen_location.column >= self.column_count() { + gst::warning!(CAT, "Unsupported scroll direction right-to-left"); + } else { + self.pen_location.column += 1; + } + } + + fn carriage_return(&mut self) { + match (self.attrs.print_direction, self.attrs.scroll_direction) { + (Direction::LeftToRight, Direction::TopToBottom) => { + self.scroll_top_to_bottom(); + self.move_to_line_beginning(); + } + (Direction::LeftToRight, Direction::BottomToTop) => { + self.scroll_bottom_to_top(); + self.move_to_line_beginning(); + } + (Direction::RightToLeft, Direction::TopToBottom) => { + self.scroll_top_to_bottom(); + self.move_to_line_beginning(); + } + (Direction::RightToLeft, Direction::BottomToTop) => { + self.scroll_bottom_to_top(); + self.move_to_line_beginning(); + } + (Direction::TopToBottom, Direction::LeftToRight) => { + self.scroll_left_to_right(); + self.move_to_line_beginning(); + } + (Direction::TopToBottom, Direction::RightToLeft) => { + self.scroll_right_to_left(); + self.move_to_line_beginning(); + } + (Direction::BottomToTop, Direction::LeftToRight) => { + self.scroll_left_to_right(); + self.move_to_line_beginning(); + } + (Direction::BottomToTop, Direction::RightToLeft) => { + self.scroll_right_to_left(); + self.move_to_line_beginning(); + } + // all other variants invalid + (print, scroll) => { + gst::warning!( + CAT, + "Unspecified print direction ({print:?}) and scroll direction ({scroll:?})" + ); + return; + } + } + gst::trace!( + CAT, + "carriage return after position {},{}", + self.pen_location.row, + self.pen_location.column + ); + self.rectangle.take(); + } + + fn horizontal_carriage_return(&mut self) { + let min_row; + let max_row; + let min_column; + let max_column; + match self.attrs.print_direction { + Direction::LeftToRight => { + min_row = self.pen_location.row; + max_row = self.pen_location.row; + max_column = self.pen_location.column; + min_column = 0; + self.pen_location.column = 0; + } + Direction::RightToLeft => { + min_row = self.pen_location.row; + max_row = self.pen_location.row; + min_column = self.pen_location.column; + max_column = self.row_count(); + self.pen_location.column = self.row_count(); + } + Direction::TopToBottom => { + min_column = self.pen_location.column; + max_column = self.pen_location.column; + min_row = self.pen_location.row; + max_row = 0; + self.pen_location.row = 0; + } + Direction::BottomToTop => { + min_column = self.pen_location.column; + max_column = self.pen_location.column; + min_row = self.pen_location.row; + max_row = self.column_count(); + self.pen_location.row = self.column_count(); + } + } + for row in min_row..=max_row { + for column in min_column..=max_column { + self.ensure_cell(row as usize, column as usize); + let cell = self.cell_mut(row as usize, column as usize).unwrap(); + cell.character = None; + } + } + self.rectangle.take(); + } + + fn push_char(&mut self, ch: char) { + if self.pen_location.row > self.row_count() { + gst::warning!( + CAT, + "row {} outside configured window row count {}", + self.pen_location.row, + self.row_count() + ); + return; + } + if self.pen_location.column > self.column_count() { + gst::warning!( + CAT, + "column {} outside configured window column count {}", + self.pen_location.column, + self.column_count() + ); + return; + } + gst::trace!( + CAT, + "push char \'{ch}\' at row {} column {}", + self.pen_location.row, + self.pen_location.column + ); + self.ensure_cell( + self.pen_location.row as usize, + self.pen_location.column as usize, + ); + let cell = self + .cell_mut( + self.pen_location.row as usize, + self.pen_location.column as usize, + ) + .unwrap(); + cell.character = Some(ch); + self.rectangle.take(); + + match self.attrs.print_direction { + Direction::LeftToRight => { + self.pen_location.column = (self.pen_location.column + 1).min(self.column_count()); + } + Direction::RightToLeft => { + self.pen_location.column = self.pen_location.column.max(1) - 1; + } + Direction::TopToBottom => { + self.pen_location.row = (self.pen_location.row + 1).min(self.row_count()); + } + Direction::BottomToTop => { + self.pen_location.row = self.pen_location.row.max(1) - 1; + } + } + } + + fn recalculate_window_position(&mut self) { + self.rectangle.take(); + + // XXX: may need a better implementation for 'skinny' (horizontal or vertical) output + // sizes. + + let (max_layout_width, max_layout_height) = + recalculate_pango_layout(&self.layout, self.video_dims.w, self.video_dims.h); + self.max_layout_dims = Dimensions { + w: max_layout_width as u32, + h: max_layout_height as u32, + }; + + let char_width = max_layout_width as u32 / 32; + let char_height = max_layout_height as u32 / 15; + let height = self.row_count() as u32 * char_height; + let width = self.column_count() as u32 * char_width; + self.window_dims = Dimensions { + w: width, + h: height, + }; + + let padding = Dimensions { + w: self.video_dims.w / 10, + h: self.video_dims.h / 10, + }; + let safe_area = Dimensions { + w: self.video_dims.w - self.video_dims.w / 5, + h: self.video_dims.h - self.video_dims.h / 5, + }; + + self.window_position = if self.define.relative_positioning { + let halign = Align::horizontal_from_anchor(self.define.anchor_point); + let valign = Align::vertical_from_anchor(self.define.anchor_point); + let x = safe_area + .w + .mul_div_round(self.define.anchor_horizontal.min(100) as u32, 100) + .unwrap(); + let x = padding.w + + match halign { + Align::First => x, + Align::Center => x.max(self.max_layout_dims.w / 2) - self.max_layout_dims.w / 2, + Align::Last => x.max(self.window_dims.w) - self.window_dims.w, + }; + let y = safe_area + .h + .mul_div_round(self.define.anchor_vertical.min(100) as u32, 100) + .unwrap(); + let y = padding.h + + match valign { + Align::First => y, + Align::Center => y.max(self.max_layout_dims.h / 2), + Align::Last => y.max(self.window_dims.h) - self.window_dims.h, + }; + Dimensions { w: x, h: y } + } else { + // FIXME + gst::fixme!(CAT, "Handle non-relative-positioning"); + padding + }; + + gst::trace!( + CAT, + "char sizes {char_width}x{char_height}, row/columns {}x{}, safe area {:?} window dimensions: {:?}, window position: {:?}, max layout {:?}, define {:?}", + self.row_count(), + self.column_count(), + safe_area, + self.window_dims, + self.window_position, + self.max_layout_dims, + self.define, + ); + } + + fn set_video_size(&mut self, video_width: u32, video_height: u32) { + let new_dims = Dimensions { + w: video_width, + h: video_height, + }; + if new_dims == self.video_dims { + return; + } + self.video_dims = new_dims; + + self.recalculate_window_position(); + } + + fn generate_rectangle(&mut self) -> Option { + if !self.visible { + return None; + } + + if self.rectangle.is_some() { + return self.rectangle.clone(); + } + self.dump(); + + // 1. generate the pango layout for the text + let mut text = String::new(); + let attrs = pango::AttrList::new(); + let mut last_color = None; + let mut last_attrs = None; + let mut background_color_attr = pango::AttrColor::new_background(0, 0, 0); + let mut background_opacity_attr = pango::AttrInt::new_background_alpha(0); + let mut foreground_color_attr = + pango::AttrColor::new_background(u16::MAX, u16::MAX, u16::MAX); + let mut foreground_opacity_attr = pango::AttrInt::new_background_alpha(u16::MAX); + let mut underline_attr = pango::AttrInt::new_underline(pango::Underline::None); + let mut italic_attr = None::; + let mut last_row = 0; + for line in self.lines.iter() { + for _ in 0..line.no - last_row { + text.push('\n'); + } + last_row = line.no; + for c in line.line.iter() { + // XXX: Need to double check these indices with more complicated text characters + let start_idx = text.len(); + if last_color.map(|col| col != c.pen_color).unwrap_or(true) { + background_color_attr.set_end_index(start_idx as u32); + attrs.insert(background_color_attr.clone()); + background_color_attr = pango_background_color_from_708(&c.pen_color); + background_color_attr.set_start_index(start_idx as u32); + + background_opacity_attr.set_end_index(start_idx as u32); + attrs.insert(background_opacity_attr.clone()); + background_opacity_attr = pango_background_opacity_from_708(&c.pen_color); + background_opacity_attr.set_start_index(start_idx as u32); + + foreground_color_attr.set_end_index(start_idx as u32); + attrs.insert(foreground_color_attr.clone()); + foreground_color_attr = pango_foreground_color_from_708(&c.pen_color); + foreground_color_attr.set_start_index(start_idx as u32); + + foreground_opacity_attr.set_end_index(start_idx as u32); + attrs.insert(foreground_opacity_attr.clone()); + foreground_opacity_attr = pango_foreground_opacity_from_708(&c.pen_color); + foreground_opacity_attr.set_start_index(start_idx as u32); + + last_color = Some(c.pen_color); + } + if last_attrs.map(|attrs| attrs != c.pen_attrs).unwrap_or(true) { + underline_attr.set_end_index(start_idx as u32); + attrs.insert(underline_attr.clone()); + let underline_type = if c.pen_attrs.underline { + pango::Underline::Single + } else { + pango::Underline::None + }; + underline_attr = pango::AttrInt::new_underline(underline_type); + underline_attr.set_start_index(start_idx as u32); + + if !c.pen_attrs.italics { + if let Some(mut italic) = italic_attr.take() { + italic.set_end_index(start_idx as u32); + attrs.insert(italic.clone()); + } + } else if c.pen_attrs.italics && italic_attr.is_none() { + let mut attr = pango::AttrInt::new_style(pango::Style::Italic); + attr.set_start_index(start_idx as u32); + italic_attr = Some(attr); + } + + last_attrs = Some(c.pen_attrs); + } + + let Some(character) = c.character else { + text.push(' '); + continue; + }; + text.push(character); + } + } + let start_idx = text.len(); + background_color_attr.set_end_index(start_idx as u32); + attrs.insert(background_color_attr.clone()); + background_opacity_attr.set_end_index(start_idx as u32); + attrs.insert(background_opacity_attr.clone()); + foreground_color_attr.set_end_index(start_idx as u32); + attrs.insert(foreground_color_attr.clone()); + foreground_opacity_attr.set_end_index(start_idx as u32); + attrs.insert(foreground_opacity_attr.clone()); + underline_attr.set_end_index(start_idx as u32); + attrs.insert(underline_attr.clone()); + if let Some(mut italic) = italic_attr { + italic.set_end_index(start_idx as u32); + attrs.insert(italic); + } + + self.layout.set_text(&text); + self.layout.set_attributes(Some(&attrs)); + let (_ink_rect, logical_rect) = self.layout.extents(); + let height = logical_rect.height() / pango::SCALE; + let width = logical_rect.width() / pango::SCALE; + + // 2. render text and window + let render_buffer = || -> Result { + let mut buffer = gst::Buffer::with_size((width * height) as usize * 4)?; + + gst_video::VideoMeta::add( + buffer.get_mut().unwrap(), + gst_video::VideoFrameFlags::empty(), + #[cfg(target_endian = "little")] + gst_video::VideoFormat::Bgra, + #[cfg(target_endian = "big")] + gst_video::VideoFormat::Argb, + width as u32, + height as u32, + )?; + let buffer = buffer.into_mapped_buffer_writable().unwrap(); + + // Pass ownership of the buffer to the cairo surface but keep around + // a raw pointer so we can later retrieve it again when the surface + // is done + let buffer_ptr = buffer.buffer().as_ptr(); + let surface = cairo::ImageSurface::create_for_data( + buffer, + cairo::Format::ARgb32, + width, + height, + width * 4, + )?; + + let cr = cairo::Context::new(&surface)?; + + // Clear background + cr.set_operator(cairo::Operator::Source); + cr.set_source_rgba(0.0, 0.0, 0.0, 0.0); + cr.paint()?; + + // Render text outline + cr.save()?; + cr.set_operator(cairo::Operator::Over); + + cr.set_source_rgba(0.0, 0.0, 0.0, 1.0); + + pangocairo::functions::layout_path(&cr, &self.layout); + cr.stroke()?; + cr.restore()?; + + // Render text + cr.save()?; + cr.set_source_rgba(255.0, 255.0, 255.0, 1.0); + + pangocairo::functions::show_layout(&cr, &self.layout); + + cr.restore()?; + drop(cr); + + // Safety: The surface still owns a mutable reference to the buffer but our reference + // to the surface here is the last one. After dropping the surface the buffer would be + // freed, so we keep an additional strong reference here before dropping the surface, + // which is then returned. As such it's guaranteed that nothing is using the buffer + // anymore mutably. + unsafe { + assert_eq!( + cairo::ffi::cairo_surface_get_reference_count(surface.to_raw_none()), + 1 + ); + let buffer = glib::translate::from_glib_none(buffer_ptr); + drop(surface); + Ok(buffer) + } + }; + + let buffer = match render_buffer() { + Ok(buffer) => buffer, + Err(e) => { + self.dump(); + gst::error!(CAT, "Failed to render buffer: \"{e}\""); + return None; + } + }; + gst::trace!( + CAT, + "sizes: video {:?}, window {:?} overlay {}x{}", + self.video_dims, + self.window_dims, + width, + height + ); + + // 3. generate overlay rectangle + // FIXME: use the window location values to place the overlay + let ret = Some(gst_video::VideoOverlayRectangle::new_raw( + &buffer, + self.window_position.w as i32, + self.window_position.h as i32, + width as u32, + height as u32, + gst_video::VideoOverlayFormatFlags::PREMULTIPLIED_ALPHA, + )); + self.rectangle = ret.clone(); + ret + } +} + +struct WindowLine { + no: usize, + line: VecDeque, +} + +impl PartialOrd for WindowLine { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for WindowLine { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.no.cmp(&other.no) + } +} + +impl PartialEq for WindowLine { + fn eq(&self, other: &Self) -> bool { + self.no == other.no + } +} + +impl Eq for WindowLine {} + +struct Cell { + character: Option, + pen_attrs: SetPenAttributesArgs, + pen_color: SetPenColorArgs, +} + +impl Cell { + fn new_empty(attrs: SetPenAttributesArgs, color: SetPenColorArgs) -> Self { + Self { + character: None, + pen_attrs: attrs, + pen_color: color, + } + } +} diff --git a/video/closedcaption/src/lib.rs b/video/closedcaption/src/lib.rs index ed3f2fec..5a34a248 100644 --- a/video/closedcaption/src/lib.rs +++ b/video/closedcaption/src/lib.rs @@ -25,6 +25,7 @@ mod cea608tojson; mod cea608tott; mod cea608utils; mod cea708mux; +mod cea708overlay; mod cea708utils; mod jsontovtt; mod line_reader; @@ -60,6 +61,7 @@ fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { cea608tocea708::register(plugin)?; cea708mux::register(plugin)?; tttocea708::register(plugin)?; + cea708overlay::register(plugin)?; Ok(()) }