From d8fe1c64f1c0b8dfc246a07afedcda5e000134ca Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 22 Mar 2024 17:30:23 +1100 Subject: [PATCH] cea608overlay: use or own CEA-608 caption frame handling instead of libcaption Part-of: --- video/closedcaption/src/ccutils.rs | 41 ++ video/closedcaption/src/cea608overlay/imp.rs | 339 +++------- video/closedcaption/src/cea608utils.rs | 618 +++++++++++++++++++ 3 files changed, 740 insertions(+), 258 deletions(-) diff --git a/video/closedcaption/src/ccutils.rs b/video/closedcaption/src/ccutils.rs index 5be0ed17..32e74327 100644 --- a/video/closedcaption/src/ccutils.rs +++ b/video/closedcaption/src/ccutils.rs @@ -141,3 +141,44 @@ impl fmt::Display for ParseError { } impl std::error::Error for ParseError {} + +// FIXME: we want to render the text in the largest 32 x 15 characters +// that will fit the viewport. This is a truly terrible way to determine +// the appropriate font size, but we only need to run that on resolution +// changes, and the API that would allow us to precisely control the +// line height has not yet been exposed by the bindings: +// +// https://blogs.gnome.org/mclasen/2019/07/27/more-text-rendering-updates/ +// +// TODO: switch to the API presented in this post once it's been exposed +pub(crate) fn recalculate_pango_layout( + layout: &pango::Layout, + video_width: u32, + video_height: u32, +) -> i32 { + let mut font_desc = pango::FontDescription::from_string("monospace"); + let video_width = video_width * 80 / 100; + let video_height = video_height * 80 / 100; + + let mut font_size = 1; + let mut left_alignment = 0; + loop { + font_desc.set_size(font_size * pango::SCALE); + layout.set_font_description(Some(&font_desc)); + layout + .set_text("12345678901234567890123456789012\n2\n3\n4\n5\n6\n7\n8\n9\n0\n1\n2\n3\n4\n5"); + let (_ink_rect, logical_rect) = layout.extents(); + if logical_rect.width() > video_width as i32 * pango::SCALE + || logical_rect.height() > video_height as i32 * pango::SCALE + { + font_desc.set_size((font_size - 1) * pango::SCALE); + layout.set_font_description(Some(&font_desc)); + break; + } + left_alignment = (video_width as i32 - logical_rect.width() / pango::SCALE) / 2 + + video_width as i32 / 10; + font_size += 1; + } + + left_alignment +} diff --git a/video/closedcaption/src/cea608overlay/imp.rs b/video/closedcaption/src/cea608overlay/imp.rs index c3cba027..e0b6b9a4 100644 --- a/video/closedcaption/src/cea608overlay/imp.rs +++ b/video/closedcaption/src/cea608overlay/imp.rs @@ -15,10 +15,8 @@ use once_cell::sync::Lazy; use std::sync::Mutex; -use pango::prelude::*; - -use crate::caption_frame::{CaptionFrame, Status}; use crate::ccutils::extract_cdp; +use crate::cea608utils::Cea608Renderer; static CAT: Lazy = Lazy::new(|| { gst::DebugCategory::new( @@ -50,10 +48,8 @@ impl Default for Settings { struct State { video_info: Option, - layout: Option, - caption_frame: CaptionFrame, + renderer: Cea608Renderer, composition: Option, - left_alignment: i32, attach: bool, selected_field: Option, last_cc_pts: Option, @@ -68,10 +64,8 @@ impl Default for State { fn default() -> Self { Self { video_info: None, - layout: None, - caption_frame: CaptionFrame::default(), + renderer: Cea608Renderer::new(), composition: None, - left_alignment: 0, attach: false, selected_field: None, last_cc_pts: gst::ClockTime::NONE, @@ -87,165 +81,10 @@ pub struct Cea608Overlay { } impl Cea608Overlay { - // FIXME: we want to render the text in the largest 32 x 15 characters - // that will fit the viewport. This is a truly terrible way to determine - // the appropriate font size, but we only need to run that on resolution - // changes, and the API that would allow us to precisely control the - // line height has not yet been exposed by the bindings: - // - // https://blogs.gnome.org/mclasen/2019/07/27/more-text-rendering-updates/ - // - // TODO: switch to the API presented in this post once it's been exposed - fn recalculate_layout(&self, state: &mut State) -> Result { - let video_info = state.video_info.as_ref().unwrap(); - let fontmap = pangocairo::FontMap::new(); - let context = fontmap.create_context(); - context.set_language(Some(&pango::Language::from_string("en_US"))); - context.set_base_dir(pango::Direction::Ltr); - let layout = pango::Layout::new(&context); - layout.set_alignment(pango::Alignment::Left); - let mut font_desc = pango::FontDescription::from_string("monospace"); - - let mut font_size = 1; - let mut left_alignment = 0; - loop { - font_desc.set_size(font_size * pango::SCALE); - layout.set_font_description(Some(&font_desc)); - layout.set_text( - "12345678901234567890123456789012\n2\n3\n4\n5\n6\n7\n8\n9\n0\n1\n2\n3\n4\n5", - ); - let (_ink_rect, logical_rect) = layout.extents(); - if logical_rect.width() > video_info.width() as i32 * pango::SCALE - || logical_rect.height() > video_info.height() as i32 * pango::SCALE - { - font_desc.set_size((font_size - 1) * pango::SCALE); - layout.set_font_description(Some(&font_desc)); - break; - } - left_alignment = (video_info.width() as i32 - logical_rect.width() / pango::SCALE) / 2; - font_size += 1; + fn overlay(&self, state: &mut State) { + if let Some(rect) = state.renderer.generate_rectangle() { + state.composition = gst_video::VideoOverlayComposition::new(Some(&rect)).ok(); } - - if self.settings.lock().unwrap().black_background { - let attrs = pango::AttrList::new(); - let attr = pango::AttrColor::new_background(0, 0, 0); - attrs.insert(attr); - layout.set_attributes(Some(&attrs)); - } - - state.left_alignment = left_alignment; - state.layout = Some(layout); - - Ok(gst::FlowSuccess::Ok) - } - - fn overlay_text(&self, text: &str, state: &mut State) { - let video_info = state.video_info.as_ref().unwrap(); - let layout = state.layout.as_ref().unwrap(); - layout.set_text(text); - let (_ink_rect, logical_rect) = layout.extents(); - let height = logical_rect.height() / pango::SCALE; - let width = logical_rect.width() / pango::SCALE; - - // No text actually needs rendering - if width == 0 || height == 0 { - state.composition = None; - return; - } - - let render_buffer = || -> Option { - let mut buffer = gst::Buffer::with_size((width * height) as usize * 4).ok()?; - - gst_video::VideoMeta::add( - buffer.get_mut().unwrap(), - gst_video::VideoFrameFlags::empty(), - #[cfg(target_endian = "little")] - gst_video::VideoFormat::Bgra, - #[cfg(target_endian = "big")] - gst_video::VideoFormat::Argb, - width as u32, - height as u32, - ) - .ok()?; - let buffer = buffer.into_mapped_buffer_writable().unwrap(); - - // Pass ownership of the buffer to the cairo surface but keep around - // a raw pointer so we can later retrieve it again when the surface - // is done - let buffer_ptr = buffer.buffer().as_ptr(); - let surface = cairo::ImageSurface::create_for_data( - buffer, - cairo::Format::ARgb32, - width, - height, - width * 4, - ) - .ok()?; - - let cr = cairo::Context::new(&surface).ok()?; - - // Clear background - cr.set_operator(cairo::Operator::Source); - cr.set_source_rgba(0.0, 0.0, 0.0, 0.0); - cr.paint().ok()?; - - // Render text outline - cr.save().ok()?; - cr.set_operator(cairo::Operator::Over); - - cr.set_source_rgba(0.0, 0.0, 0.0, 1.0); - - pangocairo::functions::layout_path(&cr, layout); - cr.stroke().ok()?; - cr.restore().ok()?; - - // Render text - cr.save().ok()?; - cr.set_source_rgba(255.0, 255.0, 255.0, 1.0); - - pangocairo::functions::show_layout(&cr, layout); - - cr.restore().ok()?; - drop(cr); - - // Safety: The surface still owns a mutable reference to the buffer but our reference - // to the surface here is the last one. After dropping the surface the buffer would be - // freed, so we keep an additional strong reference here before dropping the surface, - // which is then returned. As such it's guaranteed that nothing is using the buffer - // anymore mutably. - unsafe { - assert_eq!( - cairo::ffi::cairo_surface_get_reference_count(surface.to_raw_none()), - 1 - ); - let buffer = glib::translate::from_glib_none(buffer_ptr); - drop(surface); - buffer - } - }; - - let buffer = match render_buffer() { - Some(buffer) => buffer, - None => { - gst::error!(CAT, imp: self, "Failed to render buffer"); - state.composition = None; - return; - } - }; - - let rect = gst_video::VideoOverlayRectangle::new_raw( - &buffer, - state.left_alignment, - (video_info.height() as i32 - height) / 2, - width as u32, - height as u32, - gst_video::VideoOverlayFormatFlags::PREMULTIPLIED_ALPHA, - ); - - state.composition = match gst_video::VideoOverlayComposition::new(Some(&rect)) { - Ok(composition) => Some(composition), - Err(_) => None, - }; } fn negotiate(&self, state: &mut State) -> Result { @@ -285,7 +124,12 @@ impl Cea608Overlay { state.attach = upstream_has_meta || downstream_accepts_meta; - let _ = state.layout.take(); + state + .renderer + .set_video_size(video_info.width(), video_info.height()); + state + .renderer + .set_channel(cea608_types::tables::Channel::ONE); if !self.srcpad.push_event(gst::event::Caps::new(&caps)) { Err(gst::FlowError::NotNegotiated) @@ -294,7 +138,7 @@ impl Cea608Overlay { } } - fn decode_cc_data(&self, pad: &gst::Pad, state: &mut State, data: &[u8], pts: gst::ClockTime) { + fn decode_cc_data(&self, state: &mut State, data: &[u8], pts: gst::ClockTime) { if data.len() % 3 != 0 { gst::warning!(CAT, "cc_data length is not a multiple of 3, truncating"); } @@ -303,57 +147,39 @@ impl Cea608Overlay { let cc_valid = (triple[0] & 0x04) == 0x04; let cc_type = triple[0] & 0x03; - if cc_valid { - if cc_type == 0x00 || cc_type == 0x01 { - if state.selected_field.is_none() { - state.selected_field = Some(cc_type); - gst::info!(CAT, imp: self, "Selected field {} automatically", cc_type); - } - - if Some(cc_type) == state.selected_field { - match state - .caption_frame - .decode((triple[1] as u16) << 8 | triple[2] as u16, 0.0) - { - Ok(Status::Ready) => { - let text = match state.caption_frame.to_text(true) { - Ok(text) => text, - Err(_) => { - gst::error!( - CAT, - obj: pad, - "Failed to convert caption frame to text" - ); - continue; - } - }; - - self.overlay_text(&text, state); - } - Ok(Status::Clear) => { - self.overlay_text("", state); - } - Ok(Status::Ok) => (), - Err(err) => { - gst::error!( - CAT, - obj: pad, - "Failed to decode caption frame: {:?}", - err - ); - } - } - - self.reset_timeout(state, pts); - } - } else { - break; + if !cc_valid { + continue; + } + if cc_type == 0x00 || cc_type == 0x01 { + if state.selected_field.is_none() { + state.selected_field = Some(cc_type); + gst::info!(CAT, imp: self, "Selected field {} automatically", cc_type); } + + if Some(cc_type) != state.selected_field { + continue; + }; + match state.renderer.push_pair([triple[1], triple[2]]) { + Err(e) => { + gst::warning!(CAT, imp: self, "Failed to parse incoming CEA-608: {e:?}"); + continue; + } + Ok(true) => { + state.composition.take(); + } + Ok(false) => continue, + }; + + self.overlay(state); + + self.reset_timeout(state, pts); + } else { + break; } } } - fn decode_s334_1a(&self, pad: &gst::Pad, state: &mut State, data: &[u8], pts: gst::ClockTime) { + fn decode_s334_1a(&self, state: &mut State, data: &[u8], pts: gst::ClockTime) { if data.len() % 3 != 0 { gst::warning!(CAT, "cc_data length is not a multiple of 3, truncating"); } @@ -365,24 +191,22 @@ impl Cea608Overlay { gst::info!(CAT, imp: self, "Selected field {} automatically", cc_type); } - if Some(cc_type) == state.selected_field { - if let Ok(Status::Ready) = state - .caption_frame - .decode((triple[1] as u16) << 8 | triple[2] as u16, 0.0) - { - let text = match state.caption_frame.to_text(true) { - Ok(text) => text, - Err(_) => { - gst::error!(CAT, obj: pad, "Failed to convert caption frame to text"); - continue; - } - }; - - self.overlay_text(&text, state); + if Some(cc_type) != state.selected_field { + continue; + }; + match state.renderer.push_pair([triple[1], triple[2]]) { + Err(e) => { + gst::warning!(CAT, imp: self, "Failed to parse incoming CEA-608: {e:?}"); + continue; } + Ok(true) => { + state.composition.take(); + } + Ok(false) => continue, + }; + self.overlay(state); - self.reset_timeout(state, pts); - } + self.reset_timeout(state, pts); } } @@ -408,15 +232,11 @@ impl Cea608Overlay { self.negotiate(&mut state)?; } - if state.layout.is_none() { - self.recalculate_layout(&mut state)?; - } - for meta in buffer.iter_meta::() { if meta.caption_type() == gst_video::VideoCaptionType::Cea708Cdp { match extract_cdp(meta.data()) { Ok(data) => { - self.decode_cc_data(pad, &mut state, data, pts); + self.decode_cc_data(&mut state, data, pts); } Err(e) => { gst::warning!(CAT, "{e}"); @@ -424,31 +244,25 @@ impl Cea608Overlay { } } } else if meta.caption_type() == gst_video::VideoCaptionType::Cea708Raw { - self.decode_cc_data(pad, &mut state, meta.data(), pts); + self.decode_cc_data(&mut state, meta.data(), pts); } else if meta.caption_type() == gst_video::VideoCaptionType::Cea608S3341a { - self.decode_s334_1a(pad, &mut state, meta.data(), pts); + self.decode_s334_1a(&mut state, meta.data(), pts); } else if meta.caption_type() == gst_video::VideoCaptionType::Cea608Raw { let data = meta.data(); assert!(data.len() % 2 == 0); - for i in 0..data.len() / 2 { - if let Ok(Status::Ready) = state - .caption_frame - .decode((data[i * 2] as u16) << 8 | data[i * 2 + 1] as u16, 0.0) - { - let text = match state.caption_frame.to_text(true) { - Ok(text) => text, - Err(_) => { - gst::error!( - CAT, - obj: pad, - "Failed to convert caption frame to text" - ); - continue; - } - }; + for pair in data.chunks_exact(2) { + match state.renderer.push_pair([pair[0], pair[1]]) { + Err(e) => { + gst::warning!(CAT, imp: self, "Failed to parse incoming CEA-608: {e:?}"); + continue; + } + Ok(true) => { + state.composition.take(); + } + Ok(false) => continue, + }; - self.overlay_text(&text, &mut state); - } + self.overlay(&mut state); self.reset_timeout(&mut state, pts); } @@ -504,9 +318,15 @@ impl Cea608Overlay { } } EventView::FlushStop(..) => { + let settings = self.settings.lock().unwrap(); let mut state = self.state.lock().unwrap(); - state.caption_frame = CaptionFrame::default(); + state.renderer = Cea608Renderer::new(); + state + .renderer + .set_black_background(settings.black_background); state.composition = None; + drop(state); + drop(settings); gst::Pad::event_default(pad, Some(&*self.obj()), event) } _ => gst::Pad::event_default(pad, Some(&*self.obj()), event), @@ -604,7 +424,10 @@ impl ObjectImpl for Cea608Overlay { let mut state = self.state.lock().unwrap(); settings.black_background = value.get().expect("type checked upstream"); - let _ = state.layout.take(); + state + .renderer + .set_black_background(settings.black_background); + state.composition.take(); } "timeout" => { let mut settings = self.settings.lock().unwrap(); diff --git a/video/closedcaption/src/cea608utils.rs b/video/closedcaption/src/cea608utils.rs index 9da5ede8..49ebe5ee 100644 --- a/video/closedcaption/src/cea608utils.rs +++ b/video/closedcaption/src/cea608utils.rs @@ -6,9 +6,30 @@ // // SPDX-License-Identifier: MPL-2.0 +use std::collections::VecDeque; + +use cea608_types::{ + tables::{Channel, Color, MidRow, PreambleAddressCode, PreambleType}, + Cea608, Cea608State, Mode, +}; use gst::glib; +use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; +use pango::prelude::*; + +use crate::ccutils::recalculate_pango_layout; + +use gst::prelude::MulDiv; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "cea608utils", + gst::DebugColorFlags::empty(), + Some("CEA 608 utilities"), + ) +}); + #[derive( Serialize, Deserialize, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy, glib::Enum, )] @@ -115,3 +136,600 @@ impl From for TextStyle { } } } + +const MAX_ROW: usize = 14; +const MAX_COLUMN: usize = 31; + +#[derive(Debug)] +pub struct Cea608Frame { + display_lines: VecDeque, + undisplay_lines: VecDeque, + mode: Option, + selected_channel: Option, + column: usize, + row: usize, + base_row: u8, + preamble: PreambleAddressCode, +} + +impl Cea608Frame { + pub fn new() -> Self { + Self { + display_lines: VecDeque::new(), + undisplay_lines: VecDeque::new(), + mode: None, + selected_channel: None, + column: 0, + row: MAX_ROW, + base_row: MAX_ROW as u8, + preamble: PreambleAddressCode::new(0, false, PreambleType::Indent0), + } + } + + fn write_lines(&mut self) -> Option<&mut VecDeque> { + match self.mode { + None => None, + Some(Mode::PopOn) => Some(&mut self.undisplay_lines), + _ => Some(&mut self.display_lines), + } + } + + fn line_mut(&mut self, row: usize) -> Option<&mut Cea608Line> { + self.write_lines() + .and_then(|lines| lines.iter_mut().find(|line| line.no == row)) + } + + fn ensure_cell(&mut self, row: usize, column: usize) { + let line = if let Some(line) = self.line_mut(row) { + line + } else { + let Some(lines) = self.write_lines() else { + return; + }; + lines.push_back(Cea608Line { + no: row, + line: VecDeque::new(), + initial_preamble: None, + }); + lines.back_mut().unwrap() + }; + while line.line.len() <= column { + line.line.push_back(Cea608Cell::Empty); + } + } + + fn cell_mut(&mut self, row: usize, column: usize) -> Option<&mut Cea608Cell> { + self.write_lines().and_then(|lines| { + lines + .iter_mut() + .find(|line| line.no == row) + .and_then(|line| line.line.get_mut(column)) + }) + } + + fn reset(&mut self) { + self.display_lines.clear(); + self.undisplay_lines.clear(); + self.mode = None; + self.column = 0; + self.selected_channel = None; + } + + pub fn set_channel(&mut self, channel: Channel) { + if Some(channel) != self.selected_channel { + self.reset(); + self.selected_channel = Some(channel); + } + } + + pub fn push_code(&mut self, cea608: Cea608) -> bool { + if self.selected_channel.is_none() { + self.selected_channel = Some(cea608.channel()); + } + if Some(cea608.channel()) != self.selected_channel { + return false; + } + match cea608 { + Cea608::Text(text) => { + let mut ret = false; + if text.needs_backspace { + ret |= self.backspace(); + } + if let Some(c) = text.char1 { + ret |= self.push_char(c); + } + if let Some(c) = text.char2 { + ret |= self.push_char(c); + } + ret + } + Cea608::NewMode(_chan, new_mode) => self.new_mode(new_mode), + Cea608::Preamble(_chan, preamble) => self.preamble(preamble), + Cea608::EraseDisplay(_chan) => { + self.display_lines.clear(); + true + } + Cea608::EraseNonDisplay(_chan) => { + self.undisplay_lines.clear(); + false + } + Cea608::EndOfCaption(_chan) => { + std::mem::swap(&mut self.display_lines, &mut self.undisplay_lines); + self.new_mode(cea608_types::Mode::PopOn); + true + } + Cea608::Backspace(_chan) => self.backspace(), + Cea608::TabOffset(_chan, n_tabs) => { + self.column = (self.column + n_tabs as usize).min(MAX_COLUMN); + false + } + Cea608::MidRowChange(_chan, midrow) => self.midrow(midrow), + Cea608::CarriageReturn(_chan) => self.carriage_return(), + Cea608::DeleteToEndOfRow(_chan) => self.delete_to_end_of_row(), + } + } + + fn push_char(&mut self, c: char) -> bool { + let row = self.mode.map_or(self.row, |mode| { + if mode.is_rollup() { + self.base_row as usize + } else { + self.row + } + }); + self.ensure_cell(row, self.column); + if self.column == 0 { + let preamble = self.preamble; + let Some(line) = self.line_mut(row) else { + return false; + }; + line.initial_preamble = Some(preamble); + } + let Some(cell) = self.cell_mut(row, self.column) else { + return false; + }; + *cell = Cea608Cell::Char(c); + self.column = (self.column + 1).min(MAX_COLUMN); + true + } + + fn new_mode(&mut self, new_mode: cea608_types::Mode) -> bool { + if Some(new_mode) == self.mode { + return false; + } + + // if we are changing to roll up mode, we need to reset + if new_mode.is_rollup() + && !self + .mode + .map_or(!new_mode.is_rollup(), |mode| mode.is_rollup()) + { + self.base_row = MAX_ROW as u8; + self.reset(); + } + + self.mode = Some(new_mode); + if new_mode.is_rollup() { + self.column = 0; + // XXX: do we need to move any existing captions? + } + true + } + + fn preamble(&mut self, preamble: PreambleAddressCode) -> bool { + self.preamble = preamble; + self.column = preamble.column() as usize; + let Some(mode) = self.mode else { + self.row = preamble.row() as usize; + return false; + }; + match mode { + Mode::PopOn | Mode::PaintOn => { + self.row = preamble.row() as usize; + } + Mode::RollUp2 | Mode::RollUp3 | Mode::RollUp4 => { + let base_row = preamble.row().max(mode.rollup_rows().unwrap_or(0)); + if self.base_row != base_row { + gst::debug!( + CAT, + "roll up base row change from {} to {base_row}", + self.base_row + ); + let diff = base_row as i8 - self.base_row as i8; + self.display_lines.retain(|line| { + (0..=MAX_ROW as isize).contains(&(line.no as isize + diff as isize)) + }); + for line in self.display_lines.iter_mut() { + line.no = (line.no as isize + diff as isize) as usize; + } + self.base_row = preamble.row(); + } + } + } + true + } + + fn midrow(&mut self, midrow: MidRow) -> bool { + self.ensure_cell(self.row, self.column); + let Some(cell) = self.cell_mut(self.row, self.column) else { + return false; + }; + *cell = Cea608Cell::MidRow(midrow); + self.column = (self.column + 1).min(MAX_COLUMN); + true + } + + fn carriage_return(&mut self) -> bool { + if !matches!( + self.mode, + Some( + cea608_types::Mode::RollUp2 + | cea608_types::Mode::RollUp3 + | cea608_types::Mode::RollUp4 + ) + ) { + // no-op for non roll up modes + return false; + } + let n_rows = self.mode.unwrap().rollup_rows().unwrap(); + self.display_lines + .retain(|line| line.no > (self.base_row - n_rows + 1) as usize); + for line in self.display_lines.iter_mut() { + line.no -= 1; + } + self.column = 0; + true + } + + fn backspace(&mut self) -> bool { + if self.column == 0 { + return false; + } + self.ensure_cell(self.row, self.column - 1); + let Some(cell) = self.cell_mut(self.row, self.column - 1) else { + return false; + }; + *cell = Cea608Cell::Empty; + self.column -= 1; + true + } + + fn delete_to_end_of_row(&mut self) -> bool { + let column = self.column; + let Some(line) = self.line_mut(self.row) else { + return false; + }; + while line.line.len() > column { + line.line.pop_back(); + } + true + } + + pub fn iter(&self) -> impl Iterator { + self.display_lines.iter() + } +} + +#[derive(Debug)] +pub struct Cea608Line { + no: usize, + line: VecDeque, + initial_preamble: Option, +} + +impl Cea608Line { + pub fn initial_preamble(&self) -> PreambleAddressCode { + self.initial_preamble + .unwrap_or_else(|| PreambleAddressCode::new(0, false, PreambleType::Indent0)) + } + + pub fn display_iter(&self) -> impl Iterator { + self.line.iter().enumerate() + } +} + +#[derive(Debug)] +pub enum Cea608Cell { + Empty, + Char(char), + MidRow(cea608_types::tables::MidRow), +} + +fn pango_foreground_color_from_608(color: Color) -> pango::AttrColor { + let (r, g, b) = match color { + Color::White => (u16::MAX, u16::MAX, u16::MAX), + Color::Green => (0, u16::MAX, 0), + Color::Blue => (0, 0, u16::MAX), + Color::Cyan => (0, u16::MAX, u16::MAX), + Color::Red => (u16::MAX, 0, 0), + Color::Yellow => (u16::MAX, u16::MAX, 0), + Color::Magenta => (u16::MAX, 0, u16::MAX), + }; + pango::AttrColor::new_foreground(r, g, b) +} + +pub struct Cea608Renderer { + frame: Cea608Frame, + state: Cea608State, + context: pango::Context, + layout: pango::Layout, + rectangle: Option, + video_width: u32, + video_height: u32, + left_alignment: i32, + black_background: bool, +} + +impl Cea608Renderer { + pub fn new() -> Self { + let video_width = 0; + let video_height = 0; + let fontmap = pangocairo::FontMap::new(); + let context = fontmap.create_context(); + context.set_language(Some(&pango::Language::from_string("en_US"))); + context.set_base_dir(pango::Direction::Ltr); + let layout = pango::Layout::new(&context); + layout.set_alignment(pango::Alignment::Left); + let left_alignment = recalculate_pango_layout(&layout, video_width, video_height); + Self { + frame: Cea608Frame::new(), + state: Cea608State::default(), + context, + layout, + rectangle: None, + video_width, + video_height, + left_alignment, + black_background: false, + } + } + + pub fn push_pair(&mut self, pair: [u8; 2]) -> Result { + if let Some(cea608) = self.state.decode(pair)? { + gst::trace!(CAT, "Decoded {cea608:?}"); + if self.frame.push_code(cea608) { + self.rectangle.take(); + return Ok(true); + } + } + Ok(false) + } + + pub fn set_channel(&mut self, channel: cea608_types::tables::Channel) { + if self.frame.selected_channel != Some(channel) { + self.frame.set_channel(channel); + self.rectangle.take(); + } + } + + pub fn set_video_size(&mut self, width: u32, height: u32) { + if width != self.video_width || height != self.video_height { + self.video_width = width; + self.video_height = height; + self.layout = pango::Layout::new(&self.context); + self.layout.set_alignment(pango::Alignment::Left); + self.left_alignment = recalculate_pango_layout(&self.layout, width, height); + self.rectangle.take(); + } + } + + pub fn set_black_background(&mut self, bg: bool) { + self.black_background = bg; + self.rectangle.take(); + } + + pub fn generate_rectangle(&mut self) -> Option { + if let Some(rectangle) = self.rectangle.clone() { + return Some(rectangle); + } + + // 1. generate pango layout + let mut text = String::new(); + let attrs = pango::AttrList::new(); + if self.black_background { + let attr = pango::AttrColor::new_background(0, 0, 0); + attrs.insert(attr); + } + + let mut first_row = 0; + let mut last_row = None; + for line in self.frame.iter() { + if let Some(last_row) = last_row { + for _ in 0..line.no - last_row { + text.push('\n'); + } + } else { + first_row = line.no; + } + last_row = Some(line.no); + + let idx = text.len() as u32; + let initial = line.initial_preamble(); + + let mut foreground_color_attr = pango_foreground_color_from_608(initial.color()); + foreground_color_attr.set_start_index(idx); + let mut last_color = initial.color(); + + let mut underline_attr = pango::AttrInt::new_underline(if initial.underline() { + pango::Underline::Single + } else { + pango::Underline::None + }); + underline_attr.set_start_index(idx); + let mut last_underline = initial.underline(); + + let mut italic_attr = if initial.italics() { + let mut attr = pango::AttrInt::new_style(pango::Style::Italic); + attr.set_start_index(idx); + Some(attr) + } else { + None + }; + + for (_char_no, cell) in line.display_iter() { + match cell { + Cea608Cell::MidRow(midrow) => { + text.push(' '); + let idx = text.len() as u32; + + if last_underline != midrow.underline() { + underline_attr.set_end_index(idx); + attrs.insert(underline_attr); + underline_attr = pango::AttrInt::new_underline(if midrow.underline() { + pango::Underline::Single + } else { + pango::Underline::None + }); + underline_attr.set_start_index(idx); + last_underline = midrow.underline(); + } + + if !midrow.italics() { + if let Some(mut italic) = italic_attr.take() { + italic.set_end_index(idx); + attrs.insert(italic); + } + } else if midrow.italics() && italic_attr.is_none() { + let mut attr = pango::AttrInt::new_style(pango::Style::Italic); + attr.set_start_index(idx); + italic_attr = Some(attr); + } + + if let Some(color) = midrow.color() { + if color != last_color { + foreground_color_attr.set_end_index(idx); + attrs.insert(foreground_color_attr); + foreground_color_attr = pango_foreground_color_from_608(color); + foreground_color_attr.set_start_index(idx); + last_color = color; + } + } + } + Cea608Cell::Empty => text.push(' '), + Cea608Cell::Char(c) => text.push(*c), + } + } + + let idx = text.len() as u32; + + foreground_color_attr.set_end_index(idx); + attrs.insert(foreground_color_attr); + underline_attr.set_end_index(idx); + attrs.insert(underline_attr); + if let Some(mut italics) = italic_attr.take() { + italics.set_end_index(idx); + attrs.insert(italics); + } + } + + // 2. render text + self.layout.set_text(&text); + self.layout.set_attributes(Some(&attrs)); + let (_ink_rect, logical_rect) = self.layout.extents(); + let height = logical_rect.height() / pango::SCALE; + let width = logical_rect.width() / pango::SCALE; + gst::debug!(CAT, "overlaying size {width}x{height}, text {text}"); + + // No text actually needs rendering + if width == 0 || height == 0 { + return None; + } + + let render_buffer = || -> Option { + let mut buffer = gst::Buffer::with_size((width * height) as usize * 4).ok()?; + + gst_video::VideoMeta::add( + buffer.get_mut().unwrap(), + gst_video::VideoFrameFlags::empty(), + #[cfg(target_endian = "little")] + gst_video::VideoFormat::Bgra, + #[cfg(target_endian = "big")] + gst_video::VideoFormat::Argb, + width as u32, + height as u32, + ) + .ok()?; + let buffer = buffer.into_mapped_buffer_writable().unwrap(); + + // Pass ownership of the buffer to the cairo surface but keep around + // a raw pointer so we can later retrieve it again when the surface + // is done + let buffer_ptr = buffer.buffer().as_ptr(); + let surface = cairo::ImageSurface::create_for_data( + buffer, + cairo::Format::ARgb32, + width, + height, + width * 4, + ) + .ok()?; + + let cr = cairo::Context::new(&surface).ok()?; + + // Clear background + cr.set_operator(cairo::Operator::Source); + cr.set_source_rgba(0.0, 0.0, 0.0, 0.0); + cr.paint().ok()?; + + // Render text outline + cr.save().ok()?; + cr.set_operator(cairo::Operator::Over); + + cr.set_source_rgba(0.0, 0.0, 0.0, 1.0); + + pangocairo::functions::layout_path(&cr, &self.layout); + cr.stroke().ok()?; + cr.restore().ok()?; + + // Render text + cr.save().ok()?; + cr.set_source_rgba(255.0, 255.0, 255.0, 1.0); + + pangocairo::functions::show_layout(&cr, &self.layout); + + cr.restore().ok()?; + drop(cr); + + // Safety: The surface still owns a mutable reference to the buffer but our reference + // to the surface here is the last one. After dropping the surface the buffer would be + // freed, so we keep an additional strong reference here before dropping the surface, + // which is then returned. As such it's guaranteed that nothing is using the buffer + // anymore mutably. + unsafe { + assert_eq!( + cairo::ffi::cairo_surface_get_reference_count(surface.to_raw_none()), + 1 + ); + let buffer = glib::translate::from_glib_none(buffer_ptr); + drop(surface); + buffer + } + }; + + let buffer = match render_buffer() { + Some(buffer) => buffer, + None => { + gst::error!(CAT, "Failed to render buffer"); + return None; + } + }; + + let vertical_padding = self.video_height / 10; + let safe_height = self.video_height.mul_div_floor(80, 100).unwrap(); + let first_row_position = (safe_height as i32) + .mul_div_round(first_row as i32, MAX_ROW as i32 + 1) + .unwrap(); + + let rect = gst_video::VideoOverlayRectangle::new_raw( + &buffer, + self.left_alignment, + first_row_position + vertical_padding as i32, + width as u32, + height as u32, + gst_video::VideoOverlayFormatFlags::PREMULTIPLIED_ALPHA, + ); + + self.rectangle = Some(rect.clone()); + Some(rect) + } +}