From 6c876a56814779b8b0b48e443163f9081131188e Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Thu, 28 May 2020 14:37:23 +1000 Subject: [PATCH] examples: Add a VideoOverlayComposition example Replicate the pango-cairo cairooverlay example, but drawing into a gst::VideoOverlayComposition using the overlaycomposition element. --- examples/Cargo.toml | 6 + examples/src/bin/overlay-composition.rs | 339 ++++++++++++++++++++++++ 2 files changed, 345 insertions(+) create mode 100644 examples/src/bin/overlay-composition.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 47d702681..291fad0b0 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -28,6 +28,7 @@ derive_more = "0.99.5" futures = "0.3" byte-slice-cast = "0.3" cairo-rs = { git = "https://github.com/gtk-rs/cairo", features=["use_glib"], optional = true } +cairo-sys-rs = { git = "https://github.com/gtk-rs/cairo", features=["use_glib"], optional = true } pango = { git = "https://github.com/gtk-rs/pango", optional = true } pangocairo = { git = "https://github.com/gtk-rs/pangocairo", optional = true } glutin = { version = "0.21", optional = true } @@ -49,6 +50,7 @@ gst-rtsp-server = ["gstreamer-rtsp-server", "gstreamer-rtsp", "gstreamer-sdp"] gst-rtsp-server-record = ["gstreamer-rtsp-server-sys", "gstreamer-rtsp-server", "gstreamer-rtsp", "gio"] v1_10 = ["gstreamer/v1_10"] pango-cairo = ["pango", "pangocairo", "cairo-rs"] +overlay-composition = ["pango", "pangocairo", "cairo-rs", "cairo-sys-rs" ] gl = ["gstreamer-gl", "gl_generator", "glutin"] gl-egl = ["gstreamer-gl/egl"] gl-x11 = ["gstreamer-gl/x11"] @@ -143,6 +145,10 @@ name = "discoverer" name = "pango-cairo" required-features = ["pango-cairo"] +[[bin]] +name = "overlay-composition" +required-features = ["overlay-composition"] + [[bin]] name = "ges" required-features = ["ges"] diff --git a/examples/src/bin/overlay-composition.rs b/examples/src/bin/overlay-composition.rs new file mode 100644 index 000000000..d5d4fcadc --- /dev/null +++ b/examples/src/bin/overlay-composition.rs @@ -0,0 +1,339 @@ +// This example demonstrates how to draw an overlay on a video stream using +// cairo and the overlay composition element. +// Additionally, this example uses functionality of the pango library, which handles +// text layouting. The pangocairo crate is a nice wrapper combining both libraries +// into a nice interface. + +// {videotestsrc} - {overlaycomposition} - {capsfilter} - {videoconvert} - {autovideosink} +// The capsfilter element allows us to dictate the video resolution we want for the +// videotestsrc and the overlaycomposition element. +// +// There is a small amount of unsafe code that demonstrates how to work around +// Cairo's internal refcounting of the target buffer surface + +extern crate gstreamer as gst; +use gst::prelude::*; + +extern crate gstreamer_video as gst_video; +use pango::prelude::*; + +use std::ops; +use std::sync::{Arc, Mutex}; + +use anyhow::Error; +use derive_more::{Display, Error}; + +#[path = "../examples-common.rs"] +mod examples_common; + +#[derive(Debug, Display, Error)] +#[display(fmt = "Missing element {}", _0)] +struct MissingElement(#[error(not(source))] &'static str); + +#[derive(Debug, Display, Error)] +#[display(fmt = "Received error from {}: {} (debug: {:?})", src, error, debug)] +struct ErrorMessage { + src: String, + error: String, + debug: Option, + source: glib::Error, +} + +struct DrawingContext { + layout: glib::SendUniqueCell, + info: Option, +} + +#[derive(Debug)] +struct LayoutWrapper(pango::Layout); + +impl ops::Deref for LayoutWrapper { + type Target = pango::Layout; + + fn deref(&self) -> &pango::Layout { + &self.0 + } +} + +unsafe impl glib::SendUnique for LayoutWrapper { + fn is_unique(&self) -> bool { + self.0.ref_count() == 1 + } +} + +fn create_pipeline() -> Result { + gst::init()?; + + let pipeline = gst::Pipeline::new(None); + let src = gst::ElementFactory::make("videotestsrc", None) + .map_err(|_| MissingElement("videotestsrc"))?; + let overlay = gst::ElementFactory::make("overlaycomposition", None) + .map_err(|_| MissingElement("overlaycomposition"))?; + let capsfilter = + gst::ElementFactory::make("capsfilter", None).map_err(|_| MissingElement("capsfilter"))?; + let videoconvert = gst::ElementFactory::make("videoconvert", None) + .map_err(|_| MissingElement("videoconvert"))?; + let sink = gst::ElementFactory::make("autovideosink", None) + .map_err(|_| MissingElement("autovideosink"))?; + + pipeline.add_many(&[&src, &overlay, &capsfilter, &videoconvert, &sink])?; + gst::Element::link_many(&[&src, &overlay, &capsfilter, &videoconvert, &sink])?; + + // Plug in a capsfilter element that will force the videotestsrc and the overlay to work + // with images of the size 800x800, and framerate of 15 fps, since my laptop struggles + // rendering it at the default 30 fps + let caps = gst::Caps::builder("video/x-raw") + .field("width", &800i32) + .field("height", &800i32) + .field("framerate", &gst::Fraction::new(15, 1)) + .build(); + capsfilter.set_property("caps", &caps).unwrap(); + + // The videotestsrc supports multiple test patterns. In this example, we will use the + // pattern with a white ball moving around the video's center point. + src.set_property_from_str("pattern", "ball"); + + // The PangoFontMap represents the set of fonts available for a particular rendering system. + let fontmap = pangocairo::FontMap::new().unwrap(); + // Create a new pango layouting context for the fontmap. + let context = fontmap.create_context().unwrap(); + // Create a pango layout object. This object is a string of text we want to layout. + // It is wrapped in a LayoutWrapper (defined above) to be able to send it across threads. + let layout = LayoutWrapper(pango::Layout::new(&context)); + + // Select the text content and the font we want to use for the piece of text. + let font_desc = pango::FontDescription::from_string("Sans Bold 26"); + layout.set_font_description(Some(&font_desc)); + layout.set_text("GStreamer"); + + // The following is a context struct (containing the pango layout and the configured video info). + // We have to wrap it in an Arc (or Rc) to get reference counting, that is: to be able to have + // shared ownership of it in multiple different places (the two signal handlers here). + // We have to wrap it in a Mutex because Rust's type-system can't know that both signals are + // only ever called from a single thread (the streaming thread). It would be enough to have + // something that is Send in theory but that's not how signal handlers are generated unfortunately. + // The Mutex (or otherwise if we didn't need the Sync bound we could use a RefCell) is to implement + // interior mutability (see Rust docs). Via this we can get a mutable reference to the contained + // data which is checked at runtime for uniqueness (blocking in case of mutex, panic in case + // of refcell) instead of compile-time (like with normal references). + let drawer = Arc::new(Mutex::new(DrawingContext { + layout: glib::SendUniqueCell::new(layout).unwrap(), + info: None, + })); + + let drawer_clone = drawer.clone(); + // Connect to the overlaycomposition element's "draw" signal, which is emitted for + // each videoframe piped through the element. The signal handler needs to + // return a gst_video::VideoOverlayComposition to be drawn on the frame + // + // Signals connected with the connect(, ...) API get their arguments + // passed as array of glib::Value. For a documentation about the actual arguments + // it is always a good idea to check the element's signals using either + // gst-inspect, or the online documentation. + // + // In this case, the signal passes the gst::Element and a gst::Sample with + // the current buffer + overlay + .connect("draw", false, move |args| { + use std::f64::consts::PI; + + let drawer = &drawer_clone; + let drawer = drawer.lock().unwrap(); + + // Get the signal's arguments + let _overlay = args[0].get::().unwrap().unwrap(); + let sample = args[1].get::().unwrap().unwrap(); + let buffer = sample.get_buffer().unwrap(); + let timestamp = buffer.get_pts(); + + let info = drawer.info.as_ref().unwrap(); + let layout = drawer.layout.borrow(); + + let angle = 2.0 + * PI + * ((timestamp % (10 * gst::SECOND)).unwrap() as f64 + / (10.0 * gst::SECOND_VAL as f64)); + + /* Create a gst::Buffer for Cairo to draw into */ + let frame_width = info.width() as usize; + let frame_height = info.height() as usize; + let stride = 4 * frame_width; + let frame_size = stride * frame_height; + + /* Create an RGBA buffer, and add a video meta that the videooverlaycomposition expects */ + let mut buffer = gst::Buffer::with_size(frame_size).unwrap(); + + gst_video::VideoMeta::add( + buffer.get_mut().unwrap(), + gst_video::VideoFrameFlags::NONE, + gst_video::VideoFormat::Bgra, + frame_width as u32, + frame_height as u32, + ); + + let buffer = buffer.into_mapped_buffer_writable().unwrap(); + let buffer = { + let buffer_ptr = unsafe { buffer.get_buffer().as_ptr() }; + let surface = cairo::ImageSurface::create_for_data( + buffer, + cairo::Format::ARgb32, + frame_width as i32, + frame_height as i32, + stride as i32, + ) + .unwrap(); + + let cr = cairo::Context::new(&surface); + + cr.save(); + cr.set_operator(cairo::Operator::Clear); + cr.paint(); + cr.restore(); + + // The image we draw (the text) will be static, but we will change the + // transformation on the drawing context, which rotates and shifts everything + // that we draw afterwards. Like this, we have no complicated calulations + // in the actual drawing below. + // Calling multiple transformation methods after each other will apply the + // new transformation on top. If you repeat the cr.rotate(angle) line below + // this a second time, everything in the canvas will rotate twice as fast. + cr.translate( + f64::from(info.width()) / 2.0, + f64::from(info.height()) / 2.0, + ); + cr.rotate(angle); + + // This loop will render 10 times the string "GStreamer" in a circle + for i in 0..10 { + // Cairo, like most rendering frameworks, is using a stack for transformations + // with this, we push our current transformation onto this stack - allowing us + // to make temporary changes / render something / and then returning to the + // previous transformations. + cr.save(); + + let angle = (360. * f64::from(i)) / 10.0; + let red = (1.0 + f64::cos((angle - 60.0) * PI / 180.0)) / 2.0; + cr.set_source_rgb(red, 0.0, 1.0 - red); + cr.rotate(angle * PI / 180.0); + + // Update the text layout. This function is only updating pango's internal state. + // So e.g. that after a 90 degree rotation it knows that what was previously going + // to end up as a 200x100 rectangle would now be 100x200. + pangocairo::functions::update_layout(&cr, &**layout); + let (width, _height) = layout.get_size(); + // Using width and height of the text, we can properly possition it within + // our canvas. + cr.move_to( + -(f64::from(width) / f64::from(pango::SCALE)) / 2.0, + -(f64::from(info.height())) / 2.0, + ); + // After telling the layout object where to draw itself, we actually tell + // it to draw itself into our cairo context. + pangocairo::functions::show_layout(&cr, &**layout); + + // Here we go one step up in our stack of transformations, removing any + // changes we did to them since the last call to cr.save(); + cr.restore(); + } + + // Safety: The surface still owns a mutable reference to the buffer but our reference + // to the surface here is the last one. After dropping the surface the buffer would be + // freed, so we keep an additional strong reference here before dropping the surface, + // which is then returned. As such it's guaranteed that nothing is using the buffer + // anymore mutably. + drop(cr); + unsafe { + assert_eq!( + cairo_sys::cairo_surface_get_reference_count(surface.to_raw_none()), + 1 + ); + let buffer = glib::translate::from_glib_none(buffer_ptr); + drop(surface); + buffer + } + }; + + /* Turn the buffer into a VideoOverlayRectangle, then place + * that into a VideoOverlayComposition and return it. + * + * A VideoOverlayComposition can take a Vec of such rectangles + * spaced around the video frame, but we're just outputting 1 + * here */ + let rect = gst_video::VideoOverlayRectangle::new_raw( + &buffer, + 0, 0, frame_width as u32, frame_height as u32, + gst_video::VideoOverlayFormatFlags::PREMULTIPLIED_ALPHA, + ); + + Some(gst_video::VideoOverlayComposition::new(Some(&rect)).unwrap().to_value()) + }) + .unwrap(); + + // Add a signal handler to the overlay's "caps-changed" signal. This could e.g. + // be called when the sink that we render to does not support resizing the image + // itself - but the user just changed the window-size. The element after the overlay + // will then change its caps and we use the notification about this change to + // resize our canvas's size. + // Another possibility for when this might happen is, when our video is a network + // stream that dynamically changes resolution when enough bandwith is available. + overlay + .connect("caps-changed", false, move |args| { + let _overlay = args[0].get::().unwrap().unwrap(); + let caps = args[1].get::().unwrap().unwrap(); + + let mut drawer = drawer.lock().unwrap(); + drawer.info = Some(gst_video::VideoInfo::from_caps(&caps).unwrap()); + + None + }) + .unwrap(); + + Ok(pipeline) +} + +fn main_loop(pipeline: gst::Pipeline) -> Result<(), Error> { + pipeline.set_state(gst::State::Playing)?; + + let bus = pipeline + .get_bus() + .expect("Pipeline without bus. Shouldn't happen!"); + + for msg in bus.iter_timed(gst::CLOCK_TIME_NONE) { + use gst::MessageView; + + match msg.view() { + MessageView::Eos(..) => break, + MessageView::Error(err) => { + pipeline.set_state(gst::State::Null)?; + return Err(ErrorMessage { + src: msg + .get_src() + .map(|s| String::from(s.get_path_string())) + .unwrap_or_else(|| String::from("None")), + error: err.get_error().to_string(), + debug: err.get_debug(), + source: err.get_error(), + } + .into()); + } + _ => (), + } + } + + pipeline.set_state(gst::State::Null)?; + + Ok(()) +} + +fn example_main() { + match create_pipeline().and_then(main_loop) { + Ok(r) => r, + Err(e) => eprintln!("Error! {}", e), + } +} + +fn main() { + // tutorials_common::run is only required to set up the application environent on macOS + // (but not necessary in normal Cocoa applications where this is set up autmatically) + examples_common::run(example_main); +}