// This example demonstrates how to draw an overlay on a video stream using // Direct2D/DirectWrite/WIC and the overlay composition element. // {videotestsrc} - {overlaycomposition} - {capsfilter} - {videoconvert} - {autovideosink} // The capsfilter element allows us to dictate the video resolution we want for the // videotestsrc and the overlaycomposition element. use std::sync::{Arc, Mutex}; use byte_slice_cast::*; use anyhow::Error; use derive_more::{Display, Error}; use gst::prelude::*; use windows::{ Foundation::Numerics::*, Win32::{ Graphics::{ Direct2D::{Common::*, *}, DirectWrite::*, Dxgi::Common::*, Imaging::*, }, System::Com::*, }, }; #[derive(Debug, Display, Error)] #[display(fmt = "Received error from {}: {} (debug: {:?})", src, error, debug)] struct ErrorMessage { src: glib::GString, error: glib::Error, debug: Option, } struct DrawingContext { // Factory for creating render target d2d_factory: ID2D1Factory, // Used to create WIC bitmap surface wic_factory: IWICImagingFactory, // text layout holding text information (string, font, size, etc) text_layout: IDWriteTextLayout, // Holding rendred image bitmap: Option, // Bound to bitmap and used to actual Direct2D rendering render_target: Option, info: Option, } // Required for IWICBitmap unsafe impl Send for DrawingContext {} fn create_pipeline() -> Result { gst::init()?; let pipeline = gst::Pipeline::default(); // The videotestsrc supports multiple test patterns. In this example, we will use the // pattern with a white ball moving around the video's center point. let src = gst::ElementFactory::make("videotestsrc") .property_from_str("pattern", "ball") .build()?; let overlay = gst::ElementFactory::make("overlaycomposition").build()?; let caps = gst_video::VideoCapsBuilder::new() .width(800) .height(800) .framerate((30, 1).into()) .build(); let capsfilter = gst::ElementFactory::make("capsfilter") .property("caps", &caps) .build()?; let videoconvert = gst::ElementFactory::make("videoconvert").build()?; let sink = gst::ElementFactory::make("autovideosink").build()?; pipeline.add_many(&[&src, &overlay, &capsfilter, &videoconvert, &sink])?; gst::Element::link_many(&[&src, &overlay, &capsfilter, &videoconvert, &sink])?; // Most Direct2D/DirectWrite APIs (including factory methods) are marked as // "unsafe", but they shouldn't fail in practice let drawer = unsafe { let d2d_factory = D2D1CreateFactory::(D2D1_FACTORY_TYPE_MULTI_THREADED, None).unwrap(); let dwrite_factory = DWriteCreateFactory::(DWRITE_FACTORY_TYPE_SHARED).unwrap(); let text_format = dwrite_factory .CreateTextFormat( windows::w!("Arial"), None, DWRITE_FONT_WEIGHT_BOLD, DWRITE_FONT_STYLE_NORMAL, DWRITE_FONT_STRETCH_NORMAL, 32f32, windows::w!("en-us"), ) .unwrap(); let text_layout = dwrite_factory .CreateTextLayout( windows::w!("GStreamer").as_wide(), &text_format, // Size will be updated later on "caps-changed" signal 800f32, 800f32, ) .unwrap(); // Top (default) and center alignment text_layout .SetTextAlignment(DWRITE_TEXT_ALIGNMENT_CENTER) .unwrap(); let wic_factory: IWICImagingFactory = CoCreateInstance(&CLSID_WICImagingFactory, None, CLSCTX_ALL).unwrap(); Arc::new(Mutex::new(DrawingContext { d2d_factory, wic_factory, text_layout, bitmap: None, render_target: None, info: None, })) }; overlay.connect_closure( "draw", false, glib::closure!(@strong drawer => move |_overlay: &gst::Element, sample: &gst::Sample| { use std::f64::consts::PI; let drawer = drawer.lock().unwrap(); let buffer = sample.buffer().unwrap(); let timestamp = buffer.pts().unwrap(); let info = drawer.info.as_ref().unwrap(); let text_layout = &drawer.text_layout; let bitmap = drawer.bitmap.as_ref().unwrap(); let render_target = drawer.render_target.as_ref().unwrap(); let global_angle = 360. * (timestamp % (10 * gst::ClockTime::SECOND)).nseconds() as f64 / (10.0 * gst::ClockTime::SECOND.nseconds() as f64); let center_x = (info.width() / 2) as f32; let center_y = (info.height() / 2) as f32; let top_margin = (info.height() / 20) as f32; unsafe { // Begin drawing render_target.BeginDraw(); // Clear background render_target.Clear(Some(&D2D1_COLOR_F { r: 0f32, g: 0f32, b: 0f32, a: 0f32, })); // This loop will render 10 times the string "GStreamer" in a circle for i in 0..10 { let angle = (360. * f64::from(i)) / 10.0; let red = ((1.0 + f64::cos((angle - 60.0) * PI / 180.0)) / 2.0) as f32; let text_brush = render_target .CreateSolidColorBrush( &D2D1_COLOR_F { r: red, g: 0f32, b: 1f32 - red, a: 1f32, }, None, ) .unwrap(); let angle = (angle + global_angle) as f32; let matrix = Matrix3x2::rotation(angle, center_x, center_y); render_target.SetTransform(&matrix); render_target.DrawTextLayout( D2D_POINT_2F { x: 0f32, y: top_margin }, text_layout, &text_brush, D2D1_DRAW_TEXT_OPTIONS_NONE, ); } // EndDraw may not be successful for some reasons. // Ignores any error in this example let _ = render_target.EndDraw(None, None); // Make sure all operations is completed before copying // bitmap to buffer let _ = render_target.Flush(None::<*mut u64>, None::<*mut u64>); } let mut buffer = gst::Buffer::with_size((info.width() * info.height() * 4) as usize).unwrap(); { let buffer_mut = buffer.get_mut().unwrap(); let mut map = buffer_mut.map_writable().unwrap(); let dst = map.as_mut_slice_of::().unwrap(); unsafe { // Bitmap size is equal to the background image size. // Copy entire memory bitmap.CopyPixels(std::ptr::null(), info.width() * 4, dst).unwrap(); } } gst_video::VideoMeta::add_full( buffer.get_mut().unwrap(), gst_video::VideoFrameFlags::empty(), gst_video::VideoFormat::Bgra, info.width(), info.height(), &[0], &[(info.width() * 4) as i32], ) .unwrap(); // Turn the buffer into a VideoOverlayRectangle, then place // that into a VideoOverlayComposition and return it. // // A VideoOverlayComposition can take a Vec of such rectangles // spaced around the video frame, but we're just outputting 1 // here let rect = gst_video::VideoOverlayRectangle::new_raw( &buffer, 0, 0, info.width(), info.height(), gst_video::VideoOverlayFormatFlags::PREMULTIPLIED_ALPHA, ); gst_video::VideoOverlayComposition::new(Some(&rect)) .unwrap() }), ); // Add a signal handler to the overlay's "caps-changed" signal. This could e.g. // be called when the sink that we render to does not support resizing the image // itself - but the user just changed the window-size. The element after the overlay // will then change its caps and we use the notification about this change to // resize our canvas's size. // Another possibility for when this might happen is, when our video is a network // stream that dynamically changes resolution when enough bandwidth is available. overlay.connect_closure( "caps-changed", false, glib::closure!(move |_overlay: &gst::Element, caps: &gst::Caps, _width: u32, _height: u32| { let mut drawer = drawer.lock().unwrap(); let info = gst_video::VideoInfo::from_caps(caps).unwrap(); unsafe { // Update text layout to be identical to new video resolution drawer.text_layout.SetMaxWidth(info.width() as f32).unwrap(); drawer .text_layout .SetMaxHeight(info.height() as f32) .unwrap(); // Create new WIC bitmap with PBGRA format (pre-multiplied BGRA) let bitmap = drawer .wic_factory .CreateBitmap( info.width(), info.height(), &GUID_WICPixelFormat32bppPBGRA, WICBitmapCacheOnDemand, ) .unwrap(); let render_target = drawer .d2d_factory .CreateWicBitmapRenderTarget( &bitmap, &D2D1_RENDER_TARGET_PROPERTIES { r#type: D2D1_RENDER_TARGET_TYPE_DEFAULT, pixelFormat: D2D1_PIXEL_FORMAT { format: DXGI_FORMAT_B8G8R8A8_UNORM, alphaMode: D2D1_ALPHA_MODE_PREMULTIPLIED, }, // zero means default DPI dpiX: 0f32, dpiY: 0f32, usage: D2D1_RENDER_TARGET_USAGE_NONE, minLevel: D2D1_FEATURE_LEVEL_DEFAULT, }, ) .unwrap(); drawer.render_target = Some(render_target); drawer.bitmap = Some(bitmap); } drawer.info = Some(info); }), ); Ok(pipeline) } fn main_loop(pipeline: gst::Pipeline) -> Result<(), Error> { pipeline.set_state(gst::State::Playing)?; let bus = pipeline .bus() .expect("Pipeline without bus. Shouldn't happen!"); for msg in bus.iter_timed(gst::ClockTime::NONE) { use gst::MessageView; match msg.view() { MessageView::Eos(..) => break, MessageView::Error(err) => { pipeline.set_state(gst::State::Null)?; return Err(ErrorMessage { src: msg .src() .map(|s| s.path_string()) .unwrap_or_else(|| glib::GString::from("UNKNOWN")), error: err.error(), debug: err.debug(), } .into()); } _ => (), } } pipeline.set_state(gst::State::Null)?; Ok(()) } fn main() { // WIC requires COM initialization unsafe { CoInitializeEx(None, COINIT_MULTITHREADED).unwrap(); } match create_pipeline().and_then(main_loop) { Ok(r) => r, Err(e) => eprintln!("Error! {}", e), } unsafe { CoUninitialize(); } }