From 6cb19c1f18f7d3f5c24fbb2f154ca603a4d8fdc6 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Fri, 27 Jan 2023 02:11:37 +0900 Subject: [PATCH] examples: Add Direct2D/DirectWrite text rendering example Similar to overlay-composition example but for Direct2D/DirectWrite Part-of: --- examples/Cargo.toml | 6 +- examples/src/bin/overlay-composition-d2d.rs | 359 ++++++++++++++++++++ 2 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 examples/src/bin/overlay-composition-d2d.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index be0ae93a4..04a88eca6 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -47,7 +47,7 @@ windows = { version = "0.48", features=["Win32_Graphics_Direct3D11", "Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Direct2D", "Win32_Graphics_Direct2D_Common", "Win32_Graphics_DirectWrite", - "Foundation_Numerics"], optional = true } + "Win32_Graphics_Imaging", "Win32_System_Com", "Foundation_Numerics"], optional = true } [target.'cfg(target_os = "macos")'.dependencies] cocoa = "0.24" @@ -173,6 +173,10 @@ required-features = ["pango-cairo"] name = "overlay-composition" required-features = ["overlay-composition"] +[[bin]] +name = "overlay-composition-d2d" +required-features = ["windows"] + [[bin]] name = "ges" required-features = ["ges"] diff --git a/examples/src/bin/overlay-composition-d2d.rs b/examples/src/bin/overlay-composition-d2d.rs new file mode 100644 index 000000000..88fb9f387 --- /dev/null +++ b/examples/src/bin/overlay-composition-d2d.rs @@ -0,0 +1,359 @@ +// This example demonstrates how to draw an overlay on a video stream using +// Direct2D/DirectWrite/WIC and the overlay composition element. + +// {videotestsrc} - {overlaycomposition} - {capsfilter} - {videoconvert} - {autovideosink} +// The capsfilter element allows us to dictate the video resolution we want for the +// videotestsrc and the overlaycomposition element. + +use std::sync::{Arc, Mutex}; + +use byte_slice_cast::*; + +use anyhow::Error; +use derive_more::{Display, Error}; +use gst::prelude::*; +use windows::{ + Foundation::Numerics::*, + Win32::{ + Graphics::{ + Direct2D::{Common::*, *}, + DirectWrite::*, + Dxgi::Common::*, + Imaging::*, + }, + System::Com::*, + }, +}; + +#[derive(Debug, Display, Error)] +#[display(fmt = "Received error from {}: {} (debug: {:?})", src, error, debug)] +struct ErrorMessage { + src: glib::GString, + error: glib::Error, + debug: Option, +} + +struct DrawingContext { + // Factory for creating render target + d2d_factory: ID2D1Factory, + + // Used to create WIC bitmap surface + wic_factory: IWICImagingFactory, + + // text layout holding text information (string, font, size, etc) + text_layout: IDWriteTextLayout, + + // Holding rendred image + bitmap: Option, + + // Bound to bitmap and used to actual Direct2D rendering + render_target: Option, + + info: Option, +} + +// Required for IWICBitmap +unsafe impl Send for DrawingContext {} + +fn create_pipeline() -> Result { + gst::init()?; + + let pipeline = gst::Pipeline::default(); + + // The videotestsrc supports multiple test patterns. In this example, we will use the + // pattern with a white ball moving around the video's center point. + let src = gst::ElementFactory::make("videotestsrc") + .property_from_str("pattern", "ball") + .build()?; + + let overlay = gst::ElementFactory::make("overlaycomposition").build()?; + + let caps = gst_video::VideoCapsBuilder::new() + .width(800) + .height(800) + .framerate((30, 1).into()) + .build(); + let capsfilter = gst::ElementFactory::make("capsfilter") + .property("caps", &caps) + .build()?; + + let videoconvert = gst::ElementFactory::make("videoconvert").build()?; + let sink = gst::ElementFactory::make("autovideosink").build()?; + + pipeline.add_many(&[&src, &overlay, &capsfilter, &videoconvert, &sink])?; + gst::Element::link_many(&[&src, &overlay, &capsfilter, &videoconvert, &sink])?; + + // Most Direct2D/DirectWrite APIs (including factory methods) are marked as + // "unsafe", but they shouldn't fail in practice + let drawer = unsafe { + let d2d_factory = + D2D1CreateFactory::(D2D1_FACTORY_TYPE_MULTI_THREADED, None).unwrap(); + let dwrite_factory = + DWriteCreateFactory::(DWRITE_FACTORY_TYPE_SHARED).unwrap(); + let text_format = dwrite_factory + .CreateTextFormat( + windows::w!("Arial"), + None, + DWRITE_FONT_WEIGHT_BOLD, + DWRITE_FONT_STYLE_NORMAL, + DWRITE_FONT_STRETCH_NORMAL, + 32f32, + windows::w!("en-us"), + ) + .unwrap(); + let text_layout = dwrite_factory + .CreateTextLayout( + windows::w!("GStreamer").as_wide(), + &text_format, + // Size will be updated later on "caps-changed" signal + 800f32, + 800f32, + ) + .unwrap(); + + // Top (default) and center alignment + text_layout + .SetTextAlignment(DWRITE_TEXT_ALIGNMENT_CENTER) + .unwrap(); + + let wic_factory: IWICImagingFactory = + CoCreateInstance(&CLSID_WICImagingFactory, None, CLSCTX_ALL).unwrap(); + + Arc::new(Mutex::new(DrawingContext { + d2d_factory, + wic_factory, + text_layout, + bitmap: None, + render_target: None, + info: None, + })) + }; + + overlay.connect_closure( + "draw", + false, + glib::closure!(@strong drawer => move |_overlay: &gst::Element, + sample: &gst::Sample| { + use std::f64::consts::PI; + + let drawer = drawer.lock().unwrap(); + + let buffer = sample.buffer().unwrap(); + let timestamp = buffer.pts().unwrap(); + + let info = drawer.info.as_ref().unwrap(); + let text_layout = &drawer.text_layout; + let bitmap = drawer.bitmap.as_ref().unwrap(); + let render_target = drawer.render_target.as_ref().unwrap(); + + let global_angle = 360. * (timestamp % (10 * gst::ClockTime::SECOND)).nseconds() as f64 + / (10.0 * gst::ClockTime::SECOND.nseconds() as f64); + let center_x = (info.width() / 2) as f32; + let center_y = (info.height() / 2) as f32; + let top_margin = (info.height() / 20) as f32; + + unsafe { + // Begin drawing + render_target.BeginDraw(); + + // Clear background + render_target.Clear(Some(&D2D1_COLOR_F { + r: 0f32, + g: 0f32, + b: 0f32, + a: 0f32, + })); + + // This loop will render 10 times the string "GStreamer" in a circle + for i in 0..10 { + let angle = (360. * f64::from(i)) / 10.0; + let red = ((1.0 + f64::cos((angle - 60.0) * PI / 180.0)) / 2.0) as f32; + let text_brush = render_target + .CreateSolidColorBrush( + &D2D1_COLOR_F { + r: red, + g: 0f32, + b: 1f32 - red, + a: 1f32, + }, + None, + ) + .unwrap(); + + let angle = (angle + global_angle) as f32; + let matrix = Matrix3x2::rotation(angle, center_x, center_y); + render_target.SetTransform(&matrix); + render_target.DrawTextLayout( + D2D_POINT_2F { x: 0f32, y: top_margin }, + text_layout, + &text_brush, + D2D1_DRAW_TEXT_OPTIONS_NONE, + ); + } + + // EndDraw may not be successful for some reasons. + // Ignores any error in this example + let _ = render_target.EndDraw(None, None); + + // Make sure all operations is completed before copying + // bitmap to buffer + let _ = render_target.Flush(None::<*mut u64>, None::<*mut u64>); + } + + let mut buffer = gst::Buffer::with_size((info.width() * info.height() * 4) as usize).unwrap(); + { + let buffer_mut = buffer.get_mut().unwrap(); + let mut map = buffer_mut.map_writable().unwrap(); + let dst = map.as_mut_slice_of::().unwrap(); + + unsafe { + // Bitmap size is equal to the background image size. + // Copy entire memory + bitmap.CopyPixels(std::ptr::null(), info.width() * 4, dst).unwrap(); + } + } + + gst_video::VideoMeta::add_full( + buffer.get_mut().unwrap(), + gst_video::VideoFrameFlags::empty(), + gst_video::VideoFormat::Bgra, + info.width(), + info.height(), + &[0], + &[(info.width() * 4) as i32], + ) + .unwrap(); + + // Turn the buffer into a VideoOverlayRectangle, then place + // that into a VideoOverlayComposition and return it. + // + // A VideoOverlayComposition can take a Vec of such rectangles + // spaced around the video frame, but we're just outputting 1 + // here + let rect = gst_video::VideoOverlayRectangle::new_raw( + &buffer, + 0, + 0, + info.width(), + info.height(), + gst_video::VideoOverlayFormatFlags::PREMULTIPLIED_ALPHA, + ); + + gst_video::VideoOverlayComposition::new(Some(&rect)) + .unwrap() + }), + ); + + // Add a signal handler to the overlay's "caps-changed" signal. This could e.g. + // be called when the sink that we render to does not support resizing the image + // itself - but the user just changed the window-size. The element after the overlay + // will then change its caps and we use the notification about this change to + // resize our canvas's size. + // Another possibility for when this might happen is, when our video is a network + // stream that dynamically changes resolution when enough bandwith is available. + overlay.connect_closure( + "caps-changed", + false, + glib::closure!(move |_overlay: &gst::Element, + caps: &gst::Caps, + _width: u32, + _height: u32| { + let mut drawer = drawer.lock().unwrap(); + let info = gst_video::VideoInfo::from_caps(caps).unwrap(); + + unsafe { + // Update text layout to be identical to new video resolution + drawer.text_layout.SetMaxWidth(info.width() as f32).unwrap(); + drawer + .text_layout + .SetMaxHeight(info.height() as f32) + .unwrap(); + + // Create new WIC bitmap with PBGRA format (pre-multiplied BGRA) + let bitmap = drawer + .wic_factory + .CreateBitmap( + info.width(), + info.height(), + &GUID_WICPixelFormat32bppPBGRA, + WICBitmapCacheOnDemand, + ) + .unwrap(); + + let render_target = drawer + .d2d_factory + .CreateWicBitmapRenderTarget( + &bitmap, + &D2D1_RENDER_TARGET_PROPERTIES { + r#type: D2D1_RENDER_TARGET_TYPE_DEFAULT, + pixelFormat: D2D1_PIXEL_FORMAT { + format: DXGI_FORMAT_B8G8R8A8_UNORM, + alphaMode: D2D1_ALPHA_MODE_PREMULTIPLIED, + }, + // zero means default DPI + dpiX: 0f32, + dpiY: 0f32, + usage: D2D1_RENDER_TARGET_USAGE_NONE, + minLevel: D2D1_FEATURE_LEVEL_DEFAULT, + }, + ) + .unwrap(); + + drawer.render_target = Some(render_target); + drawer.bitmap = Some(bitmap); + } + drawer.info = Some(info); + }), + ); + + Ok(pipeline) +} + +fn main_loop(pipeline: gst::Pipeline) -> Result<(), Error> { + pipeline.set_state(gst::State::Playing)?; + + let bus = pipeline + .bus() + .expect("Pipeline without bus. Shouldn't happen!"); + + for msg in bus.iter_timed(gst::ClockTime::NONE) { + use gst::MessageView; + + match msg.view() { + MessageView::Eos(..) => break, + MessageView::Error(err) => { + pipeline.set_state(gst::State::Null)?; + return Err(ErrorMessage { + src: msg + .src() + .map(|s| s.path_string()) + .unwrap_or_else(|| glib::GString::from("UNKNOWN")), + error: err.error(), + debug: err.debug(), + } + .into()); + } + _ => (), + } + } + + pipeline.set_state(gst::State::Null)?; + + Ok(()) +} + +fn main() { + // WIC requires COM initialization + unsafe { + CoInitializeEx(None, COINIT_MULTITHREADED).unwrap(); + } + + match create_pipeline().and_then(main_loop) { + Ok(r) => r, + Err(e) => eprintln!("Error! {}", e), + } + + unsafe { + CoUninitialize(); + } +}