examples: Add d3d11videosink example with Direct2D/DirectWrite interop

Demonstartes the use of d3d11videosink's present signal
introduced in
https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2923
with Direct2D/DirectWrite API interop using the windows crate.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-rs/-/merge_requests/1089>

examples/Cargo.toml

@@ -42,6 +42,13 @@ memmap2 = { version = "0.5", optional = true }
 memfd = { version = "0.6", optional = true }
 uds = { version = "0.2", optional = true }
 
+[target.'cfg(windows)'.dependencies]
+windows = { version = "0.43", features=["Win32_Graphics_Direct3D11",
+    "Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Dxgi",
+    "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Direct2D",
+    "Win32_Graphics_Direct2D_Common", "Win32_Graphics_DirectWrite",
+    "Foundation_Numerics"], optional = true }
+
 [target.'cfg(target_os = "macos")'.dependencies]
 cocoa = "0.24"
 
@@ -195,3 +202,7 @@ required-features = ["allocators"]
 [[bin]]
 name = "cairo_compositor"
 required-features = ["cairo-rs", "gst-video/v1_18"]
+
+[[bin]]
+name = "d3d11videosink"
+required-features = ["windows"]

examples/src/bin/d3d11videosink.rs

@@ -0,0 +1,362 @@
+// This example demonstrates the use of the d3d11videosink's "present"
+// signal and the use of Direct2D/DirectWrite APIs in Rust.
+//
+// Application can perform various hardware-acceleated 2D graphics operation
+// (e.g., like cairo can support) and text rendering via the Windows APIs.
+// In this example, 2D graphics operation and text rendering will happen
+// directly to the on the DXGI swapchain's backbuffer via Windows API in
+// strictly zero-copy manner
+
+use gst::glib;
+use gst::prelude::*;
+use std::collections::VecDeque;
+use std::sync::{Arc, Mutex};
+use std::time::SystemTime;
+use windows::{
+    core::*,
+    Win32::Graphics::{
+        Direct2D::Common::*, Direct2D::*, Direct3D11::*, DirectWrite::*, Dxgi::Common::*, Dxgi::*,
+    },
+};
+
+struct OverlayContext {
+    d2d_factory: ID2D1Factory,
+    dwrite_factory: IDWriteFactory,
+    text_format: IDWriteTextFormat,
+    texture_desc: D3D11_TEXTURE2D_DESC,
+    text_layout: Option<IDWriteTextLayout>,
+    timestamp_queue: VecDeque<SystemTime>,
+    avg_fps: f32,
+    display_fps: f32,
+    font_size: f32,
+}
+
+fn create_overlay_context() -> Arc<Mutex<OverlayContext>> {
+    // Lots of DirectX APIs are marked as unsafe but the below operations
+    // are not expected to be failed unless GPU hang or device remove condition
+    // happens
+    let d2d_factory = unsafe {
+        D2D1CreateFactory::<ID2D1Factory>(D2D1_FACTORY_TYPE_MULTI_THREADED, None).unwrap()
+    };
+    let dwrite_factory =
+        unsafe { DWriteCreateFactory::<IDWriteFactory>(DWRITE_FACTORY_TYPE_SHARED).unwrap() };
+
+    // Font size can be updated later
+    let text_format = unsafe {
+        dwrite_factory
+            .CreateTextFormat(
+                w!("Consolas"),
+                None,
+                DWRITE_FONT_WEIGHT_REGULAR,
+                DWRITE_FONT_STYLE_NORMAL,
+                DWRITE_FONT_STRETCH_NORMAL,
+                12f32,
+                w!("en-us"),
+            )
+            .unwrap()
+    };
+
+    Arc::new(Mutex::new(OverlayContext {
+        d2d_factory,
+        dwrite_factory,
+        text_format,
+        texture_desc: D3D11_TEXTURE2D_DESC::default(),
+        text_layout: None,
+        timestamp_queue: VecDeque::with_capacity(10),
+        avg_fps: 0f32,
+        display_fps: 0f32,
+        font_size: 12f32,
+    }))
+}
+
+fn main() -> Result<()> {
+    gst::init().unwrap();
+
+    let args: Vec<String> = std::env::args().collect();
+
+    if args.len() != 2 {
+        println!("URI must be specified");
+        return Ok(());
+    }
+
+    let main_loop = glib::MainLoop::new(None, false);
+
+    let overlay_context = create_overlay_context();
+    let overlay_context_weak = Arc::downgrade(&overlay_context);
+    // Needs BGRA or RGBA swapchain for D2D interop,
+    // and "present" signal must be explicitly enabled
+    let videosink = gst::ElementFactory::make("d3d11videosink")
+        .property("emit-present", true)
+        .property_from_str("display-format", "DXGI_FORMAT_B8G8R8A8_UNORM")
+        .build()
+        .unwrap();
+
+    // Listen "present" signal and draw overlay from the callback
+    // Required operations here:
+    // 1) Gets IDXGISurface and ID3D11Texture2D interface from
+    //    given ID3D11RenderTargetView COM object
+    //   - ID3D11Texture2D: To get texture resolution
+    //   - IDXGISurface: To create Direct2D render target
+    // 2) Creates or reuses IDWriteTextLayout interface
+    //   - This object represents text layout we want to draw on render target
+    // 3) Draw rectangle (overlay background) and text on render target
+    //
+    // NOTE: ID2D1Factory, IDWriteFactory, IDWriteTextFormat, and
+    // IDWriteTextLayout objects are device-independent. Which can be created
+    // earlier instead of creating them in the callback.
+    // But ID2D1RenderTarget is a device-dependent resource.
+    // The client should not hold the d2d render target object outside of
+    // this callback scope because the resource must be cleared before
+    // releasing/resizing DXGI swapchain.
+    videosink.connect_closure(
+        "present",
+        false,
+        glib::closure!(move |_sink: &gst::Element,
+                             _device: &gst::Object,
+                             rtv_raw: glib::Pointer| {
+            let overlay_context = overlay_context_weak.upgrade().unwrap();
+            let mut context = overlay_context.lock().unwrap();
+            let dwrite_factory = context.dwrite_factory.clone();
+            let d2d_factory = context.d2d_factory.clone();
+
+            // SAFETY: transmute() below is clearly unsafe operation here.
+            // Regarding the other part of the below block, all DirectX
+            // APIs are marked as unsafe, except for cast.
+            //
+            // In theory, all the Direct3D/Direct2D APIs could fail for
+            // some reasons (it's hardware!), but in pratice, it's very unexpected
+            // situation and any of failure below would mean we are doing
+            // something in wrong way or driver bug or so.
+            unsafe {
+                let rtv = ID3D11RenderTargetView::from_raw_borrowed(&rtv_raw);
+                let resource = {
+                    let mut resource = None;
+                    rtv.GetResource(&mut resource);
+                    resource.unwrap()
+                };
+
+                let texture = resource.cast::<ID3D11Texture2D>().unwrap();
+                let desc = {
+                    let mut desc = D3D11_TEXTURE2D_DESC::default();
+                    texture.GetDesc(&mut desc);
+                    desc
+                };
+
+                // Window size was updated, creates new text layout
+                let calculate_font_size = if desc != context.texture_desc {
+                    context.texture_desc = desc;
+                    context.text_layout = None;
+                    true
+                } else {
+                    false
+                };
+
+                // New fps, creates new layout
+                if context.avg_fps != context.display_fps {
+                    context.display_fps = context.avg_fps;
+                    context.text_layout = None;
+                }
+
+                if context.text_layout.is_none() {
+                    let overlay_string = format!("TextOverlay, Fps {:.1}", context.display_fps);
+                    let overlay_wstring = overlay_string.encode_utf16().collect::<Vec<_>>();
+                    let layout = dwrite_factory
+                        .CreateTextLayout(
+                            &overlay_wstring,
+                            &context.text_format,
+                            desc.Width as f32,
+                            desc.Height as f32 / 5f32,
+                        )
+                        .unwrap();
+
+                    // Adjust alignment
+                    layout
+                        .SetTextAlignment(DWRITE_TEXT_ALIGNMENT_CENTER)
+                        .unwrap();
+                    layout
+                        .SetParagraphAlignment(DWRITE_PARAGRAPH_ALIGNMENT_CENTER)
+                        .unwrap();
+
+                    // XXX: This is not an efficient approach.
+                    // The font size can be pre-calculated for a pre-defined
+                    // window size and string length
+                    let mut range = DWRITE_TEXT_RANGE {
+                        startPosition: 0u32,
+                        length: overlay_wstring.len() as u32,
+                    };
+
+                    if calculate_font_size {
+                        let mut font_size = 12f32;
+                        let mut was_decreased = false;
+
+                        loop {
+                            let metrics = layout.GetMetrics().unwrap();
+                            layout
+                                .GetFontSize2(0, &mut font_size, Some(&mut range))
+                                .unwrap();
+
+                            if metrics.widthIncludingTrailingWhitespace >= desc.Width as f32 {
+                                if font_size > 1f32 {
+                                    font_size -= 0.5f32;
+                                    was_decreased = true;
+                                    layout.SetFontSize(font_size, range).unwrap();
+                                    continue;
+                                }
+
+                                break;
+                            }
+
+                            if was_decreased {
+                                break;
+                            }
+
+                            if metrics.widthIncludingTrailingWhitespace < desc.Width as f32 {
+                                if metrics.widthIncludingTrailingWhitespace
+                                    >= desc.Width as f32 * 0.7f32
+                                {
+                                    break;
+                                }
+
+                                font_size += 0.5f32;
+                                layout.SetFontSize(font_size, range).unwrap();
+                            }
+                        }
+
+                        context.font_size = font_size;
+                    } else {
+                        layout.SetFontSize(context.font_size, range).unwrap();
+                    }
+
+                    context.text_layout = Some(layout);
+                };
+
+                let dxgi_surf = resource.cast::<IDXGISurface>().unwrap();
+                let render_target = d2d_factory
+                    .CreateDxgiSurfaceRenderTarget(
+                        &dxgi_surf,
+                        &D2D1_RENDER_TARGET_PROPERTIES {
+                            r#type: D2D1_RENDER_TARGET_TYPE_DEFAULT,
+                            pixelFormat: D2D1_PIXEL_FORMAT {
+                                format: DXGI_FORMAT_B8G8R8A8_UNORM,
+                                alphaMode: D2D1_ALPHA_MODE_PREMULTIPLIED,
+                            },
+                            // zero means default DPI
+                            dpiX: 0f32,
+                            dpiY: 0f32,
+                            usage: D2D1_RENDER_TARGET_USAGE_NONE,
+                            minLevel: D2D1_FEATURE_LEVEL_DEFAULT,
+                        },
+                    )
+                    .unwrap();
+                let text_brush = render_target
+                    .CreateSolidColorBrush(
+                        &D2D1_COLOR_F {
+                            r: 0f32,
+                            g: 0f32,
+                            b: 0f32,
+                            a: 1f32,
+                        },
+                        None,
+                    )
+                    .unwrap();
+                let overlay_brush = render_target
+                    .CreateSolidColorBrush(
+                        &D2D1_COLOR_F {
+                            r: 0f32,
+                            g: 0.5f32,
+                            b: 0.5f32,
+                            a: 0.3f32,
+                        },
+                        None,
+                    )
+                    .unwrap();
+
+                render_target.BeginDraw();
+                // Draws overlay background. It will blend overlay's background
+                // color with already rendred video frame
+                render_target.FillRectangle(
+                    &D2D_RECT_F {
+                        left: 0f32,
+                        top: 0f32,
+                        right: desc.Width as f32,
+                        bottom: desc.Height as f32 / 5f32,
+                    },
+                    &overlay_brush,
+                );
+
+                // Then, renders text
+                render_target.DrawTextLayout(
+                    D2D_POINT_2F { x: 0f32, y: 0f32 },
+                    context.text_layout.as_ref(),
+                    &text_brush,
+                    D2D1_DRAW_TEXT_OPTIONS_NONE,
+                );
+
+                // EndDraw may not be successful for some reasons.
+                // Ignores any error in this example
+                let _ = render_target.EndDraw(None, None);
+            }
+        }),
+    );
+
+    // Add pad probe to calculate framerate
+    let sinkpad = videosink.static_pad("sink").unwrap();
+    let overlay_context_weak = Arc::downgrade(&overlay_context);
+    sinkpad.add_probe(gst::PadProbeType::BUFFER, move |_, probe_info| {
+        if let Some(gst::PadProbeData::Buffer(_)) = probe_info.data {
+            let overlay_context = overlay_context_weak.upgrade().unwrap();
+            let mut context = overlay_context.lock().unwrap();
+            context.timestamp_queue.push_back(SystemTime::now());
+            // Updates framerate per 10 frames
+            if context.timestamp_queue.len() >= 10 {
+                let now = context.timestamp_queue.back().unwrap();
+                let front = context.timestamp_queue.front().unwrap();
+                let duration = now.duration_since(*front).unwrap().as_millis() as f32;
+                context.avg_fps = 1000f32 * (context.timestamp_queue.len() - 1) as f32 / duration;
+                context.timestamp_queue.clear();
+            }
+        }
+        gst::PadProbeReturn::Ok
+    });
+
+    let playbin = gst::ElementFactory::make("playbin")
+        .property("uri", &args[1])
+        .property("video-sink", &videosink)
+        .build()
+        .unwrap();
+
+    let main_loop_clone = main_loop.clone();
+    let bus = playbin.bus().unwrap();
+    bus.add_watch(move |_, msg| {
+        use gst::MessageView;
+
+        let main_loop = &main_loop_clone;
+        match msg.view() {
+            MessageView::Eos(..) => {
+                println!("received eos");
+                main_loop.quit()
+            }
+            MessageView::Error(err) => {
+                println!(
+                    "Error from {:?}: {} ({:?})",
+                    err.src().map(|s| s.path_string()),
+                    err.error(),
+                    err.debug()
+                );
+                main_loop.quit();
+            }
+            _ => (),
+        };
+
+        glib::Continue(true)
+    })
+    .unwrap();
+
+    playbin.set_state(gst::State::Playing).unwrap();
+
+    main_loop.run();
+
+    playbin.set_state(gst::State::Null).unwrap();
+
+    Ok(())
+}