mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer-rs.git
synced 2024-12-28 11:00:34 +00:00
examples: Add d3d11videosink example with Direct2D/DirectWrite interop
Demonstartes the use of d3d11videosink's present signal introduced in https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2923 with Direct2D/DirectWrite API interop using the windows crate. Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-rs/-/merge_requests/1089>
This commit is contained in:
Seungha Yang
Jordan Petridіs
parent
e6e5d25e48
commit
b4d3bf297e
2 changed files with 373 additions and 0 deletions
|
|
@ -42,6 +42,13 @@ memmap2 = { version = "0.5", optional = true }
|
|||
memfd = { version = "0.6", optional = true }
|
||||
uds = { version = "0.2", optional = true }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows = { version = "0.43", features=["Win32_Graphics_Direct3D11",
|
||||
"Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Dxgi",
|
||||
"Win32_Graphics_Dxgi_Common", "Win32_Graphics_Direct2D",
|
||||
"Win32_Graphics_Direct2D_Common", "Win32_Graphics_DirectWrite",
|
||||
"Foundation_Numerics"], optional = true }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
cocoa = "0.24"
|
||||
|
||||
|
|
@ -195,3 +202,7 @@ required-features = ["allocators"]
|
|||
[[bin]]
|
||||
name = "cairo_compositor"
|
||||
required-features = ["cairo-rs", "gst-video/v1_18"]
|
||||
|
||||
[[bin]]
|
||||
name = "d3d11videosink"
|
||||
required-features = ["windows"]
|
||||
|
|
|
|||
362
examples/src/bin/d3d11videosink.rs
Normal file
362
examples/src/bin/d3d11videosink.rs
Normal file
|
|
@ -0,0 +1,362 @@
|
|||
// This example demonstrates the use of the d3d11videosink's "present"
|
||||
// signal and the use of Direct2D/DirectWrite APIs in Rust.
|
||||
//
|
||||
// Application can perform various hardware-acceleated 2D graphics operation
|
||||
// (e.g., like cairo can support) and text rendering via the Windows APIs.
|
||||
// In this example, 2D graphics operation and text rendering will happen
|
||||
// directly to the on the DXGI swapchain's backbuffer via Windows API in
|
||||
// strictly zero-copy manner
|
||||
|
||||
use gst::glib;
|
||||
use gst::prelude::*;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::SystemTime;
|
||||
use windows::{
|
||||
core::*,
|
||||
Win32::Graphics::{
|
||||
Direct2D::Common::*, Direct2D::*, Direct3D11::*, DirectWrite::*, Dxgi::Common::*, Dxgi::*,
|
||||
},
|
||||
};
|
||||
|
||||
struct OverlayContext {
|
||||
d2d_factory: ID2D1Factory,
|
||||
dwrite_factory: IDWriteFactory,
|
||||
text_format: IDWriteTextFormat,
|
||||
texture_desc: D3D11_TEXTURE2D_DESC,
|
||||
text_layout: Option<IDWriteTextLayout>,
|
||||
timestamp_queue: VecDeque<SystemTime>,
|
||||
avg_fps: f32,
|
||||
display_fps: f32,
|
||||
font_size: f32,
|
||||
}
|
||||
|
||||
fn create_overlay_context() -> Arc<Mutex<OverlayContext>> {
|
||||
// Lots of DirectX APIs are marked as unsafe but the below operations
|
||||
// are not expected to be failed unless GPU hang or device remove condition
|
||||
// happens
|
||||
let d2d_factory = unsafe {
|
||||
D2D1CreateFactory::<ID2D1Factory>(D2D1_FACTORY_TYPE_MULTI_THREADED, None).unwrap()
|
||||
};
|
||||
let dwrite_factory =
|
||||
unsafe { DWriteCreateFactory::<IDWriteFactory>(DWRITE_FACTORY_TYPE_SHARED).unwrap() };
|
||||
|
||||
// Font size can be updated later
|
||||
let text_format = unsafe {
|
||||
dwrite_factory
|
||||
.CreateTextFormat(
|
||||
w!("Consolas"),
|
||||
None,
|
||||
DWRITE_FONT_WEIGHT_REGULAR,
|
||||
DWRITE_FONT_STYLE_NORMAL,
|
||||
DWRITE_FONT_STRETCH_NORMAL,
|
||||
12f32,
|
||||
w!("en-us"),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
Arc::new(Mutex::new(OverlayContext {
|
||||
d2d_factory,
|
||||
dwrite_factory,
|
||||
text_format,
|
||||
texture_desc: D3D11_TEXTURE2D_DESC::default(),
|
||||
text_layout: None,
|
||||
timestamp_queue: VecDeque::with_capacity(10),
|
||||
avg_fps: 0f32,
|
||||
display_fps: 0f32,
|
||||
font_size: 12f32,
|
||||
}))
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
gst::init().unwrap();
|
||||
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
|
||||
if args.len() != 2 {
|
||||
println!("URI must be specified");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let main_loop = glib::MainLoop::new(None, false);
|
||||
|
||||
let overlay_context = create_overlay_context();
|
||||
let overlay_context_weak = Arc::downgrade(&overlay_context);
|
||||
// Needs BGRA or RGBA swapchain for D2D interop,
|
||||
// and "present" signal must be explicitly enabled
|
||||
let videosink = gst::ElementFactory::make("d3d11videosink")
|
||||
.property("emit-present", true)
|
||||
.property_from_str("display-format", "DXGI_FORMAT_B8G8R8A8_UNORM")
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Listen "present" signal and draw overlay from the callback
|
||||
// Required operations here:
|
||||
// 1) Gets IDXGISurface and ID3D11Texture2D interface from
|
||||
// given ID3D11RenderTargetView COM object
|
||||
// - ID3D11Texture2D: To get texture resolution
|
||||
// - IDXGISurface: To create Direct2D render target
|
||||
// 2) Creates or reuses IDWriteTextLayout interface
|
||||
// - This object represents text layout we want to draw on render target
|
||||
// 3) Draw rectangle (overlay background) and text on render target
|
||||
//
|
||||
// NOTE: ID2D1Factory, IDWriteFactory, IDWriteTextFormat, and
|
||||
// IDWriteTextLayout objects are device-independent. Which can be created
|
||||
// earlier instead of creating them in the callback.
|
||||
// But ID2D1RenderTarget is a device-dependent resource.
|
||||
// The client should not hold the d2d render target object outside of
|
||||
// this callback scope because the resource must be cleared before
|
||||
// releasing/resizing DXGI swapchain.
|
||||
videosink.connect_closure(
|
||||
"present",
|
||||
false,
|
||||
glib::closure!(move |_sink: &gst::Element,
|
||||
_device: &gst::Object,
|
||||
rtv_raw: glib::Pointer| {
|
||||
let overlay_context = overlay_context_weak.upgrade().unwrap();
|
||||
let mut context = overlay_context.lock().unwrap();
|
||||
let dwrite_factory = context.dwrite_factory.clone();
|
||||
let d2d_factory = context.d2d_factory.clone();
|
||||
|
||||
// SAFETY: transmute() below is clearly unsafe operation here.
|
||||
// Regarding the other part of the below block, all DirectX
|
||||
// APIs are marked as unsafe, except for cast.
|
||||
//
|
||||
// In theory, all the Direct3D/Direct2D APIs could fail for
|
||||
// some reasons (it's hardware!), but in pratice, it's very unexpected
|
||||
// situation and any of failure below would mean we are doing
|
||||
// something in wrong way or driver bug or so.
|
||||
unsafe {
|
||||
let rtv = ID3D11RenderTargetView::from_raw_borrowed(&rtv_raw);
|
||||
let resource = {
|
||||
let mut resource = None;
|
||||
rtv.GetResource(&mut resource);
|
||||
resource.unwrap()
|
||||
};
|
||||
|
||||
let texture = resource.cast::<ID3D11Texture2D>().unwrap();
|
||||
let desc = {
|
||||
let mut desc = D3D11_TEXTURE2D_DESC::default();
|
||||
texture.GetDesc(&mut desc);
|
||||
desc
|
||||
};
|
||||
|
||||
// Window size was updated, creates new text layout
|
||||
let calculate_font_size = if desc != context.texture_desc {
|
||||
context.texture_desc = desc;
|
||||
context.text_layout = None;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
// New fps, creates new layout
|
||||
if context.avg_fps != context.display_fps {
|
||||
context.display_fps = context.avg_fps;
|
||||
context.text_layout = None;
|
||||
}
|
||||
|
||||
if context.text_layout.is_none() {
|
||||
let overlay_string = format!("TextOverlay, Fps {:.1}", context.display_fps);
|
||||
let overlay_wstring = overlay_string.encode_utf16().collect::<Vec<_>>();
|
||||
let layout = dwrite_factory
|
||||
.CreateTextLayout(
|
||||
&overlay_wstring,
|
||||
&context.text_format,
|
||||
desc.Width as f32,
|
||||
desc.Height as f32 / 5f32,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Adjust alignment
|
||||
layout
|
||||
.SetTextAlignment(DWRITE_TEXT_ALIGNMENT_CENTER)
|
||||
.unwrap();
|
||||
layout
|
||||
.SetParagraphAlignment(DWRITE_PARAGRAPH_ALIGNMENT_CENTER)
|
||||
.unwrap();
|
||||
|
||||
// XXX: This is not an efficient approach.
|
||||
// The font size can be pre-calculated for a pre-defined
|
||||
// window size and string length
|
||||
let mut range = DWRITE_TEXT_RANGE {
|
||||
startPosition: 0u32,
|
||||
length: overlay_wstring.len() as u32,
|
||||
};
|
||||
|
||||
if calculate_font_size {
|
||||
let mut font_size = 12f32;
|
||||
let mut was_decreased = false;
|
||||
|
||||
loop {
|
||||
let metrics = layout.GetMetrics().unwrap();
|
||||
layout
|
||||
.GetFontSize2(0, &mut font_size, Some(&mut range))
|
||||
.unwrap();
|
||||
|
||||
if metrics.widthIncludingTrailingWhitespace >= desc.Width as f32 {
|
||||
if font_size > 1f32 {
|
||||
font_size -= 0.5f32;
|
||||
was_decreased = true;
|
||||
layout.SetFontSize(font_size, range).unwrap();
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if was_decreased {
|
||||
break;
|
||||
}
|
||||
|
||||
if metrics.widthIncludingTrailingWhitespace < desc.Width as f32 {
|
||||
if metrics.widthIncludingTrailingWhitespace
|
||||
>= desc.Width as f32 * 0.7f32
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
font_size += 0.5f32;
|
||||
layout.SetFontSize(font_size, range).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
context.font_size = font_size;
|
||||
} else {
|
||||
layout.SetFontSize(context.font_size, range).unwrap();
|
||||
}
|
||||
|
||||
context.text_layout = Some(layout);
|
||||
};
|
||||
|
||||
let dxgi_surf = resource.cast::<IDXGISurface>().unwrap();
|
||||
let render_target = d2d_factory
|
||||
.CreateDxgiSurfaceRenderTarget(
|
||||
&dxgi_surf,
|
||||
&D2D1_RENDER_TARGET_PROPERTIES {
|
||||
r#type: D2D1_RENDER_TARGET_TYPE_DEFAULT,
|
||||
pixelFormat: D2D1_PIXEL_FORMAT {
|
||||
format: DXGI_FORMAT_B8G8R8A8_UNORM,
|
||||
alphaMode: D2D1_ALPHA_MODE_PREMULTIPLIED,
|
||||
},
|
||||
// zero means default DPI
|
||||
dpiX: 0f32,
|
||||
dpiY: 0f32,
|
||||
usage: D2D1_RENDER_TARGET_USAGE_NONE,
|
||||
minLevel: D2D1_FEATURE_LEVEL_DEFAULT,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let text_brush = render_target
|
||||
.CreateSolidColorBrush(
|
||||
&D2D1_COLOR_F {
|
||||
r: 0f32,
|
||||
g: 0f32,
|
||||
b: 0f32,
|
||||
a: 1f32,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let overlay_brush = render_target
|
||||
.CreateSolidColorBrush(
|
||||
&D2D1_COLOR_F {
|
||||
r: 0f32,
|
||||
g: 0.5f32,
|
||||
b: 0.5f32,
|
||||
a: 0.3f32,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
render_target.BeginDraw();
|
||||
// Draws overlay background. It will blend overlay's background
|
||||
// color with already rendred video frame
|
||||
render_target.FillRectangle(
|
||||
&D2D_RECT_F {
|
||||
left: 0f32,
|
||||
top: 0f32,
|
||||
right: desc.Width as f32,
|
||||
bottom: desc.Height as f32 / 5f32,
|
||||
},
|
||||
&overlay_brush,
|
||||
);
|
||||
|
||||
// Then, renders text
|
||||
render_target.DrawTextLayout(
|
||||
D2D_POINT_2F { x: 0f32, y: 0f32 },
|
||||
context.text_layout.as_ref(),
|
||||
&text_brush,
|
||||
D2D1_DRAW_TEXT_OPTIONS_NONE,
|
||||
);
|
||||
|
||||
// EndDraw may not be successful for some reasons.
|
||||
// Ignores any error in this example
|
||||
let _ = render_target.EndDraw(None, None);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
// Add pad probe to calculate framerate
|
||||
let sinkpad = videosink.static_pad("sink").unwrap();
|
||||
let overlay_context_weak = Arc::downgrade(&overlay_context);
|
||||
sinkpad.add_probe(gst::PadProbeType::BUFFER, move |_, probe_info| {
|
||||
if let Some(gst::PadProbeData::Buffer(_)) = probe_info.data {
|
||||
let overlay_context = overlay_context_weak.upgrade().unwrap();
|
||||
let mut context = overlay_context.lock().unwrap();
|
||||
context.timestamp_queue.push_back(SystemTime::now());
|
||||
// Updates framerate per 10 frames
|
||||
if context.timestamp_queue.len() >= 10 {
|
||||
let now = context.timestamp_queue.back().unwrap();
|
||||
let front = context.timestamp_queue.front().unwrap();
|
||||
let duration = now.duration_since(*front).unwrap().as_millis() as f32;
|
||||
context.avg_fps = 1000f32 * (context.timestamp_queue.len() - 1) as f32 / duration;
|
||||
context.timestamp_queue.clear();
|
||||
}
|
||||
}
|
||||
gst::PadProbeReturn::Ok
|
||||
});
|
||||
|
||||
let playbin = gst::ElementFactory::make("playbin")
|
||||
.property("uri", &args[1])
|
||||
.property("video-sink", &videosink)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let main_loop_clone = main_loop.clone();
|
||||
let bus = playbin.bus().unwrap();
|
||||
bus.add_watch(move |_, msg| {
|
||||
use gst::MessageView;
|
||||
|
||||
let main_loop = &main_loop_clone;
|
||||
match msg.view() {
|
||||
MessageView::Eos(..) => {
|
||||
println!("received eos");
|
||||
main_loop.quit()
|
||||
}
|
||||
MessageView::Error(err) => {
|
||||
println!(
|
||||
"Error from {:?}: {} ({:?})",
|
||||
err.src().map(|s| s.path_string()),
|
||||
err.error(),
|
||||
err.debug()
|
||||
);
|
||||
main_loop.quit();
|
||||
}
|
||||
_ => (),
|
||||
};
|
||||
|
||||
glib::Continue(true)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
playbin.set_state(gst::State::Playing).unwrap();
|
||||
|
||||
main_loop.run();
|
||||
|
||||
playbin.set_state(gst::State::Null).unwrap();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Reference in a new issue