From 15434ce51d7ffa5ee7b744dc0c50e2ae47d66af1 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Wed, 10 Apr 2024 01:26:38 +0900 Subject: [PATCH] dwrite: D3D12 integration Adding d3d12 backend text renderer/blender by using d3d11on12 interop. And subclassing renderer object per backend (i.e., d3d11, d3d12, and bitmap) Part-of: --- .../sys/dwrite/gstdwritebaseoverlay.cpp | 24 +- .../sys/dwrite/gstdwriteoverlayobject.cpp | 1076 ++--------------- .../sys/dwrite/gstdwriteoverlayobject.h | 5 +- .../sys/dwrite/gstdwriterender.cpp | 124 ++ .../sys/dwrite/gstdwriterender.h | 87 ++ .../sys/dwrite/gstdwriterender_bitmap.cpp | 268 ++++ .../sys/dwrite/gstdwriterender_bitmap.h | 34 + .../sys/dwrite/gstdwriterender_d3d11.cpp | 745 ++++++++++++ .../sys/dwrite/gstdwriterender_d3d11.h | 35 + .../sys/dwrite/gstdwriterender_d3d12.cpp | 898 ++++++++++++++ .../sys/dwrite/gstdwriterender_d3d12.h | 36 + .../gst-plugins-bad/sys/dwrite/meson.build | 13 +- 12 files changed, 2356 insertions(+), 989 deletions(-) create mode 100644 subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender.cpp create mode 100644 subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender.h create mode 100644 subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_bitmap.cpp create mode 100644 subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_bitmap.h create mode 100644 subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d11.cpp create mode 100644 subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d11.h create mode 100644 subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d12.cpp create mode 100644 subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d12.h diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp index 9f93b422e8..7e84de2d5f 100644 --- a/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp @@ -119,7 +119,7 @@ struct _GstDWriteBaseOverlayPrivate std::wstring prev_text; std::wstring cur_text; - GstDWriteBlendMode blend_mode = GstDWriteBlendMode::NOT_SUPPORTED; + gboolean force_passthrough = FALSE; /* properties */ gboolean visible = DEFAULT_VISIBLE; @@ -700,11 +700,14 @@ gst_dwrite_base_overlay_set_caps (GstBaseTransform * trans, GstCaps * incaps, gst_dwrite_base_overlay_clear_layout (self); + priv->force_passthrough = FALSE; if (!gst_dwrite_overlay_object_set_caps (priv->overlay, - GST_ELEMENT (self), incaps, outcaps, &self->info, - &priv->blend_mode)) { - GST_ERROR_OBJECT (self, "Set caps failed"); - return FALSE; + GST_ELEMENT (self), incaps, outcaps, &self->info)) { + GST_WARNING_OBJECT (self, "Set caps failed"); + priv->force_passthrough = TRUE; + gst_base_transform_set_passthrough (trans, TRUE); + } else { + gst_base_transform_set_passthrough (trans, FALSE); } priv->prop_lock.lock (); @@ -712,11 +715,6 @@ gst_dwrite_base_overlay_set_caps (GstBaseTransform * trans, GstCaps * incaps, priv->layout = nullptr; priv->prop_lock.unlock (); - if (priv->blend_mode == GstDWriteBlendMode::NOT_SUPPORTED) - gst_base_transform_set_passthrough (trans, TRUE); - else - gst_base_transform_set_passthrough (trans, FALSE); - return TRUE; } @@ -728,7 +726,7 @@ gst_dwrite_base_overlay_before_transform (GstBaseTransform * trans, GstDWriteBaseOverlayPrivate *priv = self->priv; if (gst_dwrite_overlay_object_update_device (priv->overlay, buf)) - gst_base_transform_reconfigure (trans); + gst_base_transform_reconfigure_src (trans); } static GstFlowReturn @@ -739,9 +737,7 @@ gst_dwrite_base_overlay_prepare_output_buffer (GstBaseTransform * trans, GstDWriteBaseOverlayPrivate *priv = self->priv; GstDWriteBaseOverlayClass *klass = GST_DWRITE_BASE_OVERLAY_GET_CLASS (self); - if (priv->blend_mode == GstDWriteBlendMode::NOT_SUPPORTED) { - GST_TRACE_OBJECT (self, "Force passthrough"); - + if (priv->force_passthrough) { return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (trans, inbuf, outbuf); diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.cpp index 8159ef6b41..04e033fdeb 100644 --- a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.cpp +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.cpp @@ -23,13 +23,17 @@ #include #include "gstdwriteoverlayobject.h" -#include "gstdwritebitmappool.h" #include "gstdwrite-renderer.h" -#include "gstdwrite-effect.h" +#include "gstdwriterender_bitmap.h" +#include "gstdwriterender_d3d11.h" #include #include -GST_DEBUG_CATEGORY_STATIC (dwrite_overlay_object_debug); +#ifdef HAVE_GST_D3D12 +#include "gstdwriterender_d3d12.h" +#endif + +GST_DEBUG_CATEGORY (dwrite_overlay_object_debug); #define GST_CAT_DEFAULT dwrite_overlay_object_debug /* *INDENT-OFF* */ @@ -37,72 +41,42 @@ using namespace Microsoft::WRL; struct GstDWriteOverlayObjectPrivate { - GstDWriteOverlayObjectPrivate () - { - gst_video_info_init (&info); - gst_video_info_init (&bgra_info); - gst_video_info_init (&layout_info); - } - ~GstDWriteOverlayObjectPrivate () { ClearResource (true); - gst_clear_caps (&outcaps); gst_clear_object (&device); +#ifdef HAVE_GST_D3D12 + gst_clear_object (&device12); +#endif } void ClearResource (bool hard) { - if (hard) - blend_mode = GstDWriteBlendMode::NOT_SUPPORTED; - g_clear_pointer (&overlay_rect, gst_video_overlay_rectangle_unref); gst_clear_buffer (&layout_buf); layout = nullptr; - if (layout_pool) { - gst_buffer_pool_set_active (layout_pool, FALSE); - gst_clear_object (&layout_pool); - } - - if (blend_pool) { - gst_buffer_pool_set_active (blend_pool, FALSE); - gst_clear_object (&blend_pool); - } - - gst_clear_object (&pre_conv); - gst_clear_object (&blend_conv); - gst_clear_object (&post_conv); + gst_clear_object (&render); } GstVideoInfo info; - GstVideoInfo bgra_info; GstVideoInfo layout_info; GstD3D11Device *device = nullptr; - GstCaps *outcaps = nullptr; +#ifdef HAVE_GST_D3D12 + GstD3D12Device *device12 = nullptr; +#endif ComPtr d2d_factory; ComPtr dwrite_factory; ComPtr layout; - ComPtr renderer; - GstDWriteBlendMode blend_mode = GstDWriteBlendMode::NOT_SUPPORTED; + GstDWriteRender *render = nullptr; - GstBufferPool *layout_pool = nullptr; - GstBufferPool *blend_pool = nullptr; GstBuffer *layout_buf = nullptr; - /* Convert input texture to BGRA */ - GstD3D11Converter *pre_conv = nullptr; - /* Blend converted BGRA texture with rendered text texture */ - GstD3D11Converter *blend_conv = nullptr; - /* Convert blended texture to original format */ - GstD3D11Converter *post_conv = nullptr; GstVideoOverlayRectangle *overlay_rect = nullptr; - gboolean is_d3d11 = FALSE; gboolean attach_meta = FALSE; - gboolean use_bitmap = FALSE; std::recursive_mutex ctx_lock; }; @@ -148,329 +122,6 @@ gst_dwrite_overlay_object_finalize (GObject * object) G_OBJECT_CLASS (parent_class)->finalize (object); } -static void -gst_dwrite_overlay_object_decide_blend_mode (GstDWriteOverlayObject * self) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - - if (priv->attach_meta) { - if (priv->is_d3d11) - priv->blend_mode = GstDWriteBlendMode::ATTACH_TEXTURE; - else - priv->blend_mode = GstDWriteBlendMode::ATTACH_BITMAP; - return; - } - - if (!priv->is_d3d11) { - priv->blend_mode = GstDWriteBlendMode::SW_BLEND; - return; - } - - /* Decide best blend mode to use based on format */ - switch (GST_VIDEO_INFO_FORMAT (&priv->info)) { - case GST_VIDEO_FORMAT_BGRA: - case GST_VIDEO_FORMAT_RGBA: - case GST_VIDEO_FORMAT_BGRx: - case GST_VIDEO_FORMAT_RGBx: - case GST_VIDEO_FORMAT_VUYA: - case GST_VIDEO_FORMAT_RGBA64_LE: - case GST_VIDEO_FORMAT_RGB10A2_LE: - /* Alpha aware formats */ - priv->blend_mode = GstDWriteBlendMode::BLEND; - break; - case GST_VIDEO_FORMAT_NV12: - case GST_VIDEO_FORMAT_NV21: - case GST_VIDEO_FORMAT_I420: - case GST_VIDEO_FORMAT_YV12: - case GST_VIDEO_FORMAT_Y42B: - case GST_VIDEO_FORMAT_Y444: - case GST_VIDEO_FORMAT_GRAY8: - case GST_VIDEO_FORMAT_AYUV: - case GST_VIDEO_FORMAT_RGBP: - case GST_VIDEO_FORMAT_BGRP: - case GST_VIDEO_FORMAT_GBR: - case GST_VIDEO_FORMAT_GBRA: - /* 8bits formats */ - priv->blend_mode = GstDWriteBlendMode::CONVERT; - break; - case GST_VIDEO_FORMAT_P010_10LE: - case GST_VIDEO_FORMAT_P012_LE: - case GST_VIDEO_FORMAT_P016_LE: - case GST_VIDEO_FORMAT_I420_10LE: - case GST_VIDEO_FORMAT_I420_12LE: - case GST_VIDEO_FORMAT_I422_10LE: - case GST_VIDEO_FORMAT_I422_12LE: - case GST_VIDEO_FORMAT_Y444_10LE: - case GST_VIDEO_FORMAT_Y444_12LE: - case GST_VIDEO_FORMAT_Y444_16LE: - case GST_VIDEO_FORMAT_GRAY16_LE: - case GST_VIDEO_FORMAT_AYUV64: - case GST_VIDEO_FORMAT_GBR_10LE: - case GST_VIDEO_FORMAT_GBR_12LE: - case GST_VIDEO_FORMAT_GBRA_10LE: - case GST_VIDEO_FORMAT_GBRA_12LE: - /* high bitdept formats */ - priv->blend_mode = GstDWriteBlendMode::CONVERT_64; - break; - default: - /* d3d11 blending is not supported, fallback to software blending */ - priv->blend_mode = GstDWriteBlendMode::SW_BLEND; - break; - } -} - -static gboolean -is_subsampled_yuv (const GstVideoInfo * info) -{ - if (!GST_VIDEO_INFO_IS_YUV (info)) - return FALSE; - - for (guint i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) { - if (info->finfo->w_sub[i] != 0 || info->finfo->h_sub[i] != 0) - return TRUE; - } - - return FALSE; -} - -static GstD3D11Converter * -gst_dwrite_overlay_object_create_converter (GstDWriteOverlayObject * self, - const GstVideoInfo * in_info, const GstVideoInfo * out_info, - gboolean is_blend) -{ - GstD3D11Converter *ret; - GstDWriteOverlayObjectPrivate *priv = self->priv; - GstStructure *config; - D3D11_FILTER filter = D3D11_FILTER_MIN_MAG_MIP_POINT; - - if (is_subsampled_yuv (in_info) || is_subsampled_yuv (out_info)) - filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT; - - config = gst_structure_new ("convert-config", - GST_D3D11_CONVERTER_OPT_BACKEND, GST_TYPE_D3D11_CONVERTER_BACKEND, - GST_D3D11_CONVERTER_BACKEND_SHADER, - GST_D3D11_CONVERTER_OPT_SAMPLER_FILTER, - GST_TYPE_D3D11_CONVERTER_SAMPLER_FILTER, filter, nullptr); - if (is_blend) { - gst_structure_set (config, GST_D3D11_CONVERTER_OPT_SRC_ALPHA_MODE, - GST_TYPE_D3D11_CONVERTER_ALPHA_MODE, - GST_D3D11_CONVERTER_ALPHA_MODE_PREMULTIPLIED, nullptr); - } - - ret = gst_d3d11_converter_new (priv->device, in_info, out_info, config); - if (!ret) - GST_ERROR_OBJECT (self, "Couldn't create converter"); - - return ret; -} - -static GstBufferPool * -gst_dwrite_overlay_object_create_d3d11_pool (GstDWriteOverlayObject * self, - const GstVideoInfo * info) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - GstCaps *caps = nullptr; - GstStructure *config; - GstD3D11AllocationParams *params; - GstBufferPool *pool = nullptr; - - caps = gst_video_info_to_caps (info); - if (!caps) { - GST_ERROR_OBJECT (self, "Couldn't create caps"); - return nullptr; - } - - pool = gst_d3d11_buffer_pool_new (priv->device); - config = gst_buffer_pool_get_config (pool); - - params = gst_d3d11_allocation_params_new (priv->device, info, - GST_D3D11_ALLOCATION_FLAG_DEFAULT, - D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET, 0); - gst_buffer_pool_config_set_d3d11_allocation_params (config, params); - gst_d3d11_allocation_params_free (params); - gst_buffer_pool_config_set_params (config, caps, info->size, 0, 0); - gst_caps_unref (caps); - - if (!gst_buffer_pool_set_config (pool, config)) { - GST_ERROR_OBJECT (self, "Couldn't set pool config"); - goto error; - } - - if (!gst_buffer_pool_set_active (pool, TRUE)) { - GST_ERROR_OBJECT (self, "Couldn't activate pool"); - goto error; - } - - return pool; - -error: - gst_clear_object (&pool); - - return nullptr; -} - -static GstBufferPool * -gst_dwrite_overlay_object_create_bitmap_pool (GstDWriteOverlayObject * self, - const GstVideoInfo * info) -{ - GstCaps *caps = nullptr; - GstStructure *config; - GstBufferPool *pool = nullptr; - - caps = gst_video_info_to_caps (info); - if (!caps) { - GST_ERROR_OBJECT (self, "Couldn't create caps"); - return nullptr; - } - - pool = gst_dwrite_bitmap_pool_new (); - config = gst_buffer_pool_get_config (pool); - gst_buffer_pool_config_set_params (config, caps, info->size, 0, 0); - gst_caps_unref (caps); - - if (!gst_buffer_pool_set_config (pool, config)) { - GST_ERROR_OBJECT (self, "Couldn't set pool config"); - goto error; - } - - if (!gst_buffer_pool_set_active (pool, TRUE)) { - GST_ERROR_OBJECT (self, "Couldn't activate pool"); - goto error; - } - - return pool; - -error: - gst_clear_object (&pool); - - return nullptr; -} - -static gboolean -gst_dwrite_overlay_object_prepare_resource (GstDWriteOverlayObject * self) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - - switch (priv->blend_mode) { - case GstDWriteBlendMode::ATTACH_TEXTURE: - case GstDWriteBlendMode::ATTACH_BITMAP: - case GstDWriteBlendMode::SW_BLEND: - /* Updated later */ - break; - case GstDWriteBlendMode::BLEND: - priv->blend_conv = gst_dwrite_overlay_object_create_converter (self, - &priv->bgra_info, &priv->info, TRUE); - if (!priv->blend_conv) - return FALSE; - break; - case GstDWriteBlendMode::CONVERT: - priv->blend_pool = gst_dwrite_overlay_object_create_d3d11_pool (self, - &priv->bgra_info); - if (!priv->blend_pool) - return FALSE; - - priv->pre_conv = gst_dwrite_overlay_object_create_converter (self, - &priv->info, &priv->bgra_info, FALSE); - if (!priv->pre_conv) - return FALSE; - - priv->blend_conv = gst_dwrite_overlay_object_create_converter (self, - &priv->bgra_info, &priv->bgra_info, TRUE); - if (!priv->blend_conv) - return FALSE; - - priv->post_conv = gst_dwrite_overlay_object_create_converter (self, - &priv->bgra_info, &priv->info, FALSE); - if (!priv->blend_conv) - return FALSE; - break; - case GstDWriteBlendMode::CONVERT_64: - { - GstVideoInfo blend_info; - gst_video_info_set_format (&blend_info, GST_VIDEO_FORMAT_RGBA64_LE, - priv->info.width, priv->info.height); - - priv->blend_pool = gst_dwrite_overlay_object_create_d3d11_pool (self, - &blend_info); - if (!priv->blend_pool) - return FALSE; - - priv->pre_conv = gst_dwrite_overlay_object_create_converter (self, - &priv->info, &blend_info, FALSE); - if (!priv->pre_conv) - return FALSE; - - priv->blend_conv = gst_dwrite_overlay_object_create_converter (self, - &priv->bgra_info, &blend_info, TRUE); - if (!priv->pre_conv) - return FALSE; - - priv->post_conv = gst_dwrite_overlay_object_create_converter (self, - &blend_info, &priv->info, FALSE); - if (!priv->post_conv) - return FALSE; - - break; - } - default: - g_assert_not_reached (); - return FALSE; - } - - if (priv->blend_conv) { - D3D11_BLEND_DESC desc = { 0, }; - ComPtr < ID3D11BlendState > blend; - ID3D11Device *device = gst_d3d11_device_get_device_handle (priv->device); - HRESULT hr; - - desc.AlphaToCoverageEnable = FALSE; - desc.IndependentBlendEnable = FALSE; - desc.RenderTarget[0].BlendEnable = TRUE; - desc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA; - desc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA; - desc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD; - desc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE; - desc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_INV_SRC_ALPHA; - desc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD; - desc.RenderTarget[0].RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL; - - hr = device->CreateBlendState (&desc, &blend); - if (!gst_d3d11_result (hr, priv->device)) { - GST_ERROR_OBJECT (self, "Couldn't create blend state"); - return FALSE; - } - - g_object_set (priv->blend_conv, "blend-state", blend.Get (), nullptr); - } - - return TRUE; -} - -static const gchar * -gst_dwrite_overlay_mode_to_string (GstDWriteBlendMode mode) -{ - switch (mode) { - case GstDWriteBlendMode::NOT_SUPPORTED: - return "not-supported"; - case GstDWriteBlendMode::ATTACH_TEXTURE: - return "attach-texture"; - case GstDWriteBlendMode::ATTACH_BITMAP: - return "attach-bitmap"; - case GstDWriteBlendMode::SW_BLEND: - return "sw-blend"; - case GstDWriteBlendMode::BLEND: - return "blend"; - case GstDWriteBlendMode::CONVERT: - return "convert"; - case GstDWriteBlendMode::CONVERT_64: - return "convert-64"; - default: - break; - } - - return "unknown"; -} - GstDWriteOverlayObject * gst_dwrite_overlay_object_new (void) { @@ -487,7 +138,7 @@ gboolean gst_dwrite_overlay_object_start (GstDWriteOverlayObject * object, IDWriteFactory * dwrite_factory) { - GstDWriteOverlayObjectPrivate *priv = object->priv; + auto priv = object->priv; HRESULT hr; ComPtr < ID2D1Factory > d2d_factory; @@ -500,7 +151,6 @@ gst_dwrite_overlay_object_start (GstDWriteOverlayObject * object, priv->d2d_factory = d2d_factory; priv->dwrite_factory = dwrite_factory; - IGstDWriteTextRenderer::CreateInstance (dwrite_factory, &priv->renderer); return TRUE; } @@ -508,14 +158,12 @@ gst_dwrite_overlay_object_start (GstDWriteOverlayObject * object, gboolean gst_dwrite_overlay_object_stop (GstDWriteOverlayObject * object) { - GstDWriteOverlayObjectPrivate *priv = object->priv; + auto priv = object->priv; priv->ClearResource (true); priv->dwrite_factory = nullptr; priv->d2d_factory = nullptr; - priv->renderer = nullptr; gst_clear_object (&priv->device); - gst_clear_caps (&priv->outcaps); return TRUE; } @@ -524,17 +172,20 @@ void gst_dwrite_overlay_object_set_context (GstDWriteOverlayObject * object, GstElement * elem, GstContext * context) { - GstDWriteOverlayObjectPrivate *priv = object->priv; + auto priv = object->priv; std::lock_guard < std::recursive_mutex > lk (priv->ctx_lock); gst_d3d11_handle_set_context (elem, context, -1, &priv->device); +#ifdef HAVE_GST_D3D12 + gst_d3d12_handle_set_context (elem, context, -1, &priv->device12); +#endif } gboolean gst_dwrite_overlay_object_handle_query (GstDWriteOverlayObject * object, GstElement * elem, GstQuery * query) { - GstDWriteOverlayObjectPrivate *priv = object->priv; + auto priv = object->priv; if (GST_QUERY_TYPE (query) != GST_QUERY_CONTEXT) return FALSE; @@ -543,6 +194,11 @@ gst_dwrite_overlay_object_handle_query (GstDWriteOverlayObject * object, if (gst_d3d11_handle_context_query (elem, query, priv->device)) return TRUE; +#ifdef HAVE_GST_D3D12 + if (gst_d3d12_handle_context_query (elem, query, priv->device12)) + return TRUE; +#endif + return FALSE; } @@ -550,211 +206,38 @@ gboolean gst_dwrite_overlay_object_decide_allocation (GstDWriteOverlayObject * object, GstElement * elem, GstQuery * query) { - GstDWriteOverlayObjectPrivate *priv = object->priv; - guint min, max, size; - gboolean update_pool; - GstCaps *caps = nullptr; - GstVideoInfo info; - GstBufferPool *pool = nullptr; - GstCapsFeatures *features; - GstStructure *config; - GstD3D11AllocationParams *params; - guint bind_flags = 0; - GstD3D11Format d3d11_format; - - GST_DEBUG_OBJECT (elem, "Decide allocation"); - - gst_query_parse_allocation (query, &caps, nullptr); - if (!caps) { - GST_WARNING_OBJECT (elem, "Query without caps"); - return FALSE; - } - - if (!gst_video_info_from_caps (&info, caps)) { - GST_ERROR_OBJECT (elem, "Invalid caps %" GST_PTR_FORMAT, caps); - return FALSE; - } - - features = gst_caps_get_features (caps, 0); - if (!features || !gst_caps_features_contains (features, - GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) { - GST_DEBUG_OBJECT (elem, "Not a d3d11 memory"); + auto priv = object->priv; + if (!priv->render) { + GST_DEBUG_OBJECT (object, "Render object is not configured"); return TRUE; } - if (gst_query_get_n_allocation_pools (query) > 0) { - gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); - update_pool = TRUE; - } else { - min = max = 0; - size = info.size; - } - - if (pool) { - std::lock_guard < std::recursive_mutex > lk (priv->ctx_lock); - if (!gst_d3d11_ensure_element_data (elem, -1, &priv->device)) { - GST_ERROR_OBJECT (elem, "Couldn't create deice"); - return FALSE; - } - - if (!GST_IS_D3D11_BUFFER_POOL (pool)) { - gst_clear_object (&pool); - } else { - GstD3D11BufferPool *dpool = GST_D3D11_BUFFER_POOL (pool); - if (dpool->device != priv->device) - gst_clear_object (&pool); - } - } - - if (!pool) - pool = gst_d3d11_buffer_pool_new (priv->device); - - config = gst_buffer_pool_get_config (pool); - gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); - gst_buffer_pool_config_set_params (config, caps, size, min, max); - - gst_d3d11_device_get_format (priv->device, GST_VIDEO_INFO_FORMAT (&info), - &d3d11_format); - if ((d3d11_format.format_support[0] & - D3D11_FORMAT_SUPPORT_SHADER_SAMPLE) != 0) { - bind_flags |= D3D11_BIND_SHADER_RESOURCE; - } - - if ((d3d11_format.format_support[0] & - D3D11_FORMAT_SUPPORT_RENDER_TARGET) != 0) { - bind_flags |= D3D11_BIND_RENDER_TARGET; - } - - params = gst_buffer_pool_config_get_d3d11_allocation_params (config); - if (!params) { - params = gst_d3d11_allocation_params_new (priv->device, &info, - GST_D3D11_ALLOCATION_FLAG_DEFAULT, bind_flags, 0); - } else { - gst_d3d11_allocation_params_set_bind_flags (params, bind_flags); - } - - gst_buffer_pool_config_set_d3d11_allocation_params (config, params); - gst_d3d11_allocation_params_free (params); - - if (!gst_buffer_pool_set_config (pool, config)) { - GST_ERROR_OBJECT (elem, "Couldn't set config"); - gst_object_unref (pool); - return FALSE; - } - - /* Get updated size in case of d3d11 */ - config = gst_buffer_pool_get_config (pool); - gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, nullptr); - gst_structure_free (config); - - if (update_pool) - gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); - else - gst_query_add_allocation_pool (query, pool, size, min, max); - - gst_object_unref (pool); - - return TRUE; + return gst_dwrite_render_handle_allocation_query (priv->render, elem, query); } gboolean gst_dwrite_overlay_object_propose_allocation (GstDWriteOverlayObject * object, GstElement * elem, GstQuery * query) { - GstCaps *caps = nullptr; - GstVideoInfo info; - GstCapsFeatures *features; - guint min, max, size; - GstBufferPool *pool; - GstD3D11BufferPool *dpool; - GstStructure *config; - GstD3D11AllocationParams *params; - guint bind_flags = 0; - GstD3D11Format d3d11_format; - - GST_DEBUG_OBJECT (elem, "Propose allocation"); - - gst_query_parse_allocation (query, &caps, nullptr); - if (!caps) { - GST_WARNING_OBJECT (elem, "Query without caps"); - return FALSE; - } - - if (!gst_video_info_from_caps (&info, caps)) { - GST_ERROR_OBJECT (elem, "Invalid caps %" GST_PTR_FORMAT, caps); - return FALSE; - } - - features = gst_caps_get_features (caps, 0); - if (!features || !gst_caps_features_contains (features, - GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) { - GST_DEBUG_OBJECT (elem, "Not a d3d11 memory"); + auto priv = object->priv; + if (!priv->render) { + GST_DEBUG_OBJECT (object, "Render object is not configured"); return TRUE; } - if (gst_query_get_n_allocation_pools (query) == 0) - return TRUE; - - gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); - if (!pool) - return TRUE; - - if (!GST_IS_D3D11_BUFFER_POOL (pool)) { - gst_object_unref (pool); - return TRUE; - } - - dpool = GST_D3D11_BUFFER_POOL (pool); - gst_d3d11_device_get_format (dpool->device, GST_VIDEO_INFO_FORMAT (&info), - &d3d11_format); - if ((d3d11_format.format_support[0] & - D3D11_FORMAT_SUPPORT_SHADER_SAMPLE) != 0) { - bind_flags |= D3D11_BIND_SHADER_RESOURCE; - } - - if ((d3d11_format.format_support[0] & - D3D11_FORMAT_SUPPORT_RENDER_TARGET) != 0) { - bind_flags |= D3D11_BIND_RENDER_TARGET; - } - - config = gst_buffer_pool_get_config (pool); - params = gst_buffer_pool_config_get_d3d11_allocation_params (config); - if (!params) { - params = gst_d3d11_allocation_params_new (dpool->device, &info, - GST_D3D11_ALLOCATION_FLAG_DEFAULT, bind_flags, 0); - } else { - gst_d3d11_allocation_params_set_bind_flags (params, bind_flags); - } - - gst_buffer_pool_config_set_d3d11_allocation_params (config, params); - gst_d3d11_allocation_params_free (params); - - gst_buffer_pool_config_set_params (config, caps, size, min, max); - if (!gst_buffer_pool_set_config (pool, config)) { - GST_ERROR_OBJECT (elem, "Couldn't set config"); - gst_object_unref (pool); - return FALSE; - } - - gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); - gst_object_unref (pool); - - return TRUE; + return gst_dwrite_render_handle_allocation_query (priv->render, elem, query); } gboolean gst_dwrite_overlay_object_set_caps (GstDWriteOverlayObject * object, GstElement * elem, GstCaps * in_caps, GstCaps * out_caps, - GstVideoInfo * info, GstDWriteBlendMode * selected_mode) + GstVideoInfo * info) { - GstDWriteOverlayObjectPrivate *priv = object->priv; + auto priv = object->priv; gboolean is_system; GstCapsFeatures *features; - *selected_mode = GstDWriteBlendMode::NOT_SUPPORTED; - priv->ClearResource (true); - gst_caps_replace (&priv->outcaps, out_caps); if (!gst_video_info_from_caps (info, in_caps)) { GST_WARNING_OBJECT (elem, "Invalid caps %" GST_PTR_FORMAT, in_caps); @@ -762,53 +245,64 @@ gst_dwrite_overlay_object_set_caps (GstDWriteOverlayObject * object, } if (!gst_video_info_from_caps (&priv->info, out_caps)) { - GST_ERROR ("Invalid caps %" GST_PTR_FORMAT, out_caps); + GST_ERROR_OBJECT (elem, "Invalid caps %" GST_PTR_FORMAT, out_caps); return FALSE; } - gst_video_info_set_format (&priv->bgra_info, GST_VIDEO_FORMAT_BGRA, - priv->info.width, priv->info.height); - features = gst_caps_get_features (out_caps, 0); - priv->is_d3d11 = gst_caps_features_contains (features, + auto is_d3d11 = gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY); +#ifdef HAVE_GST_D3D12 + auto is_d3d12 = gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY); +#endif is_system = gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); priv->attach_meta = gst_caps_features_contains (features, GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); - if (priv->is_d3d11) { + if (is_d3d11) { std::lock_guard < std::recursive_mutex > lk (priv->ctx_lock); - if (!gst_d3d11_ensure_element_data (elem, -1, &priv->device)) { - GST_ERROR_OBJECT (elem, "Couldn't create deice"); - return FALSE; - } + is_d3d11 = gst_d3d11_ensure_element_data (elem, -1, &priv->device); } +#ifdef HAVE_GST_D3D12 + if (is_d3d12) { + std::lock_guard < std::recursive_mutex > lk (priv->ctx_lock); + is_d3d12 = gst_d3d12_ensure_element_data (elem, -1, &priv->device12); + } +#endif - if (!priv->is_d3d11 && !is_system && !priv->attach_meta) { + if (!is_d3d11 && !is_system && !priv->attach_meta +#ifdef HAVE_GST_D3D12 + && !is_d3d12 +#endif + ) { GST_WARNING_OBJECT (elem, "Not d3d11/system memory without composition meta support"); - return TRUE; + return FALSE; + } +#ifdef HAVE_GST_D3D12 + if (is_d3d12) { + priv->render = gst_dwrite_d3d12_render_new (priv->device12, &priv->info, + priv->d2d_factory.Get (), priv->dwrite_factory.Get ()); + } +#endif + + if (!priv->render && is_d3d11) { + priv->render = gst_dwrite_d3d11_render_new (priv->device, &priv->info, + priv->d2d_factory.Get (), priv->dwrite_factory.Get ()); } - gst_dwrite_overlay_object_decide_blend_mode (object); - GST_INFO_OBJECT (elem, "Selected blend mode: %s", - gst_dwrite_overlay_mode_to_string (priv->blend_mode)); - - if (priv->blend_mode == GstDWriteBlendMode::SW_BLEND || - priv->blend_mode == GstDWriteBlendMode::ATTACH_BITMAP) { - priv->use_bitmap = TRUE; - } else { - priv->use_bitmap = FALSE; + if (!priv->render) { + priv->render = gst_dwrite_bitmap_render_new (&priv->info, + priv->d2d_factory.Get (), priv->dwrite_factory.Get ()); } - if (!gst_dwrite_overlay_object_prepare_resource (object)) { - GST_ERROR_OBJECT (elem, "Couldn't prepare resource"); - priv->ClearResource (true); + if (!priv->render) { + GST_ERROR_OBJECT (elem, "Couldn't create render object"); return FALSE; } - *selected_mode = priv->blend_mode; return TRUE; } @@ -816,130 +310,12 @@ gboolean gst_dwrite_overlay_object_update_device (GstDWriteOverlayObject * object, GstBuffer * buffer) { - GstDWriteOverlayObjectPrivate *priv = object->priv; - GstMemory *mem; - GstD3D11Memory *dmem; + auto priv = object->priv; - if (priv->blend_mode == GstDWriteBlendMode::NOT_SUPPORTED || priv->use_bitmap) + if (!priv->render) return FALSE; - mem = gst_buffer_peek_memory (buffer, 0); - if (!gst_is_d3d11_memory (mem)) - return FALSE; - - dmem = GST_D3D11_MEMORY_CAST (mem); - std::lock_guard < std::recursive_mutex > lk (priv->ctx_lock); - if (dmem->device == priv->device) - return FALSE; - - GST_DEBUG_OBJECT (object, "Updating device"); - gst_clear_object (&priv->device); - priv->device = (GstD3D11Device *) gst_object_ref (dmem->device); - priv->ClearResource (false); - gst_dwrite_overlay_object_prepare_resource (object); - - return TRUE; -} - -static gboolean -gst_dwrite_overlay_object_upload_system (GstDWriteOverlayObject * self, - GstBuffer * dst, GstBuffer * src) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - GstVideoFrame in_frame, out_frame; - gboolean ret; - - GST_TRACE_OBJECT (self, "system copy"); - - if (!gst_video_frame_map (&in_frame, &priv->info, src, - (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF))) { - GST_ERROR_OBJECT (self, "Couldn't map input frame"); - return FALSE; - } - - if (!gst_video_frame_map (&out_frame, &priv->info, dst, - (GstMapFlags) (GST_MAP_WRITE | GST_VIDEO_FRAME_MAP_FLAG_NO_REF))) { - gst_video_frame_unmap (&in_frame); - GST_ERROR_OBJECT (self, "Couldn't map output frame"); - return FALSE; - } - - ret = gst_video_frame_copy (&out_frame, &in_frame); - - gst_video_frame_unmap (&in_frame); - gst_video_frame_unmap (&out_frame); - - return ret; -} - -static gboolean -gst_dwrite_overlay_object_upload_d3d11 (GstDWriteOverlayObject * self, - GstBuffer * dst, GstBuffer * src) -{ - GST_TRACE_OBJECT (self, "d3d11 copy"); - - for (guint i = 0; i < gst_buffer_n_memory (dst); i++) { - GstMemory *dst_mem, *src_mem; - GstD3D11Memory *dst_dmem, *src_dmem; - GstMapInfo dst_info; - GstMapInfo src_info; - ID3D11Resource *dst_texture, *src_texture; - ID3D11DeviceContext *device_context; - GstD3D11Device *device; - D3D11_BOX src_box = { 0, }; - D3D11_TEXTURE2D_DESC dst_desc, src_desc; - guint dst_subidx, src_subidx; - - dst_mem = gst_buffer_peek_memory (dst, i); - src_mem = gst_buffer_peek_memory (src, i); - - dst_dmem = (GstD3D11Memory *) dst_mem; - src_dmem = (GstD3D11Memory *) src_mem; - - device = dst_dmem->device; - - gst_d3d11_memory_get_texture_desc (dst_dmem, &dst_desc); - gst_d3d11_memory_get_texture_desc (src_dmem, &src_desc); - - device_context = gst_d3d11_device_get_device_context_handle (device); - - if (!gst_memory_map (dst_mem, &dst_info, - (GstMapFlags) (GST_MAP_WRITE | GST_MAP_D3D11))) { - GST_ERROR_OBJECT (self, "Cannot map dst d3d11 memory"); - return FALSE; - } - - if (!gst_memory_map (src_mem, &src_info, - (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) { - GST_ERROR_OBJECT (self, "Cannot map src d3d11 memory"); - gst_memory_unmap (dst_mem, &dst_info); - return FALSE; - } - - dst_texture = (ID3D11Resource *) dst_info.data; - src_texture = (ID3D11Resource *) src_info.data; - - /* src/dst texture size might be different if padding was used. - * select smaller size */ - src_box.left = 0; - src_box.top = 0; - src_box.front = 0; - src_box.back = 1; - src_box.right = MIN (src_desc.Width, dst_desc.Width); - src_box.bottom = MIN (src_desc.Height, dst_desc.Height); - - dst_subidx = gst_d3d11_memory_get_subresource_index (dst_dmem); - src_subidx = gst_d3d11_memory_get_subresource_index (src_dmem); - - GstD3D11DeviceLockGuard lk (device); - device_context->CopySubresourceRegion (dst_texture, dst_subidx, 0, 0, 0, - src_texture, src_subidx, &src_box); - - gst_memory_unmap (src_mem, &src_info); - gst_memory_unmap (dst_mem, &dst_info); - } - - return TRUE; + return gst_dwrite_render_update_device (priv->render, buffer); } GstFlowReturn @@ -947,39 +323,16 @@ gst_dwrite_overlay_object_prepare_output (GstDWriteOverlayObject * object, GstBaseTransform * trans, gpointer trans_class, GstBuffer * inbuf, GstBuffer ** outbuf) { - GstDWriteOverlayObjectPrivate *priv = object->priv; - GstMemory *mem = gst_buffer_peek_memory (inbuf, 0); + auto priv = object->priv; GstFlowReturn ret; - gboolean is_d3d11 = FALSE; - gboolean upload_ret; - /* attaching meta or software blending can be in-place processing */ - switch (priv->blend_mode) { - case GstDWriteBlendMode::ATTACH_TEXTURE: - case GstDWriteBlendMode::ATTACH_BITMAP: - case GstDWriteBlendMode::SW_BLEND: - goto inplace; - default: - break; + if (!priv->render) { + GST_ERROR_OBJECT (object, "Render object is not configured"); + return GST_FLOW_ERROR; } - if (gst_is_d3d11_memory (mem)) { - D3D11_TEXTURE2D_DESC desc; - GstD3D11Memory *dmem; - const guint bind_flags = (D3D11_BIND_RENDER_TARGET | - D3D11_BIND_SHADER_RESOURCE); - - is_d3d11 = TRUE; - - dmem = GST_D3D11_MEMORY_CAST (mem); - gst_d3d11_memory_get_texture_desc (dmem, &desc); - - /* Cannot write on decoder resource */ - if ((desc.BindFlags & D3D11_BIND_DECODER) == 0 && - (desc.BindFlags & bind_flags) == bind_flags) { - goto inplace; - } - } + if (priv->attach_meta || gst_dwrite_render_can_inplace (priv->render, inbuf)) + goto inplace; /* Needs to allocate new buffer */ ret = GST_BASE_TRANSFORM_CLASS (trans_class)->prepare_output_buffer (trans, @@ -987,15 +340,9 @@ gst_dwrite_overlay_object_prepare_output (GstDWriteOverlayObject * object, if (ret != GST_FLOW_OK) return ret; - if (is_d3d11) { - upload_ret = gst_dwrite_overlay_object_upload_d3d11 (object, - *outbuf, inbuf); - } else { - upload_ret = gst_dwrite_overlay_object_upload_system (object, - *outbuf, inbuf); - } + GST_LOG_OBJECT (object, "Needs upload"); - if (!upload_ret) { + if (!gst_dwrite_render_upload (priv->render, &priv->info, inbuf, *outbuf)) { gst_clear_buffer (outbuf); return GST_FLOW_ERROR; } @@ -1003,68 +350,22 @@ gst_dwrite_overlay_object_prepare_output (GstDWriteOverlayObject * object, return GST_FLOW_OK; inplace: - if (gst_buffer_is_writable (inbuf)) + GST_LOG_OBJECT (object, "Inplace render is possible"); + if (gst_buffer_is_writable (inbuf)) { *outbuf = inbuf; - else + } else { *outbuf = gst_buffer_copy (inbuf); + } return GST_FLOW_OK; } -static gboolean -gst_dwrite_overlay_object_get_target_from_d3d11 (GstDWriteOverlayObject * self, - GstMemory * mem, ID2D1RenderTarget ** target) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - ComPtr < IDXGISurface > surface; - GstD3D11Memory *dmem = GST_D3D11_MEMORY_CAST (mem); - ID3D11Resource *texture; - HRESULT hr; - static const D2D1_RENDER_TARGET_PROPERTIES props = { - D2D1_RENDER_TARGET_TYPE_DEFAULT, DXGI_FORMAT_B8G8R8A8_UNORM, - D2D1_ALPHA_MODE_PREMULTIPLIED, 0, 0, D2D1_RENDER_TARGET_USAGE_NONE, - D2D1_FEATURE_LEVEL_DEFAULT - }; - - texture = gst_d3d11_memory_get_resource_handle (dmem); - hr = texture->QueryInterface (IID_PPV_ARGS (&surface)); - if (!gst_d3d11_result (hr, priv->device)) - return FALSE; - - hr = priv->d2d_factory->CreateDxgiSurfaceRenderTarget (surface.Get (), props, - target); - - return gst_d3d11_result (hr, priv->device); -} - -static gboolean -gst_dwrite_overlay_object_get_target_from_bitmap (GstDWriteOverlayObject * self, - GstMemory * mem, ID2D1RenderTarget ** target) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - GstDWriteBitmapMemory *dmem = (GstDWriteBitmapMemory *) mem; - HRESULT hr; - static const D2D1_RENDER_TARGET_PROPERTIES props = { - D2D1_RENDER_TARGET_TYPE_DEFAULT, DXGI_FORMAT_B8G8R8A8_UNORM, - D2D1_ALPHA_MODE_PREMULTIPLIED, 0, 0, D2D1_RENDER_TARGET_USAGE_NONE, - D2D1_FEATURE_LEVEL_DEFAULT - }; - - hr = priv->d2d_factory->CreateWicBitmapRenderTarget (dmem->bitmap, &props, - target); - - return SUCCEEDED (hr); -} - static gboolean gst_dwrite_overlay_object_draw_layout (GstDWriteOverlayObject * self, IDWriteTextLayout * layout, gint x, gint y) { - GstDWriteOverlayObjectPrivate *priv = self->priv; + auto priv = self->priv; gint width, height; - GstMemory *mem; - ComPtr < ID2D1RenderTarget > target; - GstMapInfo info; if (priv->layout_buf) { if (priv->layout && priv->layout.Get () == layout) @@ -1080,75 +381,16 @@ gst_dwrite_overlay_object_draw_layout (GstDWriteOverlayObject * self, if (priv->layout_buf) return TRUE; + priv->layout_buf = gst_dwrite_render_draw_layout (priv->render, layout, x, y); + + if (!priv->layout_buf) { + GST_ERROR_OBJECT (self, "Couldn't create layout buffer"); + return FALSE; + } + width = (gint) layout->GetMaxWidth (); height = (gint) layout->GetMaxHeight (); - if (priv->layout_pool && - (priv->layout_info.width != width - || priv->layout_info.height != height)) { - gst_buffer_pool_set_active (priv->layout_pool, FALSE); - gst_clear_object (&priv->layout_pool); - } - - if (!priv->layout_pool) { - gst_video_info_set_format (&priv->layout_info, GST_VIDEO_FORMAT_BGRA, - width, height); - if (priv->use_bitmap) { - priv->layout_pool = gst_dwrite_overlay_object_create_bitmap_pool (self, - &priv->layout_info); - } else { - priv->layout_pool = gst_dwrite_overlay_object_create_d3d11_pool (self, - &priv->layout_info); - } - } - - if (!priv->layout_pool) - return FALSE; - - gst_buffer_pool_acquire_buffer (priv->layout_pool, - &priv->layout_buf, nullptr); - if (!priv->layout_buf) { - GST_ERROR_OBJECT (self, "Couldn't acquire layout buffer"); - return FALSE; - } - - mem = gst_buffer_peek_memory (priv->layout_buf, 0); - if (priv->use_bitmap) { - if (!gst_dwrite_overlay_object_get_target_from_bitmap (self, mem, &target)) { - GST_ERROR_OBJECT (self, "Couldn't get target from bitmap"); - gst_clear_buffer (&priv->layout_buf); - return FALSE; - } - } else { - if (!gst_memory_map (mem, &info, - (GstMapFlags) (GST_MAP_WRITE | GST_MAP_D3D11))) { - GST_ERROR_OBJECT (self, "Could not map buffer"); - gst_clear_buffer (&priv->layout_buf); - return FALSE; - } - - if (!gst_dwrite_overlay_object_get_target_from_d3d11 (self, mem, &target)) { - GST_ERROR_OBJECT (self, "Couldn't get target from texture"); - gst_memory_unmap (mem, &info); - gst_clear_buffer (&priv->layout_buf); - return FALSE; - } - } - - target->BeginDraw (); - target->Clear (D2D1::ColorF (D2D1::ColorF::Black, 0.0)); - priv->renderer->Draw (D2D1::Point2F (), - D2D1::Rect (0, 0, width, height), layout, target.Get ()); - target->EndDraw (); - - /* Release render target before unmapping. Otherwise pending GPU operations - * can be executed after releasing keyed-mutex, if texture was allocated with - * keyed-mutex enabled */ - target = nullptr; - - if (!priv->use_bitmap) - gst_memory_unmap (mem, &info); - priv->overlay_rect = gst_video_overlay_rectangle_new_raw (priv->layout_buf, x, y, width, height, GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA); @@ -1159,7 +401,7 @@ static gboolean gst_dwrite_overlay_object_mode_attach (GstDWriteOverlayObject * self, GstBuffer * buffer) { - GstDWriteOverlayObjectPrivate *priv = self->priv; + auto priv = self->priv; GstVideoOverlayCompositionMeta *meta; meta = gst_buffer_get_video_overlay_composition_meta (buffer); @@ -1182,131 +424,21 @@ gst_dwrite_overlay_object_mode_attach (GstDWriteOverlayObject * self, return TRUE; } -static gboolean -gst_dwrite_overlay_mode_sw_blend (GstDWriteOverlayObject * self, - GstBuffer * buffer, gint x, gint y) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - GstVideoFrame dst_frame, src_frame; - gboolean ret; - - if (!gst_video_frame_map (&dst_frame, &priv->info, buffer, - (GstMapFlags) (GST_MAP_WRITE | GST_VIDEO_FRAME_MAP_FLAG_NO_REF))) { - GST_ERROR_OBJECT (self, "Couldn't map input buffer"); - return FALSE; - } - - if (!gst_video_frame_map (&src_frame, &priv->layout_info, priv->layout_buf, - (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF))) { - gst_video_frame_unmap (&dst_frame); - GST_ERROR_OBJECT (self, "Couldn't map text buffer"); - return FALSE; - } - - src_frame.info.flags = (GstVideoFlags) - (src_frame.info.flags | GST_VIDEO_FLAG_PREMULTIPLIED_ALPHA); - ret = gst_video_blend (&dst_frame, &src_frame, x, y, 1.0); - gst_video_frame_unmap (&src_frame); - gst_video_frame_unmap (&dst_frame); - - return ret; -} - -static gboolean -gst_dwrite_overlay_mode_blend (GstDWriteOverlayObject * self, - GstBuffer * buffer, gint x, gint y) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - - g_object_set (priv->blend_conv, "src-width", priv->layout_info.width, - "src-height", priv->layout_info.height, - "dest-x", x, "dest-y", y, "dest-width", priv->layout_info.width, - "dest-height", priv->layout_info.height, nullptr); - - return gst_d3d11_converter_convert_buffer (priv->blend_conv, - priv->layout_buf, buffer); -} - -static gboolean -gst_dwrite_overlay_mode_convert (GstDWriteOverlayObject * self, - GstBuffer * buffer, gint x, gint y) -{ - GstDWriteOverlayObjectPrivate *priv = self->priv; - GstBuffer *pre_buf = nullptr; - - g_object_set (priv->blend_conv, "src-width", priv->layout_info.width, - "src-height", priv->layout_info.height, - "dest-x", x, "dest-y", y, "dest-width", priv->layout_info.width, - "dest-height", priv->layout_info.height, nullptr); - - gst_buffer_pool_acquire_buffer (priv->blend_pool, &pre_buf, nullptr); - if (!pre_buf) { - GST_ERROR_OBJECT (self, "Couldn't acquire preconv buffer"); - return FALSE; - } - - if (!gst_d3d11_converter_convert_buffer_unlocked (priv->pre_conv, - buffer, pre_buf)) { - GST_ERROR_OBJECT (self, "pre-convert failed"); - goto error; - } - - if (!gst_d3d11_converter_convert_buffer_unlocked (priv->blend_conv, - priv->layout_buf, pre_buf)) { - GST_ERROR_OBJECT (self, "blend-convert failed"); - goto error; - } - - if (!gst_d3d11_converter_convert_buffer_unlocked (priv->post_conv, - pre_buf, buffer)) { - GST_ERROR_OBJECT (self, "post-convert failed"); - goto error; - } - - gst_buffer_unref (pre_buf); - - return TRUE; - -error: - gst_clear_buffer (&pre_buf); - return FALSE; -} - gboolean gst_dwrite_overlay_object_draw (GstDWriteOverlayObject * object, GstBuffer * buffer, IDWriteTextLayout * layout, gint x, gint y) { - GstDWriteOverlayObjectPrivate *priv = object->priv; + auto priv = object->priv; gboolean ret = FALSE; - if (priv->device) - gst_d3d11_device_lock (priv->device); - if (!gst_dwrite_overlay_object_draw_layout (object, layout, x, y)) - goto out; + return FALSE; - switch (priv->blend_mode) { - case GstDWriteBlendMode::ATTACH_TEXTURE: - case GstDWriteBlendMode::ATTACH_BITMAP: - ret = gst_dwrite_overlay_object_mode_attach (object, buffer); - break; - case GstDWriteBlendMode::SW_BLEND: - ret = gst_dwrite_overlay_mode_sw_blend (object, buffer, x, y); - break; - case GstDWriteBlendMode::BLEND: - ret = gst_dwrite_overlay_mode_blend (object, buffer, x, y); - break; - case GstDWriteBlendMode::CONVERT: - case GstDWriteBlendMode::CONVERT_64: - ret = gst_dwrite_overlay_mode_convert (object, buffer, x, y); - break; - default: - g_assert_not_reached (); - break; - } + if (priv->attach_meta) + ret = gst_dwrite_overlay_object_mode_attach (object, buffer); + else + ret = gst_dwrite_render_blend (priv->render, priv->layout_buf, + x, y, buffer); -out: - if (priv->device) - gst_d3d11_device_unlock (priv->device); return ret; } diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.h b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.h index cc0078e166..2a6acbc5da 100644 --- a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.h +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.h @@ -27,6 +27,7 @@ G_BEGIN_DECLS +#if 0 enum class GstDWriteBlendMode { NOT_SUPPORTED, @@ -56,6 +57,7 @@ enum class GstDWriteBlendMode * 3) converts back original format */ CONVERT_64, }; +#endif #define GST_TYPE_DWRITE_OVERLAY_OBJECT (gst_dwrite_overlay_object_get_type()) G_DECLARE_FINAL_TYPE (GstDWriteOverlayObject, @@ -88,8 +90,7 @@ gboolean gst_dwrite_overlay_object_set_caps (GstDWriteOverlayObject * object, GstElement * elem, GstCaps * in_caps, GstCaps * out_caps, - GstVideoInfo * info, - GstDWriteBlendMode * selected_mode); + GstVideoInfo * info); gboolean gst_dwrite_overlay_object_update_device (GstDWriteOverlayObject * object, GstBuffer * buffer); diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender.cpp new file mode 100644 index 0000000000..1b6e2865f5 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender.cpp @@ -0,0 +1,124 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstdwriterender.h" + +GST_DEBUG_CATEGORY_EXTERN (dwrite_overlay_object_debug); +#define GST_CAT_DEFAULT dwrite_overlay_object_debug + +static gboolean gst_dwrite_render_upload_default (GstDWriteRender * render, + const GstVideoInfo * info, GstBuffer * in_buf, GstBuffer * out_buf); + +#define gst_dwrite_render_parent_class parent_class +G_DEFINE_ABSTRACT_TYPE (GstDWriteRender, gst_dwrite_render, GST_TYPE_OBJECT); + +static void +gst_dwrite_render_class_init (GstDWriteRenderClass * klass) +{ + klass->upload = gst_dwrite_render_upload_default; +} + +static void +gst_dwrite_render_init (GstDWriteRender * self) +{ +} + +GstBuffer * +gst_dwrite_render_draw_layout (GstDWriteRender * render, + IDWriteTextLayout * layout, gint x, gint y) +{ + auto klass = GST_DWRITE_RENDER_GET_CLASS (render); + + return klass->draw_layout (render, layout, x, y); +} + +gboolean +gst_dwrite_render_blend (GstDWriteRender * render, GstBuffer * layout_buf, + gint x, gint y, GstBuffer * output) +{ + auto klass = GST_DWRITE_RENDER_GET_CLASS (render); + + return klass->blend (render, layout_buf, x, y, output); +} + +gboolean +gst_dwrite_render_update_device (GstDWriteRender * render, GstBuffer * buffer) +{ + auto klass = GST_DWRITE_RENDER_GET_CLASS (render); + + return klass->update_device (render, buffer); +} + +gboolean +gst_dwrite_render_handle_allocation_query (GstDWriteRender * render, + GstElement * elem, GstQuery * query) +{ + auto klass = GST_DWRITE_RENDER_GET_CLASS (render); + + return klass->handle_allocation_query (render, elem, query); +} + +gboolean +gst_dwrite_render_can_inplace (GstDWriteRender * render, GstBuffer * buffer) +{ + auto klass = GST_DWRITE_RENDER_GET_CLASS (render); + + return klass->can_inplace (render, buffer); +} + +static gboolean +gst_dwrite_render_upload_default (GstDWriteRender * self, + const GstVideoInfo * info, GstBuffer * in_buf, GstBuffer * out_buf) +{ + GstVideoFrame in_frame, out_frame; + gboolean ret; + + GST_TRACE_OBJECT (self, "system copy"); + + if (!gst_video_frame_map (&in_frame, info, in_buf, GST_MAP_READ)) { + GST_ERROR_OBJECT (self, "Couldn't map input frame"); + return FALSE; + } + + if (!gst_video_frame_map (&out_frame, info, out_buf, GST_MAP_WRITE)) { + gst_video_frame_unmap (&in_frame); + GST_ERROR_OBJECT (self, "Couldn't map output frame"); + return FALSE; + } + + ret = gst_video_frame_copy (&out_frame, &in_frame); + + gst_video_frame_unmap (&in_frame); + gst_video_frame_unmap (&out_frame); + + return ret; +} + +gboolean +gst_dwrite_render_upload (GstDWriteRender * render, const GstVideoInfo * info, + GstBuffer * in_buf, GstBuffer * out_buf) +{ + auto klass = GST_DWRITE_RENDER_GET_CLASS (render); + + return klass->upload (render, info, in_buf, out_buf); +} diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender.h b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender.h new file mode 100644 index 0000000000..1c141e5fd0 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender.h @@ -0,0 +1,87 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include "gstdwriteoverlayobject.h" + +G_BEGIN_DECLS + +#define GST_TYPE_DWRITE_RENDER (gst_dwrite_render_get_type ()) +G_DECLARE_DERIVABLE_TYPE (GstDWriteRender, + gst_dwrite_render, GST, DWRITE_RENDER, GstObject); + +struct _GstDWriteRenderClass +{ + GstObjectClass parent_class; + + GstBuffer * (*draw_layout) (GstDWriteRender * render, + IDWriteTextLayout * layout, + gint x, + gint y); + + gboolean (*blend) (GstDWriteRender * render, + GstBuffer * layout_buf, + gint x, + gint y, + GstBuffer * output); + + gboolean (*update_device) (GstDWriteRender * renderer, + GstBuffer * buffer); + + gboolean (*handle_allocation_query) (GstDWriteRender * render, + GstElement * elem, + GstQuery * query); + + gboolean (*can_inplace) (GstDWriteRender * render, + GstBuffer * buffer); + + gboolean (*upload) (GstDWriteRender * render, + const GstVideoInfo * info, + GstBuffer * in_buf, + GstBuffer * out_buf); +}; + +GstBuffer * gst_dwrite_render_draw_layout (GstDWriteRender * render, + IDWriteTextLayout * layout, + gint x, + gint y); + +gboolean gst_dwrite_render_blend (GstDWriteRender * render, + GstBuffer * layout_buf, + gint x, + gint y, + GstBuffer * output); + +gboolean gst_dwrite_render_update_device (GstDWriteRender * render, + GstBuffer * buffer); + +gboolean gst_dwrite_render_handle_allocation_query (GstDWriteRender * render, + GstElement * elem, + GstQuery * query); + +gboolean gst_dwrite_render_can_inplace (GstDWriteRender * render, + GstBuffer * buffer); + +gboolean gst_dwrite_render_upload (GstDWriteRender * render, + const GstVideoInfo * info, + GstBuffer * in_buf, + GstBuffer * out_buf); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_bitmap.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_bitmap.cpp new file mode 100644 index 0000000000..d224890f1e --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_bitmap.cpp @@ -0,0 +1,268 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstdwriterender_bitmap.h" +#include "gstdwritebitmappool.h" +#include "gstdwrite-renderer.h" +#include + +GST_DEBUG_CATEGORY_EXTERN (dwrite_overlay_object_debug); +#define GST_CAT_DEFAULT dwrite_overlay_object_debug + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; + +struct GstDWriteBitmapRenderPrivate +{ + ~GstDWriteBitmapRenderPrivate () + { + renderer = nullptr; + dwrite_factory = nullptr; + d2d_factory = nullptr; + if (layout_pool) + gst_buffer_pool_set_active (layout_pool, FALSE); + gst_clear_object (&layout_pool); + } + ComPtr d2d_factory; + ComPtr dwrite_factory; + ComPtr renderer; + GstBufferPool *layout_pool = nullptr; + GstVideoInfo layout_info; + GstVideoInfo info; +}; +/* *INDENT-ON* */ + +struct _GstDWriteBitmapRender +{ + GstDWriteRender parent; + GstDWriteBitmapRenderPrivate *priv; +}; + +static void gst_dwrite_bitmap_render_finalize (GObject * object); +static GstBuffer *gst_dwrite_bitmap_render_draw_layout (GstDWriteRender * + render, IDWriteTextLayout * layout, gint x, gint y); +static gboolean gst_dwrite_bitmap_render_blend (GstDWriteRender * render, + GstBuffer * layout_buf, gint x, gint y, GstBuffer * output); +static gboolean gst_dwrite_bitmap_render_update_device (GstDWriteRender * + render, GstBuffer * buffer); +static gboolean +gst_dwrite_bitmap_render_handle_allocation_query (GstDWriteRender * render, + GstElement * elem, GstQuery * query); +static gboolean gst_dwrite_bitmap_render_can_inplace (GstDWriteRender * render, + GstBuffer * buffer); + +#define gst_dwrite_bitmap_render_parent_class parent_class +G_DEFINE_FINAL_TYPE (GstDWriteBitmapRender, gst_dwrite_bitmap_render, + GST_TYPE_DWRITE_RENDER); + +static void +gst_dwrite_bitmap_render_class_init (GstDWriteBitmapRenderClass * klass) +{ + auto object_class = G_OBJECT_CLASS (klass); + auto render_class = GST_DWRITE_RENDER_CLASS (klass); + + object_class->finalize = gst_dwrite_bitmap_render_finalize; + + render_class->draw_layout = gst_dwrite_bitmap_render_draw_layout; + render_class->blend = gst_dwrite_bitmap_render_blend; + render_class->update_device = gst_dwrite_bitmap_render_update_device; + render_class->handle_allocation_query = + gst_dwrite_bitmap_render_handle_allocation_query; + render_class->can_inplace = gst_dwrite_bitmap_render_can_inplace; +} + +static void +gst_dwrite_bitmap_render_init (GstDWriteBitmapRender * self) +{ + self->priv = new GstDWriteBitmapRenderPrivate (); +} + +static void +gst_dwrite_bitmap_render_finalize (GObject * object) +{ + auto self = GST_DWRITE_BITMAP_RENDER (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static GstBufferPool * +gst_dwrite_bitmap_render_create_pool (GstDWriteBitmapRender * self, + const GstVideoInfo * info) +{ + auto caps = gst_video_info_to_caps (info); + if (!caps) { + GST_ERROR_OBJECT (self, "Invalid info"); + return nullptr; + } + + auto pool = gst_dwrite_bitmap_pool_new (); + auto config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, caps, info->size, 0, 0); + gst_caps_unref (caps); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (self, "Couldn't set pool config"); + gst_object_unref (pool); + return nullptr; + } + + if (!gst_buffer_pool_set_active (pool, TRUE)) { + GST_ERROR_OBJECT (self, "Couldn't activate pool"); + gst_object_unref (pool); + return nullptr; + } + + return pool; +} + +static GstBuffer * +gst_dwrite_bitmap_render_draw_layout (GstDWriteRender * render, + IDWriteTextLayout * layout, gint x, gint y) +{ + auto self = GST_DWRITE_BITMAP_RENDER (render); + auto priv = self->priv; + + auto width = (gint) layout->GetMaxWidth (); + auto height = (gint) layout->GetMaxHeight (); + + if (priv->layout_pool && (priv->layout_info.width != width || + priv->layout_info.height != height)) { + gst_buffer_pool_set_active (priv->layout_pool, FALSE); + gst_clear_object (&priv->layout_pool); + } + + if (!priv->layout_pool) { + gst_video_info_set_format (&priv->layout_info, GST_VIDEO_FORMAT_BGRA, + width, height); + priv->layout_pool = gst_dwrite_bitmap_render_create_pool (self, + &priv->layout_info); + if (!priv->layout_pool) { + GST_ERROR_OBJECT (self, "Couldn't create pool"); + return nullptr; + } + } + + GstBuffer *layout_buf = nullptr; + gst_buffer_pool_acquire_buffer (priv->layout_pool, &layout_buf, nullptr); + if (!layout_buf) { + GST_ERROR_OBJECT (self, "Couldn't acquire buffer"); + return nullptr; + } + + ComPtr < IDXGISurface > surface; + auto bmem = (GstDWriteBitmapMemory *) gst_buffer_peek_memory (layout_buf, 0); + static const D2D1_RENDER_TARGET_PROPERTIES props = { + D2D1_RENDER_TARGET_TYPE_DEFAULT, DXGI_FORMAT_B8G8R8A8_UNORM, + D2D1_ALPHA_MODE_PREMULTIPLIED, 0, 0, D2D1_RENDER_TARGET_USAGE_NONE, + D2D1_FEATURE_LEVEL_DEFAULT + }; + + ComPtr < ID2D1RenderTarget > target; + auto hr = priv->d2d_factory->CreateWicBitmapRenderTarget (bmem->bitmap, props, + &target); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Couldn't create d2d render target"); + gst_buffer_unref (layout_buf); + return nullptr; + } + + target->BeginDraw (); + target->Clear (D2D1::ColorF (D2D1::ColorF::Black, 0.0)); + priv->renderer->Draw (D2D1::Point2F (), + D2D1::Rect (0, 0, width, height), layout, target.Get ()); + target->EndDraw (); + target = nullptr; + + return layout_buf; +} + +static gboolean +gst_dwrite_bitmap_render_blend (GstDWriteRender * render, + GstBuffer * layout_buf, gint x, gint y, GstBuffer * output) +{ + auto self = GST_DWRITE_BITMAP_RENDER (render); + auto priv = self->priv; + + GstVideoFrame dst_frame, src_frame; + gboolean ret; + + if (!gst_video_frame_map (&dst_frame, &priv->info, output, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (self, "Couldn't map output buffer"); + return FALSE; + } + + if (!gst_video_frame_map (&src_frame, &priv->layout_info, layout_buf, + GST_MAP_READ)) { + gst_video_frame_unmap (&dst_frame); + GST_ERROR_OBJECT (self, "Couldn't map layout buffer"); + return FALSE; + } + + src_frame.info.flags = (GstVideoFlags) + (src_frame.info.flags | GST_VIDEO_FLAG_PREMULTIPLIED_ALPHA); + ret = gst_video_blend (&dst_frame, &src_frame, x, y, 1.0); + gst_video_frame_unmap (&src_frame); + gst_video_frame_unmap (&dst_frame); + + return ret; +} + +static gboolean +gst_dwrite_bitmap_render_update_device (GstDWriteRender * render, + GstBuffer * buffer) +{ + return FALSE; +} + +static gboolean +gst_dwrite_bitmap_render_handle_allocation_query (GstDWriteRender * render, + GstElement * elem, GstQuery * query) +{ + return TRUE; +} + +static gboolean +gst_dwrite_bitmap_render_can_inplace (GstDWriteRender * render, + GstBuffer * buffer) +{ + return TRUE; +} + +GstDWriteRender * +gst_dwrite_bitmap_render_new (const GstVideoInfo * info, + ID2D1Factory * d2d_factory, IDWriteFactory * dwrite_factory) +{ + auto self = (GstDWriteBitmapRender *) + g_object_new (GST_TYPE_DWRITE_BITMAP_RENDER, nullptr); + gst_object_ref_sink (self); + + auto priv = self->priv; + priv->info = *info; + priv->d2d_factory = d2d_factory; + priv->dwrite_factory = dwrite_factory; + IGstDWriteTextRenderer::CreateInstance (dwrite_factory, &priv->renderer); + + return GST_DWRITE_RENDER (self); +} diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_bitmap.h b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_bitmap.h new file mode 100644 index 0000000000..1f877f358c --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_bitmap.h @@ -0,0 +1,34 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include "gstdwriterender.h" + +G_BEGIN_DECLS + +#define GST_TYPE_DWRITE_BITMAP_RENDER (gst_dwrite_bitmap_render_get_type ()) +G_DECLARE_FINAL_TYPE (GstDWriteBitmapRender, + gst_dwrite_bitmap_render, GST, DWRITE_BITMAP_RENDER, GstDWriteRender); + +GstDWriteRender * gst_dwrite_bitmap_render_new (const GstVideoInfo * info, + ID2D1Factory * d2d_factory, + IDWriteFactory * dwrite_factory); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d11.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d11.cpp new file mode 100644 index 0000000000..9f8de39cd7 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d11.cpp @@ -0,0 +1,745 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "gstdwriterender_d3d11.h" +#include "gstdwrite-renderer.h" +#include + +GST_DEBUG_CATEGORY_EXTERN (dwrite_overlay_object_debug); +#define GST_CAT_DEFAULT dwrite_overlay_object_debug + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; + +struct GstDWriteD3D11RenderPrivate +{ + ~GstDWriteD3D11RenderPrivate () + { + renderer = nullptr; + dwrite_factory = nullptr; + d2d_factory = nullptr; + ClearResource (); + } + + void ClearResource () + { + if (layout_pool) + gst_buffer_pool_set_active (layout_pool, FALSE); + gst_clear_object (&layout_pool); + if (blend_pool) + gst_buffer_pool_set_active (blend_pool, FALSE); + gst_clear_object (&blend_pool); + gst_clear_object (&pre_conv); + gst_clear_object (&blend_conv); + gst_clear_object (&post_conv); + gst_clear_object (&device); + prepared = FALSE; + } + + GstD3D11Device *device = nullptr; + ComPtr d2d_factory; + ComPtr dwrite_factory; + ComPtr renderer; + GstBufferPool *layout_pool = nullptr; + GstBufferPool *blend_pool = nullptr; + GstVideoInfo layout_info; + GstVideoInfo blend_info; + GstVideoInfo info; + gboolean direct_blend = FALSE; + gboolean prepared = FALSE; + + GstD3D11Converter *pre_conv = nullptr; + GstD3D11Converter *blend_conv = nullptr; + GstD3D11Converter *post_conv = nullptr; +}; +/* *INDENT-ON* */ + +struct _GstDWriteD3D11Render +{ + GstDWriteRender parent; + GstDWriteD3D11RenderPrivate *priv; +}; + +static void gst_dwrite_d3d11_render_finalize (GObject * object); +static GstBuffer *gst_dwrite_d3d11_render_draw_layout (GstDWriteRender * render, + IDWriteTextLayout * layout, gint x, gint y); +static gboolean gst_dwrite_d3d11_render_blend (GstDWriteRender * render, + GstBuffer * layout_buf, gint x, gint y, GstBuffer * output); +static gboolean +gst_dwrite_d3d11_render_update_device (GstDWriteRender * render, + GstBuffer * buffer); +static gboolean +gst_dwrite_d3d11_render_handle_allocation_query (GstDWriteRender * render, + GstElement * elem, GstQuery * query); +static gboolean gst_dwrite_d3d11_render_can_inplace (GstDWriteRender * render, + GstBuffer * buffer); +static gboolean gst_dwrite_d3d11_render_upload (GstDWriteRender * render, + const GstVideoInfo * info, GstBuffer * in_buf, GstBuffer * out_buf); + +static gboolean gst_dwrite_d3d11_render_prepare (GstDWriteD3D11Render * self); + +#define gst_dwrite_d3d11_render_parent_class parent_class +G_DEFINE_FINAL_TYPE (GstDWriteD3D11Render, gst_dwrite_d3d11_render, + GST_TYPE_DWRITE_RENDER); + +static void +gst_dwrite_d3d11_render_class_init (GstDWriteD3D11RenderClass * klass) +{ + auto object_class = G_OBJECT_CLASS (klass); + auto render_class = GST_DWRITE_RENDER_CLASS (klass); + + object_class->finalize = gst_dwrite_d3d11_render_finalize; + + render_class->draw_layout = gst_dwrite_d3d11_render_draw_layout; + render_class->blend = gst_dwrite_d3d11_render_blend; + render_class->update_device = gst_dwrite_d3d11_render_update_device; + render_class->handle_allocation_query = + gst_dwrite_d3d11_render_handle_allocation_query; + render_class->can_inplace = gst_dwrite_d3d11_render_can_inplace; + render_class->upload = gst_dwrite_d3d11_render_upload; +} + +static void +gst_dwrite_d3d11_render_init (GstDWriteD3D11Render * self) +{ + self->priv = new GstDWriteD3D11RenderPrivate (); +} + +static void +gst_dwrite_d3d11_render_finalize (GObject * object) +{ + auto self = GST_DWRITE_D3D11_RENDER (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static gboolean +gst_dwrite_d3d11_render_decide_bind_flags (GstDWriteD3D11Render * self, + const GstVideoInfo * info, guint * bind_flags) +{ + auto priv = self->priv; + + GstD3D11Format d3d11_format; + if (!gst_d3d11_device_get_format (priv->device, + GST_VIDEO_INFO_FORMAT (info), &d3d11_format)) { + GST_ERROR_OBJECT (self, "Unknown device format"); + return FALSE; + } + + DXGI_FORMAT dxgi_format = d3d11_format.dxgi_format; + if (dxgi_format == DXGI_FORMAT_UNKNOWN) + dxgi_format = d3d11_format.resource_format[0]; + + auto device = gst_d3d11_device_get_device_handle (priv->device); + UINT support = 0; + auto hr = device->CheckFormatSupport (dxgi_format, &support); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't query format support"); + return FALSE; + } + + guint flags = 0; + if ((support & D3D11_FORMAT_SUPPORT_SHADER_SAMPLE) != 0) + flags |= D3D11_BIND_SHADER_RESOURCE; + + if ((support & D3D11_FORMAT_SUPPORT_RENDER_TARGET) != 0) { + flags |= D3D11_BIND_RENDER_TARGET; + if (d3d11_format.dxgi_format == DXGI_FORMAT_UNKNOWN && + (support & D3D11_FORMAT_SUPPORT_TYPED_UNORDERED_ACCESS_VIEW) != 0) { + flags |= D3D11_BIND_UNORDERED_ACCESS; + } + } else if ((support & D3D11_FORMAT_SUPPORT_TYPED_UNORDERED_ACCESS_VIEW) != 0) { + flags |= D3D11_BIND_UNORDERED_ACCESS; + } + + *bind_flags = flags; + + return TRUE; +} + +static GstBufferPool * +gst_dwrite_d3d11_render_create_pool (GstDWriteD3D11Render * self, + const GstVideoInfo * info, guint bind_flags) +{ + auto priv = self->priv; + + auto caps = gst_video_info_to_caps (info); + if (!caps) { + GST_ERROR_OBJECT (self, "Invalid info"); + return nullptr; + } + + auto pool = gst_d3d11_buffer_pool_new (priv->device); + auto config = gst_buffer_pool_get_config (pool); + auto params = gst_d3d11_allocation_params_new (priv->device, info, + GST_D3D11_ALLOCATION_FLAG_DEFAULT, bind_flags, 0); + gst_buffer_pool_config_set_d3d11_allocation_params (config, params); + gst_d3d11_allocation_params_free (params); + gst_buffer_pool_config_set_params (config, caps, 0, 0, 0); + gst_caps_unref (caps); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (self, "Couldn't set config"); + gst_object_unref (pool); + return nullptr; + } + + if (!gst_buffer_pool_set_active (pool, TRUE)) { + GST_ERROR_OBJECT (self, "Couldn't set active"); + gst_object_unref (pool); + return nullptr; + } + + return pool; +} + +static GstBuffer * +gst_dwrite_d3d11_render_draw_layout (GstDWriteRender * render, + IDWriteTextLayout * layout, gint x, gint y) +{ + auto self = GST_DWRITE_D3D11_RENDER (render); + auto priv = self->priv; + + if (!priv->prepared) { + GST_ERROR_OBJECT (self, "Not prepapred"); + return nullptr; + } + + auto width = (gint) layout->GetMaxWidth (); + auto height = (gint) layout->GetMaxHeight (); + + if (priv->layout_pool && (priv->layout_info.width != width || + priv->layout_info.height != height)) { + gst_buffer_pool_set_active (priv->layout_pool, FALSE); + gst_clear_object (&priv->layout_pool); + } + + if (!priv->layout_pool) { + gst_video_info_set_format (&priv->layout_info, GST_VIDEO_FORMAT_BGRA, + width, height); + + guint bind_flags = 0; + if (!gst_dwrite_d3d11_render_decide_bind_flags (self, &priv->layout_info, + &bind_flags)) { + GST_ERROR_OBJECT (self, "Couldn't decide bind flags"); + return nullptr; + } + priv->layout_pool = gst_dwrite_d3d11_render_create_pool (self, + &priv->layout_info, bind_flags); + if (!priv->layout_pool) { + GST_ERROR_OBJECT (self, "Couldn't create pool"); + return nullptr; + } + } + + GstBuffer *layout_buf = nullptr; + gst_buffer_pool_acquire_buffer (priv->layout_pool, &layout_buf, nullptr); + if (!layout_buf) { + GST_ERROR_OBJECT (self, "Couldn't acquire buffer"); + return nullptr; + } + + GstD3D11DeviceLockGuard lk (priv->device); + ComPtr < IDXGISurface > surface; + auto dmem = (GstD3D11Memory *) gst_buffer_peek_memory (layout_buf, 0); + static const D2D1_RENDER_TARGET_PROPERTIES props = { + D2D1_RENDER_TARGET_TYPE_DEFAULT, DXGI_FORMAT_B8G8R8A8_UNORM, + D2D1_ALPHA_MODE_PREMULTIPLIED, 0, 0, D2D1_RENDER_TARGET_USAGE_NONE, + D2D1_FEATURE_LEVEL_DEFAULT + }; + + auto texture = gst_d3d11_memory_get_resource_handle (dmem); + auto hr = texture->QueryInterface (IID_PPV_ARGS (&surface)); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't get DXGI surface"); + gst_buffer_unref (layout_buf); + return nullptr; + } + + ComPtr < ID2D1RenderTarget > target; + hr = priv->d2d_factory->CreateDxgiSurfaceRenderTarget (surface.Get (), props, + &target); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Couldn't create d2d render target"); + gst_buffer_unref (layout_buf); + return nullptr; + } + + target->BeginDraw (); + target->Clear (D2D1::ColorF (D2D1::ColorF::Black, 0.0)); + priv->renderer->Draw (D2D1::Point2F (), + D2D1::Rect (0, 0, width, height), layout, target.Get ()); + target->EndDraw (); + target = nullptr; + + return layout_buf; +} + +static gboolean +gst_dwrite_d3d11_render_blend (GstDWriteRender * render, GstBuffer * layout_buf, + gint x, gint y, GstBuffer * output) +{ + auto self = GST_DWRITE_D3D11_RENDER (render); + auto priv = self->priv; + + if (!priv->prepared) { + GST_ERROR_OBJECT (self, "Not prepapred"); + return FALSE; + } + + g_object_set (priv->blend_conv, "src-width", priv->layout_info.width, + "src-height", priv->layout_info.height, + "dest-x", x, "dest-y", y, "dest-width", priv->layout_info.width, + "dest-height", priv->layout_info.height, nullptr); + + if (priv->direct_blend) { + return gst_d3d11_converter_convert_buffer (priv->blend_conv, + layout_buf, output); + } + + GstBuffer *bgra_buf = nullptr; + gst_buffer_pool_acquire_buffer (priv->blend_pool, &bgra_buf, nullptr); + if (!bgra_buf) { + GST_ERROR_OBJECT (self, "Couldn't acquire preconv buffer"); + return FALSE; + } + + if (!gst_d3d11_converter_convert_buffer_unlocked (priv->pre_conv, + output, bgra_buf)) { + GST_ERROR_OBJECT (self, "pre-convert failed"); + goto error; + } + + if (!gst_d3d11_converter_convert_buffer_unlocked (priv->blend_conv, + layout_buf, bgra_buf)) { + GST_ERROR_OBJECT (self, "blend-convert failed"); + goto error; + } + + if (!gst_d3d11_converter_convert_buffer_unlocked (priv->post_conv, + bgra_buf, output)) { + GST_ERROR_OBJECT (self, "post-convert failed"); + goto error; + } + + gst_buffer_unref (bgra_buf); + + return TRUE; + +error: + gst_clear_buffer (&bgra_buf); + return FALSE; +} + +static gboolean +gst_dwrite_d3d11_render_update_device (GstDWriteRender * render, + GstBuffer * buffer) +{ + auto self = GST_DWRITE_D3D11_RENDER (render); + auto priv = self->priv; + + auto mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_d3d11_memory (mem)) + return FALSE; + + auto dmem = GST_D3D11_MEMORY_CAST (mem); + if (dmem->device != priv->device) { + priv->ClearResource (); + priv->device = (GstD3D11Device *) gst_object_ref (dmem->device); + gst_dwrite_d3d11_render_prepare (self); + return TRUE; + } + + return FALSE; +} + +static gboolean +gst_dwrite_d3d11_render_handle_allocation_query (GstDWriteRender * render, + GstElement * elem, GstQuery * query) +{ + auto self = GST_DWRITE_D3D11_RENDER (render); + auto priv = self->priv; + + GstCaps *caps = nullptr; + gst_query_parse_allocation (query, &caps, nullptr); + if (!caps) { + GST_WARNING_OBJECT (elem, "Query without caps"); + return FALSE; + } + + GstVideoInfo info; + if (!gst_video_info_from_caps (&info, caps)) { + GST_ERROR_OBJECT (elem, "Invalid caps %" GST_PTR_FORMAT, caps); + return FALSE; + } + + auto features = gst_caps_get_features (caps, 0); + if (!gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) { + GST_DEBUG_OBJECT (elem, "Not a d3d11 caps"); + return TRUE; + } + + guint bind_flags = 0; + if (!gst_dwrite_d3d11_render_decide_bind_flags (self, &info, &bind_flags)) { + GST_ERROR_OBJECT (self, "Couldn't decide bind flags"); + return FALSE; + } + + gboolean update_pool = FALSE; + guint min, max, size; + GstBufferPool *pool = nullptr; + + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + update_pool = TRUE; + } else { + min = max = 0; + size = info.size; + } + + if (pool) { + if (!GST_IS_D3D11_BUFFER_POOL (pool)) { + gst_clear_object (&pool); + } else { + auto dpool = GST_D3D11_BUFFER_POOL (pool); + if (dpool->device != priv->device) + gst_clear_object (&pool); + } + } + + if (!pool) + pool = gst_d3d11_buffer_pool_new (priv->device); + + auto config = gst_buffer_pool_get_config (pool); + auto params = gst_buffer_pool_config_get_d3d11_allocation_params (config); + if (!params) { + params = gst_d3d11_allocation_params_new (priv->device, &info, + GST_D3D11_ALLOCATION_FLAG_DEFAULT, bind_flags, 0); + } else { + gst_d3d11_allocation_params_set_bind_flags (params, bind_flags); + } + + gst_buffer_pool_config_set_d3d11_allocation_params (config, params); + gst_d3d11_allocation_params_free (params); + + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_config_set_params (config, caps, size, min, max); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (self, "Couldn't set config"); + gst_object_unref (pool); + return FALSE; + } + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, nullptr); + gst_structure_free (config); + + if (update_pool) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + + gst_object_unref (pool); + + return TRUE; +} + +static gboolean +gst_dwrite_d3d11_render_can_inplace (GstDWriteRender * render, + GstBuffer * buffer) +{ + auto self = GST_DWRITE_D3D11_RENDER (render); + auto priv = self->priv; + + auto mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_d3d11_memory (mem)) + return FALSE; + + auto dmem = GST_D3D11_MEMORY_CAST (mem); + if (dmem->device != priv->device) + return FALSE; + + D3D11_TEXTURE2D_DESC desc; + gst_d3d11_memory_get_texture_desc (dmem, &desc); + + if ((desc.BindFlags & D3D11_BIND_DECODER) != 0) + return FALSE; + + return TRUE; +} + +static gboolean +gst_dwrite_d3d11_render_upload_d3d11 (GstDWriteD3D11Render * self, + GstBuffer * dst, GstBuffer * src) +{ + auto priv = self->priv; + + GST_TRACE_OBJECT (self, "d3d11 copy"); + + GstD3D11DeviceLockGuard lk (priv->device); + + for (guint i = 0; i < gst_buffer_n_memory (dst); i++) { + GstMemory *dst_mem, *src_mem; + GstD3D11Memory *dst_dmem, *src_dmem; + GstMapInfo dst_info; + GstMapInfo src_info; + ID3D11Resource *dst_texture, *src_texture; + ID3D11DeviceContext *device_context; + GstD3D11Device *device; + D3D11_BOX src_box = { 0, }; + D3D11_TEXTURE2D_DESC dst_desc, src_desc; + guint dst_subidx, src_subidx; + + dst_mem = gst_buffer_peek_memory (dst, i); + src_mem = gst_buffer_peek_memory (src, i); + + dst_dmem = (GstD3D11Memory *) dst_mem; + src_dmem = (GstD3D11Memory *) src_mem; + + device = dst_dmem->device; + + gst_d3d11_memory_get_texture_desc (dst_dmem, &dst_desc); + gst_d3d11_memory_get_texture_desc (src_dmem, &src_desc); + + device_context = gst_d3d11_device_get_device_context_handle (device); + + if (!gst_memory_map (dst_mem, &dst_info, + (GstMapFlags) (GST_MAP_WRITE | GST_MAP_D3D11))) { + GST_ERROR_OBJECT (self, "Cannot map dst d3d11 memory"); + return FALSE; + } + + if (!gst_memory_map (src_mem, &src_info, + (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) { + GST_ERROR_OBJECT (self, "Cannot map src d3d11 memory"); + gst_memory_unmap (dst_mem, &dst_info); + return FALSE; + } + + dst_texture = (ID3D11Resource *) dst_info.data; + src_texture = (ID3D11Resource *) src_info.data; + + /* src/dst texture size might be different if padding was used. + * select smaller size */ + src_box.left = 0; + src_box.top = 0; + src_box.front = 0; + src_box.back = 1; + src_box.right = MIN (src_desc.Width, dst_desc.Width); + src_box.bottom = MIN (src_desc.Height, dst_desc.Height); + + dst_subidx = gst_d3d11_memory_get_subresource_index (dst_dmem); + src_subidx = gst_d3d11_memory_get_subresource_index (src_dmem); + + GstD3D11DeviceLockGuard lk (device); + device_context->CopySubresourceRegion (dst_texture, dst_subidx, 0, 0, 0, + src_texture, src_subidx, &src_box); + + gst_memory_unmap (src_mem, &src_info); + gst_memory_unmap (dst_mem, &dst_info); + } + + return TRUE; +} + +static gboolean +gst_dwrite_d3d11_render_upload (GstDWriteRender * render, + const GstVideoInfo * info, GstBuffer * in_buf, GstBuffer * out_buf) +{ + auto self = GST_DWRITE_D3D11_RENDER (render); + auto priv = self->priv; + + if (!priv->prepared) { + GST_ERROR_OBJECT (self, "Not prepared"); + return FALSE; + } + + auto mem = gst_buffer_peek_memory (in_buf, 0); + if (gst_is_d3d11_memory (mem) && GST_D3D11_MEMORY_CAST (mem)->device == + priv->device) { + return gst_dwrite_d3d11_render_upload_d3d11 (self, out_buf, in_buf); + } + + return GST_DWRITE_RENDER_CLASS (parent_class)->upload (render, + info, in_buf, out_buf); +} + +static gboolean +is_subsampled_yuv (const GstVideoInfo * info) +{ + if (!GST_VIDEO_INFO_IS_YUV (info)) + return FALSE; + + for (guint i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) { + if (info->finfo->w_sub[i] != 0 || info->finfo->h_sub[i] != 0) + return TRUE; + } + + return FALSE; +} + +static GstD3D11Converter * +create_converter (GstDWriteD3D11Render * self, const GstVideoInfo * in_info, + const GstVideoInfo * out_info, gboolean is_blend) +{ + auto priv = self->priv; + D3D11_FILTER filter = D3D11_FILTER_MIN_MAG_MIP_POINT; + + if (is_subsampled_yuv (in_info) || is_subsampled_yuv (out_info)) + filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT; + + auto config = gst_structure_new ("convert-config", + GST_D3D11_CONVERTER_OPT_BACKEND, GST_TYPE_D3D11_CONVERTER_BACKEND, + GST_D3D11_CONVERTER_BACKEND_SHADER, + GST_D3D11_CONVERTER_OPT_SAMPLER_FILTER, + GST_TYPE_D3D11_CONVERTER_SAMPLER_FILTER, filter, nullptr); + if (is_blend) { + gst_structure_set (config, GST_D3D11_CONVERTER_OPT_SRC_ALPHA_MODE, + GST_TYPE_D3D11_CONVERTER_ALPHA_MODE, + GST_D3D11_CONVERTER_ALPHA_MODE_PREMULTIPLIED, nullptr); + } + + auto ret = gst_d3d11_converter_new (priv->device, in_info, out_info, config); + if (!ret) + GST_ERROR_OBJECT (self, "Couldn't create converter"); + + return ret; +} + +static gboolean +gst_dwrite_d3d11_render_prepare (GstDWriteD3D11Render * self) +{ + auto priv = self->priv; + GstVideoInfo bgra_info; + gst_video_info_set_format (&bgra_info, + GST_VIDEO_FORMAT_BGRA, priv->info.width, priv->info.height); + + if (priv->direct_blend) { + priv->blend_conv = create_converter (self, &bgra_info, &priv->blend_info, + TRUE); + } else { + guint bind_flags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET; + priv->blend_pool = gst_dwrite_d3d11_render_create_pool (self, + &priv->blend_info, bind_flags); + if (!priv->blend_pool) + return FALSE; + + priv->pre_conv = create_converter (self, + &priv->info, &priv->blend_info, FALSE); + if (!priv->pre_conv) + return FALSE; + + priv->blend_conv = create_converter (self, + &bgra_info, &priv->blend_info, TRUE); + if (!priv->blend_conv) + return FALSE; + + priv->post_conv = create_converter (self, + &priv->blend_info, &priv->info, FALSE); + if (!priv->post_conv) + return FALSE; + } + + D3D11_BLEND_DESC desc = { }; + ComPtr < ID3D11BlendState > blend; + auto device = gst_d3d11_device_get_device_handle (priv->device); + + desc.AlphaToCoverageEnable = FALSE; + desc.IndependentBlendEnable = FALSE; + desc.RenderTarget[0].BlendEnable = TRUE; + desc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA; + desc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA; + desc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD; + desc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE; + desc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_INV_SRC_ALPHA; + desc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD; + desc.RenderTarget[0].RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL; + + auto hr = device->CreateBlendState (&desc, &blend); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't create blend state"); + return FALSE; + } + + g_object_set (priv->blend_conv, "blend-state", blend.Get (), nullptr); + + GST_DEBUG_OBJECT (self, "Resource prepared"); + + priv->prepared = TRUE; + + return TRUE; +} + +GstDWriteRender * +gst_dwrite_d3d11_render_new (GstD3D11Device * device, const GstVideoInfo * info, + ID2D1Factory * d2d_factory, IDWriteFactory * dwrite_factory) +{ + auto self = (GstDWriteD3D11Render *) + g_object_new (GST_TYPE_DWRITE_D3D11_RENDER, nullptr); + gst_object_ref_sink (self); + + auto priv = self->priv; + priv->device = (GstD3D11Device *) gst_object_ref (device); + priv->info = *info; + + auto format = GST_VIDEO_INFO_FORMAT (info); + switch (format) { + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRx: + case GST_VIDEO_FORMAT_RGBx: + case GST_VIDEO_FORMAT_VUYA: + case GST_VIDEO_FORMAT_RGBA64_LE: + case GST_VIDEO_FORMAT_RGB10A2_LE: + priv->direct_blend = TRUE; + gst_video_info_set_format (&priv->blend_info, + format, info->width, info->height); + break; + default: + priv->direct_blend = FALSE; + if (GST_VIDEO_INFO_COMP_DEPTH (info, 0) > 8) { + gst_video_info_set_format (&priv->blend_info, + GST_VIDEO_FORMAT_RGBA64_LE, info->width, info->height); + } else { + gst_video_info_set_format (&priv->blend_info, + GST_VIDEO_FORMAT_BGRA, info->width, info->height); + } + break; + } + + if (!gst_dwrite_d3d11_render_prepare (self)) { + gst_object_unref (self); + return nullptr; + } + + priv->d2d_factory = d2d_factory; + priv->dwrite_factory = dwrite_factory; + IGstDWriteTextRenderer::CreateInstance (dwrite_factory, &priv->renderer); + + return GST_DWRITE_RENDER (self); +} diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d11.h b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d11.h new file mode 100644 index 0000000000..b76c1e4820 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d11.h @@ -0,0 +1,35 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include "gstdwriterender.h" + +G_BEGIN_DECLS + +#define GST_TYPE_DWRITE_D3D11_RENDER (gst_dwrite_d3d11_render_get_type ()) +G_DECLARE_FINAL_TYPE (GstDWriteD3D11Render, + gst_dwrite_d3d11_render, GST, DWRITE_D3D11_RENDER, GstDWriteRender); + +GstDWriteRender * gst_dwrite_d3d11_render_new (GstD3D11Device * device, + const GstVideoInfo * info, + ID2D1Factory * d2d_factory, + IDWriteFactory * dwrite_factory); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d12.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d12.cpp new file mode 100644 index 0000000000..b6a6f14636 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d12.cpp @@ -0,0 +1,898 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstdwriterender_d3d12.h" +#include "gstdwrite-renderer.h" +#include +#include +#include +#include +#include + +GST_DEBUG_CATEGORY_EXTERN (dwrite_overlay_object_debug); +#define GST_CAT_DEFAULT dwrite_overlay_object_debug + +#define ASYNC_DEPTH 4 + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; + +struct GstDWriteD3D12RenderPrivate +{ + GstDWriteD3D12RenderPrivate () + { + fence_data_pool = gst_d3d12_fence_data_pool_new (); + event_handle = CreateEventEx (nullptr, nullptr, 0, EVENT_ALL_ACCESS); + } + + ~GstDWriteD3D12RenderPrivate () + { + renderer = nullptr; + dwrite_factory = nullptr; + d2d_factory = nullptr; + ClearResource (); + gst_clear_object (&fence_data_pool); + CloseHandle (event_handle); + } + + void ClearResource () + { + if (device) { + gst_d3d12_device_fence_wait (device, D3D12_COMMAND_LIST_TYPE_DIRECT, + fence_val, event_handle); + } + + gst_clear_object (&ca_pool); + cl = nullptr; + d2d_target = nullptr; + wrapped_texture = nullptr; + layout_resource = nullptr; + device11on12 = nullptr; + d3d11_context = nullptr; + device11 = nullptr; + + if (layout_pool) + gst_buffer_pool_set_active (layout_pool, FALSE); + gst_clear_object (&layout_pool); + if (blend_pool) + gst_buffer_pool_set_active (blend_pool, FALSE); + gst_clear_object (&blend_pool); + gst_clear_object (&pre_conv); + gst_clear_object (&blend_conv); + gst_clear_object (&post_conv); + gst_clear_object (&device); + prepared = FALSE; + fence_val = 0; + scheduled = { }; + } + + GstD3D12Device *device = nullptr; + ComPtr d2d_factory; + ComPtr dwrite_factory; + ComPtr renderer; + ComPtr wrapped_texture; + ComPtr layout_resource; + ComPtr d2d_target; + GstBufferPool *layout_pool = nullptr; + GstBufferPool *blend_pool = nullptr; + GstVideoInfo layout_info; + GstVideoInfo blend_info; + GstVideoInfo info; + gboolean direct_blend = FALSE; + gboolean prepared = FALSE; + + GstD3D12Converter *pre_conv = nullptr; + GstD3D12Converter *blend_conv = nullptr; + GstD3D12Converter *post_conv = nullptr; + + HANDLE event_handle; + guint64 fence_val = 0; + + ComPtr cl; + GstD3D12FenceDataPool *fence_data_pool; + GstD3D12CommandAllocatorPool *ca_pool = nullptr; + ComPtr device11on12; + ComPtr device11; + ComPtr d3d11_context; + std::queue scheduled; +}; +/* *INDENT-ON* */ + +struct _GstDWriteD3D12Render +{ + GstDWriteRender parent; + GstDWriteD3D12RenderPrivate *priv; +}; + +static void gst_dwrite_d3d12_render_finalize (GObject * object); +static GstBuffer *gst_dwrite_d3d12_render_draw_layout (GstDWriteRender * render, + IDWriteTextLayout * layout, gint x, gint y); +static gboolean gst_dwrite_d3d12_render_blend (GstDWriteRender * render, + GstBuffer * layout_buf, gint x, gint y, GstBuffer * output); +static gboolean +gst_dwrite_d3d12_render_update_device (GstDWriteRender * render, + GstBuffer * buffer); +static gboolean +gst_dwrite_d3d12_render_handle_allocation_query (GstDWriteRender * render, + GstElement * elem, GstQuery * query); +static gboolean gst_dwrite_d3d12_render_can_inplace (GstDWriteRender * render, + GstBuffer * buffer); +static gboolean gst_dwrite_d3d12_render_upload (GstDWriteRender * render, + const GstVideoInfo * info, GstBuffer * in_buf, GstBuffer * out_buf); + +static gboolean gst_dwrite_d3d12_render_prepare (GstDWriteD3D12Render * self); + +#define gst_dwrite_d3d12_render_parent_class parent_class +G_DEFINE_FINAL_TYPE (GstDWriteD3D12Render, gst_dwrite_d3d12_render, + GST_TYPE_DWRITE_RENDER); + +static void +gst_dwrite_d3d12_render_class_init (GstDWriteD3D12RenderClass * klass) +{ + auto object_class = G_OBJECT_CLASS (klass); + auto render_class = GST_DWRITE_RENDER_CLASS (klass); + + object_class->finalize = gst_dwrite_d3d12_render_finalize; + + render_class->draw_layout = gst_dwrite_d3d12_render_draw_layout; + render_class->blend = gst_dwrite_d3d12_render_blend; + render_class->update_device = gst_dwrite_d3d12_render_update_device; + render_class->handle_allocation_query = + gst_dwrite_d3d12_render_handle_allocation_query; + render_class->can_inplace = gst_dwrite_d3d12_render_can_inplace; + render_class->upload = gst_dwrite_d3d12_render_upload; +} + +static void +gst_dwrite_d3d12_render_init (GstDWriteD3D12Render * self) +{ + self->priv = new GstDWriteD3D12RenderPrivate (); +} + +static void +gst_dwrite_d3d12_render_finalize (GObject * object) +{ + auto self = GST_DWRITE_D3D12_RENDER (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static GstBufferPool * +gst_dwrite_d3d12_render_create_pool (GstDWriteD3D12Render * self, + const GstVideoInfo * info) +{ + auto priv = self->priv; + + auto caps = gst_video_info_to_caps (info); + if (!caps) { + GST_ERROR_OBJECT (self, "Invalid info"); + return nullptr; + } + + auto pool = gst_d3d12_buffer_pool_new (priv->device); + auto config = gst_buffer_pool_get_config (pool); + auto params = gst_d3d12_allocation_params_new (priv->device, info, + GST_D3D12_ALLOCATION_FLAG_DEFAULT, + D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS | + D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET, D3D12_HEAP_FLAG_NONE); + gst_buffer_pool_config_set_d3d12_allocation_params (config, params); + gst_d3d12_allocation_params_free (params); + gst_buffer_pool_config_set_params (config, caps, 0, 0, 0); + gst_caps_unref (caps); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (self, "Couldn't set config"); + gst_object_unref (pool); + return nullptr; + } + + if (!gst_buffer_pool_set_active (pool, TRUE)) { + GST_ERROR_OBJECT (self, "Couldn't set active"); + gst_object_unref (pool); + return nullptr; + } + + return pool; +} + +static GstBuffer * +gst_dwrite_d3d12_render_draw_layout (GstDWriteRender * render, + IDWriteTextLayout * layout, gint x, gint y) +{ + auto self = GST_DWRITE_D3D12_RENDER (render); + auto priv = self->priv; + + if (!priv->prepared) { + GST_ERROR_OBJECT (self, "Not prepapred"); + return nullptr; + } + + auto width = (gint) layout->GetMaxWidth (); + auto height = (gint) layout->GetMaxHeight (); + + if (priv->layout_pool && (priv->layout_info.width != width || + priv->layout_info.height != height)) { + gst_buffer_pool_set_active (priv->layout_pool, FALSE); + gst_clear_object (&priv->layout_pool); + priv->d2d_target = nullptr; + priv->wrapped_texture = nullptr; + priv->layout_resource = nullptr; + } + + if (!priv->layout_pool) { + gst_video_info_set_format (&priv->layout_info, GST_VIDEO_FORMAT_BGRA, + width, height); + + priv->layout_pool = gst_dwrite_d3d12_render_create_pool (self, + &priv->layout_info); + if (!priv->layout_pool) { + GST_ERROR_OBJECT (self, "Couldn't create pool"); + return nullptr; + } + } + + if (!priv->layout_resource) { + auto device = gst_d3d12_device_get_device_handle (priv->device); + D3D12_HEAP_PROPERTIES heap_prop = { }; + heap_prop.Type = D3D12_HEAP_TYPE_DEFAULT; + heap_prop.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heap_prop.CreationNodeMask = 1; + heap_prop.VisibleNodeMask = 1; + + D3D12_RESOURCE_DESC desc = { }; + desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.Width = width; + desc.Height = height; + desc.DepthOrArraySize = 1; + desc.MipLevels = 1; + desc.SampleDesc.Count = 1; + desc.SampleDesc.Quality = 0; + desc.Flags = D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | + D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS; + desc.Layout = D3D12_TEXTURE_LAYOUT_UNKNOWN; + + auto hr = device->CreateCommittedResource (&heap_prop, + D3D12_HEAP_FLAG_SHARED, &desc, D3D12_RESOURCE_STATE_COMMON, + nullptr, IID_PPV_ARGS (&priv->layout_resource)); + if (!gst_d3d12_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't create layout texture"); + return nullptr; + } + + D3D11_RESOURCE_FLAGS flags11 = { }; + flags11.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + flags11.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + hr = priv->device11on12->CreateWrappedResource (priv-> + layout_resource.Get (), &flags11, D3D12_RESOURCE_STATE_RENDER_TARGET, + D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE, + IID_PPV_ARGS (&priv->wrapped_texture)); + if (!gst_d3d12_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't create wrappred resource"); + priv->layout_resource = nullptr; + return nullptr; + } + + ComPtr < IDXGISurface > surface; + static const D2D1_RENDER_TARGET_PROPERTIES props = { + D2D1_RENDER_TARGET_TYPE_DEFAULT, DXGI_FORMAT_B8G8R8A8_UNORM, + D2D1_ALPHA_MODE_PREMULTIPLIED, 0, 0, D2D1_RENDER_TARGET_USAGE_NONE, + D2D1_FEATURE_LEVEL_DEFAULT + }; + + hr = priv->wrapped_texture->QueryInterface (IID_PPV_ARGS (&surface)); + if (!gst_d3d12_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't get DXGI surface"); + priv->wrapped_texture = nullptr; + priv->layout_resource = nullptr; + return nullptr; + } + + hr = priv->d2d_factory->CreateDxgiSurfaceRenderTarget (surface.Get (), + props, &priv->d2d_target); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Couldn't create d2d render target"); + priv->wrapped_texture = nullptr; + priv->layout_resource = nullptr; + return nullptr; + } + } + + if (priv->scheduled.size () >= ASYNC_DEPTH) { + auto fence_to_wait = priv->scheduled.front (); + priv->scheduled.pop (); + gst_d3d12_device_fence_wait (priv->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, fence_to_wait, priv->event_handle); + } + + GstBuffer *layout_buf = nullptr; + gst_buffer_pool_acquire_buffer (priv->layout_pool, &layout_buf, nullptr); + if (!layout_buf) { + GST_ERROR_OBJECT (self, "Couldn't acquire buffer"); + return nullptr; + } + + ID3D11Resource *wrapped[] = { priv->wrapped_texture.Get () }; + priv->device11on12->AcquireWrappedResources (wrapped, 1); + priv->d2d_target->BeginDraw (); + priv->d2d_target->Clear (D2D1::ColorF (D2D1::ColorF::Black, 0.0)); + priv->renderer->Draw (D2D1::Point2F (), + D2D1::Rect (0, 0, width, height), layout, priv->d2d_target.Get ()); + priv->d2d_target->EndDraw (); + priv->device11on12->ReleaseWrappedResources (wrapped, 1); + priv->d3d11_context->Flush (); + + auto dmem = (GstD3D12Memory *) gst_buffer_peek_memory (layout_buf, 0); + auto texture = gst_d3d12_memory_get_resource_handle (dmem); + + GstD3D12CopyTextureRegionArgs args = { }; + args.src.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + args.src.pResource = priv->layout_resource.Get (); + args.dst.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + args.dst.pResource = texture; + + gst_d3d12_device_copy_texture_region (priv->device, + 1, &args, D3D12_COMMAND_LIST_TYPE_DIRECT, &priv->fence_val); + + priv->scheduled.push (priv->fence_val); + dmem->fence_value = priv->fence_val; + + GstD3D12FenceData *fence_data; + gst_d3d12_fence_data_pool_acquire (priv->fence_data_pool, &fence_data); + auto resource_clone = priv->layout_resource; + auto wrapped_clone = priv->wrapped_texture; + + gst_d3d12_fence_data_add_notify_com (fence_data, resource_clone.Detach ()); + gst_d3d12_fence_data_add_notify_com (fence_data, wrapped_clone.Detach ()); + + gst_d3d12_device_set_fence_notify (priv->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, dmem->fence_value, fence_data); + + GST_MINI_OBJECT_FLAG_SET (dmem, GST_D3D12_MEMORY_TRANSFER_NEED_DOWNLOAD); + GST_MINI_OBJECT_FLAG_UNSET (dmem, GST_D3D12_MEMORY_TRANSFER_NEED_UPLOAD); + + return layout_buf; +} + +static gboolean +gst_dwrite_d3d12_render_blend (GstDWriteRender * render, GstBuffer * layout_buf, + gint x, gint y, GstBuffer * output) +{ + auto self = GST_DWRITE_D3D12_RENDER (render); + auto priv = self->priv; + + if (!priv->prepared) { + GST_ERROR_OBJECT (self, "Not prepapred"); + return FALSE; + } + + if (priv->scheduled.size () >= ASYNC_DEPTH) { + auto fence_to_wait = priv->scheduled.front (); + priv->scheduled.pop (); + gst_d3d12_device_fence_wait (priv->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, fence_to_wait, priv->event_handle); + } + + GstD3D12Frame out_frame; + if (!gst_d3d12_frame_map (&out_frame, &priv->info, output, + GST_MAP_WRITE_D3D12, GST_D3D12_FRAME_MAP_FLAG_RTV)) { + GST_ERROR_OBJECT (self, "Couldn't map output frame"); + return FALSE; + } + gst_d3d12_frame_unmap (&out_frame); + + GstD3D12CommandAllocator *gst_ca; + if (!gst_d3d12_command_allocator_pool_acquire (priv->ca_pool, &gst_ca)) { + GST_ERROR_OBJECT (self, "Couldn't acquire command allocator"); + return FALSE; + } + + auto ca = gst_d3d12_command_allocator_get_handle (gst_ca); + auto hr = ca->Reset (); + if (!gst_d3d12_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't reset command allocator"); + gst_d3d12_command_allocator_unref (gst_ca); + return FALSE; + } + + if (!priv->cl) { + auto device = gst_d3d12_device_get_device_handle (priv->device); + hr = device->CreateCommandList (0, D3D12_COMMAND_LIST_TYPE_DIRECT, + ca, nullptr, IID_PPV_ARGS (&priv->cl)); + if (!gst_d3d12_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't create command list"); + gst_d3d12_command_allocator_unref (gst_ca); + return FALSE; + } + } else { + hr = priv->cl->Reset (ca, nullptr); + if (!gst_d3d12_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't reset command list"); + gst_d3d12_command_allocator_unref (gst_ca); + return FALSE; + } + } + + GstD3D12FenceData *fence_data; + gst_d3d12_fence_data_pool_acquire (priv->fence_data_pool, &fence_data); + gst_d3d12_fence_data_add_notify_mini_object (fence_data, gst_ca); + + g_object_set (priv->blend_conv, "src-width", priv->layout_info.width, + "src-height", priv->layout_info.height, + "dest-x", x, "dest-y", y, "dest-width", priv->layout_info.width, + "dest-height", priv->layout_info.height, nullptr); + + gboolean ret = TRUE; + GstBuffer *bgra_buf = nullptr; + if (priv->direct_blend) { + GST_LOG_OBJECT (self, "Direct blend"); + ret = gst_d3d12_converter_convert_buffer (priv->blend_conv, + layout_buf, output, fence_data, priv->cl.Get ()); + } else { + GST_LOG_OBJECT (self, "Need conversion for blending"); + + gst_buffer_pool_acquire_buffer (priv->blend_pool, &bgra_buf, nullptr); + if (!bgra_buf) { + GST_ERROR_OBJECT (self, "Couldn't acquire preconv buffer"); + ret = FALSE; + } + + if (ret) { + ret = gst_d3d12_converter_convert_buffer (priv->pre_conv, + output, bgra_buf, fence_data, priv->cl.Get ()); + } + + if (ret) { + auto dmem = (GstD3D12Memory *) gst_buffer_peek_memory (bgra_buf, 0); + auto resource = gst_d3d12_memory_get_resource_handle (dmem); + D3D12_RESOURCE_BARRIER barrier = { }; + barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + barrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; + barrier.Transition.pResource = resource; + barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_RENDER_TARGET; + barrier.Transition.StateAfter = + D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE; + barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + priv->cl->ResourceBarrier (1, &barrier); + + ret = gst_d3d12_converter_convert_buffer (priv->blend_conv, + layout_buf, bgra_buf, fence_data, priv->cl.Get ()); + } + + if (ret) { + std::vector < D3D12_RESOURCE_BARRIER > barriers; + auto dmem = (GstD3D12Memory *) gst_buffer_peek_memory (bgra_buf, 0); + auto resource = gst_d3d12_memory_get_resource_handle (dmem); + + D3D12_RESOURCE_BARRIER barrier = { }; + barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + barrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; + barrier.Transition.pResource = resource; + barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_RENDER_TARGET; + barrier.Transition.StateAfter = + D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE; + barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + barriers.push_back (barrier); + + for (guint i = 0; i < gst_buffer_n_memory (output); i++) { + dmem = (GstD3D12Memory *) gst_buffer_peek_memory (output, i); + resource = gst_d3d12_memory_get_resource_handle (dmem); + barrier.Transition.pResource = resource; + barrier.Transition.StateBefore = + D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE; + barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_RENDER_TARGET; + barriers.push_back (barrier); + } + + priv->cl->ResourceBarrier (barriers.size (), barriers.data ()); + + ret = gst_d3d12_converter_convert_buffer (priv->post_conv, + bgra_buf, output, fence_data, priv->cl.Get ()); + } + + gst_clear_buffer (&bgra_buf); + } + + hr = priv->cl->Close (); + if (ret) + ret = gst_d3d12_result (hr, priv->device); + + if (ret) { + ID3D12CommandList *cl[] = { priv->cl.Get () }; + ret = gst_d3d12_device_execute_command_lists (priv->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, 1, cl, &priv->fence_val); + } + + if (ret) { + gst_d3d12_device_set_fence_notify (priv->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, priv->fence_val, fence_data); + + priv->scheduled.push (priv->fence_val); + + for (guint i = 0; i < gst_buffer_n_memory (output); i++) { + auto dmem = (GstD3D12Memory *) gst_buffer_peek_memory (output, i); + dmem->fence_value = priv->fence_val; + GST_MINI_OBJECT_FLAG_SET (dmem, GST_D3D12_MEMORY_TRANSFER_NEED_DOWNLOAD); + GST_MINI_OBJECT_FLAG_UNSET (dmem, GST_D3D12_MEMORY_TRANSFER_NEED_UPLOAD); + } + } else { + gst_d3d12_fence_data_unref (fence_data); + } + + return ret; +} + +static gboolean +gst_dwrite_d3d12_render_update_device (GstDWriteRender * render, + GstBuffer * buffer) +{ + auto self = GST_DWRITE_D3D12_RENDER (render); + auto priv = self->priv; + + auto mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_d3d12_memory (mem)) + return FALSE; + + auto dmem = GST_D3D12_MEMORY_CAST (mem); + if (!gst_d3d12_device_is_equal (dmem->device, priv->device)) { + priv->ClearResource (); + priv->device = (GstD3D12Device *) gst_object_ref (dmem->device); + gst_dwrite_d3d12_render_prepare (self); + return TRUE; + } + + return FALSE; +} + +static gboolean +gst_dwrite_d3d12_render_handle_allocation_query (GstDWriteRender * render, + GstElement * elem, GstQuery * query) +{ + auto self = GST_DWRITE_D3D12_RENDER (render); + auto priv = self->priv; + + GstCaps *caps = nullptr; + gst_query_parse_allocation (query, &caps, nullptr); + if (!caps) { + GST_WARNING_OBJECT (elem, "Query without caps"); + return FALSE; + } + + GstVideoInfo info; + if (!gst_video_info_from_caps (&info, caps)) { + GST_ERROR_OBJECT (elem, "Invalid caps %" GST_PTR_FORMAT, caps); + return FALSE; + } + + auto features = gst_caps_get_features (caps, 0); + if (!gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY)) { + GST_DEBUG_OBJECT (elem, "Not a d3d12 caps"); + return TRUE; + } + + gboolean update_pool = FALSE; + guint min, max, size; + GstBufferPool *pool = nullptr; + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + update_pool = TRUE; + } else { + min = max = 0; + size = info.size; + } + + if (pool) { + if (!GST_IS_D3D12_BUFFER_POOL (pool)) { + gst_clear_object (&pool); + } else { + auto dpool = GST_D3D12_BUFFER_POOL (pool); + if (!gst_d3d12_device_is_equal (dpool->device, priv->device)) + gst_clear_object (&pool); + } + } + + if (!pool) + pool = gst_d3d12_buffer_pool_new (priv->device); + + auto config = gst_buffer_pool_get_config (pool); + auto params = gst_buffer_pool_config_get_d3d12_allocation_params (config); + if (!params) { + params = gst_d3d12_allocation_params_new (priv->device, &info, + GST_D3D12_ALLOCATION_FLAG_DEFAULT, + D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS | + D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET, D3D12_HEAP_FLAG_NONE); + } else { + gst_d3d12_allocation_params_set_resource_flags (params, + D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS | + D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET); + gst_d3d12_allocation_params_unset_resource_flags (params, + D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE); + } + + gst_buffer_pool_config_set_d3d12_allocation_params (config, params); + gst_d3d12_allocation_params_free (params); + + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_config_set_params (config, caps, size, min, max); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (self, "Couldn't set config"); + gst_object_unref (pool); + return FALSE; + } + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, nullptr); + gst_structure_free (config); + + if (update_pool) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + + gst_object_unref (pool); + + return TRUE; +} + +static gboolean +gst_dwrite_d3d12_render_can_inplace (GstDWriteRender * render, + GstBuffer * buffer) +{ + auto self = GST_DWRITE_D3D12_RENDER (render); + auto priv = self->priv; + + auto mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_d3d12_memory (mem)) + return FALSE; + + auto dmem = GST_D3D12_MEMORY_CAST (mem); + if (!gst_d3d12_device_is_equal (dmem->device, priv->device)) + return FALSE; + + D3D12_RESOURCE_DESC desc; + auto resource = gst_d3d12_memory_get_resource_handle (dmem); + desc = resource->GetDesc (); + + if ((desc.Flags & D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE) != 0 || + (desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET) == 0) { + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_dwrite_d3d12_render_upload_d3d12 (GstDWriteD3D12Render * self, + GstBuffer * dst, GstBuffer * src) +{ + auto priv = self->priv; + + GST_TRACE_OBJECT (self, "d3d12 copy"); + + return gst_d3d12_buffer_copy_into (dst, src, &priv->info); +} + +static gboolean +gst_dwrite_d3d12_render_upload (GstDWriteRender * render, + const GstVideoInfo * info, GstBuffer * in_buf, GstBuffer * out_buf) +{ + auto self = GST_DWRITE_D3D12_RENDER (render); + auto priv = self->priv; + + if (!priv->prepared) { + GST_ERROR_OBJECT (self, "Not prepared"); + return FALSE; + } + + auto mem = gst_buffer_peek_memory (in_buf, 0); + if (gst_is_d3d12_memory (mem) && + gst_d3d12_device_is_equal (GST_D3D12_MEMORY_CAST (mem)->device, + priv->device)) { + return gst_dwrite_d3d12_render_upload_d3d12 (self, out_buf, in_buf); + } + + return GST_DWRITE_RENDER_CLASS (parent_class)->upload (render, + info, in_buf, out_buf); +} + +static gboolean +is_subsampled_yuv (const GstVideoInfo * info) +{ + if (!GST_VIDEO_INFO_IS_YUV (info)) + return FALSE; + + for (guint i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) { + if (info->finfo->w_sub[i] != 0 || info->finfo->h_sub[i] != 0) + return TRUE; + } + + return FALSE; +} + +static GstD3D12Converter * +create_converter (GstDWriteD3D12Render * self, const GstVideoInfo * in_info, + const GstVideoInfo * out_info, gboolean is_blend) +{ + auto priv = self->priv; + D3D12_FILTER filter = D3D12_FILTER_MIN_MAG_MIP_POINT; + D3D12_BLEND_DESC blend_desc = { }; + blend_desc.AlphaToCoverageEnable = FALSE; + blend_desc.IndependentBlendEnable = FALSE; + blend_desc.RenderTarget[0].BlendEnable = FALSE; + blend_desc.RenderTarget[0].LogicOpEnable = FALSE; + blend_desc.RenderTarget[0].SrcBlend = D3D12_BLEND_ONE; + blend_desc.RenderTarget[0].DestBlend = D3D12_BLEND_ZERO; + blend_desc.RenderTarget[0].BlendOp = D3D12_BLEND_OP_ADD; + blend_desc.RenderTarget[0].SrcBlendAlpha = D3D12_BLEND_ONE; + blend_desc.RenderTarget[0].DestBlendAlpha = D3D12_BLEND_ZERO; + blend_desc.RenderTarget[0].BlendOpAlpha = D3D12_BLEND_OP_ADD; + blend_desc.RenderTarget[0].LogicOp = D3D12_LOGIC_OP_NOOP; + blend_desc.RenderTarget[0].RenderTargetWriteMask = + D3D12_COLOR_WRITE_ENABLE_ALL; + + if (is_subsampled_yuv (in_info) || is_subsampled_yuv (out_info)) + filter = D3D12_FILTER_MIN_MAG_LINEAR_MIP_POINT; + + auto config = gst_structure_new ("convert-config", + GST_D3D12_CONVERTER_OPT_SAMPLER_FILTER, + GST_TYPE_D3D12_CONVERTER_SAMPLER_FILTER, filter, nullptr); + + if (is_blend) { + blend_desc.RenderTarget[0].BlendEnable = TRUE; + blend_desc.RenderTarget[0].SrcBlend = D3D12_BLEND_SRC_ALPHA; + blend_desc.RenderTarget[0].DestBlend = D3D12_BLEND_INV_SRC_ALPHA; + blend_desc.RenderTarget[0].BlendOp = D3D12_BLEND_OP_ADD; + blend_desc.RenderTarget[0].SrcBlendAlpha = D3D12_BLEND_ONE; + blend_desc.RenderTarget[0].DestBlendAlpha = D3D12_BLEND_INV_SRC_ALPHA; + blend_desc.RenderTarget[0].BlendOpAlpha = D3D12_BLEND_OP_ADD; + gst_structure_set (config, GST_D3D11_CONVERTER_OPT_SRC_ALPHA_MODE, + GST_TYPE_D3D12_CONVERTER_ALPHA_MODE, + GST_D3D12_CONVERTER_ALPHA_MODE_PREMULTIPLIED, nullptr); + } + + auto ret = gst_d3d12_converter_new (priv->device, in_info, out_info, + &blend_desc, nullptr, config); + + if (!ret) + GST_ERROR_OBJECT (self, "Couldn't create converter"); + + return ret; +} + +static gboolean +gst_dwrite_d3d12_render_prepare (GstDWriteD3D12Render * self) +{ + auto priv = self->priv; + GstVideoInfo bgra_info; + gst_video_info_set_format (&bgra_info, + GST_VIDEO_FORMAT_BGRA, priv->info.width, priv->info.height); + + if (priv->direct_blend) { + priv->blend_conv = create_converter (self, &bgra_info, &priv->blend_info, + TRUE); + } else { + priv->blend_pool = gst_dwrite_d3d12_render_create_pool (self, + &priv->blend_info); + if (!priv->blend_pool) + return FALSE; + + priv->pre_conv = create_converter (self, + &priv->info, &priv->blend_info, FALSE); + if (!priv->pre_conv) + return FALSE; + + priv->blend_conv = create_converter (self, + &bgra_info, &priv->blend_info, TRUE); + if (!priv->blend_conv) + return FALSE; + + priv->post_conv = create_converter (self, + &priv->blend_info, &priv->info, FALSE); + if (!priv->post_conv) + return FALSE; + } + + auto device = gst_d3d12_device_get_device_handle (priv->device); + auto cq = gst_d3d12_device_get_command_queue (priv->device, + D3D12_COMMAND_LIST_TYPE_DIRECT); + auto cq_handle = gst_d3d12_command_queue_get_handle (cq); + IUnknown *cq_list[] = { cq_handle }; + + static const D3D_FEATURE_LEVEL feature_levels[] = { + D3D_FEATURE_LEVEL_12_1, + D3D_FEATURE_LEVEL_12_0, + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + + auto hr = D3D11On12CreateDevice (device, D3D11_CREATE_DEVICE_BGRA_SUPPORT, + feature_levels, G_N_ELEMENTS (feature_levels), cq_list, 1, 0, + &priv->device11, &priv->d3d11_context, nullptr); + if (!gst_d3d12_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't create d3d11 device"); + return FALSE; + } + + priv->device11.As (&priv->device11on12); + priv->ca_pool = gst_d3d12_command_allocator_pool_new (device, + D3D12_COMMAND_LIST_TYPE_DIRECT); + + GST_DEBUG_OBJECT (self, "Resource prepared"); + + priv->prepared = TRUE; + + return TRUE; +} + +GstDWriteRender * +gst_dwrite_d3d12_render_new (GstD3D12Device * device, const GstVideoInfo * info, + ID2D1Factory * d2d_factory, IDWriteFactory * dwrite_factory) +{ + auto self = (GstDWriteD3D12Render *) + g_object_new (GST_TYPE_DWRITE_D3D12_RENDER, nullptr); + gst_object_ref_sink (self); + + auto priv = self->priv; + priv->device = (GstD3D12Device *) gst_object_ref (device); + priv->info = *info; + + auto format = GST_VIDEO_INFO_FORMAT (info); + switch (format) { + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRx: + case GST_VIDEO_FORMAT_RGBx: + case GST_VIDEO_FORMAT_VUYA: + case GST_VIDEO_FORMAT_RGBA64_LE: + case GST_VIDEO_FORMAT_RGB10A2_LE: + priv->direct_blend = TRUE; + gst_video_info_set_format (&priv->blend_info, + format, info->width, info->height); + break; + default: + priv->direct_blend = FALSE; + if (GST_VIDEO_INFO_COMP_DEPTH (info, 0) > 8) { + gst_video_info_set_format (&priv->blend_info, + GST_VIDEO_FORMAT_RGBA64_LE, info->width, info->height); + } else { + gst_video_info_set_format (&priv->blend_info, + GST_VIDEO_FORMAT_BGRA, info->width, info->height); + } + break; + } + + if (!gst_dwrite_d3d12_render_prepare (self)) { + gst_object_unref (self); + return nullptr; + } + + priv->d2d_factory = d2d_factory; + priv->dwrite_factory = dwrite_factory; + IGstDWriteTextRenderer::CreateInstance (dwrite_factory, &priv->renderer); + + return GST_DWRITE_RENDER (self); +} diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d12.h b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d12.h new file mode 100644 index 0000000000..a78790af8e --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriterender_d3d12.h @@ -0,0 +1,36 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include "gstdwriterender.h" + +G_BEGIN_DECLS + +#define GST_TYPE_DWRITE_D3D12_RENDER (gst_dwrite_d3d12_render_get_type ()) +G_DECLARE_FINAL_TYPE (GstDWriteD3D12Render, + gst_dwrite_d3d12_render, GST, DWRITE_D3D12_RENDER, GstDWriteRender); + +GstDWriteRender * gst_dwrite_d3d12_render_new (GstD3D12Device * device, + const GstVideoInfo * info, + ID2D1Factory * d2d_factory, + IDWriteFactory * dwrite_factory); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/dwrite/meson.build b/subprojects/gst-plugins-bad/sys/dwrite/meson.build index b7296f695d..4fcf7a9e9c 100644 --- a/subprojects/gst-plugins-bad/sys/dwrite/meson.build +++ b/subprojects/gst-plugins-bad/sys/dwrite/meson.build @@ -8,6 +8,9 @@ dwrite_sources = [ 'gstdwritebitmappool.cpp', 'gstdwriteclockoverlay.cpp', 'gstdwriteoverlayobject.cpp', + 'gstdwriterender_bitmap.cpp', + 'gstdwriterender_d3d11.cpp', + 'gstdwriterender.cpp', 'gstdwritesubtitlemux.cpp', 'gstdwritesubtitleoverlay.cpp', 'gstdwritetextoverlay.cpp', @@ -16,6 +19,7 @@ dwrite_sources = [ ] extra_args = ['-DGST_USE_UNSTABLE_API'] +extra_deps = [] dwrite_option = get_option('dwrite') if host_system != 'windows' or dwrite_option.disabled() @@ -63,12 +67,19 @@ endif subdir('libcaption') +if gstd3d12_dep.found() and cc.has_header('d3d11on12.h') + extra_args += ['-DHAVE_GST_D3D12'] + extra_deps += [gstd3d12_dep] + dwrite_sources += ['gstdwriterender_d3d12.cpp'] +endif + gstdwrite = library('gstdwrite', dwrite_sources, c_args : gst_plugins_bad_args + extra_args, cpp_args: gst_plugins_bad_args + extra_args, include_directories : [configinc], - dependencies : [gstbase_dep, gstvideo_dep, gstd3d11_dep, d2d_dep, dwrite_lib, dwrite_libcaption_dep], + dependencies : [gstbase_dep, gstvideo_dep, gstd3d11_dep, d2d_dep, dwrite_lib, + dwrite_libcaption_dep] + extra_deps, install : true, install_dir : plugins_install_dir, )