d3d12decoder: Add support for D3D11 interop

As a short-term solution before full d3d12 rendering feature,
copy decoded d3d12 texture to shared d3d11 texture in order to use
existing various d3d11 implementations such as conversion, resizing,
and videosink.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5356>
This commit is contained in:
Seungha Yang 2023-09-20 01:37:30 +09:00 committed by GStreamer Marge Bot
parent 2aa88033b2
commit d9a89cce06
12 changed files with 163 additions and 42 deletions

View file

@ -276,7 +276,7 @@ gst_d3d12_av1_dec_output_picture (GstDxvaAV1Decoder * decoder,
void
gst_d3d12_av1_dec_register (GstPlugin * plugin, GstD3D12Device * device,
ID3D12VideoDevice * video_device, guint rank)
ID3D12VideoDevice * video_device, guint rank, gboolean d3d11_interop)
{
GType type;
gchar *type_name;
@ -299,7 +299,7 @@ gst_d3d12_av1_dec_register (GstPlugin * plugin, GstD3D12Device * device,
type_info.class_data =
gst_d3d12_decoder_check_feature_support (device, video_device,
GST_DXVA_CODEC_AV1);
GST_DXVA_CODEC_AV1, d3d11_interop);
if (!type_info.class_data)
return;

View file

@ -26,7 +26,8 @@ G_BEGIN_DECLS
void gst_d3d12_av1_dec_register (GstPlugin * plugin,
GstD3D12Device * device,
ID3D12VideoDevice * video_device,
guint rank);
guint rank,
gboolean d3d11_interop);
G_END_DECLS

View file

@ -183,6 +183,13 @@ struct GstD3D12DecoderPicture : public GstMiniObject
#define GST_TYPE_D3D12_DECODER_PICTURE (gst_d3d12_decoder_picture_get_type ())
GST_DEFINE_MINI_OBJECT_TYPE (GstD3D12DecoderPicture, gst_d3d12_decoder_picture);
typedef enum
{
GST_D3D12_DECODER_OUTPUT_D3D12,
GST_D3D12_DECODER_OUTPUT_D3D11,
GST_D3D12_DECODER_OUTPUT_SYSTEM,
} GstD3D12DecoderOutputType;
struct GstD3D12DecoderPrivate
{
~GstD3D12DecoderPrivate()
@ -228,7 +235,7 @@ struct GstD3D12DecoderPrivate
DXGI_FORMAT decoder_format = DXGI_FORMAT_UNKNOWN;
gboolean reference_only = FALSE;
gboolean use_array_of_texture = FALSE;
gboolean downstream_supports_d3d12 = FALSE;
GstD3D12DecoderOutputType output_type = GST_D3D12_DECODER_OUTPUT_SYSTEM;
guint downstream_min_buffers = 0;
gboolean need_crop = FALSE;
gboolean use_crop_meta = FALSE;
@ -279,6 +286,7 @@ struct _GstD3D12Decoder
GstDxvaCodec codec;
GstD3D12Device *device;
GstD3D11Device *d3d11_device;
gint64 adapter_luid;
CRITICAL_SECTION context_lock;
@ -453,6 +461,7 @@ gst_d3d12_decoder_close (GstD3D12Decoder * decoder)
}
gst_clear_object (&decoder->device);
gst_clear_object (&decoder->d3d11_device);
return TRUE;
}
@ -1055,7 +1064,7 @@ gst_d3d12_decoder_can_direct_render (GstD3D12Decoder * self,
if (videodec->input_segment.rate < 0)
return FALSE;
if (!priv->downstream_supports_d3d12)
if (priv->output_type != GST_D3D12_DECODER_OUTPUT_D3D12)
return FALSE;
if (display_width != GST_VIDEO_INFO_WIDTH (&priv->info) ||
@ -1188,6 +1197,7 @@ gst_d3d12_decoder_output_picture (GstD3D12Decoder * decoder,
ID3D12Resource *out_resource = nullptr;
UINT out_subresource[2];
GstD3D12Fence *out_fence = priv->fence;
ComPtr < ID3D12Resource > shared_resource;
ret = gst_video_decoder_allocate_output_frame (videodec, frame);
if (ret != GST_FLOW_OK) {
@ -1209,6 +1219,25 @@ gst_d3d12_decoder_output_picture (GstD3D12Decoder * decoder,
GST_MINI_OBJECT_FLAG_UNSET (dmem,
GST_D3D12_MEMORY_TRANSFER_NEED_UPLOAD);
}
} else if (gst_is_d3d11_memory (mem) &&
GST_D3D11_MEMORY_CAST (mem)->device == decoder->d3d11_device) {
HANDLE resource_handle;
if (gst_d3d11_memory_get_nt_handle (GST_D3D11_MEMORY_CAST (mem),
&resource_handle)) {
ID3D12Device *device_handle =
gst_d3d12_device_get_device_handle (decoder->device);
hr = device_handle->OpenSharedHandle (resource_handle,
IID_PPV_ARGS (&shared_resource));
if (gst_d3d12_result (hr, decoder->device)) {
out_resource = shared_resource.Get ();
out_subresource[0] = 0;
out_subresource[1] = 1;
GST_MINI_OBJECT_FLAG_SET (mem,
GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD);
GST_MINI_OBJECT_FLAG_UNSET (mem,
GST_D3D11_MEMORY_TRANSFER_NEED_UPLOAD);
}
}
}
if (!out_resource && !gst_d3d12_decoder_ensure_staging_texture (decoder)) {
@ -1235,6 +1264,9 @@ gst_d3d12_decoder_output_picture (GstD3D12Decoder * decoder,
goto error;
}
if (shared_resource)
gst_d3d11_device_lock (decoder->d3d11_device);
/* simultaneous access must be enabled already, so,barrier is not needed */
for (guint i = 0; i < 2; i++) {
D3D12_TEXTURE_COPY_LOCATION src =
@ -1275,6 +1307,8 @@ gst_d3d12_decoder_output_picture (GstD3D12Decoder * decoder,
hr = priv->copy_cl->Close ();
if (!gst_d3d12_result (hr, decoder->device)) {
GST_ERROR_OBJECT (videodec, "Couldn't record copy command");
if (shared_resource)
gst_d3d11_device_unlock (decoder->d3d11_device);
ret = GST_FLOW_ERROR;
goto error;
}
@ -1286,6 +1320,8 @@ gst_d3d12_decoder_output_picture (GstD3D12Decoder * decoder,
hr = copy_queue->Signal (gst_d3d12_fence_get_handle (out_fence),
fence_value);
if (!gst_d3d12_result (hr, decoder->device)) {
if (shared_resource)
gst_d3d11_device_unlock (decoder->d3d11_device);
ret = GST_FLOW_ERROR;
goto error;
}
@ -1319,6 +1355,9 @@ gst_d3d12_decoder_output_picture (GstD3D12Decoder * decoder,
priv->staging->Unmap (0, nullptr);
gst_video_frame_unmap (&vframe);
} else if (shared_resource) {
gst_d3d12_fence_wait (out_fence);
gst_d3d11_device_unlock (decoder->d3d11_device);
}
}
@ -1491,6 +1530,7 @@ gst_d3d12_decoder_negotiate (GstD3D12Decoder * decoder,
GstStructure *s;
const gchar *str;
gboolean d3d12_supported = FALSE;
gboolean d3d11_supported = FALSE;
peer_caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (videodec));
GST_DEBUG_OBJECT (videodec, "Allowed caps %" GST_PTR_FORMAT, peer_caps);
@ -1512,6 +1552,9 @@ gst_d3d12_decoder_negotiate (GstD3D12Decoder * decoder,
if (gst_caps_features_contains (features,
GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY)) {
d3d12_supported = TRUE;
} else if (gst_caps_features_contains (features,
GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) {
d3d11_supported = TRUE;
}
}
}
@ -1548,13 +1591,19 @@ gst_d3d12_decoder_negotiate (GstD3D12Decoder * decoder,
g_clear_pointer (&priv->output_state, gst_video_codec_state_unref);
priv->output_state = state;
priv->output_type = GST_D3D12_DECODER_OUTPUT_SYSTEM;
if (d3d12_supported) {
gst_caps_set_features (state->caps, 0,
gst_caps_features_new_single (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY));
priv->output_type = GST_D3D12_DECODER_OUTPUT_D3D12;
} else if (d3d11_supported
&& gst_d3d11_ensure_element_data_for_adapter_luid (GST_ELEMENT (videodec),
decoder->adapter_luid, &decoder->d3d11_device)) {
gst_caps_set_features (state->caps, 0,
gst_caps_features_new_single (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY));
priv->output_type = GST_D3D12_DECODER_OUTPUT_D3D11;
}
priv->downstream_supports_d3d12 = d3d12_supported;
return gst_d3d12_decoder_create (decoder);
}
@ -1568,7 +1617,6 @@ gst_d3d12_decoder_decide_allocation (GstD3D12Decoder * decoder,
guint n, size, min = 0, max = 0;
GstVideoInfo vinfo = { 0, };
GstStructure *config;
gboolean use_d3d12_pool;
g_return_val_if_fail (GST_IS_D3D12_DECODER (decoder), FALSE);
g_return_val_if_fail (GST_IS_VIDEO_DECODER (videodec), FALSE);
@ -1586,8 +1634,7 @@ gst_d3d12_decoder_decide_allocation (GstD3D12Decoder * decoder,
return FALSE;
}
use_d3d12_pool = priv->downstream_supports_d3d12;
if (use_d3d12_pool) {
if (priv->output_type == GST_D3D12_DECODER_OUTPUT_D3D12) {
priv->use_crop_meta =
gst_query_find_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE,
nullptr);
@ -1600,25 +1647,46 @@ gst_d3d12_decoder_decide_allocation (GstD3D12Decoder * decoder,
if (n > 0)
gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
if (pool && use_d3d12_pool) {
if (!GST_IS_D3D12_BUFFER_POOL (pool)) {
GST_DEBUG_OBJECT (videodec,
"Downstream pool is not d3d12, will create new one");
gst_clear_object (&pool);
} else {
GstD3D12BufferPool *dpool = GST_D3D12_BUFFER_POOL (pool);
if (dpool->device != decoder->device) {
GST_DEBUG_OBJECT (videodec, "Different device, will create new one");
if (pool) {
if (priv->output_type == GST_D3D12_DECODER_OUTPUT_D3D12) {
if (!GST_IS_D3D12_BUFFER_POOL (pool)) {
GST_DEBUG_OBJECT (videodec,
"Downstream pool is not d3d12, will create new one");
gst_clear_object (&pool);
} else {
GstD3D12BufferPool *dpool = GST_D3D12_BUFFER_POOL (pool);
if (dpool->device != decoder->device) {
GST_DEBUG_OBJECT (videodec, "Different device, will create new one");
gst_clear_object (&pool);
}
}
} else if (priv->output_type == GST_D3D12_DECODER_OUTPUT_D3D11) {
if (!GST_IS_D3D11_BUFFER_POOL (pool)) {
GST_DEBUG_OBJECT (videodec,
"Downstream pool is not d3d11, will create new one");
gst_clear_object (&pool);
} else {
GstD3D11BufferPool *dpool = GST_D3D11_BUFFER_POOL (pool);
if (dpool->device != decoder->d3d11_device) {
GST_DEBUG_OBJECT (videodec, "Different device, will create new one");
gst_clear_object (&pool);
}
}
}
}
if (!pool) {
if (use_d3d12_pool)
pool = gst_d3d12_buffer_pool_new (decoder->device);
else
pool = gst_video_buffer_pool_new ();
switch (priv->output_type) {
case GST_D3D12_DECODER_OUTPUT_D3D12:
pool = gst_d3d12_buffer_pool_new (decoder->device);
break;
case GST_D3D12_DECODER_OUTPUT_D3D11:
pool = gst_d3d11_buffer_pool_new (decoder->d3d11_device);
break;
default:
pool = gst_video_buffer_pool_new ();
break;
}
size = (guint) vinfo.size;
}
@ -1627,7 +1695,7 @@ gst_d3d12_decoder_decide_allocation (GstD3D12Decoder * decoder,
gst_buffer_pool_config_set_params (config, outcaps, size, min, max);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
if (use_d3d12_pool) {
if (priv->output_type == GST_D3D12_DECODER_OUTPUT_D3D12) {
GstD3D12AllocationParams *params;
GstVideoAlignment align;
gint width, height;
@ -1690,6 +1758,17 @@ gst_d3d12_decoder_decide_allocation (GstD3D12Decoder * decoder,
/* We will not use downstream pool for decoding, and therefore preallocation
* is unnecessary. So, Non-zero min buffer will be a waste of GPU memory */
min = 0;
} else if (priv->output_type == GST_D3D12_DECODER_OUTPUT_D3D11) {
GstD3D11AllocationParams *params;
const guint bind_flags = D3D11_BIND_SHADER_RESOURCE |
D3D11_BIND_RENDER_TARGET;
const guint misc_flags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE |
D3D11_RESOURCE_MISC_SHARED;
params = gst_d3d11_allocation_params_new (decoder->d3d11_device, &vinfo,
GST_D3D11_ALLOCATION_FLAG_DEFAULT, bind_flags, misc_flags);
gst_buffer_pool_config_set_d3d11_allocation_params (config, params);
gst_d3d11_allocation_params_free (params);
}
gst_buffer_pool_set_config (pool, config);
@ -1744,6 +1823,8 @@ gst_d3d12_decoder_set_context (GstD3D12Decoder * decoder, GstElement * element,
gst_d3d12_handle_set_context_for_adapter_luid (element,
context, decoder->adapter_luid, &decoder->device);
gst_d3d11_handle_set_context_for_adapter_luid (element,
context, decoder->adapter_luid, &decoder->d3d11_device);
}
gboolean
@ -1754,7 +1835,12 @@ gst_d3d12_decoder_handle_query (GstD3D12Decoder * decoder, GstElement * element,
return FALSE;
GstD3D12CSLockGuard lk (&decoder->context_lock);
return gst_d3d12_handle_context_query (element, query, decoder->device);
if (gst_d3d12_handle_context_query (element, query, decoder->device) ||
gst_d3d11_handle_context_query (element, query, decoder->d3d11_device)) {
return TRUE;
}
return FALSE;
}
enum
@ -1800,7 +1886,8 @@ gst_d3d12_decoder_get_profiles (const GUID & profile,
GstD3D12DecoderClassData *
gst_d3d12_decoder_check_feature_support (GstD3D12Device * device,
ID3D12VideoDevice * video_device, GstDxvaCodec codec)
ID3D12VideoDevice * video_device, GstDxvaCodec codec,
gboolean d3d11_interop)
{
HRESULT hr;
GstDxvaResolution max_resolution = { 0, 0 };
@ -1963,6 +2050,12 @@ gst_d3d12_decoder_check_feature_support (GstD3D12Device * device,
GstCaps *src_caps = gst_caps_copy (raw_caps);
gst_caps_set_features_simple (src_caps,
gst_caps_features_new_single (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY));
if (d3d11_interop) {
GstCaps *d3d11_caps = gst_caps_copy (raw_caps);
gst_caps_set_features_simple (d3d11_caps,
gst_caps_features_new_single (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY));
gst_caps_append (src_caps, d3d11_caps);
}
gst_caps_append (src_caps, raw_caps);
gint max_res = MAX (max_resolution.width, max_resolution.height);

View file

@ -23,6 +23,7 @@
#include <gst/video/video.h>
#include <gst/codecs/gstcodecpicture.h>
#include <gst/dxva/gstdxva.h>
#include <gst/d3d11/gstd3d11.h>
#include "gstd3d12_fwd.h"
G_BEGIN_DECLS
@ -162,7 +163,8 @@ gboolean gst_d3d12_decoder_handle_query (GstD3D12Decoder * decoder
/* Utils for element registration */
GstD3D12DecoderClassData * gst_d3d12_decoder_check_feature_support (GstD3D12Device * device,
ID3D12VideoDevice * video_device,
GstDxvaCodec codec);
GstDxvaCodec codec,
gboolean d3d11_interop);
void gst_d3d12_decoder_class_data_fill_subclass_data (GstD3D12DecoderClassData * data,
GstD3D12DecoderSubClassData * subclass_data);

View file

@ -276,7 +276,7 @@ gst_d3d12_h264_dec_output_picture (GstDxvaH264Decoder * decoder,
void
gst_d3d12_h264_dec_register (GstPlugin * plugin, GstD3D12Device * device,
ID3D12VideoDevice * video_device, guint rank)
ID3D12VideoDevice * video_device, guint rank, gboolean d3d11_interop)
{
GType type;
gchar *type_name;
@ -299,7 +299,7 @@ gst_d3d12_h264_dec_register (GstPlugin * plugin, GstD3D12Device * device,
type_info.class_data =
gst_d3d12_decoder_check_feature_support (device, video_device,
GST_DXVA_CODEC_H264);
GST_DXVA_CODEC_H264, d3d11_interop);
if (!type_info.class_data)
return;

View file

@ -26,7 +26,8 @@ G_BEGIN_DECLS
void gst_d3d12_h264_dec_register (GstPlugin * plugin,
GstD3D12Device * device,
ID3D12VideoDevice * video_device,
guint rank);
guint rank,
gboolean d3d11_interop);
G_END_DECLS

View file

@ -265,7 +265,7 @@ gst_d3d12_h265_dec_output_picture (GstDxvaH265Decoder * decoder,
void
gst_d3d12_h265_dec_register (GstPlugin * plugin, GstD3D12Device * device,
ID3D12VideoDevice * video_device, guint rank)
ID3D12VideoDevice * video_device, guint rank, gboolean d3d11_interop)
{
GType type;
gchar *type_name;
@ -288,7 +288,7 @@ gst_d3d12_h265_dec_register (GstPlugin * plugin, GstD3D12Device * device,
type_info.class_data =
gst_d3d12_decoder_check_feature_support (device, video_device,
GST_DXVA_CODEC_H265);
GST_DXVA_CODEC_H265, d3d11_interop);
if (!type_info.class_data)
return;

View file

@ -26,7 +26,8 @@ G_BEGIN_DECLS
void gst_d3d12_h265_dec_register (GstPlugin * plugin,
GstD3D12Device * device,
ID3D12VideoDevice * video_device,
guint rank);
guint rank,
gboolean d3d11_interop);
G_END_DECLS

View file

@ -276,7 +276,7 @@ gst_d3d12_vp9_dec_output_picture (GstDxvaVp9Decoder * decoder,
void
gst_d3d12_vp9_dec_register (GstPlugin * plugin, GstD3D12Device * device,
ID3D12VideoDevice * video_device, guint rank)
ID3D12VideoDevice * video_device, guint rank, gboolean d3d11_interop)
{
GType type;
gchar *type_name;
@ -299,7 +299,7 @@ gst_d3d12_vp9_dec_register (GstPlugin * plugin, GstD3D12Device * device,
type_info.class_data =
gst_d3d12_decoder_check_feature_support (device, video_device,
GST_DXVA_CODEC_VP9);
GST_DXVA_CODEC_VP9, d3d11_interop);
if (!type_info.class_data)
return;

View file

@ -26,7 +26,8 @@ G_BEGIN_DECLS
void gst_d3d12_vp9_dec_register (GstPlugin * plugin,
GstD3D12Device * device,
ID3D12VideoDevice * video_device,
guint rank);
guint rank,
gboolean d3d11_interop);
G_END_DECLS

View file

@ -26,7 +26,7 @@ endif
d3d12_lib = cc.find_library('d3d12', required : d3d12_option)
dxgi_lib = cc.find_library('dxgi', required : d3d12_option)
if not gstdxva_dep.found() or not d3d12_lib.found() or not dxgi_lib.found()
if not gstdxva_dep.found() or not gstd3d11_dep.found() or not d3d12_lib.found() or not dxgi_lib.found()
if d3d12_option.enabled()
error('The d3d12 was enabled explicitly, but required GstD3D11 dependencies were not found.')
endif
@ -74,7 +74,7 @@ gstd3d12 = library('gstd3d12',
cpp_args: gst_plugins_bad_args + extra_args,
include_directories : [configinc],
dependencies : [gstbase_dep, gstvideo_dep, gstcodecs_dep,
gstdxva_dep, d3d12_lib, dxgi_lib],
gstdxva_dep, gstd3d11_dep, d3d12_lib, dxgi_lib],
install : true,
install_dir : plugins_install_dir,
)

View file

@ -30,6 +30,7 @@
#include "gstd3d12av1dec.h"
#include <wrl.h>
#include <d3d11_4.h>
/* *INDENT-OFF* */
using namespace Microsoft::WRL;
@ -66,7 +67,10 @@ plugin_init (GstPlugin * plugin)
GstD3D12Device *device = nullptr;
ID3D12Device *device_handle;
ComPtr < ID3D12VideoDevice > video_device;
GstD3D11Device *d3d11_device;
HRESULT hr;
gint64 luid;
gboolean d3d11_interop = FALSE;
device = gst_d3d12_device_new (i);
if (!device)
@ -79,14 +83,32 @@ plugin_init (GstPlugin * plugin)
continue;
}
g_object_get (device, "adapter-luid", &luid, nullptr);
d3d11_device = gst_d3d11_device_new_for_adapter_luid (luid,
D3D11_CREATE_DEVICE_BGRA_SUPPORT);
/* Enable d3d11 interop only if extended NV12 shared texture feature
* is supported by GPU */
if (d3d11_device) {
ID3D11Device *d3d11_handle =
gst_d3d11_device_get_device_handle (d3d11_device);
D3D11_FEATURE_DATA_D3D11_OPTIONS4 option4 = { FALSE };
hr = d3d11_handle->CheckFeatureSupport (D3D11_FEATURE_D3D11_OPTIONS4,
&option4, sizeof (D3D11_FEATURE_DATA_D3D11_OPTIONS4));
if (SUCCEEDED (hr) && option4.ExtendedNV12SharedTextureSupported)
d3d11_interop = TRUE;
gst_object_unref (d3d11_device);
}
gst_d3d12_h264_dec_register (plugin, device, video_device.Get (),
GST_RANK_NONE);
GST_RANK_NONE, d3d11_interop);
gst_d3d12_h265_dec_register (plugin, device, video_device.Get (),
GST_RANK_NONE);
GST_RANK_NONE, d3d11_interop);
gst_d3d12_vp9_dec_register (plugin, device, video_device.Get (),
GST_RANK_NONE);
GST_RANK_NONE, d3d11_interop);
gst_d3d12_av1_dec_register (plugin, device, video_device.Get (),
GST_RANK_NONE);
GST_RANK_NONE, d3d11_interop);
gst_object_unref (device);
}