mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-12-26 02:00:33 +00:00
nvdecoder: Add support for D3D12 output
Enable D3D12 output if device can support D3D12 interop Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7529>
This commit is contained in:
Seungha Yang
parent
16e94b7fc3
commit
d9114db5d0
3 changed files with 227 additions and 8 deletions
|
|
@ -50,6 +50,11 @@
|
|||
#include <gst/gl/gstglfuncs.h>
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
#include <gst/d3d12/gstd3d12.h>
|
||||
#include "gstcudainterop_d3d12.h"
|
||||
#endif
|
||||
|
||||
#ifdef G_OS_WIN32
|
||||
#include <gst/d3d11/gstd3d11.h>
|
||||
#endif
|
||||
|
|
@ -58,6 +63,7 @@
|
|||
#include <gst/cuda/gstcuda-private.h>
|
||||
#include "gstnvdecoder.h"
|
||||
#include "gstcudaconverter.h"
|
||||
#include "gstnvcodecutils.h"
|
||||
#include <string.h>
|
||||
#include <string>
|
||||
#include <set>
|
||||
|
|
@ -80,6 +86,7 @@ typedef enum
|
|||
GST_NV_DECODER_OUTPUT_TYPE_GL = (1 << 1),
|
||||
GST_NV_DECODER_OUTPUT_TYPE_CUDA = (1 << 2),
|
||||
GST_NV_DECODER_OUTPUT_TYPE_D3D11 = (1 << 3),
|
||||
GST_NV_DECODER_OUTPUT_TYPE_D3D12 = (1 << 4),
|
||||
} GstNvDecoderOutputType;
|
||||
|
||||
struct _GstNvDecoder
|
||||
|
|
@ -118,6 +125,11 @@ struct _GstNvDecoder
|
|||
GstBuffer *convert_buf;
|
||||
GstVideoInfo output_info;
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
GstCudaD3D12Interop *interop_12;
|
||||
GstD3D12Device *device_12;
|
||||
#endif
|
||||
|
||||
GstNvDecoderOutputType output_type;
|
||||
};
|
||||
|
||||
|
|
@ -264,6 +276,11 @@ gst_nv_decoder_close (GstNvDecoder * decoder)
|
|||
gst_clear_buffer (&decoder->convert_buf);
|
||||
gst_clear_buffer (&decoder->export_buf);
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
gst_clear_object (&decoder->interop_12);
|
||||
gst_clear_object (&decoder->device_12);
|
||||
#endif
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
|
|
@ -296,6 +313,9 @@ gst_nv_decoder_configure (GstNvDecoder * decoder, cudaVideoCodec codec,
|
|||
gst_clear_buffer (&decoder->export_buf);
|
||||
gst_clear_buffer (&decoder->convert_buf);
|
||||
gst_clear_object (&decoder->converter);
|
||||
#ifdef HAVE_GST_D3D12
|
||||
gst_clear_object (&decoder->interop_12);
|
||||
#endif
|
||||
|
||||
format = GST_VIDEO_INFO_FORMAT (info);
|
||||
if (decoder->info.finfo)
|
||||
|
|
@ -730,6 +750,50 @@ gst_nv_decoder_copy_frame_to_d3d11 (GstNvDecoder * self,
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
static GstFlowReturn
|
||||
gst_nv_decoder_copy_frame_to_d3d12 (GstNvDecoder * self,
|
||||
GstNvDecSurface * surface, GstBuffer * buffer)
|
||||
{
|
||||
if (!self->interop_12) {
|
||||
GST_ERROR_OBJECT (self, "D3D12 output is not configured");
|
||||
return GST_FLOW_ERROR;
|
||||
}
|
||||
|
||||
GstMemory *mem;
|
||||
auto ret = gst_nv_dec_object_export_surface (self->object,
|
||||
surface, self->stream, &mem);
|
||||
if (ret != GST_FLOW_OK) {
|
||||
GST_WARNING_OBJECT (self, "Couldn't export surface");
|
||||
gst_nv_dec_object_unmap_surface (self->object, surface);
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto cmem = GST_CUDA_MEMORY_CAST (mem);
|
||||
if (!self->export_buf) {
|
||||
self->export_buf = gst_buffer_new ();
|
||||
gst_buffer_add_video_meta_full (self->export_buf, GST_VIDEO_FRAME_FLAG_NONE,
|
||||
GST_VIDEO_INFO_FORMAT (&self->info),
|
||||
GST_VIDEO_INFO_WIDTH (&self->info),
|
||||
GST_VIDEO_INFO_HEIGHT (&self->info),
|
||||
GST_VIDEO_INFO_N_PLANES (&self->info),
|
||||
cmem->info.offset, cmem->info.stride);
|
||||
}
|
||||
|
||||
gst_buffer_append_memory (self->export_buf, mem);
|
||||
auto download_ret = gst_cuda_d3d12_interop_download_async (self->interop_12,
|
||||
buffer, self->export_buf, self->stream);
|
||||
gst_buffer_remove_all_memory (self->export_buf);
|
||||
|
||||
if (!download_ret) {
|
||||
GST_ERROR_OBJECT (self, "Couldn't convert frame");
|
||||
return GST_FLOW_ERROR;
|
||||
}
|
||||
|
||||
return GST_FLOW_OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
static GstFlowReturn
|
||||
gst_nv_decoder_copy_frame_to_system (GstNvDecoder * decoder,
|
||||
GstNvDecSurface * surface, GstBuffer * buffer)
|
||||
|
|
@ -946,6 +1010,13 @@ gst_nv_decoder_output_picture (GstNvDecoder * decoder,
|
|||
ret = gst_nv_decoder_copy_frame_to_system (decoder,
|
||||
surface, frame->output_buffer);
|
||||
break;
|
||||
#ifdef HAVE_GST_D3D12
|
||||
case GST_NV_DECODER_OUTPUT_TYPE_D3D12:
|
||||
ret = gst_nv_decoder_copy_frame_to_d3d12 (decoder, surface,
|
||||
frame->output_buffer);
|
||||
need_unmap = FALSE;
|
||||
break;
|
||||
#endif
|
||||
#ifdef G_OS_WIN32
|
||||
case GST_NV_DECODER_OUTPUT_TYPE_D3D11:
|
||||
ret = gst_nv_decoder_copy_frame_to_d3d11 (decoder, surface,
|
||||
|
|
@ -1185,8 +1256,8 @@ const GstNvdecoderCodecMap codec_map_list[] = {
|
|||
};
|
||||
|
||||
gboolean
|
||||
gst_nv_decoder_check_device_caps (CUcontext cuda_ctx, cudaVideoCodec codec,
|
||||
GstCaps ** sink_template, GstCaps ** src_template)
|
||||
gst_nv_decoder_check_device_caps (GstCudaContext * context,
|
||||
cudaVideoCodec codec, GstCaps ** sink_template, GstCaps ** src_template)
|
||||
{
|
||||
CUresult cuda_ret;
|
||||
guint max_width = 0, min_width = G_MAXINT;
|
||||
|
|
@ -1213,6 +1284,7 @@ gst_nv_decoder_check_device_caps (CUcontext cuda_ctx, cudaVideoCodec codec,
|
|||
gboolean ret = FALSE;
|
||||
std::set < std::string > formats;
|
||||
std::set < std::string > planar_formats;
|
||||
CUcontext cuda_ctx = gst_cuda_context_get_handle (context);
|
||||
#ifdef G_OS_WIN32
|
||||
gboolean is_stateless = FALSE;
|
||||
|
||||
|
|
@ -1419,6 +1491,21 @@ gst_nv_decoder_check_device_caps (CUcontext cuda_ctx, cudaVideoCodec codec,
|
|||
gst_caps_features_new_single_static_str
|
||||
(GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY));
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
if (is_stateless && gst_nvcodec_is_windows_10_or_greater ()) {
|
||||
gboolean have_interop = FALSE;
|
||||
g_object_get (context, "external-resource-interop",
|
||||
&have_interop, nullptr);
|
||||
if (have_interop) {
|
||||
auto d3d12_caps = gst_caps_copy (raw_caps);
|
||||
gst_caps_set_features_simple (d3d12_caps,
|
||||
gst_caps_features_new_single
|
||||
(GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY));
|
||||
gst_caps_append (src_templ, d3d12_caps);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef G_OS_WIN32
|
||||
if (is_stateless) {
|
||||
format_str.clear ();
|
||||
|
|
@ -1535,6 +1622,13 @@ gst_nv_decoder_handle_set_context (GstNvDecoder * decoder,
|
|||
&decoder->context)) {
|
||||
return;
|
||||
}
|
||||
#ifdef HAVE_GST_D3D12
|
||||
if (gst_d3d12_handle_set_context_for_adapter_luid (element, context,
|
||||
decoder->adapter_luid, &decoder->device_12)) {
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef G_OS_WIN32
|
||||
if (gst_d3d11_handle_set_context_for_adapter_luid (element, context,
|
||||
decoder->adapter_luid, (GstD3D11Device **) & decoder->d3d11_device)) {
|
||||
|
|
@ -1559,6 +1653,12 @@ gst_nv_decoder_handle_query (GstNvDecoder * decoder, GstElement * element,
|
|||
if (gst_cuda_handle_context_query (element, query, decoder->context))
|
||||
return TRUE;
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
if (gst_d3d12_handle_context_query (element, query, decoder->device_12)) {
|
||||
return TRUE;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef G_OS_WIN32
|
||||
if (gst_d3d11_handle_context_query (element,
|
||||
query, (GstD3D11Device *) decoder->d3d11_device)) {
|
||||
|
|
@ -1746,6 +1846,30 @@ gst_nv_decoder_ensure_d3d11_output (GstNvDecoder * self, GstElement * element)
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
static gboolean
|
||||
gst_nv_decoder_ensure_d3d12_output (GstNvDecoder * self, GstElement * element)
|
||||
{
|
||||
if (!gst_d3d12_ensure_element_data_for_adapter_luid (GST_ELEMENT (element),
|
||||
self->adapter_luid, &self->device_12)) {
|
||||
GST_WARNING_OBJECT (element, "D3D12 device is not available");
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
if (!self->interop_12) {
|
||||
self->interop_12 = gst_cuda_d3d12_interop_new (self->context,
|
||||
self->device_12, &self->info, FALSE);
|
||||
}
|
||||
|
||||
if (!self->interop_12) {
|
||||
GST_WARNING_OBJECT (element, "Couldn't create downloader");
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
#endif
|
||||
|
||||
gboolean
|
||||
gst_nv_decoder_negotiate (GstNvDecoder * decoder,
|
||||
GstVideoDecoder * videodec, GstVideoCodecState * input_state)
|
||||
|
|
@ -1786,6 +1910,13 @@ gst_nv_decoder_negotiate (GstNvDecoder * decoder,
|
|||
GST_DEBUG_OBJECT (videodec, "found CUDA memory feature");
|
||||
available_types |= GST_NV_DECODER_OUTPUT_TYPE_CUDA;
|
||||
}
|
||||
#ifdef HAVE_GST_D3D12
|
||||
if (features && gst_caps_features_contains (features,
|
||||
GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY)) {
|
||||
GST_DEBUG_OBJECT (videodec, "found D3D12 memory feature");
|
||||
available_types |= GST_NV_DECODER_OUTPUT_TYPE_D3D12;
|
||||
}
|
||||
#endif
|
||||
#ifdef G_OS_WIN32
|
||||
if (features && gst_caps_features_contains (features,
|
||||
GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) {
|
||||
|
|
@ -1815,6 +1946,8 @@ gst_nv_decoder_negotiate (GstNvDecoder * decoder,
|
|||
if (selected_type == GST_NV_DECODER_OUTPUT_TYPE_UNKNOWN) {
|
||||
if ((available_types & GST_NV_DECODER_OUTPUT_TYPE_CUDA) != 0)
|
||||
selected_type = GST_NV_DECODER_OUTPUT_TYPE_CUDA;
|
||||
else if ((available_types & GST_NV_DECODER_OUTPUT_TYPE_D3D12) != 0)
|
||||
selected_type = GST_NV_DECODER_OUTPUT_TYPE_D3D12;
|
||||
else if ((available_types & GST_NV_DECODER_OUTPUT_TYPE_D3D11) != 0)
|
||||
selected_type = GST_NV_DECODER_OUTPUT_TYPE_D3D11;
|
||||
else if ((available_types & GST_NV_DECODER_OUTPUT_TYPE_GL) != 0)
|
||||
|
|
@ -1829,6 +1962,15 @@ gst_nv_decoder_negotiate (GstNvDecoder * decoder,
|
|||
gst_clear_caps (&caps);
|
||||
}
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
if (decoder->output_type == GST_NV_DECODER_OUTPUT_TYPE_D3D12 &&
|
||||
!gst_nv_decoder_ensure_d3d12_output (decoder, GST_ELEMENT (videodec))) {
|
||||
GST_WARNING_OBJECT (videodec, "D3D12 setup failed");
|
||||
decoder->output_type = GST_NV_DECODER_OUTPUT_TYPE_SYSTEM;
|
||||
decoder->output_info = decoder->info;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef G_OS_WIN32
|
||||
if (decoder->output_type == GST_NV_DECODER_OUTPUT_TYPE_D3D11 &&
|
||||
!gst_nv_decoder_ensure_d3d11_output (decoder, GST_ELEMENT (videodec))) {
|
||||
|
|
@ -1861,6 +2003,12 @@ gst_nv_decoder_negotiate (GstNvDecoder * decoder,
|
|||
gst_caps_features_new_single_static_str
|
||||
(GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY));
|
||||
break;
|
||||
#ifdef HAVE_GST_D3D12
|
||||
case GST_NV_DECODER_OUTPUT_TYPE_D3D12:
|
||||
gst_caps_set_features (state->caps, 0,
|
||||
gst_caps_features_new_single (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY));
|
||||
break;
|
||||
#endif
|
||||
#ifdef G_OS_WIN32
|
||||
case GST_NV_DECODER_OUTPUT_TYPE_D3D11:
|
||||
gst_caps_set_features (state->caps, 0,
|
||||
|
|
@ -2095,6 +2243,75 @@ gst_nv_decoder_ensure_d3d11_pool (GstNvDecoder * self, GstElement * element,
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_GST_D3D12
|
||||
static gboolean
|
||||
gst_nv_decoder_ensure_d3d12_pool (GstNvDecoder * self, GstElement * element,
|
||||
GstQuery * query)
|
||||
{
|
||||
GstCaps *outcaps;
|
||||
GstBufferPool *pool = NULL;
|
||||
guint n, size, min = 0, max = 0;
|
||||
GstVideoInfo info;
|
||||
|
||||
gst_query_parse_allocation (query, &outcaps, nullptr);
|
||||
|
||||
if (!outcaps) {
|
||||
GST_DEBUG_OBJECT (element, "No output caps");
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
if (!gst_d3d12_ensure_element_data_for_adapter_luid (element,
|
||||
self->adapter_luid, &self->device_12)) {
|
||||
GST_ERROR_OBJECT (element, "Couldn't create d3d11 device");
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
gst_video_info_from_caps (&info, outcaps);
|
||||
n = gst_query_get_n_allocation_pools (query);
|
||||
if (n > 0)
|
||||
gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
|
||||
|
||||
/* create our own pool */
|
||||
if (pool) {
|
||||
if (!GST_IS_D3D12_BUFFER_POOL (pool)) {
|
||||
GST_DEBUG_OBJECT (self,
|
||||
"Downstream pool is not d3d11, will create new one");
|
||||
gst_clear_object (&pool);
|
||||
} else {
|
||||
auto dpool = GST_D3D12_BUFFER_POOL (pool);
|
||||
if (!gst_d3d12_device_is_equal (dpool->device, self->device_12)) {
|
||||
GST_DEBUG_OBJECT (element, "Different device, will create new one");
|
||||
gst_clear_object (&pool);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!pool) {
|
||||
pool = gst_d3d12_buffer_pool_new (self->device_12);
|
||||
size = info.size;
|
||||
}
|
||||
|
||||
auto config = gst_buffer_pool_get_config (pool);
|
||||
gst_buffer_pool_config_set_params (config, outcaps, size, min, max);
|
||||
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
|
||||
|
||||
gst_buffer_pool_set_config (pool, config);
|
||||
/* d3d12 buffer pool will update buffer size based on allocated texture,
|
||||
* get size from config again */
|
||||
config = gst_buffer_pool_get_config (pool);
|
||||
gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, nullptr);
|
||||
gst_structure_free (config);
|
||||
|
||||
if (n > 0)
|
||||
gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
|
||||
else
|
||||
gst_query_add_allocation_pool (query, pool, size, min, max);
|
||||
gst_object_unref (pool);
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
#endif
|
||||
|
||||
gboolean
|
||||
gst_nv_decoder_decide_allocation (GstNvDecoder * decoder,
|
||||
GstVideoDecoder * videodec, GstQuery * query)
|
||||
|
|
@ -2114,6 +2331,12 @@ gst_nv_decoder_decide_allocation (GstNvDecoder * decoder,
|
|||
query);
|
||||
break;
|
||||
#endif
|
||||
#ifdef HAVE_GST_D3D12
|
||||
case GST_NV_DECODER_OUTPUT_TYPE_D3D12:
|
||||
ret = gst_nv_decoder_ensure_d3d12_pool (decoder, GST_ELEMENT (videodec),
|
||||
query);
|
||||
break;
|
||||
#endif
|
||||
#ifdef HAVE_CUDA_GST_GL
|
||||
case GST_NV_DECODER_OUTPUT_TYPE_GL:
|
||||
ret = gst_nv_decoder_ensure_gl_pool (decoder, query);
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ void gst_nv_decoder_set_flushing (GstNvDecoder * decoder,
|
|||
void gst_nv_decoder_reset (GstNvDecoder * decoder);
|
||||
|
||||
/* utils for class registration */
|
||||
gboolean gst_nv_decoder_check_device_caps (CUcontext cuda_ctx,
|
||||
gboolean gst_nv_decoder_check_device_caps (GstCudaContext * context,
|
||||
cudaVideoCodec codec,
|
||||
GstCaps **sink_template,
|
||||
GstCaps **src_template);
|
||||
|
|
|
|||
|
|
@ -201,9 +201,6 @@ plugin_init (GstPlugin * plugin)
|
|||
|
||||
for (i = 0; i < dev_count; i++) {
|
||||
GstCudaContext *context = gst_cuda_context_new (i);
|
||||
#ifdef HAVE_NVCODEC_DGPU
|
||||
CUcontext cuda_ctx;
|
||||
#endif
|
||||
#if defined(G_OS_WIN32) || defined(HAVE_NVCODEC_DGPU)
|
||||
gint64 adapter_luid = 0;
|
||||
#endif
|
||||
|
|
@ -217,7 +214,6 @@ plugin_init (GstPlugin * plugin)
|
|||
#endif
|
||||
|
||||
#ifdef HAVE_NVCODEC_DGPU
|
||||
cuda_ctx = gst_cuda_context_get_handle (context);
|
||||
if (nvdec_available) {
|
||||
gint j;
|
||||
|
||||
|
|
@ -227,7 +223,7 @@ plugin_init (GstPlugin * plugin)
|
|||
cudaVideoCodec codec = (cudaVideoCodec) j;
|
||||
gboolean register_cuviddec = FALSE;
|
||||
|
||||
if (gst_nv_decoder_check_device_caps (cuda_ctx,
|
||||
if (gst_nv_decoder_check_device_caps (context,
|
||||
codec, &sink_template, &src_template)) {
|
||||
const gchar *codec_name = gst_cuda_video_codec_to_string (codec);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue