From f5ce4d10b16c4da924b033cd7a7a3af886f6925d Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Mon, 9 Sep 2024 00:31:21 +0900 Subject: [PATCH] nvencoder: Add support for d3d12 memory Use d3d12 -> cuda memory copy helper object in cuda mode encoder Part-of: --- .../sys/nvcodec/gstnvav1encoder.cpp | 18 ++++ .../sys/nvcodec/gstnvcodecutils.cpp | 39 +++++++++ .../sys/nvcodec/gstnvcodecutils.h | 2 + .../sys/nvcodec/gstnvencoder.cpp | 83 +++++++++++++++++++ .../sys/nvcodec/gstnvencoder.h | 5 ++ .../sys/nvcodec/gstnvh264encoder.cpp | 18 ++++ .../sys/nvcodec/gstnvh265encoder.cpp | 18 ++++ .../gst-plugins-bad/sys/nvcodec/meson.build | 1 + 8 files changed, 184 insertions(+) create mode 100644 subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.cpp diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvav1encoder.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvav1encoder.cpp index 4f639a234b..b067f80908 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvav1encoder.cpp +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvav1encoder.cpp @@ -1464,6 +1464,20 @@ gst_nv_av1_encoder_create_class_data (GstObject * device, gpointer session, if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) { gst_caps_set_features (sink_caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, nullptr)); +#ifdef HAVE_GST_D3D12 + if (gst_nvcodec_is_windows_10_or_greater ()) { + gboolean have_interop = FALSE; + g_object_get (device, + "external-resource-interop", &have_interop, nullptr); + if (have_interop) { + auto d3d12_caps = gst_caps_copy (system_caps); + gst_caps_set_features_simple (d3d12_caps, + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, + nullptr)); + gst_caps_append (sink_caps, d3d12_caps); + } + } +#endif #ifdef HAVE_CUDA_GST_GL GstCaps *gl_caps = gst_caps_copy (system_caps); gst_caps_set_features (gl_caps, 0, @@ -1485,8 +1499,12 @@ gst_nv_av1_encoder_create_class_data (GstObject * device, gpointer session, cdata->formats = g_list_append (cdata->formats, g_strdup (iter.c_str())); /* *INDENT-ON* */ +#ifdef G_OS_WIN32 if (device_mode == GST_NV_ENCODER_DEVICE_D3D11) g_object_get (device, "adapter-luid", &cdata->adapter_luid, nullptr); + else + g_object_get (device, "dxgi-adapter-luid", &cdata->adapter_luid, nullptr); +#endif if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) g_object_get (device, "cuda-device-id", &cdata->cuda_device_id, nullptr); diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.cpp new file mode 100644 index 0000000000..3350830dac --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.cpp @@ -0,0 +1,39 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstnvcodecutils.h" +#include + +gboolean +gst_nvcodec_is_windows_10_or_greater (void) +{ + static gboolean ret = FALSE; + +#ifdef G_OS_WIN32 + GST_CUDA_CALL_ONCE_BEGIN { + ret = g_win32_check_windows_version (10, 0, 0, G_WIN32_OS_ANY); + } GST_CUDA_CALL_ONCE_END; +#endif + + return ret; +} diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.h b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.h index 7e12d408fc..0726dc0801 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.h +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.h @@ -25,4 +25,6 @@ G_BEGIN_DECLS void gst_cuda_ipc_client_deinit (void); +gboolean gst_nvcodec_is_windows_10_or_greater (void); + G_END_DECLS \ No newline at end of file diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp index 57c2e65c09..7a8d565e30 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp @@ -37,6 +37,10 @@ #include #include "gstnvencobject.h" +#ifdef HAVE_GST_D3D12 +#include "gstcudainterop_d3d12.h" +#endif + #ifdef G_OS_WIN32 #include @@ -87,6 +91,11 @@ struct _GstNvEncoderPrivate gboolean gl_interop = FALSE; #endif +#ifdef HAVE_GST_D3D12 + GstD3D12Device *device_12 = nullptr; + GstCudaD3D12Interop *interop_12 = nullptr; +#endif + std::shared_ptr < GstNvEncObject > object; GstNvEncoderDeviceMode subclass_device_mode; @@ -285,6 +294,10 @@ gst_nv_encoder_set_context (GstElement * element, GstContext * context) if (priv->gl_display) gst_gl_display_filter_gl_api (priv->gl_display, SUPPORTED_GL_APIS); } +#endif +#ifdef HAVE_GST_D3D12 + gst_d3d12_handle_set_context_for_adapter_luid (element, + context, priv->dxgi_adapter_luid, &priv->device_12); #endif break; default: @@ -472,6 +485,10 @@ gst_nv_encoder_close (GstVideoEncoder * encoder) gst_clear_object (&priv->gl_context); gst_clear_object (&priv->other_gl_context); #endif +#ifdef HAVE_GST_D3D12 + gst_clear_object (&priv->interop_12); + gst_clear_object (&priv->device_12); +#endif return TRUE; } @@ -609,6 +626,12 @@ gst_nv_encoder_handle_context_query (GstNvEncoder * self, GstQuery * query) if (ret) return ret; } +#endif +#ifdef HAVE_GST_D3D12 + if (gst_d3d12_handle_context_query (GST_ELEMENT (self), query, + priv->device_12)) { + return TRUE; + } #endif ret = gst_cuda_handle_context_query (GST_ELEMENT (self), query, priv->context); @@ -1283,6 +1306,20 @@ gst_nv_encoder_set_format (GstVideoEncoder * encoder, } #endif +#ifdef HAVE_GST_D3D12 + { + auto features = gst_caps_get_features (state->caps, 0); + gst_clear_object (&priv->interop_12); + if (gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY) && + gst_d3d12_ensure_element_data_for_adapter_luid (GST_ELEMENT (self), + priv->dxgi_adapter_luid, &priv->device_12)) { + priv->interop_12 = gst_cuda_d3d12_interop_new (priv->context, + priv->device_12, &state->info); + } + } +#endif + /* select device again on next buffer */ if (priv->subclass_device_mode == GST_NV_ENCODER_DEVICE_AUTO_SELECT) priv->selected_device_mode = GST_NV_ENCODER_DEVICE_AUTO_SELECT; @@ -1554,6 +1591,52 @@ gst_nv_encoder_prepare_task_input_cuda (GstNvEncoder * self, } #endif +#ifdef HAVE_GST_D3D12 + if (priv->interop_12) { + if (gst_is_d3d12_memory (mem)) { + auto dmem = GST_D3D12_MEMORY_CAST (mem); + if (!gst_d3d12_device_is_equal (dmem->device, priv->device_12)) { + GST_DEBUG_OBJECT (self, "Different d3d12 device"); + gst_clear_object (&priv->interop_12); + } + } else { + GST_WARNING_OBJECT (self, "Not a d3d12 buffer"); + gst_clear_object (&priv->interop_12); + } + } + + if (priv->interop_12) { + GstBuffer *copy = nullptr; + gst_buffer_pool_acquire_buffer (priv->internal_pool, ©, nullptr); + if (!copy) { + GST_ERROR_OBJECT (self, "Couldn't acquire buffer"); + return GST_FLOW_ERROR; + } + + if (!gst_cuda_d3d12_interop_upload_async (priv->interop_12, + copy, buffer, priv->stream)) { + GST_WARNING_OBJECT (self, "Couldn't upload d3d12 to cuda"); + gst_buffer_unref (copy); + gst_clear_object (&priv->interop_12); + } else { + mem = gst_buffer_peek_memory (copy, 0); + + status = object->AcquireResource (mem, &resource); + if (status != NV_ENC_SUCCESS) { + GST_ERROR_OBJECT (self, "Failed to get resource, status %" + GST_NVENC_STATUS_FORMAT, GST_NVENC_STATUS_ARGS (status)); + gst_buffer_unref (copy); + + return GST_FLOW_ERROR; + } + + gst_nv_enc_task_set_resource (task, copy, resource); + + return GST_FLOW_OK; + } + } +#endif + if (!gst_is_cuda_memory (mem)) { GST_LOG_OBJECT (self, "Not a CUDA buffer, system copy"); return gst_nv_encoder_copy_system (self, info, buffer, task); diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h index 6de23a5f39..9681e07fbe 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h @@ -22,6 +22,10 @@ #include #include +#ifdef HAVE_GST_D3D12 +#include +#endif + #ifdef G_OS_WIN32 #include #endif @@ -35,6 +39,7 @@ #include #include "nvEncodeAPI.h" #include "gstnvenc.h" +#include "gstnvcodecutils.h" G_BEGIN_DECLS diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp index b7c30db120..c5b3a76a88 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp @@ -2203,6 +2203,20 @@ gst_nv_h264_encoder_create_class_data (GstObject * device, gpointer session, if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) { gst_caps_set_features (sink_caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, nullptr)); +#ifdef HAVE_GST_D3D12 + if (gst_nvcodec_is_windows_10_or_greater ()) { + gboolean have_interop = FALSE; + g_object_get (device, + "external-resource-interop", &have_interop, nullptr); + if (have_interop) { + auto d3d12_caps = gst_caps_copy (system_caps); + gst_caps_set_features_simple (d3d12_caps, + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, + nullptr)); + gst_caps_append (sink_caps, d3d12_caps); + } + } +#endif #ifdef HAVE_CUDA_GST_GL GstCaps *gl_caps = gst_caps_copy (system_caps); gst_caps_set_features (gl_caps, 0, @@ -2227,8 +2241,12 @@ gst_nv_h264_encoder_create_class_data (GstObject * device, gpointer session, cdata->profiles = g_list_append (cdata->profiles, g_strdup (iter.c_str())); /* *INDENT-ON* */ +#ifdef G_OS_WIN32 if (device_mode == GST_NV_ENCODER_DEVICE_D3D11) g_object_get (device, "adapter-luid", &cdata->adapter_luid, nullptr); + else + g_object_get (device, "dxgi-adapter-luid", &cdata->adapter_luid, nullptr); +#endif if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) g_object_get (device, "cuda-device-id", &cdata->cuda_device_id, nullptr); diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp index 80dec43a51..81525bef6d 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp @@ -2261,6 +2261,20 @@ gst_nv_h265_encoder_create_class_data (GstObject * device, gpointer session, if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) { gst_caps_set_features (sink_caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, nullptr)); +#ifdef HAVE_GST_D3D12 + if (gst_nvcodec_is_windows_10_or_greater ()) { + gboolean have_interop = FALSE; + g_object_get (device, + "external-resource-interop", &have_interop, nullptr); + if (have_interop) { + auto d3d12_caps = gst_caps_copy (system_caps); + gst_caps_set_features_simple (d3d12_caps, + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, + nullptr)); + gst_caps_append (sink_caps, d3d12_caps); + } + } +#endif #ifdef HAVE_CUDA_GST_GL GstCaps *gl_caps = gst_caps_copy (system_caps); gst_caps_set_features (gl_caps, 0, @@ -2285,8 +2299,12 @@ gst_nv_h265_encoder_create_class_data (GstObject * device, gpointer session, cdata->profiles = g_list_append (cdata->profiles, g_strdup (iter.c_str())); /* *INDENT-ON* */ +#ifdef G_OS_WIN32 if (device_mode == GST_NV_ENCODER_DEVICE_D3D11) g_object_get (device, "adapter-luid", &cdata->adapter_luid, nullptr); + else + g_object_get (device, "dxgi-adapter-luid", &cdata->adapter_luid, nullptr); +#endif if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) g_object_get (device, "cuda-device-id", &cdata->cuda_device_id, nullptr); diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/meson.build b/subprojects/gst-plugins-bad/sys/nvcodec/meson.build index dbc4b833a4..69bbc2b1bb 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/meson.build +++ b/subprojects/gst-plugins-bad/sys/nvcodec/meson.build @@ -8,6 +8,7 @@ nvcodec_sources = [ 'gstcudaipcsink.cpp', 'gstcudaipcsrc.cpp', 'gstcudamemorycopy.c', + 'gstnvcodecutils.cpp', 'plugin.c' ]