nvencoder: Add support for d3d12 memory

Use d3d12 -> cuda memory copy helper object in cuda mode encoder

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7480>

subprojects/gst-plugins-bad/sys/nvcodec/gstnvav1encoder.cpp

@@ -1464,6 +1464,20 @@ gst_nv_av1_encoder_create_class_data (GstObject * device, gpointer session,
   if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) {
     gst_caps_set_features (sink_caps, 0,
         gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, nullptr));
+#ifdef HAVE_GST_D3D12
+    if (gst_nvcodec_is_windows_10_or_greater ()) {
+      gboolean have_interop = FALSE;
+      g_object_get (device,
+          "external-resource-interop", &have_interop, nullptr);
+      if (have_interop) {
+        auto d3d12_caps = gst_caps_copy (system_caps);
+        gst_caps_set_features_simple (d3d12_caps,
+            gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY,
+                nullptr));
+        gst_caps_append (sink_caps, d3d12_caps);
+      }
+    }
+#endif
 #ifdef HAVE_CUDA_GST_GL
     GstCaps *gl_caps = gst_caps_copy (system_caps);
     gst_caps_set_features (gl_caps, 0,
@@ -1485,8 +1499,12 @@ gst_nv_av1_encoder_create_class_data (GstObject * device, gpointer session,
     cdata->formats = g_list_append (cdata->formats, g_strdup (iter.c_str()));
   /* *INDENT-ON* */
 
+#ifdef G_OS_WIN32
   if (device_mode == GST_NV_ENCODER_DEVICE_D3D11)
     g_object_get (device, "adapter-luid", &cdata->adapter_luid, nullptr);
+  else
+    g_object_get (device, "dxgi-adapter-luid", &cdata->adapter_luid, nullptr);
+#endif
 
   if (device_mode == GST_NV_ENCODER_DEVICE_CUDA)
     g_object_get (device, "cuda-device-id", &cdata->cuda_device_id, nullptr);

subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.cpp

@@ -0,0 +1,39 @@
+/* GStreamer
+ * Copyright (C) 2024 Seungha Yang <seungha@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstnvcodecutils.h"
+#include <gst/cuda/gstcuda-private.h>
+
+gboolean
+gst_nvcodec_is_windows_10_or_greater (void)
+{
+  static gboolean ret = FALSE;
+
+#ifdef G_OS_WIN32
+  GST_CUDA_CALL_ONCE_BEGIN {
+    ret = g_win32_check_windows_version (10, 0, 0, G_WIN32_OS_ANY);
+  } GST_CUDA_CALL_ONCE_END;
+#endif
+
+  return ret;
+}

subprojects/gst-plugins-bad/sys/nvcodec/gstnvcodecutils.h

@@ -25,4 +25,6 @@ G_BEGIN_DECLS
 
 void gst_cuda_ipc_client_deinit (void);
 
+gboolean gst_nvcodec_is_windows_10_or_greater (void);
+
 G_END_DECLS

subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp

@@ -37,6 +37,10 @@
 #include <atomic>
 #include "gstnvencobject.h"
 
+#ifdef HAVE_GST_D3D12
+#include "gstcudainterop_d3d12.h"
+#endif
+
 #ifdef G_OS_WIN32
 #include <wrl.h>
 
@@ -87,6 +91,11 @@ struct _GstNvEncoderPrivate
   gboolean gl_interop = FALSE;
 #endif
 
+#ifdef HAVE_GST_D3D12
+  GstD3D12Device *device_12 = nullptr;
+  GstCudaD3D12Interop *interop_12 = nullptr;
+#endif
+
   std::shared_ptr < GstNvEncObject > object;
 
   GstNvEncoderDeviceMode subclass_device_mode;
@@ -285,6 +294,10 @@ gst_nv_encoder_set_context (GstElement * element, GstContext * context)
         if (priv->gl_display)
           gst_gl_display_filter_gl_api (priv->gl_display, SUPPORTED_GL_APIS);
       }
+#endif
+#ifdef HAVE_GST_D3D12
+      gst_d3d12_handle_set_context_for_adapter_luid (element,
+          context, priv->dxgi_adapter_luid, &priv->device_12);
 #endif
       break;
     default:
@@ -472,6 +485,10 @@ gst_nv_encoder_close (GstVideoEncoder * encoder)
   gst_clear_object (&priv->gl_context);
   gst_clear_object (&priv->other_gl_context);
 #endif
+#ifdef HAVE_GST_D3D12
+  gst_clear_object (&priv->interop_12);
+  gst_clear_object (&priv->device_12);
+#endif
 
   return TRUE;
 }
@@ -609,6 +626,12 @@ gst_nv_encoder_handle_context_query (GstNvEncoder * self, GstQuery * query)
       if (ret)
         return ret;
     }
+#endif
+#ifdef HAVE_GST_D3D12
+      if (gst_d3d12_handle_context_query (GST_ELEMENT (self), query,
+              priv->device_12)) {
+        return TRUE;
+      }
 #endif
       ret = gst_cuda_handle_context_query (GST_ELEMENT (self),
           query, priv->context);
@@ -1283,6 +1306,20 @@ gst_nv_encoder_set_format (GstVideoEncoder * encoder,
   }
 #endif
 
+#ifdef HAVE_GST_D3D12
+  {
+    auto features = gst_caps_get_features (state->caps, 0);
+    gst_clear_object (&priv->interop_12);
+    if (gst_caps_features_contains (features,
+            GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY) &&
+        gst_d3d12_ensure_element_data_for_adapter_luid (GST_ELEMENT (self),
+            priv->dxgi_adapter_luid, &priv->device_12)) {
+      priv->interop_12 = gst_cuda_d3d12_interop_new (priv->context,
+          priv->device_12, &state->info);
+    }
+  }
+#endif
+
   /* select device again on next buffer */
   if (priv->subclass_device_mode == GST_NV_ENCODER_DEVICE_AUTO_SELECT)
     priv->selected_device_mode = GST_NV_ENCODER_DEVICE_AUTO_SELECT;
@@ -1554,6 +1591,52 @@ gst_nv_encoder_prepare_task_input_cuda (GstNvEncoder * self,
   }
 #endif
 
+#ifdef HAVE_GST_D3D12
+  if (priv->interop_12) {
+    if (gst_is_d3d12_memory (mem)) {
+      auto dmem = GST_D3D12_MEMORY_CAST (mem);
+      if (!gst_d3d12_device_is_equal (dmem->device, priv->device_12)) {
+        GST_DEBUG_OBJECT (self, "Different d3d12 device");
+        gst_clear_object (&priv->interop_12);
+      }
+    } else {
+      GST_WARNING_OBJECT (self, "Not a d3d12 buffer");
+      gst_clear_object (&priv->interop_12);
+    }
+  }
+
+  if (priv->interop_12) {
+    GstBuffer *copy = nullptr;
+    gst_buffer_pool_acquire_buffer (priv->internal_pool, &copy, nullptr);
+    if (!copy) {
+      GST_ERROR_OBJECT (self, "Couldn't acquire buffer");
+      return GST_FLOW_ERROR;
+    }
+
+    if (!gst_cuda_d3d12_interop_upload_async (priv->interop_12,
+            copy, buffer, priv->stream)) {
+      GST_WARNING_OBJECT (self, "Couldn't upload d3d12 to cuda");
+      gst_buffer_unref (copy);
+      gst_clear_object (&priv->interop_12);
+    } else {
+      mem = gst_buffer_peek_memory (copy, 0);
+
+      status = object->AcquireResource (mem, &resource);
+      if (status != NV_ENC_SUCCESS) {
+        GST_ERROR_OBJECT (self, "Failed to get resource, status %"
+            GST_NVENC_STATUS_FORMAT, GST_NVENC_STATUS_ARGS (status));
+        gst_buffer_unref (copy);
+
+        return GST_FLOW_ERROR;
+      }
+
+      gst_nv_enc_task_set_resource (task, copy, resource);
+
+      return GST_FLOW_OK;
+    }
+  }
+#endif
+
   if (!gst_is_cuda_memory (mem)) {
     GST_LOG_OBJECT (self, "Not a CUDA buffer, system copy");
     return gst_nv_encoder_copy_system (self, info, buffer, task);

subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h

@@ -22,6 +22,10 @@
 #include <gst/gst.h>
 #include <gst/video/video.h>
 
+#ifdef HAVE_GST_D3D12
+#include <gst/d3d12/gstd3d12.h>
+#endif
+
 #ifdef G_OS_WIN32
 #include <gst/d3d11/gstd3d11.h>
 #endif
@@ -35,6 +39,7 @@
 #include <gst/cuda/gstcuda.h>
 #include "nvEncodeAPI.h"
 #include "gstnvenc.h"
+#include "gstnvcodecutils.h"
 
 G_BEGIN_DECLS
 

subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp

@@ -2203,6 +2203,20 @@ gst_nv_h264_encoder_create_class_data (GstObject * device, gpointer session,
   if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) {
     gst_caps_set_features (sink_caps, 0,
         gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, nullptr));
+#ifdef HAVE_GST_D3D12
+    if (gst_nvcodec_is_windows_10_or_greater ()) {
+      gboolean have_interop = FALSE;
+      g_object_get (device,
+          "external-resource-interop", &have_interop, nullptr);
+      if (have_interop) {
+        auto d3d12_caps = gst_caps_copy (system_caps);
+        gst_caps_set_features_simple (d3d12_caps,
+            gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY,
+                nullptr));
+        gst_caps_append (sink_caps, d3d12_caps);
+      }
+    }
+#endif
 #ifdef HAVE_CUDA_GST_GL
     GstCaps *gl_caps = gst_caps_copy (system_caps);
     gst_caps_set_features (gl_caps, 0,
@@ -2227,8 +2241,12 @@ gst_nv_h264_encoder_create_class_data (GstObject * device, gpointer session,
     cdata->profiles = g_list_append (cdata->profiles, g_strdup (iter.c_str()));
   /* *INDENT-ON* */
 
+#ifdef G_OS_WIN32
   if (device_mode == GST_NV_ENCODER_DEVICE_D3D11)
     g_object_get (device, "adapter-luid", &cdata->adapter_luid, nullptr);
+  else
+    g_object_get (device, "dxgi-adapter-luid", &cdata->adapter_luid, nullptr);
+#endif
 
   if (device_mode == GST_NV_ENCODER_DEVICE_CUDA)
     g_object_get (device, "cuda-device-id", &cdata->cuda_device_id, nullptr);

subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp

@@ -2261,6 +2261,20 @@ gst_nv_h265_encoder_create_class_data (GstObject * device, gpointer session,
   if (device_mode == GST_NV_ENCODER_DEVICE_CUDA) {
     gst_caps_set_features (sink_caps, 0,
         gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, nullptr));
+#ifdef HAVE_GST_D3D12
+    if (gst_nvcodec_is_windows_10_or_greater ()) {
+      gboolean have_interop = FALSE;
+      g_object_get (device,
+          "external-resource-interop", &have_interop, nullptr);
+      if (have_interop) {
+        auto d3d12_caps = gst_caps_copy (system_caps);
+        gst_caps_set_features_simple (d3d12_caps,
+            gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY,
+                nullptr));
+        gst_caps_append (sink_caps, d3d12_caps);
+      }
+    }
+#endif
 #ifdef HAVE_CUDA_GST_GL
     GstCaps *gl_caps = gst_caps_copy (system_caps);
     gst_caps_set_features (gl_caps, 0,
@@ -2285,8 +2299,12 @@ gst_nv_h265_encoder_create_class_data (GstObject * device, gpointer session,
     cdata->profiles = g_list_append (cdata->profiles, g_strdup (iter.c_str()));
   /* *INDENT-ON* */
 
+#ifdef G_OS_WIN32
   if (device_mode == GST_NV_ENCODER_DEVICE_D3D11)
     g_object_get (device, "adapter-luid", &cdata->adapter_luid, nullptr);
+  else
+    g_object_get (device, "dxgi-adapter-luid", &cdata->adapter_luid, nullptr);
+#endif
 
   if (device_mode == GST_NV_ENCODER_DEVICE_CUDA)
     g_object_get (device, "cuda-device-id", &cdata->cuda_device_id, nullptr);

subprojects/gst-plugins-bad/sys/nvcodec/meson.build

@@ -8,6 +8,7 @@ nvcodec_sources = [
   'gstcudaipcsink.cpp',
   'gstcudaipcsrc.cpp',
   'gstcudamemorycopy.c',
+  'gstnvcodecutils.cpp',
   'plugin.c'
 ]