nvcudaenc: Don't use default CUDA stream

Set non-default CUDA stream via NvEncSetIOCudaStreams() if possible,
so that NVENC's internal kernel function can run on the given CUDA
stream instead of default CUDA stream

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3615>

subprojects/gst-plugins-bad/sys/nvcodec/gstnvenc.c

@@ -26,6 +26,7 @@
 #include "gstnvh265enc.h"
 #include <gst/cuda/gstcudautils.h>
 #include <gst/cuda/gstcudabufferpool.h>
+#include <string.h>
 
 #include <gmodule.h>
 
@@ -51,6 +52,7 @@
 #define GST_NVENCAPI_STRUCT_VERSION(ver,api_ver) ((uint32_t)(api_ver) | ((ver)<<16) | (0x7 << 28))
 
 static guint32 gst_nvenc_api_version = NVENCAPI_VERSION;
+static gboolean gst_nvenc_supports_cuda_stream = FALSE;
 
 typedef NVENCSTATUS NVENCAPI
 tNvEncodeAPICreateInstance (NV_ENCODE_API_FUNCTION_LIST * functionList);
@@ -283,6 +285,14 @@ NvEncUnregisterAsyncEvent (void *encoder, NV_ENC_EVENT_PARAMS * event_params)
   return nvenc_api.nvEncUnregisterAsyncEvent (encoder, event_params);
 }
 
+NVENCSTATUS NVENCAPI
+NvEncSetIOCudaStreams (void *encoder, NV_ENC_CUSTREAM_PTR input_stream,
+    NV_ENC_CUSTREAM_PTR output_stream)
+{
+  g_assert (nvenc_api.nvEncSetIOCudaStreams != NULL);
+  return nvenc_api.nvEncSetIOCudaStreams (encoder, input_stream, output_stream);
+}
+
 gboolean
 gst_nvenc_cmp_guid (GUID g1, GUID g2)
 {
@@ -891,6 +901,7 @@ gst_nvenc_load_library (guint * api_major_ver, guint * api_minor_ver)
   gint i;
   static const GstNvEncVersion version_list[] = {
     {NVENCAPI_MAJOR_VERSION, NVENCAPI_MINOR_VERSION},
+    {9, 1},
     {9, 0},
     {GST_NVENC_MIN_API_MAJOR_VERSION, GST_NVENC_MIN_API_MINOR_VERSION}
   };
@@ -969,9 +980,13 @@ gst_nvenc_load_library (guint * api_major_ver, guint * api_minor_ver)
       continue;
     }
 
+    GST_INFO ("Checking version %d.%d", version_list[i].major,
+        version_list[i].minor);
+
     gst_nvenc_api_version =
         GST_NVENCAPI_VERSION (version_list[i].major, version_list[i].minor);
 
+    memset (&nvenc_api, 0, sizeof (NV_ENCODE_API_FUNCTION_LIST));
     nvenc_api.version = GST_NVENCAPI_STRUCT_VERSION (2, gst_nvenc_api_version);
     ret = nvEncodeAPICreateInstance (&nvenc_api);
 
@@ -981,7 +996,18 @@ gst_nvenc_load_library (guint * api_major_ver, guint * api_minor_ver)
 
       *api_major_ver = version_list[i].major;
       *api_minor_ver = version_list[i].minor;
+
+      if ((version_list[i].major > 9 ||
+              (version_list[i].major == 9 && version_list[i].minor > 0)) &&
+          nvenc_api.nvEncSetIOCudaStreams) {
+        GST_INFO ("nvEncSetIOCudaStreams is supported");
+        gst_nvenc_supports_cuda_stream = TRUE;
+      }
+
       break;
+    } else {
+      GST_INFO ("Version %d.%d is not supported", version_list[i].major,
+          version_list[i].minor);
     }
   }
 
@@ -1149,3 +1175,9 @@ gst_nvenc_get_open_encode_session_ex_params_version (void)
   /* NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER == NVENCAPI_STRUCT_VERSION(1) */
   return GST_NVENCAPI_STRUCT_VERSION (1, gst_nvenc_api_version);
 }
+
+gboolean
+gst_nvenc_have_set_io_cuda_streams (void)
+{
+  return gst_nvenc_supports_cuda_stream;
+}

subprojects/gst-plugins-bad/sys/nvcodec/gstnvenc.h

@@ -88,6 +88,8 @@ guint32                 gst_nvenc_get_event_params_version (void);
 
 guint32                 gst_nvenc_get_open_encode_session_ex_params_version (void);
 
+gboolean                gst_nvenc_have_set_io_cuda_streams (void);
+
 gboolean                gst_nvenc_load_library (guint * api_major_ver,
                                                 guint * api_minor_ver);
 

subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp

@@ -58,6 +58,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_nv_encoder_debug);
 struct _GstNvEncoderPrivate
 {
   GstCudaContext *context;
+  CUstream cuda_stream;
+
 #ifdef GST_CUDA_HAS_D3D
   GstD3D11Device *device;
   GstD3D11Fence *fence;
@@ -243,6 +245,13 @@ gst_nv_encoder_reset (GstNvEncoder * self)
     priv->session = NULL;
   }
 
+  if (priv->context && priv->cuda_stream) {
+    gst_cuda_context_push (priv->context);
+    CuStreamDestroy (priv->cuda_stream);
+    gst_cuda_context_pop (nullptr);
+    priv->cuda_stream = nullptr;
+  }
+
   g_queue_clear (&priv->free_tasks);
   g_queue_clear (&priv->output_tasks);
 
@@ -1259,6 +1268,21 @@ gst_nv_encoder_init_session (GstNvEncoder * self, GstBuffer * in_buf)
     goto error;
   }
 
+  if (priv->selected_device_mode == GST_NV_ENCODER_DEVICE_CUDA &&
+      gst_nvenc_have_set_io_cuda_streams ()) {
+    CUresult cuda_ret = CuStreamCreate (&priv->cuda_stream, CU_STREAM_DEFAULT);
+
+    if (gst_cuda_result (cuda_ret)) {
+      status = NvEncSetIOCudaStreams (priv->session,
+          (NV_ENC_CUSTREAM_PTR) & priv->cuda_stream,
+          (NV_ENC_CUSTREAM_PTR) & priv->cuda_stream);
+      if (status != NV_ENC_SUCCESS) {
+        GST_WARNING_OBJECT (self, "NvEncSetIOCudaStreams failed, status: %"
+            GST_NVENC_STATUS_FORMAT, GST_NVENC_STATUS_ARGS (status));
+      }
+    }
+  }
+
   task_pool_size = gst_nv_encoder_calculate_task_pool_size (self,
       &priv->config);
   g_array_set_size (priv->task_pool, task_pool_size);