From 51162acc3198c179b00ede4f5fc27bd5945a457e Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha@centricular.com>
Date: Mon, 29 Jan 2024 23:25:35 +0900
Subject: [PATCH] cuda: Report device open error

Call gst_cuda_result() with CUDA_ERROR_NO_DEVICE error code if
we could not open device, so that application can catch the error

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6006>
---
 girs/CudaGst-1.0.gir                                      | 2 ++
 subprojects/gst-plugins-bad/gst-libs/gst/cuda/stub/cuda.h | 1 +
 subprojects/gst-plugins-bad/sys/nvcodec/gstnvbaseenc.c    | 8 +++++++-
 .../gst-plugins-bad/sys/nvcodec/gstnvencobject.cpp        | 2 ++
 subprojects/gst-plugins-bad/sys/nvcodec/plugin.c          | 3 +++
 5 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/girs/CudaGst-1.0.gir b/girs/CudaGst-1.0.gir
index 921f7b76b5..c38a0d2ed9 100644
--- a/girs/CudaGst-1.0.gir
+++ b/girs/CudaGst-1.0.gir
@@ -489,6 +489,8 @@ and/or use gtk-doc annotations.  -->
       <source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/cuda/stub/cuda.h"/>
       <member name="success" value="0" c:identifier="CUDA_SUCCESS">
       </member>
+      <member name="error_no_device" value="100" c:identifier="CUDA_ERROR_NO_DEVICE">
+      </member>
       <member name="error_already_mapped" value="208" c:identifier="CUDA_ERROR_ALREADY_MAPPED">
       </member>
       <member name="error_not_supported" value="801" c:identifier="CUDA_ERROR_NOT_SUPPORTED">
diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/cuda/stub/cuda.h b/subprojects/gst-plugins-bad/gst-libs/gst/cuda/stub/cuda.h
index a070743ece..fb07171180 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/cuda/stub/cuda.h
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/cuda/stub/cuda.h
@@ -39,6 +39,7 @@ typedef gint CUdevice;
 typedef enum
 {
   CUDA_SUCCESS = 0,
+  CUDA_ERROR_NO_DEVICE = 100,
   CUDA_ERROR_ALREADY_MAPPED = 208,
   CUDA_ERROR_NOT_SUPPORTED = 801,
 } CUresult;
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvbaseenc.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvbaseenc.c
index be964a555f..b66c08aff0 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvbaseenc.c
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvbaseenc.c
@@ -460,7 +460,13 @@ gst_nv_base_enc_open_encode_session (GstNvBaseEnc * nvenc)
   params.deviceType = NV_ENC_DEVICE_TYPE_CUDA;
   nv_ret = NvEncOpenEncodeSessionEx (&params, &nvenc->encoder);
 
-  return nv_ret == NV_ENC_SUCCESS;
+  if (nv_ret != NV_ENC_SUCCESS) {
+    /* Report error to abort if GST_CUDA_CRITICAL_ERRORS is configured */
+    gst_cuda_result (CUDA_ERROR_NO_DEVICE);
+    return FALSE;
+  }
+
+  return TRUE;
 }
 
 static gboolean
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencobject.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencobject.cpp
index 65f27f6408..6520c2a472 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencobject.cpp
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencobject.cpp
@@ -173,6 +173,8 @@ GstNvEncObject::CreateInstance (GstElement * client, GstObject * device,
   status = NvEncOpenEncodeSessionEx (params, &session);
   if (!NVENC_IS_SUCCESS (status, nullptr)) {
     GST_ERROR_OBJECT (device, "NvEncOpenEncodeSessionEx failed");
+    /* Report error to abort if GST_CUDA_CRITICAL_ERRORS is configured */
+    gst_cuda_result (CUDA_ERROR_NO_DEVICE);
     return nullptr;
   }
 
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/plugin.c b/subprojects/gst-plugins-bad/sys/nvcodec/plugin.c
index 6f51eb262c..82c3617c2d 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/plugin.c
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/plugin.c
@@ -121,6 +121,9 @@ plugin_init (GstPlugin * plugin)
     CuGetErrorString (cuda_ret, &err_desc);
     GST_ERROR ("Failed to init cuda, cuInit ret: 0x%x: %s: %s",
         (int) cuda_ret, err_name, err_desc);
+
+    /* to abort if GST_CUDA_CRITICAL_ERRORS is configured */
+    gst_cuda_result (CUDA_ERROR_NO_DEVICE);
     return TRUE;
   }