From 012222bcb34632530c72811c31e563cd407b0168 Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha@centricular.com>
Date: Fri, 1 Dec 2023 21:38:17 +0900
Subject: [PATCH] cudaipcsink: Fix deadlock on stop

Manually close connection if client does not hold any shared memory
on stop.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5747>
---
 .../sys/nvcodec/gstcudaipcserver.cpp               | 14 ++++++++++++++
 .../gst-plugins-bad/sys/nvcodec/gstcudaipcserver.h |  2 ++
 .../sys/nvcodec/gstcudaipcserver_unix.cpp          |  8 +++++++-
 .../sys/nvcodec/gstcudaipcserver_win32.cpp         |  8 +++++++-
 4 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver.cpp
index 5491168643..64b6d8a2a5 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver.cpp
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver.cpp
@@ -630,14 +630,28 @@ gst_cuda_ipc_server_on_idle (GstCudaIpcServer * server)
 
     GST_DEBUG_OBJECT (server, "Have %" G_GSIZE_FORMAT " alive connections",
         priv->conn_map.size());
+
+    size_t num_closed = 0;
     for (auto it : priv->conn_map) {
       auto conn = it.second;
       GST_DEBUG_OBJECT (server, "conn-id %u"
           " peer handle size %" G_GSIZE_FORMAT, conn->id,
           conn->peer_handles.size ());
+
+      /* Cannot erase conn since it's still referenced.
+       * Manually close connection */
+      if (conn->peer_handles.empty ()) {
+        conn->CloseConn ();
+        num_closed++;
+      }
     }
     /* *INDENT-ON* */
 
+    if (priv->conn_map.size () == num_closed) {
+      GST_DEBUG_OBJECT (server, "All connections were closed");
+      klass->terminate (server);
+    }
+
     return;
   }
 
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver.h b/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver.h
index 87433f953d..d759638a85 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver.h
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver.h
@@ -150,6 +150,8 @@ struct GstCudaIpcServerConn : public OVERLAPPED
     gst_clear_caps (&caps);
   }
 
+  virtual void CloseConn () = 0;
+
   GstCudaIpcServer *server;
 
   GstCudaContext *context = nullptr;
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver_unix.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver_unix.cpp
index 1329fdb09c..d8baf9c537 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver_unix.cpp
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver_unix.cpp
@@ -47,8 +47,14 @@ struct GstCudaIpcServerConnUnix : public GstCudaIpcServerConn
     g_clear_object (&socket_conn);
   }
 
+  void CloseConn ()
+  {
+    if (socket_conn)
+      g_io_stream_close (G_IO_STREAM (socket_conn), nullptr, nullptr);
+  }
+
   /* Holds ref */
-  GSocketConnection *socket_conn;
+  GSocketConnection *socket_conn = nullptr;
 
   /* Owned by socket_conn */
   GInputStream *istream;
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver_win32.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver_win32.cpp
index 7feb626f37..3fa2e2be7d 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver_win32.cpp
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstcudaipcserver_win32.cpp
@@ -41,15 +41,21 @@ struct GstCudaIpcServerConnWin32 : public GstCudaIpcServerConn
   }
 
   virtual ~GstCudaIpcServerConnWin32 ()
+  {
+    CloseConn ();
+  }
+
+  void CloseConn ()
   {
     if (pipe != INVALID_HANDLE_VALUE) {
       CancelIo (pipe);
       DisconnectNamedPipe (pipe);
       CloseHandle (pipe);
+      pipe = INVALID_HANDLE_VALUE;
     }
   }
 
-  HANDLE pipe;
+  HANDLE pipe = INVALID_HANDLE_VALUE;
 };
 
 struct GstCudaIpcServerWin32Private