From 0445ed6ba54a203580c495deae03c1262de1526d Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha.yang@navercorp.com>
Date: Tue, 30 Jul 2019 17:49:25 +0900
Subject: [PATCH] nvenc: Fix deadlock when pad_push return was not GST_FLOW_OK

Encoding thread is terminated without any notification so
upstream streaming thread is locked because there is nothing
to pop from GAsyncQueue. If downstream returns error,
we need put SHUTDOWN_COOKIE to GAsyncQueue for chain function
can wakeup.
---
 sys/nvcodec/gstnvbaseenc.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sys/nvcodec/gstnvbaseenc.c b/sys/nvcodec/gstnvbaseenc.c
index dc35d5c4b1..1cf3295324 100644
--- a/sys/nvcodec/gstnvbaseenc.c
+++ b/sys/nvcodec/gstnvbaseenc.c
@@ -726,6 +726,7 @@ gst_nv_base_enc_bitstream_thread (gpointer user_data)
     if (flow != GST_FLOW_OK) {
       GST_INFO_OBJECT (enc, "got flow %s", gst_flow_get_name (flow));
       g_atomic_int_set (&nvenc->last_flow, flow);
+      g_async_queue_push (nvenc->in_bufs_pool, SHUTDOWN_COOKIE);
       break;
     }
   }
@@ -1462,6 +1463,9 @@ _acquire_input_buffer (GstNvBaseEnc * nvenc, gpointer * input)
   *input = g_async_queue_pop (nvenc->in_bufs_pool);
   GST_VIDEO_ENCODER_STREAM_LOCK (nvenc);
 
+  if (*input == SHUTDOWN_COOKIE)
+    return g_atomic_int_get (&nvenc->last_flow);
+
   return GST_FLOW_OK;
 }
 
@@ -1569,6 +1573,8 @@ gst_nv_base_enc_handle_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame)
   flow = _acquire_input_buffer (nvenc, &input_buffer);
   if (flow != GST_FLOW_OK)
     goto out;
+  else if (input_buffer == SHUTDOWN_COOKIE)
+    goto out;
   if (input_buffer == NULL)
     goto error;