From 52dfbbe5dac4ece0a7fa39b09b3aca31bd8b8c6c Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha.yang@navercorp.com>
Date: Fri, 30 Aug 2019 21:35:44 +0900
Subject: [PATCH] nvenc: Early terminate handle_frame if the last flow was not
 GST_FLOW_OK

If the last flow was not GST_FLOW_OK, the encoding thread is not running
and there is nothing to pop from GAsyncQueue (this causes deadlock).

To prevent deadlock, just return the handle_frame without further encoding
process if the last flow was not GST_FLOW_OK. Note that the last flow
will be cleared per FLUSH_STOP and STREAM_START event.
---
 sys/nvcodec/gstnvbaseenc.c | 38 ++++++++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/sys/nvcodec/gstnvbaseenc.c b/sys/nvcodec/gstnvbaseenc.c
index e41d268f3d..7a97f7f61a 100644
--- a/sys/nvcodec/gstnvbaseenc.c
+++ b/sys/nvcodec/gstnvbaseenc.c
@@ -34,10 +34,6 @@
 #include <gst/gl/gl.h>
 #endif
 
-/* TODO:
- *  - reset last_flow on FLUSH_STOP (seeking)
- */
-
 /* This currently supports both 5.x and 6.x versions of the NvEncodeAPI.h
  * header which are mostly API compatible. */
 
@@ -234,6 +230,8 @@ static void gst_nv_base_enc_set_context (GstElement * element,
     GstContext * context);
 static gboolean gst_nv_base_enc_sink_query (GstVideoEncoder * enc,
     GstQuery * query);
+static gboolean gst_nv_base_enc_sink_event (GstVideoEncoder * enc,
+    GstEvent * event);
 static gboolean gst_nv_base_enc_set_format (GstVideoEncoder * enc,
     GstVideoCodecState * state);
 static GstFlowReturn gst_nv_base_enc_handle_frame (GstVideoEncoder * enc,
@@ -276,6 +274,7 @@ gst_nv_base_enc_class_init (GstNvBaseEncClass * klass)
       GST_DEBUG_FUNCPTR (gst_nv_base_enc_handle_frame);
   videoenc_class->finish = GST_DEBUG_FUNCPTR (gst_nv_base_enc_finish);
   videoenc_class->sink_query = GST_DEBUG_FUNCPTR (gst_nv_base_enc_sink_query);
+  videoenc_class->sink_event = GST_DEBUG_FUNCPTR (gst_nv_base_enc_sink_event);
 
   g_object_class_install_property (gobject_class, PROP_DEVICE_ID,
       g_param_spec_uint ("cuda-device-id",
@@ -560,6 +559,26 @@ gst_nv_base_enc_sink_query (GstVideoEncoder * enc, GstQuery * query)
   return GST_VIDEO_ENCODER_CLASS (parent_class)->sink_query (enc, query);
 }
 
+static gboolean
+gst_nv_base_enc_sink_event (GstVideoEncoder * enc, GstEvent * event)
+{
+  GstNvBaseEnc *nvenc = GST_NV_BASE_ENC (enc);
+  gboolean ret;
+
+  ret = GST_VIDEO_ENCODER_CLASS (parent_class)->sink_event (enc, event);
+
+  switch (GST_EVENT_TYPE (event)) {
+    case GST_EVENT_STREAM_START:
+    case GST_EVENT_FLUSH_STOP:
+      nvenc->last_flow = GST_FLOW_OK;
+      break;
+    default:
+      break;
+  }
+
+  return ret;
+}
+
 static gboolean
 gst_nv_base_enc_start (GstVideoEncoder * enc)
 {
@@ -2240,6 +2259,17 @@ gst_nv_base_enc_handle_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame)
 
   g_assert (nvenc->encoder != NULL);
 
+  /* check last flow and if it's not OK, just return the last flow,
+   * non-OK flow means that encoding thread was terminated */
+  flow = g_atomic_int_get (&nvenc->last_flow);
+  if (flow != GST_FLOW_OK) {
+    GST_DEBUG_OBJECT (nvenc, "last flow was %s", gst_flow_get_name (flow));
+    /* just drop this frame */
+    gst_video_encoder_finish_frame (enc, frame);
+
+    return flow;
+  }
+
   if (g_atomic_int_compare_and_exchange (&nvenc->reconfig, TRUE, FALSE)) {
     if (!gst_nv_base_enc_set_format (enc, nvenc->input_state)) {
       flow = GST_FLOW_NOT_NEGOTIATED;