From 9a232f798375c46eb37a92462db9003054ee3059 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Brzezi=C5=84ski?= <piotr@centricular.com>
Date: Mon, 9 Sep 2024 15:53:25 +0200
Subject: [PATCH] vtenc: Don't call drop_frame() when flushing

Slipped through with earlier changes to use drop/release_frame() explicitly.
We should only drop when something goes wrong in the encoder, and just release otherwise.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7475>
---
 .../gst-plugins-bad/sys/applemedia/vtenc.c    | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c
index a748fc3885..3571ef4d9b 100644
--- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c
+++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c
@@ -1976,7 +1976,6 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
   GstFlowReturn ret = GST_FLOW_OK;
   CFDictionaryRef frame_props = NULL;
   GstTaskState task_state;
-  gboolean is_flushing;
 
   /* If this condition changes later while we're still in this function,
    * it'll just fail on next frame encode or in _finish() */
@@ -1984,28 +1983,30 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
   if (task_state == GST_TASK_STOPPED || task_state == GST_TASK_PAUSED) {
     /* Abort if our loop failed to push frames downstream... */
     if (self->downstream_ret != GST_FLOW_OK) {
-      if (self->downstream_ret == GST_FLOW_FLUSHING)
+      ret = self->downstream_ret;
+
+      if (ret == GST_FLOW_FLUSHING) {
         GST_DEBUG_OBJECT (self,
             "Output loop stopped because of flushing, ignoring frame");
-      else
+        goto release;
+      } else {
         GST_WARNING_OBJECT (self,
             "Output loop stopped with error (%s), leaving",
-            gst_flow_get_name (self->downstream_ret));
-
-      ret = self->downstream_ret;
-      goto drop;
+            gst_flow_get_name (ret));
+        goto drop;
+      }
     }
 
     /* ...or if it stopped because of the flushing flag while the queue
      * was empty, in which case we didn't get GST_FLOW_FLUSHING... */
     g_mutex_lock (&self->queue_mutex);
-    is_flushing = self->is_flushing;
-    g_mutex_unlock (&self->queue_mutex);
-    if (is_flushing) {
+    if (self->is_flushing) {
+      g_mutex_unlock (&self->queue_mutex);
       GST_DEBUG_OBJECT (self, "Flushing flag set, ignoring frame");
       ret = GST_FLOW_FLUSHING;
-      goto drop;
+      goto release;
     }
+    g_mutex_unlock (&self->queue_mutex);
 
     /* .. or if it refuses to resume - e.g. it was stopped instead of paused */
     if (!gst_vtenc_ensure_output_loop (self)) {
@@ -2225,17 +2226,16 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
 
   return ret;
 
-drop:
-  {
-    gst_video_encoder_drop_frame (GST_VIDEO_ENCODER_CAST (self), frame);
-    return ret;
-  }
-
 cv_error:
-  {
-    gst_video_encoder_drop_frame (GST_VIDEO_ENCODER_CAST (self), frame);
-    return GST_FLOW_ERROR;
-  }
+  ret = GST_FLOW_ERROR;
+
+drop:
+  gst_video_encoder_drop_frame (GST_VIDEO_ENCODER_CAST (self), frame);
+  return ret;
+
+release:
+  gst_video_encoder_release_frame (GST_VIDEO_ENCODER_CAST (self), frame);
+  return ret;
 }
 
 static void