From 64c12189487ada045e92e18737ce80f09649c4be Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha@centricular.com>
Date: Mon, 9 Nov 2020 23:04:32 +0900
Subject: [PATCH] codecs: h264decoder: Try reference picture marking process in
 any case

... even if there is some invalid conditions
(because of broken stream, our implementation fault or so).
Otherwise baseclass will keep such reference pictures and
it would result to DPB full.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1783>
---
 gst-libs/gst/codecs/gsth264decoder.c | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/gst-libs/gst/codecs/gsth264decoder.c b/gst-libs/gst/codecs/gsth264decoder.c
index 4e1d59597f..6cbdcb2a54 100644
--- a/gst-libs/gst/codecs/gsth264decoder.c
+++ b/gst-libs/gst/codecs/gsth264decoder.c
@@ -1399,7 +1399,8 @@ gst_h264_decoder_handle_memory_management_opt (GstH264Decoder * self,
             ref_pic_marking, picture)) {
       GST_WARNING_OBJECT (self, "memory management operation type %d failed",
           type);
-      return FALSE;
+      /* Most likely our implementation fault, but let's just perform
+       * next MMCO if any */
     }
   }
 
@@ -1423,20 +1424,27 @@ gst_h264_decoder_sliding_window_picture_marking (GstH264Decoder * self)
   num_ref_pics = gst_h264_dpb_num_ref_pictures (priv->dpb);
   max_num_ref_frames = MAX (1, sps->num_ref_frames);
 
-  if (num_ref_pics > max_num_ref_frames) {
-    GST_WARNING_OBJECT (self,
-        "num_ref_pics %d is larger than allowed maximum %d",
-        num_ref_pics, max_num_ref_frames);
-    return FALSE;
-  }
+  if (num_ref_pics < max_num_ref_frames)
+    return TRUE;
 
-  if (num_ref_pics == max_num_ref_frames) {
+  /* In theory, num_ref_pics shouldn't be larger than max_num_ref_frames
+   * but it could happen if our implementation is wrong somehow or so.
+   * Just try to remove reference pictures as many as possible in order to
+   * avoid DPB overflow.
+   */
+  while (num_ref_pics >= max_num_ref_frames) {
     /* Max number of reference pics reached, need to remove one of the short
      * term ones. Find smallest frame_num_wrap short reference picture and mark
      * it as unused */
     GstH264Picture *to_unmark =
         gst_h264_dpb_get_lowest_frame_num_short_ref (priv->dpb);
 
+    if (num_ref_pics > max_num_ref_frames) {
+      GST_WARNING_OBJECT (self,
+          "num_ref_pics %d is larger than allowed maximum %d",
+          num_ref_pics, max_num_ref_frames);
+    }
+
     if (!to_unmark) {
       GST_WARNING_OBJECT (self, "Could not find a short ref picture to unmark");
       return FALSE;
@@ -1448,6 +1456,8 @@ gst_h264_decoder_sliding_window_picture_marking (GstH264Decoder * self)
 
     to_unmark->ref = FALSE;
     gst_h264_picture_unref (to_unmark);
+
+    num_ref_pics--;
   }
 
   return TRUE;