From d885cc0f1ad68b60f54ac533e3ef252ff52e9853 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= <sebastian@centricular.com>
Date: Mon, 19 Oct 2020 18:23:25 +0300
Subject: [PATCH] v4l2codec: Garbage collect old frames if they accumulate
 because of codec bugs

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/778>
---
 sys/v4l2/gstv4l2videodec.c | 36 +++++++++++++++++++++++++++++++++++-
 sys/v4l2/gstv4l2videoenc.c | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/sys/v4l2/gstv4l2videodec.c b/sys/v4l2/gstv4l2videodec.c
index 4f0ff30c24..89ad3e8174 100644
--- a/sys/v4l2/gstv4l2videodec.c
+++ b/sys/v4l2/gstv4l2videodec.c
@@ -472,6 +472,22 @@ gst_v4l2_video_dec_drain (GstVideoDecoder * decoder)
   return GST_FLOW_OK;
 }
 
+static gboolean
+check_system_frame_number_too_old (guint32 current, guint32 old)
+{
+  guint32 absdiff = current > old ? current - old : old - current;
+
+  /* More than 100 frames in the past, or current wrapped around */
+  if (absdiff > 100) {
+    /* Wraparound and difference is actually smaller than 100 */
+    if (absdiff > G_MAXUINT32 - 100)
+      return FALSE;
+    return TRUE;
+  }
+
+  return FALSE;
+}
+
 static void
 gst_v4l2_video_dec_loop (GstVideoDecoder * decoder)
 {
@@ -517,11 +533,29 @@ gst_v4l2_video_dec_loop (GstVideoDecoder * decoder)
   GST_LOG_OBJECT (decoder, "Got buffer for frame number %u",
       (guint32) (GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND));
 
-  /* FIXME: Add garbage collection for the frames */
   frame =
       gst_video_decoder_get_frame (decoder,
       GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND);
   if (frame) {
+    GstVideoCodecFrame *oldest_frame;
+    gboolean warned = FALSE;
+
+    /* Garbage collect old frames in case of codec bugs */
+    while ((oldest_frame = gst_video_decoder_get_oldest_frame (decoder)) &&
+        check_system_frame_number_too_old (frame->system_frame_number,
+            oldest_frame->system_frame_number)) {
+      gst_video_decoder_drop_frame (decoder, oldest_frame);
+      oldest_frame = NULL;
+
+      if (!warned) {
+        g_warning ("%s: Too old frames, bug in decoder -- please file a bug",
+            GST_ELEMENT_NAME (decoder));
+        warned = TRUE;
+      }
+    }
+    if (oldest_frame)
+      gst_video_codec_frame_unref (oldest_frame);
+
     frame->output_buffer = buffer;
     buffer = NULL;
     ret = gst_video_decoder_finish_frame (decoder, frame);
diff --git a/sys/v4l2/gstv4l2videoenc.c b/sys/v4l2/gstv4l2videoenc.c
index 193d1f7234..117d1400a3 100644
--- a/sys/v4l2/gstv4l2videoenc.c
+++ b/sys/v4l2/gstv4l2videoenc.c
@@ -612,6 +612,22 @@ not_negotiated:
   return FALSE;
 }
 
+static gboolean
+check_system_frame_number_too_old (guint32 current, guint32 old)
+{
+  guint32 absdiff = current > old ? current - old : old - current;
+
+  /* More than 100 frames in the past, or current wrapped around */
+  if (absdiff > 100) {
+    /* Wraparound and difference is actually smaller than 100 */
+    if (absdiff > G_MAXUINT32 - 100)
+      return FALSE;
+    return TRUE;
+  }
+
+  return FALSE;
+}
+
 static void
 gst_v4l2_video_enc_loop (GstVideoEncoder * encoder)
 {
@@ -650,6 +666,25 @@ gst_v4l2_video_enc_loop (GstVideoEncoder * encoder)
       GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND);
 
   if (frame) {
+    GstVideoCodecFrame *oldest_frame;
+    gboolean warned = FALSE;
+
+    /* Garbage collect old frames in case of codec bugs */
+    while ((oldest_frame = gst_video_encoder_get_oldest_frame (encoder)) &&
+        check_system_frame_number_too_old (frame->system_frame_number,
+            oldest_frame->system_frame_number)) {
+      gst_video_encoder_finish_frame (encoder, oldest_frame);
+      oldest_frame = NULL;
+
+      if (!warned) {
+        g_warning ("%s: Too old frames, bug in encoder -- please file a bug",
+            GST_ELEMENT_NAME (encoder));
+        warned = TRUE;
+      }
+    }
+    if (oldest_frame)
+      gst_video_codec_frame_unref (oldest_frame);
+
     /* At this point, the delta unit buffer flag is already correctly set by
      * gst_v4l2_buffer_pool_process. Since gst_video_encoder_finish_frame
      * will overwrite it from GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame),