v4l2codec: Pass system frame number as timestamp and use it to retrieve back frames reliably

System frame numbers are supposed to be unique and correct drivers are
passing through timestamps without modification from the output/sink to the
capture/src side.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/778>

sys/v4l2/gstv4l2bufferpool.c

@@ -1139,10 +1139,9 @@ no_buffers:
 
 static GstFlowReturn
 gst_v4l2_buffer_pool_qbuf (GstV4l2BufferPool * pool, GstBuffer * buf,
-    GstV4l2MemoryGroup * group)
+    GstV4l2MemoryGroup * group, guint32 * frame_number)
 {
   const GstV4l2Object *obj = pool->obj;
-  GstClockTime timestamp;
   gint index;
 
   index = group->buffer.index;
@@ -1164,9 +1163,17 @@ gst_v4l2_buffer_pool_qbuf (GstV4l2BufferPool * pool, GstBuffer * buf,
     group->buffer.field = field;
   }
 
-  if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) {
+  if (frame_number) {
-    timestamp = GST_BUFFER_TIMESTAMP (buf);
+    group->buffer.timestamp.tv_sec = *frame_number;
-    GST_TIME_TO_TIMEVAL (timestamp, group->buffer.timestamp);
+    group->buffer.timestamp.tv_usec = 0;
+  } else {
+    if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) {
+      GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buf);
+      GST_TIME_TO_TIMEVAL (timestamp, group->buffer.timestamp);
+    } else {
+      group->buffer.timestamp.tv_sec = -1;
+      group->buffer.timestamp.tv_usec = -1;
+    }
   }
 
   GST_OBJECT_LOCK (pool);
@@ -1506,7 +1513,8 @@ gst_v4l2_buffer_pool_release_buffer (GstBufferPool * bpool, GstBuffer * buffer)
             if (pool->other_pool)
               ret = gst_v4l2_buffer_pool_prepare_buffer (pool, buffer, NULL);
             if (ret != GST_FLOW_OK ||
-                gst_v4l2_buffer_pool_qbuf (pool, buffer, group) != GST_FLOW_OK)
+                gst_v4l2_buffer_pool_qbuf (pool, buffer, group,
+                    NULL) != GST_FLOW_OK)
               pclass->release_buffer (bpool, buffer);
           } else {
             /* Simply release invalid/modified buffer, the allocator will
@@ -1795,15 +1803,20 @@ cleanup:
  * gst_v4l2_buffer_pool_process:
  * @bpool: a #GstBufferPool
  * @buf: a #GstBuffer, maybe be replaced
+ * @frame_number: 32 bit frame number or %NULL
  *
  * Process @buf in @bpool. For capture devices, this functions fills @buf with
  * data from the device. For output devices, this functions send the contents of
  * @buf to the device for playback.
  *
+ * If non-%NULL and an output device, @frame_number is stored inside the timestamp for output devices and read
+ * back from the timestamp for capture devices.
+ *
  * Returns: %GST_FLOW_OK on success.
  */
 GstFlowReturn
-gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf)
+gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf,
+    guint32 * frame_number)
 {
   GstFlowReturn ret = GST_FLOW_OK;
   GstBufferPool *bpool = GST_BUFFER_POOL_CAST (pool);
@@ -2006,7 +2019,9 @@ gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf)
             gst_v4l2_is_buffer_valid (to_queue, &group);
           }
 
-          if ((ret = gst_v4l2_buffer_pool_qbuf (pool, to_queue, group))
+          if ((ret =
+                  gst_v4l2_buffer_pool_qbuf (pool, to_queue, group,
+                      frame_number))
               != GST_FLOW_OK)
             goto queue_failed;
 

sys/v4l2/gstv4l2bufferpool.h

@@ -103,7 +103,7 @@ GType gst_v4l2_buffer_pool_get_type (void);
 
 GstBufferPool *     gst_v4l2_buffer_pool_new     (GstV4l2Object *obj, GstCaps *caps);
 
-GstFlowReturn       gst_v4l2_buffer_pool_process (GstV4l2BufferPool * bpool, GstBuffer ** buf);
+GstFlowReturn       gst_v4l2_buffer_pool_process (GstV4l2BufferPool * bpool, GstBuffer ** buf, guint32 * frame_number);
 
 void                gst_v4l2_buffer_pool_set_other_pool (GstV4l2BufferPool * pool,
                                                          GstBufferPool * other_pool);

sys/v4l2/gstv4l2sink.c

@@ -610,7 +610,7 @@ gst_v4l2sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
   gst_buffer_ref (buf);
 again:
   ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL_CAST (obj->pool),
-      &buf);
+      &buf, NULL);
   if (ret == GST_FLOW_FLUSHING) {
     ret = gst_base_sink_wait_preroll (GST_BASE_SINK (vsink));
     if (ret == GST_FLOW_OK)

sys/v4l2/gstv4l2src.c

@@ -844,7 +844,7 @@ gst_v4l2src_create (GstPushSrc * src, GstBuffer ** buf)
     if (G_UNLIKELY (ret != GST_FLOW_OK))
       goto alloc_failed;
 
-    ret = gst_v4l2_buffer_pool_process (pool, buf);
+    ret = gst_v4l2_buffer_pool_process (pool, buf, NULL);
 
   } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER);
 

sys/v4l2/gstv4l2transform.c

@@ -927,7 +927,8 @@ gst_v4l2_transform_prepare_output_buffer (GstBaseTransform * trans,
   }
 
   GST_DEBUG_OBJECT (self, "Queue input buffer");
-  ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), &inbuf);
+  ret =
+      gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), &inbuf, NULL);
   if (G_UNLIKELY (ret != GST_FLOW_OK))
     goto beach;
 
@@ -945,7 +946,9 @@ gst_v4l2_transform_prepare_output_buffer (GstBaseTransform * trans,
       goto alloc_failed;
 
     pool = self->v4l2capture->pool;
-    ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), outbuf);
+    ret =
+        gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), outbuf,
+        NULL);
 
   } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER);
 

sys/v4l2/gstv4l2videodec.c

@@ -438,7 +438,7 @@ gst_v4l2_video_dec_finish (GstVideoDecoder * decoder)
       buffer = gst_buffer_new ();
       ret =
           gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->
-              v4l2output->pool), &buffer);
+              v4l2output->pool), &buffer, NULL);
       gst_buffer_unref (buffer);
     }
   }
@@ -472,36 +472,6 @@ gst_v4l2_video_dec_drain (GstVideoDecoder * decoder)
   return GST_FLOW_OK;
 }
 
-static GstVideoCodecFrame *
-gst_v4l2_video_dec_get_oldest_frame (GstVideoDecoder * decoder)
-{
-  GstVideoCodecFrame *frame = NULL;
-  GList *frames, *l;
-  gint count = 0;
-
-  frames = gst_video_decoder_get_frames (decoder);
-
-  for (l = frames; l != NULL; l = l->next) {
-    GstVideoCodecFrame *f = l->data;
-
-    if (!frame || frame->pts > f->pts)
-      frame = f;
-
-    count++;
-  }
-
-  if (frame) {
-    GST_LOG_OBJECT (decoder,
-        "Oldest frame is %d %" GST_TIME_FORMAT " and %d frames left",
-        frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1);
-    gst_video_codec_frame_ref (frame);
-  }
-
-  g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref);
-
-  return frame;
-}
-
 static void
 gst_v4l2_video_dec_loop (GstVideoDecoder * decoder)
 {
@@ -535,15 +505,22 @@ gst_v4l2_video_dec_loop (GstVideoDecoder * decoder)
       goto beach;
 
     GST_LOG_OBJECT (decoder, "Process output buffer");
-    ret = gst_v4l2_buffer_pool_process (v4l2_pool, &buffer);
+    ret = gst_v4l2_buffer_pool_process (v4l2_pool, &buffer, NULL);
-
   } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER);
 
   if (ret != GST_FLOW_OK)
     goto beach;
 
-  frame = gst_v4l2_video_dec_get_oldest_frame (decoder);
+  if (GST_BUFFER_TIMESTAMP (buffer) % GST_SECOND != 0)
+    GST_ERROR_OBJECT (decoder,
+        "Driver bug detected - check driver with v4l2-compliance from http://git.linuxtv.org/v4l-utils.git");
+  GST_LOG_OBJECT (decoder, "Got buffer for frame number %u",
+      (guint32) (GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND));
 
+  /* FIXME: Add garbage collection for the frames */
+  frame =
+      gst_video_decoder_get_frame (decoder,
+      GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND);
   if (frame) {
     frame->output_buffer = buffer;
     buffer = NULL;
@@ -633,6 +610,7 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder,
     GstBuffer *codec_data;
     GstCaps *acquired_caps, *available_caps, *caps, *filter;
     GstStructure *st;
+    guint32 dummy_frame_number = 0;
 
     GST_DEBUG_OBJECT (self, "Sending header");
 
@@ -668,9 +646,12 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder,
     }
 
     GST_VIDEO_DECODER_STREAM_UNLOCK (decoder);
+    GST_LOG_OBJECT (decoder, "Passing buffer with system frame number %u",
+        processed ? frame->system_frame_number : 0);
     ret =
         gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->
-            v4l2output->pool), &codec_data);
+            v4l2output->pool), &codec_data,
+        processed ? &frame->system_frame_number : &dummy_frame_number);
     GST_VIDEO_DECODER_STREAM_LOCK (decoder);
 
     gst_buffer_unref (codec_data);
@@ -765,9 +746,11 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder,
 
   if (!processed) {
     GST_VIDEO_DECODER_STREAM_UNLOCK (decoder);
+    GST_LOG_OBJECT (decoder, "Passing buffer with system frame number %u",
+        frame->system_frame_number);
     ret =
         gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->v4l2output->
-            pool), &frame->input_buffer);
+            pool), &frame->input_buffer, &frame->system_frame_number);
     GST_VIDEO_DECODER_STREAM_LOCK (decoder);
 
     if (ret == GST_FLOW_FLUSHING) {

sys/v4l2/gstv4l2videoenc.c

@@ -612,37 +612,6 @@ not_negotiated:
   return FALSE;
 }
 
-static GstVideoCodecFrame *
-gst_v4l2_video_enc_get_oldest_frame (GstVideoEncoder * encoder)
-{
-  GstVideoCodecFrame *frame = NULL;
-  GList *frames, *l;
-  gint count = 0;
-
-  frames = gst_video_encoder_get_frames (encoder);
-
-  for (l = frames; l != NULL; l = l->next) {
-    GstVideoCodecFrame *f = l->data;
-
-    if (!frame || frame->pts > f->pts)
-      frame = f;
-
-    count++;
-  }
-
-  if (frame) {
-    GST_LOG_OBJECT (encoder,
-        "Oldest frame is %d %" GST_TIME_FORMAT
-        " and %d frames left",
-        frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1);
-    gst_video_codec_frame_ref (frame);
-  }
-
-  g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref);
-
-  return frame;
-}
-
 static void
 gst_v4l2_video_enc_loop (GstVideoEncoder * encoder)
 {
@@ -661,18 +630,24 @@ gst_v4l2_video_enc_loop (GstVideoEncoder * encoder)
     goto beach;
   }
 
-
   /* FIXME Check if buffer isn't the last one here */
 
   GST_LOG_OBJECT (encoder, "Process output buffer");
   ret =
       gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL
-      (self->v4l2capture->pool), &buffer);
+      (self->v4l2capture->pool), &buffer, NULL);
 
   if (ret != GST_FLOW_OK)
     goto beach;
 
-  frame = gst_v4l2_video_enc_get_oldest_frame (encoder);
+  if (GST_BUFFER_TIMESTAMP (buffer) % GST_SECOND != 0)
+    GST_ERROR_OBJECT (encoder,
+        "Driver bug detected - check driver with v4l2-compliance from http://git.linuxtv.org/v4l-utils.git");
+  GST_LOG_OBJECT (encoder, "Got buffer for frame number %u",
+      (guint32) (GST_BUFFER_PTS (buffer) / GST_SECOND));
+  frame =
+      gst_video_encoder_get_frame (encoder,
+      GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND);
 
   if (frame) {
     /* At this point, the delta unit buffer flag is already correctly set by
@@ -781,9 +756,12 @@ gst_v4l2_video_enc_handle_frame (GstVideoEncoder * encoder,
 
   if (frame->input_buffer) {
     GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);
+    GST_LOG_OBJECT (encoder, "Passing buffer with frame number %u",
+        frame->system_frame_number);
     ret =
-        gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL
+        gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->
-        (self->v4l2output->pool), &frame->input_buffer);
+            v4l2output->pool), &frame->input_buffer,
+        &frame->system_frame_number);
     GST_VIDEO_ENCODER_STREAM_LOCK (encoder);
 
     if (ret == GST_FLOW_FLUSHING) {