From bcb3428ed06e59945dc1c6034f6e68c0e9d3a964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 19 Oct 2020 17:56:04 +0300 Subject: [PATCH] v4l2codec: Pass system frame number as timestamp and use it to retrieve back frames reliably System frame numbers are supposed to be unique and correct drivers are passing through timestamps without modification from the output/sink to the capture/src side. Part-of: --- sys/v4l2/gstv4l2bufferpool.c | 31 ++++++++++++++------ sys/v4l2/gstv4l2bufferpool.h | 2 +- sys/v4l2/gstv4l2sink.c | 2 +- sys/v4l2/gstv4l2src.c | 2 +- sys/v4l2/gstv4l2transform.c | 7 +++-- sys/v4l2/gstv4l2videodec.c | 55 +++++++++++++----------------------- sys/v4l2/gstv4l2videoenc.c | 50 +++++++++----------------------- 7 files changed, 64 insertions(+), 85 deletions(-) diff --git a/sys/v4l2/gstv4l2bufferpool.c b/sys/v4l2/gstv4l2bufferpool.c index 18f019a28a..52c99163e2 100644 --- a/sys/v4l2/gstv4l2bufferpool.c +++ b/sys/v4l2/gstv4l2bufferpool.c @@ -1139,10 +1139,9 @@ no_buffers: static GstFlowReturn gst_v4l2_buffer_pool_qbuf (GstV4l2BufferPool * pool, GstBuffer * buf, - GstV4l2MemoryGroup * group) + GstV4l2MemoryGroup * group, guint32 * frame_number) { const GstV4l2Object *obj = pool->obj; - GstClockTime timestamp; gint index; index = group->buffer.index; @@ -1164,9 +1163,17 @@ gst_v4l2_buffer_pool_qbuf (GstV4l2BufferPool * pool, GstBuffer * buf, group->buffer.field = field; } - if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { - timestamp = GST_BUFFER_TIMESTAMP (buf); - GST_TIME_TO_TIMEVAL (timestamp, group->buffer.timestamp); + if (frame_number) { + group->buffer.timestamp.tv_sec = *frame_number; + group->buffer.timestamp.tv_usec = 0; + } else { + if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { + GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buf); + GST_TIME_TO_TIMEVAL (timestamp, group->buffer.timestamp); + } else { + group->buffer.timestamp.tv_sec = -1; + group->buffer.timestamp.tv_usec = -1; + } } GST_OBJECT_LOCK (pool); @@ -1506,7 +1513,8 @@ gst_v4l2_buffer_pool_release_buffer (GstBufferPool * bpool, GstBuffer * buffer) if (pool->other_pool) ret = gst_v4l2_buffer_pool_prepare_buffer (pool, buffer, NULL); if (ret != GST_FLOW_OK || - gst_v4l2_buffer_pool_qbuf (pool, buffer, group) != GST_FLOW_OK) + gst_v4l2_buffer_pool_qbuf (pool, buffer, group, + NULL) != GST_FLOW_OK) pclass->release_buffer (bpool, buffer); } else { /* Simply release invalid/modified buffer, the allocator will @@ -1795,15 +1803,20 @@ cleanup: * gst_v4l2_buffer_pool_process: * @bpool: a #GstBufferPool * @buf: a #GstBuffer, maybe be replaced + * @frame_number: 32 bit frame number or %NULL * * Process @buf in @bpool. For capture devices, this functions fills @buf with * data from the device. For output devices, this functions send the contents of * @buf to the device for playback. * + * If non-%NULL and an output device, @frame_number is stored inside the timestamp for output devices and read + * back from the timestamp for capture devices. + * * Returns: %GST_FLOW_OK on success. */ GstFlowReturn -gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf) +gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf, + guint32 * frame_number) { GstFlowReturn ret = GST_FLOW_OK; GstBufferPool *bpool = GST_BUFFER_POOL_CAST (pool); @@ -2006,7 +2019,9 @@ gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf) gst_v4l2_is_buffer_valid (to_queue, &group); } - if ((ret = gst_v4l2_buffer_pool_qbuf (pool, to_queue, group)) + if ((ret = + gst_v4l2_buffer_pool_qbuf (pool, to_queue, group, + frame_number)) != GST_FLOW_OK) goto queue_failed; diff --git a/sys/v4l2/gstv4l2bufferpool.h b/sys/v4l2/gstv4l2bufferpool.h index 62bcb0f3c8..f8972b6417 100644 --- a/sys/v4l2/gstv4l2bufferpool.h +++ b/sys/v4l2/gstv4l2bufferpool.h @@ -103,7 +103,7 @@ GType gst_v4l2_buffer_pool_get_type (void); GstBufferPool * gst_v4l2_buffer_pool_new (GstV4l2Object *obj, GstCaps *caps); -GstFlowReturn gst_v4l2_buffer_pool_process (GstV4l2BufferPool * bpool, GstBuffer ** buf); +GstFlowReturn gst_v4l2_buffer_pool_process (GstV4l2BufferPool * bpool, GstBuffer ** buf, guint32 * frame_number); void gst_v4l2_buffer_pool_set_other_pool (GstV4l2BufferPool * pool, GstBufferPool * other_pool); diff --git a/sys/v4l2/gstv4l2sink.c b/sys/v4l2/gstv4l2sink.c index a971ef4915..e4728ac706 100644 --- a/sys/v4l2/gstv4l2sink.c +++ b/sys/v4l2/gstv4l2sink.c @@ -610,7 +610,7 @@ gst_v4l2sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) gst_buffer_ref (buf); again: ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL_CAST (obj->pool), - &buf); + &buf, NULL); if (ret == GST_FLOW_FLUSHING) { ret = gst_base_sink_wait_preroll (GST_BASE_SINK (vsink)); if (ret == GST_FLOW_OK) diff --git a/sys/v4l2/gstv4l2src.c b/sys/v4l2/gstv4l2src.c index a518aca845..2c30a51115 100644 --- a/sys/v4l2/gstv4l2src.c +++ b/sys/v4l2/gstv4l2src.c @@ -844,7 +844,7 @@ gst_v4l2src_create (GstPushSrc * src, GstBuffer ** buf) if (G_UNLIKELY (ret != GST_FLOW_OK)) goto alloc_failed; - ret = gst_v4l2_buffer_pool_process (pool, buf); + ret = gst_v4l2_buffer_pool_process (pool, buf, NULL); } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER); diff --git a/sys/v4l2/gstv4l2transform.c b/sys/v4l2/gstv4l2transform.c index cc5564d446..edb3203630 100644 --- a/sys/v4l2/gstv4l2transform.c +++ b/sys/v4l2/gstv4l2transform.c @@ -927,7 +927,8 @@ gst_v4l2_transform_prepare_output_buffer (GstBaseTransform * trans, } GST_DEBUG_OBJECT (self, "Queue input buffer"); - ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), &inbuf); + ret = + gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), &inbuf, NULL); if (G_UNLIKELY (ret != GST_FLOW_OK)) goto beach; @@ -945,7 +946,9 @@ gst_v4l2_transform_prepare_output_buffer (GstBaseTransform * trans, goto alloc_failed; pool = self->v4l2capture->pool; - ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), outbuf); + ret = + gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), outbuf, + NULL); } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER); diff --git a/sys/v4l2/gstv4l2videodec.c b/sys/v4l2/gstv4l2videodec.c index 8c06c90135..4f0ff30c24 100644 --- a/sys/v4l2/gstv4l2videodec.c +++ b/sys/v4l2/gstv4l2videodec.c @@ -438,7 +438,7 @@ gst_v4l2_video_dec_finish (GstVideoDecoder * decoder) buffer = gst_buffer_new (); ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self-> - v4l2output->pool), &buffer); + v4l2output->pool), &buffer, NULL); gst_buffer_unref (buffer); } } @@ -472,36 +472,6 @@ gst_v4l2_video_dec_drain (GstVideoDecoder * decoder) return GST_FLOW_OK; } -static GstVideoCodecFrame * -gst_v4l2_video_dec_get_oldest_frame (GstVideoDecoder * decoder) -{ - GstVideoCodecFrame *frame = NULL; - GList *frames, *l; - gint count = 0; - - frames = gst_video_decoder_get_frames (decoder); - - for (l = frames; l != NULL; l = l->next) { - GstVideoCodecFrame *f = l->data; - - if (!frame || frame->pts > f->pts) - frame = f; - - count++; - } - - if (frame) { - GST_LOG_OBJECT (decoder, - "Oldest frame is %d %" GST_TIME_FORMAT " and %d frames left", - frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1); - gst_video_codec_frame_ref (frame); - } - - g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref); - - return frame; -} - static void gst_v4l2_video_dec_loop (GstVideoDecoder * decoder) { @@ -535,15 +505,22 @@ gst_v4l2_video_dec_loop (GstVideoDecoder * decoder) goto beach; GST_LOG_OBJECT (decoder, "Process output buffer"); - ret = gst_v4l2_buffer_pool_process (v4l2_pool, &buffer); - + ret = gst_v4l2_buffer_pool_process (v4l2_pool, &buffer, NULL); } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER); if (ret != GST_FLOW_OK) goto beach; - frame = gst_v4l2_video_dec_get_oldest_frame (decoder); + if (GST_BUFFER_TIMESTAMP (buffer) % GST_SECOND != 0) + GST_ERROR_OBJECT (decoder, + "Driver bug detected - check driver with v4l2-compliance from http://git.linuxtv.org/v4l-utils.git"); + GST_LOG_OBJECT (decoder, "Got buffer for frame number %u", + (guint32) (GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND)); + /* FIXME: Add garbage collection for the frames */ + frame = + gst_video_decoder_get_frame (decoder, + GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND); if (frame) { frame->output_buffer = buffer; buffer = NULL; @@ -633,6 +610,7 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder, GstBuffer *codec_data; GstCaps *acquired_caps, *available_caps, *caps, *filter; GstStructure *st; + guint32 dummy_frame_number = 0; GST_DEBUG_OBJECT (self, "Sending header"); @@ -668,9 +646,12 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder, } GST_VIDEO_DECODER_STREAM_UNLOCK (decoder); + GST_LOG_OBJECT (decoder, "Passing buffer with system frame number %u", + processed ? frame->system_frame_number : 0); ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self-> - v4l2output->pool), &codec_data); + v4l2output->pool), &codec_data, + processed ? &frame->system_frame_number : &dummy_frame_number); GST_VIDEO_DECODER_STREAM_LOCK (decoder); gst_buffer_unref (codec_data); @@ -765,9 +746,11 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder, if (!processed) { GST_VIDEO_DECODER_STREAM_UNLOCK (decoder); + GST_LOG_OBJECT (decoder, "Passing buffer with system frame number %u", + frame->system_frame_number); ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->v4l2output-> - pool), &frame->input_buffer); + pool), &frame->input_buffer, &frame->system_frame_number); GST_VIDEO_DECODER_STREAM_LOCK (decoder); if (ret == GST_FLOW_FLUSHING) { diff --git a/sys/v4l2/gstv4l2videoenc.c b/sys/v4l2/gstv4l2videoenc.c index a6c44cbc6e..193d1f7234 100644 --- a/sys/v4l2/gstv4l2videoenc.c +++ b/sys/v4l2/gstv4l2videoenc.c @@ -612,37 +612,6 @@ not_negotiated: return FALSE; } -static GstVideoCodecFrame * -gst_v4l2_video_enc_get_oldest_frame (GstVideoEncoder * encoder) -{ - GstVideoCodecFrame *frame = NULL; - GList *frames, *l; - gint count = 0; - - frames = gst_video_encoder_get_frames (encoder); - - for (l = frames; l != NULL; l = l->next) { - GstVideoCodecFrame *f = l->data; - - if (!frame || frame->pts > f->pts) - frame = f; - - count++; - } - - if (frame) { - GST_LOG_OBJECT (encoder, - "Oldest frame is %d %" GST_TIME_FORMAT - " and %d frames left", - frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1); - gst_video_codec_frame_ref (frame); - } - - g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref); - - return frame; -} - static void gst_v4l2_video_enc_loop (GstVideoEncoder * encoder) { @@ -661,18 +630,24 @@ gst_v4l2_video_enc_loop (GstVideoEncoder * encoder) goto beach; } - /* FIXME Check if buffer isn't the last one here */ GST_LOG_OBJECT (encoder, "Process output buffer"); ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL - (self->v4l2capture->pool), &buffer); + (self->v4l2capture->pool), &buffer, NULL); if (ret != GST_FLOW_OK) goto beach; - frame = gst_v4l2_video_enc_get_oldest_frame (encoder); + if (GST_BUFFER_TIMESTAMP (buffer) % GST_SECOND != 0) + GST_ERROR_OBJECT (encoder, + "Driver bug detected - check driver with v4l2-compliance from http://git.linuxtv.org/v4l-utils.git"); + GST_LOG_OBJECT (encoder, "Got buffer for frame number %u", + (guint32) (GST_BUFFER_PTS (buffer) / GST_SECOND)); + frame = + gst_video_encoder_get_frame (encoder, + GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND); if (frame) { /* At this point, the delta unit buffer flag is already correctly set by @@ -781,9 +756,12 @@ gst_v4l2_video_enc_handle_frame (GstVideoEncoder * encoder, if (frame->input_buffer) { GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder); + GST_LOG_OBJECT (encoder, "Passing buffer with frame number %u", + frame->system_frame_number); ret = - gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL - (self->v4l2output->pool), &frame->input_buffer); + gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self-> + v4l2output->pool), &frame->input_buffer, + &frame->system_frame_number); GST_VIDEO_ENCODER_STREAM_LOCK (encoder); if (ret == GST_FLOW_FLUSHING) {