video{de,en}coder: fix missing timestamp estimating

... by having some more timestamp tracking in a private frame field.
Not doing so would lead to (a.o.) losing the needed minimum timestamp in
an earlier sent frame.

gst-libs/gst/video/gstvideodecoder.c

@@ -2147,21 +2147,44 @@ gst_video_decoder_prepare_finish_frame (GstVideoDecoder *
         GST_TIME_ARGS (frame->duration));
   }
 
-  if (frame->pts == GST_CLOCK_TIME_NONE &&
-      GST_CLOCK_TIME_IS_VALID (priv->pts_delta) && priv->frames) {
-    GstVideoCodecFrame *oframe = priv->frames->data;
+  /* PTS is expected montone ascending,
+   * so a good guess is lowest unsent DTS */
+  {
+    GstClockTime min_ts = GST_CLOCK_TIME_NONE;
+    GstVideoCodecFrame *oframe = NULL;
+    gboolean seen_none = FALSE;
 
-    /* valid delta, so we have some reasonable DTS input,
-     * then outgoing PTS = smallest DTS = DTS of oldest frame */
-    if (GST_CLOCK_TIME_IS_VALID (oframe->dts)) {
-      frame->pts = oframe->dts + priv->pts_delta;
-      GST_LOG_OBJECT (decoder,
-          "Guessing timestamp %" GST_TIME_FORMAT
-          " from DTS %" GST_TIME_FORMAT " for frame...",
-          GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (oframe->dts));
+    /* some maintenance regardless */
+    for (l = priv->frames; l; l = l->next) {
+      GstVideoCodecFrame *tmp = l->data;
+
+      if (!GST_CLOCK_TIME_IS_VALID (tmp->abidata.ABI.ts)) {
+        seen_none = TRUE;
+        continue;
+      }
+
+      if (!GST_CLOCK_TIME_IS_VALID (min_ts) || tmp->abidata.ABI.ts < min_ts) {
+        min_ts = tmp->abidata.ABI.ts;
+        oframe = tmp;
+      }
+    }
+    /* save a ts if needed */
+    if (oframe && oframe != frame) {
+      oframe->abidata.ABI.ts = frame->abidata.ABI.ts;
+    }
+
+    /* and set if needed;
+     * valid delta means we have reasonable DTS input */
+    if (!GST_CLOCK_TIME_IS_VALID (frame->pts) && !seen_none &&
+        GST_CLOCK_TIME_IS_VALID (priv->pts_delta)) {
+      frame->pts = min_ts + priv->pts_delta;
+      GST_DEBUG_OBJECT (decoder,
+          "no valid PTS, using oldest DTS %" GST_TIME_FORMAT,
+          GST_TIME_ARGS (frame->pts));
     }
   }
 
+
   if (frame->pts == GST_CLOCK_TIME_NONE) {
     /* Last ditch timestamp guess: Just add the duration to the previous
      * frame */
@@ -2568,6 +2591,7 @@ gst_video_decoder_decode_frame (GstVideoDecoder * decoder,
   frame->pts = GST_BUFFER_PTS (frame->input_buffer);
   frame->dts = GST_BUFFER_DTS (frame->input_buffer);
   frame->duration = GST_BUFFER_DURATION (frame->input_buffer);
+  frame->abidata.ABI.ts = frame->dts;
 
   /* For keyframes, PTS = DTS */
   if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) {

gst-libs/gst/video/gstvideoencoder.c

@@ -1217,6 +1217,7 @@ gst_video_encoder_new_frame (GstVideoEncoder * encoder, GstBuffer * buf,
   frame->pts = pts;
   frame->dts = dts;
   frame->duration = duration;
+  frame->abidata.ABI.ts = pts;
 
   return frame;
 }
@@ -1757,15 +1758,39 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder,
     GST_BUFFER_FLAG_SET (frame->output_buffer, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  /* DTS is expected monotone ascending, so a good guess is the lowest PTS
-   * of all pending frames, i.e. the oldest frame's PTS (all being OK) */
-  if (!GST_CLOCK_TIME_IS_VALID (frame->dts) && encoder->priv->frames) {
-    GstVideoCodecFrame *oframe = encoder->priv->frames->data;
+  /* DTS is expected monotone ascending,
+   * so a good guess is the lowest unsent PTS (all being OK) */
+  {
+    GstClockTime min_ts = GST_CLOCK_TIME_NONE;
+    GstVideoCodecFrame *oframe = NULL;
+    gboolean seen_none = FALSE;
 
-    frame->dts = oframe->pts;
-    GST_DEBUG_OBJECT (encoder,
-        "no valid DTS, using oldest frame's PTS %" GST_TIME_FORMAT,
-        GST_TIME_ARGS (frame->pts));
+    /* some maintenance regardless */
+    for (l = priv->frames; l; l = l->next) {
+      GstVideoCodecFrame *tmp = l->data;
+
+      if (!GST_CLOCK_TIME_IS_VALID (tmp->abidata.ABI.ts)) {
+        seen_none = TRUE;
+        continue;
+      }
+
+      if (!GST_CLOCK_TIME_IS_VALID (min_ts) || tmp->abidata.ABI.ts < min_ts) {
+        min_ts = tmp->abidata.ABI.ts;
+        oframe = tmp;
+      }
+    }
+    /* save a ts if needed */
+    if (oframe && oframe != frame) {
+      oframe->abidata.ABI.ts = frame->abidata.ABI.ts;
+    }
+
+    /* and set if needed */
+    if (!GST_CLOCK_TIME_IS_VALID (frame->dts) && !seen_none) {
+      frame->dts = min_ts;
+      GST_DEBUG_OBJECT (encoder,
+          "no valid DTS, using oldest PTS %" GST_TIME_FORMAT,
+          GST_TIME_ARGS (frame->pts));
+    }
   }
 
   frame->distance_from_sync = priv->distance_from_sync;

gst-libs/gst/video/gstvideoutils.h

@@ -246,7 +246,12 @@ struct _GstVideoCodecFrame
   gpointer       user_data;
   GDestroyNotify user_data_destroy_notify;
 
-  void         *padding[GST_PADDING_LARGE];
+  union {
+    struct {
+      GstClockTime ts;
+    } ABI;
+    void         *padding[GST_PADDING_LARGE];
+  } abidata;
 };
 
 /* GstVideoCodecState */