qtmux: adjust nasty case timestamp tracking

That is, all sorts of problems arise with re-ordered input timestamps that tend to defy automagic handling for every case, so allow for a few variations that can be tried depending on circumstances. Also try to document accordingly. Also fixes #638288.
2024-07-11 09:06:05 +00:00 · 2011-01-03 17:24:23 +01:00 · 2011-01-03 17:24:23 +01:00 · ae644937fc
parent 62c728464e
commit ae644937fc
2 changed files with 217 additions and 26 deletions
--- a/gst/quicktime/gstqtmux.c
+++ b/gst/quicktime/gstqtmux.c
@ -82,6 +82,18 @@
 * <link linkend="GstQTMux--streamable">streamable</link> allows foregoing to add
 * index metadata (at the end of file).
 *
+ * <link linkend="GstQTMux--dts-method">dts-method</link> allows selecting a
+ * method for managing input timestamps (stay tuned for 0.11 to have this
+ * automagically settled).  The default delta/duration method should handle nice
+ * (aka perfect streams) just fine, but may experience problems otherwise
+ * (e.g. input stream with re-ordered B-frames and/or with frame dropping).
+ * The re-ordering approach re-assigns incoming timestamps in ascending order
+ * to incoming buffers and offers an alternative in such cases.  In cases where
+ * that might fail, the remaining method can be tried, which is exact and
+ * according to specs, but might experience playback on not so spec-wise players.
+ * Note that this latter approach also requires one to enable
+ * <link linkend="GstQTMux--presentation-timestamp">presentation-timestamp</link>.
+ *
 * <refsect2>
 * <title>Example pipelines</title>
 * |[
@ -128,6 +140,36 @@
 GST_DEBUG_CATEGORY_STATIC (gst_qt_mux_debug);
 #define GST_CAT_DEFAULT gst_qt_mux_debug

+enum
+{
+  DTS_METHOD_DD,
+  DTS_METHOD_REORDER,
+  DTS_METHOD_ASC
+};
+
+static GType
+gst_qt_mux_dts_method_get_type (void)
+{
+  static GType gst_qt_mux_dts_method = 0;
+
+  if (!gst_qt_mux_dts_method) {
+    static const GEnumValue dts_methods[] = {
+      {DTS_METHOD_DD, "delta/duration", "dd"},
+      {DTS_METHOD_REORDER, "reorder", "reorder"},
+      {DTS_METHOD_ASC, "ascending", "asc"},
+      {0, NULL, NULL},
+    };
+
+    gst_qt_mux_dts_method =
+        g_enum_register_static ("GstQTMuxDtsMethods", dts_methods);
+  }
+
+  return gst_qt_mux_dts_method;
+}
+
+#define GST_TYPE_QT_MUX_DTS_METHOD \
+  (gst_qt_mux_dts_method_get_type ())
+
 /* QTMux signals and args */
 enum
 {
@ -140,12 +182,13 @@ enum
  PROP_0,
  PROP_MOVIE_TIMESCALE,
  PROP_TRAK_TIMESCALE,
-  PROP_DO_CTTS,
  PROP_FAST_START,
  PROP_FAST_START_TEMP_FILE,
  PROP_MOOV_RECOV_FILE,
  PROP_FRAGMENT_DURATION,
-  PROP_STREAMABLE
+  PROP_STREAMABLE,
+  PROP_DTS_METHOD,
+  PROP_DO_CTTS,
 };

 /* some spare for header size as well */
@ -160,6 +203,7 @@ enum
 #define DEFAULT_MOOV_RECOV_FILE         NULL
 #define DEFAULT_FRAGMENT_DURATION       0
 #define DEFAULT_STREAMABLE              FALSE
+#define DEFAULT_DTS_METHOD              DTS_METHOD_DD


 static void gst_qt_mux_finalize (GObject * object);
@ -264,6 +308,11 @@ gst_qt_mux_class_init (GstQTMuxClass * klass)
          "(in addition to decoding time) (use with caution)",
          DEFAULT_DO_CTTS,
          G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
+  g_object_class_install_property (gobject_class, PROP_DTS_METHOD,
+      g_param_spec_enum ("dts-method", "dts-method",
+          "Method to determine DTS time",
+          GST_TYPE_QT_MUX_DTS_METHOD, DEFAULT_DTS_METHOD,
+          G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_FAST_START,
      g_param_spec_boolean ("faststart", "Format file to faststart",
          "If the file should be formated for faststart (headers first). ",
@ -304,6 +353,8 @@ gst_qt_mux_class_init (GstQTMuxClass * klass)
 static void
 gst_qt_mux_pad_reset (GstQTPad * qtpad)
 {
+  gint i;
+
  qtpad->fourcc = 0;
  qtpad->is_out_of_order = FALSE;
  qtpad->have_dts = FALSE;
@ -314,6 +365,16 @@ gst_qt_mux_pad_reset (GstQTPad * qtpad)
  qtpad->prepare_buf_func = NULL;
  qtpad->avg_bitrate = 0;
  qtpad->max_bitrate = 0;
+  qtpad->ts_n_entries = 0;
+
+  qtpad->buf_head = 0;
+  qtpad->buf_tail = 0;
+  for (i = 0; i < G_N_ELEMENTS (qtpad->buf_entries); i++) {
+    if (qtpad->buf_entries[i]) {
+      gst_buffer_unref (qtpad->buf_entries[i]);
+      qtpad->buf_entries[i] = NULL;
+    }
+  }

  if (qtpad->last_buf)
    gst_buffer_replace (&qtpad->last_buf, NULL);
@ -1930,12 +1991,50 @@ init:
  return ret;
 }

-/* check whether @a differs from @b by order of @magn */
-static gboolean inline
-gst_qtmux_check_difference (GstQTMux * qtmux, GstClockTime a,
-    GstClockTime b, GstClockTime magn)
+/* sigh, tiny list helpers to re-order stuff */
+static void
+gst_qt_mux_push_ts (GstQTMux * qtmux, GstQTPad * pad, GstClockTime ts)
 {
-  return ((a >= b) ? (a - b >= (magn >> 1)) : (b - a >= (magn >> 1)));
+  gint i;
+
+  for (i = 0; (i < QTMUX_NO_OF_TS) && (i < pad->ts_n_entries); i++) {
+    if (ts > pad->ts_entries[i])
+      break;
+  }
+  memmove (&pad->ts_entries[i + 1], &pad->ts_entries[i],
+      sizeof (GstClockTime) * (pad->ts_n_entries - i));
+  pad->ts_entries[i] = ts;
+  pad->ts_n_entries++;
+}
+
+/* takes ownership of @buf */
+static GstBuffer *
+gst_qt_mux_get_asc_buffer_ts (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
+{
+  const gint wrap = G_N_ELEMENTS (pad->buf_entries);
+  GstClockTime ts;
+
+  /* store buffer and ts, latter ordered */
+  if (buf) {
+    pad->buf_entries[pad->buf_tail++] = buf;
+    pad->buf_tail %= wrap;
+    gst_qt_mux_push_ts (qtmux, pad, GST_BUFFER_TIMESTAMP (buf));
+  }
+
+  if (pad->ts_n_entries && (!buf || pad->ts_n_entries >= QTMUX_NO_OF_TS)) {
+    ts = pad->ts_entries[--pad->ts_n_entries];
+    buf = pad->buf_entries[pad->buf_head];
+    pad->buf_entries[pad->buf_head++] = NULL;
+    pad->buf_head %= wrap;
+    buf = gst_buffer_make_metadata_writable (buf);
+    GST_BUFFER_TIMESTAMP (buf) = ts;
+    GST_DEBUG_OBJECT (qtmux, "next buffer uses reordered ts %" GST_TIME_FORMAT,
+        GST_TIME_ARGS (ts));
+  } else {
+    buf = NULL;
+  }
+
+  return buf;
 }

 /*
@ -1947,10 +2046,12 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
  GstBuffer *last_buf = NULL;
  GstClockTime duration;
  guint nsamples, sample_size;
-  guint64 scaled_duration, chunk_offset;
-  gint64 last_dts;
+  guint64 chunk_offset;
+  gint64 last_dts, scaled_duration;
  gint64 pts_offset = 0;
  gboolean sync = FALSE, do_pts = FALSE;
+  gboolean drain = (buf == NULL);
+  GstFlowReturn ret;

  if (!pad->fourcc)
    goto not_negotiated;
@ -1960,6 +2061,15 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
    buf = pad->prepare_buf_func (pad, buf, qtmux);
  }

+again:
+  if (G_UNLIKELY (qtmux->dts_method == DTS_METHOD_REORDER)) {
+    buf = gst_qt_mux_get_asc_buffer_ts (qtmux, pad, buf);
+    if (!buf) {
+      GST_DEBUG_OBJECT (qtmux, "no reordered buffer yet");
+      return GST_FLOW_OK;
+    }
+  }
+
  last_buf = pad->last_buf;
  if (last_buf == NULL) {
 #ifndef GST_DISABLE_GST_DEBUG
@ -1979,16 +2089,29 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
    gst_buffer_ref (last_buf);

  /* nasty heuristic mess to guestimate dealing with DTS/PTS,
-   * while also trying to stay close to input ts to preserve sync, so:
+   * while also trying to stay close to input ts to preserve sync,
+   * so in DTS_METHOD_DD:
   * - prefer using input ts where possible
-   * - if those detected out-of-order (*), and input duration available,
-   *   mark as out-of-order and fallback to duration
-   * - if in out-of-order, need to preserve sync between streams, and adding
-   *   durations might drift, so try to resync when we expect
-   *   input ts == (sum of durations), which is at some keyframe input frame.
+   * - if those detected out-of-order (*), mark as out-of-order
+   * - if in out-of-order, then
+   *   - if duration available, use that as delta
+   *     Also mind to preserve sync between streams, and adding
+   *     durations might drift, so try to resync when we expect
+   *     input ts == (sum of durations), which is at some keyframe input frame.
+   *   - if no duration available, we are actually in serious trouble and need
+   *     to hack around that, so we fail.
+   * To remedy failure, alternatively, in DTS_METHOD_REORDER:
+   * - collect some buffers and re-order timestamp,
+   *   then process the oldest buffer with smallest timestamps.
+   *   This should typically compensate for some codec's handywork with ts.
+   * ... but in case this makes ts end up where not expected:
+   * - keep each ts with its buffer and still keep a list of most recent X ts,
+   *   use the (ascending) minimum of those as DTS (and the difference as ts delta),
+   *   and use this DTS as a basis to obtain a (positive) CTS offset.
+   *   This should yield exact PTS == buffer ts, but it seems not all players
+   *   out there are aware of ctts pts ...
   *
-   * (*) if input ts out-of-order, or if ts differs from (sum of durations)
-   *     by an (approx) order-of-duration magnitude
+   * 0.11 Phew, can we (pretty) please please sort out DTS/PTS on buffers ...
   */
  if (G_LIKELY (buf) && !pad->is_out_of_order) {
    if (G_LIKELY (GST_BUFFER_TIMESTAMP_IS_VALID (last_buf) &&
@ -2005,9 +2128,16 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
    }
  }

+  /* would have to be some unusual input, but not impossible */
+  if (G_UNLIKELY (qtmux->dts_method == DTS_METHOD_REORDER &&
+          pad->is_out_of_order)) {
+    goto no_order;
+  }
+
  /* fall back to duration if last buffer or
   * out-of-order (determined previously), otherwise use input ts */
-  if (buf == NULL || pad->is_out_of_order) {
+  if (buf == NULL ||
+      (pad->is_out_of_order && qtmux->dts_method == DTS_METHOD_DD)) {
    if (!GST_BUFFER_DURATION_IS_VALID (last_buf)) {
      /* be forgiving for some possibly last upstream flushed buffer */
      if (buf)
@ -2020,9 +2150,7 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
      /* avoid drift in sum timestamps,
       * so use input timestamp for suitable keyframe */
      if (buf && !GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT) &&
-          GST_BUFFER_TIMESTAMP (buf) >= pad->last_dts &&
-          !gst_qtmux_check_difference (qtmux, pad->last_dts + duration,
-              GST_BUFFER_TIMESTAMP (buf), duration)) {
+          GST_BUFFER_TIMESTAMP (buf) >= pad->last_dts) {
        GST_DEBUG_OBJECT (qtmux, "resyncing out-of-order input to ts; "
            "replacing %" GST_TIME_FORMAT " by %" GST_TIME_FORMAT,
            GST_TIME_ARGS (pad->last_dts + duration),
@ -2030,8 +2158,37 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
        duration = GST_BUFFER_TIMESTAMP (buf) - pad->last_dts;
      }
    }
-  } else {
+  } else if (qtmux->dts_method != DTS_METHOD_ASC) {
    duration = GST_BUFFER_TIMESTAMP (buf) - GST_BUFFER_TIMESTAMP (last_buf);
+  } else {
+    GstClockTime ts;
+
+    g_assert (qtmux->dts_method == DTS_METHOD_ASC);
+    if (!qtmux->guess_pts)
+      goto need_pts;
+
+    /* add timestamp to queue; keeps in descending order */
+    gst_qt_mux_push_ts (qtmux, pad, GST_BUFFER_TIMESTAMP (last_buf));
+    /* chuck out smallest/last one if we have enough */
+    if (G_LIKELY (pad->ts_n_entries > QTMUX_NO_OF_TS))
+      pad->ts_n_entries--;
+    /* peek the now smallest timestamp */
+    ts = pad->ts_entries[pad->ts_n_entries - 1];
+    /* these tails are expected to be (strictly) ascending with
+     * large enough history */
+    GST_DEBUG_OBJECT (qtmux, "ASC method; base timestamp %" GST_TIME_FORMAT,
+        GST_TIME_ARGS (ts));
+    if (ts >= pad->last_dts) {
+      duration = ts - pad->last_dts;
+    } else {
+      /* fallback to previous value, negative ct offset might handle */
+      GST_WARNING_OBJECT (qtmux, "unexpected decrease in timestamp");
+      duration = 0;
+    }
+    /* arrange for small non-zero duration/delta << expected frame time */
+    ts = gst_util_uint64_scale (10, GST_SECOND,
+        atom_trak_get_timescale (pad->trak));
+    duration = MAX (duration, ts);
  }

  gst_buffer_replace (&pad->last_buf, buf);
@ -2166,15 +2323,23 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)

  if (qtmux->fragment_sequence) {
    /* ensure that always sync samples are marked as such */
-    return gst_qt_mux_pad_fragment_add_buffer (qtmux, pad, last_buf,
-        buf == NULL, nsamples, last_dts, scaled_duration, sample_size,
+    ret = gst_qt_mux_pad_fragment_add_buffer (qtmux, pad, last_buf,
+        buf == NULL, nsamples, last_dts, (gint32) scaled_duration, sample_size,
        !pad->sync || sync, pts_offset);
  } else {
-    atom_trak_add_samples (pad->trak, nsamples, scaled_duration,
+    atom_trak_add_samples (pad->trak, nsamples, (gint32) scaled_duration,
        sample_size, chunk_offset, sync, pts_offset);
-    return gst_qt_mux_send_buffer (qtmux, last_buf, &qtmux->mdat_size, TRUE);
+    ret = gst_qt_mux_send_buffer (qtmux, last_buf, &qtmux->mdat_size, TRUE);
  }

+  if (G_UNLIKELY (drain && qtmux->dts_method == DTS_METHOD_REORDER &&
+          ret == GST_FLOW_OK)) {
+    buf = NULL;
+    goto again;
+  }
+
+  return ret;
+
  /* ERRORS */
 bail:
  {
@ -2189,6 +2354,18 @@ no_time:
        ("Received buffer without timestamp/duration."));
    goto bail;
  }
+no_order:
+  {
+    GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
+        ("DTS method failed to re-order timestamps."));
+    goto bail;
+  }
+need_pts:
+  {
+    GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
+        ("Selected DTS method also needs PTS enabled."));
+    goto bail;
+  }
 fragmented_sample:
  {
    GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
@ -3087,6 +3264,9 @@ gst_qt_mux_get_property (GObject * object,
    case PROP_DO_CTTS:
      g_value_set_boolean (value, qtmux->guess_pts);
      break;
+    case PROP_DTS_METHOD:
+      g_value_set_enum (value, qtmux->dts_method);
+      break;
    case PROP_FAST_START:
      g_value_set_boolean (value, qtmux->fast_start);
      break;
@ -3139,6 +3319,9 @@ gst_qt_mux_set_property (GObject * object,
    case PROP_DO_CTTS:
      qtmux->guess_pts = g_value_get_boolean (value);
      break;
+    case PROP_DTS_METHOD:
+      qtmux->dts_method = g_value_get_enum (value);
+      break;
    case PROP_FAST_START:
      qtmux->fast_start = g_value_get_boolean (value);
      break;
--- a/gst/quicktime/gstqtmux.h
+++ b/gst/quicktime/gstqtmux.h
@ -78,6 +78,8 @@ typedef struct _GstQTPad GstQTPad;
 typedef GstBuffer * (*GstQTPadPrepareBufferFunc) (GstQTPad * pad,
    GstBuffer * buf, GstQTMux * qtmux);

+#define QTMUX_NO_OF_TS   10
+
 struct _GstQTPad
 {
  GstCollectData collect;       /* we extend the CollectData */
@ -102,6 +104,11 @@ struct _GstQTPad
  /* store the first timestamp for comparing with other streams and
   * know if there are late streams */
  GstClockTime first_ts;
+  GstClockTime ts_entries[QTMUX_NO_OF_TS + 2];
+  guint ts_n_entries;
+  GstBuffer *buf_entries[QTMUX_NO_OF_TS + 2];
+  guint buf_head;
+  guint buf_tail;

  /* all the atom and chunk book-keeping is delegated here
   * unowned/uncounted reference, parent MOOV owns */
@ -174,6 +181,7 @@ struct _GstQTMux
  AtomsTreeFlavor flavor;
  gboolean fast_start;
  gboolean guess_pts;
+  gint dts_method;
  gchar *fast_start_file_path;
  gchar *moov_recov_file_path;
  guint32 fragment_duration;