diff --git a/gst/quicktime/gstqtmux.c b/gst/quicktime/gstqtmux.c index 29edc2a23d..0ac1e91141 100644 --- a/gst/quicktime/gstqtmux.c +++ b/gst/quicktime/gstqtmux.c @@ -82,6 +82,18 @@ * streamable allows foregoing to add * index metadata (at the end of file). * + * dts-method allows selecting a + * method for managing input timestamps (stay tuned for 0.11 to have this + * automagically settled). The default delta/duration method should handle nice + * (aka perfect streams) just fine, but may experience problems otherwise + * (e.g. input stream with re-ordered B-frames and/or with frame dropping). + * The re-ordering approach re-assigns incoming timestamps in ascending order + * to incoming buffers and offers an alternative in such cases. In cases where + * that might fail, the remaining method can be tried, which is exact and + * according to specs, but might experience playback on not so spec-wise players. + * Note that this latter approach also requires one to enable + * presentation-timestamp. + * * * Example pipelines * |[ @@ -128,6 +140,36 @@ GST_DEBUG_CATEGORY_STATIC (gst_qt_mux_debug); #define GST_CAT_DEFAULT gst_qt_mux_debug +enum +{ + DTS_METHOD_DD, + DTS_METHOD_REORDER, + DTS_METHOD_ASC +}; + +static GType +gst_qt_mux_dts_method_get_type (void) +{ + static GType gst_qt_mux_dts_method = 0; + + if (!gst_qt_mux_dts_method) { + static const GEnumValue dts_methods[] = { + {DTS_METHOD_DD, "delta/duration", "dd"}, + {DTS_METHOD_REORDER, "reorder", "reorder"}, + {DTS_METHOD_ASC, "ascending", "asc"}, + {0, NULL, NULL}, + }; + + gst_qt_mux_dts_method = + g_enum_register_static ("GstQTMuxDtsMethods", dts_methods); + } + + return gst_qt_mux_dts_method; +} + +#define GST_TYPE_QT_MUX_DTS_METHOD \ + (gst_qt_mux_dts_method_get_type ()) + /* QTMux signals and args */ enum { @@ -140,12 +182,13 @@ enum PROP_0, PROP_MOVIE_TIMESCALE, PROP_TRAK_TIMESCALE, - PROP_DO_CTTS, PROP_FAST_START, PROP_FAST_START_TEMP_FILE, PROP_MOOV_RECOV_FILE, PROP_FRAGMENT_DURATION, - PROP_STREAMABLE + PROP_STREAMABLE, + PROP_DTS_METHOD, + PROP_DO_CTTS, }; /* some spare for header size as well */ @@ -160,6 +203,7 @@ enum #define DEFAULT_MOOV_RECOV_FILE NULL #define DEFAULT_FRAGMENT_DURATION 0 #define DEFAULT_STREAMABLE FALSE +#define DEFAULT_DTS_METHOD DTS_METHOD_DD static void gst_qt_mux_finalize (GObject * object); @@ -264,6 +308,11 @@ gst_qt_mux_class_init (GstQTMuxClass * klass) "(in addition to decoding time) (use with caution)", DEFAULT_DO_CTTS, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_DTS_METHOD, + g_param_spec_enum ("dts-method", "dts-method", + "Method to determine DTS time", + GST_TYPE_QT_MUX_DTS_METHOD, DEFAULT_DTS_METHOD, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_FAST_START, g_param_spec_boolean ("faststart", "Format file to faststart", "If the file should be formated for faststart (headers first). ", @@ -304,6 +353,8 @@ gst_qt_mux_class_init (GstQTMuxClass * klass) static void gst_qt_mux_pad_reset (GstQTPad * qtpad) { + gint i; + qtpad->fourcc = 0; qtpad->is_out_of_order = FALSE; qtpad->have_dts = FALSE; @@ -314,6 +365,16 @@ gst_qt_mux_pad_reset (GstQTPad * qtpad) qtpad->prepare_buf_func = NULL; qtpad->avg_bitrate = 0; qtpad->max_bitrate = 0; + qtpad->ts_n_entries = 0; + + qtpad->buf_head = 0; + qtpad->buf_tail = 0; + for (i = 0; i < G_N_ELEMENTS (qtpad->buf_entries); i++) { + if (qtpad->buf_entries[i]) { + gst_buffer_unref (qtpad->buf_entries[i]); + qtpad->buf_entries[i] = NULL; + } + } if (qtpad->last_buf) gst_buffer_replace (&qtpad->last_buf, NULL); @@ -1930,12 +1991,50 @@ init: return ret; } -/* check whether @a differs from @b by order of @magn */ -static gboolean inline -gst_qtmux_check_difference (GstQTMux * qtmux, GstClockTime a, - GstClockTime b, GstClockTime magn) +/* sigh, tiny list helpers to re-order stuff */ +static void +gst_qt_mux_push_ts (GstQTMux * qtmux, GstQTPad * pad, GstClockTime ts) { - return ((a >= b) ? (a - b >= (magn >> 1)) : (b - a >= (magn >> 1))); + gint i; + + for (i = 0; (i < QTMUX_NO_OF_TS) && (i < pad->ts_n_entries); i++) { + if (ts > pad->ts_entries[i]) + break; + } + memmove (&pad->ts_entries[i + 1], &pad->ts_entries[i], + sizeof (GstClockTime) * (pad->ts_n_entries - i)); + pad->ts_entries[i] = ts; + pad->ts_n_entries++; +} + +/* takes ownership of @buf */ +static GstBuffer * +gst_qt_mux_get_asc_buffer_ts (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) +{ + const gint wrap = G_N_ELEMENTS (pad->buf_entries); + GstClockTime ts; + + /* store buffer and ts, latter ordered */ + if (buf) { + pad->buf_entries[pad->buf_tail++] = buf; + pad->buf_tail %= wrap; + gst_qt_mux_push_ts (qtmux, pad, GST_BUFFER_TIMESTAMP (buf)); + } + + if (pad->ts_n_entries && (!buf || pad->ts_n_entries >= QTMUX_NO_OF_TS)) { + ts = pad->ts_entries[--pad->ts_n_entries]; + buf = pad->buf_entries[pad->buf_head]; + pad->buf_entries[pad->buf_head++] = NULL; + pad->buf_head %= wrap; + buf = gst_buffer_make_metadata_writable (buf); + GST_BUFFER_TIMESTAMP (buf) = ts; + GST_DEBUG_OBJECT (qtmux, "next buffer uses reordered ts %" GST_TIME_FORMAT, + GST_TIME_ARGS (ts)); + } else { + buf = NULL; + } + + return buf; } /* @@ -1947,10 +2046,12 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) GstBuffer *last_buf = NULL; GstClockTime duration; guint nsamples, sample_size; - guint64 scaled_duration, chunk_offset; - gint64 last_dts; + guint64 chunk_offset; + gint64 last_dts, scaled_duration; gint64 pts_offset = 0; gboolean sync = FALSE, do_pts = FALSE; + gboolean drain = (buf == NULL); + GstFlowReturn ret; if (!pad->fourcc) goto not_negotiated; @@ -1960,6 +2061,15 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) buf = pad->prepare_buf_func (pad, buf, qtmux); } +again: + if (G_UNLIKELY (qtmux->dts_method == DTS_METHOD_REORDER)) { + buf = gst_qt_mux_get_asc_buffer_ts (qtmux, pad, buf); + if (!buf) { + GST_DEBUG_OBJECT (qtmux, "no reordered buffer yet"); + return GST_FLOW_OK; + } + } + last_buf = pad->last_buf; if (last_buf == NULL) { #ifndef GST_DISABLE_GST_DEBUG @@ -1979,16 +2089,29 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) gst_buffer_ref (last_buf); /* nasty heuristic mess to guestimate dealing with DTS/PTS, - * while also trying to stay close to input ts to preserve sync, so: + * while also trying to stay close to input ts to preserve sync, + * so in DTS_METHOD_DD: * - prefer using input ts where possible - * - if those detected out-of-order (*), and input duration available, - * mark as out-of-order and fallback to duration - * - if in out-of-order, need to preserve sync between streams, and adding - * durations might drift, so try to resync when we expect - * input ts == (sum of durations), which is at some keyframe input frame. + * - if those detected out-of-order (*), mark as out-of-order + * - if in out-of-order, then + * - if duration available, use that as delta + * Also mind to preserve sync between streams, and adding + * durations might drift, so try to resync when we expect + * input ts == (sum of durations), which is at some keyframe input frame. + * - if no duration available, we are actually in serious trouble and need + * to hack around that, so we fail. + * To remedy failure, alternatively, in DTS_METHOD_REORDER: + * - collect some buffers and re-order timestamp, + * then process the oldest buffer with smallest timestamps. + * This should typically compensate for some codec's handywork with ts. + * ... but in case this makes ts end up where not expected: + * - keep each ts with its buffer and still keep a list of most recent X ts, + * use the (ascending) minimum of those as DTS (and the difference as ts delta), + * and use this DTS as a basis to obtain a (positive) CTS offset. + * This should yield exact PTS == buffer ts, but it seems not all players + * out there are aware of ctts pts ... * - * (*) if input ts out-of-order, or if ts differs from (sum of durations) - * by an (approx) order-of-duration magnitude + * 0.11 Phew, can we (pretty) please please sort out DTS/PTS on buffers ... */ if (G_LIKELY (buf) && !pad->is_out_of_order) { if (G_LIKELY (GST_BUFFER_TIMESTAMP_IS_VALID (last_buf) && @@ -2005,9 +2128,16 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) } } + /* would have to be some unusual input, but not impossible */ + if (G_UNLIKELY (qtmux->dts_method == DTS_METHOD_REORDER && + pad->is_out_of_order)) { + goto no_order; + } + /* fall back to duration if last buffer or * out-of-order (determined previously), otherwise use input ts */ - if (buf == NULL || pad->is_out_of_order) { + if (buf == NULL || + (pad->is_out_of_order && qtmux->dts_method == DTS_METHOD_DD)) { if (!GST_BUFFER_DURATION_IS_VALID (last_buf)) { /* be forgiving for some possibly last upstream flushed buffer */ if (buf) @@ -2020,9 +2150,7 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) /* avoid drift in sum timestamps, * so use input timestamp for suitable keyframe */ if (buf && !GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT) && - GST_BUFFER_TIMESTAMP (buf) >= pad->last_dts && - !gst_qtmux_check_difference (qtmux, pad->last_dts + duration, - GST_BUFFER_TIMESTAMP (buf), duration)) { + GST_BUFFER_TIMESTAMP (buf) >= pad->last_dts) { GST_DEBUG_OBJECT (qtmux, "resyncing out-of-order input to ts; " "replacing %" GST_TIME_FORMAT " by %" GST_TIME_FORMAT, GST_TIME_ARGS (pad->last_dts + duration), @@ -2030,8 +2158,37 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) duration = GST_BUFFER_TIMESTAMP (buf) - pad->last_dts; } } - } else { + } else if (qtmux->dts_method != DTS_METHOD_ASC) { duration = GST_BUFFER_TIMESTAMP (buf) - GST_BUFFER_TIMESTAMP (last_buf); + } else { + GstClockTime ts; + + g_assert (qtmux->dts_method == DTS_METHOD_ASC); + if (!qtmux->guess_pts) + goto need_pts; + + /* add timestamp to queue; keeps in descending order */ + gst_qt_mux_push_ts (qtmux, pad, GST_BUFFER_TIMESTAMP (last_buf)); + /* chuck out smallest/last one if we have enough */ + if (G_LIKELY (pad->ts_n_entries > QTMUX_NO_OF_TS)) + pad->ts_n_entries--; + /* peek the now smallest timestamp */ + ts = pad->ts_entries[pad->ts_n_entries - 1]; + /* these tails are expected to be (strictly) ascending with + * large enough history */ + GST_DEBUG_OBJECT (qtmux, "ASC method; base timestamp %" GST_TIME_FORMAT, + GST_TIME_ARGS (ts)); + if (ts >= pad->last_dts) { + duration = ts - pad->last_dts; + } else { + /* fallback to previous value, negative ct offset might handle */ + GST_WARNING_OBJECT (qtmux, "unexpected decrease in timestamp"); + duration = 0; + } + /* arrange for small non-zero duration/delta << expected frame time */ + ts = gst_util_uint64_scale (10, GST_SECOND, + atom_trak_get_timescale (pad->trak)); + duration = MAX (duration, ts); } gst_buffer_replace (&pad->last_buf, buf); @@ -2166,15 +2323,23 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) if (qtmux->fragment_sequence) { /* ensure that always sync samples are marked as such */ - return gst_qt_mux_pad_fragment_add_buffer (qtmux, pad, last_buf, - buf == NULL, nsamples, last_dts, scaled_duration, sample_size, + ret = gst_qt_mux_pad_fragment_add_buffer (qtmux, pad, last_buf, + buf == NULL, nsamples, last_dts, (gint32) scaled_duration, sample_size, !pad->sync || sync, pts_offset); } else { - atom_trak_add_samples (pad->trak, nsamples, scaled_duration, + atom_trak_add_samples (pad->trak, nsamples, (gint32) scaled_duration, sample_size, chunk_offset, sync, pts_offset); - return gst_qt_mux_send_buffer (qtmux, last_buf, &qtmux->mdat_size, TRUE); + ret = gst_qt_mux_send_buffer (qtmux, last_buf, &qtmux->mdat_size, TRUE); } + if (G_UNLIKELY (drain && qtmux->dts_method == DTS_METHOD_REORDER && + ret == GST_FLOW_OK)) { + buf = NULL; + goto again; + } + + return ret; + /* ERRORS */ bail: { @@ -2189,6 +2354,18 @@ no_time: ("Received buffer without timestamp/duration.")); goto bail; } +no_order: + { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("DTS method failed to re-order timestamps.")); + goto bail; + } +need_pts: + { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("Selected DTS method also needs PTS enabled.")); + goto bail; + } fragmented_sample: { GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), @@ -3087,6 +3264,9 @@ gst_qt_mux_get_property (GObject * object, case PROP_DO_CTTS: g_value_set_boolean (value, qtmux->guess_pts); break; + case PROP_DTS_METHOD: + g_value_set_enum (value, qtmux->dts_method); + break; case PROP_FAST_START: g_value_set_boolean (value, qtmux->fast_start); break; @@ -3139,6 +3319,9 @@ gst_qt_mux_set_property (GObject * object, case PROP_DO_CTTS: qtmux->guess_pts = g_value_get_boolean (value); break; + case PROP_DTS_METHOD: + qtmux->dts_method = g_value_get_enum (value); + break; case PROP_FAST_START: qtmux->fast_start = g_value_get_boolean (value); break; diff --git a/gst/quicktime/gstqtmux.h b/gst/quicktime/gstqtmux.h index 4df0c3e2b9..3a2cb492fe 100644 --- a/gst/quicktime/gstqtmux.h +++ b/gst/quicktime/gstqtmux.h @@ -78,6 +78,8 @@ typedef struct _GstQTPad GstQTPad; typedef GstBuffer * (*GstQTPadPrepareBufferFunc) (GstQTPad * pad, GstBuffer * buf, GstQTMux * qtmux); +#define QTMUX_NO_OF_TS 10 + struct _GstQTPad { GstCollectData collect; /* we extend the CollectData */ @@ -102,6 +104,11 @@ struct _GstQTPad /* store the first timestamp for comparing with other streams and * know if there are late streams */ GstClockTime first_ts; + GstClockTime ts_entries[QTMUX_NO_OF_TS + 2]; + guint ts_n_entries; + GstBuffer *buf_entries[QTMUX_NO_OF_TS + 2]; + guint buf_head; + guint buf_tail; /* all the atom and chunk book-keeping is delegated here * unowned/uncounted reference, parent MOOV owns */ @@ -174,6 +181,7 @@ struct _GstQTMux AtomsTreeFlavor flavor; gboolean fast_start; gboolean guess_pts; + gint dts_method; gchar *fast_start_file_path; gchar *moov_recov_file_path; guint32 fragment_duration;