From c1b100cf9cbe50dd677fddef0eba68c7a195573f Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Mon, 8 Nov 2010 14:06:15 +0100 Subject: [PATCH] deinterlace: Add support for deinterlacing using buffer caps/flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When not using the fieldanalysis element immediately upstream of deinterlace, behaviour should remain unchanged. fieldanalysis will set the caps and flags on the buffers such that they can be interpreted and acted upon to produce progressive output. There are two main modes of operation: - Passive pattern locking Passive pattern locking is a non-blocking, low-latency mode of operation that is suitable for close-to-live usage. Initially a telecine stream will be output as variable framerate with naïve timestamp adjustment. With each incoming buffer, an attempt is made to lock onto a pattern. When a lock is obtained, the src pad and output buffer caps will reflect the pattern and timestamps will be accurately interpolated between pattern repeats. This means that initially and at pattern transitions there will be short periods of inaccurate timestamping. - Active pattern locking Active pattern locking is a blocking, high-latency mode of operation that is targeted at use-cases where timestamp accuracy is paramount. Buffers will be queued until enough are present to make a lock. When locked, timestamps will be accurately interpolated between pattern repeats. Orphan fields can be dropped or deinterlaced. If no lock can be obtained, a single field might be pushed through to be deinterlaced. Locking can also be disabled or 'auto' chooses between passive and active locking modes depending on whether upstream is live. --- gst/deinterlace/gstdeinterlace.c | 1196 +++++++++++++++++++++++++----- gst/deinterlace/gstdeinterlace.h | 59 +- 2 files changed, 1075 insertions(+), 180 deletions(-) diff --git a/gst/deinterlace/gstdeinterlace.c b/gst/deinterlace/gstdeinterlace.c index ce76f720c8..0ea263b5c6 100644 --- a/gst/deinterlace/gstdeinterlace.c +++ b/gst/deinterlace/gstdeinterlace.c @@ -2,6 +2,7 @@ * GStreamer * Copyright (C) 2005 Martin Eikermann * Copyright (C) 2008-2010 Sebastian Dröge + * Copyright (C) 2011 Robert Swain * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -55,6 +56,9 @@ GST_DEBUG_CATEGORY_STATIC (deinterlace_debug); #define DEFAULT_METHOD GST_DEINTERLACE_LINEAR #define DEFAULT_FIELDS GST_DEINTERLACE_ALL #define DEFAULT_FIELD_LAYOUT GST_DEINTERLACE_LAYOUT_AUTO +#define DEFAULT_LOCKING GST_DEINTERLACE_LOCKING_NONE +#define DEFAULT_IGNORE_OBSCURE TRUE +#define DEFAULT_DROP_ORPHANS TRUE enum { @@ -63,9 +67,73 @@ enum PROP_METHOD, PROP_FIELDS, PROP_FIELD_LAYOUT, + PROP_LOCKING, + PROP_IGNORE_OBSCURE, + PROP_DROP_ORPHANS, PROP_LAST }; +#define GST_DEINTERLACE_BUFFER_STATE_P (1<<0) +#define GST_DEINTERLACE_BUFFER_STATE_I (1<<1) +#define GST_DEINTERLACE_BUFFER_STATE_TC_B (1<<2) +#define GST_DEINTERLACE_BUFFER_STATE_TC_T (1<<3) +#define GST_DEINTERLACE_BUFFER_STATE_TC_P (1<<4) +#define GST_DEINTERLACE_BUFFER_STATE_TC_M (1<<5) +#define GST_DEINTERLACE_BUFFER_STATE_DROP (1<<6) + +#define GST_ONE \ + (GST_DEINTERLACE_BUFFER_STATE_TC_T | GST_DEINTERLACE_BUFFER_STATE_TC_B) +#define GST_PRG \ + (GST_DEINTERLACE_BUFFER_STATE_P | GST_DEINTERLACE_BUFFER_STATE_TC_P) +#define GST_INT \ + (GST_DEINTERLACE_BUFFER_STATE_I | GST_DEINTERLACE_BUFFER_STATE_TC_M) +#define GST_DRP (GST_DEINTERLACE_BUFFER_STATE_DROP) + +#define GST_DEINTERLACE_OBSCURE_THRESHOLD 5 + +static const TelecinePattern telecine_patterns[] = { + /* 60i -> 60p or 50i -> 50p (NOTE THE WEIRD RATIOS) */ + {"1:1", 1, 2, 1, {GST_ONE,}}, + /* 60i -> 30p or 50i -> 25p */ + {"2:2", 1, 1, 1, {GST_INT,}}, + /* 60i telecine -> 24p */ + {"2:3", 5, 4, 5, {GST_PRG, GST_PRG, GST_ONE, GST_ONE, GST_PRG,}}, + {"3:2:2:3", 5, 4, 5, {GST_PRG, GST_ONE, GST_INT, GST_ONE, GST_PRG,}}, + {"2:3:3:2", 5, 4, 5, {GST_PRG, GST_PRG, GST_DRP, GST_PRG, GST_PRG,}}, + + /* The following patterns are obscure and are ignored if ignore-obscure is + * set to true. If any patterns are added above this line, check and edit + * GST_DEINTERLACE_OBSCURE_THRESHOLD */ + + /* 50i Euro pulldown -> 24p */ + {"2-11:3", 25, 24, 25, {GST_PRG, GST_PRG, GST_PRG, GST_PRG, GST_PRG, + GST_PRG, GST_PRG, GST_PRG, GST_PRG, GST_PRG, + GST_PRG, GST_PRG, GST_ONE, GST_INT, GST_INT, + GST_INT, GST_INT, GST_INT, GST_INT, GST_INT, + GST_INT, GST_INT, GST_INT, GST_ONE, GST_PRG,}}, + /* 60i (NTSC 30000/1001) -> 16p (16000/1001) */ + {"3:4-3", 15, 8, 15, {GST_PRG, GST_DRP, GST_PRG, GST_DRP, GST_PRG, + GST_DRP, GST_PRG, GST_DRP, GST_PRG, GST_DRP, + GST_PRG, GST_DRP, GST_PRG, GST_DRP, GST_PRG,}}, + /* 50i (PAL) -> 16p */ + {"3-7:4", 25, 16, 25, {GST_PRG, GST_DRP, GST_PRG, GST_PRG, GST_DRP, + GST_PRG, GST_PRG, GST_DRP, GST_PRG, GST_PRG, + GST_DRP, GST_PRG, GST_DRP, GST_PRG, GST_PRG, + GST_DRP, GST_PRG, GST_PRG, GST_DRP, GST_PRG, + GST_PRG, GST_DRP, GST_PRG, GST_PRG, GST_DRP,}}, + /* NTSC 60i -> 18p */ + {"3:3:4", 5, 3, 5, {GST_PRG, GST_DRP, GST_PRG, GST_DRP, GST_PRG,}}, + /* NTSC 60i -> 20p */ + {"3:3", 3, 2, 3, {GST_PRG, GST_DRP, GST_PRG,}}, + /* NTSC 60i -> 27.5 */ + {"3:2-4", 11, 10, 11, {GST_PRG, GST_PRG, GST_PRG, GST_PRG, GST_PRG, + GST_PRG, GST_ONE, GST_INT, GST_INT, GST_INT, + GST_ONE,}}, + /* PAL 50i -> 27.5 */ + {"1:2-4", 9, 9, 10, {GST_PRG, GST_PRG, GST_PRG, GST_PRG, GST_INT, + GST_INT, GST_INT, GST_INT, GST_INT,}}, +}; + static const GEnumValue methods_types[] = { {GST_DEINTERLACE_TOMSMOCOMP, "Motion Adaptive: Motion Search", "tomsmocomp"}, @@ -85,6 +153,21 @@ static const GEnumValue methods_types[] = { {0, NULL, NULL}, }; +static const GEnumValue locking_types[] = { + {GST_DEINTERLACE_LOCKING_NONE, + "No pattern locking", "none"}, + {GST_DEINTERLACE_LOCKING_AUTO, + "Choose passive/active locking depending on whether upstream is live", + "auto"}, + {GST_DEINTERLACE_LOCKING_ACTIVE, + "Block until pattern-locked. Use accurate timestamp interpolation within a pattern repeat.", + "active"}, + {GST_DEINTERLACE_LOCKING_PASSIVE, + "Do not block. Use naïve timestamp adjustment until pattern-locked based on state history.", + "passive"}, + {0, NULL, NULL}, +}; + #define GST_TYPE_DEINTERLACE_METHODS (gst_deinterlace_methods_get_type ()) static GType @@ -160,6 +243,21 @@ gst_deinterlace_modes_get_type (void) return deinterlace_modes_type; } +#define GST_TYPE_DEINTERLACE_LOCKING (gst_deinterlace_locking_get_type ()) +static GType +gst_deinterlace_locking_get_type (void) +{ + static GType deinterlace_locking_type = 0; + + if (!deinterlace_locking_type) { + deinterlace_locking_type = + g_enum_register_static ("GstDeinterlaceLocking", locking_types); + } + + return deinterlace_locking_type; +} + + #define DEINTERLACE_CAPS \ GST_VIDEO_CAPS_YUV ("{ AYUV, Y444, YUY2, YVYU, UYVY, Y42B, I420, YV12, Y41B, NV12, NV21 }") ";" \ GST_VIDEO_CAPS_ARGB ";" GST_VIDEO_CAPS_ABGR ";" \ @@ -504,6 +602,51 @@ gst_deinterlace_class_init (GstDeinterlaceClass * klass) DEFAULT_FIELD_LAYOUT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) ); + /** + * GstDeinterlace:locking + * + * This selects which approach to pattern locking is used which affects + * processing latency and accuracy of timestamp adjustment for telecine + * streams. + * + * Since: 0.10.29. + * + */ + g_object_class_install_property (gobject_class, PROP_LOCKING, + g_param_spec_enum ("locking", "locking", "Pattern locking mode", + GST_TYPE_DEINTERLACE_LOCKING, DEFAULT_LOCKING, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + /** + * GstDeinterlace:ignore-obscure + * + * This selects whether to ignore obscure/rare telecine patterns. + * NTSC 2:3 pulldown variants are the only really common patterns. + * + * Since: 0.10.29. + * + */ + g_object_class_install_property (gobject_class, PROP_IGNORE_OBSCURE, + g_param_spec_boolean ("ignore-obscure", "ignore-obscure", + "Ignore obscure telecine patterns (only consider P, I and 2:3 " + "variants).", DEFAULT_IGNORE_OBSCURE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + /** + * GstDeinterlace:drop-orphans + * + * This selects whether to drop orphan fields at the beginning of telecine + * patterns in active locking mode. + * + * Since: 0.10.29. + * + */ + g_object_class_install_property (gobject_class, PROP_DROP_ORPHANS, + g_param_spec_boolean ("drop-orphans", "drop-orphans", + "Drop orphan fields at the beginning of telecine patterns in " + "active locking mode.", DEFAULT_DROP_ORPHANS, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + element_class->change_state = GST_DEBUG_FUNCPTR (gst_deinterlace_change_state); } @@ -571,7 +714,17 @@ gst_deinterlace_init (GstDeinterlace * self, GstDeinterlaceClass * klass) gst_deinterlace_set_method (self, self->user_set_method_id); self->fields = DEFAULT_FIELDS; self->field_layout = DEFAULT_FIELD_LAYOUT; + self->locking = DEFAULT_LOCKING; + self->ignore_obscure = DEFAULT_IGNORE_OBSCURE; + self->drop_orphans = DEFAULT_DROP_ORPHANS; + self->low_latency = -1; + self->pattern = -1; + self->pattern_phase = -1; + self->pattern_count = 0; + self->output_count = 0; + self->pattern_base_ts = GST_CLOCK_TIME_NONE; + self->pattern_buf_dur = GST_CLOCK_TIME_NONE; self->still_frame_mode = FALSE; gst_deinterlace_reset (self); @@ -606,6 +759,12 @@ gst_deinterlace_reset_history (GstDeinterlace * self, gboolean drop_all) memset (self->field_history, 0, GST_DEINTERLACE_MAX_FIELD_HISTORY * sizeof (GstDeinterlaceField)); self->history_count = 0; + memset (self->buf_states, 0, + GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY * + sizeof (GstDeinterlaceBufferState)); + self->state_count = 0; + self->pattern_lock = FALSE; + self->pattern_refresh = TRUE; if (!self->still_frame_mode && self->last_buffer) { gst_buffer_unref (self->last_buffer); @@ -650,6 +809,9 @@ gst_deinterlace_reset (GstDeinterlace * self) gst_deinterlace_reset_history (self, TRUE); gst_deinterlace_reset_qos (self); + + self->need_more = FALSE; + self->have_eos = FALSE; } static void @@ -698,6 +860,15 @@ gst_deinterlace_set_property (GObject * object, guint prop_id, case PROP_FIELD_LAYOUT: self->field_layout = g_value_get_enum (value); break; + case PROP_LOCKING: + self->locking = g_value_get_enum (value); + break; + case PROP_IGNORE_OBSCURE: + self->ignore_obscure = g_value_get_boolean (value); + break; + case PROP_DROP_ORPHANS: + self->drop_orphans = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); } @@ -726,6 +897,15 @@ gst_deinterlace_get_property (GObject * object, guint prop_id, case PROP_FIELD_LAYOUT: g_value_set_enum (value, self->field_layout); break; + case PROP_LOCKING: + g_value_set_enum (value, self->locking); + break; + case PROP_IGNORE_OBSCURE: + g_value_set_boolean (value, self->ignore_obscure); + break; + case PROP_DROP_ORPHANS: + g_value_set_boolean (value, self->drop_orphans); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); } @@ -746,6 +926,32 @@ gst_deinterlace_finalize (GObject * object) G_OBJECT_CLASS (parent_class)->finalize (object); } +static void +gst_deinterlace_update_pattern_timestamps (GstDeinterlace * self) +{ + gint state_idx; + if (self->low_latency) { + /* in low-latency mode the buffer state history contains old buffer + * states as well as the current one and perhaps some future ones. + * the current buffer's state is given by the number of field pairs + * rounded up, minus 1. the below is equivalent */ + state_idx = (self->history_count - 1) >> 1; + } else { + /* in high-latency mode state_count - 1 is the current buffer's state */ + state_idx = self->state_count - 1; + } + + self->pattern_base_ts = self->buf_states[state_idx].timestamp; + self->pattern_buf_dur = + (self->buf_states[state_idx].duration * + telecine_patterns[self->pattern].ratio_d) / + telecine_patterns[self->pattern].ratio_n; + GST_DEBUG_OBJECT (self, + "Starting a new pattern repeat with base ts %" GST_TIME_FORMAT + " and dur %" GST_TIME_FORMAT, GST_TIME_ARGS (self->pattern_base_ts), + GST_TIME_ARGS (self->pattern_buf_dur)); +} + static GstBuffer * gst_deinterlace_pop_history (GstDeinterlace * self) { @@ -759,6 +965,21 @@ gst_deinterlace_pop_history (GstDeinterlace * self) buffer = self->field_history[self->history_count - 1].buf; self->history_count--; + if (self->locking != GST_DEINTERLACE_LOCKING_NONE && (!self->history_count + || GST_BUFFER_DATA (buffer) != + GST_BUFFER_DATA (self->field_history[self->history_count - 1].buf))) { + if (!self->low_latency) + self->state_count--; + if (self->pattern_lock) { + self->pattern_count++; + if (self->pattern != -1 + && self->pattern_count >= telecine_patterns[self->pattern].length) { + self->pattern_count = 0; + self->output_count = 0; + gst_deinterlace_update_pattern_timestamps (self); + } + } + } GST_DEBUG_OBJECT (self, "Returning buffer: %p %" GST_TIME_FORMAT " with duration %" GST_TIME_FORMAT " and size %u", buffer, @@ -768,6 +989,78 @@ gst_deinterlace_pop_history (GstDeinterlace * self) return buffer; } +typedef enum +{ + GST_DEINTERLACE_PROGRESSIVE, + GST_DEINTERLACE_INTERLACED, + GST_DEINTERLACE_TELECINE, +} GstDeinterlaceInterlacingMethod; + +static GstDeinterlaceInterlacingMethod +gst_deinterlace_get_interlacing_method (const GstCaps * caps) +{ + GstDeinterlaceInterlacingMethod method = 0; + gboolean interlaced; + + /* check interlaced cap */ + gst_structure_get_boolean (gst_caps_get_structure (caps, 0), "interlaced", + &interlaced); + + method = + interlaced ? GST_DEINTERLACE_INTERLACED : GST_DEINTERLACE_PROGRESSIVE; + + if (method == GST_DEINTERLACE_INTERLACED) { + const gchar *temp = + gst_structure_get_string (gst_caps_get_structure (caps, 0), + "interlacing-method"); + if (temp && g_str_equal (temp, "telecine")) + method = GST_DEINTERLACE_TELECINE; + } + + return method; +} + +static void +gst_deinterlace_get_buffer_state (GstDeinterlace * self, GstBuffer * buffer, + guint8 * state, GstDeinterlaceInterlacingMethod * i_method) +{ + GstDeinterlaceInterlacingMethod interlacing_method; + + if (!(i_method || state)) + return; + + interlacing_method = + gst_deinterlace_get_interlacing_method (GST_BUFFER_CAPS (buffer)); + + if (state) { + if (interlacing_method == GST_DEINTERLACE_TELECINE) { + if (GST_BUFFER_FLAG_IS_SET (buffer, GST_VIDEO_BUFFER_RFF)) { + *state = GST_DEINTERLACE_BUFFER_STATE_DROP; + } else if (GST_BUFFER_FLAG_IS_SET (buffer, GST_VIDEO_BUFFER_ONEFIELD)) { + /* tc top if tff, tc bottom otherwise */ + if (GST_BUFFER_FLAG_IS_SET (buffer, GST_VIDEO_BUFFER_TFF)) { + *state = GST_DEINTERLACE_BUFFER_STATE_TC_T; + } else { + *state = GST_DEINTERLACE_BUFFER_STATE_TC_B; + } + } else if (GST_BUFFER_FLAG_IS_SET (buffer, GST_VIDEO_BUFFER_PROGRESSIVE)) { + *state = GST_DEINTERLACE_BUFFER_STATE_TC_P; + } else { + *state = GST_DEINTERLACE_BUFFER_STATE_TC_M; + } + } else { + if (interlacing_method == GST_DEINTERLACE_INTERLACED) { + *state = GST_DEINTERLACE_BUFFER_STATE_I; + } else { + *state = GST_DEINTERLACE_BUFFER_STATE_P; + } + } + } + + if (i_method) + *i_method = interlacing_method; +} + static void gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer) { @@ -781,14 +1074,47 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer) GstBuffer *field1, *field2; guint fields_to_push = (onefield) ? 1 : (!repeated) ? 2 : 3; gint field1_flags, field2_flags; + GstDeinterlaceInterlacingMethod interlacing_method; + guint8 buf_state; g_return_if_fail (self->history_count < GST_DEINTERLACE_MAX_FIELD_HISTORY - fields_to_push); - GST_DEBUG_OBJECT (self, "Pushing new buffer to the history: %" GST_TIME_FORMAT - " with duration %" GST_TIME_FORMAT " and size %u", + gst_deinterlace_get_buffer_state (self, buffer, &buf_state, + &interlacing_method); + + GST_DEBUG_OBJECT (self, + "Pushing new buffer to the history: ptr %p at %" GST_TIME_FORMAT + " with duration %" GST_TIME_FORMAT + ", size %u, state %u, interlacing method %d", GST_BUFFER_DATA (buffer), GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer)), - GST_TIME_ARGS (GST_BUFFER_DURATION (buffer)), GST_BUFFER_SIZE (buffer)); + GST_TIME_ARGS (GST_BUFFER_DURATION (buffer)), GST_BUFFER_SIZE (buffer), + buf_state, + interlacing_method == + GST_DEINTERLACE_TELECINE ? "TC" : interlacing_method == + GST_DEINTERLACE_INTERLACED ? "I" : "P"); + + /* move up for new state */ + memmove (&self->buf_states[1], &self->buf_states[0], + (GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY - 1) * + sizeof (GstDeinterlaceBufferState)); + self->buf_states[0].state = buf_state; + self->buf_states[0].timestamp = GST_BUFFER_TIMESTAMP (buffer); + self->buf_states[0].duration = GST_BUFFER_DURATION (buffer); + if (self->state_count < GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY) + self->state_count++; + + if (buf_state == GST_DEINTERLACE_BUFFER_STATE_DROP) { + GST_DEBUG_OBJECT (self, + "Buffer contains only unneeded repeated fields, dropping and not" + "adding to field history"); + gst_buffer_unref (buffer); + return; + } + + /* telecine does not make use of repeated fields */ + if (interlacing_method == GST_DEINTERLACE_TELECINE) + repeated = FALSE; for (i = GST_DEINTERLACE_MAX_FIELD_HISTORY - 1; i >= fields_to_push; i--) { self->field_history[i].buf = self->field_history[i - fields_to_push].buf; @@ -821,12 +1147,16 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer) field2_flags = PICTURE_INTERLACED_TOP; } - /* Timestamps are assigned to the field buffers under the assumption that - the timestamp of the buffer equals the first fields timestamp */ + if (interlacing_method != GST_DEINTERLACE_TELECINE) { + /* Timestamps are assigned to the field buffers under the assumption that + the timestamp of the buffer equals the first fields timestamp */ - timestamp = GST_BUFFER_TIMESTAMP (buffer); - GST_BUFFER_TIMESTAMP (field1) = timestamp; - GST_BUFFER_TIMESTAMP (field2) = timestamp + self->field_duration; + timestamp = GST_BUFFER_TIMESTAMP (buffer); + GST_BUFFER_TIMESTAMP (field1) = timestamp; + GST_BUFFER_TIMESTAMP (field2) = timestamp + self->field_duration; + if (repeated) + GST_BUFFER_TIMESTAMP (field2) += self->field_duration; + } if (repeated) { self->field_history[2].buf = field1; @@ -943,84 +1273,553 @@ gst_deinterlace_do_qos (GstDeinterlace * self, GstClockTime timestamp) return TRUE; } +static gboolean +gst_deinterlace_fix_timestamps (GstDeinterlace * self, + GstDeinterlaceField * field1, GstDeinterlaceField * field2) +{ + GstDeinterlaceField *field3, *field4; + GstDeinterlaceInterlacingMethod interlacing_method; + + if (self->pattern_lock && self->pattern > -1) { + /* accurate pattern-locked timestamp adjustment */ + if (!self->pattern_count) + gst_deinterlace_update_pattern_timestamps (self); + + GST_BUFFER_TIMESTAMP (field1->buf) = + self->pattern_base_ts + self->output_count * self->pattern_buf_dur; + GST_BUFFER_DURATION (field1->buf) = self->pattern_buf_dur; + self->output_count++; + } else { + /* naive (but low-latency) timestamp adjustment based on subsequent + * fields/buffers */ + if (field2 + && GST_BUFFER_DATA (field1->buf) != GST_BUFFER_DATA (field2->buf)) { + if (GST_BUFFER_TIMESTAMP (field1->buf) + + GST_BUFFER_DURATION (field1->buf) == + GST_BUFFER_TIMESTAMP (field2->buf)) { + GST_BUFFER_TIMESTAMP (field1->buf) = + GST_BUFFER_TIMESTAMP (field2->buf) = + (GST_BUFFER_TIMESTAMP (field1->buf) + + GST_BUFFER_TIMESTAMP (field2->buf)) / 2; + } else { + GST_BUFFER_TIMESTAMP (field2->buf) = GST_BUFFER_TIMESTAMP (field1->buf); + } + } + + if (self->history_count < 3) { + GST_DEBUG_OBJECT (self, "Need more fields (have %d, need 3)", + self->history_count); + return FALSE; + } + + field3 = &self->field_history[self->history_count - 3]; + interlacing_method = + gst_deinterlace_get_interlacing_method (GST_BUFFER_CAPS (field3->buf)); + if (interlacing_method == GST_DEINTERLACE_TELECINE) { + if (self->history_count < 4) { + GST_DEBUG_OBJECT (self, "Need more fields (have %d, need 4)", + self->history_count); + return FALSE; + } + + field4 = &self->field_history[self->history_count - 4]; + if (GST_BUFFER_DATA (field3->buf) != GST_BUFFER_DATA (field4->buf)) { + /* telecine fields in separate buffers */ + GST_BUFFER_TIMESTAMP (field3->buf) = + (GST_BUFFER_TIMESTAMP (field3->buf) + + GST_BUFFER_TIMESTAMP (field4->buf)) / 2; + } + } + + GST_BUFFER_DURATION (field1->buf) = + GST_BUFFER_TIMESTAMP (field3->buf) - GST_BUFFER_TIMESTAMP (field1->buf); + } + + GST_DEBUG_OBJECT (self, + "Field 1 adjusted to ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT, + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (field1->buf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (field1->buf))); + return TRUE; +} + +static void +gst_deinterlace_get_pattern_lock (GstDeinterlace * self, gboolean * flush_one) +{ + /* loop over all possible patterns and all possible phases + * giving each a score. the highest score gets the lock */ + /* the score is calculated as the number of matched buffers in the + * sequence starting at the phase offset with those from the history + * then the longest duration pattern match is taken. if there is more than + * one pattern matching all buffers, we take the longest pattern of those. + * matches to complete patterns are preferred. if no non-trivial pattern is + * matched, trivial patterns are tested. */ + gint i, j, k, score, pattern, phase; + const gint state_count = self->state_count; + const gint n_required = self->ignore_obscure ? + GST_DEINTERLACE_OBSCURE_THRESHOLD : + GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY; + + /* set unknown pattern as this is used in logic outside this function */ + self->pattern = -1; + + /* wait for more buffers */ + if (!self->have_eos && state_count < n_required) { + GST_DEBUG_OBJECT (self, "Need more buffers in state history - %d/%d", + state_count, n_required); + return; + } + + score = pattern = phase = -1; + + /* loop over all patterns */ + for (i = 0; i < G_N_ELEMENTS (telecine_patterns); i++) { + const guint8 length = telecine_patterns[i].length; + + if (self->ignore_obscure && i >= GST_DEINTERLACE_OBSCURE_THRESHOLD) + break; + + if (state_count < length) + continue; + + /* loop over all phases */ + for (j = 0; j < length; j++) { + /* low-latency mode looks at past buffers, high latency at future buffers */ + const gint state_idx = (self->low_latency ? length : state_count) - 1; + /* loop over history, breaking on differing buffer states */ + for (k = 0; k < length && k < state_count; k++) { + const guint8 hist = self->buf_states[state_idx - k].state; + const guint8 patt = telecine_patterns[i].states[(j + k) % length]; + if (!(hist & patt)) + break; + } + + /* make complete matches more signficant */ + if (k == length) + k += GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY; + + /* take as new best pattern if the number of matched buffers is more than + * for other patterns */ + if (k > score) { + score = k; + pattern = i; + phase = j; + if (self->low_latency) { + /* state_idx + 1 is the number of buffers yet to be pushed out + * so length - state_idx - 1 is the number of old buffers in the + * pattern */ + phase = (phase + length - state_idx - 1) % length; + } + } + } + } + + GST_DEBUG_OBJECT (self, + "Final pattern match result: pa %d, ph %d, l %d, s %d", pattern, phase, + telecine_patterns[pattern].length, score); + self->pattern = pattern; + self->pattern_phase = phase; + self->pattern_count = 0; + self->output_count = 0; + self->pattern_lock = TRUE; + + /* check for the case that the first field of the pattern is an orphan */ + if (pattern > 1 + && telecine_patterns[pattern].states[phase] & (GST_ONE | GST_INT)) { + gint i = phase, field_count = 0; + guint8 state = telecine_patterns[pattern].states[i]; + + do { + if (state & GST_ONE) { + field_count++; + } else if (!(state & GST_DRP)) { + field_count += 2; + } + i++; + i %= telecine_patterns[pattern].length; + state = telecine_patterns[pattern].states[i]; + } while (!(state & GST_PRG)); + + /* if field_count is odd, we have an orphan field at the beginning of the + * sequence + * note - don't do this in low-latency mode as we are somewhere within the + * pattern already */ + if (!self->low_latency && (*flush_one = field_count & 1)) { + GST_DEBUG_OBJECT (self, "Orphan field detected at the beginning of the " + "pattern - it will be deinterlaced."); + } + } +} + static GstFlowReturn gst_deinterlace_output_frame (GstDeinterlace * self, gboolean flushing) { GstClockTime timestamp; - GstFlowReturn ret = GST_FLOW_OK; - gint fields_required = 0; - gint cur_field_idx = 0; + GstFlowReturn ret; + gint fields_required; + gint cur_field_idx; GstBuffer *buf, *outbuf; + GstDeinterlaceField *field1, *field2; + GstDeinterlaceInterlacingMethod interlacing_method; + guint8 buf_state; + gboolean hl_no_lock; /* indicates high latency timestamp adjustment but no pattern lock (could be ONEF or I) */ + gboolean same_buffer; /* are field1 and field2 in the same buffer? */ + gboolean flush_one; /* used for flushing one field when in high latency mode and not locked */ + TelecinePattern pattern; + guint8 phase, count; + const GstDeinterlaceLocking locking = self->locking; - gst_deinterlace_set_method (self, self->user_set_method_id); - fields_required = gst_deinterlace_method_get_fields_required (self->method); +restart: + ret = GST_FLOW_OK; + fields_required = 0; + cur_field_idx = 0; + hl_no_lock = FALSE; + same_buffer = FALSE; + flush_one = FALSE; + self->need_more = FALSE; + phase = self->pattern_phase; + count = self->pattern_count; - if (self->history_count < fields_required) { - if (flushing) { - /* FIXME: if there are any methods implemented that output different - * dimensions (e.g. half height) that require more than one field of - * history, it is desirable to degrade to something that outputs - * half-height also */ - gst_deinterlace_set_method (self, - self->history_count >= 2 ? - GST_DEINTERLACE_VFIR : GST_DEINTERLACE_LINEAR); + if (!self->history_count) { + GST_DEBUG_OBJECT (self, "History is empty, waiting for more buffers!"); + goto need_more; + } + + field1 = &self->field_history[self->history_count - 1]; + + if (locking != GST_DEINTERLACE_LOCKING_NONE) { + if (!self->state_count) { + GST_ERROR_OBJECT (self, + "BROKEN! Fields in history + no states should not happen!"); + return GST_FLOW_ERROR; + } + + gst_deinterlace_get_buffer_state (self, field1->buf, &buf_state, + &interlacing_method); + + if (self->pattern != -1) + pattern = telecine_patterns[self->pattern]; + + /* patterns 0 and 1 are interlaced, the rest are telecine */ + if (self->pattern > 1) + interlacing_method = GST_DEINTERLACE_TELECINE; + + if (self->pattern == -1 || self->pattern_refresh + || !(buf_state & pattern.states[(phase + count) % pattern.length])) { + /* no pattern, pattern refresh set or unexpected buffer state */ + self->pattern_lock = FALSE; + self->pattern_refresh = TRUE; + + /* refresh pattern lock */ + gst_deinterlace_get_pattern_lock (self, &flush_one); + + if (self->pattern != -1) { + /* locked onto a valid pattern so refresh complete */ + GST_DEBUG_OBJECT (self, "Pattern locked! %s starting at %d", + telecine_patterns[self->pattern].nick, self->pattern_phase); + self->pattern_refresh = FALSE; + } else if (!self->low_latency) { + if (!self->pattern_lock) { + goto need_more; + } else { + hl_no_lock = TRUE; + } + } + + /* setcaps on sink and src pads */ + gst_deinterlace_setcaps (self->sinkpad, GST_PAD_CAPS (self->sinkpad)); + + if (flush_one && self->drop_orphans) { + GST_DEBUG_OBJECT (self, "Dropping orphan first field"); + gst_buffer_unref (gst_deinterlace_pop_history (self)); + goto restart; + } + } + } else { + gst_deinterlace_get_buffer_state (self, field1->buf, NULL, + &interlacing_method); + } + + same_buffer = self->history_count >= 2 + && (GST_BUFFER_DATA (field1->buf) == + GST_BUFFER_DATA (self->field_history[self->history_count - 2].buf)); + + if ((flushing && self->history_count == 1) || (flush_one + && !self->drop_orphans) || (hl_no_lock && (self->history_count == 1 + || !same_buffer))) { + GST_DEBUG_OBJECT (self, "Flushing one field using linear method"); + gst_deinterlace_set_method (self, GST_DEINTERLACE_LINEAR); + fields_required = gst_deinterlace_method_get_fields_required (self->method); + } else if (interlacing_method == GST_DEINTERLACE_TELECINE + && (self->low_latency > 0 || self->pattern != -1 || (hl_no_lock + && same_buffer + && GST_BUFFER_FLAG_IS_SET (field1->buf, + GST_VIDEO_BUFFER_PROGRESSIVE)))) { + /* telecined - we reconstruct frames by weaving pairs of fields */ + fields_required = 2; + if (!flushing && self->history_count < fields_required) { + GST_DEBUG_OBJECT (self, "Need more fields (have %d, need %d)", + self->history_count, fields_required); + goto need_more; + } + + field2 = &self->field_history[self->history_count - 2]; + if (!gst_deinterlace_fix_timestamps (self, field1, field2) && !flushing) + goto need_more; + + if (same_buffer) { + /* telecine progressive */ + GstBuffer *field1_buf; + + GST_DEBUG_OBJECT (self, + "Frame type: Telecine Progressive; pushing buffer as a frame"); + /* pop and push */ + field1_buf = gst_deinterlace_pop_history (self); + /* field2 is the same buffer as field1, but we need to remove it from + * the history anyway */ + gst_buffer_unref (gst_deinterlace_pop_history (self)); + /* set the caps from the src pad on the buffer as they should be correct */ + gst_buffer_set_caps (field1_buf, GST_PAD_CAPS (self->srcpad)); + GST_DEBUG_OBJECT (self, + "[OUT] ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", end %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (field1_buf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (field1_buf)), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (field1_buf) + + GST_BUFFER_DURATION (field1_buf))); + return gst_pad_push (self->srcpad, field1_buf); + } else { + /* telecine fields in separate buffers */ + + /* check field1 and field2 buffer caps and flags are corresponding */ + if (field1->flags == field2->flags) { + /* ERROR - fields are of same parity - what should be done here? + * perhaps deinterlace the tip field and start again? */ + GST_ERROR_OBJECT (self, "Telecine mixed with fields of same parity!"); + } + GST_DEBUG_OBJECT (self, + "Frame type: Telecine Mixed; weaving tip two fields into a frame"); + /* set method to WEAVE */ + gst_deinterlace_set_method (self, GST_DEINTERLACE_WEAVE); + } + } else if (interlacing_method == GST_DEINTERLACE_INTERLACED || (hl_no_lock + && interlacing_method == GST_DEINTERLACE_TELECINE && same_buffer + && !GST_BUFFER_FLAG_IS_SET (field1->buf, + GST_VIDEO_BUFFER_PROGRESSIVE))) { + gst_deinterlace_set_method (self, self->user_set_method_id); + fields_required = gst_deinterlace_method_get_fields_required (self->method); + if (flushing && self->history_count < fields_required) { + /* note: we already checked for flushing with history count == 1 above + * so we must have 2 or more fields in here */ + gst_deinterlace_set_method (self, GST_DEINTERLACE_VFIR); fields_required = gst_deinterlace_method_get_fields_required (self->method); GST_DEBUG_OBJECT (self, "Flushing field(s) using %s method", methods_types[self->method_id].value_nick); - } else { - /* Not enough fields in the history */ + } + + /* Not enough fields in the history */ + if (self->history_count < fields_required) { GST_DEBUG_OBJECT (self, "Need more fields (have %d, need %d)", self->history_count, fields_required); - return GST_FLOW_OK; + goto need_more; + } + + GST_DEBUG_OBJECT (self, + "Frame type: Interlaced; deinterlacing using %s method", + methods_types[self->method_id].value_nick); + } else { + GstBuffer *field1_buf; + + /* progressive */ + fields_required = 2; + + /* Not enough fields in the history */ + if (self->history_count < fields_required) { + GST_DEBUG_OBJECT (self, "Need more fields (have %d, need %d)", + self->history_count, fields_required); + goto need_more; + } + + field2 = &self->field_history[self->history_count - 2]; + if (GST_BUFFER_DATA (field1->buf) != GST_BUFFER_DATA (field2->buf)) { + /* ERROR - next two fields in field history are not one progressive buffer - weave? */ + GST_ERROR_OBJECT (self, + "Progressive buffer but two fields at tip aren't in the same buffer!"); + } + + GST_DEBUG_OBJECT (self, + "Frame type: Progressive; pushing buffer as a frame"); + /* pop and push */ + field1_buf = gst_deinterlace_pop_history (self); + /* field2 is the same buffer as field1, but we need to remove it from the + * history anyway */ + gst_buffer_unref (gst_deinterlace_pop_history (self)); + GST_DEBUG_OBJECT (self, + "[OUT] ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", end %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (field1_buf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (field1_buf)), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (field1_buf) + + GST_BUFFER_DURATION (field1_buf))); + return gst_pad_push (self->srcpad, field1_buf); + } + + if (self->fields == GST_DEINTERLACE_ALL + || interlacing_method == GST_DEINTERLACE_TELECINE) + GST_DEBUG_OBJECT (self, "All fields"); + else if (self->fields == GST_DEINTERLACE_TF) + GST_DEBUG_OBJECT (self, "Top fields"); + else if (self->fields == GST_DEINTERLACE_BF) + GST_DEBUG_OBJECT (self, "Bottom fields"); + + cur_field_idx = self->history_count - fields_required; + + if ((self->field_history[cur_field_idx].flags == PICTURE_INTERLACED_TOP + && (self->fields == GST_DEINTERLACE_TF + || interlacing_method == GST_DEINTERLACE_TELECINE)) + || self->fields == GST_DEINTERLACE_ALL) { + GST_DEBUG_OBJECT (self, "deinterlacing top field"); + + /* create new buffer */ + ret = + gst_pad_alloc_buffer (self->srcpad, GST_BUFFER_OFFSET_NONE, + self->frame_size, GST_PAD_CAPS (self->srcpad), &outbuf); + if (ret != GST_FLOW_OK) + return ret; + + if (GST_PAD_CAPS (self->srcpad) != GST_BUFFER_CAPS (outbuf) && + !gst_caps_is_equal (GST_PAD_CAPS (self->srcpad), + GST_BUFFER_CAPS (outbuf))) { + gst_caps_replace (&self->request_caps, GST_BUFFER_CAPS (outbuf)); + GST_DEBUG_OBJECT (self, "Upstream wants new caps %" GST_PTR_FORMAT, + self->request_caps); + + gst_buffer_unref (outbuf); + outbuf = gst_buffer_try_new_and_alloc (self->frame_size); + + if (!outbuf) + return GST_FLOW_ERROR; + + gst_buffer_set_caps (outbuf, GST_PAD_CAPS (self->srcpad)); + } + + g_return_val_if_fail (self->history_count - 1 - + gst_deinterlace_method_get_latency (self->method) >= 0, GST_FLOW_ERROR); + + buf = + self->field_history[self->history_count - 1 - + gst_deinterlace_method_get_latency (self->method)].buf; + + if (interlacing_method != GST_DEINTERLACE_TELECINE) { + timestamp = GST_BUFFER_TIMESTAMP (buf); + + GST_BUFFER_TIMESTAMP (outbuf) = timestamp; + if (self->fields == GST_DEINTERLACE_ALL) + GST_BUFFER_DURATION (outbuf) = self->field_duration; + else + GST_BUFFER_DURATION (outbuf) = 2 * self->field_duration; + } else { + GST_BUFFER_TIMESTAMP (outbuf) = GST_BUFFER_TIMESTAMP (buf); + GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (buf); + } + + /* Check if we need to drop the frame because of QoS */ + if (!gst_deinterlace_do_qos (self, GST_BUFFER_TIMESTAMP (buf))) { + gst_buffer_unref (gst_deinterlace_pop_history (self)); + gst_buffer_unref (outbuf); + outbuf = NULL; + ret = GST_FLOW_OK; + } else { + /* do magic calculus */ + gst_deinterlace_method_deinterlace_frame (self->method, + self->field_history, self->history_count, outbuf); + + gst_buffer_unref (gst_deinterlace_pop_history (self)); + + if (gst_deinterlace_clip_buffer (self, outbuf)) { + GST_DEBUG_OBJECT (self, + "[OUT] ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", end %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (outbuf)), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf) + + GST_BUFFER_DURATION (outbuf))); + ret = gst_pad_push (self->srcpad, outbuf); + } else { + ret = GST_FLOW_OK; + gst_buffer_unref (outbuf); + } + + outbuf = NULL; + if (ret != GST_FLOW_OK) + return ret; + if (interlacing_method == GST_DEINTERLACE_TELECINE + && self->method_id == GST_DEINTERLACE_WEAVE) { + /* pop off the second field */ + GST_DEBUG_OBJECT (self, "Removing unused field (count: %d)", + self->history_count); + gst_buffer_unref (gst_deinterlace_pop_history (self)); + interlacing_method = GST_DEINTERLACE_INTERLACED; + return ret; + } + } + + if (flush_one && !self->drop_orphans) { + GST_DEBUG_OBJECT (self, "Orphan field deinterlaced - reconfiguring"); + goto restart; + } + } + /* no calculation done: remove excess field */ + else if (self->field_history[cur_field_idx].flags == + PICTURE_INTERLACED_TOP && (self->fields == GST_DEINTERLACE_BF + && interlacing_method != GST_DEINTERLACE_TELECINE)) { + GST_DEBUG_OBJECT (self, "Removing unused top field"); + gst_buffer_unref (gst_deinterlace_pop_history (self)); + + if (flush_one && !self->drop_orphans) { + GST_DEBUG_OBJECT (self, "Orphan field deinterlaced - reconfiguring"); + goto restart; } } - while (self->history_count >= fields_required) { - if (self->fields == GST_DEINTERLACE_ALL) - GST_DEBUG_OBJECT (self, "All fields"); - else if (self->fields == GST_DEINTERLACE_TF) - GST_DEBUG_OBJECT (self, "Top fields"); - else if (self->fields == GST_DEINTERLACE_BF) - GST_DEBUG_OBJECT (self, "Bottom fields"); + cur_field_idx = self->history_count - fields_required; + if (self->history_count < fields_required) + return ret; - cur_field_idx = self->history_count - fields_required; + /* deinterlace bottom_field */ + if ((self->field_history[cur_field_idx].flags == PICTURE_INTERLACED_BOTTOM + && (self->fields == GST_DEINTERLACE_BF + || interlacing_method == GST_DEINTERLACE_TELECINE)) + || self->fields == GST_DEINTERLACE_ALL) { + GST_DEBUG_OBJECT (self, "deinterlacing bottom field"); - if ((self->field_history[cur_field_idx].flags == PICTURE_INTERLACED_TOP - && self->fields == GST_DEINTERLACE_TF) || - self->fields == GST_DEINTERLACE_ALL) { - GST_DEBUG_OBJECT (self, "deinterlacing top field"); + /* create new buffer */ + ret = + gst_pad_alloc_buffer (self->srcpad, GST_BUFFER_OFFSET_NONE, + self->frame_size, GST_PAD_CAPS (self->srcpad), &outbuf); + if (ret != GST_FLOW_OK) + return ret; - /* create new buffer */ - ret = - gst_pad_alloc_buffer (self->srcpad, GST_BUFFER_OFFSET_NONE, - self->frame_size, GST_PAD_CAPS (self->srcpad), &outbuf); - if (ret != GST_FLOW_OK) - return ret; + if (GST_PAD_CAPS (self->srcpad) != GST_BUFFER_CAPS (outbuf) && + !gst_caps_is_equal (GST_PAD_CAPS (self->srcpad), + GST_BUFFER_CAPS (outbuf))) { + gst_caps_replace (&self->request_caps, GST_BUFFER_CAPS (outbuf)); + GST_DEBUG_OBJECT (self, "Upstream wants new caps %" GST_PTR_FORMAT, + self->request_caps); - if (GST_PAD_CAPS (self->srcpad) != GST_BUFFER_CAPS (outbuf) - && !gst_caps_is_equal (GST_PAD_CAPS (self->srcpad), - GST_BUFFER_CAPS (outbuf))) { - gst_caps_replace (&self->request_caps, GST_BUFFER_CAPS (outbuf)); - GST_DEBUG_OBJECT (self, "Upstream wants new caps %" GST_PTR_FORMAT, - self->request_caps); + gst_buffer_unref (outbuf); + outbuf = gst_buffer_try_new_and_alloc (self->frame_size); - gst_buffer_unref (outbuf); - outbuf = gst_buffer_try_new_and_alloc (self->frame_size); + if (!outbuf) + return GST_FLOW_ERROR; - if (!outbuf) - return GST_FLOW_ERROR; + gst_buffer_set_caps (outbuf, GST_PAD_CAPS (self->srcpad)); + } - gst_buffer_set_caps (outbuf, GST_PAD_CAPS (self->srcpad)); - } + g_return_val_if_fail (self->history_count - 1 - + gst_deinterlace_method_get_latency (self->method) >= 0, GST_FLOW_ERROR); - g_return_val_if_fail (self->history_count - 1 - - gst_deinterlace_method_get_latency (self->method) >= 0, - GST_FLOW_ERROR); - - buf = - self->field_history[self->history_count - 1 - - gst_deinterlace_method_get_latency (self->method)].buf; + buf = + self->field_history[self->history_count - 1 - + gst_deinterlace_method_get_latency (self->method)].buf; + if (interlacing_method != GST_DEINTERLACE_TELECINE) { timestamp = GST_BUFFER_TIMESTAMP (buf); GST_BUFFER_TIMESTAMP (outbuf) = timestamp; @@ -1028,127 +1827,111 @@ gst_deinterlace_output_frame (GstDeinterlace * self, gboolean flushing) GST_BUFFER_DURATION (outbuf) = self->field_duration; else GST_BUFFER_DURATION (outbuf) = 2 * self->field_duration; - - /* Check if we need to drop the frame because of QoS */ - if (!gst_deinterlace_do_qos (self, GST_BUFFER_TIMESTAMP (buf))) { - gst_buffer_unref (gst_deinterlace_pop_history (self)); - gst_buffer_unref (outbuf); - outbuf = NULL; - ret = GST_FLOW_OK; - } else { - /* do magic calculus */ - gst_deinterlace_method_deinterlace_frame (self->method, - self->field_history, self->history_count, outbuf); - - gst_buffer_unref (gst_deinterlace_pop_history (self)); - - if (gst_deinterlace_clip_buffer (self, outbuf)) { - ret = gst_pad_push (self->srcpad, outbuf); - } else { - ret = GST_FLOW_OK; - gst_buffer_unref (outbuf); - } - - outbuf = NULL; - if (ret != GST_FLOW_OK) - return ret; - } + } else { + GST_BUFFER_TIMESTAMP (outbuf) = GST_BUFFER_TIMESTAMP (buf); + GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (buf); } - /* no calculation done: remove excess field */ - else if (self->field_history[cur_field_idx].flags == - PICTURE_INTERLACED_TOP && self->fields == GST_DEINTERLACE_BF) { - GST_DEBUG_OBJECT (self, "Removing unused top field"); + + /* Check if we need to drop the frame because of QoS */ + if (!gst_deinterlace_do_qos (self, GST_BUFFER_TIMESTAMP (buf))) { gst_buffer_unref (gst_deinterlace_pop_history (self)); - } + gst_buffer_unref (outbuf); + outbuf = NULL; + ret = GST_FLOW_OK; + } else { + /* do magic calculus */ + gst_deinterlace_method_deinterlace_frame (self->method, + self->field_history, self->history_count, outbuf); - cur_field_idx = self->history_count - fields_required; - if (self->history_count < fields_required) - break; + gst_buffer_unref (gst_deinterlace_pop_history (self)); - /* deinterlace bottom_field */ - if ((self->field_history[cur_field_idx].flags == PICTURE_INTERLACED_BOTTOM - && self->fields == GST_DEINTERLACE_BF) || - self->fields == GST_DEINTERLACE_ALL) { - GST_DEBUG_OBJECT (self, "deinterlacing bottom field"); + if (gst_deinterlace_clip_buffer (self, outbuf)) { + GST_DEBUG_OBJECT (self, + "[OUT] ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", end %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (outbuf)), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf) + + GST_BUFFER_DURATION (outbuf))); + ret = gst_pad_push (self->srcpad, outbuf); + } else { + ret = GST_FLOW_OK; + gst_buffer_unref (outbuf); + } - /* create new buffer */ - ret = - gst_pad_alloc_buffer (self->srcpad, GST_BUFFER_OFFSET_NONE, - self->frame_size, GST_PAD_CAPS (self->srcpad), &outbuf); + outbuf = NULL; if (ret != GST_FLOW_OK) return ret; - - if (GST_PAD_CAPS (self->srcpad) != GST_BUFFER_CAPS (outbuf) - && !gst_caps_is_equal (GST_PAD_CAPS (self->srcpad), - GST_BUFFER_CAPS (outbuf))) { - gst_caps_replace (&self->request_caps, GST_BUFFER_CAPS (outbuf)); - GST_DEBUG_OBJECT (self, "Upstream wants new caps %" GST_PTR_FORMAT, - self->request_caps); - - gst_buffer_unref (outbuf); - outbuf = gst_buffer_try_new_and_alloc (self->frame_size); - - if (!outbuf) - return GST_FLOW_ERROR; - - gst_buffer_set_caps (outbuf, GST_PAD_CAPS (self->srcpad)); - } - - g_return_val_if_fail (self->history_count - 1 - - gst_deinterlace_method_get_latency (self->method) >= 0, - GST_FLOW_ERROR); - - buf = - self->field_history[self->history_count - 1 - - gst_deinterlace_method_get_latency (self->method)].buf; - timestamp = GST_BUFFER_TIMESTAMP (buf); - - GST_BUFFER_TIMESTAMP (outbuf) = timestamp; - if (self->fields == GST_DEINTERLACE_ALL) - GST_BUFFER_DURATION (outbuf) = self->field_duration; - else - GST_BUFFER_DURATION (outbuf) = 2 * self->field_duration; - - /* Check if we need to drop the frame because of QoS */ - if (!gst_deinterlace_do_qos (self, GST_BUFFER_TIMESTAMP (buf))) { + if (interlacing_method == GST_DEINTERLACE_TELECINE + && self->method_id == GST_DEINTERLACE_WEAVE) { + /* pop off the second field */ + GST_DEBUG_OBJECT (self, "Removing unused field (count: %d)", + self->history_count); gst_buffer_unref (gst_deinterlace_pop_history (self)); - gst_buffer_unref (outbuf); - outbuf = NULL; - ret = GST_FLOW_OK; - } else { - /* do magic calculus */ - gst_deinterlace_method_deinterlace_frame (self->method, - self->field_history, self->history_count, outbuf); - - gst_buffer_unref (gst_deinterlace_pop_history (self)); - - if (gst_deinterlace_clip_buffer (self, outbuf)) { - ret = gst_pad_push (self->srcpad, outbuf); - } else { - ret = GST_FLOW_OK; - gst_buffer_unref (outbuf); - } - - outbuf = NULL; - if (ret != GST_FLOW_OK) - return ret; + interlacing_method = GST_DEINTERLACE_INTERLACED; + return ret; } } - /* no calculation done: remove excess field */ - else if (self->field_history[cur_field_idx].flags == - PICTURE_INTERLACED_BOTTOM && self->fields == GST_DEINTERLACE_TF) { - GST_DEBUG_OBJECT (self, "Removing unused bottom field"); - gst_buffer_unref (gst_deinterlace_pop_history (self)); + + if (flush_one && !self->drop_orphans) { + GST_DEBUG_OBJECT (self, "Orphan field deinterlaced - reconfiguring"); + goto restart; + } + } + /* no calculation done: remove excess field */ + else if (self->field_history[cur_field_idx].flags == + PICTURE_INTERLACED_BOTTOM && (self->fields == GST_DEINTERLACE_TF + && interlacing_method != GST_DEINTERLACE_TELECINE)) { + GST_DEBUG_OBJECT (self, "Removing unused bottom field"); + gst_buffer_unref (gst_deinterlace_pop_history (self)); + + if (flush_one && !self->drop_orphans) { + GST_DEBUG_OBJECT (self, "Orphan field deinterlaced - reconfiguring"); + goto restart; } } return ret; + +need_more: + self->need_more = TRUE; + return ret; +} + +static gboolean +gst_deinterlace_get_latency (GstDeinterlace * self) +{ + if (self->locking == GST_DEINTERLACE_LOCKING_AUTO) { + gboolean res; + GstQuery *query; + + query = gst_query_new_latency (); + if ((res = gst_pad_peer_query (self->sinkpad, query))) { + gboolean is_live; + /* if upstream is live, we use low-latency passive locking mode + * else high-latency active locking mode */ + gst_query_parse_latency (query, &is_live, NULL, NULL); + GST_DEBUG_OBJECT (self, "Latency query indicates stream is %s", + is_live ? "live - using passive locking" : + "not live - using active locking"); + gst_query_unref (query); + return is_live; + } else { + /* conservatively use passive locking if the query fails */ + GST_WARNING_OBJECT (self, + "Latency query failed - fall back to using passive locking"); + gst_query_unref (query); + return TRUE; + } + } else { + return self->locking - 2; + } } static GstFlowReturn gst_deinterlace_chain (GstPad * pad, GstBuffer * buf) { GstDeinterlace *self = GST_DEINTERLACE (GST_PAD_PARENT (pad)); + GstFlowReturn ret = GST_FLOW_OK; GST_OBJECT_LOCK (self); if (self->reconfigure) { @@ -1166,8 +1949,23 @@ gst_deinterlace_chain (GstPad * pad, GstBuffer * buf) GST_OBJECT_UNLOCK (self); } - if (self->still_frame_mode || self->passthrough) + GST_DEBUG_OBJECT (self, + "[IN] ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", end %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (buf)), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf) + GST_BUFFER_DURATION (buf))); + + if (self->still_frame_mode || self->passthrough) { + GST_DEBUG_OBJECT (self, + "Frame type: Progressive?; pushing buffer using pass-through"); + GST_DEBUG_OBJECT (self, + "[OUT] ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", end %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (buf)), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf) + GST_BUFFER_DURATION (buf))); + return gst_pad_push (self->srcpad, buf); + } if (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)) { GST_DEBUG_OBJECT (self, "DISCONT buffer, resetting history"); @@ -1177,7 +1975,11 @@ gst_deinterlace_chain (GstPad * pad, GstBuffer * buf) gst_deinterlace_push_history (self, buf); buf = NULL; - return gst_deinterlace_output_frame (self, FALSE); + do { + ret = gst_deinterlace_output_frame (self, FALSE); + } while (!self->need_more && self->history_count > 0 && ret == GST_FLOW_OK); + + return ret; } static gint @@ -1386,6 +2188,23 @@ gst_deinterlace_setcaps (GstPad * pad, GstCaps * caps) gboolean res = TRUE; GstDeinterlace *self = GST_DEINTERLACE (gst_pad_get_parent (pad)); GstCaps *srccaps; + GstDeinterlaceInterlacingMethod interlacing_method; + + if (self->locking != GST_DEINTERLACE_LOCKING_NONE) { + if (self->low_latency == -1) + self->low_latency = gst_deinterlace_get_latency (self); + + if (self->pattern_lock) { + /* refresh has been successful - we have a lock now */ + self->pattern_refresh = FALSE; + } else { + /* if we were not refreshing (!pattern_refresh) the caps have changed + * so we need to refresh and we don't have a lock anymore + * otherwise we have pattern_fresh and !pattern_lock anyway */ + self->pattern_refresh = TRUE; + self->pattern_lock = FALSE; + } + } res = gst_video_format_parse_caps (caps, &self->format, &self->width, @@ -1398,27 +2217,51 @@ gst_deinterlace_setcaps (GstPad * pad, GstCaps * caps) gst_deinterlace_update_passthrough (self); - if (!self->passthrough && self->fields == GST_DEINTERLACE_ALL) { - gint fps_n = self->fps_n, fps_d = self->fps_d; - - if (!gst_fraction_double (&fps_n, &fps_d, FALSE)) - goto invalid_caps; + interlacing_method = gst_deinterlace_get_interlacing_method (caps); + if (self->pattern_lock) { srccaps = gst_caps_copy (caps); + if (self->pattern != -1 + && G_UNLIKELY (!gst_util_fraction_multiply (self->fps_n, self->fps_d, + telecine_patterns[self->pattern].ratio_n, + telecine_patterns[self->pattern].ratio_d, &self->fps_n, + &self->fps_d))) + GST_ERROR_OBJECT (self, + "Multiplying the framerate by the telecine pattern ratio overflowed!"); + gst_caps_set_simple (srccaps, "framerate", GST_TYPE_FRACTION, self->fps_n, + self->fps_d, NULL); + } else if (self->low_latency > 0) { + if (interlacing_method == GST_DEINTERLACE_TELECINE) { + /* for initial buffers of a telecine pattern, until there is a lock we + * we output naïvely adjusted timestamps */ + srccaps = gst_caps_copy (caps); + gst_caps_set_simple (srccaps, "framerate", GST_TYPE_FRACTION, 0, 1, NULL); + } else if (!self->passthrough && self->fields == GST_DEINTERLACE_ALL) { + gint fps_n = self->fps_n, fps_d = self->fps_d; - gst_caps_set_simple (srccaps, "framerate", GST_TYPE_FRACTION, fps_n, - fps_d, NULL); + if (!gst_fraction_double (&fps_n, &fps_d, FALSE)) + goto invalid_caps; + + srccaps = gst_caps_copy (caps); + + gst_caps_set_simple (srccaps, "framerate", GST_TYPE_FRACTION, fps_n, + fps_d, NULL); + } else { + srccaps = gst_caps_ref (caps); + } } else { + /* in high latency pattern locking mode if we don't have a pattern lock, + * the sink pad caps are the best we know */ srccaps = gst_caps_ref (caps); } if (self->mode != GST_DEINTERLACE_MODE_DISABLED) { srccaps = gst_caps_make_writable (srccaps); + gst_structure_remove_field (gst_caps_get_structure (srccaps, 0), + "interlacing-method"); gst_caps_set_simple (srccaps, "interlaced", G_TYPE_BOOLEAN, FALSE, NULL); } - gst_deinterlace_reset_history (self, FALSE); - if (!gst_pad_set_caps (self->srcpad, srccaps)) goto caps_not_accepted; @@ -1475,8 +2318,8 @@ gst_deinterlace_sink_event (GstPad * pad, GstEvent * event) gint64 start, end, base; gdouble rate, applied_rate; - gst_event_parse_new_segment_full (event, &is_update, &rate, &applied_rate, - &fmt, &start, &end, &base); + gst_event_parse_new_segment_full (event, &is_update, &rate, + &applied_rate, &fmt, &start, &end, &base); gst_deinterlace_reset_qos (self); gst_deinterlace_reset_history (self, FALSE); @@ -1524,6 +2367,7 @@ gst_deinterlace_sink_event (GstPad * pad, GstEvent * event) } /* fall through */ case GST_EVENT_EOS: + self->have_eos = TRUE; gst_deinterlace_reset_history (self, FALSE); /* fall through */ diff --git a/gst/deinterlace/gstdeinterlace.h b/gst/deinterlace/gstdeinterlace.h index df996f8425..ac97a555ec 100644 --- a/gst/deinterlace/gstdeinterlace.h +++ b/gst/deinterlace/gstdeinterlace.h @@ -43,8 +43,6 @@ G_BEGIN_DECLS typedef struct _GstDeinterlace GstDeinterlace; typedef struct _GstDeinterlaceClass GstDeinterlaceClass; -#define GST_DEINTERLACE_MAX_FIELD_HISTORY 10 - typedef enum { GST_DEINTERLACE_TOMSMOCOMP, @@ -79,6 +77,39 @@ typedef enum { GST_DEINTERLACE_MODE_DISABLED } GstDeinterlaceMode; +typedef enum +{ + GST_DEINTERLACE_LOCKING_NONE, + GST_DEINTERLACE_LOCKING_AUTO, + GST_DEINTERLACE_LOCKING_ACTIVE, + GST_DEINTERLACE_LOCKING_PASSIVE, +} GstDeinterlaceLocking; + +#define GST_DEINTERLACE_MAX_FIELD_HISTORY 10 +#define GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY 50 +/* check max field history is large enough */ +#if GST_DEINTERLACE_MAX_FIELD_HISTORY < GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY * 3 +#undef GST_DEINTERLACE_MAX_FIELD_HISTORY +#define GST_DEINTERLACE_MAX_FIELD_HISTORY (GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY * 3) +#endif + +typedef struct _TelecinePattern TelecinePattern; +struct _TelecinePattern +{ + const gchar *nick; + guint8 length; + guint8 ratio_n, ratio_d; + guint8 states[GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY]; +}; + +typedef struct _GstDeinterlaceBufferState GstDeinterlaceBufferState; +struct _GstDeinterlaceBufferState +{ + GstClockTime timestamp; + GstClockTime duration; + guint8 state; +}; + struct _GstDeinterlace { GstElement parent; @@ -92,8 +123,10 @@ struct _GstDeinterlace GstDeinterlaceFields fields; - GstDeinterlaceMethods method_id; /* current state (differs when flushing) */ - GstDeinterlaceMethods user_set_method_id; /* property value */ + /* current state (differs when flushing/inverse telecine using weave) */ + GstDeinterlaceMethods method_id; + /* property value */ + GstDeinterlaceMethods user_set_method_id; GstDeinterlaceMethod *method; GstVideoFormat format; @@ -134,6 +167,24 @@ struct _GstDeinterlace gboolean reconfigure; GstDeinterlaceMode new_mode; GstDeinterlaceFields new_fields; + + GstDeinterlaceLocking locking; + gint low_latency; + gboolean drop_orphans; + gboolean ignore_obscure; + gboolean pattern_lock; + gboolean pattern_refresh; + GstDeinterlaceBufferState buf_states[GST_DEINTERLACE_MAX_BUFFER_STATE_HISTORY]; + gint state_count; + gint pattern; + guint8 pattern_phase; + guint8 pattern_count; + guint8 output_count; + GstClockTime pattern_base_ts; + GstClockTime pattern_buf_dur; + + gboolean need_more; + gboolean have_eos; }; struct _GstDeinterlaceClass