From f2c2560db27b97ae9fa00e3a06a485b8e1cd79aa Mon Sep 17 00:00:00 2001 From: Vivia Nikolaidou Date: Wed, 25 Jul 2018 17:15:53 +0300 Subject: [PATCH] deinterlace: Closed caption pass-through Pass through closed caption data when deinterlacing. When two deinterlaced frames are created for the same interlaced frame (e.g. fields=all), the second of the two frames will have no closed caption data. Also fixed memory leaks related to timecode meta pass-through. https://bugzilla.gnome.org/show_bug.cgi?id=796876 --- gst/deinterlace/gstdeinterlace.c | 192 +++++++++++++++++-------- gst/deinterlace/gstdeinterlacemethod.h | 1 + 2 files changed, 133 insertions(+), 60 deletions(-) diff --git a/gst/deinterlace/gstdeinterlace.c b/gst/deinterlace/gstdeinterlace.c index ed19a5c9b9..51e91336c2 100644 --- a/gst/deinterlace/gstdeinterlace.c +++ b/gst/deinterlace/gstdeinterlace.c @@ -760,6 +760,77 @@ gst_video_frame_unmap_and_free (GstVideoFrame * frame) g_free (frame); } +static GstVideoFrame * +gst_deinterlace_pop_history (GstDeinterlace * self) +{ + GstVideoFrame *frame; + + g_return_val_if_fail (self->history_count > 0, NULL); + + GST_DEBUG_OBJECT (self, "Pop last history frame -- current history size %d", + self->history_count); + + frame = self->field_history[self->history_count - 1].frame; + + self->history_count--; + if (self->locking != GST_DEINTERLACE_LOCKING_NONE && (!self->history_count + || GST_VIDEO_FRAME_PLANE_DATA (frame, 0) != + GST_VIDEO_FRAME_PLANE_DATA (self->field_history[self->history_count - + 1].frame, 0))) { + if (!self->low_latency) + self->state_count--; + if (self->pattern_lock) { + self->pattern_count++; + if (self->pattern != -1 + && self->pattern_count >= telecine_patterns[self->pattern].length) { + self->pattern_count = 0; + self->output_count = 0; + } + } + } + + GST_DEBUG_OBJECT (self, "Returning frame: %p %" GST_TIME_FORMAT + " with duration %" GST_TIME_FORMAT " and size %" G_GSIZE_FORMAT, frame, + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (frame->buffer)), + GST_TIME_ARGS (GST_BUFFER_DURATION (frame->buffer)), + GST_VIDEO_FRAME_SIZE (frame)); + + return frame; +} + +static void +gst_deinterlace_delete_meta_at (GstDeinterlace * self, gint idx) +{ + if (self->field_history[idx].frame) { + if (self->field_history[idx].tc) { + gst_video_time_code_free (self->field_history[idx].tc); + self->field_history[idx].tc = NULL; + } + if (self->field_history[idx].caption) { + g_free (self->field_history[idx].caption->data); + g_free (self->field_history[idx].caption); + self->field_history[idx].caption = NULL; + } + } +} + +static void +gst_deinterlace_pop_and_clear (GstDeinterlace * self) +{ + gint idx; + + if (self->history_count <= 0) + return; + + idx = self->history_count - 1; + gst_deinterlace_delete_meta_at (self, idx); + + /* FIXME: pop_history should return a structure with the frame and its meta. + * Currently we're just doing guesswork with the indices. Maybe just + * refactor the history functionality to make something clearer */ + gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); +} + static void gst_deinterlace_reset_history (GstDeinterlace * self, gboolean drop_all) { @@ -783,9 +854,7 @@ gst_deinterlace_reset_history (GstDeinterlace * self, gboolean drop_all) if (self->field_history[i].frame) { gst_video_frame_unmap_and_free (self->field_history[i].frame); self->field_history[i].frame = NULL; - if (self->field_history[i].tc) { - gst_video_time_code_free (self->field_history[i].tc); - } + gst_deinterlace_delete_meta_at (self, i); } } } @@ -985,44 +1054,6 @@ gst_deinterlace_update_pattern_timestamps (GstDeinterlace * self) GST_TIME_ARGS (self->pattern_buf_dur)); } -static GstVideoFrame * -gst_deinterlace_pop_history (GstDeinterlace * self) -{ - GstVideoFrame *frame; - - g_return_val_if_fail (self->history_count > 0, NULL); - - GST_DEBUG_OBJECT (self, "Pop last history frame -- current history size %d", - self->history_count); - - frame = self->field_history[self->history_count - 1].frame; - - self->history_count--; - if (self->locking != GST_DEINTERLACE_LOCKING_NONE && (!self->history_count - || GST_VIDEO_FRAME_PLANE_DATA (frame, 0) != - GST_VIDEO_FRAME_PLANE_DATA (self->field_history[self->history_count - - 1].frame, 0))) { - if (!self->low_latency) - self->state_count--; - if (self->pattern_lock) { - self->pattern_count++; - if (self->pattern != -1 - && self->pattern_count >= telecine_patterns[self->pattern].length) { - self->pattern_count = 0; - self->output_count = 0; - } - } - } - - GST_DEBUG_OBJECT (self, "Returning frame: %p %" GST_TIME_FORMAT - " with duration %" GST_TIME_FORMAT " and size %" G_GSIZE_FORMAT, frame, - GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (frame->buffer)), - GST_TIME_ARGS (GST_BUFFER_DURATION (frame->buffer)), - GST_VIDEO_FRAME_SIZE (frame)); - - return frame; -} - static void gst_deinterlace_get_buffer_state (GstDeinterlace * self, GstVideoFrame * frame, guint8 * state, GstVideoInterlaceMode * i_mode) @@ -1129,14 +1160,9 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer) self->field_history[i - fields_to_push].frame; self->field_history[i].flags = self->field_history[i - fields_to_push].flags; - if (self->field_history[i].tc) - gst_video_time_code_free (self->field_history[i].tc); - if (self->field_history[i - fields_to_push].tc) { - self->field_history[i].tc = - gst_video_time_code_copy (self->field_history[i - fields_to_push].tc); - } else { - self->field_history[i].tc = NULL; - } + self->field_history[i].tc = self->field_history[i - fields_to_push].tc; + self->field_history[i].caption = + self->field_history[i - fields_to_push].caption; } if (field_layout == GST_DEINTERLACE_LAYOUT_AUTO) { @@ -1171,6 +1197,7 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer) if (!onefield) { GstVideoTimeCodeMeta *meta = gst_buffer_get_video_time_code_meta (buffer); + GstVideoCaptionMeta *cc_meta = gst_buffer_get_video_caption_meta (buffer); GST_DEBUG_OBJECT (self, "Two fields"); self->field_history[1].frame = field1; @@ -1187,8 +1214,24 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer) self->field_history[1].tc->config.flags &= ~GST_VIDEO_TIME_CODE_FLAGS_INTERLACED; } + + if (cc_meta) { + self->field_history[0].caption = g_new (GstVideoCaptionMeta, 1); + self->field_history[0].caption->data = g_malloc (cc_meta->size); + self->field_history[0].caption->caption_type = cc_meta->caption_type; + self->field_history[0].caption->size = cc_meta->size; + memcpy (self->field_history[0].caption->data, cc_meta->data, + cc_meta->size); + self->field_history[1].caption = g_new (GstVideoCaptionMeta, 1); + self->field_history[1].caption->data = g_malloc (cc_meta->size); + self->field_history[1].caption->caption_type = cc_meta->caption_type; + self->field_history[1].caption->size = cc_meta->size; + memcpy (self->field_history[1].caption->data, cc_meta->data, + cc_meta->size); + } } else { /* onefield */ GstVideoTimeCodeMeta *meta = gst_buffer_get_video_time_code_meta (buffer); + GstVideoCaptionMeta *cc_meta = gst_buffer_get_video_caption_meta (buffer); GST_DEBUG_OBJECT (self, "One field"); self->field_history[0].frame = field1; @@ -1198,6 +1241,15 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer) self->field_history[0].tc->config.flags &= ~GST_VIDEO_TIME_CODE_FLAGS_INTERLACED; } + + if (cc_meta) { + self->field_history[0].caption = g_new (GstVideoCaptionMeta, 1); + self->field_history[0].caption->data = g_malloc (cc_meta->size); + self->field_history[0].caption->caption_type = cc_meta->caption_type; + self->field_history[0].caption->size = cc_meta->size; + memcpy (self->field_history[0].caption->data, cc_meta->data, + cc_meta->size); + } gst_video_frame_unmap_and_free (field2); } @@ -1525,6 +1577,7 @@ gst_deinterlace_output_frame (GstDeinterlace * self, gboolean flushing) TelecinePattern pattern; guint8 phase, count; const GstDeinterlaceLocking locking = self->locking; + gboolean cc_added = FALSE; memset (&pattern, 0, sizeof (pattern)); @@ -1604,7 +1657,7 @@ restart: if (flush_one && self->drop_orphans) { GST_DEBUG_OBJECT (self, "Dropping orphan first field"); self->cur_field_idx--; - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); goto restart; } } @@ -1664,15 +1717,17 @@ restart: GST_DEBUG_OBJECT (self, "Frame type: Progressive; pushing buffer as a frame"); /* pop and push */ + gst_deinterlace_delete_meta_at (self, self->history_count - 1); self->cur_field_idx--; field1_frame = gst_deinterlace_pop_history (self); field1_buffer = field1_frame->buffer; gst_buffer_ref (field1_buffer); gst_video_frame_unmap_and_free (field1_frame); + /* field2 is the same buffer as field1, but we need to remove it from the * history anyway */ self->cur_field_idx--; - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); GST_DEBUG_OBJECT (self, "[OUT] ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", end %" GST_TIME_FORMAT, @@ -1786,6 +1841,15 @@ restart: gst_buffer_add_video_time_code_meta (outbuf, self->field_history[index].tc); } + if (self->field_history[index].caption) { + g_assert (self->field_history[index].caption->data != NULL); + g_assert (!cc_added); + gst_buffer_add_video_caption_meta (outbuf, + self->field_history[index].caption->caption_type, + self->field_history[index].caption->data, + self->field_history[index].caption->size); + cc_added = TRUE; + } if (IS_TELECINE (interlacing_mode) && !self->telecine_tc_warned) { self->telecine_tc_warned = TRUE; GST_FIXME_OBJECT (self, @@ -1828,14 +1892,14 @@ restart: /* Check if we need to drop the frame because of QoS */ if (!gst_deinterlace_do_qos (self, buf)) { self->cur_field_idx--; - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); gst_buffer_unref (outbuf); outbuf = NULL; ret = GST_FLOW_OK; } else { if (self->cur_field_idx < 0 && flushing) { if (self->history_count == 1) { - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); goto need_more; } self->cur_field_idx++; @@ -1866,7 +1930,7 @@ restart: || self->cur_field_idx + 1 + gst_deinterlace_method_get_latency (self->method) < self->history_count || flushing) { - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); } if (gst_deinterlace_clip_buffer (self, outbuf)) { @@ -1895,7 +1959,7 @@ restart: GST_DEBUG_OBJECT (self, "Removing unused field (count: %d)", self->history_count); self->cur_field_idx--; - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); interlacing_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED; return ret; } @@ -1912,7 +1976,7 @@ restart: && !IS_TELECINE (interlacing_mode))) { GST_DEBUG_OBJECT (self, "Removing unused top field"); self->cur_field_idx--; - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); if (flush_one && !self->drop_orphans) { GST_DEBUG_OBJECT (self, "Orphan field deinterlaced - reconfiguring"); @@ -1953,6 +2017,14 @@ restart: gst_buffer_add_video_time_code_meta (outbuf, self->field_history[index].tc); } + if (self->field_history[index].caption && !cc_added) { + g_assert (self->field_history[index].caption->data != NULL); + gst_buffer_add_video_caption_meta (outbuf, + self->field_history[index].caption->caption_type, + self->field_history[index].caption->data, + self->field_history[index].caption->size); + cc_added = TRUE; + } if (IS_TELECINE (interlacing_mode) && !self->telecine_tc_warned) { self->telecine_tc_warned = TRUE; GST_FIXME_OBJECT (self, @@ -1995,7 +2067,7 @@ restart: /* Check if we need to drop the frame because of QoS */ if (!gst_deinterlace_do_qos (self, buf)) { self->cur_field_idx--; - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); gst_buffer_unref (outbuf); outbuf = NULL; ret = GST_FLOW_OK; @@ -2019,7 +2091,7 @@ restart: || self->cur_field_idx + 1 + gst_deinterlace_method_get_latency (self->method) < self->history_count) { - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); } if (gst_deinterlace_clip_buffer (self, outbuf)) { @@ -2044,7 +2116,7 @@ restart: GST_DEBUG_OBJECT (self, "Removing unused field (count: %d)", self->history_count); self->cur_field_idx--; - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); interlacing_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED; return ret; } @@ -2061,7 +2133,7 @@ restart: && !IS_TELECINE (interlacing_mode))) { GST_DEBUG_OBJECT (self, "Removing unused bottom field"); self->cur_field_idx--; - gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self)); + gst_deinterlace_pop_and_clear (self); if (flush_one && !self->drop_orphans) { GST_DEBUG_OBJECT (self, "Orphan field deinterlaced - reconfiguring"); diff --git a/gst/deinterlace/gstdeinterlacemethod.h b/gst/deinterlace/gstdeinterlacemethod.h index c4ab201c05..e967f2c8ff 100644 --- a/gst/deinterlace/gstdeinterlacemethod.h +++ b/gst/deinterlace/gstdeinterlacemethod.h @@ -55,6 +55,7 @@ typedef struct /* see PICTURE_ flags in *.c */ guint flags; GstVideoTimeCode *tc; + GstVideoCaptionMeta *caption; } GstDeinterlaceField; /*