From be8cec5348783df9f20e0aad8ba499224d3203a2 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Wed, 8 Apr 2020 15:37:03 +0900 Subject: [PATCH] h264parse: Add support for inband timecode update Add new property "update-timecode" to allow updating timecode in picture timing SEI depending on timecode meta. Since the picture timing SEI message requires proper VUI setting but we don't support re-writing SPS, this might not work for some streams --- gst/videoparsers/gsth264parse.c | 246 ++++++++++++++++++++++++++++++-- gst/videoparsers/gsth264parse.h | 5 +- 2 files changed, 242 insertions(+), 9 deletions(-) diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c index 99658f4e6d..822c50fe7c 100644 --- a/gst/videoparsers/gsth264parse.c +++ b/gst/videoparsers/gsth264parse.c @@ -37,11 +37,13 @@ GST_DEBUG_CATEGORY (h264_parse_debug); #define GST_CAT_DEFAULT h264_parse_debug #define DEFAULT_CONFIG_INTERVAL (0) +#define DEFAULT_UPDATE_TIMECODE FALSE enum { PROP_0, - PROP_CONFIG_INTERVAL + PROP_CONFIG_INTERVAL, + PROP_UPDATE_TIMECODE, }; enum @@ -144,6 +146,29 @@ gst_h264_parse_class_init (GstH264ParseClass * klass) -1, 3600, DEFAULT_CONFIG_INTERVAL, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + /** + * GstH264Parse:update-timecode: + * + * If the stream contains Picture Timing SEI, update their timecode values + * using upstream GstVideoTimeCodeMeta. However, if there are no Picture + * Timing SEI in bitstream, this property will not insert the SEI into the + * bitstream - it only modifies existing ones. + * Moreover, even if both GstVideoTimeCodeMeta and Picture Timing SEI + * are present, if pic_struct_present_flag of VUI is equal to zero, + * timecode values will not updated as there is not enough information + * in the stream to do so. + * + * Since: 1.18 + */ + g_object_class_install_property (gobject_class, PROP_UPDATE_TIMECODE, + g_param_spec_boolean ("update-timecode", + "Update Timecode", + "Update time code values in Picture Timing SEI if GstVideoTimeCodeMeta " + "is attached to incoming buffer and also Picture Timing SEI exists " + "in the bitstream. To make this property work, SPS must contain " + "VUI and pic_struct_present_flag of VUI must be non-zero", + DEFAULT_CONFIG_INTERVAL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /* Override BaseParse vfuncs */ parse_class->start = GST_DEBUG_FUNCPTR (gst_h264_parse_start); parse_class->stop = GST_DEBUG_FUNCPTR (gst_h264_parse_stop); @@ -175,9 +200,9 @@ gst_h264_parse_init (GstH264Parse * h264parse) h264parse->aud_needed = TRUE; h264parse->aud_insert = TRUE; + h264parse->update_timecode = FALSE; } - static void gst_h264_parse_finalize (GObject * object) { @@ -200,6 +225,8 @@ gst_h264_parse_reset_frame (GstH264Parse * h264parse) h264parse->update_caps = FALSE; h264parse->idr_pos = -1; h264parse->sei_pos = -1; + h264parse->pic_timing_sei_pos = -1; + h264parse->pic_timing_sei_size = -1; h264parse->keyframe = FALSE; h264parse->predicted = FALSE; h264parse->bidirectional = FALSE; @@ -604,13 +631,23 @@ gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu) } h264parse->num_clock_timestamp = 0; + memcpy (&h264parse->pic_timing_sei, &sei.payload.pic_timing, + sizeof (GstH264PicTiming)); for (j = 0; j < 3; j++) { if (sei.payload.pic_timing.clock_timestamp_flag[j]) { - memcpy (&h264parse-> - clock_timestamp[h264parse->num_clock_timestamp++], - &sei.payload.pic_timing.clock_timestamp[j], - sizeof (GstH264ClockTimestamp)); + h264parse->num_clock_timestamp++; + } + } + + if (h264parse->sei_pic_struct_pres_flag && h264parse->update_timecode) { + /* FIXME: add support multiple messages in a SEI nalu. + * Updating only this SEI message and preserving the others + * is a bit complicated */ + if (messages->len == 1) { + h264parse->pic_timing_sei_pos = nalu->sc_offset; + h264parse->pic_timing_sei_size = + nalu->size + (nalu->offset - nalu->sc_offset); } } @@ -2745,11 +2782,181 @@ gst_h264_parse_handle_sps_pps_nals (GstH264Parse * h264parse, return send_done; } +static GstBuffer * +gst_h264_parse_create_pic_timing_sei (GstH264Parse * h264parse, + GstBuffer * buffer) +{ + guint num_meta; + const guint8 num_clock_ts_table[9] = { + 1, 1, 1, 2, 2, 3, 3, 2, 3 + }; + guint num_clock_ts; + GstBuffer *out_buf = NULL; + GstMemory *sei_mem; + GArray *msg_array; + gint i, j; + GstH264SEIMessage sei; + GstH264PicTiming *pic_timing; + GstVideoTimeCodeMeta *tc_meta; + gpointer iter = NULL; + guint8 ct_type = GST_H264_CT_TYPE_PROGRESSIVE; + + if (!h264parse->update_timecode) + return NULL; + + num_meta = gst_buffer_get_n_meta (buffer, GST_VIDEO_TIME_CODE_META_API_TYPE); + if (num_meta == 0) + return NULL; + + if (!h264parse->sei_pic_struct_pres_flag || h264parse->pic_timing_sei_pos < 0) { + GST_ELEMENT_WARNING (h264parse, STREAM, NOT_IMPLEMENTED, (NULL), + ("timecode update was requested but VUI doesn't support timecode")); + return NULL; + } + + g_assert (h264parse->sei_pic_struct >= GST_H264_SEI_PIC_STRUCT_FRAME); + g_assert (h264parse->sei_pic_struct <= + GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING); + + num_clock_ts = num_clock_ts_table[h264parse->sei_pic_struct]; + + if (num_meta != num_clock_ts) { + GST_LOG_OBJECT (h264parse, + "The number of timecode meta %d is not equal to required %d", + num_meta, num_clock_ts); + + return NULL; + } + + GST_LOG_OBJECT (h264parse, + "The number of timecode meta %d is equal", num_meta); + + memset (&sei, 0, sizeof (GstH264SEIMessage)); + sei.payloadType = GST_H264_SEI_PIC_TIMING; + memcpy (&sei.payload.pic_timing, + &h264parse->pic_timing_sei, sizeof (GstH264PicTiming)); + + pic_timing = &sei.payload.pic_timing; + + switch (h264parse->sei_pic_struct) { + case GST_H264_SEI_PIC_STRUCT_FRAME: + case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING: + case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING: + ct_type = GST_H264_CT_TYPE_PROGRESSIVE; + break; + case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM: + case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP: + case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP: + case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM: + ct_type = GST_H264_CT_TYPE_INTERLACED; + break; + default: + ct_type = GST_H264_CT_TYPE_UNKNOWN; + break; + } + + i = 0; + while ((tc_meta = + (GstVideoTimeCodeMeta *) gst_buffer_iterate_meta_filtered (buffer, + &iter, GST_VIDEO_TIME_CODE_META_API_TYPE))) { + GstH264ClockTimestamp *tim = &pic_timing->clock_timestamp[i]; + GstVideoTimeCode *tc = &tc_meta->tc; + + pic_timing->clock_timestamp_flag[i] = 1; + tim->ct_type = ct_type; + tim->nuit_field_based_flag = 1; + tim->counting_type = 0; + + if ((tc->config.flags & GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME) + == GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME) + tim->counting_type = 4; + + tim->discontinuity_flag = 0; + tim->cnt_dropped_flag = 0; + tim->n_frames = tc->frames; + + tim->hours_value = tc->hours; + tim->minutes_value = tc->minutes; + tim->seconds_value = tc->seconds; + + tim->full_timestamp_flag = + tim->seconds_flag = tim->minutes_flag = tim->hours_flag = 0; + + if (tc->hours > 0) + tim->full_timestamp_flag = 1; + else if (tc->minutes > 0) + tim->seconds_flag = tim->minutes_flag = 1; + else if (tc->seconds > 0) + tim->seconds_flag = 1; + + GST_LOG_OBJECT (h264parse, + "New time code value %02u:%02u:%02u:%02u", + tim->hours_value, tim->minutes_value, tim->seconds_value, + tim->n_frames); + + i++; + } + + for (j = i; j < 3; j++) + pic_timing->clock_timestamp_flag[j] = 0; + + msg_array = g_array_new (FALSE, FALSE, sizeof (GstH264SEIMessage)); + g_array_set_clear_func (msg_array, (GDestroyNotify) gst_h264_sei_clear); + + g_array_append_val (msg_array, sei); + if (h264parse->format == GST_H264_PARSE_FORMAT_BYTE) { + sei_mem = gst_h264_create_sei_memory (3, msg_array); + } else { + sei_mem = gst_h264_create_sei_memory_avc (h264parse->nal_length_size, + msg_array); + } + g_array_unref (msg_array); + + if (!sei_mem) { + GST_WARNING_OBJECT (h264parse, "Cannot create Picture Timing SEI memory"); + return NULL; + } + + out_buf = gst_buffer_new (); + gst_buffer_copy_into (out_buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1); + + if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) { + gst_buffer_append_memory (out_buf, sei_mem); + } else { + gsize mem_size; + + mem_size = gst_memory_get_sizes (sei_mem, NULL, NULL); + + /* copy every data except for the SEI */ + if (h264parse->pic_timing_sei_pos > 0) { + gst_buffer_copy_into (out_buf, buffer, GST_BUFFER_COPY_MEMORY, 0, + h264parse->pic_timing_sei_pos); + } + + /* insert new SEI */ + gst_buffer_append_memory (out_buf, sei_mem); + + if (gst_buffer_get_size (buffer) > + h264parse->pic_timing_sei_pos + h264parse->pic_timing_sei_size) { + gst_buffer_copy_into (out_buf, buffer, GST_BUFFER_COPY_MEMORY, + h264parse->pic_timing_sei_pos + h264parse->pic_timing_sei_size, -1); + } + + if (h264parse->idr_pos >= 0) { + h264parse->idr_pos += mem_size; + h264parse->idr_pos -= h264parse->pic_timing_sei_size; + } + } + + return out_buf; +} + static GstFlowReturn gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame) { GstH264Parse *h264parse; GstBuffer *buffer; + GstBuffer *new_buf; GstEvent *event; GstBuffer *parse_buffer = NULL; gboolean is_interlaced = FALSE; @@ -2817,6 +3024,15 @@ gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame) gst_h264_parse_prepare_key_unit (h264parse, event); } + /* handle timecode */ + new_buf = gst_h264_parse_create_pic_timing_sei (h264parse, buffer); + if (new_buf) { + if (frame->out_buffer) + gst_buffer_unref (frame->out_buffer); + + buffer = frame->out_buffer = new_buf; + } + /* periodic SPS/PPS sending */ if (h264parse->interval > 0 || h264parse->push_codec) { GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer); @@ -2899,12 +3115,16 @@ gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame) if (!gst_buffer_get_video_time_code_meta (buffer)) { guint i = 0; - for (i = 0; i < h264parse->num_clock_timestamp; i++) { - GstH264ClockTimestamp *tim = &h264parse->clock_timestamp[i]; + for (i = 0; i < 3 && h264parse->num_clock_timestamp; i++) { + GstH264ClockTimestamp *tim = + &h264parse->pic_timing_sei.clock_timestamp[i]; gint field_count = -1; guint n_frames; GstVideoTimeCodeFlags flags = 0; + if (!h264parse->pic_timing_sei.clock_timestamp_flag[i]) + continue; + /* Table D-1 */ switch (h264parse->sei_pic_struct) { case GST_H264_SEI_PIC_STRUCT_FRAME: @@ -2951,6 +3171,10 @@ gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame) gst_util_uint64_scale_int (tim->n_frames, 1, 2 - tim->nuit_field_based_flag); + GST_LOG_OBJECT (h264parse, + "Add time code meta %02u:%02u:%02u:%02u", + tim->hours_value, tim->minutes_value, tim->seconds_value, n_frames); + gst_buffer_add_video_time_code_meta_full (buffer, h264parse->parsed_fps_n, h264parse->parsed_fps_d, @@ -3416,6 +3640,9 @@ gst_h264_parse_set_property (GObject * object, guint prop_id, case PROP_CONFIG_INTERVAL: parse->interval = g_value_get_int (value); break; + case PROP_UPDATE_TIMECODE: + parse->update_timecode = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -3434,6 +3661,9 @@ gst_h264_parse_get_property (GObject * object, guint prop_id, case PROP_CONFIG_INTERVAL: g_value_set_int (value, parse->interval); break; + case PROP_UPDATE_TIMECODE: + g_value_set_boolean (value, parse->update_timecode); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/gst/videoparsers/gsth264parse.h b/gst/videoparsers/gsth264parse.h index 5d45017893..61579c65e7 100644 --- a/gst/videoparsers/gsth264parse.h +++ b/gst/videoparsers/gsth264parse.h @@ -100,7 +100,7 @@ struct _GstH264Parse /* collected SEI timestamps */ guint num_clock_timestamp; - GstH264ClockTimestamp clock_timestamp[3]; + GstH264PicTiming pic_timing_sei; /* Infos we need to keep track of */ guint32 sei_cpb_removal_delay; @@ -121,6 +121,8 @@ struct _GstH264Parse /*guint last_nal_pos;*/ /*guint next_sc_pos;*/ gint idr_pos, sei_pos; + gint pic_timing_sei_pos; + gint pic_timing_sei_size; gboolean update_caps; GstAdapter *frame_out; gboolean keyframe; @@ -133,6 +135,7 @@ struct _GstH264Parse /* props */ gint interval; + gboolean update_timecode; GstClockTime pending_key_unit_ts; GstEvent *force_key_unit_event;