h264parse: extract CEA-708 closed captions

Expose SEI data in the H.264 bitstream parser API and
extract closed captions and other things that are not
specified in the H.264 spec itself in the videoparser.

Based on patch by: Mathieu Duponchelle <mathieu@centricular.com>

https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/940
This commit is contained in:
Tim-Philipp Müller 2019-03-29 21:02:44 +00:00
parent 2e442b801b
commit 76f1ed15fb
4 changed files with 185 additions and 0 deletions

View file

@ -1008,6 +1008,51 @@ error:
return GST_H264_PARSER_ERROR;
}
static GstH264ParserResult
gst_h264_parser_parse_registered_user_data (GstH264NalParser * nalparser,
GstH264RegisteredUserData * rud, NalReader * nr, guint payload_size)
{
guint8 *data = NULL;
guint i;
rud->data = NULL;
rud->size = 0;
if (payload_size < 2)
return GST_H264_PARSER_ERROR;
READ_UINT8 (nr, rud->country_code, 8);
--payload_size;
if (rud->country_code == 0xFF) {
READ_UINT8 (nr, rud->country_code_extension, 8);
--payload_size;
} else {
rud->country_code_extension = 0;
}
if (payload_size < 8)
return GST_H264_PARSER_ERROR;
data = g_malloc (payload_size);
for (i = 0; i < payload_size / 8; ++i) {
READ_UINT8 (nr, data[i], 8);
}
GST_MEMDUMP ("SEI user data", data, payload_size / 8);
rud->data = data;
rud->size = payload_size;
return GST_H264_PARSER_OK;
error:
{
GST_WARNING ("error parsing \"Registered User Data\"");
g_free (data);
return GST_H264_PARSER_ERROR;
}
}
static GstH264ParserResult
gst_h264_parser_parse_recovery_point (GstH264NalParser * nalparser,
GstH264RecoveryPoint * rp, NalReader * nr)
@ -1157,6 +1202,10 @@ gst_h264_parser_parse_sei_message (GstH264NalParser * nalparser,
res = gst_h264_parser_parse_pic_timing (nalparser,
&sei->payload.pic_timing, nr);
break;
case GST_H264_SEI_REGISTERED_USER_DATA:
res = gst_h264_parser_parse_registered_user_data (nalparser,
&sei->payload.registered_user_data, nr, payload_size);
break;
case GST_H264_SEI_RECOVERY_POINT:
res = gst_h264_parser_parse_recovery_point (nalparser,
&sei->payload.recovery_point, nr);
@ -2276,6 +2325,22 @@ gst_h264_sps_clear (GstH264SPS * sps)
}
}
static void
h264_sei_message_clear (GstH264SEIMessage * sei_msg)
{
switch (sei_msg->payloadType) {
case GST_H264_SEI_REGISTERED_USER_DATA:{
GstH264RegisteredUserData *rud = &sei_msg->payload.registered_user_data;
g_free ((guint8 *) rud->data);
rud->data = NULL;
break;
}
default:
break;
}
}
/**
* gst_h264_parser_parse_sei:
* @nalparser: a #GstH264NalParser
@ -2299,6 +2364,7 @@ gst_h264_parser_parse_sei (GstH264NalParser * nalparser, GstH264NalUnit * nalu,
nal_reader_init (&nr, nalu->data + nalu->offset + nalu->header_bytes,
nalu->size - nalu->header_bytes);
*messages = g_array_new (FALSE, FALSE, sizeof (GstH264SEIMessage));
g_array_set_clear_func (*messages, (GDestroyNotify) h264_sei_message_clear);
do {
res = gst_h264_parser_parse_sei_message (nalparser, &nr, &sei);

View file

@ -240,6 +240,7 @@ typedef enum
* GstH264SEIPayloadType:
* @GST_H264_SEI_BUF_PERIOD: Buffering Period SEI Message
* @GST_H264_SEI_PIC_TIMING: Picture Timing SEI Message
* @GST_H264_SEI_REGISTERED_USER_DATA: Registered user data (D.2.5)
* @GST_H264_SEI_RECOVERY_POINT: Recovery Point SEI Message (D.2.7)
* @GST_H264_SEI_STEREO_VIDEO_INFO: stereo video info SEI message (Since: 1.6)
* @GST_H264_SEI_FRAME_PACKING: Frame Packing Arrangement (FPA) message that
@ -252,6 +253,7 @@ typedef enum
{
GST_H264_SEI_BUF_PERIOD = 0,
GST_H264_SEI_PIC_TIMING = 1,
GST_H264_SEI_REGISTERED_USER_DATA = 4,
GST_H264_SEI_RECOVERY_POINT = 6,
GST_H264_SEI_STEREO_VIDEO_INFO = 21,
GST_H264_SEI_FRAME_PACKING = 45
@ -345,6 +347,7 @@ typedef struct _GstH264SliceHdr GstH264SliceHdr;
typedef struct _GstH264ClockTimestamp GstH264ClockTimestamp;
typedef struct _GstH264PicTiming GstH264PicTiming;
typedef struct _GstH264RegisteredUserData GstH264RegisteredUserData;
typedef struct _GstH264BufferingPeriod GstH264BufferingPeriod;
typedef struct _GstH264RecoveryPoint GstH264RecoveryPoint;
typedef struct _GstH264StereoVideoInfo GstH264StereoVideoInfo;
@ -994,6 +997,14 @@ struct _GstH264PicTiming
GstH264ClockTimestamp clock_timestamp[3];
};
struct _GstH264RegisteredUserData
{
guint8 country_code;
guint8 country_code_extension;
const guint8 *data;
guint size;
};
struct _GstH264BufferingPeriod
{
GstH264SPS *sps;
@ -1022,6 +1033,7 @@ struct _GstH264SEIMessage
union {
GstH264BufferingPeriod buffering_period;
GstH264PicTiming pic_timing;
GstH264RegisteredUserData registered_user_data;
GstH264RecoveryPoint recovery_point;
GstH264StereoVideoInfo stereo_video_info;
GstH264FramePacking frame_packing;

View file

@ -519,6 +519,95 @@ _nal_name (GstH264NalUnitType nal_type)
}
#endif
static void
gst_h264_parse_process_sei_user_data (GstH264Parse * h264parse,
GstH264RegisteredUserData * rud)
{
guint16 provider_code;
guint32 atsc_user_id;
GstByteReader br;
if (rud->country_code != 0xB5)
return;
if (rud->data == NULL || rud->size < 2)
return;
gst_byte_reader_init (&br, rud->data, rud->size);
provider_code = gst_byte_reader_get_uint16_be_unchecked (&br);
/* There is also 0x29 / 47 for DirecTV, but we don't handle that for now.
* https://en.wikipedia.org/wiki/CEA-708#Picture_User_Data */
if (provider_code != 0x0031)
return;
/* ANSI/SCTE 128-2010a section 8.1.2 */
if (!gst_byte_reader_get_uint32_be (&br, &atsc_user_id))
return;
atsc_user_id = GUINT32_FROM_BE (atsc_user_id);
switch (atsc_user_id) {
case GST_MAKE_FOURCC ('G', 'A', '9', '4'):{
guint8 user_data_type_code, m;
/* 8.2.1 ATSC1_data() Semantics, Table 15 */
if (!gst_byte_reader_get_uint8 (&br, &user_data_type_code))
return;
switch (user_data_type_code) {
case 0x03:{
guint8 process_cc_data_flag;
guint8 cc_count, em_data, b;
guint i;
if (!gst_byte_reader_get_uint8 (&br, &b) || (b & 0x20) != 0)
break;
if (!gst_byte_reader_get_uint8 (&br, &em_data) || em_data != 0xff)
break;
process_cc_data_flag = (b & 0x40) != 0;
cc_count = b & 0x1f;
if (cc_count * 3 > gst_byte_reader_get_remaining (&br))
break;
if (!process_cc_data_flag || cc_count == 0) {
gst_byte_reader_skip_unchecked (&br, cc_count * 3);
break;
}
/* Shouldn't really happen so let's not go out of our way to handle it */
if (h264parse->closedcaptions_size > 0) {
GST_WARNING_OBJECT (h264parse, "unused pending closed captions!");
GST_MEMDUMP_OBJECT (h264parse, "unused captions being dropped",
h264parse->closedcaptions, h264parse->closedcaptions_size);
}
for (i = 0; i < cc_count * 3; i++) {
h264parse->closedcaptions[i] =
gst_byte_reader_get_uint8_unchecked (&br);
}
h264parse->closedcaptions_size = cc_count * 3;
h264parse->closedcaptions_type = GST_VIDEO_CAPTION_TYPE_CEA708_RAW;
GST_LOG_OBJECT (h264parse, "Extracted closed captions");
break;
}
default:
GST_LOG ("Unhandled atsc1 user data type %d", atsc_user_id);
break;
}
if (!gst_byte_reader_get_uint8 (&br, &m) || m != 0xff)
GST_WARNING_OBJECT (h264parse, "Marker bits mismatch after ATSC1_data");
break;
}
default:
break;
}
}
static void
gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
{
@ -563,6 +652,10 @@ gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
GST_LOG_OBJECT (h264parse, "pic timing updated");
break;
}
case GST_H264_SEI_REGISTERED_USER_DATA:
gst_h264_parse_process_sei_user_data (h264parse,
&sei.payload.registered_user_data);
break;
case GST_H264_SEI_BUF_PERIOD:
if (h264parse->ts_trn_nb == GST_CLOCK_TIME_NONE ||
h264parse->dts == GST_CLOCK_TIME_NONE)
@ -2486,6 +2579,15 @@ gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
buffer = frame->buffer;
}
if (h264parse->closedcaptions_size > 0) {
gst_buffer_add_video_caption_meta (buffer,
h264parse->closedcaptions_type, h264parse->closedcaptions,
h264parse->closedcaptions_size);
h264parse->closedcaptions_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
h264parse->closedcaptions_size = 0;
}
if ((event = check_pending_key_unit_event (h264parse->force_key_unit_event,
&parse->segment, GST_BUFFER_TIMESTAMP (buffer),
GST_BUFFER_FLAGS (buffer), h264parse->pending_key_unit_ts))) {

View file

@ -141,6 +141,11 @@ struct _GstH264Parse
/* For insertion of AU Delimiter */
gboolean aud_needed;
gboolean aud_insert;
/* pending closed captions */
guint8 closedcaptions[96];
guint closedcaptions_size;
GstVideoCaptionType closedcaptions_type;
};
struct _GstH264ParseClass