From 2c514ac2f1eb285d896da545b818e97865771b62 Mon Sep 17 00:00:00 2001 From: Julian Bouzas Date: Mon, 19 Aug 2024 12:16:49 -0400 Subject: [PATCH] h264parse: attach LCEVC meta to buffers if it is present in SEI This improves the h264parse element to attach LCEVC enhancement data to buffers using the new GstLcevcMeta API. This metadata will eventually be used downstream by LCEVC decoders to enhance the RAW video frame. Part-of: --- subprojects/gst-integration-testsuites/medias | 2 +- .../flow-expectations/log-parse-src-expected | 2 +- .../flow-expectations/log-parse-src-expected | 2 +- .../gst/videoparsers/gsth264parse.c | 13 +++++-- .../gst/videoparsers/gsth265parse.c | 2 +- .../gst/videoparsers/gstmpegvideoparse.c | 2 +- .../gst/videoparsers/gstvideoparseutils.c | 36 ++++++++++++++++++- .../gst/videoparsers/gstvideoparseutils.h | 12 +++++-- 8 files changed, 61 insertions(+), 10 deletions(-) diff --git a/subprojects/gst-integration-testsuites/medias b/subprojects/gst-integration-testsuites/medias index fa0b7ec4a3..2df2532dc7 160000 --- a/subprojects/gst-integration-testsuites/medias +++ b/subprojects/gst-integration-testsuites/medias @@ -1 +1 @@ -Subproject commit fa0b7ec4a3e48371ff25d608ff5278e6fac1060c +Subproject commit 2df2532dc766b95df8b8ee42fb4cb2dcf2e5ae59 diff --git a/subprojects/gst-integration-testsuites/testsuites/validate/h264/parse.trickmode_predicted.seek_trickmode_predicted/flow-expectations/log-parse-src-expected b/subprojects/gst-integration-testsuites/testsuites/validate/h264/parse.trickmode_predicted.seek_trickmode_predicted/flow-expectations/log-parse-src-expected index 35126eb6b2..06c9c3a0de 100644 --- a/subprojects/gst-integration-testsuites/testsuites/validate/h264/parse.trickmode_predicted.seek_trickmode_predicted/flow-expectations/log-parse-src-expected +++ b/subprojects/gst-integration-testsuites/testsuites/validate/h264/parse.trickmode_predicted.seek_trickmode_predicted/flow-expectations/log-parse-src-expected @@ -1,5 +1,5 @@ event stream-start: GstEventStreamStart, flags=(GstStreamFlags)GST_STREAM_FLAG_NONE, group-id=(uint)1; -event caps: video/x-h264, alignment=(string)au, bit-depth-chroma=(uint)8, bit-depth-luma=(uint)8, chroma-format=(string)4:2:0, codec_data=(buffer)01640014ffe1001867640014acd94141fb0110000003001773594000f142996001000568ebecb22c, coded-picture-structure=(string)frame, framerate=(fraction)30/1, height=(int)240, level=(string)2, parsed=(boolean)true, pixel-aspect-ratio=(fraction)1/1, profile=(string)high, stream-format=(string)avc, width=(int)320; +event caps: video/x-h264, alignment=(string)au, bit-depth-chroma=(uint)8, bit-depth-luma=(uint)8, chroma-format=(string)4:2:0, codec_data=(buffer)01640014ffe1001867640014acd94141fb0110000003001773594000f142996001000568ebecb22c, coded-picture-structure=(string)frame, framerate=(fraction)30/1, height=(int)240, lcevc=(boolean)false, level=(string)2, parsed=(boolean)true, pixel-aspect-ratio=(fraction)1/1, profile=(string)high, stream-format=(string)avc, width=(int)320; event segment: format=TIME, start=0:00:00.066666666, offset=0:00:00.000000000, stop=0:00:10.066666666, time=0:00:00.000000000, base=0:00:00.000000000, position=0:00:00.066666666 event tag: GstTagList-stream, taglist=(taglist)"taglist\,\ video-codec\=\(string\)\"H.264\\\ /\\\ AVC\"\,\ bitrate\=\(uint\)99582\,\ language-code\=\(string\)en\,\ container-specific-track-id\=\(string\)1\;"; event tag: GstTagList-global, taglist=(taglist)"taglist\,\ datetime\=\(datetime\)2013-09-03T16:21:39Z\,\ description\=\(string\)\"audiotest\\\ wave\"\,\ encoder\=\(string\)x264\,\ container-format\=\(string\)Quicktime\;"; diff --git a/subprojects/gst-integration-testsuites/testsuites/validate/mp4/qtdemux_reverse_playback_full_gop.reverse_playback_full_gop/flow-expectations/log-parse-src-expected b/subprojects/gst-integration-testsuites/testsuites/validate/mp4/qtdemux_reverse_playback_full_gop.reverse_playback_full_gop/flow-expectations/log-parse-src-expected index 939c3149aa..5b75e561cb 100644 --- a/subprojects/gst-integration-testsuites/testsuites/validate/mp4/qtdemux_reverse_playback_full_gop.reverse_playback_full_gop/flow-expectations/log-parse-src-expected +++ b/subprojects/gst-integration-testsuites/testsuites/validate/mp4/qtdemux_reverse_playback_full_gop.reverse_playback_full_gop/flow-expectations/log-parse-src-expected @@ -1,5 +1,5 @@ event stream-start: GstEventStreamStart, flags=(GstStreamFlags)GST_STREAM_FLAG_NONE, group-id=(uint)1; -event caps: video/x-h264, alignment=(string)au, bit-depth-chroma=(uint)8, bit-depth-luma=(uint)8, chroma-format=(string)4:2:0, codec_data=(buffer)01640014ffe1001867640014acd94141fb0110000003001773594000f142996001000568ebecb22c, coded-picture-structure=(string)frame, framerate=(fraction)30/1, height=(int)240, level=(string)2, parsed=(boolean)true, pixel-aspect-ratio=(fraction)1/1, profile=(string)high, stream-format=(string)avc, width=(int)320; +event caps: video/x-h264, alignment=(string)au, bit-depth-chroma=(uint)8, bit-depth-luma=(uint)8, chroma-format=(string)4:2:0, codec_data=(buffer)01640014ffe1001867640014acd94141fb0110000003001773594000f142996001000568ebecb22c, coded-picture-structure=(string)frame, framerate=(fraction)30/1, height=(int)240, lcevc=(boolean)false, level=(string)2, parsed=(boolean)true, pixel-aspect-ratio=(fraction)1/1, profile=(string)high, stream-format=(string)avc, width=(int)320; event segment: format=TIME, start=0:00:00.066666666, offset=0:00:00.000000000, stop=0:00:10.066666666, time=0:00:00.000000000, base=0:00:00.000000000, position=0:00:00.066666666 event tag: GstTagList-stream, taglist=(taglist)"taglist\,\ video-codec\=\(string\)\"H.264\\\ /\\\ AVC\"\,\ bitrate\=\(uint\)99582\,\ language-code\=\(string\)en\,\ container-specific-track-id\=\(string\)1\;"; event tag: GstTagList-global, taglist=(taglist)"taglist\,\ datetime\=\(datetime\)2013-09-03T16:21:39Z\,\ description\=\(string\)\"audiotest\\\ wave\"\,\ encoder\=\(string\)x264\,\ container-format\=\(string\)Quicktime\;"; diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c index 56543d8acd..675a2440e8 100644 --- a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c +++ b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c @@ -212,6 +212,9 @@ gst_h264_parse_finalize (GObject * object) { GstH264Parse *h264parse = GST_H264_PARSE (object); + gst_video_clear_user_data_unregistered (&h264parse->user_data_unregistered, + TRUE); + gst_video_clear_user_data (&h264parse->user_data, TRUE); gst_video_clear_user_data_unregistered (&h264parse->user_data_unregistered, TRUE); @@ -242,7 +245,7 @@ gst_h264_parse_reset_frame (GstH264Parse * h264parse) h264parse->have_pps_in_frame = FALSE; h264parse->have_aud_in_frame = FALSE; gst_adapter_clear (h264parse->frame_out); - gst_video_clear_user_data (&h264parse->user_data); + gst_video_clear_user_data (&h264parse->user_data, FALSE); gst_video_clear_user_data_unregistered (&h264parse->user_data_unregistered, FALSE); } @@ -588,8 +591,9 @@ gst_h264_parse_process_sei_user_data (GstH264Parse * h264parse, GstByteReader br; GstVideoParseUtilsField field = GST_VIDEO_PARSE_UTILS_FIELD_1; - /* only US country code is currently supported */ + /* only US and UK country codes are currently supported */ switch (rud->country_code) { + case ITU_T_T35_COUNTRY_CODE_UK: case ITU_T_T35_COUNTRY_CODE_US: break; default: @@ -2445,6 +2449,11 @@ gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps) "Couldn't set content light level to caps"); } + if (h264parse->user_data.lcevc_enhancement_data) + gst_caps_set_simple (caps, "lcevc", G_TYPE_BOOLEAN, TRUE, NULL); + else + gst_caps_set_simple (caps, "lcevc", G_TYPE_BOOLEAN, FALSE, NULL); + src_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (h264parse)); if (src_caps) { diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gsth265parse.c b/subprojects/gst-plugins-bad/gst/videoparsers/gsth265parse.c index 491c1c8c24..6fb36e469d 100644 --- a/subprojects/gst-plugins-bad/gst/videoparsers/gsth265parse.c +++ b/subprojects/gst-plugins-bad/gst/videoparsers/gsth265parse.c @@ -209,7 +209,7 @@ gst_h265_parse_reset_frame (GstH265Parse * h265parse) h265parse->have_sps_in_frame = FALSE; h265parse->have_pps_in_frame = FALSE; gst_adapter_clear (h265parse->frame_out); - gst_video_clear_user_data (&h265parse->user_data); + gst_video_clear_user_data (&h265parse->user_data, FALSE); gst_video_clear_user_data_unregistered (&h265parse->user_data_unregistered, FALSE); } diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gstmpegvideoparse.c b/subprojects/gst-plugins-bad/gst/videoparsers/gstmpegvideoparse.c index 5132bf7569..57a301e586 100644 --- a/subprojects/gst-plugins-bad/gst/videoparsers/gstmpegvideoparse.c +++ b/subprojects/gst-plugins-bad/gst/videoparsers/gstmpegvideoparse.c @@ -196,7 +196,7 @@ gst_mpegv_parse_reset_frame (GstMpegvParse * mpvparse) mpvparse->ext_count = 0; mpvparse->slice_count = 0; mpvparse->slice_offset = 0; - gst_video_clear_user_data (&mpvparse->user_data); + gst_video_clear_user_data (&mpvparse->user_data, FALSE); } static void diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gstvideoparseutils.c b/subprojects/gst-plugins-bad/gst/videoparsers/gstvideoparseutils.c index 137da924c7..e9ca41a5bb 100644 --- a/subprojects/gst-plugins-bad/gst/videoparsers/gstvideoparseutils.c +++ b/subprojects/gst-plugins-bad/gst/videoparsers/gstvideoparseutils.c @@ -28,6 +28,7 @@ #include #include #include +#include #include GST_DEBUG_CATEGORY_EXTERN (videoparseutils_debug); @@ -98,6 +99,9 @@ gst_video_parse_user_data (GstElement * elt, GstVideoParseUserData * user_data, case ITU_T_T35_MANUFACTURER_US_DIRECTV: user_data_id = USER_DATA_ID_DIRECTV_CC; break; + case ITU_T_T35_MANUFACTURER_UK_LCEVC: + user_data_id = USER_DATA_ID_LCEVC_ENHANCEMENT; + break; default: GST_LOG_OBJECT (elt, "Unsupported provider code %d", provider_code); return; @@ -233,6 +237,22 @@ gst_video_parse_user_data (GstElement * elt, GstVideoParseUserData * user_data, break; } break; + case USER_DATA_ID_LCEVC_ENHANCEMENT: + if (!gst_byte_reader_get_uint8 (br, &user_data_type_code)) { + GST_WARNING_OBJECT (elt, "Missing user data type code, ignoring"); + break; + } + bar_size = gst_byte_reader_get_remaining (br); + if (bar_size == 0) { + GST_WARNING_OBJECT (elt, "Bar data packet too short, ignoring"); + break; + } + if (!gst_byte_reader_get_data (br, bar_size, &data)) + break; + g_clear_pointer (&user_data->lcevc_enhancement_data, gst_buffer_unref); + user_data->lcevc_enhancement_data = + gst_buffer_new_memdup (data, bar_size); + break; default: GST_DEBUG_OBJECT (elt, "Unrecognized user data id %d of size %d", user_data_id, @@ -288,6 +308,18 @@ gst_video_push_user_data (GstElement * elt, GstVideoParseUserData * user_data, gst_buffer_add_video_bar_meta (buf, bar.field, bar.is_letterbox, bar.bar_data[0], bar.bar_data[1]); } + + /* 4. handle LCEVC */ + if (user_data->lcevc_enhancement_data) { + if (!gst_buffer_get_meta (buf, GST_LCEVC_META_API_TYPE)) { + gst_buffer_add_lcevc_meta (buf, user_data->lcevc_enhancement_data); + } else { + GST_DEBUG_OBJECT (elt, "LCEVC data already found on buffer, " + "discarding to avoid duplication"); + } + + g_clear_pointer (&user_data->lcevc_enhancement_data, gst_buffer_unref); + } } /* @@ -447,11 +479,13 @@ gst_video_parse_utils_parse_afd (const guint8 data, GstVideoAFDSpec spec, * Clears the user data, resetting it for the next frame */ void -gst_video_clear_user_data (GstVideoParseUserData * user_data) +gst_video_clear_user_data (GstVideoParseUserData * user_data, gboolean free) { user_data->closedcaptions_size = 0; user_data->bar_data_size = 0; user_data->active_format_flag = 0; + if (free) + g_clear_pointer (&user_data->lcevc_enhancement_data, gst_buffer_unref); } /* diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gstvideoparseutils.h b/subprojects/gst-plugins-bad/gst/videoparsers/gstvideoparseutils.h index 9d86a4f6b9..cb34d4f21c 100644 --- a/subprojects/gst-plugins-bad/gst/videoparsers/gstvideoparseutils.h +++ b/subprojects/gst-plugins-bad/gst/videoparsers/gstvideoparseutils.h @@ -32,6 +32,8 @@ #define A53_USER_DATA_ID_GA94 0x47413934 #define A53_USER_DATA_ID_DTG1 0x44544731 +/* custom id for LCEVC */ +#define USER_DATA_ID_LCEVC_ENHANCEMENT 0xFFFFFFFD /* custom id for SCTE 20 608 */ #define USER_DATA_ID_SCTE_20_CC 0xFFFFFFFE /* custom id for DirecTV */ @@ -49,11 +51,13 @@ #define CEA_708_PROCESS_EM_DATA_FLAG 0x80 /* country codes */ +#define ITU_T_T35_COUNTRY_CODE_UK 0xB4 #define ITU_T_T35_COUNTRY_CODE_US 0xB5 /* provider codes */ #define ITU_T_T35_MANUFACTURER_US_ATSC 0x31 #define ITU_T_T35_MANUFACTURER_US_DIRECTV 0x2F +#define ITU_T_T35_MANUFACTURER_UK_LCEVC 0x50 /* * GstVideoAFDAspectRatio: @@ -144,7 +148,7 @@ typedef struct { /* * GstVideoParseUserData * - * Holds unparsed and parsed user data for closed captions, AFD and Bar data. + * Holds unparsed and parsed user data for closed captions, LCEVC, AFD and Bar data. */ typedef struct { @@ -164,6 +168,9 @@ typedef struct gboolean active_format_flag; GstVideoAFDSpec afd_spec; + /* pending LCEVC data */ + GstBuffer *lcevc_enhancement_data; + } GstVideoParseUserData; /* @@ -200,7 +207,8 @@ void gst_video_push_user_data (GstElement * elt, GstVideoParseUserData * user_data, GstBuffer * buf); -void gst_video_clear_user_data (GstVideoParseUserData * user_data); +void gst_video_clear_user_data (GstVideoParseUserData * user_data, + gboolean free); void gst_video_parse_user_data_unregistered (GstElement * elt,