msdkh264enc: Add support for CEA708 closed caption insertion

Currently supported caption format is CEA708_RAW

sys/msdk/gstmsdkh264enc.c

@@ -37,6 +37,7 @@
 
 #include <gst/base/base.h>
 #include <gst/pbutils/pbutils.h>
+#include <string.h>
 
 GST_DEBUG_CATEGORY_EXTERN (gst_msdkh264enc_debug);
 #define GST_CAT_DEFAULT gst_msdkh264enc_debug
@@ -185,11 +186,91 @@ done:
   gst_buffer_unmap (frame->output_buffer, &map);
 }
 
+static void
+gst_msdkh264enc_add_cc (GstMsdkH264Enc * thiz, GstVideoCodecFrame * frame)
+{
+  GstVideoCaptionMeta *cc_meta;
+  gpointer iter = NULL;
+  GstBuffer *in_buf = frame->input_buffer;
+  GstMemory *mem = NULL;
+  GstBuffer *new_buffer = NULL;
+
+  g_array_set_size (thiz->extra_sei, 0);
+
+  while ((cc_meta =
+          (GstVideoCaptionMeta *) gst_buffer_iterate_meta_filtered (in_buf,
+              &iter, GST_VIDEO_CAPTION_META_API_TYPE))) {
+    GstH264SEIMessage sei;
+    GstH264RegisteredUserData *rud;
+    guint8 *data;
+
+    if (cc_meta->caption_type != GST_VIDEO_CAPTION_TYPE_CEA708_RAW)
+      continue;
+
+    memset (&sei, 0, sizeof (GstH264SEIMessage));
+    sei.payloadType = GST_H264_SEI_REGISTERED_USER_DATA;
+    rud = &sei.payload.registered_user_data;
+
+    rud->country_code = 181;
+    rud->size = cc_meta->size + 10;
+
+    data = g_malloc (rud->size);
+    memcpy (data + 9, cc_meta->data, cc_meta->size);
+
+    data[0] = 0;                /* 16-bits itu_t_t35_provider_code */
+    data[1] = 49;
+    data[2] = 'G';              /* 32-bits ATSC_user_identifier */
+    data[3] = 'A';
+    data[4] = '9';
+    data[5] = '4';
+    data[6] = 3;                /* 8-bits ATSC1_data_user_data_type_code */
+    /* 8-bits:
+     * 1 bit process_em_data_flag (0)
+     * 1 bit process_cc_data_flag (1)
+     * 1 bit additional_data_flag (0)
+     * 5-bits cc_count
+     */
+    data[7] = ((cc_meta->size / 3) & 0x1f) | 0x40;
+    data[8] = 255;              /* 8 bits em_data, unused */
+    data[cc_meta->size + 9] = 255;      /* 8 marker bits */
+
+    rud->data = data;
+
+    g_array_append_val (thiz->extra_sei, sei);
+  }
+
+  if (!thiz->extra_sei->len)
+    return;
+
+  mem = gst_h264_create_sei_memory (4, thiz->extra_sei);
+
+  if (!mem) {
+    GST_WARNING_OBJECT (thiz, "Cannot create SEI nal unit");
+    return;
+  }
+
+  GST_DEBUG_OBJECT (thiz,
+      "Inserting %d closed caption SEI message(s)", thiz->extra_sei->len);
+
+  new_buffer = gst_h264_parser_insert_sei (thiz->parser,
+      frame->output_buffer, mem);
+  gst_memory_unref (mem);
+
+  if (!new_buffer) {
+    GST_WARNING_OBJECT (thiz, "Cannot insert SEI nal into AU buffer");
+    return;
+  }
+
+  gst_buffer_unref (frame->output_buffer);
+  frame->output_buffer = new_buffer;
+}
+
 static GstFlowReturn
 gst_msdkh264enc_pre_push (GstVideoEncoder * encoder, GstVideoCodecFrame * frame)
 {
   GstMsdkH264Enc *thiz = GST_MSDKH264ENC (encoder);
 
+  /* FIXME: port to generic sei insertion logic */
   if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) &&
       (thiz->frame_packing != GST_VIDEO_MULTIVIEW_MODE_NONE ||
           ((GST_VIDEO_INFO_MULTIVIEW_MODE (&thiz->base.input_state->info) !=
@@ -202,6 +283,8 @@ gst_msdkh264enc_pre_push (GstVideoEncoder * encoder, GstVideoCodecFrame * frame)
         GST_VIDEO_INFO_MULTIVIEW_MODE (&thiz->base.input_state->info));
   }
 
+  gst_msdkh264enc_add_cc (thiz, frame);
+
   return GST_FLOW_OK;
 }
 
@@ -400,6 +483,17 @@ gst_msdkh264enc_set_src_caps (GstMsdkEnc * encoder)
   return caps;
 }
 
+static void
+gst_msdkh264enc_finalize (GObject * object)
+{
+  GstMsdkH264Enc *thiz = GST_MSDKH264ENC (object);
+
+  gst_h264_nal_parser_free (thiz->parser);
+  g_array_unref (thiz->extra_sei);
+
+  G_OBJECT_CLASS (parent_class)->finalize (object);
+}
+
 static void
 gst_msdkh264enc_set_property (GObject * object, guint prop_id,
     const GValue * value, GParamSpec * pspec)
@@ -511,6 +605,7 @@ gst_msdkh264enc_class_init (GstMsdkH264EncClass * klass)
   videoencoder_class = GST_VIDEO_ENCODER_CLASS (klass);
   encoder_class = GST_MSDKENC_CLASS (klass);
 
+  gobject_class->finalize = gst_msdkh264enc_finalize;
   gobject_class->set_property = gst_msdkh264enc_set_property;
   gobject_class->get_property = gst_msdkh264enc_get_property;
 
@@ -579,4 +674,8 @@ gst_msdkh264enc_init (GstMsdkH264Enc * thiz)
   thiz->trellis = PROP_TRELLIS_DEFAULT;
   thiz->max_slice_size = PROP_MAX_SLICE_SIZE_DEFAULT;
   thiz->b_pyramid = PROP_B_PYRAMID_DEFAULT;
+
+  thiz->parser = gst_h264_nal_parser_new ();
+  thiz->extra_sei = g_array_new (FALSE, FALSE, sizeof (GstH264SEIMessage));
+  g_array_set_clear_func (thiz->extra_sei, (GDestroyNotify) gst_h264_sei_clear);
 }

sys/msdk/gstmsdkh264enc.h

@@ -33,6 +33,7 @@
 #define __GST_MSDKH264ENC_H__
 
 #include "gstmsdkenc.h"
+#include <gst/codecparsers/gsth264parser.h>
 
 G_BEGIN_DECLS
 
@@ -68,6 +69,9 @@ struct _GstMsdkH264Enc
   guint trellis;
   guint max_slice_size;
   guint b_pyramid;
+
+  GstH264NalParser *parser;
+  GArray *extra_sei;
 };
 
 struct _GstMsdkH264EncClass

sys/msdk/meson.build

@@ -116,9 +116,9 @@ endif
 if msdk_deps_found
   gstmsdktag = library('gstmsdk',
     msdk_sources,
-    c_args : gst_plugins_bad_args,
+    c_args : gst_plugins_bad_args + ['-DGST_USE_UNSTABLE_API'],
     include_directories : [configinc, mfx_inc],
-    dependencies : [gstbase_dep, gstvideo_dep, gstpbutils_dep, gstallocators_dep, mfx_dep, msdk_deps],
+    dependencies : [gstbase_dep, gstvideo_dep, gstpbutils_dep, gstallocators_dep, gstcodecparsers_dep, mfx_dep, msdk_deps],
     install : true,
     install_dir : plugins_install_dir,
   )