decklinkvideosrc: Add support for extracting Closed Caption

If the "output-cc" property is set to TRUE and there is CC present
in the VBI Ancillary Data, they will be extracted and set on the
outgoing buffer as GstVideoCaptionMeta.

Only CDP packets are supported.

https://bugzilla.gnome.org/show_bug.cgi?id=773863

sys/decklink/gstdecklink.cpp

@@ -232,10 +232,10 @@ gst_decklink_audio_channels_get_type (void)
   return (GType) id;
 }
 
-#define NTSC 10, 11, false, "bt601"
-#define PAL 12, 11, true, "bt601"
-#define HD 1, 1, true, "bt709"
-#define UHD 1, 1, true, "bt2020"
+#define NTSC 10, 11, false, "bt601", FALSE
+#define PAL 12, 11, true, "bt601", FALSE
+#define HD 1, 1, true, "bt709", TRUE
+#define UHD 1, 1, true, "bt2020", TRUE
 
 static const GstDecklinkMode modes[] = {
   {bmdModeNTSC, 720, 486, 30000, 1001, true, NTSC},     // default is ntsc
@@ -462,6 +462,20 @@ gst_decklink_type_from_video_format (GstVideoFormat f)
   return GST_DECKLINK_VIDEO_FORMAT_AUTO;
 }
 
+GstVideoFormat
+gst_decklink_video_format_from_type (BMDPixelFormat pf)
+{
+  guint i;
+
+  for (i = 1; i < G_N_ELEMENTS (formats); i++) {
+    if (formats[i].format == pf)
+      return formats[i].vformat;
+  }
+  GST_WARNING ("Unknown pixel format 0x%x", pf);
+  return GST_VIDEO_FORMAT_UNKNOWN;
+}
+
+
 const BMDTimecodeFormat
 gst_decklink_timecode_format_from_enum (GstDecklinkTimecodeFormat f)
 {

sys/decklink/gstdecklink.h

@@ -178,6 +178,7 @@ enum _BMDKeyerMode
 const BMDPixelFormat gst_decklink_pixel_format_from_type (GstDecklinkVideoFormat t);
 const gint gst_decklink_bpp_from_type (GstDecklinkVideoFormat t);
 const GstDecklinkVideoFormat gst_decklink_type_from_video_format (GstVideoFormat f);
+GstVideoFormat gst_decklink_video_format_from_type (BMDPixelFormat pf);
 const BMDTimecodeFormat gst_decklink_timecode_format_from_enum (GstDecklinkTimecodeFormat f);
 const GstDecklinkTimecodeFormat gst_decklink_timecode_format_to_enum (BMDTimecodeFormat f);
 const BMDKeyerMode gst_decklink_keyer_mode_from_enum (GstDecklinkKeyerMode m);
@@ -195,6 +196,7 @@ struct _GstDecklinkMode {
   int par_d;
   gboolean tff;
   const gchar *colorimetry;
+  gboolean vanc;
 };
 
 const GstDecklinkMode * gst_decklink_get_mode (GstDecklinkModeEnum e);

sys/decklink/gstdecklinkvideosrc.cpp

@@ -35,6 +35,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_decklink_video_src_debug);
 #define DEFAULT_OUTPUT_STREAM_TIME (FALSE)
 #define DEFAULT_SKIP_FIRST_TIME (0)
 #define DEFAULT_DROP_NO_SIGNAL_FRAMES (FALSE)
+#define DEFAULT_OUTPUT_CC (FALSE)
 
 #ifndef ABSDIFF
 #define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) )
@@ -53,7 +54,8 @@ enum
   PROP_SKIP_FIRST_TIME,
   PROP_DROP_NO_SIGNAL_FRAMES,
   PROP_SIGNAL,
-  PROP_HW_SERIAL_NUMBER
+  PROP_HW_SERIAL_NUMBER,
+  PROP_OUTPUT_CC
 };
 
 typedef struct
@@ -221,6 +223,12 @@ gst_decklink_video_src_class_init (GstDecklinkVideoSrcClass * klass)
           "The serial number (hardware ID) of the Decklink card",
           NULL, (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
 
+  g_object_class_install_property (gobject_class, PROP_OUTPUT_CC,
+      g_param_spec_boolean ("output-cc", "Output Closed Caption",
+          "Extract and output CC as GstMeta (if present)",
+          DEFAULT_OUTPUT_CC,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
   templ_caps = gst_decklink_mode_get_template_caps (TRUE);
   gst_element_class_add_pad_template (element_class,
       gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, templ_caps));
@@ -325,6 +333,9 @@ gst_decklink_video_src_set_property (GObject * object, guint property_id,
     case PROP_DROP_NO_SIGNAL_FRAMES:
       self->drop_no_signal_frames = g_value_get_boolean (value);
       break;
+    case PROP_OUTPUT_CC:
+      self->output_cc = g_value_get_boolean (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
       break;
@@ -375,6 +386,9 @@ gst_decklink_video_src_get_property (GObject * object, guint property_id,
       else
         g_value_set_string (value, NULL);
       break;
+    case PROP_OUTPUT_CC:
+      g_value_set_boolean (value, self->output_cc);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
       break;
@@ -764,6 +778,76 @@ gst_decklink_video_src_got_frame (GstElement * element,
   g_mutex_unlock (&self->lock);
 }
 
+static void
+extract_cc_from_vbi (GstDecklinkVideoSrc * self, GstBuffer ** buffer,
+    VideoFrame * vf, const GstDecklinkMode * mode)
+{
+  IDeckLinkVideoFrameAncillary *vanc_frame = NULL;
+  gint fi;
+  guint8 *vancdata;
+  GstVideoFormat videoformat;
+  gboolean found = FALSE;
+
+  if (vf->frame->GetAncillaryData (&vanc_frame) != S_OK)
+    return;
+
+  videoformat =
+      gst_decklink_video_format_from_type (vanc_frame->GetPixelFormat ());
+
+  if (videoformat == GST_VIDEO_FORMAT_UNKNOWN) {
+    GST_DEBUG_OBJECT (self, "Unknown video format for Ancillary data");
+    vanc_frame->Release ();
+    return;
+  }
+
+  if (videoformat != self->anc_vformat) {
+    gst_video_vbi_parser_free (self->vbiparser);
+    self->vbiparser = NULL;
+  }
+
+  GST_DEBUG_OBJECT (self, "Checking for ancillary data in VBI");
+
+  fi = self->last_cc_vbi_line;
+  if (fi == -1)
+    fi = 1;
+
+  while (fi < 22 && !found) {
+    if (vanc_frame->GetBufferForVerticalBlankingLine (fi,
+            (void **) &vancdata) == S_OK) {
+      GstVideoAncillary gstanc;
+      if (self->vbiparser == NULL) {
+        self->vbiparser = gst_video_vbi_parser_new (videoformat, mode->width);
+        self->anc_vformat = videoformat;
+      }
+      GST_DEBUG_OBJECT (self, "Might have data on line %d", fi);
+      gst_video_vbi_parser_add_line (self->vbiparser, vancdata);
+
+      while (gst_video_vbi_parser_get_ancillary (self->vbiparser,
+              &gstanc) == GST_VIDEO_VBI_PARSER_RESULT_OK) {
+        if (GST_VIDEO_ANCILLARY_DID16 (&gstanc) ==
+            GST_VIDEO_ANCILLARY_DID16_S334_EIA_708) {
+          GST_DEBUG_OBJECT (self,
+              "Adding CEA-708 CDP meta to buffer for line %d", fi);
+          GST_MEMDUMP_OBJECT (self, "CDP", gstanc.data, gstanc.data_count);
+          gst_buffer_add_video_caption_meta (*buffer,
+              GST_VIDEO_CAPTION_TYPE_CEA708_CDP, gstanc.data,
+              gstanc.data_count);
+          found = TRUE;
+          self->last_cc_vbi_line = fi;
+          break;
+        }
+      }
+    }
+
+    fi++;
+  }
+
+  if (!found)
+    self->last_cc_vbi_line = -1;
+
+  vanc_frame->Release ();
+}
+
 static GstFlowReturn
 gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer)
 {
@@ -854,13 +938,18 @@ gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer)
 
   g_mutex_unlock (&self->lock);
   if (caps_changed) {
+    self->last_cc_vbi_line = -1;
     caps = gst_decklink_mode_get_caps (f.mode, f.format, TRUE);
     gst_video_info_from_caps (&self->info, caps);
     gst_base_src_set_caps (GST_BASE_SRC_CAST (bsrc), caps);
     gst_element_post_message (GST_ELEMENT_CAST (self),
         gst_message_new_latency (GST_OBJECT_CAST (self)));
     gst_caps_unref (caps);
-
+    if (self->vbiparser) {
+      gst_video_vbi_parser_free (self->vbiparser);
+      self->vbiparser = NULL;
+      self->anc_vformat = GST_VIDEO_FORMAT_UNKNOWN;
+    }
   }
 
   f.frame->GetBytes ((gpointer *) & data);
@@ -894,6 +983,13 @@ gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer)
     }
   }
 
+  mode = gst_decklink_get_mode (self->mode);
+
+  // If we have a format that supports VANC and we are asked to extract CC,
+  // then do it here.
+  if (self->output_cc && mode->vanc)
+    extract_cc_from_vbi (self, buffer, vf, mode);
+
   if (f.no_signal)
     GST_BUFFER_FLAG_SET (*buffer, GST_BUFFER_FLAG_GAP);
   GST_BUFFER_TIMESTAMP (*buffer) = f.timestamp;
@@ -907,7 +1003,6 @@ gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer)
       gst_static_caps_get (&hardware_reference), f.hardware_timestamp,
       f.hardware_duration);
 
-  mode = gst_decklink_get_mode (self->mode);
   if (mode->interlaced && mode->tff)
     GST_BUFFER_FLAG_SET (*buffer,
         GST_VIDEO_BUFFER_FLAG_TFF | GST_VIDEO_BUFFER_FLAG_INTERLACED);
@@ -1057,6 +1152,12 @@ gst_decklink_video_src_stop (GstDecklinkVideoSrc * self)
     self->input->input->DisableVideoInput ();
   }
 
+  if (self->vbiparser) {
+    gst_video_vbi_parser_free (self->vbiparser);
+    self->vbiparser = NULL;
+    self->anc_vformat = GST_VIDEO_FORMAT_UNKNOWN;
+  }
+
   return TRUE;
 }
 
@@ -1120,6 +1221,8 @@ gst_decklink_video_src_change_state (GstElement * element,
         GST_WARNING_OBJECT (self, "Warning: mode=auto and format!=auto may \
                             not work");
       }
+      self->vbiparser = NULL;
+      self->anc_vformat = GST_VIDEO_FORMAT_UNKNOWN;
       break;
     case GST_STATE_CHANGE_READY_TO_PAUSED:
       self->flushing = FALSE;

sys/decklink/gstdecklinkvideosrc.h

@@ -94,6 +94,11 @@ struct _GstDecklinkVideoSrc
     GstClockTime num, den;
   } next_time_mapping;
   gboolean next_time_mapping_pending;
+
+  GstVideoVBIParser *vbiparser;
+  GstVideoFormat anc_vformat;
+  gboolean output_cc;
+  guint last_cc_vbi_line;
 };
 
 struct _GstDecklinkVideoSrcClass