From 44390d9d1dae9a98403bd8c85d5a294fe7a3b901 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Fri, 9 Mar 2018 12:23:04 +0100 Subject: [PATCH] decklinkvideosrc: Add support for extracting Closed Caption If the "output-cc" property is set to TRUE and there is CC present in the VBI Ancillary Data, they will be extracted and set on the outgoing buffer as GstVideoCaptionMeta. Only CDP packets are supported. https://bugzilla.gnome.org/show_bug.cgi?id=773863 --- sys/decklink/gstdecklink.cpp | 22 +++++- sys/decklink/gstdecklink.h | 2 + sys/decklink/gstdecklinkvideosrc.cpp | 109 ++++++++++++++++++++++++++- sys/decklink/gstdecklinkvideosrc.h | 5 ++ 4 files changed, 131 insertions(+), 7 deletions(-) diff --git a/sys/decklink/gstdecklink.cpp b/sys/decklink/gstdecklink.cpp index fb7276a8cd..63d7887012 100644 --- a/sys/decklink/gstdecklink.cpp +++ b/sys/decklink/gstdecklink.cpp @@ -232,10 +232,10 @@ gst_decklink_audio_channels_get_type (void) return (GType) id; } -#define NTSC 10, 11, false, "bt601" -#define PAL 12, 11, true, "bt601" -#define HD 1, 1, true, "bt709" -#define UHD 1, 1, true, "bt2020" +#define NTSC 10, 11, false, "bt601", FALSE +#define PAL 12, 11, true, "bt601", FALSE +#define HD 1, 1, true, "bt709", TRUE +#define UHD 1, 1, true, "bt2020", TRUE static const GstDecklinkMode modes[] = { {bmdModeNTSC, 720, 486, 30000, 1001, true, NTSC}, // default is ntsc @@ -462,6 +462,20 @@ gst_decklink_type_from_video_format (GstVideoFormat f) return GST_DECKLINK_VIDEO_FORMAT_AUTO; } +GstVideoFormat +gst_decklink_video_format_from_type (BMDPixelFormat pf) +{ + guint i; + + for (i = 1; i < G_N_ELEMENTS (formats); i++) { + if (formats[i].format == pf) + return formats[i].vformat; + } + GST_WARNING ("Unknown pixel format 0x%x", pf); + return GST_VIDEO_FORMAT_UNKNOWN; +} + + const BMDTimecodeFormat gst_decklink_timecode_format_from_enum (GstDecklinkTimecodeFormat f) { diff --git a/sys/decklink/gstdecklink.h b/sys/decklink/gstdecklink.h index 0b65bb25a4..3c55608ccb 100644 --- a/sys/decklink/gstdecklink.h +++ b/sys/decklink/gstdecklink.h @@ -178,6 +178,7 @@ enum _BMDKeyerMode const BMDPixelFormat gst_decklink_pixel_format_from_type (GstDecklinkVideoFormat t); const gint gst_decklink_bpp_from_type (GstDecklinkVideoFormat t); const GstDecklinkVideoFormat gst_decklink_type_from_video_format (GstVideoFormat f); +GstVideoFormat gst_decklink_video_format_from_type (BMDPixelFormat pf); const BMDTimecodeFormat gst_decklink_timecode_format_from_enum (GstDecklinkTimecodeFormat f); const GstDecklinkTimecodeFormat gst_decklink_timecode_format_to_enum (BMDTimecodeFormat f); const BMDKeyerMode gst_decklink_keyer_mode_from_enum (GstDecklinkKeyerMode m); @@ -195,6 +196,7 @@ struct _GstDecklinkMode { int par_d; gboolean tff; const gchar *colorimetry; + gboolean vanc; }; const GstDecklinkMode * gst_decklink_get_mode (GstDecklinkModeEnum e); diff --git a/sys/decklink/gstdecklinkvideosrc.cpp b/sys/decklink/gstdecklinkvideosrc.cpp index 5e97ac1761..b57e8380f9 100644 --- a/sys/decklink/gstdecklinkvideosrc.cpp +++ b/sys/decklink/gstdecklinkvideosrc.cpp @@ -35,6 +35,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_decklink_video_src_debug); #define DEFAULT_OUTPUT_STREAM_TIME (FALSE) #define DEFAULT_SKIP_FIRST_TIME (0) #define DEFAULT_DROP_NO_SIGNAL_FRAMES (FALSE) +#define DEFAULT_OUTPUT_CC (FALSE) #ifndef ABSDIFF #define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) ) @@ -53,7 +54,8 @@ enum PROP_SKIP_FIRST_TIME, PROP_DROP_NO_SIGNAL_FRAMES, PROP_SIGNAL, - PROP_HW_SERIAL_NUMBER + PROP_HW_SERIAL_NUMBER, + PROP_OUTPUT_CC }; typedef struct @@ -221,6 +223,12 @@ gst_decklink_video_src_class_init (GstDecklinkVideoSrcClass * klass) "The serial number (hardware ID) of the Decklink card", NULL, (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property (gobject_class, PROP_OUTPUT_CC, + g_param_spec_boolean ("output-cc", "Output Closed Caption", + "Extract and output CC as GstMeta (if present)", + DEFAULT_OUTPUT_CC, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + templ_caps = gst_decklink_mode_get_template_caps (TRUE); gst_element_class_add_pad_template (element_class, gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, templ_caps)); @@ -325,6 +333,9 @@ gst_decklink_video_src_set_property (GObject * object, guint property_id, case PROP_DROP_NO_SIGNAL_FRAMES: self->drop_no_signal_frames = g_value_get_boolean (value); break; + case PROP_OUTPUT_CC: + self->output_cc = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -375,6 +386,9 @@ gst_decklink_video_src_get_property (GObject * object, guint property_id, else g_value_set_string (value, NULL); break; + case PROP_OUTPUT_CC: + g_value_set_boolean (value, self->output_cc); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -764,6 +778,76 @@ gst_decklink_video_src_got_frame (GstElement * element, g_mutex_unlock (&self->lock); } +static void +extract_cc_from_vbi (GstDecklinkVideoSrc * self, GstBuffer ** buffer, + VideoFrame * vf, const GstDecklinkMode * mode) +{ + IDeckLinkVideoFrameAncillary *vanc_frame = NULL; + gint fi; + guint8 *vancdata; + GstVideoFormat videoformat; + gboolean found = FALSE; + + if (vf->frame->GetAncillaryData (&vanc_frame) != S_OK) + return; + + videoformat = + gst_decklink_video_format_from_type (vanc_frame->GetPixelFormat ()); + + if (videoformat == GST_VIDEO_FORMAT_UNKNOWN) { + GST_DEBUG_OBJECT (self, "Unknown video format for Ancillary data"); + vanc_frame->Release (); + return; + } + + if (videoformat != self->anc_vformat) { + gst_video_vbi_parser_free (self->vbiparser); + self->vbiparser = NULL; + } + + GST_DEBUG_OBJECT (self, "Checking for ancillary data in VBI"); + + fi = self->last_cc_vbi_line; + if (fi == -1) + fi = 1; + + while (fi < 22 && !found) { + if (vanc_frame->GetBufferForVerticalBlankingLine (fi, + (void **) &vancdata) == S_OK) { + GstVideoAncillary gstanc; + if (self->vbiparser == NULL) { + self->vbiparser = gst_video_vbi_parser_new (videoformat, mode->width); + self->anc_vformat = videoformat; + } + GST_DEBUG_OBJECT (self, "Might have data on line %d", fi); + gst_video_vbi_parser_add_line (self->vbiparser, vancdata); + + while (gst_video_vbi_parser_get_ancillary (self->vbiparser, + &gstanc) == GST_VIDEO_VBI_PARSER_RESULT_OK) { + if (GST_VIDEO_ANCILLARY_DID16 (&gstanc) == + GST_VIDEO_ANCILLARY_DID16_S334_EIA_708) { + GST_DEBUG_OBJECT (self, + "Adding CEA-708 CDP meta to buffer for line %d", fi); + GST_MEMDUMP_OBJECT (self, "CDP", gstanc.data, gstanc.data_count); + gst_buffer_add_video_caption_meta (*buffer, + GST_VIDEO_CAPTION_TYPE_CEA708_CDP, gstanc.data, + gstanc.data_count); + found = TRUE; + self->last_cc_vbi_line = fi; + break; + } + } + } + + fi++; + } + + if (!found) + self->last_cc_vbi_line = -1; + + vanc_frame->Release (); +} + static GstFlowReturn gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer) { @@ -854,13 +938,18 @@ gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer) g_mutex_unlock (&self->lock); if (caps_changed) { + self->last_cc_vbi_line = -1; caps = gst_decklink_mode_get_caps (f.mode, f.format, TRUE); gst_video_info_from_caps (&self->info, caps); gst_base_src_set_caps (GST_BASE_SRC_CAST (bsrc), caps); gst_element_post_message (GST_ELEMENT_CAST (self), gst_message_new_latency (GST_OBJECT_CAST (self))); gst_caps_unref (caps); - + if (self->vbiparser) { + gst_video_vbi_parser_free (self->vbiparser); + self->vbiparser = NULL; + self->anc_vformat = GST_VIDEO_FORMAT_UNKNOWN; + } } f.frame->GetBytes ((gpointer *) & data); @@ -894,6 +983,13 @@ gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer) } } + mode = gst_decklink_get_mode (self->mode); + + // If we have a format that supports VANC and we are asked to extract CC, + // then do it here. + if (self->output_cc && mode->vanc) + extract_cc_from_vbi (self, buffer, vf, mode); + if (f.no_signal) GST_BUFFER_FLAG_SET (*buffer, GST_BUFFER_FLAG_GAP); GST_BUFFER_TIMESTAMP (*buffer) = f.timestamp; @@ -907,7 +1003,6 @@ gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer) gst_static_caps_get (&hardware_reference), f.hardware_timestamp, f.hardware_duration); - mode = gst_decklink_get_mode (self->mode); if (mode->interlaced && mode->tff) GST_BUFFER_FLAG_SET (*buffer, GST_VIDEO_BUFFER_FLAG_TFF | GST_VIDEO_BUFFER_FLAG_INTERLACED); @@ -1057,6 +1152,12 @@ gst_decklink_video_src_stop (GstDecklinkVideoSrc * self) self->input->input->DisableVideoInput (); } + if (self->vbiparser) { + gst_video_vbi_parser_free (self->vbiparser); + self->vbiparser = NULL; + self->anc_vformat = GST_VIDEO_FORMAT_UNKNOWN; + } + return TRUE; } @@ -1120,6 +1221,8 @@ gst_decklink_video_src_change_state (GstElement * element, GST_WARNING_OBJECT (self, "Warning: mode=auto and format!=auto may \ not work"); } + self->vbiparser = NULL; + self->anc_vformat = GST_VIDEO_FORMAT_UNKNOWN; break; case GST_STATE_CHANGE_READY_TO_PAUSED: self->flushing = FALSE; diff --git a/sys/decklink/gstdecklinkvideosrc.h b/sys/decklink/gstdecklinkvideosrc.h index 49c5ba8cf0..367b05b513 100644 --- a/sys/decklink/gstdecklinkvideosrc.h +++ b/sys/decklink/gstdecklinkvideosrc.h @@ -94,6 +94,11 @@ struct _GstDecklinkVideoSrc GstClockTime num, den; } next_time_mapping; gboolean next_time_mapping_pending; + + GstVideoVBIParser *vbiparser; + GstVideoFormat anc_vformat; + gboolean output_cc; + guint last_cc_vbi_line; }; struct _GstDecklinkVideoSrcClass