diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp index 004d8c970e..121643654f 100644 --- a/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp @@ -2198,6 +2198,7 @@ gst_dwrite_base_overlay_transform (GstBaseTransform * trans, GstBuffer * inbuf, { GstDWriteBaseOverlay *self = GST_DWRITE_BASE_OVERLAY (trans); GstDWriteBaseOverlayPrivate *priv = self->priv; + GstDWriteBaseOverlayClass *klass = GST_DWRITE_BASE_OVERLAY_GET_CLASS (self); gboolean ret = FALSE; std::lock_guard < std::mutex > lk (priv->prop_lock); @@ -2241,5 +2242,8 @@ gst_dwrite_base_overlay_transform (GstBaseTransform * trans, GstBuffer * inbuf, if (!ret) return GST_FLOW_ERROR; + if (klass->after_transform) + klass->after_transform (self, outbuf); + return GST_FLOW_OK; } diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.h b/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.h index eaaddc8d3f..3549efc2b9 100644 --- a/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.h +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.h @@ -60,6 +60,9 @@ struct _GstDWriteBaseOverlayClass WString (*get_text) (GstDWriteBaseOverlay * overlay, const std::wstring & default_text, GstBuffer * buffer); + + void (*after_transform) (GstDWriteBaseOverlay * overlay, + GstBuffer * buffer); }; GType gst_dwrite_base_overlay_get_type (void); diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwritetextoverlay.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwritetextoverlay.cpp index 482409d0f1..3bb43fe5e3 100644 --- a/subprojects/gst-plugins-bad/sys/dwrite/gstdwritetextoverlay.cpp +++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwritetextoverlay.cpp @@ -1,4 +1,5 @@ /* GStreamer + * Copyright (C) 2020 Mathieu Duponchelle * Copyright (C) 2023 Seungha Yang * * This library is free software; you can redistribute it and/or @@ -22,17 +23,66 @@ #endif #include "gstdwritetextoverlay.h" +#include +#include +#include GST_DEBUG_CATEGORY_STATIC (dwrite_text_overlay_debug); #define GST_CAT_DEFAULT dwrite_text_overlay_debug +enum +{ + PROP_0, + PROP_ENABLE_CC, + PROP_CC_FIELD, + PROP_CC_TIMEOUT, + PROP_REMOVE_CC_META, +}; + +#define DEFAULT_ENABLE_CC TRUE +#define DEFAULT_CC_FIELD -1 +#define DEFAULT_CC_TIMEOUT GST_CLOCK_TIME_NONE +#define DEFAULT_REMOVE_CC_META FALSE + +/* *INDENT-OFF* */ +struct GstDWriteTextOverlayPrivate +{ + std::mutex lock; + caption_frame_t frame; + GstClockTime caption_running_time; + GstClockTime running_time; + guint8 selected_field; + + std::string closed_caption; + + /* properties */ + gboolean enable_cc = DEFAULT_ENABLE_CC; + gint field = DEFAULT_CC_FIELD; + GstClockTime timeout = DEFAULT_CC_TIMEOUT; + gboolean remove_caption_meta = DEFAULT_REMOVE_CC_META; +}; +/* *INDENT-ON* */ + struct _GstDWriteTextOverlay { GstDWriteBaseOverlay parent; + + GstDWriteTextOverlayPrivate *priv; }; +static void gst_dwrite_text_overlay_finalize (GObject * object); +static void gst_dwrite_text_overlay_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_dwrite_text_overlay_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static gboolean gst_dwrite_text_overlay_start (GstBaseTransform * trans); +static gboolean gst_dwrite_text_overlay_sink_event (GstBaseTransform * trans, + GstEvent * event); static WString gst_dwrite_text_overlay_get_text (GstDWriteBaseOverlay * overlay, const WString & default_text, GstBuffer * buffer); +static void +gst_dwrite_text_overlay_after_transform (GstDWriteBaseOverlay * overlay, + GstBuffer * buffer); #define gst_dwrite_text_overlay_parent_class parent_class G_DEFINE_TYPE (GstDWriteTextOverlay, gst_dwrite_text_overlay, @@ -41,17 +91,55 @@ G_DEFINE_TYPE (GstDWriteTextOverlay, gst_dwrite_text_overlay, static void gst_dwrite_text_overlay_class_init (GstDWriteTextOverlayClass * klass) { + GObjectClass *object_class = G_OBJECT_CLASS (klass); GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass); GstDWriteBaseOverlayClass *overlay_class = GST_DWRITE_BASE_OVERLAY_CLASS (klass); + object_class->finalize = gst_dwrite_text_overlay_finalize; + object_class->set_property = gst_dwrite_text_overlay_set_property; + object_class->get_property = gst_dwrite_text_overlay_get_property; + + g_object_class_install_property (object_class, PROP_ENABLE_CC, + g_param_spec_boolean ("enable-cc", "Enable CC", + "Enable closed caption rendering", + DEFAULT_ENABLE_CC, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (object_class, PROP_CC_FIELD, + g_param_spec_int ("cc-field", "CC Field", + "The closed caption field to render when available, (-1 = automatic)", + -1, 1, DEFAULT_CC_FIELD, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (object_class, PROP_CC_TIMEOUT, + g_param_spec_uint64 ("cc-timeout", "CC Timeout", + "Duration after which to erase overlay when no cc data has arrived " + "for the selected field, in nanoseconds unit", 16 * GST_SECOND, + GST_CLOCK_TIME_NONE, DEFAULT_CC_TIMEOUT, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (object_class, PROP_REMOVE_CC_META, + g_param_spec_boolean ("remove-cc-meta", "Remove CC Meta", + "Remove caption meta from output buffers " + "when closed caption rendering is enabled", + DEFAULT_REMOVE_CC_META, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + gst_element_class_set_static_metadata (element_class, "DirectWrite Text Overlay", "Filter/Editor/Video", "Adds text strings on top of a video buffer", "Seungha Yang "); + trans_class->start = GST_DEBUG_FUNCPTR (gst_dwrite_text_overlay_start); + trans_class->sink_event = + GST_DEBUG_FUNCPTR (gst_dwrite_text_overlay_sink_event); + overlay_class->get_text = GST_DEBUG_FUNCPTR (gst_dwrite_text_overlay_get_text); + overlay_class->after_transform = + GST_DEBUG_FUNCPTR (gst_dwrite_text_overlay_after_transform); GST_DEBUG_CATEGORY_INIT (dwrite_text_overlay_debug, "dwritetextoverlay", 0, "dwritetextoverlay"); @@ -60,13 +148,453 @@ gst_dwrite_text_overlay_class_init (GstDWriteTextOverlayClass * klass) static void gst_dwrite_text_overlay_init (GstDWriteTextOverlay * self) { + self->priv = new GstDWriteTextOverlayPrivate (); + self->priv->closed_caption.reserve (CAPTION_FRAME_TEXT_BYTES); + g_object_set (self, "text-alignment", DWRITE_TEXT_ALIGNMENT_CENTER, - "paragraph-alignment", DWRITE_PARAGRAPH_ALIGNMENT_FAR, nullptr); + "paragraph-alignment", DWRITE_PARAGRAPH_ALIGNMENT_FAR, + "font-size", 20.0, nullptr); +} + +static void +gst_dwrite_text_overlay_finalize (GObject * object) +{ + GstDWriteTextOverlay *self = GST_DWRITE_TEXT_OVERLAY (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_dwrite_text_overlay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstDWriteTextOverlay *self = GST_DWRITE_TEXT_OVERLAY (object); + GstDWriteTextOverlayPrivate *priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_ENABLE_CC: + priv->enable_cc = g_value_get_boolean (value); + break; + case PROP_CC_FIELD: + priv->field = g_value_get_int (value); + if (priv->field == -1) { + priv->selected_field = 0xff; + } else { + priv->selected_field = (guint) priv->field; + } + break; + case PROP_CC_TIMEOUT: + priv->timeout = g_value_get_uint64 (value); + break; + case PROP_REMOVE_CC_META: + priv->remove_caption_meta = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_dwrite_text_overlay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstDWriteTextOverlay *self = GST_DWRITE_TEXT_OVERLAY (object); + GstDWriteTextOverlayPrivate *priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_ENABLE_CC: + g_value_set_boolean (value, priv->enable_cc); + break; + case PROP_CC_FIELD: + g_value_set_int (value, priv->field); + break; + case PROP_CC_TIMEOUT: + g_value_set_uint64 (value, priv->timeout); + break; + case PROP_REMOVE_CC_META: + g_value_set_boolean (value, priv->remove_caption_meta); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_dwrite_text_overlay_start (GstBaseTransform * trans) +{ + GstDWriteTextOverlay *self = GST_DWRITE_TEXT_OVERLAY (trans); + GstDWriteTextOverlayPrivate *priv = self->priv; + + caption_frame_init (&priv->frame); + priv->running_time = GST_CLOCK_TIME_NONE; + priv->caption_running_time = GST_CLOCK_TIME_NONE; + priv->selected_field = 0xff; + priv->closed_caption.clear (); + + return GST_BASE_TRANSFORM_CLASS (parent_class)->start (trans); +} + +static gboolean +gst_dwrite_text_overlay_sink_event (GstBaseTransform * trans, GstEvent * event) +{ + GstDWriteTextOverlay *self = GST_DWRITE_TEXT_OVERLAY (trans); + GstDWriteTextOverlayPrivate *priv = self->priv; + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_SEGMENT: + priv->caption_running_time = GST_CLOCK_TIME_NONE; + priv->running_time = GST_CLOCK_TIME_NONE; + break; + default: + break; + } + + return GST_BASE_TRANSFORM_CLASS (parent_class)->sink_event (trans, event); +} + +static guint +gst_dwrite_text_overlay_extract_cdp (GstDWriteTextOverlay * self, + const guint8 * cdp, guint cdp_len, guint * pos) +{ + GstByteReader br; + guint16 u16; + guint8 u8; + guint8 flags; + guint len = 0; + + GST_TRACE_OBJECT (self, "Extracting CDP"); + + /* Header + footer length */ + if (cdp_len < 11) { + GST_WARNING_OBJECT (self, "cdp packet too short (%u). expected at " + "least %u", cdp_len, 11); + return 0; + } + + gst_byte_reader_init (&br, cdp, cdp_len); + u16 = gst_byte_reader_get_uint16_be_unchecked (&br); + if (u16 != 0x9669) { + GST_WARNING_OBJECT (self, "cdp packet does not have initial magic bytes " + "of 0x9669"); + return 0; + } + + u8 = gst_byte_reader_get_uint8_unchecked (&br); + if (u8 != cdp_len) { + GST_WARNING_OBJECT (self, "cdp packet length (%u) does not match passed " + "in value (%u)", u8, cdp_len); + return 0; + } + + /* skip framerate value */ + gst_byte_reader_skip_unchecked (&br, 1); + + flags = gst_byte_reader_get_uint8_unchecked (&br); + /* No cc_data? */ + if ((flags & 0x40) == 0) { + GST_DEBUG_OBJECT (self, "cdp packet does have any cc_data"); + return 0; + } + + /* cdp_hdr_sequence_cntr */ + gst_byte_reader_skip_unchecked (&br, 2); + + /* skip timecode */ + if (flags & 0x80) { + if (gst_byte_reader_get_remaining (&br) < 5) { + GST_WARNING_OBJECT (self, "cdp packet does not have enough data to " + "contain a timecode (%u). Need at least 5 bytes", + gst_byte_reader_get_remaining (&br)); + return 0; + } + + gst_byte_reader_skip_unchecked (&br, 5); + } + + /* ccdata_present */ + if (flags & 0x40) { + guint8 cc_count; + + if (gst_byte_reader_get_remaining (&br) < 2) { + GST_WARNING_OBJECT (self, "not enough data to contain valid cc_data"); + return 0; + } + + u8 = gst_byte_reader_get_uint8_unchecked (&br); + if (u8 != 0x72) { + GST_WARNING_OBJECT (self, "missing cc_data start code of 0x72, " + "found 0x%02x", u8); + return 0; + } + + cc_count = gst_byte_reader_get_uint8_unchecked (&br); + if ((cc_count & 0xe0) != 0xe0) { + GST_WARNING_OBJECT (self, "reserved bits are not 0xe0, found 0x%02x", u8); + return 0; + } + cc_count &= 0x1f; + + len = 3 * cc_count; + if (gst_byte_reader_get_remaining (&br) < len) { + GST_WARNING_OBJECT (self, "not enough bytes (%u) left for the " + "number of byte triples (%u)", gst_byte_reader_get_remaining (&br), + cc_count); + return 0; + } + + *pos = gst_byte_reader_get_pos (&br); + } + + /* skip everything else we don't care about */ + return len; +} + +static void +gst_dwrite_text_overlay_decode_cc_data (GstDWriteTextOverlay * self, + const guint8 * data, guint len, GstClockTime running_time) +{ + GstDWriteTextOverlayPrivate *priv = self->priv; + GstByteReader br; + + GST_TRACE_OBJECT (self, "Decoding CC data"); + + gst_byte_reader_init (&br, data, len); + while (gst_byte_reader_get_remaining (&br) >= 3) { + guint8 cc_type; + guint16 cc_data; + + cc_type = gst_byte_reader_get_uint8_unchecked (&br); + cc_data = gst_byte_reader_get_uint16_be_unchecked (&br); + + if ((cc_type & 0x04) != 0x04) + continue; + + cc_type = cc_type & 0x03; + if (cc_type != 0x00 && cc_type != 0x01) + continue; + + if (priv->selected_field == 0xff) { + GST_INFO_OBJECT (self, "Selected field %d", cc_type); + priv->selected_field = cc_type; + } + + if (cc_type != priv->selected_field) + continue; + + auto status = caption_frame_decode (&priv->frame, cc_data, 0.0); + switch (status) { + case LIBCAPTION_READY: + { + auto len = caption_frame_to_text (&priv->frame, + &priv->closed_caption[0], TRUE); + priv->closed_caption.resize (len); + break; + } + case LIBCAPTION_CLEAR: + priv->closed_caption.clear (); + break; + default: + break; + } + + priv->caption_running_time = running_time; + } +} + +static void +gst_dwrite_text_overlay_decode_s334_1a (GstDWriteTextOverlay * self, + const guint8 * data, guint len, GstClockTime running_time) +{ + GstDWriteTextOverlayPrivate *priv = self->priv; + GstByteReader br; + + GST_TRACE_OBJECT (self, "Decoding S334-1A"); + + gst_byte_reader_init (&br, data, len); + while (gst_byte_reader_get_remaining (&br) >= 3) { + guint8 cc_type; + guint16 cc_data; + + cc_type = gst_byte_reader_get_uint8_unchecked (&br); + cc_data = gst_byte_reader_get_uint16_be_unchecked (&br); + + cc_type = cc_type & 0x01; + if (priv->selected_field == 0xff) { + GST_INFO_OBJECT (self, "Selected field %d", cc_type); + priv->selected_field = cc_type; + } + + if (cc_type != priv->selected_field) + continue; + + auto status = caption_frame_decode (&priv->frame, cc_data, 0.0); + switch (status) { + case LIBCAPTION_READY: + { + auto len = caption_frame_to_text (&priv->frame, + &priv->closed_caption[0], TRUE); + priv->closed_caption.resize (len); + break; + } + case LIBCAPTION_CLEAR: + priv->closed_caption.clear (); + break; + default: + break; + } + + priv->caption_running_time = running_time; + } +} + +static void +gst_dwrite_text_overlay_decode_raw (GstDWriteTextOverlay * self, + const guint8 * data, guint len, GstClockTime running_time) +{ + GstDWriteTextOverlayPrivate *priv = self->priv; + GstByteReader br; + + GST_TRACE_OBJECT (self, "Decoding CEA608 RAW"); + + gst_byte_reader_init (&br, data, len); + while (gst_byte_reader_get_remaining (&br) >= 2) { + guint16 cc_data; + + cc_data = gst_byte_reader_get_uint16_be_unchecked (&br); + + auto status = caption_frame_decode (&priv->frame, cc_data, 0.0); + switch (status) { + case LIBCAPTION_READY: + { + auto len = caption_frame_to_text (&priv->frame, + &priv->closed_caption[0], TRUE); + priv->closed_caption.resize (len); + break; + } + case LIBCAPTION_CLEAR: + priv->closed_caption.clear (); + break; + default: + break; + } + + priv->caption_running_time = running_time; + } +} + +static gboolean +gst_dwrite_text_overlay_foreach_meta (GstBuffer * buffer, GstMeta ** meta, + GstDWriteTextOverlay * self) +{ + GstDWriteTextOverlayPrivate *priv = self->priv; + GstVideoCaptionMeta *cc_meta; + + if ((*meta)->info->api != GST_VIDEO_CAPTION_META_API_TYPE) + return TRUE; + + cc_meta = (GstVideoCaptionMeta *) (*meta); + switch (cc_meta->caption_type) { + case GST_VIDEO_CAPTION_TYPE_CEA608_RAW: + gst_dwrite_text_overlay_decode_raw (self, cc_meta->data, cc_meta->size, + priv->running_time); + break; + case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A: + gst_dwrite_text_overlay_decode_s334_1a (self, cc_meta->data, + cc_meta->size, priv->running_time); + break; + case GST_VIDEO_CAPTION_TYPE_CEA708_RAW: + gst_dwrite_text_overlay_decode_cc_data (self, cc_meta->data, + cc_meta->size, priv->running_time); + break; + case GST_VIDEO_CAPTION_TYPE_CEA708_CDP: + { + guint len, pos = 0; + len = gst_dwrite_text_overlay_extract_cdp (self, cc_meta->data, + cc_meta->size, &pos); + if (len > 0) { + gst_dwrite_text_overlay_decode_cc_data (self, cc_meta->data + pos, + len, priv->running_time); + } + break; + } + default: + break; + } + + return TRUE; } static WString gst_dwrite_text_overlay_get_text (GstDWriteBaseOverlay * overlay, const WString & default_text, GstBuffer * buffer) { - return default_text; + GstBaseTransform *trans = GST_BASE_TRANSFORM (overlay); + GstDWriteTextOverlay *self = GST_DWRITE_TEXT_OVERLAY (overlay); + GstDWriteTextOverlayPrivate *priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + priv->running_time = gst_segment_to_running_time (&trans->segment, + GST_FORMAT_TIME, GST_BUFFER_PTS (buffer)); + + if (priv->enable_cc) { + gst_buffer_foreach_meta (buffer, + (GstBufferForeachMetaFunc) gst_dwrite_text_overlay_foreach_meta, self); + + if (GST_CLOCK_TIME_IS_VALID (priv->timeout) && + GST_CLOCK_TIME_IS_VALID (priv->running_time) && + GST_CLOCK_TIME_IS_VALID (priv->caption_running_time) && + priv->running_time >= priv->caption_running_time) { + GstClockTime diff = priv->running_time - priv->caption_running_time; + + if (diff > priv->timeout) { + GST_INFO_OBJECT (self, "Reached timeout, clearing text"); + priv->closed_caption.clear (); + } + } + } else { + priv->closed_caption.clear (); + } + + if (priv->closed_caption.empty ()) + return default_text; + + auto text_wide = gst_dwrite_string_to_wstring (priv->closed_caption); + if (default_text.empty ()) + return text_wide; + + return default_text + WString (L" ") + text_wide; +} + +static gboolean +gst_dwrite_text_overlay_remove_meta (GstBuffer * buffer, GstMeta ** meta, + GstDWriteTextOverlay * self) +{ + if ((*meta)->info->api != GST_VIDEO_CAPTION_META_API_TYPE) + return TRUE; + + GST_TRACE_OBJECT (self, "Removing caption meta"); + *meta = nullptr; + + return TRUE; +} + +static void +gst_dwrite_text_overlay_after_transform (GstDWriteBaseOverlay * overlay, + GstBuffer * buffer) +{ + GstDWriteTextOverlay *self = GST_DWRITE_TEXT_OVERLAY (overlay); + GstDWriteTextOverlayPrivate *priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + if (!priv->enable_cc || !priv->remove_caption_meta) + return; + + gst_buffer_foreach_meta (buffer, + (GstBufferForeachMetaFunc) gst_dwrite_text_overlay_remove_meta, self); } diff --git a/subprojects/gst-plugins-bad/sys/dwrite/libcaption/meson.build b/subprojects/gst-plugins-bad/sys/dwrite/libcaption/meson.build new file mode 100644 index 0000000000..0373b470a1 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/dwrite/libcaption/meson.build @@ -0,0 +1,19 @@ +libcaption_sources = [ + 'caption.c', + 'eia608_charmap.c', + 'eia608_from_utf8.c', + 'eia608.c', + 'utf8.c', + 'xds.c', +] + +libcaption_static = static_library('libcaption-static', + libcaption_sources, + include_directories : include_directories('.'), + override_options: ['werror=false'], +) + +dwrite_libcaption_dep = declare_dependency( + link_with : libcaption_static, + include_directories: include_directories('.') +) diff --git a/subprojects/gst-plugins-bad/sys/dwrite/meson.build b/subprojects/gst-plugins-bad/sys/dwrite/meson.build index b0f60862b1..d87ecd8460 100644 --- a/subprojects/gst-plugins-bad/sys/dwrite/meson.build +++ b/subprojects/gst-plugins-bad/sys/dwrite/meson.build @@ -58,12 +58,14 @@ if cc.get_id() != 'msvc' extra_args += extra_mingw_args endif +subdir('libcaption') + gstdwrite = library('gstdwrite', dwrite_sources, c_args : gst_plugins_bad_args + extra_args, cpp_args: gst_plugins_bad_args + extra_args, include_directories : [configinc], - dependencies : [gstbase_dep, gstvideo_dep, gstd3d11_dep, d2d_dep, dwrite_lib], + dependencies : [gstbase_dep, gstvideo_dep, gstd3d11_dep, d2d_dep, dwrite_lib, dwrite_libcaption_dep], install : true, install_dir : plugins_install_dir, )