mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-22 17:51:16 +00:00
Code Issues 1 Projects Releases Wiki Activity

dwrite: Add dwritesubtitleoverlay element

Browse source 
Adding new subtitle overlay element. It's a bin which is wrapping
two internal elements dwritesubtitlemux and dwritetextoverlay.

* dwritesubtitlemux: A new internal element to aggregate subtitle
buffers and to attach the aggregated subtitle buffers on
video buffer as meta.
* dwritetextoverlay: Extracts/renders the subtitle meta and
discard the meta after rendering.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4934>
This commit is contained in:
Seungha Yang 2023-06-27 23:40:20 +09:00 committed by GStreamer Marge Bot
parent a1ca42ad66
commit 1c4de219e4
8 changed files with 597 additions and 187 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

225
subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.cpp
View file

@ -78,6 +78,10 @@ enum
PROP_ENABLE_COLOR_FONT,
};
/* *INDENT-OFF* */
static std::vector <GParamSpec *> _pspec;
/* *INDENT-ON* */
enum class GstDWriteBaseOverlayBlendMode
{
UNKNOWN,
@ -248,128 +252,9 @@ gst_dwrite_base_overlay_class_init (GstDWriteBaseOverlayClass * klass)
object_class->set_property = gst_dwrite_base_overlay_set_property;
object_class->get_property = gst_dwrite_base_overlay_get_property;
g_object_class_install_property (object_class, PROP_VISIBLE,
g_param_spec_boolean ("visible", "Visible",
"Whether to draw text", DEFAULT_VISIBLE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_FONT_FAMILY,
g_param_spec_string ("font-family", "Font Family",
"Font family to use", DEFAULT_FONT_FAMILY,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_FONT_SIZE,
g_param_spec_float ("font-size", "Font Size",
"Font size to use", 0.1f, 1638.f, DEFAULT_FONT_SIZE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_REFERENCE_FRAME_SIZE,
g_param_spec_uint ("reference-frame-size", "Reference Frame Size",
"Reference Frame size used for \"auto-resize\"", 16, 16384,
DEFAULT_REFERENCE_FRAME_SIZE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_AUTO_RESIZE,
g_param_spec_boolean ("auto-resize", "Auto Resize",
"Calculate font size to be equivalent to \"font-size\" at "
"\"reference-frame-size\"", DEFAULT_AUTO_RESIZE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_FONT_WEIGHT,
g_param_spec_enum ("font-weight", "Font Weight",
"Font Weight", GST_TYPE_DWRITE_FONT_WEIGHT,
DEFAULT_FONT_WEIGHT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_FONT_STYLE,
g_param_spec_enum ("font-style", "Font Style",
"Font Style", GST_TYPE_DWRITE_FONT_STYLE,
DEFAULT_FONT_STYLE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_FONT_STRETCH,
g_param_spec_enum ("font-stretch", "Font Stretch",
"Font Stretch", GST_TYPE_DWRITE_FONT_STRETCH,
DEFAULT_FONT_STRETCH,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_TEXT,
g_param_spec_string ("text", "Text",
"Text to render", "",
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_COLOR,
g_param_spec_uint ("color", "Color",
"Text color to use (big-endian ARGB)", 0, G_MAXUINT32, DEFAULT_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_OUTLINE_COLOR,
g_param_spec_uint ("outline-color", "Outline Color",
"Text outline color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_OUTLINE_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_UNDERLINE_COLOR,
g_param_spec_uint ("underline-color", "Underline Color",
"Underline color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_UNDERLINE_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_UNDERLINE_OUTLINE_COLOR,
g_param_spec_uint ("underline-outline-color", "Underline Outline Color",
"Outline of underline color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_UNDERLINE_OUTLINE_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_STRIKETHROUGH_COLOR,
g_param_spec_uint ("strikethrough-color", "Strikethrough Color",
"Strikethrough color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_STRIKETHROUGH_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class,
PROP_STRIKETHROUGH_OUTLINE_COLOR,
g_param_spec_uint ("strikethrough-outline-color",
"Strikethrough Outline Color",
"Outline of strikethrough color to use (big-endian ARGB)",
0, G_MAXUINT32, DEFAULT_STRIKETHROUGH_OUTLINE_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_SHADOW_COLOR,
g_param_spec_uint ("shadow-color", "Shadow Color",
"Shadow color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_SHADOW_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_BACKGROUND_COLOR,
g_param_spec_uint ("background-color", "Background Color",
"Background color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_BACKGROUND_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_LAYOUT_X,
g_param_spec_double ("layout-x", "Layout X",
"Normalized X coordinate of text layout", 0, 1,
DEFAULT_LAYOUT_XY,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_LAYOUT_Y,
g_param_spec_double ("layout-y", "Layout Y",
"Normalized Y coordinate of text layout", 0, 1,
DEFAULT_LAYOUT_XY,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_LAYOUT_WIDTH,
g_param_spec_double ("layout-width", "Layout Width",
"Normalized width of text layout", 0, 1,
DEFAULT_LAYOUT_WH,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_LAYOUT_HEIGHT,
g_param_spec_double ("layout-height", "Layout Height",
"Normalized height of text layout", 0, 1,
DEFAULT_LAYOUT_WH,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_TEXT_ALIGNMENT,
g_param_spec_enum ("text-alignment", "Text Alignment",
"Text Alignment", GST_TYPE_DWRITE_TEXT_ALIGNMENT,
DEFAULT_TEXT_ALIGNMENT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_PARAGRAPH_ALIGNMENT,
g_param_spec_enum ("paragraph-alignment", "Paragraph alignment",
"Paragraph Alignment", GST_TYPE_DWRITE_PARAGRAPH_ALIGNMENT,
DEFAULT_PARAGRAPH_ALIGNMENT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
#ifdef HAVE_DWRITE_COLOR_FONT
if (gst_dwrite_is_windows_10_or_greater ()) {
g_object_class_install_property (object_class, PROP_ENABLE_COLOR_FONT,
g_param_spec_boolean ("color-font", "Color Font",
"Enable color font, requires Windows 10 or newer",
DEFAULT_COLOR_FONT,
(GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE |
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
}
#endif
gst_dwrite_base_overlay_build_param_specs (_pspec);
for (guint i = 0; i < (guint) _pspec.size (); i++)
g_object_class_install_property (object_class, i + 1, _pspec[i]);
gst_element_class_add_static_pad_template (element_class, &sink_template);
gst_element_class_add_static_pad_template (element_class, &src_template);
@ -2253,3 +2138,99 @@ gst_dwrite_base_overlay_transform (GstBaseTransform * trans, GstBuffer * inbuf,
return GST_FLOW_OK;
}
void
gst_dwrite_base_overlay_build_param_specs (std::vector < GParamSpec * >&pspec)
{
pspec.push_back (g_param_spec_boolean ("visible", "Visible",
"Whether to draw text", DEFAULT_VISIBLE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_string ("font-family", "Font Family",
"Font family to use", DEFAULT_FONT_FAMILY,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_float ("font-size", "Font Size",
"Font size to use", 0.1f, 1638.f, DEFAULT_FONT_SIZE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("reference-frame-size",
"Reference Frame Size",
"Reference Frame size used for \"auto-resize\"", 16, 16384,
DEFAULT_REFERENCE_FRAME_SIZE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_boolean ("auto-resize", "Auto Resize",
"Calculate font size to be equivalent to \"font-size\" at "
"\"reference-frame-size\"", DEFAULT_AUTO_RESIZE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_enum ("font-weight", "Font Weight",
"Font Weight", GST_TYPE_DWRITE_FONT_WEIGHT, DEFAULT_FONT_WEIGHT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_enum ("font-style", "Font Style", "Font Style",
GST_TYPE_DWRITE_FONT_STYLE, DEFAULT_FONT_STYLE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_enum ("font-stretch", "Font Stretch",
"Font Stretch", GST_TYPE_DWRITE_FONT_STRETCH, DEFAULT_FONT_STRETCH,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_string ("text", "Text", "Text to render", "",
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("color", "Color",
"Text color to use (big-endian ARGB)", 0, G_MAXUINT32, DEFAULT_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("outline-color", "Outline Color",
"Text outline color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_OUTLINE_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("underline-color", "Underline Color",
"Underline color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_UNDERLINE_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("underline-outline-color",
"Underline Outline Color",
"Outline of underline color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_UNDERLINE_OUTLINE_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("strikethrough-color",
"Strikethrough Color", "Strikethrough color to use (big-endian ARGB)",
0, G_MAXUINT32, DEFAULT_STRIKETHROUGH_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("strikethrough-outline-color",
"Strikethrough Outline Color",
"Outline of strikethrough color to use (big-endian ARGB)", 0,
G_MAXUINT32, DEFAULT_STRIKETHROUGH_OUTLINE_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("shadow-color", "Shadow Color",
"Shadow color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_SHADOW_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint ("background-color", "Background Color",
"Background color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_BACKGROUND_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_double ("layout-x", "Layout X",
"Normalized X coordinate of text layout", 0, 1, DEFAULT_LAYOUT_XY,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_double ("layout-y", "Layout Y",
"Normalized Y coordinate of text layout", 0, 1, DEFAULT_LAYOUT_XY,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_double ("layout-width", "Layout Width",
"Normalized width of text layout", 0, 1, DEFAULT_LAYOUT_WH,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_double ("layout-height", "Layout Height",
"Normalized height of text layout", 0, 1, DEFAULT_LAYOUT_WH,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_enum ("text-alignment", "Text Alignment",
"Text Alignment", GST_TYPE_DWRITE_TEXT_ALIGNMENT,
DEFAULT_TEXT_ALIGNMENT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_enum ("paragraph-alignment",
"Paragraph alignment", "Paragraph Alignment",
GST_TYPE_DWRITE_PARAGRAPH_ALIGNMENT, DEFAULT_PARAGRAPH_ALIGNMENT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
#ifdef HAVE_DWRITE_COLOR_FONT
if (gst_dwrite_is_windows_10_or_greater ()) {
pspec.push_back (g_param_spec_boolean ("color-font", "Color Font",
"Enable color font, requires Windows 10 or newer",
DEFAULT_COLOR_FONT,
(GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE |
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
}
#endif
}

3
subprojects/gst-plugins-bad/sys/dwrite/gstdwritebaseoverlay.h
View file

@ -25,6 +25,7 @@
#include <string>
#include "gstdwrite-utils.h"
#include "gstdwrite-enums.h"
#include <vector>
G_BEGIN_DECLS
@ -67,6 +68,8 @@ struct _GstDWriteBaseOverlayClass
GType gst_dwrite_base_overlay_get_type (void);
void gst_dwrite_base_overlay_build_param_specs (std::vector<GParamSpec *> & pspec);
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstDWriteBaseOverlay, gst_object_unref)
G_END_DECLS

259
subprojects/gst-plugins-bad/sys/dwrite/gstdwritesubtitleoverlay.cpp Normal file
View file

@ -0,0 +1,259 @@
/* GStreamer
* Copyright (C) 2023 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gst/d3d11/gstd3d11.h>
#include <gst/d3d11/gstd3d11-private.h>
#include <gst/video/video.h>
#include "gstdwritesubtitleoverlay.h"
#include "gstdwrite-utils.h"
#include "gstdwritetextoverlay.h"
#include <mutex>
GST_DEBUG_CATEGORY_STATIC (dwrite_subtitle_overlay_debug);
#define GST_CAT_DEFAULT dwrite_subtitle_overlay_debug
static GstStaticPadTemplate video_templ = GST_STATIC_PAD_TEMPLATE ("video",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_DWRITE_CAPS)
);
static GstStaticPadTemplate text_templ = GST_STATIC_PAD_TEMPLATE ("text",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("text/x-raw, format = { pango-markup, utf8 }"));
static GstStaticPadTemplate src_templ = GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_DWRITE_CAPS)
);
/* *INDENT-OFF* */
static std::vector<GParamSpec *> _pspec;
/* *INDENT-ON* */
struct GstDWriteSubtitleOverlayPrivate
{
std::mutex lock;
GstElement *mux = nullptr;
GstElement *overlay = nullptr;
GstPad *text_pad = nullptr;
GstPad *mux_pad = nullptr;
};
struct _GstDWriteSubtitleOverlay
{
GstBin parent;
GstDWriteSubtitleOverlayPrivate *priv;
};
static void gst_dwrite_subtitle_overlay_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_dwrite_subtitle_overlay_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec);
static GstPadLinkReturn gst_dwrite_subtitle_overlay_text_link (GstPad * pad,
GstObject * parent, GstPad * peer);
static void gst_dwrite_subtitle_overlay_text_unlink (GstPad * pad,
GstObject * parent);
static gboolean gst_dwrite_subtitle_overlay_src_event (GstPad * pad,
GstObject * parent, GstEvent * event);
#define gst_dwrite_subtitle_overlay_parent_class parent_class
G_DEFINE_TYPE (GstDWriteSubtitleOverlay, gst_dwrite_subtitle_overlay,
GST_TYPE_BIN);
static void
gst_dwrite_subtitle_overlay_class_init (GstDWriteSubtitleOverlayClass * klass)
{
GObjectClass *object_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
object_class->set_property = gst_dwrite_subtitle_overlay_set_property;
object_class->get_property = gst_dwrite_subtitle_overlay_get_property;
gst_dwrite_base_overlay_build_param_specs (_pspec);
gst_dwrite_text_overlay_build_param_specs (_pspec);
for (guint i = 0; i < _pspec.size (); i++)
g_object_class_install_property (object_class, i + 1, _pspec[i]);
gst_element_class_set_static_metadata (element_class,
"DirectWrite Subtitle Overlay",
"Filter/Editor/Video/Overlay/Subtitle",
"Adds subtitle strings on top of a video buffer",
"Seungha Yang <seungha@centricular.com>");
gst_element_class_add_static_pad_template (element_class, &video_templ);
gst_element_class_add_static_pad_template (element_class, &text_templ);
gst_element_class_add_static_pad_template (element_class, &src_templ);
GST_DEBUG_CATEGORY_INIT (dwrite_subtitle_overlay_debug,
"dwritesubtitleoverlay", 0, "dwritesubtitleoverlay");
}
static void
gst_dwrite_subtitle_overlay_init (GstDWriteSubtitleOverlay * self)
{
GstElement *elem = GST_ELEMENT_CAST (self);
GstPad *gpad;
GstPad *pad;
GstPadTemplate *templ;
GstDWriteSubtitleOverlayPrivate *priv;
self->priv = priv = new GstDWriteSubtitleOverlayPrivate ();
priv->mux = gst_element_factory_make ("dwritesubtitlemux", "subtitle-mux");
priv->overlay =
gst_element_factory_make ("dwritetextoverlay", "text-overlay");
gst_bin_add_many (GST_BIN_CAST (self), priv->mux, priv->overlay, nullptr);
gst_element_link (priv->mux, priv->overlay);
pad = gst_element_get_static_pad (priv->mux, "video");
gpad = gst_ghost_pad_new ("video", pad);
gst_object_unref (pad);
gst_element_add_pad (elem, gpad);
pad = gst_element_get_static_pad (priv->overlay, "src");
gpad = gst_ghost_pad_new ("src", pad);
gst_object_unref (pad);
gst_element_add_pad (elem, gpad);
pad = GST_PAD_CAST (gst_proxy_pad_get_internal (GST_PROXY_PAD (gpad)));
gst_pad_set_event_function (pad, gst_dwrite_subtitle_overlay_src_event);
gst_object_unref (pad);
templ = gst_static_pad_template_get (&text_templ);
priv->text_pad = gst_ghost_pad_new_no_target_from_template ("text", templ);
gst_object_unref (templ);
gst_element_add_pad (elem, priv->text_pad);
GST_PAD_SET_ACCEPT_INTERSECT (priv->text_pad);
GST_PAD_SET_ACCEPT_TEMPLATE (priv->text_pad);
gst_pad_set_link_function (priv->text_pad,
gst_dwrite_subtitle_overlay_text_link);
gst_pad_set_unlink_function (priv->text_pad,
gst_dwrite_subtitle_overlay_text_unlink);
}
static void
gst_dwrite_subtitle_overlay_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstDWriteSubtitleOverlay *self = GST_DWRITE_SUBTITLE_OVERLAY (object);
GstDWriteSubtitleOverlayPrivate *priv = self->priv;
g_object_set_property (G_OBJECT (priv->overlay), pspec->name, value);
}
static void
gst_dwrite_subtitle_overlay_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstDWriteSubtitleOverlay *self = GST_DWRITE_SUBTITLE_OVERLAY (object);
GstDWriteSubtitleOverlayPrivate *priv = self->priv;
g_object_get_property (G_OBJECT (priv->overlay), pspec->name, value);
}
static GstPadLinkReturn
gst_dwrite_subtitle_overlay_text_link (GstPad * pad, GstObject * parent,
GstPad * peer)
{
GstDWriteSubtitleOverlay *self = GST_DWRITE_SUBTITLE_OVERLAY (parent);
GstDWriteSubtitleOverlayPrivate *priv = self->priv;
GstPad *mux_pad;
std::lock_guard < std::mutex > lk (priv->lock);
mux_pad = gst_element_request_pad_simple (priv->mux, "text_%u");
if (!mux_pad) {
GST_ERROR_OBJECT (self, "Couldn't get mux pad");
return GST_PAD_LINK_REFUSED;
}
gst_ghost_pad_set_target (GST_GHOST_PAD_CAST (priv->text_pad), mux_pad);
gst_clear_object (&priv->mux_pad);
priv->mux_pad = mux_pad;
GST_DEBUG_OBJECT (self, "Text pad linked");
return GST_PAD_LINK_OK;
}
static void
gst_dwrite_subtitle_overlay_text_unlink (GstPad * pad, GstObject * parent)
{
GstDWriteSubtitleOverlay *self = GST_DWRITE_SUBTITLE_OVERLAY (parent);
GstDWriteSubtitleOverlayPrivate *priv = self->priv;
std::lock_guard < std::mutex > lk (priv->lock);
/* We cannot clear target on unlink function, since unlink function is
* called with GST_OBJECT_LOCK and get/set target will take the lock
* as well. Let ghostpad hold old target but it's fine */
if (!priv->mux_pad) {
GST_WARNING_OBJECT (self, "No linked mux pad");
} else {
GST_DEBUG_OBJECT (self, "Unlinking text pad");
gst_element_release_request_pad (priv->mux, priv->mux_pad);
gst_clear_object (&priv->mux_pad);
}
}
static gboolean
gst_dwrite_subtitle_overlay_src_event (GstPad * pad, GstObject * parent,
GstEvent * event)
{
guint32 seqnum;
/* subtitleoverlay elements will drop flush event if it was passed to text pad
* based on the pango element's behavior, it should be dropped since
* aggregator will forward the same flush event to text pad as well.
* Replace flush event with ours */
switch (GST_EVENT_TYPE (event)) {
case GST_EVENT_FLUSH_START:
seqnum = gst_event_get_seqnum (event);
gst_event_unref (event);
event = gst_event_new_flush_start ();
gst_event_set_seqnum (event, seqnum);
break;
case GST_EVENT_FLUSH_STOP:
{
gboolean reset;
gst_event_parse_flush_stop (event, &reset);
seqnum = gst_event_get_seqnum (event);
gst_event_unref (event);
event = gst_event_new_flush_stop (reset);
gst_event_set_seqnum (event, seqnum);
break;
}
default:
break;
}
return gst_pad_event_default (pad, parent, event);
}

31
subprojects/gst-plugins-bad/sys/dwrite/gstdwritesubtitleoverlay.h Normal file
View file

@ -0,0 +1,31 @@
/* GStreamer
* Copyright (C) 2023 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#pragma once
#include <gst/gst.h>
#include <gst/base/base.h>
G_BEGIN_DECLS
#define GST_TYPE_DWRITE_SUBTITLE_OVERLAY (gst_dwrite_subtitle_overlay_get_type())
G_DECLARE_FINAL_TYPE (GstDWriteSubtitleOverlay, gst_dwrite_subtitle_overlay,
GST, DWRITE_SUBTITLE_OVERLAY, GstBin)
G_END_DECLS

260
subprojects/gst-plugins-bad/sys/dwrite/gstdwritetextoverlay.cpp
View file

@ -39,6 +39,10 @@ enum
PROP_REMOVE_CC_META,
};
/* *INDENT-OFF* */
static std::vector<GParamSpec *> _pspec;
/* *INDENT-ON* */
#define DEFAULT_ENABLE_CC TRUE
#define DEFAULT_CC_FIELD -1
#define DEFAULT_CC_TIMEOUT GST_CLOCK_TIME_NONE
@ -54,6 +58,7 @@ struct GstDWriteTextOverlayPrivate
guint8 selected_field;
std::string closed_caption;
std::string text;
/* properties */
gboolean enable_cc = DEFAULT_ENABLE_CC;
@ -101,31 +106,9 @@ gst_dwrite_text_overlay_class_init (GstDWriteTextOverlayClass * klass)
object_class->set_property = gst_dwrite_text_overlay_set_property;
object_class->get_property = gst_dwrite_text_overlay_get_property;
g_object_class_install_property (object_class, PROP_ENABLE_CC,
g_param_spec_boolean ("enable-cc", "Enable CC",
"Enable closed caption rendering",
DEFAULT_ENABLE_CC,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_CC_FIELD,
g_param_spec_int ("cc-field", "CC Field",
"The closed caption field to render when available, (-1 = automatic)",
-1, 1, DEFAULT_CC_FIELD,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_CC_TIMEOUT,
g_param_spec_uint64 ("cc-timeout", "CC Timeout",
"Duration after which to erase overlay when no cc data has arrived "
"for the selected field, in nanoseconds unit", 16 * GST_SECOND,
GST_CLOCK_TIME_NONE, DEFAULT_CC_TIMEOUT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_REMOVE_CC_META,
g_param_spec_boolean ("remove-cc-meta", "Remove CC Meta",
"Remove caption meta from output buffers "
"when closed caption rendering is enabled",
DEFAULT_REMOVE_CC_META,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
gst_dwrite_text_overlay_build_param_specs (_pspec);
for (guint i = 0; i < _pspec.size (); i++)
g_object_class_install_property (object_class, i + 1, _pspec[i]);
gst_element_class_set_static_metadata (element_class,
"DirectWrite Text Overlay", "Filter/Editor/Video",
@ -488,6 +471,115 @@ gst_dwrite_text_overlay_decode_raw (GstDWriteTextOverlay * self,
}
}
static void
xml_text (GMarkupParseContext * context, const gchar * text, gsize text_len,
gpointer user_data, GError ** error)
{
gchar **accum = (gchar **) user_data;
gchar *concat;
if (*accum) {
concat = g_strconcat (*accum, text, NULL);
g_free (*accum);
*accum = concat;
} else {
*accum = g_strdup (text);
}
}
static gchar *
gst_dwrite_text_overlay_strip_markup (GstDWriteTextOverlay * self,
const gchar * markup)
{
GMarkupParser parser = { 0, };
GMarkupParseContext *context;
gchar *accum = nullptr;
parser.text = xml_text;
context = g_markup_parse_context_new (&parser,
(GMarkupParseFlags) 0, &accum, nullptr);
if (!g_markup_parse_context_parse (context, "<root>", 6, nullptr))
goto error;
if (!g_markup_parse_context_parse (context, markup, strlen (markup), nullptr))
goto error;
if (!g_markup_parse_context_parse (context, "</root>", 7, nullptr))
goto error;
if (!g_markup_parse_context_end_parse (context, nullptr))
goto error;
done:
g_markup_parse_context_free (context);
return accum;
error:
g_free (accum);
accum = nullptr;
goto done;
}
static void
gst_dwrite_text_overlay_extract_meta (GstDWriteTextOverlay * self,
GstDWriteSubtitleMeta * meta)
{
GstDWriteTextOverlayPrivate *priv = self->priv;
GstCaps *caps = nullptr;
GstStructure *s;
const gchar *format;
std::string str;
GstMapInfo info;
if (!meta || !meta->subtitle || !meta->stream)
return;
caps = gst_stream_get_caps (meta->stream);
if (!caps)
return;
if (gst_buffer_get_size (meta->subtitle) == 0)
goto out;
if (!gst_buffer_map (meta->subtitle, &info, GST_MAP_READ))
goto out;
s = gst_caps_get_structure (caps, 0);
format = gst_structure_get_string (s, "format");
/* TODO: parse pango attributs and make layout based on that */
if (g_strcmp0 (format, "pango-markup") == 0) {
gchar *stripped = gst_dwrite_text_overlay_strip_markup (self,
(gchar *) info.data);
gst_buffer_unmap (meta->subtitle, &info);
if (!stripped)
goto out;
if (priv->text.empty ()) {
priv->text = stripped;
} else {
priv->text += "\n";
priv->text += stripped;
}
} else {
std::string ret;
ret.resize (info.size);
memcpy (&ret[0], info.data, info.size);
gst_buffer_unmap (meta->subtitle, &info);
auto len = strlen (ret.c_str ());
ret.resize (len);
if (priv->text.empty ())
priv->text = ret;
else
priv->text += " " + ret;
}
out:
gst_clear_caps (&caps);
}
static gboolean
gst_dwrite_text_overlay_foreach_meta (GstBuffer * buffer, GstMeta ** meta,
GstDWriteTextOverlay * self)
@ -495,36 +587,38 @@ gst_dwrite_text_overlay_foreach_meta (GstBuffer * buffer, GstMeta ** meta,
GstDWriteTextOverlayPrivate *priv = self->priv;
GstVideoCaptionMeta *cc_meta;
if ((*meta)->info->api != GST_VIDEO_CAPTION_META_API_TYPE)
return TRUE;
cc_meta = (GstVideoCaptionMeta *) (*meta);
switch (cc_meta->caption_type) {
case GST_VIDEO_CAPTION_TYPE_CEA608_RAW:
gst_dwrite_text_overlay_decode_raw (self, cc_meta->data, cc_meta->size,
priv->running_time);
break;
case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A:
gst_dwrite_text_overlay_decode_s334_1a (self, cc_meta->data,
cc_meta->size, priv->running_time);
break;
case GST_VIDEO_CAPTION_TYPE_CEA708_RAW:
gst_dwrite_text_overlay_decode_cc_data (self, cc_meta->data,
cc_meta->size, priv->running_time);
break;
case GST_VIDEO_CAPTION_TYPE_CEA708_CDP:
{
guint len, pos = 0;
len = gst_dwrite_text_overlay_extract_cdp (self, cc_meta->data,
cc_meta->size, &pos);
if (len > 0) {
gst_dwrite_text_overlay_decode_cc_data (self, cc_meta->data + pos,
len, priv->running_time);
if (priv->enable_cc && (*meta)->info->api == GST_VIDEO_CAPTION_META_API_TYPE) {
cc_meta = (GstVideoCaptionMeta *) (*meta);
switch (cc_meta->caption_type) {
case GST_VIDEO_CAPTION_TYPE_CEA608_RAW:
gst_dwrite_text_overlay_decode_raw (self, cc_meta->data, cc_meta->size,
priv->running_time);
break;
case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A:
gst_dwrite_text_overlay_decode_s334_1a (self, cc_meta->data,
cc_meta->size, priv->running_time);
break;
case GST_VIDEO_CAPTION_TYPE_CEA708_RAW:
gst_dwrite_text_overlay_decode_cc_data (self, cc_meta->data,
cc_meta->size, priv->running_time);
break;
case GST_VIDEO_CAPTION_TYPE_CEA708_CDP:
{
guint len, pos = 0;
len = gst_dwrite_text_overlay_extract_cdp (self, cc_meta->data,
cc_meta->size, &pos);
if (len > 0) {
gst_dwrite_text_overlay_decode_cc_data (self, cc_meta->data + pos,
len, priv->running_time);
}
break;
}
break;
default:
break;
}
default:
break;
} else if ((*meta)->info->api == GST_DWRITE_SUBTITLE_META_API_TYPE) {
GstDWriteSubtitleMeta *smeta = (GstDWriteSubtitleMeta *) (*meta);
gst_dwrite_text_overlay_extract_meta (self, smeta);
}
return TRUE;
@ -538,14 +632,17 @@ gst_dwrite_text_overlay_get_text (GstDWriteBaseOverlay * overlay,
GstDWriteTextOverlay *self = GST_DWRITE_TEXT_OVERLAY (overlay);
GstDWriteTextOverlayPrivate *priv = self->priv;
std::lock_guard < std::mutex > lk (priv->lock);
WString text_wide;
priv->text.clear ();
priv->running_time = gst_segment_to_running_time (&trans->segment,
GST_FORMAT_TIME, GST_BUFFER_PTS (buffer));
if (priv->enable_cc) {
gst_buffer_foreach_meta (buffer,
(GstBufferForeachMetaFunc) gst_dwrite_text_overlay_foreach_meta, self);
gst_buffer_foreach_meta (buffer,
(GstBufferForeachMetaFunc) gst_dwrite_text_overlay_foreach_meta, self);
if (priv->enable_cc) {
if (GST_CLOCK_TIME_IS_VALID (priv->timeout) &&
GST_CLOCK_TIME_IS_VALID (priv->running_time) &&
GST_CLOCK_TIME_IS_VALID (priv->caption_running_time) &&
@ -561,10 +658,19 @@ gst_dwrite_text_overlay_get_text (GstDWriteBaseOverlay * overlay,
priv->closed_caption.clear ();
}
if (priv->closed_caption.empty ())
if (priv->closed_caption.empty () && priv->text.empty ())
return default_text;
auto text_wide = gst_dwrite_string_to_wstring (priv->closed_caption);
if (!priv->text.empty ())
text_wide = gst_dwrite_string_to_wstring (priv->text);
if (!priv->closed_caption.empty ()) {
if (!text_wide.empty ())
text_wide += L"\n";
text_wide += gst_dwrite_string_to_wstring (priv->closed_caption);
}
if (default_text.empty ())
return text_wide;
@ -575,11 +681,15 @@ static gboolean
gst_dwrite_text_overlay_remove_meta (GstBuffer * buffer, GstMeta ** meta,
GstDWriteTextOverlay * self)
{
if ((*meta)->info->api != GST_VIDEO_CAPTION_META_API_TYPE)
return TRUE;
GstDWriteTextOverlayPrivate *priv = self->priv;
GST_TRACE_OBJECT (self, "Removing caption meta");
*meta = nullptr;
if ((*meta)->info->api == GST_VIDEO_CAPTION_META_API_TYPE &&
priv->enable_cc && priv->remove_caption_meta) {
GST_TRACE_OBJECT (self, "Removing caption meta");
*meta = nullptr;
} else if ((*meta)->info->api == GST_DWRITE_SUBTITLE_META_API_TYPE) {
*meta = nullptr;
}
return TRUE;
}
@ -592,9 +702,29 @@ gst_dwrite_text_overlay_after_transform (GstDWriteBaseOverlay * overlay,
GstDWriteTextOverlayPrivate *priv = self->priv;
std::lock_guard < std::mutex > lk (priv->lock);
if (!priv->enable_cc || !priv->remove_caption_meta)
return;
gst_buffer_foreach_meta (buffer,
(GstBufferForeachMetaFunc) gst_dwrite_text_overlay_remove_meta, self);
}
void
gst_dwrite_text_overlay_build_param_specs (std::vector < GParamSpec * >&pspec)
{
pspec.push_back (g_param_spec_boolean ("enable-cc", "Enable CC",
"Enable closed caption rendering",
DEFAULT_ENABLE_CC,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_int ("cc-field", "CC Field",
"The closed caption field to render when available, (-1 = automatic)",
-1, 1, DEFAULT_CC_FIELD,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_uint64 ("cc-timeout", "CC Timeout",
"Duration after which to erase overlay when no cc data has arrived "
"for the selected field, in nanoseconds unit", 16 * GST_SECOND,
GST_CLOCK_TIME_NONE, DEFAULT_CC_TIMEOUT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
pspec.push_back (g_param_spec_boolean ("remove-cc-meta", "Remove CC Meta",
"Remove caption meta from output buffers "
"when closed caption rendering is enabled",
DEFAULT_REMOVE_CC_META,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
}

2
subprojects/gst-plugins-bad/sys/dwrite/gstdwritetextoverlay.h
View file

@ -27,4 +27,6 @@ G_BEGIN_DECLS
G_DECLARE_FINAL_TYPE (GstDWriteTextOverlay,
gst_dwrite_text_overlay, GST, DWRITE_TEXT_OVERLAY, GstDWriteBaseOverlay);
void gst_dwrite_text_overlay_build_param_specs (std::vector<GParamSpec *> & pspec);
G_END_DECLS

1
subprojects/gst-plugins-bad/sys/dwrite/meson.build
View file

@ -8,6 +8,7 @@ dwrite_sources = [
'gstdwritebitmappool.cpp',
'gstdwriteclockoverlay.cpp',
'gstdwritesubtitlemux.cpp',
'gstdwritesubtitleoverlay.cpp',
'gstdwritetextoverlay.cpp',
'gstdwritetimeoverlay.cpp',
'plugin.cpp',

3
subprojects/gst-plugins-bad/sys/dwrite/plugin.cpp
View file

@ -31,6 +31,7 @@
#include "gstdwriteclockoverlay.h"
#include "gstdwritetextoverlay.h"
#include "gstdwritetimeoverlay.h"
#include "gstdwritesubtitleoverlay.h"
GST_DEBUG_CATEGORY (gst_dwrite_debug);
@ -47,6 +48,8 @@ plugin_init (GstPlugin * plugin)
GST_TYPE_DWRITE_TEXT_OVERLAY);
gst_element_register (plugin, "dwritetimeoverlay", GST_RANK_NONE,
GST_TYPE_DWRITE_TIME_OVERLAY);
gst_element_register (plugin, "dwritesubtitleoverlay", GST_RANK_NONE,
GST_TYPE_DWRITE_SUBTITLE_OVERLAY);
return TRUE;
}