From 95464c89772e144088af54c1e8a4c1fecc45f09a Mon Sep 17 00:00:00 2001 From: Daniel Morin Date: Tue, 5 Dec 2023 21:11:31 -0500 Subject: [PATCH] analyticsoverlay: add object-detection overlay - Overlay analytics-meta-od attached to video buffer Part-of: --- .../docs/plugins/gst_plugins_cache.json | 81 ++ .../analyticsoverlay/gstanalyticsoverlay.c | 52 + .../gstobjectdetectionoverlay.c | 889 ++++++++++++++++++ .../gstobjectdetectionoverlay.h | 40 + .../ext/analyticsoverlay/meson.build | 30 + subprojects/gst-plugins-bad/ext/meson.build | 1 + subprojects/gst-plugins-bad/meson_options.txt | 1 + 7 files changed, 1094 insertions(+) create mode 100644 subprojects/gst-plugins-bad/ext/analyticsoverlay/gstanalyticsoverlay.c create mode 100644 subprojects/gst-plugins-bad/ext/analyticsoverlay/gstobjectdetectionoverlay.c create mode 100644 subprojects/gst-plugins-bad/ext/analyticsoverlay/gstobjectdetectionoverlay.h create mode 100644 subprojects/gst-plugins-bad/ext/analyticsoverlay/meson.build diff --git a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json index 5f07e2fcb8..6d27e43662 100644 --- a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json +++ b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json @@ -2355,6 +2355,87 @@ "tracers": {}, "url": "Unknown package origin" }, + "analyticsoverlay": { + "description": "Analytics-meta overlay elements", + "elements": { + "objectdetectionoverlay": { + "author": "Daniel Morin", + "description": "Overlay a visual representation of analytics metadata on the video", + "hierarchy": [ + "GstObjectDetectionOverlay", + "GstVideoFilter", + "GstBaseTransform", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Analyzer/Visualization/Video", + "pad-templates": { + "sink": { + "caps": "video/x-raw:\n format: { A444_16LE, A444_16BE, AYUV64, RGBA64_LE, ARGB64, ARGB64_LE, BGRA64_LE, ABGR64_LE, RGBA64_BE, ARGB64_BE, BGRA64_BE, ABGR64_BE, A422_16LE, A422_16BE, A420_16LE, A420_16BE, A444_12LE, GBRA_12LE, A444_12BE, GBRA_12BE, Y412_LE, Y412_BE, A422_12LE, A422_12BE, A420_12LE, A420_12BE, A444_10LE, GBRA_10LE, A444_10BE, GBRA_10BE, A422_10LE, A422_10BE, A420_10LE, A420_10BE, BGR10A2_LE, RGB10A2_LE, Y410, A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, A420, AV12, Y444_16LE, GBR_16LE, Y444_16BE, GBR_16BE, v216, P016_LE, P016_BE, Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y212_LE, Y212_BE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, Y444_10BE, GBR_10BE, r210, I422_10LE, I422_10BE, NV16_10LE32, Y210, UYVP, v210, I420_10LE, I420_10BE, P010_10LE, NV12_10LE40, NV12_10LE32, P010_10BE, MT2110R, MT2110T, NV12_10BE_8L128, NV12_10LE40_4L4, Y444, BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, BGR, Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "video/x-raw:\n format: { A444_16LE, A444_16BE, AYUV64, RGBA64_LE, ARGB64, ARGB64_LE, BGRA64_LE, ABGR64_LE, RGBA64_BE, ARGB64_BE, BGRA64_BE, ABGR64_BE, A422_16LE, A422_16BE, A420_16LE, A420_16BE, A444_12LE, GBRA_12LE, A444_12BE, GBRA_12BE, Y412_LE, Y412_BE, A422_12LE, A422_12BE, A420_12LE, A420_12BE, A444_10LE, GBRA_10LE, A444_10BE, GBRA_10BE, A422_10LE, A422_10BE, A420_10LE, A420_10BE, BGR10A2_LE, RGB10A2_LE, Y410, A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, A420, AV12, Y444_16LE, GBR_16LE, Y444_16BE, GBR_16BE, v216, P016_LE, P016_BE, Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y212_LE, Y212_BE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, Y444_10BE, GBR_10BE, r210, I422_10LE, I422_10BE, NV16_10LE32, Y210, UYVP, v210, I420_10LE, I420_10BE, P010_10LE, NV12_10LE40, NV12_10LE32, P010_10BE, MT2110R, MT2110T, NV12_10BE_8L128, NV12_10LE40_4L4, Y444, BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, BGR, Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "direction": "src", + "presence": "always" + } + }, + "properties": { + "draw-labels": { + "blurb": "Draw object labels", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "true", + "mutable": "null", + "readable": true, + "type": "gboolean", + "writable": true + }, + "labels-color": { + "blurb": "Color (ARGB) to use for object labels", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "-1", + "min": "0", + "mutable": "null", + "readable": true, + "type": "guint", + "writable": true + }, + "object-detection-outline-color": { + "blurb": "Color (ARGB) to use for object detection overlay outline", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "-1", + "min": "0", + "mutable": "null", + "readable": true, + "type": "guint", + "writable": true + } + }, + "rank": "none" + } + }, + "filename": "gstanalyticsoverlay", + "license": "LGPL", + "other-types": {}, + "package": "GStreamer Bad Plug-ins", + "source": "gst-plugins-bad", + "tracers": {}, + "url": "Unknown package origin" + }, "aom": { "description": "AOM plugin library", "elements": { diff --git a/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstanalyticsoverlay.c b/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstanalyticsoverlay.c new file mode 100644 index 0000000000..dc24d46929 --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstanalyticsoverlay.c @@ -0,0 +1,52 @@ +/* GStreamer object detection overlay + * Copyright (C) <2022> Collabora Ltd. + * @author: Daniel Morin + * + * gstanalyticsoverlay.c + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "gstobjectdetectionoverlay.h" + +/** + * SECTION:plugin-analyticsoverlay + * + * Analytics overlay + * + * See also: @objectdetectionoverlay + * Since: 1.24 + */ + +static gboolean +plugin_init (GstPlugin * plugin) +{ + gboolean ret = FALSE; + + ret |= GST_ELEMENT_REGISTER (objectdetectionoverlay, plugin); + + return ret; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + analyticsoverlay, + "Analytics-meta overlay elements", + plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstobjectdetectionoverlay.c b/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstobjectdetectionoverlay.c new file mode 100644 index 0000000000..a8999a3be9 --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstobjectdetectionoverlay.c @@ -0,0 +1,889 @@ +/* GStreamer object detection overlay + * Copyright (C) <2023> Collabora Ltd. + * @author: Aaron Boxer + * @author: Daniel Morin + * + * gstobjectdetectionoverlay.c + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-objectdetectionoverlay + * @title: objectdetectionoverlay + * @see_also: #GstObjectDetectionOverlay + * + * This element create a graphical representation of the analytics object + * detection metadata attached to video stream and overlay graphics above the + * video. + * + * The object detection overlay element monitor video stream for + * @GstAnalyticsRelationMeta and query @GstAnalyticsODMtd. Retrieved + * @GstAnalyticsODMtd are then used to generate an overlay highlighing objects + * detected. + * + * ## Example launch line + * |[ + * gst-launch-1.0 v4l2src device=/dev/video0 ! videoconvertscale add-borders=1 ! 'video/x-raw,width=640,height=383,framerate=2/1' ! onnxobjectdetector execution-provider=cpu model-file=ssd_mobilenet_v1_coco.onnx ! objectdetectionoverlay label-file=COCO_classes.txt ! videoconvertscale ! autovideosink + * ]| This pipeline create an overlay representing results of an object detetion + * analysis. + * + * Since: 1.24 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "gstobjectdetectionoverlay.h" + +struct _GstObjectDetectionOverlay +{ + GstVideoFilter parent; + + cairo_matrix_t cairo_matrix; + gsize render_len; + + /* stream metrics */ + GstVideoInfo *in_info; + GMutex stream_event_mutex; + gboolean flushing; + gboolean eos; + + /* properties */ + guint od_outline_color; + guint od_outline_stroke_width; + gboolean draw_labels; + guint labels_color; + gdouble labels_stroke_width; + gdouble labels_outline_ofs; + + /* composition */ + gboolean attach_compo_to_buffer; + GstBuffer *canvas; + gint canvas_length; + GstVideoOverlayComposition *composition; + GstVideoOverlayComposition *upstream_composition; + + /* Graphic Outline */ + PangoContext *pango_context; + PangoLayout *pango_layout; + +}; + + +#define MINIMUM_TEXT_OUTLINE_OFFSET 1.0 + +GST_DEBUG_CATEGORY_STATIC (objectdetectionoverlay_debug); +#define GST_CAT_DEFAULT objectdetectionoverlay_debug + +enum +{ + PROP_OD_OUTLINE_COLOR = 1, + PROP_DRAW_LABELS, + PROP_LABELS_COLOR, + _PROP_COUNT +}; + +typedef struct _GstObjectDetectionOverlayPangoCairoContext + GstObjectDetectionOverlayPangoCairoContext; + +struct _GstObjectDetectionOverlayPangoCairoContext +{ + cairo_t *cr; + cairo_surface_t *surface; + guint8 *data; + cairo_matrix_t *cairo_matrix; +}; + +#define VIDEO_FORMATS GST_VIDEO_OVERLAY_COMPOSITION_BLEND_FORMATS +#define OBJECT_DETECTION_OVERLAY_CAPS GST_VIDEO_CAPS_MAKE (VIDEO_FORMATS) + +static GstStaticCaps sw_template_caps = +GST_STATIC_CAPS (OBJECT_DETECTION_OVERLAY_CAPS); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (OBJECT_DETECTION_OVERLAY_CAPS) + ); + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (OBJECT_DETECTION_OVERLAY_CAPS) + ); + +G_DEFINE_TYPE (GstObjectDetectionOverlay, + gst_object_detection_overlay, GST_TYPE_VIDEO_FILTER); + +#define parent_class gst_object_detection_overlay_parent_class + +GST_ELEMENT_REGISTER_DEFINE (objectdetectionoverlay, "objectdetectionoverlay", + GST_RANK_NONE, GST_TYPE_OBJECT_DETECTION_OVERLAY); + +static void gst_object_detection_overlay_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); + +static void gst_object_detection_overlay_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); + +static gboolean gst_object_detection_overlay_sink_event (GstBaseTransform * + trans, GstEvent * event); + +static gboolean gst_object_detection_overlay_start (GstBaseTransform * trans); +static gboolean gst_object_detection_overlay_stop (GstBaseTransform * trans); + + +static gboolean gst_object_detection_overlay_set_info (GstVideoFilter * filter, + GstCaps * incaps, GstVideoInfo * in_info, GstCaps * outcaps, + GstVideoInfo * out_info); + +static GstFlowReturn +gst_object_detection_overlay_transform_frame_ip (GstVideoFilter * filter, + GstVideoFrame * buf); + +static void gst_object_detection_overlay_finalize (GObject * object); + +static void +gst_object_detection_overlay_render_boundingbox (GstObjectDetectionOverlay + * overlay, GstObjectDetectionOverlayPangoCairoContext * cairo_ctx, + GstAnalyticsODMtd * od_mtd); + +static void +gst_object_detection_overlay_render_text_annotation (GstObjectDetectionOverlay + * overlay, GstObjectDetectionOverlayPangoCairoContext * cairo_ctx, + GstAnalyticsODMtd * od_mtd, const gchar * annotation); + +static void +gst_object_detection_overlay_class_init (GstObjectDetectionOverlayClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *element_class; + GstBaseTransformClass *basetransform_class; + GstVideoFilterClass *videofilter_class; + + gobject_class = (GObjectClass *) klass; + gobject_class->set_property = gst_object_detection_overlay_set_property; + gobject_class->get_property = gst_object_detection_overlay_get_property; + gobject_class->finalize = gst_object_detection_overlay_finalize; + + /** + * GstObjectDetectionOverlay:object-detection-outline-color + * + * Object Detetion Overlay outline color + * ARGB format (ex. 0xFFFF0000 for red) + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_OD_OUTLINE_COLOR, + g_param_spec_uint ("object-detection-outline-color", + "Object detection outline color", + "Color (ARGB) to use for object detection overlay outline", + 0, G_MAXUINT, 0xFFFFFFFF, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + /** + * GstObjectDetectionOverlay:draw-labels + * + * Control labels drawing + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_DRAW_LABELS, + g_param_spec_boolean ("draw-labels", + "Draw labels", + "Draw object labels", + TRUE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + /** + * GstObjectDetectionOverlay:labels-color + * + * Control labels color + * Format ARGB (ex. 0xFFFF0000 for red) + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_LABELS_COLOR, + g_param_spec_uint ("labels-color", + "Labels color", + "Color (ARGB) to use for object labels", + 0, G_MAXUINT, 0xFFFFFF, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + element_class = (GstElementClass *) klass; + + gst_element_class_add_static_pad_template (element_class, &sink_template); + gst_element_class_add_static_pad_template (element_class, &src_template); + gst_element_class_set_static_metadata (element_class, + "Object Detection Overlay", + "Analyzer/Visualization/Video", + "Overlay a visual representation of analytics metadata on the video", + "Daniel Morin"); + + basetransform_class = (GstBaseTransformClass *) klass; + basetransform_class->passthrough_on_same_caps = FALSE; + basetransform_class->start = + GST_DEBUG_FUNCPTR (gst_object_detection_overlay_start); + + basetransform_class->stop = + GST_DEBUG_FUNCPTR (gst_object_detection_overlay_stop); + + basetransform_class->sink_event = + GST_DEBUG_FUNCPTR (gst_object_detection_overlay_sink_event); + + videofilter_class = (GstVideoFilterClass *) klass; + videofilter_class->set_info = + GST_DEBUG_FUNCPTR (gst_object_detection_overlay_set_info); + videofilter_class->transform_frame_ip = + GST_DEBUG_FUNCPTR (gst_object_detection_overlay_transform_frame_ip); +} + +static void +gst_object_detection_overlay_adj_labels_outline_ofs (GstObjectDetectionOverlay * + overlay, PangoFontDescription * desc) +{ + gint font_size = pango_font_description_get_size (desc) / PANGO_SCALE; + overlay->labels_outline_ofs = (double) (font_size) / 15.0; + if (overlay->labels_outline_ofs < MINIMUM_TEXT_OUTLINE_OFFSET) + overlay->labels_outline_ofs = MINIMUM_TEXT_OUTLINE_OFFSET; +} + +static void +gst_object_detection_overlay_finalize (GObject * object) +{ + gst_object_detection_overlay_stop (GST_BASE_TRANSFORM (object)); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_object_detection_overlay_init (GstObjectDetectionOverlay * overlay) +{ + overlay->pango_context = NULL; + overlay->pango_layout = NULL; + overlay->od_outline_color = 0xFFFFFFFF; + overlay->draw_labels = TRUE; + overlay->labels_color = 0xFFFFFFFF; + overlay->in_info = &GST_VIDEO_FILTER (overlay)->in_info; + overlay->attach_compo_to_buffer = TRUE; + overlay->canvas = NULL; + overlay->labels_stroke_width = 1.0; + overlay->od_outline_stroke_width = 2; + overlay->composition = NULL; + overlay->upstream_composition = NULL; + overlay->flushing = FALSE; + GST_DEBUG_CATEGORY_INIT (objectdetectionoverlay_debug, + "analytics_overlay_od", 0, "Object detection overlay"); +} + +static void +gst_object_detection_overlay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstObjectDetectionOverlay *overlay; + overlay = GST_OBJECT_DETECTION_OVERLAY (object); + + switch (prop_id) { + case PROP_OD_OUTLINE_COLOR: + overlay->od_outline_color = g_value_get_uint (value); + break; + case PROP_DRAW_LABELS: + overlay->draw_labels = g_value_get_boolean (value); + break; + case PROP_LABELS_COLOR: + overlay->labels_color = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_object_detection_overlay_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstObjectDetectionOverlay *od_overlay = GST_OBJECT_DETECTION_OVERLAY (object); + + switch (prop_id) { + case PROP_OD_OUTLINE_COLOR: + g_value_set_uint (value, od_overlay->od_outline_color); + break; + case PROP_DRAW_LABELS: + g_value_set_boolean (value, od_overlay->draw_labels); + break; + case PROP_LABELS_COLOR: + g_value_set_uint (value, od_overlay->labels_color); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_object_detection_overlay_can_handle_caps (GstCaps * incaps) +{ + gboolean ret; + GstCaps *caps; + + caps = gst_static_caps_get (&sw_template_caps); + ret = gst_caps_is_subset (incaps, caps); + gst_caps_unref (caps); + + return ret; +} + +static gboolean +gst_object_detection_overlay_negotiate (GstObjectDetectionOverlay * overlay, + GstCaps * caps) +{ + GstBaseTransform *basetransform = GST_BASE_TRANSFORM (overlay); + gboolean upstream_has_meta = FALSE; + gboolean caps_has_meta = FALSE; + gboolean alloc_has_meta = FALSE; + gboolean attach = FALSE; + gboolean ret = TRUE; + guint width, height; + GstCapsFeatures *f; + GstCaps *overlay_caps; + GstQuery *query; + guint alloc_index; + GstPad *srcpad = basetransform->srcpad; + GstPad *sinkpad = basetransform->sinkpad; + + GST_DEBUG_OBJECT (overlay, "performing negotiation"); + + /* Clear any pending reconfigure to avoid negotiating twice */ + gst_pad_check_reconfigure (sinkpad); + + /* Check if upstream caps have meta */ + if ((f = gst_caps_get_features (caps, 0))) { + GST_DEBUG_OBJECT (overlay, "upstream has caps"); + upstream_has_meta = gst_caps_features_contains (f, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + } + + /* Initialize dimensions */ + width = GST_VIDEO_INFO_WIDTH (overlay->in_info); + height = GST_VIDEO_INFO_HEIGHT (overlay->in_info); + GST_DEBUG_OBJECT (overlay, "initial dims: %ux%u", width, height); + + if (upstream_has_meta) { + overlay_caps = gst_caps_ref (caps); + } else { + GstCaps *peercaps; + + /* BaseTransform requires caps for the allocation query to work */ + overlay_caps = gst_caps_copy (caps); + f = gst_caps_get_features (overlay_caps, 0); + gst_caps_features_add (f, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + + /* Then check if downstream accept overlay composition in caps */ + /* FIXME: We should probably check if downstream *prefers* the + * overlay meta, and only enforce usage of it if we can't handle + * the format ourselves and thus would have to drop the overlays. + * Otherwise we should prefer what downstream wants here. + */ + peercaps = gst_pad_peer_query_caps (srcpad, overlay_caps); + caps_has_meta = !gst_caps_is_empty (peercaps); + gst_caps_unref (peercaps); + + GST_DEBUG_OBJECT (overlay, "caps have overlay meta %d", caps_has_meta); + } + + if (upstream_has_meta || caps_has_meta) { + /* Send caps immediately, it's needed by GstBaseTransform to get a reply + * from allocation query */ + GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (basetransform, caps, + overlay_caps); + ret = gst_pad_set_caps (srcpad, overlay_caps); + + /* First check if the allocation meta has compositon */ + query = gst_query_new_allocation (overlay_caps, FALSE); + + if (!gst_pad_peer_query (srcpad, query)) { + /* no problem, we use the query defaults */ + GST_DEBUG_OBJECT (overlay, "ALLOCATION query failed"); + + /* In case we were flushing, mark reconfigure and fail this method, + * will make it retry */ + if (overlay->flushing) + ret = FALSE; + } + + alloc_has_meta = gst_query_find_allocation_meta (query, + GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, &alloc_index); + + GST_DEBUG_OBJECT (overlay, "sink alloc has overlay meta %d", + alloc_has_meta); + + if (alloc_has_meta) { + const GstStructure *params; + + gst_query_parse_nth_allocation_meta (query, alloc_index, ¶ms); + if (params) { + if (gst_structure_get (params, "width", G_TYPE_UINT, &width, + "height", G_TYPE_UINT, &height, NULL)) { + GST_DEBUG_OBJECT (overlay, "received window size: %dx%d", width, + height); + g_assert (width != 0 && height != 0); + } + } + } + + gst_query_unref (query); + } + + /* Update render size if needed */ + overlay->canvas_length = width * height; + + /* For backward compatibility, we will prefer blitting if downstream + * allocation does not support the meta. In other case we will prefer + * attaching, and will fail the negotiation in the unlikely case we are + * force to blit, but format isn't supported. */ + + if (upstream_has_meta) { + attach = TRUE; + } else if (caps_has_meta) { + if (alloc_has_meta) { + attach = TRUE; + } else { + /* Don't attach unless we cannot handle the format */ + attach = !gst_object_detection_overlay_can_handle_caps (caps); + } + } else { + ret = gst_object_detection_overlay_can_handle_caps (caps); + } + + /* If we attach, then pick the overlay caps */ + if (attach) { + GST_DEBUG_OBJECT (overlay, "Using caps %" GST_PTR_FORMAT, overlay_caps); + /* Caps where already sent */ + } else if (ret) { + GST_DEBUG_OBJECT (overlay, "Using caps %" GST_PTR_FORMAT, caps); + GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (basetransform, caps, + caps); + ret = gst_pad_set_caps (srcpad, caps); + } + + overlay->attach_compo_to_buffer = attach; + + if (attach) { + GST_BASE_TRANSFORM_CLASS (parent_class)->passthrough_on_same_caps = FALSE; + } + + if (!ret) { + GST_DEBUG_OBJECT (overlay, "negotiation failed, schedule reconfigure"); + gst_pad_mark_reconfigure (srcpad); + } + + gst_caps_unref (overlay_caps); + + return ret; +} + +static gboolean +gst_object_detection_overlay_setcaps (GstObjectDetectionOverlay * overlay, + GstCaps * caps) +{ + gboolean ret = FALSE; + + if (!gst_video_info_from_caps (overlay->in_info, caps)) + goto invalid_caps; + + ret = gst_object_detection_overlay_negotiate (overlay, caps); + GST_VIDEO_FILTER (overlay)->negotiated = ret; + + if (!overlay->attach_compo_to_buffer && + !gst_object_detection_overlay_can_handle_caps (caps)) { + GST_DEBUG_OBJECT (overlay, "unsupported caps %" GST_PTR_FORMAT, caps); + ret = FALSE; + } + + return ret; + + /* ERRORS */ +invalid_caps: + { + GST_DEBUG_OBJECT (overlay, "could not parse caps"); + return FALSE; + } +} + +static gboolean +gst_object_detection_overlay_sink_event (GstBaseTransform * trans, + GstEvent * event) +{ + gboolean ret = FALSE; + GST_DEBUG_OBJECT (trans, "received sink event %s", + GST_EVENT_TYPE_NAME (event)); + + GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (trans); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_CAPS: + { + GstCaps *caps; + gst_event_parse_caps (event, &caps); + ret = gst_object_detection_overlay_setcaps (overlay, caps); + gst_event_unref (event); + break; + } + case GST_EVENT_EOS: + g_mutex_lock (&overlay->stream_event_mutex); + GST_INFO_OBJECT (overlay, "EOS"); + overlay->eos = TRUE; + g_mutex_unlock (&overlay->stream_event_mutex); + break; + case GST_EVENT_FLUSH_START: + g_mutex_lock (&overlay->stream_event_mutex); + GST_INFO_OBJECT (overlay, "Flush stop"); + overlay->flushing = TRUE; + g_mutex_unlock (&overlay->stream_event_mutex); + break; + case GST_EVENT_FLUSH_STOP: + g_mutex_lock (&overlay->stream_event_mutex); + GST_INFO_OBJECT (overlay, "Flush stop"); + overlay->eos = FALSE; + overlay->flushing = FALSE; + g_mutex_unlock (&overlay->stream_event_mutex); + break; + default: + ret = GST_BASE_TRANSFORM_CLASS (parent_class)->sink_event (trans, event); + break; + } + + return ret; +} + +static gboolean +gst_object_detection_overlay_start (GstBaseTransform * trans) +{ + GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (trans); + PangoFontDescription *desc; + PangoFontMap *fontmap; + + fontmap = pango_cairo_font_map_new (); + overlay->pango_context = + pango_font_map_create_context (PANGO_FONT_MAP (fontmap)); + g_object_unref (fontmap); + overlay->pango_layout = pango_layout_new (overlay->pango_context); + desc = pango_context_get_font_description (overlay->pango_context); + pango_font_description_set_size (desc, 10000); + pango_font_description_set_weight (desc, PANGO_WEIGHT_ULTRALIGHT); + pango_context_set_font_description (overlay->pango_context, desc); + pango_layout_set_alignment (overlay->pango_layout, PANGO_ALIGN_LEFT); + + gst_object_detection_overlay_adj_labels_outline_ofs (overlay, desc); + GST_DEBUG_OBJECT (overlay, "labels_outline_offset %f", + overlay->labels_outline_ofs); + + return TRUE; +} + +static gboolean +gst_object_detection_overlay_stop (GstBaseTransform * trans) +{ + GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (trans); + + g_clear_object (&overlay->pango_layout); + g_clear_object (&overlay->pango_context); + gst_clear_buffer (&overlay->canvas); + + return TRUE; +} + +static gboolean +gst_object_detection_overlay_set_info (GstVideoFilter * filter, + GstCaps * incaps, GstVideoInfo * in_info, GstCaps * outcaps, + GstVideoInfo * out_info) +{ + GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (filter); + GST_DEBUG_OBJECT (filter, "set_info incaps:%s", gst_caps_to_string (incaps)); + GST_DEBUG_OBJECT (filter, "set_info outcaps:%s", + gst_caps_to_string (outcaps)); + + filter->in_info = *in_info; + filter->out_info = *out_info; + + cairo_matrix_init_scale (&overlay->cairo_matrix, 1, 1); + overlay->render_len = GST_VIDEO_INFO_WIDTH (in_info) * + GST_VIDEO_INFO_HEIGHT (in_info) * 4; + return TRUE; +} + +static void +gst_object_detection_overlay_create_cairo_context (GstObjectDetectionOverlay * + overlay, GstObjectDetectionOverlayPangoCairoContext * cairo_ctx, + guint8 * data) +{ + cairo_ctx->cairo_matrix = &overlay->cairo_matrix; + cairo_ctx->surface = cairo_image_surface_create_for_data (data, + CAIRO_FORMAT_ARGB32, GST_VIDEO_INFO_WIDTH (overlay->in_info), + GST_VIDEO_INFO_HEIGHT (overlay->in_info), + GST_VIDEO_INFO_WIDTH (overlay->in_info) * 4); + cairo_ctx->cr = cairo_create (cairo_ctx->surface); + + /* clear surface */ + cairo_set_operator (cairo_ctx->cr, CAIRO_OPERATOR_CLEAR); + cairo_paint (cairo_ctx->cr); + cairo_set_operator (cairo_ctx->cr, CAIRO_OPERATOR_OVER); + + /* apply transformations */ + cairo_set_matrix (cairo_ctx->cr, cairo_ctx->cairo_matrix); + cairo_save (cairo_ctx->cr); +} + +static void + gst_object_detection_overlay_destroy_cairo_context + (GstObjectDetectionOverlayPangoCairoContext * cairo_ctx) +{ + cairo_restore (cairo_ctx->cr); + cairo_destroy (cairo_ctx->cr); + cairo_surface_destroy (cairo_ctx->surface); +} + +static GstFlowReturn +gst_object_detection_overlay_transform_frame_ip (GstVideoFilter * filter, + GstVideoFrame * frame) +{ + GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (filter); + GstVideoOverlayCompositionMeta *composition_meta; + gpointer state = NULL; + GstVideoOverlayRectangle *rectangle = NULL; + gchar str_buf[5]; + GstAnalyticsMtd rlt_mtd; + GstAnalyticsODMtd *od_mtd; + gint x, y, w, h; + gfloat loc_confi_lvl; + gboolean success; + + GST_DEBUG_OBJECT (filter, "buffer writeable=%d", + gst_buffer_is_writable (frame->buffer)); + + g_mutex_lock (&overlay->stream_event_mutex); + if (overlay->eos || overlay->flushing) { + g_mutex_unlock (&overlay->stream_event_mutex); + return GST_FLOW_EOS; + } + g_mutex_unlock (&overlay->stream_event_mutex); + + composition_meta = + gst_buffer_get_video_overlay_composition_meta (frame->buffer); + if (composition_meta) { + if (overlay->upstream_composition != composition_meta->overlay) { + GST_DEBUG_OBJECT (overlay, "GstVideoOverlayCompositionMeta found."); + overlay->upstream_composition = composition_meta->overlay; + } + } else if (overlay->upstream_composition != NULL) { + overlay->upstream_composition = NULL; + } + + GstAnalyticsRelationMeta *rmeta = (GstAnalyticsRelationMeta *) + gst_buffer_get_meta (GST_BUFFER (frame->buffer), + GST_ANALYTICS_RELATION_META_API_TYPE); + + if (rmeta) { + GST_DEBUG_OBJECT (filter, "received buffer with analytics relation meta"); + + GstBuffer *buffer; + GstMapInfo map; + GstObjectDetectionOverlayPangoCairoContext cairo_ctx; + + buffer = gst_buffer_new_and_alloc (overlay->render_len); + gst_buffer_add_video_meta (buffer, + GST_VIDEO_FRAME_FLAG_NONE, GST_VIDEO_OVERLAY_COMPOSITION_FORMAT_RGB, + GST_VIDEO_INFO_WIDTH (overlay->in_info), + GST_VIDEO_INFO_HEIGHT (overlay->in_info)); + + gst_buffer_replace (&overlay->canvas, buffer); + gst_buffer_unref (buffer); + + gst_buffer_map (buffer, &map, GST_MAP_READWRITE); + memset (map.data, 0, overlay->render_len); + + gst_object_detection_overlay_create_cairo_context (overlay, + &cairo_ctx, map.data); + + if (overlay->composition) + gst_video_overlay_composition_unref (overlay->composition); + + if (overlay->upstream_composition) { + overlay->composition = + gst_video_overlay_composition_copy (overlay->upstream_composition); + } else { + overlay->composition = gst_video_overlay_composition_new (NULL); + } + + /* Get quark represent object detection metadata type */ + GstAnalyticsMtdType rlt_type = gst_analytics_od_mtd_get_mtd_type (); + while (gst_analytics_relation_meta_iterate (rmeta, &state, rlt_type, + &rlt_mtd)) { + od_mtd = (GstAnalyticsODMtd *) & rlt_mtd; + GST_DEBUG_OBJECT (filter, "buffer contain OD mtd"); + + /* Quark representing the type of the object detected by OD */ + GQuark od_obj_type = gst_analytics_od_mtd_get_obj_type (od_mtd); + + // Find classification metadata attached to object detection metadata + GstAnalyticsMtd cls_rlt_mtd; + success = gst_analytics_relation_meta_get_direct_related (rmeta, + gst_analytics_mtd_get_id ( + (GstAnalyticsMtd *) od_mtd), + GST_ANALYTICS_REL_TYPE_RELATE_TO, + gst_analytics_cls_mtd_get_mtd_type (), NULL, &cls_rlt_mtd); + + gst_object_detection_overlay_render_boundingbox + (GST_OBJECT_DETECTION_OVERLAY (filter), &cairo_ctx, od_mtd); + + if (overlay->draw_labels) { + if (success) { + /* Use associated classification analytics-meta */ + g_snprintf (str_buf, sizeof (str_buf), "%04.2f", + gst_analytics_cls_mtd_get_level ( + (GstAnalyticsClsMtd *) & cls_rlt_mtd, 0)); + + od_obj_type = gst_analytics_cls_mtd_get_quark (&cls_rlt_mtd, 0); + } else { + /* Use basic class type directly on OD. + * Here we want the confidence level of the bbox but to retrieve + * we need to also retrieve the bbox location. */ + gst_analytics_od_mtd_get_location (od_mtd, &x, &y, &w, &h, + &loc_confi_lvl); + GST_TRACE_OBJECT (filter, "obj {type: %s loc:[(%u,%u)-(%ux%u)] @ %f}", + g_quark_to_string (od_obj_type), x, y, w, h, loc_confi_lvl); + + g_snprintf (str_buf, sizeof (str_buf), "%04.2f", loc_confi_lvl); + } + gchar *text = g_strdup_printf ("%s (c=%s)", + g_quark_to_string (od_obj_type), str_buf); + + gst_object_detection_overlay_render_text_annotation + (GST_OBJECT_DETECTION_OVERLAY (filter), &cairo_ctx, od_mtd, text); + + g_free (text); + } + } + + rectangle = gst_video_overlay_rectangle_new_raw (overlay->canvas, + 0, 0, GST_VIDEO_INFO_WIDTH (overlay->in_info), + GST_VIDEO_INFO_HEIGHT (overlay->in_info), + GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA); + gst_video_overlay_composition_add_rectangle (overlay->composition, + rectangle); + gst_video_overlay_rectangle_unref (rectangle); + + gst_object_detection_overlay_destroy_cairo_context (&cairo_ctx); + gst_buffer_unmap (buffer, &map); + + } + + if (overlay->composition) { + GST_DEBUG_OBJECT (filter, "have composition"); + + if (overlay->attach_compo_to_buffer) { + GST_DEBUG_OBJECT (filter, "attach"); + + gst_buffer_add_video_overlay_composition_meta (frame->buffer, + overlay->composition); + } else { + gst_video_overlay_composition_blend (overlay->composition, frame); + } + } + + return GST_FLOW_OK; +} + +static void +gst_object_detection_overlay_render_boundingbox (GstObjectDetectionOverlay + * overlay, GstObjectDetectionOverlayPangoCairoContext * ctx, + GstAnalyticsODMtd * od_mtd) +{ + gint x, y, w, h; + gfloat _dummy; + cairo_save (ctx->cr); + gst_analytics_od_mtd_get_location (od_mtd, &x, &y, &w, &h, &_dummy); + gint maxw = GST_VIDEO_INFO_WIDTH (overlay->in_info) - 1; + gint maxh = GST_VIDEO_INFO_HEIGHT (overlay->in_info) - 1; + + x = CLAMP (x, 0, maxw); + y = CLAMP (y, 0, maxh); + w = CLAMP (w, 0, maxw - x); + h = CLAMP (h, 0, maxh - y); + + /* Set bounding box stroke color and width */ + cairo_set_source_rgba (ctx->cr, + ((overlay->od_outline_color >> 16) & 0xFF) / 255.0, + ((overlay->od_outline_color >> 8) & 0xFF) / 255.0, + ((overlay->od_outline_color) & 0xFF) / 255.0, + ((overlay->od_outline_color >> 24) & 0xFF) / 255.0); + cairo_set_line_width (ctx->cr, overlay->od_outline_stroke_width); + + /* draw bounding box */ + cairo_rectangle (ctx->cr, x, y, w, h); + cairo_stroke (ctx->cr); + cairo_restore (ctx->cr); +} + +static void +gst_object_detection_overlay_render_text_annotation (GstObjectDetectionOverlay + * overlay, GstObjectDetectionOverlayPangoCairoContext * ctx, + GstAnalyticsODMtd * od_mtd, const gchar * annotation) +{ + PangoRectangle ink_rect, logical_rect; + gint x, y, w, h; + gfloat _dummy; + gint maxw = GST_VIDEO_INFO_WIDTH (overlay->in_info) - 1; + gint maxh = GST_VIDEO_INFO_HEIGHT (overlay->in_info) - 1; + + cairo_save (ctx->cr); + gst_analytics_od_mtd_get_location (od_mtd, &x, &y, &w, &h, &_dummy); + + x = CLAMP (x, 0, maxw); + y = CLAMP (y, 0, maxh); + w = CLAMP (w, 0, maxw - x); + h = CLAMP (h, 0, maxh - y); + + /* Set label strokes color and width */ + cairo_set_source_rgba (ctx->cr, + ((overlay->labels_color >> 16) & 0xFF) / 255.0, + ((overlay->labels_color >> 8) & 0xFF) / 255.0, + ((overlay->labels_color) & 0xFF) / 255.0, + ((overlay->labels_color >> 24) & 0xFF) / 255.0); + cairo_set_line_width (ctx->cr, overlay->labels_stroke_width); + + pango_layout_set_markup (overlay->pango_layout, annotation, + strlen (annotation)); + pango_layout_get_pixel_extents (overlay->pango_layout, &ink_rect, + &logical_rect); + GST_DEBUG_OBJECT (overlay, "logical_rect:(%d,%d),%dx%d", logical_rect.x, + logical_rect.y, logical_rect.width, logical_rect.height); + GST_DEBUG_OBJECT (overlay, "ink_rect:(%d,%d),%dx%d", ink_rect.x, ink_rect.y, + ink_rect.width, ink_rect.height); + cairo_move_to (ctx->cr, x + overlay->labels_outline_ofs, + y - logical_rect.height - overlay->labels_outline_ofs); + + pango_cairo_layout_path (ctx->cr, overlay->pango_layout); + cairo_stroke (ctx->cr); + cairo_restore (ctx->cr); +} diff --git a/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstobjectdetectionoverlay.h b/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstobjectdetectionoverlay.h new file mode 100644 index 0000000000..7c4e3a413c --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/analyticsoverlay/gstobjectdetectionoverlay.h @@ -0,0 +1,40 @@ +/* GStreamer object detection overlay + * Copyright (C) <2023> Collabora Ltd. + * @author: Aaron Boxer + * @author: Daniel Morin + * + * gstobjectdetectionoverlay.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_OBJECT_DETECTION_OVERLAY_H__ +#define __GST_OBJECT_DETECTION_OVERLAY_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_OBJECT_DETECTION_OVERLAY \ + (gst_object_detection_overlay_get_type()) + +G_DECLARE_FINAL_TYPE (GstObjectDetectionOverlay, gst_object_detection_overlay, + GST, OBJECT_DETECTION_OVERLAY, GstVideoFilter) + +GST_ELEMENT_REGISTER_DECLARE (objectdetectionoverlay); + +G_END_DECLS +#endif /* __GST_OBJECT_DETECTION_OVERLAY_H__ */ diff --git a/subprojects/gst-plugins-bad/ext/analyticsoverlay/meson.build b/subprojects/gst-plugins-bad/ext/analyticsoverlay/meson.build new file mode 100644 index 0000000000..3b3d5a64f0 --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/analyticsoverlay/meson.build @@ -0,0 +1,30 @@ +if get_option('analyticsoverlay').disabled() + gstanalyticsoverlay_ext_dep = disabler() + subdir_done() +endif + +analyticsoverlay_sources = [ + 'gstanalyticsoverlay.c', + 'gstobjectdetectionoverlay.c', +] + +gstanalyticsoverlay_ext_dep = dependency('pangocairo', version : '>=1.22.0', + required : get_option('analyticsoverlay'), + fallback: ['pango', 'libpangocairo_dep'], + default_options: ['cairo=enabled']) + +if gstanalyticsoverlay_ext_dep.found() + gstanalyticsoverlay = library('gstanalyticsoverlay', + analyticsoverlay_sources, + c_args : gst_plugins_bad_args, + include_directories : [configinc, libsinc], + dependencies : [ + gstbase_dep, gstvideo_dep, gstanalytics_dep, gstanalyticsoverlay_ext_dep + ], + install : true, + install_dir : plugins_install_dir, + ) + plugins += [gstanalyticsoverlay] +else + error('analyticsoverlay plugin is enabled, but dependency is missing') +endif diff --git a/subprojects/gst-plugins-bad/ext/meson.build b/subprojects/gst-plugins-bad/ext/meson.build index 9305493a2a..4f7139835f 100644 --- a/subprojects/gst-plugins-bad/ext/meson.build +++ b/subprojects/gst-plugins-bad/ext/meson.build @@ -1,4 +1,5 @@ subdir('aes') +subdir('analyticsoverlay') subdir('assrender') subdir('aom') subdir('avtp') diff --git a/subprojects/gst-plugins-bad/meson_options.txt b/subprojects/gst-plugins-bad/meson_options.txt index 7154bf840a..5f91f42a3e 100644 --- a/subprojects/gst-plugins-bad/meson_options.txt +++ b/subprojects/gst-plugins-bad/meson_options.txt @@ -6,6 +6,7 @@ option('accurip', type : 'feature', value : 'auto') option('adpcmdec', type : 'feature', value : 'auto') option('adpcmenc', type : 'feature', value : 'auto') option('aiff', type : 'feature', value : 'auto') +option('analyticsoverlay', type:'feature', value : 'auto') option('asfmux', type : 'feature', value : 'auto') option('audiobuffersplit', type : 'feature', value : 'auto') option('audiofxbad', type : 'feature', value : 'auto')