mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2025-01-03 14:08:56 +00:00
analyticsoverlay: add object-detection overlay
- Overlay analytics-meta-od attached to video buffer Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5768>
This commit is contained in:
Daniel Morin
GStreamer Marge Bot
parent
8122f623f6
commit
95464c8977
7 changed files with 1094 additions and 0 deletions
|
|
@ -2355,6 +2355,87 @@
|
||||||
"tracers": {},
|
"tracers": {},
|
||||||
"url": "Unknown package origin"
|
"url": "Unknown package origin"
|
||||||
},
|
},
|
||||||
|
"analyticsoverlay": {
|
||||||
|
"description": "Analytics-meta overlay elements",
|
||||||
|
"elements": {
|
||||||
|
"objectdetectionoverlay": {
|
||||||
|
"author": "Daniel Morin",
|
||||||
|
"description": "Overlay a visual representation of analytics metadata on the video",
|
||||||
|
"hierarchy": [
|
||||||
|
"GstObjectDetectionOverlay",
|
||||||
|
"GstVideoFilter",
|
||||||
|
"GstBaseTransform",
|
||||||
|
"GstElement",
|
||||||
|
"GstObject",
|
||||||
|
"GInitiallyUnowned",
|
||||||
|
"GObject"
|
||||||
|
],
|
||||||
|
"klass": "Analyzer/Visualization/Video",
|
||||||
|
"pad-templates": {
|
||||||
|
"sink": {
|
||||||
|
"caps": "video/x-raw:\n format: { A444_16LE, A444_16BE, AYUV64, RGBA64_LE, ARGB64, ARGB64_LE, BGRA64_LE, ABGR64_LE, RGBA64_BE, ARGB64_BE, BGRA64_BE, ABGR64_BE, A422_16LE, A422_16BE, A420_16LE, A420_16BE, A444_12LE, GBRA_12LE, A444_12BE, GBRA_12BE, Y412_LE, Y412_BE, A422_12LE, A422_12BE, A420_12LE, A420_12BE, A444_10LE, GBRA_10LE, A444_10BE, GBRA_10BE, A422_10LE, A422_10BE, A420_10LE, A420_10BE, BGR10A2_LE, RGB10A2_LE, Y410, A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, A420, AV12, Y444_16LE, GBR_16LE, Y444_16BE, GBR_16BE, v216, P016_LE, P016_BE, Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y212_LE, Y212_BE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, Y444_10BE, GBR_10BE, r210, I422_10LE, I422_10BE, NV16_10LE32, Y210, UYVP, v210, I420_10LE, I420_10BE, P010_10LE, NV12_10LE40, NV12_10LE32, P010_10BE, MT2110R, MT2110T, NV12_10BE_8L128, NV12_10LE40_4L4, Y444, BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, BGR, Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n",
|
||||||
|
"direction": "sink",
|
||||||
|
"presence": "always"
|
||||||
|
},
|
||||||
|
"src": {
|
||||||
|
"caps": "video/x-raw:\n format: { A444_16LE, A444_16BE, AYUV64, RGBA64_LE, ARGB64, ARGB64_LE, BGRA64_LE, ABGR64_LE, RGBA64_BE, ARGB64_BE, BGRA64_BE, ABGR64_BE, A422_16LE, A422_16BE, A420_16LE, A420_16BE, A444_12LE, GBRA_12LE, A444_12BE, GBRA_12BE, Y412_LE, Y412_BE, A422_12LE, A422_12BE, A420_12LE, A420_12BE, A444_10LE, GBRA_10LE, A444_10BE, GBRA_10BE, A422_10LE, A422_10BE, A420_10LE, A420_10BE, BGR10A2_LE, RGB10A2_LE, Y410, A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, A420, AV12, Y444_16LE, GBR_16LE, Y444_16BE, GBR_16BE, v216, P016_LE, P016_BE, Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y212_LE, Y212_BE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, Y444_10BE, GBR_10BE, r210, I422_10LE, I422_10BE, NV16_10LE32, Y210, UYVP, v210, I420_10LE, I420_10BE, P010_10LE, NV12_10LE40, NV12_10LE32, P010_10BE, MT2110R, MT2110T, NV12_10BE_8L128, NV12_10LE40_4L4, Y444, BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, BGR, Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n",
|
||||||
|
"direction": "src",
|
||||||
|
"presence": "always"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"properties": {
|
||||||
|
"draw-labels": {
|
||||||
|
"blurb": "Draw object labels",
|
||||||
|
"conditionally-available": false,
|
||||||
|
"construct": false,
|
||||||
|
"construct-only": false,
|
||||||
|
"controllable": false,
|
||||||
|
"default": "true",
|
||||||
|
"mutable": "null",
|
||||||
|
"readable": true,
|
||||||
|
"type": "gboolean",
|
||||||
|
"writable": true
|
||||||
|
},
|
||||||
|
"labels-color": {
|
||||||
|
"blurb": "Color (ARGB) to use for object labels",
|
||||||
|
"conditionally-available": false,
|
||||||
|
"construct": false,
|
||||||
|
"construct-only": false,
|
||||||
|
"controllable": false,
|
||||||
|
"default": "-1",
|
||||||
|
"max": "-1",
|
||||||
|
"min": "0",
|
||||||
|
"mutable": "null",
|
||||||
|
"readable": true,
|
||||||
|
"type": "guint",
|
||||||
|
"writable": true
|
||||||
|
},
|
||||||
|
"object-detection-outline-color": {
|
||||||
|
"blurb": "Color (ARGB) to use for object detection overlay outline",
|
||||||
|
"conditionally-available": false,
|
||||||
|
"construct": false,
|
||||||
|
"construct-only": false,
|
||||||
|
"controllable": false,
|
||||||
|
"default": "-1",
|
||||||
|
"max": "-1",
|
||||||
|
"min": "0",
|
||||||
|
"mutable": "null",
|
||||||
|
"readable": true,
|
||||||
|
"type": "guint",
|
||||||
|
"writable": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"rank": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"filename": "gstanalyticsoverlay",
|
||||||
|
"license": "LGPL",
|
||||||
|
"other-types": {},
|
||||||
|
"package": "GStreamer Bad Plug-ins",
|
||||||
|
"source": "gst-plugins-bad",
|
||||||
|
"tracers": {},
|
||||||
|
"url": "Unknown package origin"
|
||||||
|
},
|
||||||
"aom": {
|
"aom": {
|
||||||
"description": "AOM plugin library",
|
"description": "AOM plugin library",
|
||||||
"elements": {
|
"elements": {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
/* GStreamer object detection overlay
|
||||||
|
* Copyright (C) <2022> Collabora Ltd.
|
||||||
|
* @author: Daniel Morin <daniel.morin@collabora.com>
|
||||||
|
*
|
||||||
|
* gstanalyticsoverlay.c
|
||||||
|
*
|
||||||
|
* This library is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU Library General Public
|
||||||
|
* License as published by the Free Software Foundation; either
|
||||||
|
* version 2 of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This library is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* Library General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Library General Public
|
||||||
|
* License along with this library; if not, write to the
|
||||||
|
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
||||||
|
* Boston, MA 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifdef HAVE_CONFIG_H
|
||||||
|
# include "config.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include "gstobjectdetectionoverlay.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SECTION:plugin-analyticsoverlay
|
||||||
|
*
|
||||||
|
* Analytics overlay
|
||||||
|
*
|
||||||
|
* See also: @objectdetectionoverlay
|
||||||
|
* Since: 1.24
|
||||||
|
*/
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
plugin_init (GstPlugin * plugin)
|
||||||
|
{
|
||||||
|
gboolean ret = FALSE;
|
||||||
|
|
||||||
|
ret |= GST_ELEMENT_REGISTER (objectdetectionoverlay, plugin);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
|
||||||
|
GST_VERSION_MINOR,
|
||||||
|
analyticsoverlay,
|
||||||
|
"Analytics-meta overlay elements",
|
||||||
|
plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)
|
||||||
|
|
@ -0,0 +1,889 @@
|
||||||
|
/* GStreamer object detection overlay
|
||||||
|
* Copyright (C) <2023> Collabora Ltd.
|
||||||
|
* @author: Aaron Boxer <aaron.boxer@collabora.com>
|
||||||
|
* @author: Daniel Morin <daniel.morin@collabora.com>
|
||||||
|
*
|
||||||
|
* gstobjectdetectionoverlay.c
|
||||||
|
*
|
||||||
|
* This library is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU Library General Public
|
||||||
|
* License as published by the Free Software Foundation; either
|
||||||
|
* version 2 of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This library is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* Library General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Library General Public
|
||||||
|
* License along with this library; if not, write to the
|
||||||
|
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
||||||
|
* Boston, MA 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SECTION:element-objectdetectionoverlay
|
||||||
|
* @title: objectdetectionoverlay
|
||||||
|
* @see_also: #GstObjectDetectionOverlay
|
||||||
|
*
|
||||||
|
* This element create a graphical representation of the analytics object
|
||||||
|
* detection metadata attached to video stream and overlay graphics above the
|
||||||
|
* video.
|
||||||
|
*
|
||||||
|
* The object detection overlay element monitor video stream for
|
||||||
|
* @GstAnalyticsRelationMeta and query @GstAnalyticsODMtd. Retrieved
|
||||||
|
* @GstAnalyticsODMtd are then used to generate an overlay highlighing objects
|
||||||
|
* detected.
|
||||||
|
*
|
||||||
|
* ## Example launch line
|
||||||
|
* |[
|
||||||
|
* gst-launch-1.0 v4l2src device=/dev/video0 ! videoconvertscale add-borders=1 ! 'video/x-raw,width=640,height=383,framerate=2/1' ! onnxobjectdetector execution-provider=cpu model-file=ssd_mobilenet_v1_coco.onnx ! objectdetectionoverlay label-file=COCO_classes.txt ! videoconvertscale ! autovideosink
|
||||||
|
* ]| This pipeline create an overlay representing results of an object detetion
|
||||||
|
* analysis.
|
||||||
|
*
|
||||||
|
* Since: 1.24
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifdef HAVE_CONFIG_H
|
||||||
|
#include "config.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include <gst/gst.h>
|
||||||
|
#include <gst/video/video.h>
|
||||||
|
#include <gst/analytics/analytics.h>
|
||||||
|
#include <pango/pangocairo.h>
|
||||||
|
|
||||||
|
#include "gstobjectdetectionoverlay.h"
|
||||||
|
|
||||||
|
struct _GstObjectDetectionOverlay
|
||||||
|
{
|
||||||
|
GstVideoFilter parent;
|
||||||
|
|
||||||
|
cairo_matrix_t cairo_matrix;
|
||||||
|
gsize render_len;
|
||||||
|
|
||||||
|
/* stream metrics */
|
||||||
|
GstVideoInfo *in_info;
|
||||||
|
GMutex stream_event_mutex;
|
||||||
|
gboolean flushing;
|
||||||
|
gboolean eos;
|
||||||
|
|
||||||
|
/* properties */
|
||||||
|
guint od_outline_color;
|
||||||
|
guint od_outline_stroke_width;
|
||||||
|
gboolean draw_labels;
|
||||||
|
guint labels_color;
|
||||||
|
gdouble labels_stroke_width;
|
||||||
|
gdouble labels_outline_ofs;
|
||||||
|
|
||||||
|
/* composition */
|
||||||
|
gboolean attach_compo_to_buffer;
|
||||||
|
GstBuffer *canvas;
|
||||||
|
gint canvas_length;
|
||||||
|
GstVideoOverlayComposition *composition;
|
||||||
|
GstVideoOverlayComposition *upstream_composition;
|
||||||
|
|
||||||
|
/* Graphic Outline */
|
||||||
|
PangoContext *pango_context;
|
||||||
|
PangoLayout *pango_layout;
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#define MINIMUM_TEXT_OUTLINE_OFFSET 1.0
|
||||||
|
|
||||||
|
GST_DEBUG_CATEGORY_STATIC (objectdetectionoverlay_debug);
|
||||||
|
#define GST_CAT_DEFAULT objectdetectionoverlay_debug
|
||||||
|
|
||||||
|
enum
|
||||||
|
{
|
||||||
|
PROP_OD_OUTLINE_COLOR = 1,
|
||||||
|
PROP_DRAW_LABELS,
|
||||||
|
PROP_LABELS_COLOR,
|
||||||
|
_PROP_COUNT
|
||||||
|
};
|
||||||
|
|
||||||
|
typedef struct _GstObjectDetectionOverlayPangoCairoContext
|
||||||
|
GstObjectDetectionOverlayPangoCairoContext;
|
||||||
|
|
||||||
|
struct _GstObjectDetectionOverlayPangoCairoContext
|
||||||
|
{
|
||||||
|
cairo_t *cr;
|
||||||
|
cairo_surface_t *surface;
|
||||||
|
guint8 *data;
|
||||||
|
cairo_matrix_t *cairo_matrix;
|
||||||
|
};
|
||||||
|
|
||||||
|
#define VIDEO_FORMATS GST_VIDEO_OVERLAY_COMPOSITION_BLEND_FORMATS
|
||||||
|
#define OBJECT_DETECTION_OVERLAY_CAPS GST_VIDEO_CAPS_MAKE (VIDEO_FORMATS)
|
||||||
|
|
||||||
|
static GstStaticCaps sw_template_caps =
|
||||||
|
GST_STATIC_CAPS (OBJECT_DETECTION_OVERLAY_CAPS);
|
||||||
|
|
||||||
|
static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
|
||||||
|
GST_PAD_SRC,
|
||||||
|
GST_PAD_ALWAYS,
|
||||||
|
GST_STATIC_CAPS (OBJECT_DETECTION_OVERLAY_CAPS)
|
||||||
|
);
|
||||||
|
|
||||||
|
static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
|
||||||
|
GST_PAD_SINK,
|
||||||
|
GST_PAD_ALWAYS,
|
||||||
|
GST_STATIC_CAPS (OBJECT_DETECTION_OVERLAY_CAPS)
|
||||||
|
);
|
||||||
|
|
||||||
|
G_DEFINE_TYPE (GstObjectDetectionOverlay,
|
||||||
|
gst_object_detection_overlay, GST_TYPE_VIDEO_FILTER);
|
||||||
|
|
||||||
|
#define parent_class gst_object_detection_overlay_parent_class
|
||||||
|
|
||||||
|
GST_ELEMENT_REGISTER_DEFINE (objectdetectionoverlay, "objectdetectionoverlay",
|
||||||
|
GST_RANK_NONE, GST_TYPE_OBJECT_DETECTION_OVERLAY);
|
||||||
|
|
||||||
|
static void gst_object_detection_overlay_set_property (GObject * object,
|
||||||
|
guint prop_id, const GValue * value, GParamSpec * pspec);
|
||||||
|
|
||||||
|
static void gst_object_detection_overlay_get_property (GObject * object,
|
||||||
|
guint prop_id, GValue * value, GParamSpec * pspec);
|
||||||
|
|
||||||
|
static gboolean gst_object_detection_overlay_sink_event (GstBaseTransform *
|
||||||
|
trans, GstEvent * event);
|
||||||
|
|
||||||
|
static gboolean gst_object_detection_overlay_start (GstBaseTransform * trans);
|
||||||
|
static gboolean gst_object_detection_overlay_stop (GstBaseTransform * trans);
|
||||||
|
|
||||||
|
|
||||||
|
static gboolean gst_object_detection_overlay_set_info (GstVideoFilter * filter,
|
||||||
|
GstCaps * incaps, GstVideoInfo * in_info, GstCaps * outcaps,
|
||||||
|
GstVideoInfo * out_info);
|
||||||
|
|
||||||
|
static GstFlowReturn
|
||||||
|
gst_object_detection_overlay_transform_frame_ip (GstVideoFilter * filter,
|
||||||
|
GstVideoFrame * buf);
|
||||||
|
|
||||||
|
static void gst_object_detection_overlay_finalize (GObject * object);
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_render_boundingbox (GstObjectDetectionOverlay
|
||||||
|
* overlay, GstObjectDetectionOverlayPangoCairoContext * cairo_ctx,
|
||||||
|
GstAnalyticsODMtd * od_mtd);
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_render_text_annotation (GstObjectDetectionOverlay
|
||||||
|
* overlay, GstObjectDetectionOverlayPangoCairoContext * cairo_ctx,
|
||||||
|
GstAnalyticsODMtd * od_mtd, const gchar * annotation);
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_class_init (GstObjectDetectionOverlayClass * klass)
|
||||||
|
{
|
||||||
|
GObjectClass *gobject_class;
|
||||||
|
GstElementClass *element_class;
|
||||||
|
GstBaseTransformClass *basetransform_class;
|
||||||
|
GstVideoFilterClass *videofilter_class;
|
||||||
|
|
||||||
|
gobject_class = (GObjectClass *) klass;
|
||||||
|
gobject_class->set_property = gst_object_detection_overlay_set_property;
|
||||||
|
gobject_class->get_property = gst_object_detection_overlay_get_property;
|
||||||
|
gobject_class->finalize = gst_object_detection_overlay_finalize;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GstObjectDetectionOverlay:object-detection-outline-color
|
||||||
|
*
|
||||||
|
* Object Detetion Overlay outline color
|
||||||
|
* ARGB format (ex. 0xFFFF0000 for red)
|
||||||
|
*
|
||||||
|
* Since: 1.24
|
||||||
|
*/
|
||||||
|
g_object_class_install_property (gobject_class, PROP_OD_OUTLINE_COLOR,
|
||||||
|
g_param_spec_uint ("object-detection-outline-color",
|
||||||
|
"Object detection outline color",
|
||||||
|
"Color (ARGB) to use for object detection overlay outline",
|
||||||
|
0, G_MAXUINT, 0xFFFFFFFF,
|
||||||
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GstObjectDetectionOverlay:draw-labels
|
||||||
|
*
|
||||||
|
* Control labels drawing
|
||||||
|
*
|
||||||
|
* Since: 1.24
|
||||||
|
*/
|
||||||
|
g_object_class_install_property (gobject_class, PROP_DRAW_LABELS,
|
||||||
|
g_param_spec_boolean ("draw-labels",
|
||||||
|
"Draw labels",
|
||||||
|
"Draw object labels",
|
||||||
|
TRUE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GstObjectDetectionOverlay:labels-color
|
||||||
|
*
|
||||||
|
* Control labels color
|
||||||
|
* Format ARGB (ex. 0xFFFF0000 for red)
|
||||||
|
*
|
||||||
|
* Since: 1.24
|
||||||
|
*/
|
||||||
|
g_object_class_install_property (gobject_class, PROP_LABELS_COLOR,
|
||||||
|
g_param_spec_uint ("labels-color",
|
||||||
|
"Labels color",
|
||||||
|
"Color (ARGB) to use for object labels",
|
||||||
|
0, G_MAXUINT, 0xFFFFFF, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
||||||
|
|
||||||
|
element_class = (GstElementClass *) klass;
|
||||||
|
|
||||||
|
gst_element_class_add_static_pad_template (element_class, &sink_template);
|
||||||
|
gst_element_class_add_static_pad_template (element_class, &src_template);
|
||||||
|
gst_element_class_set_static_metadata (element_class,
|
||||||
|
"Object Detection Overlay",
|
||||||
|
"Analyzer/Visualization/Video",
|
||||||
|
"Overlay a visual representation of analytics metadata on the video",
|
||||||
|
"Daniel Morin");
|
||||||
|
|
||||||
|
basetransform_class = (GstBaseTransformClass *) klass;
|
||||||
|
basetransform_class->passthrough_on_same_caps = FALSE;
|
||||||
|
basetransform_class->start =
|
||||||
|
GST_DEBUG_FUNCPTR (gst_object_detection_overlay_start);
|
||||||
|
|
||||||
|
basetransform_class->stop =
|
||||||
|
GST_DEBUG_FUNCPTR (gst_object_detection_overlay_stop);
|
||||||
|
|
||||||
|
basetransform_class->sink_event =
|
||||||
|
GST_DEBUG_FUNCPTR (gst_object_detection_overlay_sink_event);
|
||||||
|
|
||||||
|
videofilter_class = (GstVideoFilterClass *) klass;
|
||||||
|
videofilter_class->set_info =
|
||||||
|
GST_DEBUG_FUNCPTR (gst_object_detection_overlay_set_info);
|
||||||
|
videofilter_class->transform_frame_ip =
|
||||||
|
GST_DEBUG_FUNCPTR (gst_object_detection_overlay_transform_frame_ip);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_adj_labels_outline_ofs (GstObjectDetectionOverlay *
|
||||||
|
overlay, PangoFontDescription * desc)
|
||||||
|
{
|
||||||
|
gint font_size = pango_font_description_get_size (desc) / PANGO_SCALE;
|
||||||
|
overlay->labels_outline_ofs = (double) (font_size) / 15.0;
|
||||||
|
if (overlay->labels_outline_ofs < MINIMUM_TEXT_OUTLINE_OFFSET)
|
||||||
|
overlay->labels_outline_ofs = MINIMUM_TEXT_OUTLINE_OFFSET;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_finalize (GObject * object)
|
||||||
|
{
|
||||||
|
gst_object_detection_overlay_stop (GST_BASE_TRANSFORM (object));
|
||||||
|
|
||||||
|
G_OBJECT_CLASS (parent_class)->finalize (object);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_init (GstObjectDetectionOverlay * overlay)
|
||||||
|
{
|
||||||
|
overlay->pango_context = NULL;
|
||||||
|
overlay->pango_layout = NULL;
|
||||||
|
overlay->od_outline_color = 0xFFFFFFFF;
|
||||||
|
overlay->draw_labels = TRUE;
|
||||||
|
overlay->labels_color = 0xFFFFFFFF;
|
||||||
|
overlay->in_info = &GST_VIDEO_FILTER (overlay)->in_info;
|
||||||
|
overlay->attach_compo_to_buffer = TRUE;
|
||||||
|
overlay->canvas = NULL;
|
||||||
|
overlay->labels_stroke_width = 1.0;
|
||||||
|
overlay->od_outline_stroke_width = 2;
|
||||||
|
overlay->composition = NULL;
|
||||||
|
overlay->upstream_composition = NULL;
|
||||||
|
overlay->flushing = FALSE;
|
||||||
|
GST_DEBUG_CATEGORY_INIT (objectdetectionoverlay_debug,
|
||||||
|
"analytics_overlay_od", 0, "Object detection overlay");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_set_property (GObject * object, guint prop_id,
|
||||||
|
const GValue * value, GParamSpec * pspec)
|
||||||
|
{
|
||||||
|
GstObjectDetectionOverlay *overlay;
|
||||||
|
overlay = GST_OBJECT_DETECTION_OVERLAY (object);
|
||||||
|
|
||||||
|
switch (prop_id) {
|
||||||
|
case PROP_OD_OUTLINE_COLOR:
|
||||||
|
overlay->od_outline_color = g_value_get_uint (value);
|
||||||
|
break;
|
||||||
|
case PROP_DRAW_LABELS:
|
||||||
|
overlay->draw_labels = g_value_get_boolean (value);
|
||||||
|
break;
|
||||||
|
case PROP_LABELS_COLOR:
|
||||||
|
overlay->labels_color = g_value_get_uint (value);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_get_property (GObject * object,
|
||||||
|
guint prop_id, GValue * value, GParamSpec * pspec)
|
||||||
|
{
|
||||||
|
GstObjectDetectionOverlay *od_overlay = GST_OBJECT_DETECTION_OVERLAY (object);
|
||||||
|
|
||||||
|
switch (prop_id) {
|
||||||
|
case PROP_OD_OUTLINE_COLOR:
|
||||||
|
g_value_set_uint (value, od_overlay->od_outline_color);
|
||||||
|
break;
|
||||||
|
case PROP_DRAW_LABELS:
|
||||||
|
g_value_set_boolean (value, od_overlay->draw_labels);
|
||||||
|
break;
|
||||||
|
case PROP_LABELS_COLOR:
|
||||||
|
g_value_set_uint (value, od_overlay->labels_color);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
gst_object_detection_overlay_can_handle_caps (GstCaps * incaps)
|
||||||
|
{
|
||||||
|
gboolean ret;
|
||||||
|
GstCaps *caps;
|
||||||
|
|
||||||
|
caps = gst_static_caps_get (&sw_template_caps);
|
||||||
|
ret = gst_caps_is_subset (incaps, caps);
|
||||||
|
gst_caps_unref (caps);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
gst_object_detection_overlay_negotiate (GstObjectDetectionOverlay * overlay,
|
||||||
|
GstCaps * caps)
|
||||||
|
{
|
||||||
|
GstBaseTransform *basetransform = GST_BASE_TRANSFORM (overlay);
|
||||||
|
gboolean upstream_has_meta = FALSE;
|
||||||
|
gboolean caps_has_meta = FALSE;
|
||||||
|
gboolean alloc_has_meta = FALSE;
|
||||||
|
gboolean attach = FALSE;
|
||||||
|
gboolean ret = TRUE;
|
||||||
|
guint width, height;
|
||||||
|
GstCapsFeatures *f;
|
||||||
|
GstCaps *overlay_caps;
|
||||||
|
GstQuery *query;
|
||||||
|
guint alloc_index;
|
||||||
|
GstPad *srcpad = basetransform->srcpad;
|
||||||
|
GstPad *sinkpad = basetransform->sinkpad;
|
||||||
|
|
||||||
|
GST_DEBUG_OBJECT (overlay, "performing negotiation");
|
||||||
|
|
||||||
|
/* Clear any pending reconfigure to avoid negotiating twice */
|
||||||
|
gst_pad_check_reconfigure (sinkpad);
|
||||||
|
|
||||||
|
/* Check if upstream caps have meta */
|
||||||
|
if ((f = gst_caps_get_features (caps, 0))) {
|
||||||
|
GST_DEBUG_OBJECT (overlay, "upstream has caps");
|
||||||
|
upstream_has_meta = gst_caps_features_contains (f,
|
||||||
|
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Initialize dimensions */
|
||||||
|
width = GST_VIDEO_INFO_WIDTH (overlay->in_info);
|
||||||
|
height = GST_VIDEO_INFO_HEIGHT (overlay->in_info);
|
||||||
|
GST_DEBUG_OBJECT (overlay, "initial dims: %ux%u", width, height);
|
||||||
|
|
||||||
|
if (upstream_has_meta) {
|
||||||
|
overlay_caps = gst_caps_ref (caps);
|
||||||
|
} else {
|
||||||
|
GstCaps *peercaps;
|
||||||
|
|
||||||
|
/* BaseTransform requires caps for the allocation query to work */
|
||||||
|
overlay_caps = gst_caps_copy (caps);
|
||||||
|
f = gst_caps_get_features (overlay_caps, 0);
|
||||||
|
gst_caps_features_add (f,
|
||||||
|
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
|
||||||
|
|
||||||
|
/* Then check if downstream accept overlay composition in caps */
|
||||||
|
/* FIXME: We should probably check if downstream *prefers* the
|
||||||
|
* overlay meta, and only enforce usage of it if we can't handle
|
||||||
|
* the format ourselves and thus would have to drop the overlays.
|
||||||
|
* Otherwise we should prefer what downstream wants here.
|
||||||
|
*/
|
||||||
|
peercaps = gst_pad_peer_query_caps (srcpad, overlay_caps);
|
||||||
|
caps_has_meta = !gst_caps_is_empty (peercaps);
|
||||||
|
gst_caps_unref (peercaps);
|
||||||
|
|
||||||
|
GST_DEBUG_OBJECT (overlay, "caps have overlay meta %d", caps_has_meta);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (upstream_has_meta || caps_has_meta) {
|
||||||
|
/* Send caps immediately, it's needed by GstBaseTransform to get a reply
|
||||||
|
* from allocation query */
|
||||||
|
GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (basetransform, caps,
|
||||||
|
overlay_caps);
|
||||||
|
ret = gst_pad_set_caps (srcpad, overlay_caps);
|
||||||
|
|
||||||
|
/* First check if the allocation meta has compositon */
|
||||||
|
query = gst_query_new_allocation (overlay_caps, FALSE);
|
||||||
|
|
||||||
|
if (!gst_pad_peer_query (srcpad, query)) {
|
||||||
|
/* no problem, we use the query defaults */
|
||||||
|
GST_DEBUG_OBJECT (overlay, "ALLOCATION query failed");
|
||||||
|
|
||||||
|
/* In case we were flushing, mark reconfigure and fail this method,
|
||||||
|
* will make it retry */
|
||||||
|
if (overlay->flushing)
|
||||||
|
ret = FALSE;
|
||||||
|
}
|
||||||
|
|
||||||
|
alloc_has_meta = gst_query_find_allocation_meta (query,
|
||||||
|
GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, &alloc_index);
|
||||||
|
|
||||||
|
GST_DEBUG_OBJECT (overlay, "sink alloc has overlay meta %d",
|
||||||
|
alloc_has_meta);
|
||||||
|
|
||||||
|
if (alloc_has_meta) {
|
||||||
|
const GstStructure *params;
|
||||||
|
|
||||||
|
gst_query_parse_nth_allocation_meta (query, alloc_index, ¶ms);
|
||||||
|
if (params) {
|
||||||
|
if (gst_structure_get (params, "width", G_TYPE_UINT, &width,
|
||||||
|
"height", G_TYPE_UINT, &height, NULL)) {
|
||||||
|
GST_DEBUG_OBJECT (overlay, "received window size: %dx%d", width,
|
||||||
|
height);
|
||||||
|
g_assert (width != 0 && height != 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
gst_query_unref (query);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Update render size if needed */
|
||||||
|
overlay->canvas_length = width * height;
|
||||||
|
|
||||||
|
/* For backward compatibility, we will prefer blitting if downstream
|
||||||
|
* allocation does not support the meta. In other case we will prefer
|
||||||
|
* attaching, and will fail the negotiation in the unlikely case we are
|
||||||
|
* force to blit, but format isn't supported. */
|
||||||
|
|
||||||
|
if (upstream_has_meta) {
|
||||||
|
attach = TRUE;
|
||||||
|
} else if (caps_has_meta) {
|
||||||
|
if (alloc_has_meta) {
|
||||||
|
attach = TRUE;
|
||||||
|
} else {
|
||||||
|
/* Don't attach unless we cannot handle the format */
|
||||||
|
attach = !gst_object_detection_overlay_can_handle_caps (caps);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ret = gst_object_detection_overlay_can_handle_caps (caps);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* If we attach, then pick the overlay caps */
|
||||||
|
if (attach) {
|
||||||
|
GST_DEBUG_OBJECT (overlay, "Using caps %" GST_PTR_FORMAT, overlay_caps);
|
||||||
|
/* Caps where already sent */
|
||||||
|
} else if (ret) {
|
||||||
|
GST_DEBUG_OBJECT (overlay, "Using caps %" GST_PTR_FORMAT, caps);
|
||||||
|
GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (basetransform, caps,
|
||||||
|
caps);
|
||||||
|
ret = gst_pad_set_caps (srcpad, caps);
|
||||||
|
}
|
||||||
|
|
||||||
|
overlay->attach_compo_to_buffer = attach;
|
||||||
|
|
||||||
|
if (attach) {
|
||||||
|
GST_BASE_TRANSFORM_CLASS (parent_class)->passthrough_on_same_caps = FALSE;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!ret) {
|
||||||
|
GST_DEBUG_OBJECT (overlay, "negotiation failed, schedule reconfigure");
|
||||||
|
gst_pad_mark_reconfigure (srcpad);
|
||||||
|
}
|
||||||
|
|
||||||
|
gst_caps_unref (overlay_caps);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
gst_object_detection_overlay_setcaps (GstObjectDetectionOverlay * overlay,
|
||||||
|
GstCaps * caps)
|
||||||
|
{
|
||||||
|
gboolean ret = FALSE;
|
||||||
|
|
||||||
|
if (!gst_video_info_from_caps (overlay->in_info, caps))
|
||||||
|
goto invalid_caps;
|
||||||
|
|
||||||
|
ret = gst_object_detection_overlay_negotiate (overlay, caps);
|
||||||
|
GST_VIDEO_FILTER (overlay)->negotiated = ret;
|
||||||
|
|
||||||
|
if (!overlay->attach_compo_to_buffer &&
|
||||||
|
!gst_object_detection_overlay_can_handle_caps (caps)) {
|
||||||
|
GST_DEBUG_OBJECT (overlay, "unsupported caps %" GST_PTR_FORMAT, caps);
|
||||||
|
ret = FALSE;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
/* ERRORS */
|
||||||
|
invalid_caps:
|
||||||
|
{
|
||||||
|
GST_DEBUG_OBJECT (overlay, "could not parse caps");
|
||||||
|
return FALSE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
gst_object_detection_overlay_sink_event (GstBaseTransform * trans,
|
||||||
|
GstEvent * event)
|
||||||
|
{
|
||||||
|
gboolean ret = FALSE;
|
||||||
|
GST_DEBUG_OBJECT (trans, "received sink event %s",
|
||||||
|
GST_EVENT_TYPE_NAME (event));
|
||||||
|
|
||||||
|
GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (trans);
|
||||||
|
|
||||||
|
switch (GST_EVENT_TYPE (event)) {
|
||||||
|
case GST_EVENT_CAPS:
|
||||||
|
{
|
||||||
|
GstCaps *caps;
|
||||||
|
gst_event_parse_caps (event, &caps);
|
||||||
|
ret = gst_object_detection_overlay_setcaps (overlay, caps);
|
||||||
|
gst_event_unref (event);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case GST_EVENT_EOS:
|
||||||
|
g_mutex_lock (&overlay->stream_event_mutex);
|
||||||
|
GST_INFO_OBJECT (overlay, "EOS");
|
||||||
|
overlay->eos = TRUE;
|
||||||
|
g_mutex_unlock (&overlay->stream_event_mutex);
|
||||||
|
break;
|
||||||
|
case GST_EVENT_FLUSH_START:
|
||||||
|
g_mutex_lock (&overlay->stream_event_mutex);
|
||||||
|
GST_INFO_OBJECT (overlay, "Flush stop");
|
||||||
|
overlay->flushing = TRUE;
|
||||||
|
g_mutex_unlock (&overlay->stream_event_mutex);
|
||||||
|
break;
|
||||||
|
case GST_EVENT_FLUSH_STOP:
|
||||||
|
g_mutex_lock (&overlay->stream_event_mutex);
|
||||||
|
GST_INFO_OBJECT (overlay, "Flush stop");
|
||||||
|
overlay->eos = FALSE;
|
||||||
|
overlay->flushing = FALSE;
|
||||||
|
g_mutex_unlock (&overlay->stream_event_mutex);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
ret = GST_BASE_TRANSFORM_CLASS (parent_class)->sink_event (trans, event);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
gst_object_detection_overlay_start (GstBaseTransform * trans)
|
||||||
|
{
|
||||||
|
GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (trans);
|
||||||
|
PangoFontDescription *desc;
|
||||||
|
PangoFontMap *fontmap;
|
||||||
|
|
||||||
|
fontmap = pango_cairo_font_map_new ();
|
||||||
|
overlay->pango_context =
|
||||||
|
pango_font_map_create_context (PANGO_FONT_MAP (fontmap));
|
||||||
|
g_object_unref (fontmap);
|
||||||
|
overlay->pango_layout = pango_layout_new (overlay->pango_context);
|
||||||
|
desc = pango_context_get_font_description (overlay->pango_context);
|
||||||
|
pango_font_description_set_size (desc, 10000);
|
||||||
|
pango_font_description_set_weight (desc, PANGO_WEIGHT_ULTRALIGHT);
|
||||||
|
pango_context_set_font_description (overlay->pango_context, desc);
|
||||||
|
pango_layout_set_alignment (overlay->pango_layout, PANGO_ALIGN_LEFT);
|
||||||
|
|
||||||
|
gst_object_detection_overlay_adj_labels_outline_ofs (overlay, desc);
|
||||||
|
GST_DEBUG_OBJECT (overlay, "labels_outline_offset %f",
|
||||||
|
overlay->labels_outline_ofs);
|
||||||
|
|
||||||
|
return TRUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
gst_object_detection_overlay_stop (GstBaseTransform * trans)
|
||||||
|
{
|
||||||
|
GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (trans);
|
||||||
|
|
||||||
|
g_clear_object (&overlay->pango_layout);
|
||||||
|
g_clear_object (&overlay->pango_context);
|
||||||
|
gst_clear_buffer (&overlay->canvas);
|
||||||
|
|
||||||
|
return TRUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
gst_object_detection_overlay_set_info (GstVideoFilter * filter,
|
||||||
|
GstCaps * incaps, GstVideoInfo * in_info, GstCaps * outcaps,
|
||||||
|
GstVideoInfo * out_info)
|
||||||
|
{
|
||||||
|
GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (filter);
|
||||||
|
GST_DEBUG_OBJECT (filter, "set_info incaps:%s", gst_caps_to_string (incaps));
|
||||||
|
GST_DEBUG_OBJECT (filter, "set_info outcaps:%s",
|
||||||
|
gst_caps_to_string (outcaps));
|
||||||
|
|
||||||
|
filter->in_info = *in_info;
|
||||||
|
filter->out_info = *out_info;
|
||||||
|
|
||||||
|
cairo_matrix_init_scale (&overlay->cairo_matrix, 1, 1);
|
||||||
|
overlay->render_len = GST_VIDEO_INFO_WIDTH (in_info) *
|
||||||
|
GST_VIDEO_INFO_HEIGHT (in_info) * 4;
|
||||||
|
return TRUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_create_cairo_context (GstObjectDetectionOverlay *
|
||||||
|
overlay, GstObjectDetectionOverlayPangoCairoContext * cairo_ctx,
|
||||||
|
guint8 * data)
|
||||||
|
{
|
||||||
|
cairo_ctx->cairo_matrix = &overlay->cairo_matrix;
|
||||||
|
cairo_ctx->surface = cairo_image_surface_create_for_data (data,
|
||||||
|
CAIRO_FORMAT_ARGB32, GST_VIDEO_INFO_WIDTH (overlay->in_info),
|
||||||
|
GST_VIDEO_INFO_HEIGHT (overlay->in_info),
|
||||||
|
GST_VIDEO_INFO_WIDTH (overlay->in_info) * 4);
|
||||||
|
cairo_ctx->cr = cairo_create (cairo_ctx->surface);
|
||||||
|
|
||||||
|
/* clear surface */
|
||||||
|
cairo_set_operator (cairo_ctx->cr, CAIRO_OPERATOR_CLEAR);
|
||||||
|
cairo_paint (cairo_ctx->cr);
|
||||||
|
cairo_set_operator (cairo_ctx->cr, CAIRO_OPERATOR_OVER);
|
||||||
|
|
||||||
|
/* apply transformations */
|
||||||
|
cairo_set_matrix (cairo_ctx->cr, cairo_ctx->cairo_matrix);
|
||||||
|
cairo_save (cairo_ctx->cr);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_destroy_cairo_context
|
||||||
|
(GstObjectDetectionOverlayPangoCairoContext * cairo_ctx)
|
||||||
|
{
|
||||||
|
cairo_restore (cairo_ctx->cr);
|
||||||
|
cairo_destroy (cairo_ctx->cr);
|
||||||
|
cairo_surface_destroy (cairo_ctx->surface);
|
||||||
|
}
|
||||||
|
|
||||||
|
static GstFlowReturn
|
||||||
|
gst_object_detection_overlay_transform_frame_ip (GstVideoFilter * filter,
|
||||||
|
GstVideoFrame * frame)
|
||||||
|
{
|
||||||
|
GstObjectDetectionOverlay *overlay = GST_OBJECT_DETECTION_OVERLAY (filter);
|
||||||
|
GstVideoOverlayCompositionMeta *composition_meta;
|
||||||
|
gpointer state = NULL;
|
||||||
|
GstVideoOverlayRectangle *rectangle = NULL;
|
||||||
|
gchar str_buf[5];
|
||||||
|
GstAnalyticsMtd rlt_mtd;
|
||||||
|
GstAnalyticsODMtd *od_mtd;
|
||||||
|
gint x, y, w, h;
|
||||||
|
gfloat loc_confi_lvl;
|
||||||
|
gboolean success;
|
||||||
|
|
||||||
|
GST_DEBUG_OBJECT (filter, "buffer writeable=%d",
|
||||||
|
gst_buffer_is_writable (frame->buffer));
|
||||||
|
|
||||||
|
g_mutex_lock (&overlay->stream_event_mutex);
|
||||||
|
if (overlay->eos || overlay->flushing) {
|
||||||
|
g_mutex_unlock (&overlay->stream_event_mutex);
|
||||||
|
return GST_FLOW_EOS;
|
||||||
|
}
|
||||||
|
g_mutex_unlock (&overlay->stream_event_mutex);
|
||||||
|
|
||||||
|
composition_meta =
|
||||||
|
gst_buffer_get_video_overlay_composition_meta (frame->buffer);
|
||||||
|
if (composition_meta) {
|
||||||
|
if (overlay->upstream_composition != composition_meta->overlay) {
|
||||||
|
GST_DEBUG_OBJECT (overlay, "GstVideoOverlayCompositionMeta found.");
|
||||||
|
overlay->upstream_composition = composition_meta->overlay;
|
||||||
|
}
|
||||||
|
} else if (overlay->upstream_composition != NULL) {
|
||||||
|
overlay->upstream_composition = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
GstAnalyticsRelationMeta *rmeta = (GstAnalyticsRelationMeta *)
|
||||||
|
gst_buffer_get_meta (GST_BUFFER (frame->buffer),
|
||||||
|
GST_ANALYTICS_RELATION_META_API_TYPE);
|
||||||
|
|
||||||
|
if (rmeta) {
|
||||||
|
GST_DEBUG_OBJECT (filter, "received buffer with analytics relation meta");
|
||||||
|
|
||||||
|
GstBuffer *buffer;
|
||||||
|
GstMapInfo map;
|
||||||
|
GstObjectDetectionOverlayPangoCairoContext cairo_ctx;
|
||||||
|
|
||||||
|
buffer = gst_buffer_new_and_alloc (overlay->render_len);
|
||||||
|
gst_buffer_add_video_meta (buffer,
|
||||||
|
GST_VIDEO_FRAME_FLAG_NONE, GST_VIDEO_OVERLAY_COMPOSITION_FORMAT_RGB,
|
||||||
|
GST_VIDEO_INFO_WIDTH (overlay->in_info),
|
||||||
|
GST_VIDEO_INFO_HEIGHT (overlay->in_info));
|
||||||
|
|
||||||
|
gst_buffer_replace (&overlay->canvas, buffer);
|
||||||
|
gst_buffer_unref (buffer);
|
||||||
|
|
||||||
|
gst_buffer_map (buffer, &map, GST_MAP_READWRITE);
|
||||||
|
memset (map.data, 0, overlay->render_len);
|
||||||
|
|
||||||
|
gst_object_detection_overlay_create_cairo_context (overlay,
|
||||||
|
&cairo_ctx, map.data);
|
||||||
|
|
||||||
|
if (overlay->composition)
|
||||||
|
gst_video_overlay_composition_unref (overlay->composition);
|
||||||
|
|
||||||
|
if (overlay->upstream_composition) {
|
||||||
|
overlay->composition =
|
||||||
|
gst_video_overlay_composition_copy (overlay->upstream_composition);
|
||||||
|
} else {
|
||||||
|
overlay->composition = gst_video_overlay_composition_new (NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Get quark represent object detection metadata type */
|
||||||
|
GstAnalyticsMtdType rlt_type = gst_analytics_od_mtd_get_mtd_type ();
|
||||||
|
while (gst_analytics_relation_meta_iterate (rmeta, &state, rlt_type,
|
||||||
|
&rlt_mtd)) {
|
||||||
|
od_mtd = (GstAnalyticsODMtd *) & rlt_mtd;
|
||||||
|
GST_DEBUG_OBJECT (filter, "buffer contain OD mtd");
|
||||||
|
|
||||||
|
/* Quark representing the type of the object detected by OD */
|
||||||
|
GQuark od_obj_type = gst_analytics_od_mtd_get_obj_type (od_mtd);
|
||||||
|
|
||||||
|
// Find classification metadata attached to object detection metadata
|
||||||
|
GstAnalyticsMtd cls_rlt_mtd;
|
||||||
|
success = gst_analytics_relation_meta_get_direct_related (rmeta,
|
||||||
|
gst_analytics_mtd_get_id (
|
||||||
|
(GstAnalyticsMtd *) od_mtd),
|
||||||
|
GST_ANALYTICS_REL_TYPE_RELATE_TO,
|
||||||
|
gst_analytics_cls_mtd_get_mtd_type (), NULL, &cls_rlt_mtd);
|
||||||
|
|
||||||
|
gst_object_detection_overlay_render_boundingbox
|
||||||
|
(GST_OBJECT_DETECTION_OVERLAY (filter), &cairo_ctx, od_mtd);
|
||||||
|
|
||||||
|
if (overlay->draw_labels) {
|
||||||
|
if (success) {
|
||||||
|
/* Use associated classification analytics-meta */
|
||||||
|
g_snprintf (str_buf, sizeof (str_buf), "%04.2f",
|
||||||
|
gst_analytics_cls_mtd_get_level (
|
||||||
|
(GstAnalyticsClsMtd *) & cls_rlt_mtd, 0));
|
||||||
|
|
||||||
|
od_obj_type = gst_analytics_cls_mtd_get_quark (&cls_rlt_mtd, 0);
|
||||||
|
} else {
|
||||||
|
/* Use basic class type directly on OD.
|
||||||
|
* Here we want the confidence level of the bbox but to retrieve
|
||||||
|
* we need to also retrieve the bbox location. */
|
||||||
|
gst_analytics_od_mtd_get_location (od_mtd, &x, &y, &w, &h,
|
||||||
|
&loc_confi_lvl);
|
||||||
|
GST_TRACE_OBJECT (filter, "obj {type: %s loc:[(%u,%u)-(%ux%u)] @ %f}",
|
||||||
|
g_quark_to_string (od_obj_type), x, y, w, h, loc_confi_lvl);
|
||||||
|
|
||||||
|
g_snprintf (str_buf, sizeof (str_buf), "%04.2f", loc_confi_lvl);
|
||||||
|
}
|
||||||
|
gchar *text = g_strdup_printf ("%s (c=%s)",
|
||||||
|
g_quark_to_string (od_obj_type), str_buf);
|
||||||
|
|
||||||
|
gst_object_detection_overlay_render_text_annotation
|
||||||
|
(GST_OBJECT_DETECTION_OVERLAY (filter), &cairo_ctx, od_mtd, text);
|
||||||
|
|
||||||
|
g_free (text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rectangle = gst_video_overlay_rectangle_new_raw (overlay->canvas,
|
||||||
|
0, 0, GST_VIDEO_INFO_WIDTH (overlay->in_info),
|
||||||
|
GST_VIDEO_INFO_HEIGHT (overlay->in_info),
|
||||||
|
GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA);
|
||||||
|
gst_video_overlay_composition_add_rectangle (overlay->composition,
|
||||||
|
rectangle);
|
||||||
|
gst_video_overlay_rectangle_unref (rectangle);
|
||||||
|
|
||||||
|
gst_object_detection_overlay_destroy_cairo_context (&cairo_ctx);
|
||||||
|
gst_buffer_unmap (buffer, &map);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if (overlay->composition) {
|
||||||
|
GST_DEBUG_OBJECT (filter, "have composition");
|
||||||
|
|
||||||
|
if (overlay->attach_compo_to_buffer) {
|
||||||
|
GST_DEBUG_OBJECT (filter, "attach");
|
||||||
|
|
||||||
|
gst_buffer_add_video_overlay_composition_meta (frame->buffer,
|
||||||
|
overlay->composition);
|
||||||
|
} else {
|
||||||
|
gst_video_overlay_composition_blend (overlay->composition, frame);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return GST_FLOW_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_render_boundingbox (GstObjectDetectionOverlay
|
||||||
|
* overlay, GstObjectDetectionOverlayPangoCairoContext * ctx,
|
||||||
|
GstAnalyticsODMtd * od_mtd)
|
||||||
|
{
|
||||||
|
gint x, y, w, h;
|
||||||
|
gfloat _dummy;
|
||||||
|
cairo_save (ctx->cr);
|
||||||
|
gst_analytics_od_mtd_get_location (od_mtd, &x, &y, &w, &h, &_dummy);
|
||||||
|
gint maxw = GST_VIDEO_INFO_WIDTH (overlay->in_info) - 1;
|
||||||
|
gint maxh = GST_VIDEO_INFO_HEIGHT (overlay->in_info) - 1;
|
||||||
|
|
||||||
|
x = CLAMP (x, 0, maxw);
|
||||||
|
y = CLAMP (y, 0, maxh);
|
||||||
|
w = CLAMP (w, 0, maxw - x);
|
||||||
|
h = CLAMP (h, 0, maxh - y);
|
||||||
|
|
||||||
|
/* Set bounding box stroke color and width */
|
||||||
|
cairo_set_source_rgba (ctx->cr,
|
||||||
|
((overlay->od_outline_color >> 16) & 0xFF) / 255.0,
|
||||||
|
((overlay->od_outline_color >> 8) & 0xFF) / 255.0,
|
||||||
|
((overlay->od_outline_color) & 0xFF) / 255.0,
|
||||||
|
((overlay->od_outline_color >> 24) & 0xFF) / 255.0);
|
||||||
|
cairo_set_line_width (ctx->cr, overlay->od_outline_stroke_width);
|
||||||
|
|
||||||
|
/* draw bounding box */
|
||||||
|
cairo_rectangle (ctx->cr, x, y, w, h);
|
||||||
|
cairo_stroke (ctx->cr);
|
||||||
|
cairo_restore (ctx->cr);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
gst_object_detection_overlay_render_text_annotation (GstObjectDetectionOverlay
|
||||||
|
* overlay, GstObjectDetectionOverlayPangoCairoContext * ctx,
|
||||||
|
GstAnalyticsODMtd * od_mtd, const gchar * annotation)
|
||||||
|
{
|
||||||
|
PangoRectangle ink_rect, logical_rect;
|
||||||
|
gint x, y, w, h;
|
||||||
|
gfloat _dummy;
|
||||||
|
gint maxw = GST_VIDEO_INFO_WIDTH (overlay->in_info) - 1;
|
||||||
|
gint maxh = GST_VIDEO_INFO_HEIGHT (overlay->in_info) - 1;
|
||||||
|
|
||||||
|
cairo_save (ctx->cr);
|
||||||
|
gst_analytics_od_mtd_get_location (od_mtd, &x, &y, &w, &h, &_dummy);
|
||||||
|
|
||||||
|
x = CLAMP (x, 0, maxw);
|
||||||
|
y = CLAMP (y, 0, maxh);
|
||||||
|
w = CLAMP (w, 0, maxw - x);
|
||||||
|
h = CLAMP (h, 0, maxh - y);
|
||||||
|
|
||||||
|
/* Set label strokes color and width */
|
||||||
|
cairo_set_source_rgba (ctx->cr,
|
||||||
|
((overlay->labels_color >> 16) & 0xFF) / 255.0,
|
||||||
|
((overlay->labels_color >> 8) & 0xFF) / 255.0,
|
||||||
|
((overlay->labels_color) & 0xFF) / 255.0,
|
||||||
|
((overlay->labels_color >> 24) & 0xFF) / 255.0);
|
||||||
|
cairo_set_line_width (ctx->cr, overlay->labels_stroke_width);
|
||||||
|
|
||||||
|
pango_layout_set_markup (overlay->pango_layout, annotation,
|
||||||
|
strlen (annotation));
|
||||||
|
pango_layout_get_pixel_extents (overlay->pango_layout, &ink_rect,
|
||||||
|
&logical_rect);
|
||||||
|
GST_DEBUG_OBJECT (overlay, "logical_rect:(%d,%d),%dx%d", logical_rect.x,
|
||||||
|
logical_rect.y, logical_rect.width, logical_rect.height);
|
||||||
|
GST_DEBUG_OBJECT (overlay, "ink_rect:(%d,%d),%dx%d", ink_rect.x, ink_rect.y,
|
||||||
|
ink_rect.width, ink_rect.height);
|
||||||
|
cairo_move_to (ctx->cr, x + overlay->labels_outline_ofs,
|
||||||
|
y - logical_rect.height - overlay->labels_outline_ofs);
|
||||||
|
|
||||||
|
pango_cairo_layout_path (ctx->cr, overlay->pango_layout);
|
||||||
|
cairo_stroke (ctx->cr);
|
||||||
|
cairo_restore (ctx->cr);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
/* GStreamer object detection overlay
|
||||||
|
* Copyright (C) <2023> Collabora Ltd.
|
||||||
|
* @author: Aaron Boxer <aaron.boxer@collabora.com>
|
||||||
|
* @author: Daniel Morin <daniel.morin@collabora.com>
|
||||||
|
*
|
||||||
|
* gstobjectdetectionoverlay.h
|
||||||
|
*
|
||||||
|
* This library is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU Library General Public
|
||||||
|
* License as published by the Free Software Foundation; either
|
||||||
|
* version 2 of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This library is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* Library General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Library General Public
|
||||||
|
* License along with this library; if not, write to the
|
||||||
|
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
||||||
|
* Boston, MA 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef __GST_OBJECT_DETECTION_OVERLAY_H__
|
||||||
|
#define __GST_OBJECT_DETECTION_OVERLAY_H__
|
||||||
|
|
||||||
|
#include <gst/video/gstvideofilter.h>
|
||||||
|
|
||||||
|
G_BEGIN_DECLS
|
||||||
|
|
||||||
|
#define GST_TYPE_OBJECT_DETECTION_OVERLAY \
|
||||||
|
(gst_object_detection_overlay_get_type())
|
||||||
|
|
||||||
|
G_DECLARE_FINAL_TYPE (GstObjectDetectionOverlay, gst_object_detection_overlay,
|
||||||
|
GST, OBJECT_DETECTION_OVERLAY, GstVideoFilter)
|
||||||
|
|
||||||
|
GST_ELEMENT_REGISTER_DECLARE (objectdetectionoverlay);
|
||||||
|
|
||||||
|
G_END_DECLS
|
||||||
|
#endif /* __GST_OBJECT_DETECTION_OVERLAY_H__ */
|
||||||
30
subprojects/gst-plugins-bad/ext/analyticsoverlay/meson.build
Normal file
30
subprojects/gst-plugins-bad/ext/analyticsoverlay/meson.build
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
if get_option('analyticsoverlay').disabled()
|
||||||
|
gstanalyticsoverlay_ext_dep = disabler()
|
||||||
|
subdir_done()
|
||||||
|
endif
|
||||||
|
|
||||||
|
analyticsoverlay_sources = [
|
||||||
|
'gstanalyticsoverlay.c',
|
||||||
|
'gstobjectdetectionoverlay.c',
|
||||||
|
]
|
||||||
|
|
||||||
|
gstanalyticsoverlay_ext_dep = dependency('pangocairo', version : '>=1.22.0',
|
||||||
|
required : get_option('analyticsoverlay'),
|
||||||
|
fallback: ['pango', 'libpangocairo_dep'],
|
||||||
|
default_options: ['cairo=enabled'])
|
||||||
|
|
||||||
|
if gstanalyticsoverlay_ext_dep.found()
|
||||||
|
gstanalyticsoverlay = library('gstanalyticsoverlay',
|
||||||
|
analyticsoverlay_sources,
|
||||||
|
c_args : gst_plugins_bad_args,
|
||||||
|
include_directories : [configinc, libsinc],
|
||||||
|
dependencies : [
|
||||||
|
gstbase_dep, gstvideo_dep, gstanalytics_dep, gstanalyticsoverlay_ext_dep
|
||||||
|
],
|
||||||
|
install : true,
|
||||||
|
install_dir : plugins_install_dir,
|
||||||
|
)
|
||||||
|
plugins += [gstanalyticsoverlay]
|
||||||
|
else
|
||||||
|
error('analyticsoverlay plugin is enabled, but dependency is missing')
|
||||||
|
endif
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
subdir('aes')
|
subdir('aes')
|
||||||
|
subdir('analyticsoverlay')
|
||||||
subdir('assrender')
|
subdir('assrender')
|
||||||
subdir('aom')
|
subdir('aom')
|
||||||
subdir('avtp')
|
subdir('avtp')
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ option('accurip', type : 'feature', value : 'auto')
|
||||||
option('adpcmdec', type : 'feature', value : 'auto')
|
option('adpcmdec', type : 'feature', value : 'auto')
|
||||||
option('adpcmenc', type : 'feature', value : 'auto')
|
option('adpcmenc', type : 'feature', value : 'auto')
|
||||||
option('aiff', type : 'feature', value : 'auto')
|
option('aiff', type : 'feature', value : 'auto')
|
||||||
|
option('analyticsoverlay', type:'feature', value : 'auto')
|
||||||
option('asfmux', type : 'feature', value : 'auto')
|
option('asfmux', type : 'feature', value : 'auto')
|
||||||
option('audiobuffersplit', type : 'feature', value : 'auto')
|
option('audiobuffersplit', type : 'feature', value : 'auto')
|
||||||
option('audiofxbad', type : 'feature', value : 'auto')
|
option('audiofxbad', type : 'feature', value : 'auto')
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue