mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-12-31 20:48:56 +00:00
analytics: Allow specific analytics-meta (Mtd) to handle their clear
- Add mtd_meta_clear to allow specific analytics-meta to handle their clear operation specific to their type. - Clear mtd's attached when analytic-meta is freed. When the buffer where analytics-meta is attached is not from a buffer pool gst_analytics_relation_meta_clear will not be called unless we explicitly call it in _free. This important otherwise _mtd_clear are not called and lead to leak if embedded mtd's allocated memory - Un-ref in transform if it's a copy Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6026>
This commit is contained in:
Daniel Morin
GStreamer Marge Bot
parent
bc5bd03723
commit
98653aa43a
3 changed files with 429 additions and 2 deletions
|
|
@ -111,6 +111,8 @@ typedef struct _GstAnalyticsRelationMeta
|
|||
static guint
|
||||
gst_analytics_relation_meta_get_next_id (GstAnalyticsRelationMeta * meta);
|
||||
|
||||
static void
|
||||
gst_analytics_relation_meta_clear (GstBuffer * buffer, GstMeta * meta);
|
||||
|
||||
static GstAnalyticsRelatableMtdData *
|
||||
gst_analytics_relation_meta_get_mtd_data_internal (const
|
||||
|
|
@ -346,6 +348,8 @@ gst_analytics_relation_meta_free (GstMeta * meta, GstBuffer * buffer)
|
|||
"Content analysis meta-data(%p) freed for buffer(%p)",
|
||||
(gpointer) rmeta, (gpointer) buffer);
|
||||
|
||||
gst_analytics_relation_meta_clear (buffer, meta);
|
||||
|
||||
g_free (rmeta->analysis_results);
|
||||
g_free (rmeta->adj_mat);
|
||||
g_free (rmeta->mtd_data_lookup);
|
||||
|
|
@ -442,6 +446,19 @@ static void
|
|||
gst_analytics_relation_meta_clear (GstBuffer * buffer, GstMeta * meta)
|
||||
{
|
||||
GstAnalyticsRelationMeta *rmeta = (GstAnalyticsRelationMeta *) meta;
|
||||
GstAnalyticsRelatableMtdData *rlt_mtd_data = NULL;
|
||||
|
||||
for (gsize index = 0; index < rmeta->length; index++) {
|
||||
rlt_mtd_data = (GstAnalyticsRelatableMtdData *)
|
||||
(rmeta->mtd_data_lookup[index] + rmeta->analysis_results);
|
||||
if (rlt_mtd_data->impl && rlt_mtd_data->impl->mtd_meta_clear) {
|
||||
GstAnalyticsMtd mtd;
|
||||
mtd.id = rlt_mtd_data->id;
|
||||
mtd.meta = rmeta;
|
||||
rlt_mtd_data->impl->mtd_meta_clear (buffer, &mtd);
|
||||
}
|
||||
}
|
||||
|
||||
gsize adj_mat_data_size =
|
||||
(sizeof (guint8) * rmeta->rel_order * rmeta->rel_order);
|
||||
|
||||
|
|
|
|||
|
|
@ -104,6 +104,9 @@ struct _GstAnalyticsMtd
|
|||
* @mtd_meta_transform: A pointer to a function that will be called
|
||||
* when the containing meta is transform to potentially copy the data
|
||||
* into a new Mtd into the new meta.
|
||||
* @mtd_meta_clear: A pointer to a function that will be called when the
|
||||
* containing meta is cleared to potetially do cleanup (ex. _unref or release)
|
||||
* resources it was using.
|
||||
*
|
||||
* This structure must be provided when registering a new type of Mtd. It must
|
||||
* have a static lifetime (never be freed).
|
||||
|
|
@ -119,8 +122,10 @@ gboolean (*mtd_meta_transform) (GstBuffer * transbuf,
|
|||
GstAnalyticsMtd * transmtd, GstBuffer * buffer, GQuark type,
|
||||
gpointer data);
|
||||
|
||||
/*< private > */
|
||||
gpointer _reserved[GST_PADDING_LARGE];
|
||||
void (*mtd_meta_clear) (GstBuffer *buffer, GstAnalyticsMtd *mtd);
|
||||
|
||||
/*< private >*/
|
||||
gpointer _reserved[GST_PADDING_LARGE - 1];
|
||||
} GstAnalyticsMtdImpl;
|
||||
|
||||
GST_ANALYTICS_META_API
|
||||
|
|
|
|||
|
|
@ -0,0 +1,405 @@
|
|||
/* GStreamer
|
||||
* Copyright (C) 2024 Collabora Ltd
|
||||
* @author: Daniel Morin <daniel.morin@collabora.com>
|
||||
*
|
||||
* gstanalyticssegmentationmtd.c
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Library General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Library General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Library General Public
|
||||
* License along with this library; if not, write to the
|
||||
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
||||
* Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
|
||||
#include "gstanalyticssegmentationmtd.h"
|
||||
#include <gst/video/video-info.h>
|
||||
|
||||
/**
|
||||
* SECTION: gstanalyticssegmentationmtd
|
||||
* @title: GstAnalyticsSegmentationMtd
|
||||
* @short_description: An analytics metadata for image segmentation inside a
|
||||
* #GstAnalyticsRelationMeta
|
||||
* @symbols:
|
||||
* - GstAnalyticsSegmentationMtd
|
||||
* @see_also: #GstAnalyticsMtd, #GstAnalyticsRelationMeta
|
||||
*
|
||||
* This type of metadata holds information on which pixels belongs to
|
||||
* a region of the image representing a type of object.
|
||||
*
|
||||
* It supports two types of segmentation, semantic or instance:
|
||||
* * Semantic: All objects of the same type have the same id
|
||||
* * Instance: Each instance of an object has a different id
|
||||
*
|
||||
* The results of the segmentation are stored in a #GstBuffer that has a
|
||||
* #GstVideoMeta associated with it. This buffer is stored in the
|
||||
* GstAnalyticsSegmentationMtd using
|
||||
* #gst_analytics_relation_meta_add_segmentation_mtd(). The #GstBuffer
|
||||
* containing the segmentation mask is image-like but the color values are
|
||||
* arbitrary values, referred by region-id in this API, without meaning beyond
|
||||
* specifying that two pixels in the original image with the same values in
|
||||
* their corresponding mask value belong to the same region.
|
||||
*
|
||||
* To further describe a region, the #GstAnalyticsSegmentationMtd can be
|
||||
* associated with other #GstAnalyticsMtd. Since region ids are
|
||||
* generated by the segmentation process itself and are not always sequential,
|
||||
* we use a map of indexes to region ids starting with 0 without discontinuity
|
||||
* which facilitate N-to-N mapping with other #GstAnalyticsMtd. For
|
||||
* example it can be associated with #GstAnalyticsClsMtd to describe the class
|
||||
* of object matching the pixels of a segmented region.
|
||||
*
|
||||
* Example: Associate Instance Segmentation with Classification
|
||||
*
|
||||
* In the following example the segmentation process will fill segmask with
|
||||
* values of 0 for background, 12 for the first region which correspond to a
|
||||
* to a strawberry, 7 for the second region that also correspond to a
|
||||
* strawberry in the image and 31 for the third region that correspond to a
|
||||
* leaf in the image.
|
||||
* region_ids is fill during segmentation post-processing
|
||||
*
|
||||
* region_ids:
|
||||
* |region-index | region-id |
|
||||
* |-------------|-----------|
|
||||
* | 0 | 0 |
|
||||
* | 1 | 12 |
|
||||
* | 2 | 7 |
|
||||
* | 3 | 31 |
|
||||
*
|
||||
* region_count = 4
|
||||
*
|
||||
* ``` C
|
||||
* GstAnalyticsSegmentationMtd segmtd;
|
||||
* GstAnalyticsClassificationMtd clsmtd;
|
||||
* GstBuffer *segmask, *img;
|
||||
* guint *region_ids;
|
||||
* gsize region_count, class_count;
|
||||
* gfloat *class_confidence;
|
||||
* GQuark *classes;
|
||||
*
|
||||
* ... (segmentation filling segmask based on img)
|
||||
*
|
||||
* gst_analytics_relation_meta_add_segmentation_mtd (rmeta, segmask,
|
||||
* GST_SEGMENTATION_TYPE_INSTANCE, region_count, region_ids, &segmtd);
|
||||
* class_count = region_count;
|
||||
*
|
||||
* ... (class-index must match and correspond to region-index)
|
||||
* classes [0] = g_quark_from_string ("background");
|
||||
* classes [1] = g_quark_from_string ("strawberry");
|
||||
* classes [2] = g_quark_from_string ("strawberry");
|
||||
* classes [3] = g_quark_from_string ("leaf");
|
||||
*
|
||||
* ... (set confidence level for each class associated with a region
|
||||
* ... where -1.0 mean undefined.)
|
||||
* class_confidence [0] = -1.0;
|
||||
* class_confidence [1] = 0.6;
|
||||
* class_confidence [2] = 0.9;
|
||||
* class_confidence [3] = 0.8;
|
||||
*
|
||||
* gst_analytics_relation_meta_add_cls_mtd (rmeta, class_count,
|
||||
* class_confidence, classes, &clsmtd);
|
||||
*
|
||||
* gst_analytics_relation_meta_set_relation (rmeta,
|
||||
* GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id);
|
||||
* ```
|
||||
*
|
||||
* Example: Associate Semantic Segmentation with Classification
|
||||
* Assuming the same context as for Instance Segmentation above but instead
|
||||
* a semantic segmentation is performed, therfore region-id-12 and region-id-7
|
||||
* are now represented by the same region-id-12
|
||||
*
|
||||
* region_ids: (here
|
||||
* |region-index | region-id |
|
||||
* |-------------|-----------|
|
||||
* | 0 | 0 |
|
||||
* | 1 | 12 |
|
||||
* | 2 | 31 |
|
||||
*
|
||||
* Code remain the same except that we set all confidence level to undefined
|
||||
* (-1.0).
|
||||
*
|
||||
* ```
|
||||
* ... (class-index must match and correspond to region-index)
|
||||
* classes [0] = g_quark_from_string ("background");
|
||||
* classes [1] = g_quark_from_string ("strawberry");
|
||||
* classes [2] = g_quark_from_string ("leaf");
|
||||
*
|
||||
* ... (set confidence level for each class associated with a region
|
||||
* ... where -1.0 mean undefined.)
|
||||
* class_confidence [0] = -1.0;
|
||||
* class_confidence [1] = -1.0;
|
||||
* class_confidence [2] = -1.0;
|
||||
*
|
||||
* gst_analytics_relation_meta_add_cls_mtd (rmeta, class_count,
|
||||
* class_confidence, classes, &clsmtd);
|
||||
*
|
||||
* gst_analytics_relation_meta_set_relation (rmeta,
|
||||
* GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id);
|
||||
* ```
|
||||
*
|
||||
* Example: Retriving class associated with a segmentation region-id-12
|
||||
* This the typical case for an overlay as we visit the segmentation mask we
|
||||
* we find region-id values
|
||||
*
|
||||
* ```
|
||||
* gsize idx;
|
||||
* gst_analytics_segmentation_mtd_get_region_index (&segmtd, &idx, 12);
|
||||
* gst_analytics_relation_meta_get_direct_related (rmeta, segmtd.id,
|
||||
* GST_ANALYTICS_REL_TYPE_RELATE_TO, gst_analytics_cls_mtd_get_type (),
|
||||
* NULL, &clsmtd);
|
||||
*
|
||||
* GQuark region_class = gst_analytics_cls_mtd_get_quark (&segmtd, idx)
|
||||
* ...
|
||||
* ```
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
|
||||
static void gst_analytics_segmentation_mtd_clear (GstBuffer * buffer,
|
||||
GstAnalyticsMtd * mtd);
|
||||
|
||||
static gboolean
|
||||
gst_analytics_segmentation_mtd_transform (GstBuffer * transbuf,
|
||||
GstAnalyticsMtd * transmtd, GstBuffer * buffer, GQuark type, gpointer data);
|
||||
|
||||
static const GstAnalyticsMtdImpl segmentation_impl = {
|
||||
"segmentation",
|
||||
gst_analytics_segmentation_mtd_transform,
|
||||
gst_analytics_segmentation_mtd_clear
|
||||
};
|
||||
|
||||
/*
|
||||
* GstAnalyticsSegMtdData:
|
||||
* @type: #GstSegmentationType indicate if the mask values are object/region-id
|
||||
* (in the case of instance segmentation) or object/region-type (in the case
|
||||
* of semantic segmentation).
|
||||
* @masks: #GstBuffer used to store segmentation masks
|
||||
* @region_count: Number of region in the segmentation masks
|
||||
* @region_ids: Indexed region ids
|
||||
*
|
||||
* Store segmentation results where each value represent a group to which
|
||||
* belong the corresponding pixel from original image where segmentation was
|
||||
* performed. All values equal in @masks form a mask defining all the
|
||||
* pixel belonging to the same segmented region from the original image. The
|
||||
* GstVideoMeta attached to the @masks, describe masks resolution, padding,
|
||||
* format, ... The format in video meta has a special meaning in the context
|
||||
* of the mask, GRAY8 mean that @masks value can take 256 values which mean
|
||||
* 256 segmented region can be represented.
|
||||
*
|
||||
*/
|
||||
typedef struct _GstAnalyticsSegMtdData
|
||||
{
|
||||
GstSegmentationType type;
|
||||
GstBuffer *masks;
|
||||
gsize region_count;
|
||||
guint32 region_ids[]; /* Must be last */
|
||||
} GstAnalyticsSegMtdData;
|
||||
|
||||
/**
|
||||
* gst_analytics_segmentation_mtd_get_mtd_type:
|
||||
*
|
||||
* Get an instance of #GstAnalyticsMtdType that represent segmentation
|
||||
* metadata type.
|
||||
*
|
||||
* Returns: A #GstAnalyticsMtdType type
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
GstAnalyticsMtdType
|
||||
gst_analytics_segmentation_mtd_get_mtd_type (void)
|
||||
{
|
||||
return (GstAnalyticsMtdType) & segmentation_impl;
|
||||
}
|
||||
|
||||
/**
|
||||
* gst_analytics_segmentation_mtd_get_mask:
|
||||
* @handle: Instance
|
||||
*
|
||||
* Get segmentation mask data.
|
||||
*
|
||||
* Returns: Segmentation mask data stored in a #GstBuffer
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
GstBuffer *
|
||||
gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle)
|
||||
{
|
||||
GstAnalyticsSegMtdData *mtddata;
|
||||
|
||||
g_return_val_if_fail (handle, FALSE);
|
||||
|
||||
mtddata = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id);
|
||||
g_return_val_if_fail (mtddata != NULL, NULL);
|
||||
|
||||
return gst_buffer_ref (mtddata->masks);
|
||||
}
|
||||
|
||||
/**
|
||||
* gst_analytics_segmentation_mtd_get_region_index:
|
||||
* @handle: Instance
|
||||
* @index: (out caller-allocates)(not nullable): Region index
|
||||
* @id: Region id
|
||||
*
|
||||
* Get region index of the region identified by @id.
|
||||
*
|
||||
* Returns: TRUE if a region with @id exist, otherwise FALSE
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
gboolean
|
||||
gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd *
|
||||
handle, gsize * index, guint id)
|
||||
{
|
||||
GstAnalyticsSegMtdData *mtddata;
|
||||
|
||||
g_return_val_if_fail (handle, FALSE);
|
||||
g_return_val_if_fail (index != NULL, FALSE);
|
||||
|
||||
mtddata = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id);
|
||||
g_return_val_if_fail (mtddata != NULL, FALSE);
|
||||
|
||||
gsize i;
|
||||
for (i = 0; i < mtddata->region_count; i++) {
|
||||
if (mtddata->region_ids[i] == id) {
|
||||
*index = i;
|
||||
return TRUE;
|
||||
}
|
||||
}
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
/**
|
||||
* gst_analytics_segmentation_mtd_get_region_id:
|
||||
* @handle: Instance
|
||||
* @index: Region index
|
||||
*
|
||||
* Get id of the region corresponding to @index. which should be
|
||||
* smaller than the return value of
|
||||
* gst_analytics_segmentation_mtd_get_region_count()
|
||||
*
|
||||
* Returns: The region ID
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
guint
|
||||
gst_analytics_segmentation_mtd_get_region_id (GstAnalyticsSegmentationMtd *
|
||||
handle, gsize index)
|
||||
{
|
||||
GstAnalyticsSegMtdData *mtddata;
|
||||
|
||||
g_return_val_if_fail (handle, 0);
|
||||
|
||||
mtddata = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id);
|
||||
g_return_val_if_fail (mtddata != NULL, 0);
|
||||
g_return_val_if_fail (index < mtddata->region_count, 0);
|
||||
|
||||
return mtddata->region_ids[index];
|
||||
}
|
||||
|
||||
/**
|
||||
* gst_analytics_segmentation_mtd_get_region_count:
|
||||
* @handle: Instance
|
||||
*
|
||||
* Get the regions count.
|
||||
*
|
||||
* Returns: Number of regions segmented
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
gsize
|
||||
gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd *
|
||||
handle)
|
||||
{
|
||||
GstAnalyticsSegMtdData *mtddata;
|
||||
|
||||
g_return_val_if_fail (handle, FALSE);
|
||||
|
||||
mtddata = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id);
|
||||
g_return_val_if_fail (mtddata != NULL, FALSE);
|
||||
|
||||
return mtddata->region_count;
|
||||
}
|
||||
|
||||
/**
|
||||
* gst_analytics_relation_meta_add_segmentation_mtd:
|
||||
* @instance: Instance of #GstAnalyticsRelationMeta where to add segmentation
|
||||
* instance.
|
||||
* @buffer:(in)(transfer full): Buffer containing segmentation masks. @buffer
|
||||
* must have a #GstVideoMeta attached
|
||||
* @segmentation_type:(in): Segmentation type
|
||||
* @region_count:(in): Number of regions in the masks
|
||||
* @region_ids:(in) (array length=region_count): Arrays of region ids present in the mask.
|
||||
* @segmentation_mtd:(out)(not nullable): Handle update with newly added segmenation meta.
|
||||
*
|
||||
* Add analytics segmentation metadata to @instance.
|
||||
*
|
||||
* Returns: TRUE if added successfully, otherwise FALSE
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
gboolean
|
||||
gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta *
|
||||
instance, GstBuffer * buffer, GstSegmentationType segmentation_type,
|
||||
gsize region_count, guint * region_ids, GstAnalyticsSegmentationMtd *
|
||||
segmentation_mtd)
|
||||
{
|
||||
const gsize region_ids_size = sizeof (guint) * region_count;
|
||||
const gsize size = sizeof (GstAnalyticsSegMtdData) + region_ids_size;
|
||||
GstVideoMeta *vmeta = gst_buffer_get_video_meta (buffer);
|
||||
g_return_val_if_fail (vmeta != NULL, FALSE);
|
||||
g_return_val_if_fail (instance != NULL, FALSE);
|
||||
g_return_val_if_fail (vmeta->format == GST_VIDEO_FORMAT_GRAY8 ||
|
||||
vmeta->format == GST_VIDEO_FORMAT_GRAY16_BE ||
|
||||
vmeta->format == GST_VIDEO_FORMAT_GRAY16_LE, FALSE);
|
||||
|
||||
GstAnalyticsSegMtdData *mtddata = NULL;
|
||||
mtddata =
|
||||
(GstAnalyticsSegMtdData *) gst_analytics_relation_meta_add_mtd (instance,
|
||||
&segmentation_impl, size, segmentation_mtd);
|
||||
|
||||
if (mtddata) {
|
||||
mtddata->masks = buffer;
|
||||
mtddata->type = segmentation_type;
|
||||
mtddata->region_count = region_count;
|
||||
memcpy (mtddata->region_ids, region_ids, region_ids_size);
|
||||
}
|
||||
|
||||
return mtddata != NULL;
|
||||
}
|
||||
|
||||
static void
|
||||
gst_analytics_segmentation_mtd_clear (GstBuffer * buffer, GstAnalyticsMtd * mtd)
|
||||
{
|
||||
GstAnalyticsSegMtdData *segdata;
|
||||
segdata = gst_analytics_relation_meta_get_mtd_data (mtd->meta, mtd->id);
|
||||
g_return_if_fail (segdata != NULL);
|
||||
gst_clear_buffer (&segdata->masks);
|
||||
}
|
||||
|
||||
static gboolean
|
||||
gst_analytics_segmentation_mtd_transform (GstBuffer * transbuf,
|
||||
GstAnalyticsMtd * transmtd, GstBuffer * buffer, GQuark type, gpointer data)
|
||||
{
|
||||
GstAnalyticsSegMtdData *segdata;
|
||||
if (GST_META_TRANSFORM_IS_COPY (type)) {
|
||||
segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta,
|
||||
transmtd->id);
|
||||
gst_buffer_ref (segdata->masks);
|
||||
} else if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type) && transbuf != buffer) {
|
||||
segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta,
|
||||
transmtd->id);
|
||||
gst_buffer_ref (segdata->masks);
|
||||
}
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
Loading…
Reference in a new issue