diff --git a/girs/GstAnalytics-1.0.gir b/girs/GstAnalytics-1.0.gir index 0e7e1e8a1c..afa602c595 100644 --- a/girs/GstAnalytics-1.0.gir +++ b/girs/GstAnalytics-1.0.gir @@ -4,6 +4,7 @@ and/or use gtk-doc annotations. --> + @@ -16,11 +17,11 @@ and/or use gtk-doc annotations. --> is generally expected to be allocated on the stack. - Instance identifier. + Instance identifier - Instance of #GstAnalyticsRelationMeta where the analysis-metadata + Instance of #GstAnalyticsRelationMeta where the analytics-metadata identified by @id is stored. @@ -125,11 +126,11 @@ infinite. is generally expected to be allocated on the stack. - Instance identifier. + Instance identifier - Instance of #GstAnalyticsRelationMeta where the analysis-metadata + Instance of #GstAnalyticsRelationMeta where the analytics-metadata identified by @id is stored. @@ -223,8 +224,24 @@ have a static lifetime (never be freed). + + + + + + + + + + + + + + + + - + @@ -234,11 +251,11 @@ have a static lifetime (never be freed). is generally expected to be allocated on the stack. - Instance identifier. + Instance identifier - Instance of #GstAnalyticsRelationMeta where the analysis-metadata + Instance of #GstAnalyticsRelationMeta where the analytics-metadata identified by @id is stored. @@ -476,6 +493,44 @@ new struct sub-classing GstAnalyticsRelatableMtd. + + Add analytics segmentation metadata to @instance. + + + TRUE if added successfully, otherwise FALSE + + + + + Instance of #GstAnalyticsRelationMeta where to add segmentation +instance. + + + + Buffer containing segmentation masks. @buffer +must have a #GstVideoMeta attached + + + + Segmentation type + + + + Number of regions in the masks + + + + Arrays of region ids present in the mask. + + + + + + Handle update with newly added segmenation meta. + + + + @@ -806,15 +861,244 @@ Ids must have been obtained a call to + + This type of metadata holds information on which pixels belongs to + a region of the image representing a type of object. + + It supports two types of segmentation, semantic or instance: + * Semantic: All objects of the same type have the same id + * Instance: Each instance of an object has a different id + + The results of the segmentation are stored in a #GstBuffer that has a + #GstVideoMeta associated with it. This buffer is stored in the + GstAnalyticsSegmentationMtd using + #gst_analytics_relation_meta_add_segmentation_mtd(). The #GstBuffer + containing the segmentation mask is image-like but the color values are + arbitrary values, referred by region-id in this API, without meaning beyond + specifying that two pixels in the original image with the same values in + their corresponding mask value belong to the same region. + + To further describe a region, the #GstAnalyticsSegmentationMtd can be + associated with other #GstAnalyticsMtd. Since region ids are + generated by the segmentation process itself and are not always sequential, + we use a map of indexes to region ids starting with 0 without discontinuity + which facilitate N-to-N mapping with other #GstAnalyticsMtd. For + example it can be associated with #GstAnalyticsClsMtd to describe the class + of object matching the pixels of a segmented region. + + Example: Associate Instance Segmentation with Classification + + In the following example the segmentation process will fill segmask with + values of 0 for background, 12 for the first region which correspond to a + to a strawberry, 7 for the second region that also correspond to a + strawberry in the image and 31 for the third region that correspond to a + leaf in the image. + region_ids is fill during segmentation post-processing + + region_ids: + |region-index | region-id | + |-------------|-----------| + | 0 | 0 | + | 1 | 12 | + | 2 | 7 | + | 3 | 31 | + + region_count = 4 + + ``` C + GstAnalyticsSegmentationMtd segmtd; + GstAnalyticsClassificationMtd clsmtd; + GstBuffer *segmask, *img; + guint *region_ids; + gsize region_count, class_count; + gfloat *class_confidence; + GQuark *classes; + + ... (segmentation filling segmask based on img) + + gst_analytics_relation_meta_add_segmentation_mtd (rmeta, segmask, + GST_SEGMENTATION_TYPE_INSTANCE, region_count, region_ids, &segmtd); + class_count = region_count; + + ... (class-index must match and correspond to region-index) + classes [0] = g_quark_from_string ("background"); + classes [1] = g_quark_from_string ("strawberry"); + classes [2] = g_quark_from_string ("strawberry"); + classes [3] = g_quark_from_string ("leaf"); + + ... (set confidence level for each class associated with a region + ... where -1.0 mean undefined.) + class_confidence [0] = -1.0; + class_confidence [1] = 0.6; + class_confidence [2] = 0.9; + class_confidence [3] = 0.8; + + gst_analytics_relation_meta_add_cls_mtd (rmeta, class_count, + class_confidence, classes, &clsmtd); + + gst_analytics_relation_meta_set_relation (rmeta, + GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id); + ``` + + Example: Associate Semantic Segmentation with Classification + Assuming the same context as for Instance Segmentation above but instead + a semantic segmentation is performed, therfore region-id-12 and region-id-7 + are now represented by the same region-id-12 + + region_ids: (here + |region-index | region-id | + |-------------|-----------| + | 0 | 0 | + | 1 | 12 | + | 2 | 31 | + + Code remain the same except that we set all confidence level to undefined + (-1.0). + + ``` + ... (class-index must match and correspond to region-index) + classes [0] = g_quark_from_string ("background"); + classes [1] = g_quark_from_string ("strawberry"); + classes [2] = g_quark_from_string ("leaf"); + + ... (set confidence level for each class associated with a region + ... where -1.0 mean undefined.) + class_confidence [0] = -1.0; + class_confidence [1] = -1.0; + class_confidence [2] = -1.0; + + gst_analytics_relation_meta_add_cls_mtd (rmeta, class_count, + class_confidence, classes, &clsmtd); + + gst_analytics_relation_meta_set_relation (rmeta, + GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id); + ``` + + Example: Retriving class associated with a segmentation region-id-12 + This the typical case for an overlay as we visit the segmentation mask we + we find region-id values + + ``` + gsize idx; + gst_analytics_segmentation_mtd_get_region_index (&segmtd, &idx, 12); + gst_analytics_relation_meta_get_direct_related (rmeta, segmtd.id, + GST_ANALYTICS_REL_TYPE_RELATE_TO, gst_analytics_cls_mtd_get_type (), + NULL, &clsmtd); + + GQuark region_class = gst_analytics_cls_mtd_get_quark (&segmtd, idx) + ... + ``` + + Since: 1.26 + + + Instance identifier + + + + Instance of #GstAnalyticsRelationMeta where the analytics-metadata +identified by @id is stored. + + + + Get segmentation mask data. + + + Segmentation mask data stored in a #GstBuffer + + + + + Instance + + + + + + Get the regions count. + + + Number of regions segmented + + + + + Instance + + + + + + Get id of the region corresponding to @index. which should be +smaller than the return value of +gst_analytics_segmentation_mtd_get_region_count() + + + The region ID + + + + + Instance + + + + Region index + + + + + + Get region index of the region identified by @id. + + + TRUE if a region with @id exist, otherwise FALSE + + + + + Instance + + + + Region index + + + + Region id + + + + + + Get an instance of #GstAnalyticsMtdType that represent segmentation +metadata type. + + + A #GstAnalyticsMtdType type + + + + + + Enum value describing supported segmentation type + + + Each region of the same type have the same id. + + + Each instance of a region have different id. + + Store information on results of object tracking - Instance identifier. + Instance identifier - Instance of #GstAnalyticsRelationMeta where the analysis-metadata + Instance of #GstAnalyticsRelationMeta where the analytics-metadata identified by @id is stored. @@ -1023,6 +1307,15 @@ information about the tracking, for example, it can be used alongside a + + Get an instance of #GstAnalyticsMtdType that represent segmentation +metadata type. + + + A #GstAnalyticsMtdType type + + + diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h index d25cf3ba35..5ae6143515 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h @@ -30,5 +30,6 @@ #include #include #include +#include #endif /* __ANALYTICS_H__ */ diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c index 776dd3d8ba..2661fb7602 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c @@ -115,7 +115,7 @@ * * Example: Associate Semantic Segmentation with Classification * Assuming the same context as for Instance Segmentation above but instead - * a semantic segmentation is performed, therfore region-id-12 and region-id-7 + * a semantic segmentation is performed, therefore region-id-12 and region-id-7 * are now represented by the same region-id-12 * * region_ids: (here @@ -147,7 +147,7 @@ * GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id); * ``` * - * Example: Retriving class associated with a segmentation region-id-12 + * Example: Retrieving class associated with a segmentation region-id-12 * This the typical case for an overlay as we visit the segmentation mask we * we find region-id values * @@ -201,6 +201,12 @@ typedef struct _GstAnalyticsSegMtdData { GstSegmentationType type; GstBuffer *masks; + + gint masks_loc_x; + gint masks_loc_y; + guint masks_loc_w; + guint masks_loc_h; + gsize region_count; guint32 region_ids[]; /* Must be last */ } GstAnalyticsSegMtdData; @@ -224,6 +230,14 @@ gst_analytics_segmentation_mtd_get_mtd_type (void) /** * gst_analytics_segmentation_mtd_get_mask: * @handle: Instance + * @masks_loc_x: (out caller-allocates)(nullable): Left coordinate of the + * rectangle corresponding to the mask in the image. + * @masks_loc_y: (out caller-allocates)(nullable): Top coordinate of the + * rectangle corresponding to the mask in the image. + * @masks_loc_w: (out caller-allocates)(nullable): Width of the rectangle + * corresponding to the mask in the image. + * @masks_loc_h: (out caller-allocates)(nullable): Height of the rectangle + * corresponding to the mask in the image. * * Get segmentation mask data. * @@ -232,7 +246,9 @@ gst_analytics_segmentation_mtd_get_mtd_type (void) * Since: 1.26 */ GstBuffer * -gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle) +gst_analytics_segmentation_mtd_get_mask (const GstAnalyticsSegmentationMtd * + handle, gint * masks_loc_x, gint * masks_loc_y, guint * masks_loc_w, guint * + masks_loc_h) { GstAnalyticsSegMtdData *mtddata; @@ -241,6 +257,15 @@ gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle) mtddata = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id); g_return_val_if_fail (mtddata != NULL, NULL); + if (masks_loc_x) + *masks_loc_x = mtddata->masks_loc_x; + if (masks_loc_y) + *masks_loc_y = mtddata->masks_loc_y; + if (masks_loc_w) + *masks_loc_w = mtddata->masks_loc_w; + if (masks_loc_h) + *masks_loc_h = mtddata->masks_loc_h; + return gst_buffer_ref (mtddata->masks); } @@ -257,8 +282,8 @@ gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle) * Since: 1.26 */ gboolean -gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd * - handle, gsize * index, guint id) +gst_analytics_segmentation_mtd_get_region_index (const + GstAnalyticsSegmentationMtd * handle, gsize * index, guint id) { GstAnalyticsSegMtdData *mtddata; @@ -283,7 +308,7 @@ gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd * * @handle: Instance * @index: Region index * - * Get id of the region corresponding to @index. which should be + * Get id of the region corresponding to @index, which should be * smaller than the return value of * gst_analytics_segmentation_mtd_get_region_count() * @@ -292,8 +317,8 @@ gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd * * Since: 1.26 */ guint -gst_analytics_segmentation_mtd_get_region_id (GstAnalyticsSegmentationMtd * - handle, gsize index) +gst_analytics_segmentation_mtd_get_region_id (const + GstAnalyticsSegmentationMtd * handle, gsize index) { GstAnalyticsSegMtdData *mtddata; @@ -317,8 +342,8 @@ gst_analytics_segmentation_mtd_get_region_id (GstAnalyticsSegmentationMtd * * Since: 1.26 */ gsize -gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd * - handle) +gst_analytics_segmentation_mtd_get_region_count (const + GstAnalyticsSegmentationMtd * handle) { GstAnalyticsSegMtdData *mtddata; @@ -339,9 +364,18 @@ gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd * * @segmentation_type:(in): Segmentation type * @region_count:(in): Number of regions in the masks * @region_ids:(in) (array length=region_count): Arrays of region ids present in the mask. - * @segmentation_mtd:(out)(not nullable): Handle update with newly added segmenation meta. + * @masks_loc_x:(in): Left coordinate of the rectangle corresponding to the masks in the image. + * @masks_loc_y:(in): Top coordinate of the rectangle corresponding to the masks in the image. + * @masks_loc_w:(in): Width of the rectangle corresponding to the masks in the image. + * @masks_loc_h:(in): Height of the rectangle corresponding to the masks in the image. + * @segmentation_mtd:(out)(not nullable): Handle update with newly added segmentation meta. * - * Add analytics segmentation metadata to @instance. + * Add analytics segmentation metadata to @instance. The rectangle (@masks_loc_x, + * @mask_loc_y, @mask_loc_w, @mask_loc_h) define a area of the image that + * correspond to the segmentation masks stored in @buffer. For example if the + * segmentation masks stored in @buffer describe the segmented regions for the + * entire image the rectangular area will be (@masks_loc_x = 0, @masks_loc_y = 0, + * @masks_loc_w = image_width, @masks_loc_h = image_height). * * Returns: TRUE if added successfully, otherwise FALSE * @@ -350,7 +384,8 @@ gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd * gboolean gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta * instance, GstBuffer * buffer, GstSegmentationType segmentation_type, - gsize region_count, guint * region_ids, GstAnalyticsSegmentationMtd * + gsize region_count, guint * region_ids, gint masks_loc_x, gint masks_loc_y, + guint masks_loc_w, guint masks_loc_h, GstAnalyticsSegmentationMtd * segmentation_mtd) { const gsize region_ids_size = sizeof (guint) * region_count; @@ -371,6 +406,10 @@ gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta * mtddata->masks = buffer; mtddata->type = segmentation_type; mtddata->region_count = region_count; + mtddata->masks_loc_x = masks_loc_x; + mtddata->masks_loc_y = masks_loc_y; + mtddata->masks_loc_w = masks_loc_w; + mtddata->masks_loc_h = masks_loc_h; memcpy (mtddata->region_ids, region_ids, region_ids_size); } @@ -395,10 +434,33 @@ gst_analytics_segmentation_mtd_transform (GstBuffer * transbuf, segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta, transmtd->id); gst_buffer_ref (segdata->masks); - } else if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type) && transbuf != buffer) { + } else if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) { + GstVideoMetaTransform *trans = data; + gint ow, oh, nw, nh; + + ow = GST_VIDEO_INFO_WIDTH (trans->in_info); + nw = GST_VIDEO_INFO_WIDTH (trans->out_info); + oh = GST_VIDEO_INFO_HEIGHT (trans->in_info); + nh = GST_VIDEO_INFO_HEIGHT (trans->out_info); + segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta, transmtd->id); - gst_buffer_ref (segdata->masks); + + segdata->masks_loc_x *= nw; + segdata->masks_loc_x /= ow; + + segdata->masks_loc_w *= nw; + segdata->masks_loc_w /= ow; + + segdata->masks_loc_y *= nh; + segdata->masks_loc_y /= oh; + + segdata->masks_loc_h *= nh; + segdata->masks_loc_h /= oh; + + if (transbuf != buffer) { + gst_buffer_ref (segdata->masks); + } } return TRUE; diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h new file mode 100644 index 0000000000..578dd6cbeb --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h @@ -0,0 +1,97 @@ +/* GStreamer + * Copyright (C) 2024 Collabora Ltd + * @author: Daniel Morin + * + * gstanalyticssegmentationmtd.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_ANALYTICS_SEGMENTATION_META_H__ +#define __GST_ANALYTICS_SEGMENTATION_META_H__ + +#include +#include +#include +#include + +G_BEGIN_DECLS + +/** + * GstAnalyticsSegmentationMtd: + * @id: Instance identifier + * @meta: Instance of #GstAnalyticsRelationMeta where the analytics-metadata + * identified by @id is stored. + * + * Handle containing data required to use gst_analytics_segmentation_mtd APIs. + * This type is generally expected to be allocated on the stack. + * + * Since: 1.26 + */ +typedef struct _GstAnalyticsMtd GstAnalyticsSegmentationMtd; + +/** + * GstSegmentationType: + * @GST_SEGMENTATION_TYPE_SEMANTIC: Segmentation where the belonging of each + * pixel to a class of objects is identified. + * @GST_SEGMENTATION_TYPE_INSTANCE: Segmentation where the belonging of each + * pixel to instance of an object is identified. + * + * Enum value describing supported segmentation type + * + * Since: 1.26 + */ +typedef enum +{ + GST_SEGMENTATION_TYPE_SEMANTIC, + GST_SEGMENTATION_TYPE_INSTANCE +} GstSegmentationType; + +GST_ANALYTICS_META_API +GstAnalyticsMtdType +gst_analytics_segmentation_mtd_get_mtd_type (void); + +GST_ANALYTICS_META_API +GstBuffer * +gst_analytics_segmentation_mtd_get_mask (const GstAnalyticsSegmentationMtd * + handle, gint * masks_loc_x, gint * masks_loc_y, guint * masks_loc_w, guint * + masks_loc_h); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_segmentation_mtd_get_region_index ( + const GstAnalyticsSegmentationMtd * handle, gsize * index, guint id); + +GST_ANALYTICS_META_API +guint +gst_analytics_segmentation_mtd_get_region_id ( + const GstAnalyticsSegmentationMtd * handle, gsize index); + +GST_ANALYTICS_META_API +gsize +gst_analytics_segmentation_mtd_get_region_count ( + const GstAnalyticsSegmentationMtd * handle); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta * + instance, GstBuffer * buffer, GstSegmentationType segmentation_type, + gsize region_count, guint * region_ids, gint masks_loc_x, gint masks_loc_y, + guint masks_loc_w, guint masks_loc_h, GstAnalyticsSegmentationMtd * + segmentation_mtd); + +G_END_DECLS +#endif // __GST_ANALYTICS_SEGMENTATION_META_H__ diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build index 7fdbdd674c..053b0b2234 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build @@ -1,14 +1,17 @@ analytics_sources = files( 'gstanalyticsmeta.c', 'gstanalyticsclassificationmtd.c', 'gstanalyticsobjectdetectionmtd.c', - 'gstanalyticsobjecttrackingmtd.c') + 'gstanalyticsobjecttrackingmtd.c', + 'gstanalyticssegmentationmtd.c') analytics_headers = files( 'analytics.h', 'gstanalyticsmeta.h', 'analytics-meta-prelude.h', 'gstanalyticsclassificationmtd.h', 'gstanalyticsobjectdetectionmtd.h', - 'gstanalyticsobjecttrackingmtd.h') + 'gstanalyticsobjecttrackingmtd.h', + 'gstanalyticssegmentationmtd.h') + install_headers(analytics_headers, subdir : 'gstreamer-1.0/gst/analytics') pkg_name = 'gstreamer-analytics-1.0' @@ -40,7 +43,7 @@ if build_gir 'identifier_prefix' : ['GstAnalytics', 'Gst'], 'symbol_prefix' : ['gst_analytics', 'gst'], 'export_packages' : pkg_name, - 'includes' : ['Gst-1.0', 'GstBase-1.0'], + 'includes' : ['Gst-1.0', 'GstBase-1.0', 'GstVideo-1.0'], 'install' : true, 'extra_args' : gir_init_section + ['-DGST_USE_UNSTABLE_API'], 'dependencies' : [gstbase_dep, gstvideo_dep]