mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-06-06 15:38:53 +00:00
Code Issues 1 Projects Releases Wiki Activity

analytics: add segmentation analytics-meta

Browse source 
- Add a new analytics-meta to store segmentation analysis result.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6026>
This commit is contained in:
Daniel Morin 2024-07-08 17:52:24 -04:00 committed by GStreamer Marge Bot
parent 98653aa43a
commit 6db6d44ff5
5 changed files with 483 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

311
girs/GstAnalytics-1.0.gir
View file

@ -4,6 +4,7 @@ and/or use gtk-doc annotations. -->
<repository xmlns="http://www.gtk.org/introspection/core/1.0" xmlns:c="http://www.gtk.org/introspection/c/1.0" xmlns:glib="http://www.gtk.org/introspection/glib/1.0" version="1.2"> <repository xmlns="http://www.gtk.org/introspection/core/1.0" xmlns:c="http://www.gtk.org/introspection/c/1.0" xmlns:glib="http://www.gtk.org/introspection/glib/1.0" version="1.2">
<include name="Gst" version="1.0"/> <include name="Gst" version="1.0"/>
<include name="GstBase" version="1.0"/> <include name="GstBase" version="1.0"/>
<include name="GstVideo" version="1.0"/>
<package name="gstreamer-analytics-1.0"/> <package name="gstreamer-analytics-1.0"/>
<namespace name="GstAnalytics" version="1.0" shared-library="libgstanalytics-1.0.so.0" c:identifier-prefixes="GstAnalytics,Gst" c:symbol-prefixes="gst_analytics,gst"> <namespace name="GstAnalytics" version="1.0" shared-library="libgstanalytics-1.0.so.0" c:identifier-prefixes="GstAnalytics,Gst" c:symbol-prefixes="gst_analytics,gst">
<alias name="MtdType" c:type="GstAnalyticsMtdType"> <alias name="MtdType" c:type="GstAnalyticsMtdType">
@ -16,11 +17,11 @@ and/or use gtk-doc annotations. -->
is generally expected to be allocated on the stack.</doc> is generally expected to be allocated on the stack.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.h"/> <source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.h"/>
<field name="id" writable="1"> <field name="id" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance identifier.</doc> <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
<type name="guint" c:type="guint"/> <type name="guint" c:type="guint"/>
</field> </field>
<field name="meta" writable="1"> <field name="meta" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance of #GstAnalyticsRelationMeta where the analysis-metadata <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
identified by @id is stored.</doc> identified by @id is stored.</doc>
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/> <type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
</field> </field>
@ -125,11 +126,11 @@ infinite.</doc>
is generally expected to be allocated on the stack.</doc> is generally expected to be allocated on the stack.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h"/> <source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h"/>
<field name="id" writable="1"> <field name="id" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance identifier.</doc> <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
<type name="guint" c:type="guint"/> <type name="guint" c:type="guint"/>
</field> </field>
<field name="meta" writable="1"> <field name="meta" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance of #GstAnalyticsRelationMeta where the analysis-metadata <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
identified by @id is stored.</doc> identified by @id is stored.</doc>
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/> <type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
</field> </field>
@ -223,8 +224,24 @@ have a static lifetime (never be freed).</doc>
</parameters> </parameters>
</callback> </callback>
</field> </field>
<field name="mtd_meta_clear">
<callback name="mtd_meta_clear">
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h"/>
<return-value transfer-ownership="none">
<type name="none" c:type="void"/>
</return-value>
<parameters>
<parameter name="buffer" transfer-ownership="none">
<type name="Gst.Buffer" c:type="GstBuffer*"/>
</parameter>
<parameter name="mtd" transfer-ownership="none">
<type name="Mtd" c:type="GstAnalyticsMtd*"/>
</parameter>
</parameters>
</callback>
</field>
<field name="_reserved" readable="0" private="1"> <field name="_reserved" readable="0" private="1">
<array zero-terminated="0" fixed-size="20"> <array zero-terminated="0" fixed-size="19">
<type name="gpointer" c:type="gpointer"/> <type name="gpointer" c:type="gpointer"/>
</array> </array>
</field> </field>
@ -234,11 +251,11 @@ have a static lifetime (never be freed).</doc>
is generally expected to be allocated on the stack.</doc> is generally expected to be allocated on the stack.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h"/> <source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h"/>
<field name="id" writable="1"> <field name="id" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance identifier.</doc> <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
<type name="guint" c:type="guint"/> <type name="guint" c:type="guint"/>
</field> </field>
<field name="meta" writable="1"> <field name="meta" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance of #GstAnalyticsRelationMeta where the analysis-metadata <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
identified by @id is stored.</doc> identified by @id is stored.</doc>
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/> <type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
</field> </field>
@ -476,6 +493,44 @@ new struct sub-classing GstAnalyticsRelatableMtd.</doc>
</parameter> </parameter>
</parameters> </parameters>
</method> </method>
<method name="add_segmentation_mtd" c:identifier="gst_analytics_relation_meta_add_segmentation_mtd" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Add analytics segmentation metadata to @instance.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<return-value transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">TRUE if added successfully, otherwise FALSE</doc>
<type name="gboolean" c:type="gboolean"/>
</return-value>
<parameters>
<instance-parameter name="instance" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance of #GstAnalyticsRelationMeta where to add segmentation
instance.</doc>
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
</instance-parameter>
<parameter name="buffer" transfer-ownership="full">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Buffer containing segmentation masks. @buffer
must have a #GstVideoMeta attached</doc>
<type name="Gst.Buffer" c:type="GstBuffer*"/>
</parameter>
<parameter name="segmentation_type" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Segmentation type</doc>
<type name="SegmentationType" c:type="GstSegmentationType"/>
</parameter>
<parameter name="region_count" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Number of regions in the masks</doc>
<type name="gsize" c:type="gsize"/>
</parameter>
<parameter name="region_ids" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Arrays of region ids present in the mask.</doc>
<array length="2" zero-terminated="0" c:type="guint*">
<type name="guint" c:type="guint"/>
</array>
</parameter>
<parameter name="segmentation_mtd" direction="out" caller-allocates="1" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Handle update with newly added segmenation meta.</doc>
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
</parameter>
</parameters>
</method>
<method name="add_tracking_mtd" c:identifier="gst_analytics_relation_meta_add_tracking_mtd" version="1.24"> <method name="add_tracking_mtd" c:identifier="gst_analytics_relation_meta_add_tracking_mtd" version="1.24">
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/> <source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/>
<return-value transfer-ownership="none"> <return-value transfer-ownership="none">
@ -806,15 +861,244 @@ Ids must have been obtained a call to
<type name="gsize" c:type="gsize"/> <type name="gsize" c:type="gsize"/>
</field> </field>
</record> </record>
<record name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">This type of metadata holds information on which pixels belongs to
a region of the image representing a type of object.
It supports two types of segmentation, semantic or instance:
* Semantic: All objects of the same type have the same id
* Instance: Each instance of an object has a different id
The results of the segmentation are stored in a #GstBuffer that has a
#GstVideoMeta associated with it. This buffer is stored in the
GstAnalyticsSegmentationMtd using
#gst_analytics_relation_meta_add_segmentation_mtd(). The #GstBuffer
containing the segmentation mask is image-like but the color values are
arbitrary values, referred by region-id in this API, without meaning beyond
specifying that two pixels in the original image with the same values in
their corresponding mask value belong to the same region.
To further describe a region, the #GstAnalyticsSegmentationMtd can be
associated with other #GstAnalyticsMtd. Since region ids are
generated by the segmentation process itself and are not always sequential,
we use a map of indexes to region ids starting with 0 without discontinuity
which facilitate N-to-N mapping with other #GstAnalyticsMtd. For
example it can be associated with #GstAnalyticsClsMtd to describe the class
of object matching the pixels of a segmented region.
Example: Associate Instance Segmentation with Classification
In the following example the segmentation process will fill segmask with
values of 0 for background, 12 for the first region which correspond to a
to a strawberry, 7 for the second region that also correspond to a
strawberry in the image and 31 for the third region that correspond to a
leaf in the image.
region_ids is fill during segmentation post-processing
region_ids:
|region-index | region-id |
|-------------|-----------|
| 0 | 0 |
| 1 | 12 |
| 2 | 7 |
| 3 | 31 |
region_count = 4
``` C
GstAnalyticsSegmentationMtd segmtd;
GstAnalyticsClassificationMtd clsmtd;
GstBuffer *segmask, *img;
guint *region_ids;
gsize region_count, class_count;
gfloat *class_confidence;
GQuark *classes;
... (segmentation filling segmask based on img)
gst_analytics_relation_meta_add_segmentation_mtd (rmeta, segmask,
GST_SEGMENTATION_TYPE_INSTANCE, region_count, region_ids, &amp;segmtd);
class_count = region_count;
... (class-index must match and correspond to region-index)
classes [0] = g_quark_from_string ("background");
classes [1] = g_quark_from_string ("strawberry");
classes [2] = g_quark_from_string ("strawberry");
classes [3] = g_quark_from_string ("leaf");
... (set confidence level for each class associated with a region
... where -1.0 mean undefined.)
class_confidence [0] = -1.0;
class_confidence [1] = 0.6;
class_confidence [2] = 0.9;
class_confidence [3] = 0.8;
gst_analytics_relation_meta_add_cls_mtd (rmeta, class_count,
class_confidence, classes, &amp;clsmtd);
gst_analytics_relation_meta_set_relation (rmeta,
GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id);
```
Example: Associate Semantic Segmentation with Classification
Assuming the same context as for Instance Segmentation above but instead
a semantic segmentation is performed, therfore region-id-12 and region-id-7
are now represented by the same region-id-12
region_ids: (here
|region-index | region-id |
|-------------|-----------|
| 0 | 0 |
| 1 | 12 |
| 2 | 31 |
Code remain the same except that we set all confidence level to undefined
(-1.0).
```
... (class-index must match and correspond to region-index)
classes [0] = g_quark_from_string ("background");
classes [1] = g_quark_from_string ("strawberry");
classes [2] = g_quark_from_string ("leaf");
... (set confidence level for each class associated with a region
... where -1.0 mean undefined.)
class_confidence [0] = -1.0;
class_confidence [1] = -1.0;
class_confidence [2] = -1.0;
gst_analytics_relation_meta_add_cls_mtd (rmeta, class_count,
class_confidence, classes, &amp;clsmtd);
gst_analytics_relation_meta_set_relation (rmeta,
GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id);
```
Example: Retriving class associated with a segmentation region-id-12
This the typical case for an overlay as we visit the segmentation mask we
we find region-id values
```
gsize idx;
gst_analytics_segmentation_mtd_get_region_index (&amp;segmtd, &amp;idx, 12);
gst_analytics_relation_meta_get_direct_related (rmeta, segmtd.id,
GST_ANALYTICS_REL_TYPE_RELATE_TO, gst_analytics_cls_mtd_get_type (),
NULL, &amp;clsmtd);
GQuark region_class = gst_analytics_cls_mtd_get_quark (&amp;segmtd, idx)
...
```
Since: 1.26</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<field name="id" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
<type name="guint" c:type="guint"/>
</field>
<field name="meta" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
identified by @id is stored.</doc>
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
</field>
<method name="get_mask" c:identifier="gst_analytics_segmentation_mtd_get_mask" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get segmentation mask data.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<return-value transfer-ownership="full">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Segmentation mask data stored in a #GstBuffer</doc>
<type name="Gst.Buffer" c:type="GstBuffer*"/>
</return-value>
<parameters>
<instance-parameter name="handle" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance</doc>
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
</instance-parameter>
</parameters>
</method>
<method name="get_region_count" c:identifier="gst_analytics_segmentation_mtd_get_region_count" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get the regions count.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<return-value transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Number of regions segmented</doc>
<type name="gsize" c:type="gsize"/>
</return-value>
<parameters>
<instance-parameter name="handle" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance</doc>
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
</instance-parameter>
</parameters>
</method>
<method name="get_region_id" c:identifier="gst_analytics_segmentation_mtd_get_region_id" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get id of the region corresponding to @index. which should be
smaller than the return value of
gst_analytics_segmentation_mtd_get_region_count()</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<return-value transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">The region ID</doc>
<type name="guint" c:type="guint"/>
</return-value>
<parameters>
<instance-parameter name="handle" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance</doc>
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
</instance-parameter>
<parameter name="index" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Region index</doc>
<type name="gsize" c:type="gsize"/>
</parameter>
</parameters>
</method>
<method name="get_region_index" c:identifier="gst_analytics_segmentation_mtd_get_region_index" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get region index of the region identified by @id.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<return-value transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">TRUE if a region with @id exist, otherwise FALSE</doc>
<type name="gboolean" c:type="gboolean"/>
</return-value>
<parameters>
<instance-parameter name="handle" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance</doc>
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
</instance-parameter>
<parameter name="index" direction="out" caller-allocates="1" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Region index</doc>
<type name="gsize" c:type="gsize*"/>
</parameter>
<parameter name="id" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Region id</doc>
<type name="guint" c:type="guint"/>
</parameter>
</parameters>
</method>
<function name="get_mtd_type" c:identifier="gst_analytics_segmentation_mtd_get_mtd_type" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get an instance of #GstAnalyticsMtdType that represent segmentation
metadata type.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<return-value transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">A #GstAnalyticsMtdType type</doc>
<type name="MtdType" c:type="GstAnalyticsMtdType"/>
</return-value>
</function>
</record>
<enumeration name="SegmentationType" version="1.26" c:type="GstSegmentationType">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Enum value describing supported segmentation type</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<member name="semantic" value="0" c:identifier="GST_SEGMENTATION_TYPE_SEMANTIC" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Each region of the same type have the same id.</doc>
</member>
<member name="instance" value="1" c:identifier="GST_SEGMENTATION_TYPE_INSTANCE" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Each instance of a region have different id.</doc>
</member>
</enumeration>
<record name="TrackingMtd" c:type="GstAnalyticsTrackingMtd" version="1.24"> <record name="TrackingMtd" c:type="GstAnalyticsTrackingMtd" version="1.24">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.c">Store information on results of object tracking</doc> <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.c">Store information on results of object tracking</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/> <source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/>
<field name="id" writable="1"> <field name="id" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance identifier.</doc> <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
<type name="guint" c:type="guint"/> <type name="guint" c:type="guint"/>
</field> </field>
<field name="meta" writable="1"> <field name="meta" writable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance of #GstAnalyticsRelationMeta where the analysis-metadata <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
identified by @id is stored.</doc> identified by @id is stored.</doc>
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/> <type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
</field> </field>
@ -1023,6 +1307,15 @@ information about the tracking, for example, it can be used alongside a
<type name="Gst.MetaInfo" c:type="const GstMetaInfo*"/> <type name="Gst.MetaInfo" c:type="const GstMetaInfo*"/>
</return-value> </return-value>
</function> </function>
<function name="segmentation_mtd_get_mtd_type" c:identifier="gst_analytics_segmentation_mtd_get_mtd_type" moved-to="SegmentationMtd.get_mtd_type" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get an instance of #GstAnalyticsMtdType that represent segmentation
metadata type.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
<return-value transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">A #GstAnalyticsMtdType type</doc>
<type name="MtdType" c:type="GstAnalyticsMtdType"/>
</return-value>
</function>
<function name="tracking_mtd_get_mtd_type" c:identifier="gst_analytics_tracking_mtd_get_mtd_type" moved-to="TrackingMtd.get_mtd_type" version="1.24"> <function name="tracking_mtd_get_mtd_type" c:identifier="gst_analytics_tracking_mtd_get_mtd_type" moved-to="TrackingMtd.get_mtd_type" version="1.24">
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/> <source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/>
<return-value transfer-ownership="none"> <return-value transfer-ownership="none">

1
subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h
View file

@ -30,5 +30,6 @@
#include <gst/analytics/gstanalyticsclassificationmtd.h> #include <gst/analytics/gstanalyticsclassificationmtd.h>
#include <gst/analytics/gstanalyticsobjectdetectionmtd.h> #include <gst/analytics/gstanalyticsobjectdetectionmtd.h>
#include <gst/analytics/gstanalyticsobjecttrackingmtd.h> #include <gst/analytics/gstanalyticsobjecttrackingmtd.h>
#include <gst/analytics/gstanalyticssegmentationmtd.h>
#endif /* __ANALYTICS_H__ */ #endif /* __ANALYTICS_H__ */

92
subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c
View file

@ -115,7 +115,7 @@
* *
* Example: Associate Semantic Segmentation with Classification * Example: Associate Semantic Segmentation with Classification
* Assuming the same context as for Instance Segmentation above but instead * Assuming the same context as for Instance Segmentation above but instead
* a semantic segmentation is performed, therfore region-id-12 and region-id-7 * a semantic segmentation is performed, therefore region-id-12 and region-id-7
* are now represented by the same region-id-12 * are now represented by the same region-id-12
* *
* region_ids: (here * region_ids: (here
@ -147,7 +147,7 @@
* GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id); * GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id);
* ``` * ```
* *
* Example: Retriving class associated with a segmentation region-id-12 * Example: Retrieving class associated with a segmentation region-id-12
* This the typical case for an overlay as we visit the segmentation mask we * This the typical case for an overlay as we visit the segmentation mask we
* we find region-id values * we find region-id values
* *
@ -201,6 +201,12 @@ typedef struct _GstAnalyticsSegMtdData
{ {
GstSegmentationType type; GstSegmentationType type;
GstBuffer *masks; GstBuffer *masks;
gint masks_loc_x;
gint masks_loc_y;
guint masks_loc_w;
guint masks_loc_h;
gsize region_count; gsize region_count;
guint32 region_ids[]; /* Must be last */ guint32 region_ids[]; /* Must be last */
} GstAnalyticsSegMtdData; } GstAnalyticsSegMtdData;
@ -224,6 +230,14 @@ gst_analytics_segmentation_mtd_get_mtd_type (void)
/** /**
* gst_analytics_segmentation_mtd_get_mask: * gst_analytics_segmentation_mtd_get_mask:
* @handle: Instance * @handle: Instance
* @masks_loc_x: (out caller-allocates)(nullable): Left coordinate of the
* rectangle corresponding to the mask in the image.
* @masks_loc_y: (out caller-allocates)(nullable): Top coordinate of the
* rectangle corresponding to the mask in the image.
* @masks_loc_w: (out caller-allocates)(nullable): Width of the rectangle
* corresponding to the mask in the image.
* @masks_loc_h: (out caller-allocates)(nullable): Height of the rectangle
* corresponding to the mask in the image.
* *
* Get segmentation mask data. * Get segmentation mask data.
* *
@ -232,7 +246,9 @@ gst_analytics_segmentation_mtd_get_mtd_type (void)
* Since: 1.26 * Since: 1.26
*/ */
GstBuffer * GstBuffer *
gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle) gst_analytics_segmentation_mtd_get_mask (const GstAnalyticsSegmentationMtd *
handle, gint * masks_loc_x, gint * masks_loc_y, guint * masks_loc_w, guint *
masks_loc_h)
{ {
GstAnalyticsSegMtdData *mtddata; GstAnalyticsSegMtdData *mtddata;
@ -241,6 +257,15 @@ gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle)
mtddata = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id); mtddata = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id);
g_return_val_if_fail (mtddata != NULL, NULL); g_return_val_if_fail (mtddata != NULL, NULL);
if (masks_loc_x)
*masks_loc_x = mtddata->masks_loc_x;
if (masks_loc_y)
*masks_loc_y = mtddata->masks_loc_y;
if (masks_loc_w)
*masks_loc_w = mtddata->masks_loc_w;
if (masks_loc_h)
*masks_loc_h = mtddata->masks_loc_h;
return gst_buffer_ref (mtddata->masks); return gst_buffer_ref (mtddata->masks);
} }
@ -257,8 +282,8 @@ gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle)
* Since: 1.26 * Since: 1.26
*/ */
gboolean gboolean
gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd * gst_analytics_segmentation_mtd_get_region_index (const
handle, gsize * index, guint id) GstAnalyticsSegmentationMtd * handle, gsize * index, guint id)
{ {
GstAnalyticsSegMtdData *mtddata; GstAnalyticsSegMtdData *mtddata;
@ -283,7 +308,7 @@ gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd *
* @handle: Instance * @handle: Instance
* @index: Region index * @index: Region index
* *
* Get id of the region corresponding to @index. which should be * Get id of the region corresponding to @index, which should be
* smaller than the return value of * smaller than the return value of
* gst_analytics_segmentation_mtd_get_region_count() * gst_analytics_segmentation_mtd_get_region_count()
* *
@ -292,8 +317,8 @@ gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd *
* Since: 1.26 * Since: 1.26
*/ */
guint guint
gst_analytics_segmentation_mtd_get_region_id (GstAnalyticsSegmentationMtd * gst_analytics_segmentation_mtd_get_region_id (const
handle, gsize index) GstAnalyticsSegmentationMtd * handle, gsize index)
{ {
GstAnalyticsSegMtdData *mtddata; GstAnalyticsSegMtdData *mtddata;
@ -317,8 +342,8 @@ gst_analytics_segmentation_mtd_get_region_id (GstAnalyticsSegmentationMtd *
* Since: 1.26 * Since: 1.26
*/ */
gsize gsize
gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd * gst_analytics_segmentation_mtd_get_region_count (const
handle) GstAnalyticsSegmentationMtd * handle)
{ {
GstAnalyticsSegMtdData *mtddata; GstAnalyticsSegMtdData *mtddata;
@ -339,9 +364,18 @@ gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd *
* @segmentation_type:(in): Segmentation type * @segmentation_type:(in): Segmentation type
* @region_count:(in): Number of regions in the masks * @region_count:(in): Number of regions in the masks
* @region_ids:(in) (array length=region_count): Arrays of region ids present in the mask. * @region_ids:(in) (array length=region_count): Arrays of region ids present in the mask.
* @segmentation_mtd:(out)(not nullable): Handle update with newly added segmenation meta. * @masks_loc_x:(in): Left coordinate of the rectangle corresponding to the masks in the image.
* @masks_loc_y:(in): Top coordinate of the rectangle corresponding to the masks in the image.
* @masks_loc_w:(in): Width of the rectangle corresponding to the masks in the image.
* @masks_loc_h:(in): Height of the rectangle corresponding to the masks in the image.
* @segmentation_mtd:(out)(not nullable): Handle update with newly added segmentation meta.
* *
* Add analytics segmentation metadata to @instance. * Add analytics segmentation metadata to @instance. The rectangle (@masks_loc_x,
* @mask_loc_y, @mask_loc_w, @mask_loc_h) define a area of the image that
* correspond to the segmentation masks stored in @buffer. For example if the
* segmentation masks stored in @buffer describe the segmented regions for the
* entire image the rectangular area will be (@masks_loc_x = 0, @masks_loc_y = 0,
* @masks_loc_w = image_width, @masks_loc_h = image_height).
* *
* Returns: TRUE if added successfully, otherwise FALSE * Returns: TRUE if added successfully, otherwise FALSE
* *
@ -350,7 +384,8 @@ gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd *
gboolean gboolean
gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta * gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta *
instance, GstBuffer * buffer, GstSegmentationType segmentation_type, instance, GstBuffer * buffer, GstSegmentationType segmentation_type,
gsize region_count, guint * region_ids, GstAnalyticsSegmentationMtd * gsize region_count, guint * region_ids, gint masks_loc_x, gint masks_loc_y,
guint masks_loc_w, guint masks_loc_h, GstAnalyticsSegmentationMtd *
segmentation_mtd) segmentation_mtd)
{ {
const gsize region_ids_size = sizeof (guint) * region_count; const gsize region_ids_size = sizeof (guint) * region_count;
@ -371,6 +406,10 @@ gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta *
mtddata->masks = buffer; mtddata->masks = buffer;
mtddata->type = segmentation_type; mtddata->type = segmentation_type;
mtddata->region_count = region_count; mtddata->region_count = region_count;
mtddata->masks_loc_x = masks_loc_x;
mtddata->masks_loc_y = masks_loc_y;
mtddata->masks_loc_w = masks_loc_w;
mtddata->masks_loc_h = masks_loc_h;
memcpy (mtddata->region_ids, region_ids, region_ids_size); memcpy (mtddata->region_ids, region_ids, region_ids_size);
} }
@ -395,10 +434,33 @@ gst_analytics_segmentation_mtd_transform (GstBuffer * transbuf,
segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta, segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta,
transmtd->id); transmtd->id);
gst_buffer_ref (segdata->masks); gst_buffer_ref (segdata->masks);
} else if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type) && transbuf != buffer) { } else if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) {
GstVideoMetaTransform *trans = data;
gint ow, oh, nw, nh;
ow = GST_VIDEO_INFO_WIDTH (trans->in_info);
nw = GST_VIDEO_INFO_WIDTH (trans->out_info);
oh = GST_VIDEO_INFO_HEIGHT (trans->in_info);
nh = GST_VIDEO_INFO_HEIGHT (trans->out_info);
segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta, segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta,
transmtd->id); transmtd->id);
gst_buffer_ref (segdata->masks);
segdata->masks_loc_x *= nw;
segdata->masks_loc_x /= ow;
segdata->masks_loc_w *= nw;
segdata->masks_loc_w /= ow;
segdata->masks_loc_y *= nh;
segdata->masks_loc_y /= oh;
segdata->masks_loc_h *= nh;
segdata->masks_loc_h /= oh;
if (transbuf != buffer) {
gst_buffer_ref (segdata->masks);
}
} }
return TRUE; return TRUE;

97
subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h Normal file
View file

@ -0,0 +1,97 @@
/* GStreamer
* Copyright (C) 2024 Collabora Ltd
* @author: Daniel Morin <daniel.morin@collabora.com>
*
* gstanalyticssegmentationmtd.h
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_ANALYTICS_SEGMENTATION_META_H__
#define __GST_ANALYTICS_SEGMENTATION_META_H__
#include <gst/gst.h>
#include <gst/video/video-info.h>
#include <gst/analytics/analytics-meta-prelude.h>
#include <gst/analytics/gstanalyticsmeta.h>
G_BEGIN_DECLS
/**
* GstAnalyticsSegmentationMtd:
* @id: Instance identifier
* @meta: Instance of #GstAnalyticsRelationMeta where the analytics-metadata
* identified by @id is stored.
*
* Handle containing data required to use gst_analytics_segmentation_mtd APIs.
* This type is generally expected to be allocated on the stack.
*
* Since: 1.26
*/
typedef struct _GstAnalyticsMtd GstAnalyticsSegmentationMtd;
/**
* GstSegmentationType:
* @GST_SEGMENTATION_TYPE_SEMANTIC: Segmentation where the belonging of each
* pixel to a class of objects is identified.
* @GST_SEGMENTATION_TYPE_INSTANCE: Segmentation where the belonging of each
* pixel to instance of an object is identified.
*
* Enum value describing supported segmentation type
*
* Since: 1.26
*/
typedef enum
{
GST_SEGMENTATION_TYPE_SEMANTIC,
GST_SEGMENTATION_TYPE_INSTANCE
} GstSegmentationType;
GST_ANALYTICS_META_API
GstAnalyticsMtdType
gst_analytics_segmentation_mtd_get_mtd_type (void);
GST_ANALYTICS_META_API
GstBuffer *
gst_analytics_segmentation_mtd_get_mask (const GstAnalyticsSegmentationMtd *
handle, gint * masks_loc_x, gint * masks_loc_y, guint * masks_loc_w, guint *
masks_loc_h);
GST_ANALYTICS_META_API
gboolean
gst_analytics_segmentation_mtd_get_region_index (
const GstAnalyticsSegmentationMtd * handle, gsize * index, guint id);
GST_ANALYTICS_META_API
guint
gst_analytics_segmentation_mtd_get_region_id (
const GstAnalyticsSegmentationMtd * handle, gsize index);
GST_ANALYTICS_META_API
gsize
gst_analytics_segmentation_mtd_get_region_count (
const GstAnalyticsSegmentationMtd * handle);
GST_ANALYTICS_META_API
gboolean
gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta *
instance, GstBuffer * buffer, GstSegmentationType segmentation_type,
gsize region_count, guint * region_ids, gint masks_loc_x, gint masks_loc_y,
guint masks_loc_w, guint masks_loc_h, GstAnalyticsSegmentationMtd *
segmentation_mtd);
G_END_DECLS
#endif // __GST_ANALYTICS_SEGMENTATION_META_H__

9
subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build
View file

@ -1,14 +1,17 @@
analytics_sources = files( 'gstanalyticsmeta.c', analytics_sources = files( 'gstanalyticsmeta.c',
'gstanalyticsclassificationmtd.c', 'gstanalyticsclassificationmtd.c',
'gstanalyticsobjectdetectionmtd.c', 'gstanalyticsobjectdetectionmtd.c',
'gstanalyticsobjecttrackingmtd.c') 'gstanalyticsobjecttrackingmtd.c',
'gstanalyticssegmentationmtd.c')
analytics_headers = files( 'analytics.h', analytics_headers = files( 'analytics.h',
'gstanalyticsmeta.h', 'gstanalyticsmeta.h',
'analytics-meta-prelude.h', 'analytics-meta-prelude.h',
'gstanalyticsclassificationmtd.h', 'gstanalyticsclassificationmtd.h',
'gstanalyticsobjectdetectionmtd.h', 'gstanalyticsobjectdetectionmtd.h',
'gstanalyticsobjecttrackingmtd.h') 'gstanalyticsobjecttrackingmtd.h',
'gstanalyticssegmentationmtd.h')
install_headers(analytics_headers, subdir : 'gstreamer-1.0/gst/analytics') install_headers(analytics_headers, subdir : 'gstreamer-1.0/gst/analytics')
pkg_name = 'gstreamer-analytics-1.0' pkg_name = 'gstreamer-analytics-1.0'
@ -40,7 +43,7 @@ if build_gir
'identifier_prefix' : ['GstAnalytics', 'Gst'], 'identifier_prefix' : ['GstAnalytics', 'Gst'],
'symbol_prefix' : ['gst_analytics', 'gst'], 'symbol_prefix' : ['gst_analytics', 'gst'],
'export_packages' : pkg_name, 'export_packages' : pkg_name,
'includes' : ['Gst-1.0', 'GstBase-1.0'], 'includes' : ['Gst-1.0', 'GstBase-1.0', 'GstVideo-1.0'],
'install' : true, 'install' : true,
'extra_args' : gir_init_section + ['-DGST_USE_UNSTABLE_API'], 'extra_args' : gir_init_section + ['-DGST_USE_UNSTABLE_API'],
'dependencies' : [gstbase_dep, gstvideo_dep] 'dependencies' : [gstbase_dep, gstvideo_dep]