mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2025-01-02 21:48:55 +00:00
analytics: add segmentation analytics-meta
- Add a new analytics-meta to store segmentation analysis result. Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6026>
This commit is contained in:
Daniel Morin
GStreamer Marge Bot
parent
98653aa43a
commit
6db6d44ff5
5 changed files with 483 additions and 27 deletions
|
|
@ -4,6 +4,7 @@ and/or use gtk-doc annotations. -->
|
|||
<repository xmlns="http://www.gtk.org/introspection/core/1.0" xmlns:c="http://www.gtk.org/introspection/c/1.0" xmlns:glib="http://www.gtk.org/introspection/glib/1.0" version="1.2">
|
||||
<include name="Gst" version="1.0"/>
|
||||
<include name="GstBase" version="1.0"/>
|
||||
<include name="GstVideo" version="1.0"/>
|
||||
<package name="gstreamer-analytics-1.0"/>
|
||||
<namespace name="GstAnalytics" version="1.0" shared-library="libgstanalytics-1.0.so.0" c:identifier-prefixes="GstAnalytics,Gst" c:symbol-prefixes="gst_analytics,gst">
|
||||
<alias name="MtdType" c:type="GstAnalyticsMtdType">
|
||||
|
|
@ -16,11 +17,11 @@ and/or use gtk-doc annotations. -->
|
|||
is generally expected to be allocated on the stack.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.h"/>
|
||||
<field name="id" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance identifier.</doc>
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
|
||||
<type name="guint" c:type="guint"/>
|
||||
</field>
|
||||
<field name="meta" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance of #GstAnalyticsRelationMeta where the analysis-metadata
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
|
||||
identified by @id is stored.</doc>
|
||||
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
|
||||
</field>
|
||||
|
|
@ -125,11 +126,11 @@ infinite.</doc>
|
|||
is generally expected to be allocated on the stack.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h"/>
|
||||
<field name="id" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance identifier.</doc>
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
|
||||
<type name="guint" c:type="guint"/>
|
||||
</field>
|
||||
<field name="meta" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance of #GstAnalyticsRelationMeta where the analysis-metadata
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
|
||||
identified by @id is stored.</doc>
|
||||
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
|
||||
</field>
|
||||
|
|
@ -223,8 +224,24 @@ have a static lifetime (never be freed).</doc>
|
|||
</parameters>
|
||||
</callback>
|
||||
</field>
|
||||
<field name="mtd_meta_clear">
|
||||
<callback name="mtd_meta_clear">
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
<type name="none" c:type="void"/>
|
||||
</return-value>
|
||||
<parameters>
|
||||
<parameter name="buffer" transfer-ownership="none">
|
||||
<type name="Gst.Buffer" c:type="GstBuffer*"/>
|
||||
</parameter>
|
||||
<parameter name="mtd" transfer-ownership="none">
|
||||
<type name="Mtd" c:type="GstAnalyticsMtd*"/>
|
||||
</parameter>
|
||||
</parameters>
|
||||
</callback>
|
||||
</field>
|
||||
<field name="_reserved" readable="0" private="1">
|
||||
<array zero-terminated="0" fixed-size="20">
|
||||
<array zero-terminated="0" fixed-size="19">
|
||||
<type name="gpointer" c:type="gpointer"/>
|
||||
</array>
|
||||
</field>
|
||||
|
|
@ -234,11 +251,11 @@ have a static lifetime (never be freed).</doc>
|
|||
is generally expected to be allocated on the stack.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h"/>
|
||||
<field name="id" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance identifier.</doc>
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
|
||||
<type name="guint" c:type="guint"/>
|
||||
</field>
|
||||
<field name="meta" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance of #GstAnalyticsRelationMeta where the analysis-metadata
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
|
||||
identified by @id is stored.</doc>
|
||||
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
|
||||
</field>
|
||||
|
|
@ -476,6 +493,44 @@ new struct sub-classing GstAnalyticsRelatableMtd.</doc>
|
|||
</parameter>
|
||||
</parameters>
|
||||
</method>
|
||||
<method name="add_segmentation_mtd" c:identifier="gst_analytics_relation_meta_add_segmentation_mtd" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Add analytics segmentation metadata to @instance.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">TRUE if added successfully, otherwise FALSE</doc>
|
||||
<type name="gboolean" c:type="gboolean"/>
|
||||
</return-value>
|
||||
<parameters>
|
||||
<instance-parameter name="instance" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance of #GstAnalyticsRelationMeta where to add segmentation
|
||||
instance.</doc>
|
||||
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
|
||||
</instance-parameter>
|
||||
<parameter name="buffer" transfer-ownership="full">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Buffer containing segmentation masks. @buffer
|
||||
must have a #GstVideoMeta attached</doc>
|
||||
<type name="Gst.Buffer" c:type="GstBuffer*"/>
|
||||
</parameter>
|
||||
<parameter name="segmentation_type" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Segmentation type</doc>
|
||||
<type name="SegmentationType" c:type="GstSegmentationType"/>
|
||||
</parameter>
|
||||
<parameter name="region_count" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Number of regions in the masks</doc>
|
||||
<type name="gsize" c:type="gsize"/>
|
||||
</parameter>
|
||||
<parameter name="region_ids" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Arrays of region ids present in the mask.</doc>
|
||||
<array length="2" zero-terminated="0" c:type="guint*">
|
||||
<type name="guint" c:type="guint"/>
|
||||
</array>
|
||||
</parameter>
|
||||
<parameter name="segmentation_mtd" direction="out" caller-allocates="1" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Handle update with newly added segmenation meta.</doc>
|
||||
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
|
||||
</parameter>
|
||||
</parameters>
|
||||
</method>
|
||||
<method name="add_tracking_mtd" c:identifier="gst_analytics_relation_meta_add_tracking_mtd" version="1.24">
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
|
|
@ -806,15 +861,244 @@ Ids must have been obtained a call to
|
|||
<type name="gsize" c:type="gsize"/>
|
||||
</field>
|
||||
</record>
|
||||
<record name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">This type of metadata holds information on which pixels belongs to
|
||||
a region of the image representing a type of object.
|
||||
|
||||
It supports two types of segmentation, semantic or instance:
|
||||
* Semantic: All objects of the same type have the same id
|
||||
* Instance: Each instance of an object has a different id
|
||||
|
||||
The results of the segmentation are stored in a #GstBuffer that has a
|
||||
#GstVideoMeta associated with it. This buffer is stored in the
|
||||
GstAnalyticsSegmentationMtd using
|
||||
#gst_analytics_relation_meta_add_segmentation_mtd(). The #GstBuffer
|
||||
containing the segmentation mask is image-like but the color values are
|
||||
arbitrary values, referred by region-id in this API, without meaning beyond
|
||||
specifying that two pixels in the original image with the same values in
|
||||
their corresponding mask value belong to the same region.
|
||||
|
||||
To further describe a region, the #GstAnalyticsSegmentationMtd can be
|
||||
associated with other #GstAnalyticsMtd. Since region ids are
|
||||
generated by the segmentation process itself and are not always sequential,
|
||||
we use a map of indexes to region ids starting with 0 without discontinuity
|
||||
which facilitate N-to-N mapping with other #GstAnalyticsMtd. For
|
||||
example it can be associated with #GstAnalyticsClsMtd to describe the class
|
||||
of object matching the pixels of a segmented region.
|
||||
|
||||
Example: Associate Instance Segmentation with Classification
|
||||
|
||||
In the following example the segmentation process will fill segmask with
|
||||
values of 0 for background, 12 for the first region which correspond to a
|
||||
to a strawberry, 7 for the second region that also correspond to a
|
||||
strawberry in the image and 31 for the third region that correspond to a
|
||||
leaf in the image.
|
||||
region_ids is fill during segmentation post-processing
|
||||
|
||||
region_ids:
|
||||
|region-index | region-id |
|
||||
|-------------|-----------|
|
||||
| 0 | 0 |
|
||||
| 1 | 12 |
|
||||
| 2 | 7 |
|
||||
| 3 | 31 |
|
||||
|
||||
region_count = 4
|
||||
|
||||
``` C
|
||||
GstAnalyticsSegmentationMtd segmtd;
|
||||
GstAnalyticsClassificationMtd clsmtd;
|
||||
GstBuffer *segmask, *img;
|
||||
guint *region_ids;
|
||||
gsize region_count, class_count;
|
||||
gfloat *class_confidence;
|
||||
GQuark *classes;
|
||||
|
||||
... (segmentation filling segmask based on img)
|
||||
|
||||
gst_analytics_relation_meta_add_segmentation_mtd (rmeta, segmask,
|
||||
GST_SEGMENTATION_TYPE_INSTANCE, region_count, region_ids, &segmtd);
|
||||
class_count = region_count;
|
||||
|
||||
... (class-index must match and correspond to region-index)
|
||||
classes [0] = g_quark_from_string ("background");
|
||||
classes [1] = g_quark_from_string ("strawberry");
|
||||
classes [2] = g_quark_from_string ("strawberry");
|
||||
classes [3] = g_quark_from_string ("leaf");
|
||||
|
||||
... (set confidence level for each class associated with a region
|
||||
... where -1.0 mean undefined.)
|
||||
class_confidence [0] = -1.0;
|
||||
class_confidence [1] = 0.6;
|
||||
class_confidence [2] = 0.9;
|
||||
class_confidence [3] = 0.8;
|
||||
|
||||
gst_analytics_relation_meta_add_cls_mtd (rmeta, class_count,
|
||||
class_confidence, classes, &clsmtd);
|
||||
|
||||
gst_analytics_relation_meta_set_relation (rmeta,
|
||||
GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id);
|
||||
```
|
||||
|
||||
Example: Associate Semantic Segmentation with Classification
|
||||
Assuming the same context as for Instance Segmentation above but instead
|
||||
a semantic segmentation is performed, therfore region-id-12 and region-id-7
|
||||
are now represented by the same region-id-12
|
||||
|
||||
region_ids: (here
|
||||
|region-index | region-id |
|
||||
|-------------|-----------|
|
||||
| 0 | 0 |
|
||||
| 1 | 12 |
|
||||
| 2 | 31 |
|
||||
|
||||
Code remain the same except that we set all confidence level to undefined
|
||||
(-1.0).
|
||||
|
||||
```
|
||||
... (class-index must match and correspond to region-index)
|
||||
classes [0] = g_quark_from_string ("background");
|
||||
classes [1] = g_quark_from_string ("strawberry");
|
||||
classes [2] = g_quark_from_string ("leaf");
|
||||
|
||||
... (set confidence level for each class associated with a region
|
||||
... where -1.0 mean undefined.)
|
||||
class_confidence [0] = -1.0;
|
||||
class_confidence [1] = -1.0;
|
||||
class_confidence [2] = -1.0;
|
||||
|
||||
gst_analytics_relation_meta_add_cls_mtd (rmeta, class_count,
|
||||
class_confidence, classes, &clsmtd);
|
||||
|
||||
gst_analytics_relation_meta_set_relation (rmeta,
|
||||
GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id);
|
||||
```
|
||||
|
||||
Example: Retriving class associated with a segmentation region-id-12
|
||||
This the typical case for an overlay as we visit the segmentation mask we
|
||||
we find region-id values
|
||||
|
||||
```
|
||||
gsize idx;
|
||||
gst_analytics_segmentation_mtd_get_region_index (&segmtd, &idx, 12);
|
||||
gst_analytics_relation_meta_get_direct_related (rmeta, segmtd.id,
|
||||
GST_ANALYTICS_REL_TYPE_RELATE_TO, gst_analytics_cls_mtd_get_type (),
|
||||
NULL, &clsmtd);
|
||||
|
||||
GQuark region_class = gst_analytics_cls_mtd_get_quark (&segmtd, idx)
|
||||
...
|
||||
```
|
||||
|
||||
Since: 1.26</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<field name="id" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
|
||||
<type name="guint" c:type="guint"/>
|
||||
</field>
|
||||
<field name="meta" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
|
||||
identified by @id is stored.</doc>
|
||||
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
|
||||
</field>
|
||||
<method name="get_mask" c:identifier="gst_analytics_segmentation_mtd_get_mask" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get segmentation mask data.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<return-value transfer-ownership="full">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Segmentation mask data stored in a #GstBuffer</doc>
|
||||
<type name="Gst.Buffer" c:type="GstBuffer*"/>
|
||||
</return-value>
|
||||
<parameters>
|
||||
<instance-parameter name="handle" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance</doc>
|
||||
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
|
||||
</instance-parameter>
|
||||
</parameters>
|
||||
</method>
|
||||
<method name="get_region_count" c:identifier="gst_analytics_segmentation_mtd_get_region_count" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get the regions count.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Number of regions segmented</doc>
|
||||
<type name="gsize" c:type="gsize"/>
|
||||
</return-value>
|
||||
<parameters>
|
||||
<instance-parameter name="handle" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance</doc>
|
||||
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
|
||||
</instance-parameter>
|
||||
</parameters>
|
||||
</method>
|
||||
<method name="get_region_id" c:identifier="gst_analytics_segmentation_mtd_get_region_id" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get id of the region corresponding to @index. which should be
|
||||
smaller than the return value of
|
||||
gst_analytics_segmentation_mtd_get_region_count()</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">The region ID</doc>
|
||||
<type name="guint" c:type="guint"/>
|
||||
</return-value>
|
||||
<parameters>
|
||||
<instance-parameter name="handle" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance</doc>
|
||||
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
|
||||
</instance-parameter>
|
||||
<parameter name="index" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Region index</doc>
|
||||
<type name="gsize" c:type="gsize"/>
|
||||
</parameter>
|
||||
</parameters>
|
||||
</method>
|
||||
<method name="get_region_index" c:identifier="gst_analytics_segmentation_mtd_get_region_index" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get region index of the region identified by @id.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">TRUE if a region with @id exist, otherwise FALSE</doc>
|
||||
<type name="gboolean" c:type="gboolean"/>
|
||||
</return-value>
|
||||
<parameters>
|
||||
<instance-parameter name="handle" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Instance</doc>
|
||||
<type name="SegmentationMtd" c:type="GstAnalyticsSegmentationMtd*"/>
|
||||
</instance-parameter>
|
||||
<parameter name="index" direction="out" caller-allocates="1" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Region index</doc>
|
||||
<type name="gsize" c:type="gsize*"/>
|
||||
</parameter>
|
||||
<parameter name="id" transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Region id</doc>
|
||||
<type name="guint" c:type="guint"/>
|
||||
</parameter>
|
||||
</parameters>
|
||||
</method>
|
||||
<function name="get_mtd_type" c:identifier="gst_analytics_segmentation_mtd_get_mtd_type" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get an instance of #GstAnalyticsMtdType that represent segmentation
|
||||
metadata type.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">A #GstAnalyticsMtdType type</doc>
|
||||
<type name="MtdType" c:type="GstAnalyticsMtdType"/>
|
||||
</return-value>
|
||||
</function>
|
||||
</record>
|
||||
<enumeration name="SegmentationType" version="1.26" c:type="GstSegmentationType">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Enum value describing supported segmentation type</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<member name="semantic" value="0" c:identifier="GST_SEGMENTATION_TYPE_SEMANTIC" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Each region of the same type have the same id.</doc>
|
||||
</member>
|
||||
<member name="instance" value="1" c:identifier="GST_SEGMENTATION_TYPE_INSTANCE" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Each instance of a region have different id.</doc>
|
||||
</member>
|
||||
</enumeration>
|
||||
<record name="TrackingMtd" c:type="GstAnalyticsTrackingMtd" version="1.24">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.c">Store information on results of object tracking</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/>
|
||||
<field name="id" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance identifier.</doc>
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance identifier</doc>
|
||||
<type name="guint" c:type="guint"/>
|
||||
</field>
|
||||
<field name="meta" writable="1">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h">Instance of #GstAnalyticsRelationMeta where the analysis-metadata
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h">Instance of #GstAnalyticsRelationMeta where the analytics-metadata
|
||||
identified by @id is stored.</doc>
|
||||
<type name="RelationMeta" c:type="GstAnalyticsRelationMeta*"/>
|
||||
</field>
|
||||
|
|
@ -1023,6 +1307,15 @@ information about the tracking, for example, it can be used alongside a
|
|||
<type name="Gst.MetaInfo" c:type="const GstMetaInfo*"/>
|
||||
</return-value>
|
||||
</function>
|
||||
<function name="segmentation_mtd_get_mtd_type" c:identifier="gst_analytics_segmentation_mtd_get_mtd_type" moved-to="SegmentationMtd.get_mtd_type" version="1.26">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">Get an instance of #GstAnalyticsMtdType that represent segmentation
|
||||
metadata type.</doc>
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticssegmentationmtd.c">A #GstAnalyticsMtdType type</doc>
|
||||
<type name="MtdType" c:type="GstAnalyticsMtdType"/>
|
||||
</return-value>
|
||||
</function>
|
||||
<function name="tracking_mtd_get_mtd_type" c:identifier="gst_analytics_tracking_mtd_get_mtd_type" moved-to="TrackingMtd.get_mtd_type" version="1.24">
|
||||
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h"/>
|
||||
<return-value transfer-ownership="none">
|
||||
|
|
|
|||
|
|
@ -30,5 +30,6 @@
|
|||
#include <gst/analytics/gstanalyticsclassificationmtd.h>
|
||||
#include <gst/analytics/gstanalyticsobjectdetectionmtd.h>
|
||||
#include <gst/analytics/gstanalyticsobjecttrackingmtd.h>
|
||||
#include <gst/analytics/gstanalyticssegmentationmtd.h>
|
||||
|
||||
#endif /* __ANALYTICS_H__ */
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@
|
|||
*
|
||||
* Example: Associate Semantic Segmentation with Classification
|
||||
* Assuming the same context as for Instance Segmentation above but instead
|
||||
* a semantic segmentation is performed, therfore region-id-12 and region-id-7
|
||||
* a semantic segmentation is performed, therefore region-id-12 and region-id-7
|
||||
* are now represented by the same region-id-12
|
||||
*
|
||||
* region_ids: (here
|
||||
|
|
@ -147,7 +147,7 @@
|
|||
* GST_ANALYTICS_REL_TYPE_RELATE_TO, segmtd.id, clsmtd.id);
|
||||
* ```
|
||||
*
|
||||
* Example: Retriving class associated with a segmentation region-id-12
|
||||
* Example: Retrieving class associated with a segmentation region-id-12
|
||||
* This the typical case for an overlay as we visit the segmentation mask we
|
||||
* we find region-id values
|
||||
*
|
||||
|
|
@ -201,6 +201,12 @@ typedef struct _GstAnalyticsSegMtdData
|
|||
{
|
||||
GstSegmentationType type;
|
||||
GstBuffer *masks;
|
||||
|
||||
gint masks_loc_x;
|
||||
gint masks_loc_y;
|
||||
guint masks_loc_w;
|
||||
guint masks_loc_h;
|
||||
|
||||
gsize region_count;
|
||||
guint32 region_ids[]; /* Must be last */
|
||||
} GstAnalyticsSegMtdData;
|
||||
|
|
@ -224,6 +230,14 @@ gst_analytics_segmentation_mtd_get_mtd_type (void)
|
|||
/**
|
||||
* gst_analytics_segmentation_mtd_get_mask:
|
||||
* @handle: Instance
|
||||
* @masks_loc_x: (out caller-allocates)(nullable): Left coordinate of the
|
||||
* rectangle corresponding to the mask in the image.
|
||||
* @masks_loc_y: (out caller-allocates)(nullable): Top coordinate of the
|
||||
* rectangle corresponding to the mask in the image.
|
||||
* @masks_loc_w: (out caller-allocates)(nullable): Width of the rectangle
|
||||
* corresponding to the mask in the image.
|
||||
* @masks_loc_h: (out caller-allocates)(nullable): Height of the rectangle
|
||||
* corresponding to the mask in the image.
|
||||
*
|
||||
* Get segmentation mask data.
|
||||
*
|
||||
|
|
@ -232,7 +246,9 @@ gst_analytics_segmentation_mtd_get_mtd_type (void)
|
|||
* Since: 1.26
|
||||
*/
|
||||
GstBuffer *
|
||||
gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle)
|
||||
gst_analytics_segmentation_mtd_get_mask (const GstAnalyticsSegmentationMtd *
|
||||
handle, gint * masks_loc_x, gint * masks_loc_y, guint * masks_loc_w, guint *
|
||||
masks_loc_h)
|
||||
{
|
||||
GstAnalyticsSegMtdData *mtddata;
|
||||
|
||||
|
|
@ -241,6 +257,15 @@ gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle)
|
|||
mtddata = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id);
|
||||
g_return_val_if_fail (mtddata != NULL, NULL);
|
||||
|
||||
if (masks_loc_x)
|
||||
*masks_loc_x = mtddata->masks_loc_x;
|
||||
if (masks_loc_y)
|
||||
*masks_loc_y = mtddata->masks_loc_y;
|
||||
if (masks_loc_w)
|
||||
*masks_loc_w = mtddata->masks_loc_w;
|
||||
if (masks_loc_h)
|
||||
*masks_loc_h = mtddata->masks_loc_h;
|
||||
|
||||
return gst_buffer_ref (mtddata->masks);
|
||||
}
|
||||
|
||||
|
|
@ -257,8 +282,8 @@ gst_analytics_segmentation_mtd_get_mask (GstAnalyticsSegmentationMtd * handle)
|
|||
* Since: 1.26
|
||||
*/
|
||||
gboolean
|
||||
gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd *
|
||||
handle, gsize * index, guint id)
|
||||
gst_analytics_segmentation_mtd_get_region_index (const
|
||||
GstAnalyticsSegmentationMtd * handle, gsize * index, guint id)
|
||||
{
|
||||
GstAnalyticsSegMtdData *mtddata;
|
||||
|
||||
|
|
@ -283,7 +308,7 @@ gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd *
|
|||
* @handle: Instance
|
||||
* @index: Region index
|
||||
*
|
||||
* Get id of the region corresponding to @index. which should be
|
||||
* Get id of the region corresponding to @index, which should be
|
||||
* smaller than the return value of
|
||||
* gst_analytics_segmentation_mtd_get_region_count()
|
||||
*
|
||||
|
|
@ -292,8 +317,8 @@ gst_analytics_segmentation_mtd_get_region_index (GstAnalyticsSegmentationMtd *
|
|||
* Since: 1.26
|
||||
*/
|
||||
guint
|
||||
gst_analytics_segmentation_mtd_get_region_id (GstAnalyticsSegmentationMtd *
|
||||
handle, gsize index)
|
||||
gst_analytics_segmentation_mtd_get_region_id (const
|
||||
GstAnalyticsSegmentationMtd * handle, gsize index)
|
||||
{
|
||||
GstAnalyticsSegMtdData *mtddata;
|
||||
|
||||
|
|
@ -317,8 +342,8 @@ gst_analytics_segmentation_mtd_get_region_id (GstAnalyticsSegmentationMtd *
|
|||
* Since: 1.26
|
||||
*/
|
||||
gsize
|
||||
gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd *
|
||||
handle)
|
||||
gst_analytics_segmentation_mtd_get_region_count (const
|
||||
GstAnalyticsSegmentationMtd * handle)
|
||||
{
|
||||
GstAnalyticsSegMtdData *mtddata;
|
||||
|
||||
|
|
@ -339,9 +364,18 @@ gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd *
|
|||
* @segmentation_type:(in): Segmentation type
|
||||
* @region_count:(in): Number of regions in the masks
|
||||
* @region_ids:(in) (array length=region_count): Arrays of region ids present in the mask.
|
||||
* @segmentation_mtd:(out)(not nullable): Handle update with newly added segmenation meta.
|
||||
* @masks_loc_x:(in): Left coordinate of the rectangle corresponding to the masks in the image.
|
||||
* @masks_loc_y:(in): Top coordinate of the rectangle corresponding to the masks in the image.
|
||||
* @masks_loc_w:(in): Width of the rectangle corresponding to the masks in the image.
|
||||
* @masks_loc_h:(in): Height of the rectangle corresponding to the masks in the image.
|
||||
* @segmentation_mtd:(out)(not nullable): Handle update with newly added segmentation meta.
|
||||
*
|
||||
* Add analytics segmentation metadata to @instance.
|
||||
* Add analytics segmentation metadata to @instance. The rectangle (@masks_loc_x,
|
||||
* @mask_loc_y, @mask_loc_w, @mask_loc_h) define a area of the image that
|
||||
* correspond to the segmentation masks stored in @buffer. For example if the
|
||||
* segmentation masks stored in @buffer describe the segmented regions for the
|
||||
* entire image the rectangular area will be (@masks_loc_x = 0, @masks_loc_y = 0,
|
||||
* @masks_loc_w = image_width, @masks_loc_h = image_height).
|
||||
*
|
||||
* Returns: TRUE if added successfully, otherwise FALSE
|
||||
*
|
||||
|
|
@ -350,7 +384,8 @@ gst_analytics_segmentation_mtd_get_region_count (GstAnalyticsSegmentationMtd *
|
|||
gboolean
|
||||
gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta *
|
||||
instance, GstBuffer * buffer, GstSegmentationType segmentation_type,
|
||||
gsize region_count, guint * region_ids, GstAnalyticsSegmentationMtd *
|
||||
gsize region_count, guint * region_ids, gint masks_loc_x, gint masks_loc_y,
|
||||
guint masks_loc_w, guint masks_loc_h, GstAnalyticsSegmentationMtd *
|
||||
segmentation_mtd)
|
||||
{
|
||||
const gsize region_ids_size = sizeof (guint) * region_count;
|
||||
|
|
@ -371,6 +406,10 @@ gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta *
|
|||
mtddata->masks = buffer;
|
||||
mtddata->type = segmentation_type;
|
||||
mtddata->region_count = region_count;
|
||||
mtddata->masks_loc_x = masks_loc_x;
|
||||
mtddata->masks_loc_y = masks_loc_y;
|
||||
mtddata->masks_loc_w = masks_loc_w;
|
||||
mtddata->masks_loc_h = masks_loc_h;
|
||||
memcpy (mtddata->region_ids, region_ids, region_ids_size);
|
||||
}
|
||||
|
||||
|
|
@ -395,11 +434,34 @@ gst_analytics_segmentation_mtd_transform (GstBuffer * transbuf,
|
|||
segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta,
|
||||
transmtd->id);
|
||||
gst_buffer_ref (segdata->masks);
|
||||
} else if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type) && transbuf != buffer) {
|
||||
} else if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) {
|
||||
GstVideoMetaTransform *trans = data;
|
||||
gint ow, oh, nw, nh;
|
||||
|
||||
ow = GST_VIDEO_INFO_WIDTH (trans->in_info);
|
||||
nw = GST_VIDEO_INFO_WIDTH (trans->out_info);
|
||||
oh = GST_VIDEO_INFO_HEIGHT (trans->in_info);
|
||||
nh = GST_VIDEO_INFO_HEIGHT (trans->out_info);
|
||||
|
||||
segdata = gst_analytics_relation_meta_get_mtd_data (transmtd->meta,
|
||||
transmtd->id);
|
||||
|
||||
segdata->masks_loc_x *= nw;
|
||||
segdata->masks_loc_x /= ow;
|
||||
|
||||
segdata->masks_loc_w *= nw;
|
||||
segdata->masks_loc_w /= ow;
|
||||
|
||||
segdata->masks_loc_y *= nh;
|
||||
segdata->masks_loc_y /= oh;
|
||||
|
||||
segdata->masks_loc_h *= nh;
|
||||
segdata->masks_loc_h /= oh;
|
||||
|
||||
if (transbuf != buffer) {
|
||||
gst_buffer_ref (segdata->masks);
|
||||
}
|
||||
}
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,97 @@
|
|||
/* GStreamer
|
||||
* Copyright (C) 2024 Collabora Ltd
|
||||
* @author: Daniel Morin <daniel.morin@collabora.com>
|
||||
*
|
||||
* gstanalyticssegmentationmtd.h
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Library General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Library General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Library General Public
|
||||
* License along with this library; if not, write to the
|
||||
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
||||
* Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef __GST_ANALYTICS_SEGMENTATION_META_H__
|
||||
#define __GST_ANALYTICS_SEGMENTATION_META_H__
|
||||
|
||||
#include <gst/gst.h>
|
||||
#include <gst/video/video-info.h>
|
||||
#include <gst/analytics/analytics-meta-prelude.h>
|
||||
#include <gst/analytics/gstanalyticsmeta.h>
|
||||
|
||||
G_BEGIN_DECLS
|
||||
|
||||
/**
|
||||
* GstAnalyticsSegmentationMtd:
|
||||
* @id: Instance identifier
|
||||
* @meta: Instance of #GstAnalyticsRelationMeta where the analytics-metadata
|
||||
* identified by @id is stored.
|
||||
*
|
||||
* Handle containing data required to use gst_analytics_segmentation_mtd APIs.
|
||||
* This type is generally expected to be allocated on the stack.
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
typedef struct _GstAnalyticsMtd GstAnalyticsSegmentationMtd;
|
||||
|
||||
/**
|
||||
* GstSegmentationType:
|
||||
* @GST_SEGMENTATION_TYPE_SEMANTIC: Segmentation where the belonging of each
|
||||
* pixel to a class of objects is identified.
|
||||
* @GST_SEGMENTATION_TYPE_INSTANCE: Segmentation where the belonging of each
|
||||
* pixel to instance of an object is identified.
|
||||
*
|
||||
* Enum value describing supported segmentation type
|
||||
*
|
||||
* Since: 1.26
|
||||
*/
|
||||
typedef enum
|
||||
{
|
||||
GST_SEGMENTATION_TYPE_SEMANTIC,
|
||||
GST_SEGMENTATION_TYPE_INSTANCE
|
||||
} GstSegmentationType;
|
||||
|
||||
GST_ANALYTICS_META_API
|
||||
GstAnalyticsMtdType
|
||||
gst_analytics_segmentation_mtd_get_mtd_type (void);
|
||||
|
||||
GST_ANALYTICS_META_API
|
||||
GstBuffer *
|
||||
gst_analytics_segmentation_mtd_get_mask (const GstAnalyticsSegmentationMtd *
|
||||
handle, gint * masks_loc_x, gint * masks_loc_y, guint * masks_loc_w, guint *
|
||||
masks_loc_h);
|
||||
|
||||
GST_ANALYTICS_META_API
|
||||
gboolean
|
||||
gst_analytics_segmentation_mtd_get_region_index (
|
||||
const GstAnalyticsSegmentationMtd * handle, gsize * index, guint id);
|
||||
|
||||
GST_ANALYTICS_META_API
|
||||
guint
|
||||
gst_analytics_segmentation_mtd_get_region_id (
|
||||
const GstAnalyticsSegmentationMtd * handle, gsize index);
|
||||
|
||||
GST_ANALYTICS_META_API
|
||||
gsize
|
||||
gst_analytics_segmentation_mtd_get_region_count (
|
||||
const GstAnalyticsSegmentationMtd * handle);
|
||||
|
||||
GST_ANALYTICS_META_API
|
||||
gboolean
|
||||
gst_analytics_relation_meta_add_segmentation_mtd (GstAnalyticsRelationMeta *
|
||||
instance, GstBuffer * buffer, GstSegmentationType segmentation_type,
|
||||
gsize region_count, guint * region_ids, gint masks_loc_x, gint masks_loc_y,
|
||||
guint masks_loc_w, guint masks_loc_h, GstAnalyticsSegmentationMtd *
|
||||
segmentation_mtd);
|
||||
|
||||
G_END_DECLS
|
||||
#endif // __GST_ANALYTICS_SEGMENTATION_META_H__
|
||||
|
|
@ -1,14 +1,17 @@
|
|||
analytics_sources = files( 'gstanalyticsmeta.c',
|
||||
'gstanalyticsclassificationmtd.c',
|
||||
'gstanalyticsobjectdetectionmtd.c',
|
||||
'gstanalyticsobjecttrackingmtd.c')
|
||||
'gstanalyticsobjecttrackingmtd.c',
|
||||
'gstanalyticssegmentationmtd.c')
|
||||
|
||||
analytics_headers = files( 'analytics.h',
|
||||
'gstanalyticsmeta.h',
|
||||
'analytics-meta-prelude.h',
|
||||
'gstanalyticsclassificationmtd.h',
|
||||
'gstanalyticsobjectdetectionmtd.h',
|
||||
'gstanalyticsobjecttrackingmtd.h')
|
||||
'gstanalyticsobjecttrackingmtd.h',
|
||||
'gstanalyticssegmentationmtd.h')
|
||||
|
||||
install_headers(analytics_headers, subdir : 'gstreamer-1.0/gst/analytics')
|
||||
|
||||
pkg_name = 'gstreamer-analytics-1.0'
|
||||
|
|
@ -40,7 +43,7 @@ if build_gir
|
|||
'identifier_prefix' : ['GstAnalytics', 'Gst'],
|
||||
'symbol_prefix' : ['gst_analytics', 'gst'],
|
||||
'export_packages' : pkg_name,
|
||||
'includes' : ['Gst-1.0', 'GstBase-1.0'],
|
||||
'includes' : ['Gst-1.0', 'GstBase-1.0', 'GstVideo-1.0'],
|
||||
'install' : true,
|
||||
'extra_args' : gir_init_section + ['-DGST_USE_UNSTABLE_API'],
|
||||
'dependencies' : [gstbase_dep, gstvideo_dep]
|
||||
|
|
|
|||
Loading…
Reference in a new issue