From d23a90cb16ff73f35407d10e3bebb256743e60da Mon Sep 17 00:00:00 2001 From: Daniel Morin Date: Mon, 3 Jul 2023 16:44:22 -0400 Subject: [PATCH] analytics: base class for analytics meta - GstAnalyticRelationMeta is a base class for analytics meta. It's able to store analytics results (GstAnalyticRelatableMtd) and describe the relation between each analysis results. - GstAnalysisRelationMeta also contain an algorithm able to explore analysis results relation using a bfs. - Relation(edge) between analysis results (vertice) are stored in an adjacency-matrix that allow to quickly identify if two analysis results are related and by which relation they related. It also work for indirect relation and can provide the path of analysis results by which two analysis results are related. - One allocation per buffer to store analysis results. Here we rely on the application to guess how much space will be required to store all analysis results. This is something that could be improved significantly but it's a starting point. - Define common analysis results, classification, object-detection, tracking that are subclass of GstAnalyticRelatableMtd. The also provide exemple of how to extend GstAnalyticRelatableMtd to have them benefit for the mechanim to express relation with other analysis results. Part-of: --- girs/Gst-1.0.gir | 10 + girs/GstAnalytics-1.0.gir | 1025 ++++++++++++++++ .../gst/analytics/analytics-meta-prelude.h | 36 + .../gst-libs/gst/analytics/analytics.h | 34 + .../analytics/gstanalyticsclassificationmtd.c | 271 +++++ .../analytics/gstanalyticsclassificationmtd.h | 82 ++ .../gst-libs/gst/analytics/gstanalyticsmeta.c | 1038 +++++++++++++++++ .../gst-libs/gst/analytics/gstanalyticsmeta.h | 226 ++++ .../gstanalyticsobjectdetectionmtd.c | 214 ++++ .../gstanalyticsobjectdetectionmtd.h | 73 ++ .../analytics/gstanalyticsobjecttrackingmtd.c | 228 ++++ .../analytics/gstanalyticsobjecttrackingmtd.h | 78 ++ .../gst-libs/gst/analytics/meson.build | 64 + .../gst-plugins-bad/gst-libs/gst/meson.build | 1 + .../tests/check/libs/analyticsmeta.c | 978 ++++++++++++++++ .../gst-plugins-bad/tests/check/meson.build | 1 + 16 files changed, 4359 insertions(+) create mode 100644 girs/GstAnalytics-1.0.gir create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics-meta-prelude.h create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.c create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.h create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.c create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.c create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.c create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h create mode 100644 subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build create mode 100644 subprojects/gst-plugins-bad/tests/check/libs/analyticsmeta.c diff --git a/girs/Gst-1.0.gir b/girs/Gst-1.0.gir index 7918b15f74..372c57b785 100644 --- a/girs/Gst-1.0.gir +++ b/girs/Gst-1.0.gir @@ -22114,6 +22114,16 @@ In hindsight, this tag should have been called "memory-layout". + + Check if the transform type is clearing the content of the meta without +freeing it. + + + + a transform type + + + Check if the transform type is a copy transform diff --git a/girs/GstAnalytics-1.0.gir b/girs/GstAnalytics-1.0.gir new file mode 100644 index 0000000000..ddf1359c2f --- /dev/null +++ b/girs/GstAnalytics-1.0.gir @@ -0,0 +1,1025 @@ + + + + + + + + Type of analytics meta data + + + + + + + + + Handle containing data required to use gst_analytics_cls_mtd APIs. This type +is generally expected to be allocated on the stack. + + + Instance identifier. + + + + Instance of #GstAnalyticsRelationMeta where the analysis-metadata +identified by @id is stored. + + + + + + index of the class associated with @quarks ( and label) or + a negative value on failure. + + + + + Instance handle + + + + Quark of the class +Get index of class represented by @quark + + + + + + + + Number of classes in this classification instance + + + + + Instance handle +Get number of classes + + + + + + Get confidence level for class at @index + + + confidence level for @index, <0.0 if the call failed. + + + + + instance handle + + + + Object class index + + + + + + + + Quark of this class (label) associated with @index + + + + + Instance handle + + + + index of the class +Get quark of the class at @index + + + + + + Get the static string representing #GstAnalyticsMtd type. + + + #GstAnalyticsMtd type name. + + + + + Get a quark identifying #GstAnalyticsMtd type. + + + Quark of #GstAnalyticsMtd type + + + + + + + + + + + + + + + + + A wildcard matching any type of analysis + + + + + Handle containing data required to use gst_analytics_mtd API. This type +is generally expected to be allocated on the stack. + + + Instance identifier. + + + + Instance of #GstAnalyticsRelationMeta where the analysis-metadata +identified by @id is stored. + + + + + + Id of @instance + + + + + Instance of #GstAnalyticsMtd +Get instance id + + + + + + @instance Instance of #GstAnalyticsRelatableMtd +Get instance size + + + Size (in bytes) of this instance or 0 on failure. + + + + + + + + + + + + quark associated with type. + + + + + Instance of #GstAnalyticsMtd +Get analysis result type. + + + + + + + Handle containing data required to use gst_analytics_od_mtd APIs. This type +is generally expected to be allocated on the stack. + + + Instance identifier. + + + + Instance of #GstAnalyticsRelationMeta where the analysis-metadata +identified by @id is stored. + + + + Retrieve location and location confidence level. + + + TRUE on success, otherwise FALSE. + + + + + instance + + + + x component of upper-left corner of the object location + + + + y component of upper-left corner of the object location + + + + bounding box width of the object location + + + + bounding box height of the object location + + + + Confidence on object location + + + + + + + + Quark different from on success and 0 on failure. + + + + + Instance handle +Quark of the class of object associated with this location. + + + + + + Get a text representing object-detection metadata type. + + + #GstAnalyticsMtd type name. + + + + + Get a quark that represent object-detection metadata type + + + Quark of #GstAnalyticsMtd type + + + + + + + + No relation + + + First analysis-meta is part of second analysis-meta + + + First analysis-meta contain second analysis-meta. + + + + + reserved + + + Only use for criteria. + + + + Base structure for analysis-metadata that can be placed in relation. Only +other analysis-metadata based on GstAnalyticsRelatableMtdData should +directly use this structure. + + + Identify the type of analysis-metadata + + + + Instance identifier. + + + + Size in bytes of the instance + + + + + + + + + + + + + + + + + + + + + + + + + Add analytic classification metadata to @instance. + + + Added successfully + + + + + Instance of #GstAnalyticsRelationMeta where to add classification instance + + + + length of @confidence_levels + + + + confidence levels + + + + + + labels of this + classification. Order define index, quark, labels relation. This array + need to exist as long has this classification meta exist. + + + + + + Handle updated to newly added classification meta. + + + + + + Add a relatable metadata to @meta. This method is meant to be used by +new struct sub-classing GstAnalyticsRelatableMtd. + + + New GstAnalyticsRelatableMtdData instance. + +Since 1.24 + + + + + Instance + + + + Type of relatable (#GstAnalyticsRelatableMtd) + + + + Size required + + + + Updated handle + + + + + + + + Added successfully + + + + + Instance of #GstAnalyticsRelationMeta where to add classification instance + + + + Quark of the object type + + + + x component of bounding box upper-left corner + + + + y component of bounding box upper-left corner + + + + bounding box width + + + + bounding box height + + + + confidence level on the object location + + + + Handle updated with newly added object detection + meta. Add an object-detetion metadata to @instance. + + + + + + Add analytic classification metadata to @instance. + + + Added successfully + + + + + Instance of #GstAnalyticsRelationMeta where to add classification instance + + + + confidence levels + + + + labels of this + classification. Order define index, quark, labels relation. This array + need to exist as long has this classification meta exist. + + + + Handle updated to newly added classification meta. + + + + + + + + Added successfully + + + + + Instance of GstAnalyticsRelationMeta where to add tracking mtd + + + + Tracking id + + + + Timestamp of first time the object was observed. + + + + Handle updated with newly added tracking meta. +Add an analytic tracking metadata to @instance. + + + + + + Verify existence of relation(s) between @an_meta_first_d and +@an_meta_second_id according to relation condition @cond_types. It optionally +also return a shortest path of relations ( compliant with @cond_types) +between @an_meta_first_id and @an_meta_second_id. + + + TRUE if a relation between exit between @an_meta_first_id and + @an_meta_second_id, otherwise FALSE. + +Since 1.24 + + + + + a #GstAnalyticsRelationMeta describing analysis-meta + relation + + + + First analysis-meta + + + + Second analysis-meta + + + + Maximum number of relation between @an_meta_first_id and + @an_meta_second_id. + A value of 1 mean only only consider direct relation. + + + + condition on relation types. + + + + + If not NULL this list will be filled with relation path between + @an_meta_first_id and + @an_meta_second_id. List value should be access with GSList API. Use + GPOINTER_TO_INT(iter->data) where iter is a GSList element to get + analysis-meta id on the relation path. Free this list with g_slist_free + (@relations_path) after using. + + + + + + + + Fill @rlt if a analytics-meta with id == @an_meta_id exist in @meta instance, +otherwise this method return FALSE and @rlt is invalid. + + + TRUE if successful. + +Since 1.24 + + + + + Instance of #GstAnalyticsRelationMeta + + + + Id of #GstAnalyticsClsMtd instance to retrieve + + + + Will be filled with relatable + meta + + + + + + + + TRUE if @rlt_mtd was updated, other wise FALSE + +Since 1.24 + + + + + GstAnalyticsRelationMeta instance where to query for + GstAnalyticsRelatableMtd. + + + + Id of GstAnalyticsMtd involved in relation to query + + + + Type of relation to filter on. + + + + Type of GstAnalyticsMtd to filter on + + + + Opaque data to store state of the query. + If @state point to NULL, the first analytics-metadata directly related + to @an_meta_id will be set in @rlt_mtd. Doesn't need to be free. + + + + Handle updated to directly related relatable meta. + + + + + + Fill @rlt if a analytics-meta with id == @an_meta_id exist in @meta instance, +otherwise this method return FALSE and @rlt is invalid. + + + TRUE if successful. + +Since 1.24 + + + + + Instance of GstAnalyticsRelationMeta + + + + Id of GstAnalyticsMtd instance to retrieve + + + + Filter on a specific type of analysis, use + %GST_ANALYTICS_MTD_TYPE_ANY to match any type + + + + Will be filled with relatable + meta + + + + + + + + Instance of GstAnalyticsRelatableMtdData + +Since 1.24 + + + + + Instance of GstAnalyticsRelationMeta + + + + Id of GstAnalyticsMtd instance to retrieve + + + + + + Fill @rlt if a analytics-meta with id == @an_meta_id exist in @meta instance, +otherwise this method return FALSE and @rlt is invalid. + + + TRUE if successful. + +Since 1.24 + + + + + + + + Id of #GstAnalyticsOdMtd instance to retrieve + + + + Will be filled with relatable + meta + + + + + + + + relation description between first and second analysis-meta. + +Get relations between first and second analysis-meta. +Ids (@an_meta_first_id and @an_meta_second_id) must be from a call to +@gst_analytics_mtd_get_id (handle). + + + + + a #GstAnalyticsRelationMeta + + + + Id of first analysis-meta + + + + Id of second analysis-meta + + + + + + Fill @rlt if a analytics-meta with id == @an_meta_id exist in @meta instance, +otherwise this method return FALSE and @rlt is invalid. + + + TRUE if successful. + +Since 1.24 + + + + + + + + Id of GstAnalyticsMtd instance to retrieve + + + + Will be filled with relatable + meta + + + + + + + + FALSE if end was reached and iteration is completed. + +Since 1.24 + + + + + Instance of GstAnalyticsRelationMeta + + + + Opaque data to store iteration state, initialize to NULL, no need to + free it. + + + + Type of GstAnalyticsMtd to iterate on or use + %GST_ANALYTICS_MTD_TYPE_ANY for any. + + + + Handle updated to iterated GstAnalyticsRelatableMtd. + + + + + + Sets the relation (#GstAnalyticsRelTypes) between @an_meta_first and + @an_meta_second. +Ids must have been obtained a call to + @gst_analytics_mtd_get_id(handle). + + + TRUE on success and FALSE on failure. + + + + + Parameter to receive new maximum number of + analysis-meta described by relation. + + + + a #GstAnalyticsRelTypes defining relation between two analysis-meta + + + + first meta id + + + + second meta id + + + + + + + + + + + + + GstAnalyticsRelationMeta initialization parameters. + + + Initial relations order. + + + + Buffer size in bytes to store relatable metadata + + + + + Store information on results of object tracking + + + Instance identifier. + + + + Instance of #GstAnalyticsRelationMeta where the analysis-metadata +identified by @id is stored. + + + + Retrieve tracking information. + + + Successfully retrieved info. + + + + + Instance of tracking metadata + + + + Updated tracking id + + + + Updated timestamp of the tracking first observation. + + + + Updated timestamp of the tracking last observation. + + + + Has the tracking been lost + + + + + + + + Update successful + + + + + Instance of GstAnalyticsTrackingMtd. +Set tracking to lost + + + + + + + + + + + + GstAnalyticsTrackingMtd instance + + + + Timestamp of last time this object was tracked + + + + + + + + + + + + + + Quark representing the type of GstAnalyticsRelatableMtd + +Get the quark identifying the relatable type + + + + + + Attach a analysis-results-meta-relation meta (#GstAnalyticsRelationMeta)to @buffer. + +A #GstAnalyticsRelationMeta is a metadata describing relation between other +analysis meta. It's more efficient to use #gst_buffer_add_analytics_relation_meta_full +and providing the maximum number of analysis meta that will attached to a buffer. + + + Newly attached #GstAnalyticsRelationMeta + +Since 1.24 + + + + + a #GstBuffer + + + + + + Attache a analysis-results relation-meta (#GstAnalyticsRelationMeta) to @buffer. + +A #GstAnalyticsRelationMeta is a metadata describing relation between other +analysis meta. + + + Newly attached #GstAnalyticsRelationMeta + +Since 1.24 + + + + + a #GstBuffer + + + + Initialization parameters + + + + + + Retrives the meta or %NULL if it doesn't exist + + + The #GstAnalyticsRelationMeta if there is one + + + + + a #GstBuffer + + + + + + Get the static string representing #GstAnalyticsMtd type. + + + #GstAnalyticsMtd type name. + + + + + Get a quark identifying #GstAnalyticsMtd type. + + + Quark of #GstAnalyticsMtd type + + + + + Get a text representing object-detection metadata type. + + + #GstAnalyticsMtd type name. + + + + + Get a quark that represent object-detection metadata type + + + Quark of #GstAnalyticsMtd type + + + + + @instance Instance of #GstAnalyticsRelationMeta +Get number of relatable meta attached to instance + + + Number of analysis-meta attached to this +instance. + + + + + + + + + + + + GType of GstAnalyticsRelationMeta + + + + + + + + + + + + + + + + + + + Quark representing the type of GstAnalyticsRelatableMtd + +Get the quark identifying the relatable type + + + + + diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics-meta-prelude.h b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics-meta-prelude.h new file mode 100644 index 0000000000..6b4ae3bf68 --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics-meta-prelude.h @@ -0,0 +1,36 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * analytics-meta-prelude.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Publicn + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_ANALYTICS_META_PRELUDE_H__ +#define __GST_ANALYTICS_META_PRELUDE_H__ + +#include + +#ifndef GST_ANALYTICS_META_API +# ifdef BUILDING_GST_ANALYTIC_META +# define GST_ANALYTICS_META_API GST_API_EXPORT +# else +# define GST_ANALYTICS_META_API GST_API_IMPORT +# endif +#endif + +#endif /* __GST_ANALYTICS_META_PRELUDE_H__ */ + diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h new file mode 100644 index 0000000000..d25cf3ba35 --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/analytics.h @@ -0,0 +1,34 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * analytics.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + + + +#ifndef __ANALYTICS_H__ +#define __ANALYTICS_H__ + +#include +#include +#include +#include +#include +#include + +#endif /* __ANALYTICS_H__ */ diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.c b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.c new file mode 100644 index 0000000000..9f5bf1991a --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.c @@ -0,0 +1,271 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * gstanalyticsclassificationmtd.c + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstanalyticsclassificationmtd.h" + +#define GST_RELATABLE_MTD_CLASSIFICATION_TYPE_NAME "classification" + +static char type[] = GST_RELATABLE_MTD_CLASSIFICATION_TYPE_NAME; + +typedef struct _GstAnalyticsClsConfLvlAndClass GstAnalyticsClsConfLvlAndClass; +typedef struct _GstAnalyticsClsMtdData GstAnalyticsClsMtdData; + +struct _GstAnalyticsClsConfLvlAndClass +{ + GQuark class; + gfloat confidence_levels; +}; + +/* + * GstAnalyticsClsMtd: + * @parent: parent + * @length: classes and confidence levels count + * @class_quarks: (array length=length): Array of quark representing a class + * @confidence_levels: (array length=length): Array of confidence levels for + * each class in @class_quarks. + * + * Information on results of a classification of buffer content. + */ +struct _GstAnalyticsClsMtdData +{ + GstAnalyticsRelatableMtdData parent; + gsize length; + GstAnalyticsClsConfLvlAndClass confidence_levels_and_classes[]; // Must be last +}; + + +static GstAnalyticsClsMtdData * +gst_analytics_cls_mtd_get_data (GstAnalyticsClsMtd * instance) +{ + GstAnalyticsRelatableMtdData *rlt_data = + gst_analytics_relation_meta_get_mtd_data (instance->meta, + instance->id); + g_return_val_if_fail (rlt_data, NULL); + g_return_val_if_fail (rlt_data->analysis_type == + gst_analytics_cls_mtd_get_type_quark (), NULL); + + return (GstAnalyticsClsMtdData *) rlt_data; +} + +/** + * gst_analytics_cls_mtd_get_type_quark: + * Get a quark identifying #GstAnalyticsMtd type. + * + * Returns: Quark of #GstAnalyticsMtd type + * + * Since: 1.24 + */ +GstAnalyticsMtdType +gst_analytics_cls_mtd_get_type_quark (void) +{ + return g_quark_from_static_string (type); +} + +/** + * gst_analytics_cls_mtd_get_type_name: + * Get the static string representing #GstAnalyticsMtd type. + * + * Returns: #GstAnalyticsMtd type name. + * + * Since: 1.24 + */ +const gchar * +gst_analytics_cls_mtd_get_type_name (void) +{ + return GST_RELATABLE_MTD_CLASSIFICATION_TYPE_NAME; +} + +/** + * gst_analytics_cls_mtd_get_level: + * @handle: instance handle + * @index: Object class index + * + * Get confidence level for class at @index + * Returns: confidence level for @index, <0.0 if the call failed. + * + * Since: 1.24 + */ +gfloat +gst_analytics_cls_mtd_get_level (GstAnalyticsClsMtd * handle, gint index) +{ + g_return_val_if_fail (handle, -1.0); + g_return_val_if_fail (index >= 0, -1.0); + g_return_val_if_fail (handle->meta != NULL, -1.0); + GstAnalyticsClsMtdData *cls_mtd_data; + cls_mtd_data = gst_analytics_cls_mtd_get_data (handle); + g_return_val_if_fail (cls_mtd_data != NULL, -1.0); + g_return_val_if_fail (cls_mtd_data->length > index, -1.0); + return cls_mtd_data->confidence_levels_and_classes[index].confidence_levels; +} + +/** + * gst_analytics_cls_mtd_get_index_by_quark: + * @handle: Instance handle + * @quark: Quark of the class + * Get index of class represented by @quark + * Returns: index of the class associated with @quarks ( and label) or + * a negative value on failure. + * + * Since: 1.24 + */ +gint +gst_analytics_cls_mtd_get_index_by_quark (GstAnalyticsClsMtd * handle, + GQuark quark) +{ + g_return_val_if_fail (handle, -1); + + GstAnalyticsClsMtdData *cls_mtd_data; + cls_mtd_data = gst_analytics_cls_mtd_get_data (handle); + g_return_val_if_fail (cls_mtd_data != NULL, -1); + + for (gint i = 0; i < cls_mtd_data->length; i++) { + if (quark == cls_mtd_data->confidence_levels_and_classes[i].class) { + return i; + } + } + return -1; +} + +/** + * gst_analytics_cls_mtd_get_length: + * @handle: Instance handle + * Get number of classes + * Returns: Number of classes in this classification instance + * + * Since: 1.24 + */ +gsize +gst_analytics_cls_mtd_get_length (GstAnalyticsClsMtd * handle) +{ + GstAnalyticsClsMtdData *cls_mtd_data; + cls_mtd_data = gst_analytics_cls_mtd_get_data (handle); + g_return_val_if_fail (cls_mtd_data != NULL, 0); + return cls_mtd_data->length; +} + +/** + * gst_analytics_cls_mtd_get_quark: + * @handle: Instance handle + * @index: index of the class + * Get quark of the class at @index + * Returns: Quark of this class (label) associated with @index + * + * Since: 1.24 + */ +GQuark +gst_analytics_cls_mtd_get_quark (GstAnalyticsClsMtd * handle, gint index) +{ + GstAnalyticsClsMtdData *cls_mtd_data; + g_return_val_if_fail (handle, 0); + cls_mtd_data = gst_analytics_cls_mtd_get_data (handle); + g_return_val_if_fail (cls_mtd_data != NULL, 0); + return cls_mtd_data->confidence_levels_and_classes[index].class; +} + +/** + * gst_analytics_relation_meta_add_cls_mtd: + * @instance: Instance of #GstAnalyticsRelationMeta where to add classification instance + * @length: length of @confidence_levels + * @confidence_levels: (array length=length): confidence levels + * @class_quarks: (array length=length): labels of this + * classification. Order define index, quark, labels relation. This array + * need to exist as long has this classification meta exist. + * @cls_mtd: (out) (not nullable): Handle updated to newly added classification meta. + * + * Add analytic classification metadata to @instance. + * Returns: Added successfully + * + * Since: 1.24 + */ +gboolean +gst_analytics_relation_meta_add_cls_mtd (GstAnalyticsRelationMeta * + instance, gsize length, gfloat * confidence_levels, GQuark * class_quarks, + GstAnalyticsClsMtd * cls_mtd) +{ + GQuark type = gst_analytics_cls_mtd_get_type_quark (); + g_return_val_if_fail (instance, FALSE); + gsize confidence_levels_size = + (sizeof (GstAnalyticsClsConfLvlAndClass) * length); + gsize size = sizeof (GstAnalyticsClsMtdData) + confidence_levels_size; + GstAnalyticsClsConfLvlAndClass *conf_lvls_and_classes; + + GstAnalyticsClsMtdData *cls_mtd_data = (GstAnalyticsClsMtdData *) + gst_analytics_relation_meta_add_mtd (instance, type, size, cls_mtd); + if (cls_mtd_data) { + cls_mtd_data->length = length; + for (gsize i = 0; i < length; i++) { + conf_lvls_and_classes = &(cls_mtd_data->confidence_levels_and_classes[i]); + conf_lvls_and_classes->class = class_quarks[i]; + conf_lvls_and_classes->confidence_levels = confidence_levels[i]; + } + } + return cls_mtd_data != NULL; +} + +/** + * gst_analytics_relation_meta_add_one_cls_mtd: + * @instance: Instance of #GstAnalyticsRelationMeta where to add classification instance + * @confidence_level: confidence levels + * @class_quark: labels of this + * classification. Order define index, quark, labels relation. This array + * need to exist as long has this classification meta exist. + * @cls_mtd: (out) (not nullable): Handle updated to newly added classification meta. + * + * Add analytic classification metadata to @instance. + * Returns: Added successfully + * + * Since: 1.24 + */ +gboolean +gst_analytics_relation_meta_add_one_cls_mtd (GstAnalyticsRelationMeta * + instance, gfloat confidence_level, GQuark class_quark, + GstAnalyticsClsMtd * cls_mtd) +{ + static const gsize len = 1; + + return gst_analytics_relation_meta_add_cls_mtd (instance, len, + &confidence_level, &class_quark, cls_mtd); +} + +/** + * gst_analytics_relation_meta_get_cls_mtd: + * @meta: Instance of #GstAnalyticsRelationMeta + * @an_meta_id: Id of #GstAnalyticsClsMtd instance to retrieve + * @rlt: (out caller-allocates)(not nullable): Will be filled with relatable + * meta + * + * Fill @rlt if a analytics-meta with id == @an_meta_id exist in @meta instance, + * otherwise this method return FALSE and @rlt is invalid. + * + * Returns: TRUE if successful. + * + * Since 1.24 + */ +gboolean +gst_analytics_relation_meta_get_cls_mtd (GstAnalyticsRelationMeta * meta, + gint an_meta_id, GstAnalyticsClsMtd * rlt) +{ + return gst_analytics_relation_meta_get_mtd (meta, an_meta_id, + gst_analytics_cls_mtd_get_type_quark (), (GstAnalyticsClsMtd *) rlt); +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.h b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.h new file mode 100644 index 0000000000..753952fa2a --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsclassificationmtd.h @@ -0,0 +1,82 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * gstanalyticsclassificationmtd.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_ANALYTICS_CLASSIFICATION_H__ +#define __GST_ANALYTICS_CLASSIFICATION_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +/** + * GstAnalyticsClsMtd: + * @id: Instance identifier. + * @meta: Instance of #GstAnalyticsRelationMeta where the analysis-metadata + * identified by @id is stored. + * + * Handle containing data required to use gst_analytics_cls_mtd APIs. This type + * is generally expected to be allocated on the stack. + * + * Since: 1.24 + */ +typedef struct _GstAnalyticsMtd GstAnalyticsClsMtd; + +GST_ANALYTICS_META_API +GstAnalyticsMtdType gst_analytics_cls_mtd_get_type_quark (void); + +GST_ANALYTICS_META_API +const gchar *gst_analytics_cls_mtd_get_type_name (void); + +GST_ANALYTICS_META_API +gfloat gst_analytics_cls_mtd_get_level (GstAnalyticsClsMtd * handle, + gint index); + +GST_ANALYTICS_META_API +gint gst_analytics_cls_mtd_get_index_by_quark (GstAnalyticsClsMtd * handle, + GQuark quark); + +GST_ANALYTICS_META_API +gsize gst_analytics_cls_mtd_get_length (GstAnalyticsClsMtd * handle); + +GST_ANALYTICS_META_API +GQuark gst_analytics_cls_mtd_get_quark (GstAnalyticsClsMtd * handle, gint index); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_add_cls_mtd (GstAnalyticsRelationMeta * + instance, gsize length, gfloat * confidence_levels, GQuark * class_quarks, + GstAnalyticsClsMtd * cls_mtd); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_add_one_cls_mtd (GstAnalyticsRelationMeta * + instance, gfloat confidence_level, GQuark class_quark, + GstAnalyticsClsMtd * cls_mtd); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_get_cls_mtd (GstAnalyticsRelationMeta * meta, + gint an_meta_id, GstAnalyticsClsMtd * rlt); + +G_END_DECLS +#endif // __GST_ANALYTICS_CLASSIFICATION_H__ diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.c b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.c new file mode 100644 index 0000000000..388d320dad --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.c @@ -0,0 +1,1038 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * gstanalyticsmeta.c + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstanalyticsmeta.h" + +GST_DEBUG_CATEGORY_STATIC (an_relation_meta_debug); +#define GST_CAT_AN_RELATION an_relation_meta_debug + +static char invalid_type_name[] = "_invalid"; + +static gint +gst_analytics_relation_meta_get_next_id (GstAnalyticsRelationMeta * meta); + +/** + * gst_analytics_mtd_get_type_quark: + * @instance: Instance of #GstAnalyticsMtd + * Get analysis result type. + * + * Returns: quark associated with type. + * + * Since: 1.24 + */ +GstAnalyticsMtdType +gst_analytics_mtd_get_type_quark (GstAnalyticsMtd * handle) +{ + GstAnalyticsRelatableMtdData *rlt; + rlt = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id); + + g_return_val_if_fail (rlt != NULL, + g_quark_from_static_string (invalid_type_name)); + + return rlt->analysis_type; +} + +/** + * gst_analytics_mtd_get_id: + * @instance: Instance of #GstAnalyticsMtd + * Get instance id + * + * Returns: Id of @instance + * + * Since: 1.24 + */ +guint +gst_analytics_mtd_get_id (GstAnalyticsMtd * handle) +{ + return handle->id; +} + +/** + * gst_analytics_mtd_get_size: + * @instance Instance of #GstAnalyticsRelatableMtd + * Get instance size + * + * Returns: Size (in bytes) of this instance or 0 on failure. + * + * Since: 1.24 + */ +gsize +gst_analytics_mtd_get_size (GstAnalyticsMtd * handle) +{ + GstAnalyticsRelatableMtdData *rlt; + rlt = gst_analytics_relation_meta_get_mtd_data (handle->meta, handle->id); + if (rlt == NULL) { + GST_CAT_ERROR (GST_CAT_AN_RELATION, "Invalid parameter"); + return 0; + } + + return rlt->size; +} + +/* + * GstAnalyticsRelationMeta: + * Meta storing analysis-metadata relation information. + * + */ +typedef struct _GstAnalyticsRelationMeta +{ + GstMeta parent_meta; + + /*< Next id > */ + guint next_id; + + /* Adjacency-matrix */ + guint8 **adj_mat; + + /* Lookup (offset relative to analysis_results) of relatable metadata */ + gsize *mtd_data_lookup; + + /* Relation order */ + gsize rel_order; + + /* Increment used when relation order grow */ + gsize rel_order_increment; + + /* Analysis metadata based on GstAnalyticsRelatableMtdData */ + gint8 *analysis_results; + + /* Current writing location in analysis_results buffer */ + gsize offset; + + /* Size of analysis_results buffer */ + gsize max_size; + + /* Increment of analysis_results */ + gsize max_size_increment; + + /* Number of analytic metadata (GstAnalyticsRelatableMtdData) in + * analysis_results */ + gsize length; + +} GstAnalyticsRelationMeta; + +/** + * gst_analytics_relation_get_length: + * @instance Instance of #GstAnalyticsRelationMeta + * Get number of relatable meta attached to instance + * + * Returns: Number of analysis-meta attached to this + * instance. + * Since: 1.24 + */ +gsize +gst_analytics_relation_get_length (GstAnalyticsRelationMeta * instance) +{ + gsize rv; + g_return_val_if_fail (instance, 0); + + rv = instance->length; + return rv; +} + +/* + * gst_analytics_relation_adj_mat_create: + * @order: Order or the adjacency-matrix to create. + * Create a new adjacency-matrix (array of MxN where M and N are equal + * to order). + * + * Returns: new adjacency-matrix + * + * Since: 1.24 + */ +static guint8 ** +gst_analytics_relation_adj_mat_create (gsize order) +{ + guint8 **adj_mat, *data; + gsize sz = sizeof (guint8 *) * order + sizeof (guint8) * order * order; + adj_mat = (guint8 **) g_malloc0 (sz); + data = (guint8 *) (adj_mat + order); + for (gsize r = 0; r < order; r++) { + adj_mat[r] = (data + order * r); + } + return adj_mat; +} + +/** + * gst_analytics_relation_adj_mat_dup: + * @adj_mat: Adjcency-matrix (array or MxN) + * @order: Order of the existing matrix + * @new_order: Order of the matrix to create + * Duplicate adj_mat to a newly allocated array new_order x new_order dimension + * while keep values of adj_mat at the same indexes in the new array. + * + * Returns: New adjacency matrix with maintained values. + * + * Since: 1.24 + */ +static guint8 ** +gst_analytics_relation_adj_mat_dup (guint8 ** adj_mat, gsize order, + gsize new_order) +{ + guint8 **new_adj_mat = gst_analytics_relation_adj_mat_create (new_order); + for (gsize r = 0; r < order; r++) { + memcpy (new_adj_mat[r], adj_mat[r], sizeof (guint8) * order); + } + return new_adj_mat; +} + +/** + * gst_analytics_relation_meta_api_get_type: + * Returns: GType of GstAnalyticsRelationMeta + * + * Since: 1.24 + */ +GType +gst_analytics_relation_meta_api_get_type (void) +{ + static GType type = 0; + static const gchar *tags[] = { NULL }; + if (g_once_init_enter (&type)) { + GType newType = + gst_meta_api_type_register ("GstAnalyticsRelationMetaAPI", tags); + GST_DEBUG_CATEGORY_INIT (an_relation_meta_debug, "anrelmeta", + GST_DEBUG_FG_BLACK, "Content analysis meta relations meta"); + g_once_init_leave (&type, newType); + } + return type; +} + +static gboolean +gst_analytics_relation_meta_init (GstMeta * meta, gpointer params, + GstBuffer * buffer) +{ + GstAnalyticsRelationMeta *rmeta = (GstAnalyticsRelationMeta *) meta; + GstAnalyticsRelationMetaInitParams *rel_params = params; + rmeta->next_id = 0; + + g_return_val_if_fail (params != NULL, FALSE); + + GST_CAT_TRACE (GST_CAT_AN_RELATION, "Relation order:%" G_GSIZE_FORMAT, + *((gsize *) params)); + + rmeta->rel_order_increment = rel_params->initial_relation_order; + rmeta->rel_order = rmeta->rel_order_increment; + if (rmeta->rel_order > 0) { + rmeta->adj_mat = gst_analytics_relation_adj_mat_create (rmeta->rel_order); + rmeta->mtd_data_lookup = g_malloc0 (sizeof (gpointer) * rmeta->rel_order); + } + rmeta->offset = 0; + rmeta->max_size = rmeta->max_size_increment = rel_params->initial_buf_size; + rmeta->analysis_results = g_malloc (rel_params->initial_buf_size); + rmeta->length = 0; + + if (buffer->pool) + GST_META_FLAG_SET (meta, GST_META_FLAG_POOLED); + + GST_CAT_DEBUG (GST_CAT_AN_RELATION, + "Content analysis meta-relation meta(%p, order=%" G_GSIZE_FORMAT + ") created for buffer(%p)", (gpointer) rmeta, *(gsize *) params, + (gpointer) buffer); + return TRUE; +} + +static void +gst_analytics_relation_meta_free (GstMeta * meta, GstBuffer * buffer) +{ + GstAnalyticsRelationMeta *rmeta = (GstAnalyticsRelationMeta *) meta; + gpointer state = NULL; + GstAnalyticsMtd mtd; + GstAnalyticsRelatableMtdData *data; + GST_CAT_TRACE (GST_CAT_AN_RELATION, + "Content analysis meta-data(%p) freed for buffer(%p)", + (gpointer) rmeta, (gpointer) buffer); + + /* call custom free function if set */ + while (gst_analytics_relation_meta_iterate (rmeta, &state, 0, &mtd)) { + data = gst_analytics_relation_meta_get_mtd_data (mtd.meta, mtd.id); + if (data->free != NULL) { + data->free (data); + } + } + + g_free (rmeta->analysis_results); + g_free (rmeta->adj_mat); + g_free (rmeta->mtd_data_lookup); +} + +static gboolean +gst_analytics_relation_meta_transform (GstBuffer * transbuf, + GstMeta * meta, GstBuffer * buffer, GQuark type, gpointer data) +{ + + GST_CAT_TRACE (GST_CAT_AN_RELATION, "meta transform %s", + g_quark_to_string (type)); + + if (GST_META_TRANSFORM_IS_COPY (type)) { + GstAnalyticsRelationMeta *rmeta = (GstAnalyticsRelationMeta *) meta; + GstAnalyticsRelationMeta *new = (GstAnalyticsRelationMeta *) + gst_buffer_get_meta (transbuf, GST_ANALYTICS_RELATION_META_API_TYPE); + + if (new == NULL) { + GstAnalyticsRelationMetaInitParams init_params = { + rmeta->rel_order, rmeta->max_size + }; + + GST_CAT_TRACE (GST_CAT_AN_RELATION, + "meta transform creating new meta rel_order:%" G_GSIZE_FORMAT + " max_size:%" G_GSIZE_FORMAT, + init_params.initial_relation_order, init_params.initial_buf_size); + + new = + gst_buffer_add_analytics_relation_meta_full (transbuf, &init_params); + } + + if (new->offset == 0) { + + if (new->rel_order < rmeta->rel_order) { + g_free (new->adj_mat); + g_free (new->mtd_data_lookup); + new->adj_mat = gst_analytics_relation_adj_mat_create (rmeta->rel_order); + new->mtd_data_lookup = g_malloc0 (sizeof (gpointer) * rmeta->rel_order); + new->rel_order = rmeta->rel_order; + } + + if (new->max_size < rmeta->max_size) { + g_free (new->analysis_results); + new->analysis_results = g_malloc (rmeta->max_size); + new->max_size = rmeta->max_size; + } + + if (rmeta->rel_order == new->rel_order) { + memcpy (new->adj_mat + new->rel_order, rmeta->adj_mat, + rmeta->rel_order * rmeta->rel_order); + } else { + /* When destination adj_mat has a higher order than source we need + * to copy by row to have the correct alignment */ + for (gsize r = 0; r < rmeta->rel_order; r++) { + memcpy (new->adj_mat[r], rmeta->adj_mat[r], rmeta->rel_order); + } + } + memcpy (new->mtd_data_lookup, rmeta->mtd_data_lookup, + sizeof (gpointer) * rmeta->rel_order); + memcpy (new->analysis_results, rmeta->analysis_results, rmeta->offset); + + new->length = rmeta->length; + new->next_id = rmeta->next_id; + new->offset = rmeta->offset; + + return TRUE; + } else { + g_warning ("Trying to copy GstAnalyticsRelationMeta into non-empty meta"); + g_debug ("ofs:%" G_GSIZE_FORMAT, new->offset); + + return FALSE; + } + } else if (GST_META_TRANSFORM_IS_CLEAR (type)) { + GstAnalyticsRelationMeta *rmeta = (GstAnalyticsRelationMeta *) meta; + gsize adj_mat_data_size = + (sizeof (guint8) * rmeta->rel_order * rmeta->rel_order); + + rmeta->next_id = 0; + rmeta->offset = 0; + rmeta->length = 0; + if (adj_mat_data_size) { + /* Only clear data and not lines addresses occupying begining of this + * array. */ + memset (rmeta->adj_mat + rmeta->rel_order, 0, adj_mat_data_size); + } + + return TRUE; + } + + return FALSE; +} + + +const GstMetaInfo * +gst_analytics_relation_meta_get_info (void) +{ + static const GstMetaInfo *info = NULL; + if (g_once_init_enter ((GstMetaInfo **) & info)) { + const GstMetaInfo *meta = + gst_meta_register (GST_ANALYTICS_RELATION_META_API_TYPE, + "GstAnalyticsRelationMeta", + sizeof (GstAnalyticsRelationMeta), + gst_analytics_relation_meta_init, + gst_analytics_relation_meta_free, + gst_analytics_relation_meta_transform); + g_once_init_leave ((GstMetaInfo **) & info, (GstMetaInfo *) meta); + } + return info; +} + +/* + * gst_analytics_relation_meta_bfs: + * @start: start vertex + * @adj_mat: graph's adjacency matrix + * @adj_mat_order: order of the adjacency matrix (number of vertex in the graph) + * @edge_mask: allow to select edge type we are interested by. + * @max_span: Maximum number of edge to traverse from start vertex while + * exploring graph. + * @level: (out)(not nullable)(array length=adj_mat_order): Array that will be + * filled with number of edge to traverse to reach @start from the vertex + * identified by the array index. (Ex: start=1 and level[3]=2, mean we need + * to traverse 2 edges from vertex 2 to vertex 3. A value of -1 in @level + * mean this vertex is un-reachable considering @edge_mask, @max_span and + * @adj_mat. + * @parent: (out)(array length=adj_mat_order): Array that will be filled with + * shortest path information. The shortest path from vertex X and vertex + * @start, where X is the index of @parent array. To find each node on the + * path we need to recursively do ...parent[parent[parent[X]]] until value + * is @start. Value at index Y equal to -1 mean there's no path from + * vertex Y to vertex @start. + * + * This function can be used to find the existence of relation paths that + * emerge from a analysis-metadata (@start). The existence verification can + * also be parametrize by only considering certain types of relation + * (@edge_mask), a maximum intermediates analysis-metadata (@max_span). A + * usage exemple can be found in @gst_analytics_relation_meta_exist. + */ +static void +gst_analytics_relation_meta_bfs (gint start, const guint8 ** adj_mat, + gsize adj_mat_order, guint8 edge_mask, gsize max_span, gint * level, + gint * parent) +{ + GSList *frontier = NULL; + GSList *iter; + GSList *next_frontier; + gsize i = 1; + memset (level, -1, sizeof (gint) * adj_mat_order); + memset (parent, -1, sizeof (gint) * adj_mat_order); + + GST_CAT_TRACE (GST_CAT_AN_RELATION, + "Performing bfs to find relation(%x) starting from %d with less than %" + G_GSIZE_FORMAT " edges from start", edge_mask, start, max_span); + + // vertex that has a relation with itself + if (adj_mat[start][start] & edge_mask) { + level[start] = 0; + } + + frontier = g_slist_prepend (frontier, GINT_TO_POINTER (start)); + + while (frontier && i <= max_span) { + next_frontier = NULL; + for (iter = frontier; iter; iter = iter->next) { + for (gsize j = 0; j < adj_mat_order; j++) { + if (adj_mat[(gsize) GPOINTER_TO_INT (iter->data)][j] & edge_mask) { + if (level[j] == -1) { + level[j] = i; + parent[j] = GPOINTER_TO_INT (iter->data); + GST_CAT_TRACE (GST_CAT_AN_RELATION, "Parent of %" G_GSIZE_FORMAT + " is %d", j, parent[j]); + next_frontier = + g_slist_prepend (next_frontier, GINT_TO_POINTER ((gint) j)); + } + } + } + } + g_slist_free (frontier); + frontier = next_frontier; + i++; + } + g_slist_free (frontier); +} + +/* + * gst_analytics_relation_meta_get_next_id: + * @meta a #GstAnalyticsRelationMeta from which we want to get next id. + * + * Get next id and prepare for future request. + * + * Returns: next id + * + */ +static gint +gst_analytics_relation_meta_get_next_id (GstAnalyticsRelationMeta * meta) +{ + g_return_val_if_fail (meta != NULL, -1); + return g_atomic_int_add (&meta->next_id, 1); +} + +/** + * gst_analytics_relation_meta_get_relation: + * @meta: (transfer none): a #GstAnalyticsRelationMeta + * @an_meta_first_id: Id of first analysis-meta + * @an_meta_second_id: Id of second analysis-meta + * + * Returns: relation description between first and second analysis-meta. + * + * Get relations between first and second analysis-meta. + * Ids (@an_meta_first_id and @an_meta_second_id) must be from a call to + * @gst_analytics_mtd_get_id (handle). + * + * Since: 1.24 + */ +GstAnalyticsRelTypes +gst_analytics_relation_meta_get_relation (GstAnalyticsRelationMeta * meta, + gint an_meta_first, gint an_meta_second) +{ + GstAnalyticsRelTypes types = GST_ANALYTICS_REL_TYPE_NONE; + g_return_val_if_fail (meta, GST_ANALYTICS_REL_TYPE_NONE); + + g_return_val_if_fail (meta->adj_mat != NULL, GST_ANALYTICS_REL_TYPE_NONE); + if (meta->rel_order > an_meta_first && meta->rel_order > an_meta_second) { + types = meta->adj_mat[an_meta_first][an_meta_second]; + } else { + GST_CAT_DEBUG (GST_CAT_AN_RELATION, + "an_meta_first(%i) and an_meta_second(%i) must be inferior to %" + G_GSIZE_FORMAT, an_meta_first, an_meta_second, meta->rel_order); + + if (an_meta_first >= meta->rel_order) { + GST_CAT_ERROR (GST_CAT_AN_RELATION, + "an_meta_first(%i) must be from a call to " + "gst_analytics_mtd_get_id(...)", an_meta_first); + } + + if (an_meta_second >= meta->rel_order) { + GST_CAT_ERROR (GST_CAT_AN_RELATION, + "an_meta_second(%i) must be from a call to " + "gst_analytics_mtd_get_id(...)", an_meta_second); + } + } + return types; +} + +/** + * gst_analytics_relation_meta_set_relation: + * @meta: (transfer none): Parameter to receive new maximum number of + * analysis-meta described by relation. + * @type: a #GstAnalyticsRelTypes defining relation between two analysis-meta + * @an_meta_first_id: first meta id + * @an_meta_second_id: second meta id + * + * Sets the relation (#GstAnalyticsRelTypes) between @an_meta_first and + * @an_meta_second. + * Ids must have been obtained a call to + * @gst_analytics_mtd_get_id(handle). + * + * Returns: TRUE on success and FALSE on failure. + * + * Since: 1.24 + */ +gboolean +gst_analytics_relation_meta_set_relation (GstAnalyticsRelationMeta * meta, + GstAnalyticsRelTypes type, gint an_meta_first_id, gint an_meta_second_id) +{ + g_return_val_if_fail (type < GST_ANALYTICS_REL_TYPE_LAST, FALSE); + g_return_val_if_fail (meta, FALSE); + if (an_meta_first_id >= meta->rel_order + || an_meta_second_id >= meta->rel_order) { + GST_CAT_ERROR (GST_CAT_AN_RELATION, "Invalid parameter"); + return FALSE; + } + meta->adj_mat[an_meta_first_id][an_meta_second_id] = type; + GST_CAT_TRACE (GST_CAT_AN_RELATION, + "Relation %x set between %d and %d", + type, an_meta_first_id, an_meta_second_id); + return TRUE; +} + +/** + * gst_analytics_relation_meta_exist: + * @rmeta: (transfer none): a #GstAnalyticsRelationMeta describing analysis-meta + * relation + * @an_meta_first_id: First analysis-meta + * @an_meta_second_id: Second analysis-meta + * @max_relation_span: Maximum number of relation between @an_meta_first_id and + * @an_meta_second_id. + * A value of 1 mean only only consider direct relation. + * @cond_types: condition on relation types. + * @relations_path: (transfer full)(nullable)(out caller-allocates)(array) + * (element-type gint): + * If not NULL this list will be filled with relation path between + * @an_meta_first_id and + * @an_meta_second_id. List value should be access with GSList API. Use + * GPOINTER_TO_INT(iter->data) where iter is a GSList element to get + * analysis-meta id on the relation path. Free this list with g_slist_free + * (@relations_path) after using. + * + * Verify existence of relation(s) between @an_meta_first_d and + * @an_meta_second_id according to relation condition @cond_types. It optionally + * also return a shortest path of relations ( compliant with @cond_types) + * between @an_meta_first_id and @an_meta_second_id. + * + * Returns: TRUE if a relation between exit between @an_meta_first_id and + * @an_meta_second_id, otherwise FALSE. + * + * Since 1.24 + */ +gboolean +gst_analytics_relation_meta_exist (GstAnalyticsRelationMeta * rmeta, + gint an_meta_first_id, + gint an_meta_second_id, + gint max_relation_span, + GstAnalyticsRelTypes cond_types, GArray ** relations_path) +{ + gboolean rv = FALSE; + guint8 **adj_mat; + gsize adj_mat_order, span; + GArray *path = NULL; + gint *level, i, path_left; + gint *parent; + g_return_val_if_fail (rmeta, FALSE); + + if (!rmeta) { + GST_CAT_ERROR (GST_CAT_AN_RELATION, "Invalid parameter"); + return EINVAL; + } + adj_mat_order = rmeta->rel_order; + + if (adj_mat_order < (an_meta_first_id + 1) + || adj_mat_order < (an_meta_second_id + 1)) { + + GST_CAT_DEBUG (GST_CAT_AN_RELATION, + "Testing relation existence for analysis-meta that have no index in " + "adj-mat."); + + return FALSE; + } + + adj_mat = rmeta->adj_mat; + if (max_relation_span < 0) { + span = G_MAXSIZE; + } + // If we're only considering the direct relation (@max_relation_span <= 1) we can directly read the + // adjacency-matrix, + if (max_relation_span == 0 || max_relation_span == 1) { + rv = (adj_mat[an_meta_first_id][an_meta_second_id] & cond_types) != 0; + if (rv && relations_path) { + path = *relations_path; + + /* re-use array if possible */ + if (path != NULL && (g_array_get_element_size (path) != sizeof (gint) + || path->len < 2)) { + g_array_free (path, TRUE); + path = NULL; + } + + if (path == NULL) { + path = g_array_sized_new (FALSE, FALSE, sizeof (gint), 2); + } + g_array_index (path, gint, 0) = an_meta_first_id; + g_array_index (path, gint, 1) = an_meta_second_id; + path->len = 2; + *relations_path = path; + } + } else { + + level = g_malloc (sizeof (gint) * adj_mat_order); + parent = g_malloc (sizeof (gint) * adj_mat_order); + gst_analytics_relation_meta_bfs (an_meta_first_id, + (const guint8 **) adj_mat, adj_mat_order, cond_types, span, level, + parent); + + GST_CAT_TRACE (GST_CAT_AN_RELATION, "Adj order:%" G_GSIZE_FORMAT, + adj_mat_order); + + rv = level[an_meta_second_id] != -1; + if (rv && relations_path) { + path_left = level[an_meta_second_id] + 1; + i = parent[an_meta_second_id]; + if (i != -1) { + path = *relations_path; + + /* re-use array if possible */ + if (path != NULL && (g_array_get_element_size (path) != sizeof (gint) + || path->len < path_left)) { + g_array_free (path, TRUE); + path = NULL; + } + + if (path == NULL) { + path = g_array_sized_new (FALSE, FALSE, sizeof (gint), path_left); + } + + path->len = path_left; + g_array_index (path, gint, --path_left) = an_meta_second_id; + //path = g_slist_prepend (path, GINT_TO_POINTER (an_meta_second_id)); + while (i != -1 && i != an_meta_second_id) { + GST_CAT_TRACE (GST_CAT_AN_RELATION, "Relation parent of %d", i); + g_array_index (path, gint, --path_left) = i; + //path = g_slist_prepend (path, GINT_TO_POINTER (i)); + i = parent[i]; + } + + while (path_left > 0) { + g_array_index (path, gint, --path_left) = -1; + } + } + *relations_path = path; + } + + g_free (level); + g_free (parent); + } + + GST_CAT_TRACE (GST_CAT_AN_RELATION, + "Relation %x between %d and %d %s", + cond_types, an_meta_first_id, an_meta_second_id, + rv ? "exist" : "does not exist"); + return rv; +} + + +/** + * gst_buffer_add_analytics_relation_meta: + * @buffer: (transfer none): a #GstBuffer + * + * Attach a analysis-results-meta-relation meta (#GstAnalyticsRelationMeta)to @buffer. + * + * A #GstAnalyticsRelationMeta is a metadata describing relation between other + * analysis meta. It's more efficient to use #gst_buffer_add_analytics_relation_meta_full + * and providing the maximum number of analysis meta that will attached to a buffer. + * + * Returns: (transfer none) (nullable) : Newly attached #GstAnalyticsRelationMeta + * + * Since 1.24 + */ +GstAnalyticsRelationMeta * +gst_buffer_add_analytics_relation_meta (GstBuffer * buffer) +{ + GstAnalyticsRelationMetaInitParams init_params = { 5, 1024 }; + return gst_buffer_add_analytics_relation_meta_full (buffer, &init_params); +} + +/** + * gst_buffer_add_analytics_relation_meta_full: + * @buffer: (transfer none): a #GstBuffer + * @init_params: Initialization parameters + * + * Attache a analysis-results relation-meta (#GstAnalyticsRelationMeta) to @buffer. + * + * A #GstAnalyticsRelationMeta is a metadata describing relation between other + * analysis meta. + * + * Returns: (transfer none) (nullable) : Newly attached #GstAnalyticsRelationMeta + * + * Since 1.24 + */ +GstAnalyticsRelationMeta * +gst_buffer_add_analytics_relation_meta_full (GstBuffer * buffer, + GstAnalyticsRelationMetaInitParams * init_params) +{ + GstAnalyticsRelationMeta *meta = NULL; + g_return_val_if_fail (init_params != NULL, NULL); + g_return_val_if_fail (buffer != NULL, NULL); + + // We only want one relation-meta on a buffer, will check if one already + // exist. + meta = (GstAnalyticsRelationMeta *) gst_buffer_get_meta (buffer, + GST_ANALYTICS_RELATION_META_API_TYPE); + + if (!meta) { + meta = + (GstAnalyticsRelationMeta *) gst_buffer_add_meta (buffer, + GST_ANALYTICS_RELATION_META_INFO, init_params); + } + + return meta; +} + +/** + * gst_buffer_get_analytics_relation_meta: + * @buffer: a #GstBuffer + * + * Retrives the meta or %NULL if it doesn't exist + * + * Returns: (transfer none) (nullable) :The #GstAnalyticsRelationMeta if there is one + * Since: 1.24: + */ +GstAnalyticsRelationMeta * +gst_buffer_get_analytics_relation_meta (GstBuffer * buffer) +{ + return (GstAnalyticsRelationMeta *) + gst_buffer_get_meta (buffer, GST_ANALYTICS_RELATION_META_API_TYPE); +} + +/** + * gst_analytics_relation_meta_add_mtd: (skip) + * @meta: Instance + * @type: Type of relatable (#GstAnalyticsRelatableMtd) + * @size: Size required + * @rlt_mtd: Updated handle + * + * Add a relatable metadata to @meta. This method is meant to be used by + * new struct sub-classing GstAnalyticsRelatableMtd. + * + * Returns: New GstAnalyticsRelatableMtdData instance. + * + * Since 1.24 + */ +GstAnalyticsRelatableMtdData * +gst_analytics_relation_meta_add_mtd (GstAnalyticsRelationMeta * meta, + GstAnalyticsMtdType type, gsize size, GstAnalyticsMtd * rlt_mtd) +{ + gsize new_size = size + meta->offset; + GstAnalyticsRelatableMtdData *dest = NULL; + gpointer mem; + guint8 **new_adj_mat; + gsize new_mem_cap, new_rel_order; + GST_CAT_TRACE (GST_CAT_AN_RELATION, "Adding relatable metadata to rmeta %p", + meta); + + if (new_size > meta->max_size) { + + if (new_size > meta->max_size_increment + meta->offset) { + new_mem_cap = new_size; + } else { + new_mem_cap = meta->max_size + meta->max_size_increment; + } + + mem = g_realloc (meta->analysis_results, new_mem_cap); + meta->max_size = new_mem_cap; + meta->analysis_results = mem; + } + + if (meta->length >= meta->rel_order) { + new_rel_order = meta->rel_order + meta->rel_order_increment; + new_adj_mat = gst_analytics_relation_adj_mat_dup (meta->adj_mat, + meta->rel_order, new_rel_order); + g_free (meta->adj_mat); + meta->adj_mat = new_adj_mat; + + mem = g_realloc (meta->mtd_data_lookup, sizeof (gpointer) * new_rel_order); + meta->mtd_data_lookup = mem; + meta->rel_order = new_rel_order; + } + + if (new_size <= meta->max_size && (meta->length < meta->rel_order)) { + dest = + (GstAnalyticsRelatableMtdData *) (meta->analysis_results + + meta->offset); + dest->analysis_type = type; + dest->id = gst_analytics_relation_meta_get_next_id (meta); + dest->size = size; + dest->free = NULL; + meta->mtd_data_lookup[dest->id] = meta->offset; + meta->offset += dest->size; + meta->length++; + rlt_mtd->id = dest->id; + rlt_mtd->meta = meta; + GST_CAT_TRACE (GST_CAT_AN_RELATION, "Add %p relatable type=%s (%" + G_GSIZE_FORMAT " / %" G_GSIZE_FORMAT ").", dest, + g_quark_to_string (type), new_size, meta->max_size); + } else { + GST_CAT_ERROR (GST_CAT_AN_RELATION, + "Failed to add relatable, out-of-space (%" G_GSIZE_FORMAT " / %" + G_GSIZE_FORMAT ").", new_size, meta->max_size); + } + return dest; +} + +/** + * gst_analytics_relation_meta_get_mtd: + * @meta: Instance of GstAnalyticsRelationMeta + * @an_meta_id: Id of GstAnalyticsMtd instance to retrieve + * @type: Filter on a specific type of analysis, use + * %GST_ANALYTICS_MTD_TYPE_ANY to match any type + * @rlt: (out caller-allocates)(not nullable): Will be filled with relatable + * meta + * + * Fill @rlt if a analytics-meta with id == @an_meta_id exist in @meta instance, + * otherwise this method return FALSE and @rlt is invalid. + * + * Returns: TRUE if successful. + * + * Since 1.24 + */ +gboolean +gst_analytics_relation_meta_get_mtd (GstAnalyticsRelationMeta * meta, + gint an_meta_id, GstAnalyticsMtdType type, GstAnalyticsMtd * rlt) +{ + GstAnalyticsRelatableMtdData *d; + + g_return_val_if_fail (meta, FALSE); + g_return_val_if_fail (rlt, FALSE); + + rlt->meta = NULL; + + if (an_meta_id < 0 || an_meta_id >= meta->length) { + GST_CAT_ERROR (GST_CAT_AN_RELATION, "Invalid parameter"); + return FALSE; + } + + d = gst_analytics_relation_meta_get_mtd_data (meta, an_meta_id); + if (d == NULL) + return FALSE; + + if (d->analysis_type == 0) { + return FALSE; + } + + if (type != GST_ANALYTICS_MTD_TYPE_ANY && d->analysis_type != type) { + return FALSE; + } + + rlt->meta = meta; + rlt->id = an_meta_id; + + return TRUE; +} + +/** + * gst_analytics_relation_meta_get_mtd_data: (skip) + * @meta: Instance of GstAnalyticsRelationMeta + * @an_meta_id: Id of GstAnalyticsMtd instance to retrieve + * + * Returns:(nullable): Instance of GstAnalyticsRelatableMtdData + * + * Since 1.24 + */ +GstAnalyticsRelatableMtdData * +gst_analytics_relation_meta_get_mtd_data (GstAnalyticsRelationMeta * + meta, gint an_meta_id) +{ + GstAnalyticsRelatableMtdData *rv; + g_return_val_if_fail (meta, NULL); + if (an_meta_id < 0 || an_meta_id >= meta->rel_order) { + GST_CAT_ERROR (GST_CAT_AN_RELATION, "Invalid parameter"); + return NULL; + } + rv = (GstAnalyticsRelatableMtdData *) + (meta->mtd_data_lookup[an_meta_id] + meta->analysis_results); + return rv; +} + + +/** + * gst_analytics_relation_meta_get_direct_related: + * @meta: GstAnalyticsRelationMeta instance where to query for + * GstAnalyticsRelatableMtd. + * @an_meta_id: Id of GstAnalyticsMtd involved in relation to query + * @relation_type: Type of relation to filter on. + * @type: Type of GstAnalyticsMtd to filter on + * @state: (inout) (not nullable): Opaque data to store state of the query. + * If @state point to NULL, the first analytics-metadata directly related + * to @an_meta_id will be set in @rlt_mtd. Doesn't need to be free. + * @rlt_mtd: Handle updated to directly related relatable meta. + * + * Returns: TRUE if @rlt_mtd was updated, other wise FALSE + * + * Since 1.24 + */ +gboolean +gst_analytics_relation_meta_get_direct_related (GstAnalyticsRelationMeta * meta, + gint an_meta_id, GstAnalyticsRelTypes relation_type, + GstAnalyticsMtdType type, gpointer * state, GstAnalyticsMtd * rlt_mtd) +{ + guint8 **adj_mat; + gsize adj_mat_order; + GstAnalyticsRelationMeta *rmeta = (GstAnalyticsRelationMeta *) meta; + GstAnalyticsRelatableMtdData *rlt_mtd_data = NULL; + gsize i; + + GST_CAT_TRACE (GST_CAT_AN_RELATION, + "Looking for %s related to %d by %d", g_quark_to_string (type), + an_meta_id, relation_type); + + g_return_val_if_fail (rmeta != NULL, FALSE); + g_return_val_if_fail (type != 0, FALSE); + + if (state) { + if (*state) { + i = ~G_MINSSIZE & (GPOINTER_TO_SIZE (*state) + 1); + } else { + i = 0; + *state = GSIZE_TO_POINTER (G_MINSSIZE | i); + } + } else { + i = 0; + } + + adj_mat_order = meta->rel_order; + + if (adj_mat_order < (an_meta_id + 1)) { + GST_CAT_DEBUG (GST_CAT_AN_RELATION, + "Testing relation existence for analysis-meta that have no index in " + "adj-mat."); + return FALSE; + } + + rlt_mtd->meta = rmeta; + adj_mat = meta->adj_mat; + for (; i < adj_mat_order; i++) { + if (adj_mat[an_meta_id][i] & relation_type) { + rlt_mtd_data = (GstAnalyticsRelatableMtdData *) + (meta->mtd_data_lookup[i] + meta->analysis_results); + rlt_mtd->id = rlt_mtd_data->id; + if (gst_analytics_mtd_get_type_quark (rlt_mtd) == type) { + if (state) { + *state = GSIZE_TO_POINTER (G_MINSSIZE | i); + } + GST_CAT_TRACE (GST_CAT_AN_RELATION, "Found match at %" G_GSIZE_FORMAT, + i); + break; + } + rlt_mtd_data = NULL; + } + } + + return rlt_mtd_data != NULL; +} + +/** + * gst_analytics_relation_meta_iterate: + * @meta: Instance of GstAnalyticsRelationMeta + * @state: Opaque data to store iteration state, initialize to NULL, no need to + * free it. + * @type: Type of GstAnalyticsMtd to iterate on or use + * %GST_ANALYTICS_MTD_TYPE_ANY for any. + * @rlt_mtd: Handle updated to iterated GstAnalyticsRelatableMtd. + * + * Returns: FALSE if end was reached and iteration is completed. + * + * Since 1.24 + */ +gboolean +gst_analytics_relation_meta_iterate (GstAnalyticsRelationMeta * meta, + gpointer * state, GstAnalyticsMtdType type, GstAnalyticsMtd * rlt_mtd) +{ + gsize index; + gsize len = gst_analytics_relation_get_length (meta); + GstAnalyticsRelatableMtdData *rlt_mtd_data = NULL; + + g_return_val_if_fail (rlt_mtd != NULL, FALSE); + + if (*state) { + index = ~G_MINSSIZE & (GPOINTER_TO_SIZE (*state) + 1); + } else { + index = 0; + *state = GSIZE_TO_POINTER (G_MINSSIZE | index); + } + for (; index < len; index++) { + rlt_mtd_data = (GstAnalyticsRelatableMtdData *) + (meta->mtd_data_lookup[index] + meta->analysis_results); + rlt_mtd->id = rlt_mtd_data->id; + rlt_mtd->meta = meta; + if (type == GST_ANALYTICS_MTD_TYPE_ANY || + gst_analytics_mtd_get_type_quark (rlt_mtd) == type) { + *state = GSIZE_TO_POINTER (G_MINSSIZE | index); + return TRUE; + } + } + + return FALSE; +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h new file mode 100644 index 0000000000..23356dee08 --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsmeta.h @@ -0,0 +1,226 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * gstanalyticsmeta.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifndef __GST_ANALYTICS_META_H__ +#define __GST_ANALYTICS_META_H__ + +#include +#include + +G_BEGIN_DECLS +#define GST_INF_RELATION_SPAN -1 +#define GST_AN_RELATION_META_TAG "GST-ANALYSIS-RELATION-META-TAG" +typedef struct _GstAnalyticsMtd GstAnalyticsMtd; + +/** + * GstAnalyticsMtdType: + * + * Type of analytics meta data + * + * Since: 1.24 + */ +typedef guint32 GstAnalyticsMtdType; + +/** + * GST_ANALYTICS_MTD_TYPE_ANY: + * + * A wildcard matching any type of analysis + * + * Since: 1.24 + */ + +#define GST_ANALYTICS_MTD_TYPE_ANY (0) + +#define GST_ANALYTICS_MTD_CAST(mtd) \ + ((GstAnalyticsMtd *)(mtd)) + +typedef struct _GstAnalyticsRelatableMtdData GstAnalyticsRelatableMtdData; +typedef struct _GstAnalyticsRelationMeta GstAnalyticsRelationMeta; + +/** + * GstAnalyticsMtd: + * @id: Instance identifier. + * @meta: Instance of #GstAnalyticsRelationMeta where the analysis-metadata + * identified by @id is stored. + * + * Handle containing data required to use gst_analytics_mtd API. This type + * is generally expected to be allocated on the stack. + * + * Since: 1.24 + */ +struct _GstAnalyticsMtd +{ + guint id; + GstAnalyticsRelationMeta *meta; +}; + +/** + * GstAnalyticsRelatableMtdData: + * @analysis_type: Identify the type of analysis-metadata + * @id: Instance identifier. + * @size: Size in bytes of the instance + * + * Base structure for analysis-metadata that can be placed in relation. Only + * other analysis-metadata based on GstAnalyticsRelatableMtdData should + * directly use this structure. + * + * Since: 1.24 + */ +struct _GstAnalyticsRelatableMtdData +{ + GstAnalyticsMtdType analysis_type; + guint id; + gsize size; + + void (*free) (GstAnalyticsRelatableMtdData * mtd_data); + + gpointer _gst_reserved[GST_PADDING]; +}; + +GST_ANALYTICS_META_API +GstAnalyticsMtdType gst_analytics_mtd_get_type_quark (GstAnalyticsMtd * instance); + +GST_ANALYTICS_META_API +guint gst_analytics_mtd_get_id (GstAnalyticsMtd * instance); + +GST_ANALYTICS_META_API +gsize gst_analytics_mtd_get_size (GstAnalyticsMtd * instance); + +typedef struct _GstAnalyticsRelationMetaInitParams + GstAnalyticsRelationMetaInitParams; + +#define GST_ANALYTICS_RELATION_META_API_TYPE \ + (gst_analytics_relation_meta_api_get_type()) + +#define GST_ANALYTICS_RELATION_META_INFO \ + (gst_analytics_relation_meta_get_info()) + +/** + * GstAnalyticsRelTypes: + * @GST_ANALYTICS_REL_TYPE_NONE: No relation + * @GST_ANALYTICS_REL_TYPE_IS_PART_OF: First analysis-meta is part of second analysis-meta + * @GST_ANALYTICS_REL_TYPE_CONTAIN: First analysis-meta contain second analysis-meta. + * @GST_ANALYTICS_REL_TYPE_RELATE: First analysis-meta relate to second analysis-meta. + * @GST_ANALYTICS_REL_TYPE_LAST: reserved + * @GST_ANALYTICS_REL_TYPE_ANY: Only use for criteria. + * + * Since: 1.24 + */ +typedef enum +{ + GST_ANALYTICS_REL_TYPE_NONE = 0, + GST_ANALYTICS_REL_TYPE_IS_PART_OF = (1 << 1), + GST_ANALYTICS_REL_TYPE_CONTAIN = (1 << 2), + GST_ANALYTICS_REL_TYPE_RELATE_TO = (1 << 3), + GST_ANALYTICS_REL_TYPE_LAST = (1 << 4), + GST_ANALYTICS_REL_TYPE_ANY = G_MAXINT +} GstAnalyticsRelTypes; + + +/** + * GstAnalyticsRelationMetaInitParams: + * @initial_relation_order: Initial relations order. + * @initial_buf_size: Buffer size in bytes to store relatable metadata + * + * GstAnalyticsRelationMeta initialization parameters. + * + * Since: 1.24 + */ +struct _GstAnalyticsRelationMetaInitParams +{ + gsize initial_relation_order; + gsize initial_buf_size; +}; + +GST_ANALYTICS_META_API GType gst_analytics_relation_meta_api_get_type (void); + +GST_ANALYTICS_META_API +const GstMetaInfo *gst_analytics_relation_meta_get_info (void); + +GST_ANALYTICS_META_API +gsize +gst_analytics_relation_get_length (GstAnalyticsRelationMeta * + instance); + +GST_ANALYTICS_META_API +GstAnalyticsRelTypes +gst_analytics_relation_meta_get_relation (GstAnalyticsRelationMeta * meta, + gint an_meta_first_id, gint an_meta_second_id); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_set_relation (GstAnalyticsRelationMeta + * meta, GstAnalyticsRelTypes type, gint an_meta_first_id, + gint an_meta_second_id); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_exist (GstAnalyticsRelationMeta * + rmeta, gint an_meta_first_id, gint an_meta_second_id, + gint max_relation_span, GstAnalyticsRelTypes cond_types, + GArray ** relations_path); + +GST_ANALYTICS_META_API +GstAnalyticsRelationMeta * +gst_buffer_add_analytics_relation_meta (GstBuffer * buffer); + +GST_ANALYTICS_META_API +GstAnalyticsRelationMeta * +gst_buffer_add_analytics_relation_meta_full (GstBuffer * buffer, + GstAnalyticsRelationMetaInitParams * init_params); + +GST_ANALYTICS_META_API +GstAnalyticsRelationMeta * +gst_buffer_get_analytics_relation_meta (GstBuffer * buffer); + +GST_ANALYTICS_META_API +GstAnalyticsRelatableMtdData * +gst_analytics_relation_meta_add_mtd (GstAnalyticsRelationMeta * meta, + GstAnalyticsMtdType type, gsize size, GstAnalyticsMtd * rlt_mtd); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_get_mtd (GstAnalyticsRelationMeta * + meta, gint an_meta_id, GstAnalyticsMtdType type, GstAnalyticsMtd * rlt); + +GST_ANALYTICS_META_API +GstAnalyticsRelatableMtdData * +gst_analytics_relation_meta_get_mtd_data (GstAnalyticsRelationMeta * meta, + gint an_meta_id); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_iterate (GstAnalyticsRelationMeta * + meta, gpointer * state, GstAnalyticsMtdType type, + GstAnalyticsMtd * rlt_mtd); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_get_direct_related (GstAnalyticsRelationMeta * meta, + gint an_meta_id, GstAnalyticsRelTypes relation_type, + GstAnalyticsMtdType type, gpointer * state, GstAnalyticsMtd * rlt_mtd); + +G_END_DECLS +#endif // __GST_ANALYTICS_META_H__ diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.c b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.c new file mode 100644 index 0000000000..4b7dc34b4f --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.c @@ -0,0 +1,214 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * gstanalyticsobjectdetectionmtd.c + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstanalyticsobjectdetectionmtd.h" + +#define GST_RELATABLE_MTD_OD_TYPE_NAME "object-detection" + +static char type[] = GST_RELATABLE_MTD_OD_TYPE_NAME; + +typedef struct _GstAnalyticsODMtdData GstAnalyticsODMtdData; + +/** + * GstAnalyticsODMtd: + * @parent: parent #GstAnalyticsMtd + * @object_type: Type of object + * @x: x component of upper-left corner + * @y: y component of upper-left corner + * @w: bounding box width + * @h: bounding box height + * @location_confidence_lvl: Confidence on object location + * + * Store information on results of object detection + * + * Since: 1.24 + */ +struct _GstAnalyticsODMtdData +{ + GstAnalyticsRelatableMtdData parent; + GQuark object_type; + gint x; + gint y; + gint w; + gint h; + gfloat location_confidence_lvl; +}; + + +/** + * gst_analytics_od_mtd_get_type_quark: + * Get a quark that represent object-detection metadata type + * + * Returns: Quark of #GstAnalyticsMtd type + * + * Since: 1.24 + */ +GstAnalyticsMtdType +gst_analytics_od_mtd_get_type_quark (void) +{ + return g_quark_from_static_string (type); +} + +/** + * gst_analytics_od_mtd_get_type_name: + * Get a text representing object-detection metadata type. + * + * Returns: #GstAnalyticsMtd type name. + * + * Since: 1.24 + */ +const gchar * +gst_analytics_od_mtd_get_type_name (void) +{ + return GST_RELATABLE_MTD_OD_TYPE_NAME; +} + +static GstAnalyticsODMtdData * +gst_analytics_od_mtd_get_data (GstAnalyticsODMtd * instance) +{ + GstAnalyticsRelatableMtdData *rlt_data = + gst_analytics_relation_meta_get_mtd_data (instance->meta, + instance->id); + g_return_val_if_fail (rlt_data, NULL); + g_return_val_if_fail (rlt_data->analysis_type == + gst_analytics_od_mtd_get_type_quark (), NULL); + + return (GstAnalyticsODMtdData *) rlt_data; +} + +/** + * gst_analytics_od_mtd_get_location: + * @instance: instance + * @x: (out): x component of upper-left corner of the object location + * @y: (out): y component of upper-left corner of the object location + * @w: (out): bounding box width of the object location + * @h: (out): bounding box height of the object location + * @loc_conf_lvl: (out)(optional): Confidence on object location + + * + * Retrieve location and location confidence level. + * + * Returns: TRUE on success, otherwise FALSE. + * + * Since: 1.24 + */ +gboolean +gst_analytics_od_mtd_get_location (GstAnalyticsODMtd * instance, + gint * x, gint * y, gint * w, gint * h, gfloat * loc_conf_lvl) +{ + g_return_val_if_fail (instance && x && y && w && h, FALSE); + GstAnalyticsODMtdData *data; + data = gst_analytics_od_mtd_get_data (instance); + g_return_val_if_fail (data != NULL, FALSE); + + if (data) { + *x = data->x; + *y = data->y; + *w = data->w; + *h = data->h; + + if (loc_conf_lvl) { + *loc_conf_lvl = data->location_confidence_lvl; + } + } + + return TRUE; +} + +/** + * gst_analytics_od_mtd_get_type: + * @handle: Instance handle + * Quark of the class of object associated with this location. + * Returns: Quark different from on success and 0 on failure. + * + * Since: 1.24 + */ +GQuark +gst_analytics_od_mtd_get_type (GstAnalyticsODMtd * handle) +{ + GstAnalyticsODMtdData *data; + g_return_val_if_fail (handle != NULL, 0); + data = gst_analytics_od_mtd_get_data (handle); + g_return_val_if_fail (data != NULL, 0); + return data->object_type; +} + +/** + * gst_analytics_relation_meta_add_od_mtd: + * @instance: Instance of #GstAnalyticsRelationMeta where to add classification instance + * @type: Quark of the object type + * @x: x component of bounding box upper-left corner + * @y: y component of bounding box upper-left corner + * @w: bounding box width + * @h: bounding box height + * @loc_conf_lvl: confidence level on the object location + * @od_mtd: (out)(nullable): Handle updated with newly added object detection + * meta. Add an object-detetion metadata to @instance. + * + * Returns: Added successfully + * + * Since: 1.24 + */ +gboolean +gst_analytics_relation_meta_add_od_mtd (GstAnalyticsRelationMeta * + instance, GQuark type, gint x, gint y, gint w, gint h, + gfloat loc_conf_lvl, GstAnalyticsODMtd * od_mtd) +{ + g_return_val_if_fail (instance != NULL, FALSE); + GstAnalyticsMtdType mtd_type = gst_analytics_od_mtd_get_type_quark (); + gsize size = sizeof (GstAnalyticsODMtdData); + GstAnalyticsODMtdData *od_mtd_data = (GstAnalyticsODMtdData *) + gst_analytics_relation_meta_add_mtd (instance, mtd_type, size, od_mtd); + if (od_mtd_data) { + od_mtd_data->x = x; + od_mtd_data->y = y; + od_mtd_data->w = w; + od_mtd_data->h = h; + od_mtd_data->location_confidence_lvl = loc_conf_lvl; + od_mtd_data->object_type = type; + } + return od_mtd_data != NULL; +} + +/** + * gst_analytics_relation_meta_get_od_mtd: + * @meta: Instance of #GstAnalyticsRelationMeta + * @an_meta_id: Id of #GstAnalyticsOdMtd instance to retrieve + * @rlt: (out caller-allocates)(not nullable): Will be filled with relatable + * meta + * + * Fill @rlt if a analytics-meta with id == @an_meta_id exist in @meta instance, + * otherwise this method return FALSE and @rlt is invalid. + * + * Returns: TRUE if successful. + * + * Since 1.24 + */ +gboolean +gst_analytics_relation_meta_get_od_mtd (GstAnalyticsRelationMeta * meta, + gint an_meta_id, GstAnalyticsODMtd * rlt) +{ + return gst_analytics_relation_meta_get_mtd (meta, an_meta_id, + gst_analytics_od_mtd_get_type_quark (), (GstAnalyticsODMtd *) rlt); +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h new file mode 100644 index 0000000000..266fb70fa7 --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjectdetectionmtd.h @@ -0,0 +1,73 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * gstanalyticsobjectdetectionmtd.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifndef __GST_ANALYTICS_OBJECT_DETECTION_MTD__ +#define __GST_ANALYTICS_OBJECT_DETECTION_MTD__ + +#include +#include +#include + +G_BEGIN_DECLS + +/** + * GstAnalyticsODMtd: + * @id: Instance identifier. + * @meta: Instance of #GstAnalyticsRelationMeta where the analysis-metadata + * identified by @id is stored. + * + * Handle containing data required to use gst_analytics_od_mtd APIs. This type + * is generally expected to be allocated on the stack. + * + * Since: 1.24 + */ +typedef struct _GstAnalyticsMtd GstAnalyticsODMtd; + +GST_ANALYTICS_META_API +GstAnalyticsMtdType gst_analytics_od_mtd_get_type_quark (void); + +GST_ANALYTICS_META_API +const gchar *gst_analytics_od_mtd_get_type_name (void); + +GST_ANALYTICS_META_API +gboolean gst_analytics_od_mtd_get_location (GstAnalyticsODMtd * instance, + gint * x, gint * y, gint * w, gint * h, gfloat * loc_conf_lvl); + +GST_ANALYTICS_META_API +GQuark gst_analytics_od_mtd_get_type (GstAnalyticsODMtd * handle); + +GST_ANALYTICS_META_API +gboolean gst_analytics_relation_meta_add_od_mtd ( + GstAnalyticsRelationMeta * instance, GQuark type, gint x, gint y, + gint w, gint h, gfloat loc_conf_lvl, GstAnalyticsODMtd * od_mtd); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_get_od_mtd (GstAnalyticsRelationMeta * xbmeta, + gint an_meta_id, GstAnalyticsODMtd * rlt); + + +G_END_DECLS +#endif // __GST_ANALYTICS_OBJECT_DETECTION_MTD__ diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.c b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.c new file mode 100644 index 0000000000..69a32759ea --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.c @@ -0,0 +1,228 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * gstanalyticsobjecttrackingmtd.c + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstanalyticsobjecttrackingmtd.h" + +#define GST_RELATABLE_MTD_TRACKING_TYPE_NAME "object-tracking" + +typedef struct _GstAnalyticsTrackingMtdData GstAnalyticsTrackingMtdData; + +/** + * GstAnalyticsTrackingMtd: + * @parent: parent #GstAnalyticsMtd + * @tracking_id: Tracking identifier + * @tracking_first_seen: Tracking creation time + * @tracking_last_seen: Tracking last observation + * @tracking_lost: Track lost + * + * Store information on results of object tracking + * + * Since: 1.24 + */ +struct _GstAnalyticsTrackingMtdData +{ + GstAnalyticsRelatableMtdData parent; + guint64 tracking_id; + GstClockTime tracking_first_seen; + GstClockTime tracking_last_seen; + gboolean tracking_lost; +}; + + +static char type[] = GST_RELATABLE_MTD_TRACKING_TYPE_NAME; + +static GstAnalyticsTrackingMtdData * +gst_analytics_tracking_mtd_get_data (GstAnalyticsTrackingMtd * instance) +{ + GstAnalyticsRelatableMtdData *rlt_data = + gst_analytics_relation_meta_get_mtd_data (instance->meta, + instance->id); + g_return_val_if_fail (rlt_data, NULL); + + g_return_val_if_fail (rlt_data->analysis_type == + gst_analytics_tracking_mtd_get_type_quark (), NULL); + + return (GstAnalyticsTrackingMtdData *) rlt_data; +} + +/** + * gst_analytics_tracking_mtd_get_type_quark: + * Returns: Quark representing the type of GstAnalyticsRelatableMtd + * + * Get the quark identifying the relatable type + * + * Since: 1.24 + */ +GstAnalyticsMtdType +gst_analytics_tracking_mtd_get_type_quark (void) +{ + return g_quark_from_static_string (type); +} + +/** + * gst_an_od_mtd_get_type_name: + * Returns: #GstAnalyticsMtd type name. + * + * Get the name identifying relatable type name + * + * Since: 1.24 + */ +const gchar * +gst_analytics_tracking_mtd_get_type_name (void) +{ + return GST_RELATABLE_MTD_TRACKING_TYPE_NAME; +} + +/** + * gst_analytics_tracking_mtd_update_last_seen: + * @instance: GstAnalyticsTrackingMtd instance + * @last_seen: Timestamp of last time this object was tracked + * + * Since: 1.24 + */ +gboolean +gst_analytics_tracking_mtd_update_last_seen (GstAnalyticsTrackingMtd * instance, + GstClockTime last_seen) +{ + GstAnalyticsTrackingMtdData *trk_mtd_data; + g_return_val_if_fail (instance, FALSE); + trk_mtd_data = gst_analytics_tracking_mtd_get_data (instance); + g_return_val_if_fail (trk_mtd_data != NULL, FALSE); + + trk_mtd_data->tracking_last_seen = last_seen; + return TRUE; +} + +/** + * gst_analytics_tracking_mtd_set_lost: + * @instance: Instance of GstAnalyticsTrackingMtd. + * Set tracking to lost + * + * Returns: Update successful + * + * Since: 1.24 + */ +gboolean +gst_analytics_tracking_mtd_set_lost (GstAnalyticsTrackingMtd * instance) +{ + GstAnalyticsTrackingMtdData *trk_mtd_data; + g_return_val_if_fail (instance, FALSE); + trk_mtd_data = gst_analytics_tracking_mtd_get_data (instance); + g_return_val_if_fail (trk_mtd_data != NULL, FALSE); + trk_mtd_data->tracking_lost = TRUE; + return TRUE; +} + +/** + * gst_analytics_tracking_mtd_get_info: + * @instance: Instance of tracking metadata + * @tracking_id: (out): Updated tracking id + * @tracking_first_seen: (out): Updated timestamp of the tracking first observation. + * @tracking_last_seen: (out): Updated timestamp of the tracking last observation. + * @tracking_lost: (out): Has the tracking been lost + * + * Retrieve tracking information. + * + * Returns: Successfully retrieved info. + * Since: 1.24 + */ +gboolean +gst_analytics_tracking_mtd_get_info (GstAnalyticsTrackingMtd * instance, + guint64 * tracking_id, GstClockTime * tracking_first_seen, GstClockTime * + tracking_last_seen, gboolean * tracking_lost) +{ + GstAnalyticsTrackingMtdData *trk_mtd_data; + g_return_val_if_fail (instance, FALSE); + trk_mtd_data = gst_analytics_tracking_mtd_get_data (instance); + + g_return_val_if_fail (trk_mtd_data != NULL, FALSE); + + if (tracking_id) + *tracking_id = trk_mtd_data->tracking_id; + if (tracking_first_seen) + *tracking_first_seen = trk_mtd_data->tracking_first_seen; + if (tracking_last_seen) + *tracking_last_seen = trk_mtd_data->tracking_last_seen; + if (tracking_lost) + *tracking_lost = trk_mtd_data->tracking_lost; + + return TRUE; +} + + +/** + * gst_analytics_relation_meta_add_tracking_mtd: + * @instance: Instance of GstAnalyticsRelationMeta where to add tracking mtd + * @tracking_id: Tracking id + * @tracking_first_seen: Timestamp of first time the object was observed. + * @trk_mtd: (out) (not nullable): Handle updated with newly added tracking meta. + * Add an analytic tracking metadata to @instance. + * Returns: Added successfully + * + * Since: 1.24 + */ +gboolean +gst_analytics_relation_meta_add_tracking_mtd (GstAnalyticsRelationMeta * + instance, guint64 tracking_id, GstClockTime tracking_first_seen, + GstAnalyticsTrackingMtd * trk_mtd) +{ + g_return_val_if_fail (instance, FALSE); + + GstAnalyticsMtdType type = gst_analytics_tracking_mtd_get_type_quark (); + gsize size = sizeof (GstAnalyticsTrackingMtdData); + GstAnalyticsTrackingMtdData *trk_mtd_data = (GstAnalyticsTrackingMtdData *) + gst_analytics_relation_meta_add_mtd (instance, + type, size, trk_mtd); + + if (trk_mtd_data) { + trk_mtd_data->tracking_id = tracking_id; + trk_mtd_data->tracking_first_seen = tracking_first_seen; + trk_mtd_data->tracking_last_seen = tracking_first_seen; + trk_mtd_data->tracking_lost = FALSE; + } + return trk_mtd_data != NULL; +} + +/** + * gst_analytics_relation_meta_get_tracking_mtd: + * @meta: Instance of GstAnalyticsRelationMeta + * @an_meta_id: Id of GstAnalyticsMtd instance to retrieve + * @rlt: (out caller-allocates)(not nullable): Will be filled with relatable + * meta + * + * Fill @rlt if a analytics-meta with id == @an_meta_id exist in @meta instance, + * otherwise this method return FALSE and @rlt is invalid. + * + * Returns: TRUE if successful. + * + * Since 1.24 + */ +gboolean +gst_analytics_relation_meta_get_tracking_mtd (GstAnalyticsRelationMeta * meta, + gint an_meta_id, GstAnalyticsTrackingMtd * rlt) +{ + return gst_analytics_relation_meta_get_mtd (meta, an_meta_id, + gst_analytics_tracking_mtd_get_type_quark (), + (GstAnalyticsTrackingMtd *) rlt); +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h new file mode 100644 index 0000000000..15353cee40 --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/gstanalyticsobjecttrackingmtd.h @@ -0,0 +1,78 @@ +/* GStreamer + * Copyright (C) 2023 Collabora Ltd + * + * gstobjecttrackingmtd.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifndef __GST_ANALYTICS_OBJECT_TRACKING_MTD__ +#define __GST_ANALYTICS_OBJECT_TRACKING_MTD__ + +#include +#include +#include + +G_BEGIN_DECLS + +/** + * GstAnalyticsTrackMtd: + * @id: Instance identifier. + * @meta: Instance of #GstAnalyticsRelationMeta where the analysis-metadata + * identified by @id is stored. + * + * Handle containing data required to use gst_analytics_track_mtd APIs. + * This type is generally expected to be allocated on the stack. + * + * Since: 1.24 + */ +typedef struct _GstAnalyticsMtd GstAnalyticsTrackingMtd; + +GST_ANALYTICS_META_API +GQuark gst_analytics_tracking_mtd_get_type_quark (void); + +GST_ANALYTICS_META_API +const gchar *gst_analytics_tracking_mtd_get_type_name (void); + +GST_ANALYTICS_META_API +gboolean gst_analytics_tracking_mtd_update_last_seen (GstAnalyticsTrackingMtd * instance, + GstClockTime last_seen); + +GST_ANALYTICS_META_API +gboolean gst_analytics_tracking_mtd_set_lost (GstAnalyticsTrackingMtd * instance); + +GST_ANALYTICS_META_API +gboolean gst_analytics_tracking_mtd_get_info (GstAnalyticsTrackingMtd * instance, + guint64 * tracking_id, GstClockTime * tracking_first_seen, GstClockTime * + tracking_last_seen, gboolean * tracking_lost); + +GST_ANALYTICS_META_API +gboolean gst_analytics_relation_meta_add_tracking_mtd ( + GstAnalyticsRelationMeta * instance, guint64 tracking_id, + GstClockTime tracking_first_seen, GstAnalyticsTrackingMtd * trk_mtd); + +GST_ANALYTICS_META_API +gboolean +gst_analytics_relation_meta_get_tracking_mtd (GstAnalyticsRelationMeta * xbmeta, + gint an_meta_id, GstAnalyticsTrackingMtd * rlt); + + +G_END_DECLS +#endif // __GST_ANALYTICS_OBJECT_TRACKING_MTD__ diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build new file mode 100644 index 0000000000..55bad96d59 --- /dev/null +++ b/subprojects/gst-plugins-bad/gst-libs/gst/analytics/meson.build @@ -0,0 +1,64 @@ +analytics_sources = files( 'gstanalyticsmeta.c', + 'gstanalyticsclassificationmtd.c', + 'gstanalyticsobjectdetectionmtd.c', + 'gstanalyticsobjecttrackingmtd.c') + +analytics_headers = files( 'analytics.h', + 'gstanalyticsmeta.h', + 'analytics-meta-prelude.h', + 'gstanalyticsclassificationmtd.h', + 'gstanalyticsobjectdetectionmtd.h', + 'gstanalyticsobjecttrackingmtd.h') +install_headers(analytics_headers, subdir : 'gstreamer-1.0/gst/analytics') + +pkg_name = 'gstreamer-analytics-1.0' +gstanalytics = library('gstanalytics-' + api_version, + analytics_sources, + c_args : gst_plugins_bad_args + ['-DGST_USE_UNSTABLE_API', '-DBUILDING_GST_ANALYTIC_META', '-DG_LOG_DOMAIN="Gstreamer-Analytic"'], + include_directories : [configinc, libsinc], + version : libversion, + soversion : soversion, + darwin_versions : osxversion, + install : true, + dependencies : [gstbase_dep]) + +gst_libraries += [[pkg_name, {'lib': gstanalytics}]] +pkgconfig.generate(gstanalytics, + libraries : [gst_dep, gstbase_dep], + variables : pkgconfig_variables, + subdirs : pkgconfig_subdirs, + name : pkg_name, + description : 'GStreamer analytics metadata library', +) +gen_sources = [] + +if build_gir + gir = { + 'sources' : analytics_sources + analytics_headers, + 'namespace' : 'GstAnalytics', + 'nsversion' : api_version, + 'identifier_prefix' : ['GstAnalytics', 'Gst'], + 'symbol_prefix' : ['gst_analytics', 'gst'], + 'export_packages' : pkg_name, + 'includes' : ['Gst-1.0', 'GstBase-1.0'], + 'install' : true, + 'extra_args' : gir_init_section + ['-DGST_USE_UNSTABLE_API'], + 'dependencies' : [gstbase_dep] + } + library_def += {'gir': [gir]} + if not static_build + analyticsmeta_gir = gnome.generate_gir(gstanalytics, kwargs: gir) + library_def += {'gir_targets': library_def.get('gir_targets', []) + [analyticsmeta_gir]} + gen_sources += analyticsmeta_gir + endif +endif +gst_libraries += [[pkg_name, library_def]] + + + +gstanalytics_dep = declare_dependency(link_with : gstanalytics, + include_directories : [libsinc], + sources: gen_sources, + dependencies : [gstbase_dep]) + +meson.override_dependency(pkg_name, gstanalytics_dep) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/meson.build index 7772d8c2b1..6207f3381a 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/meson.build +++ b/subprojects/gst-plugins-bad/gst-libs/gst/meson.build @@ -24,3 +24,4 @@ subdir('vulkan') subdir('wayland') subdir('webrtc') subdir('winrt') +subdir('analytics') diff --git a/subprojects/gst-plugins-bad/tests/check/libs/analyticsmeta.c b/subprojects/gst-plugins-bad/tests/check/libs/analyticsmeta.c new file mode 100644 index 0000000000..a6d42d8e36 --- /dev/null +++ b/subprojects/gst-plugins-bad/tests/check/libs/analyticsmeta.c @@ -0,0 +1,978 @@ +/* GStreamer + * Copyright (C) 2022 Collabora Ltd + * + * analyticmeta.c + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include +#include + +GST_START_TEST (test_add_classification_meta) +{ + /* Verify we can create a relation metadata + * and attach it classification mtd + */ + GstBuffer *buf; + GstAnalyticsRelationMeta *rmeta; + GstAnalyticsClsMtd cls_mtd; + gfloat conf_lvl[] = { 0.5f, 0.5f }; + GQuark class_quarks[2]; + gboolean ret; + + class_quarks[0] = g_quark_from_string ("dog"); + class_quarks[1] = g_quark_from_string ("cat"); + + buf = gst_buffer_new (); + rmeta = gst_buffer_add_analytics_relation_meta (buf); + ret = gst_analytics_relation_meta_add_cls_mtd (rmeta, 2, conf_lvl, + class_quarks, &cls_mtd); + fail_unless (ret == TRUE); + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_meta_pooled) +{ + GstBufferPool *pool; + GstStructure *config; + GstBuffer *buf; + GstAnalyticsRelationMeta *rmeta1, *rmeta2; + + pool = gst_buffer_pool_new (); + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, NULL, 1, 1, 1); + gst_buffer_pool_set_config (pool, config); + gst_buffer_pool_set_active (pool, TRUE); + + gst_buffer_pool_acquire_buffer (pool, &buf, NULL); + rmeta1 = gst_buffer_add_analytics_relation_meta (buf); + gst_buffer_unref (buf); + + gst_buffer_pool_acquire_buffer (pool, &buf, NULL); + + rmeta2 = gst_buffer_add_analytics_relation_meta (buf); + + fail_unless (rmeta1 == rmeta2); + gst_buffer_unref (buf); + gst_object_unref (pool); +} + +GST_END_TEST; + +GST_START_TEST (test_classification_meta_classes) +{ + /* Verify we can retrieve classification data + * from the relation metadata + */ + GstBuffer *buf; + GstAnalyticsRelationMeta *rmeta; + gboolean ret; + GQuark class_quarks[2]; + GstAnalyticsClsMtd cls_mtd, cls_mtd2; + + class_quarks[0] = g_quark_from_string ("dog"); + class_quarks[1] = g_quark_from_string ("cat"); + + buf = gst_buffer_new (); + rmeta = gst_buffer_add_analytics_relation_meta (buf); + gfloat conf_lvl[] = { 0.6f, 0.4f }; + ret = gst_analytics_relation_meta_add_cls_mtd (rmeta, 2, conf_lvl, + class_quarks, &cls_mtd); + fail_unless (ret == TRUE); + fail_unless (gst_analytics_relation_get_length (rmeta) == 1); + + gint dogIndex = gst_analytics_cls_mtd_get_index_by_quark (&cls_mtd, + class_quarks[0]); + fail_unless (dogIndex == 0); + gfloat confLvl = gst_analytics_cls_mtd_get_level (&cls_mtd, dogIndex); + GST_LOG ("dog:%f", confLvl); + assert_equals_float (confLvl, 0.6f); + + gint catIndex = gst_analytics_cls_mtd_get_index_by_quark (&cls_mtd, + g_quark_from_string ("cat")); + confLvl = gst_analytics_cls_mtd_get_level (&cls_mtd, catIndex); + GST_LOG ("Cat:%f", confLvl); + assert_equals_float (confLvl, 0.4f); + assert_equals_int (gst_analytics_mtd_get_id ((GstAnalyticsMtd *) & cls_mtd), + 0); + + conf_lvl[0] = 0.1f; + conf_lvl[1] = 0.9f; + ret = + gst_analytics_relation_meta_add_cls_mtd (rmeta, 2, conf_lvl, + class_quarks, &cls_mtd2); + fail_unless (ret == TRUE); + fail_unless (gst_analytics_relation_get_length (rmeta) == 2); + + dogIndex = gst_analytics_cls_mtd_get_index_by_quark (&cls_mtd2, + class_quarks[0]); + confLvl = gst_analytics_cls_mtd_get_level (&cls_mtd2, dogIndex); + assert_equals_float (confLvl, 0.1f); + + catIndex = gst_analytics_cls_mtd_get_index_by_quark (&cls_mtd2, + class_quarks[0]); + confLvl = gst_analytics_cls_mtd_get_level (&cls_mtd2, catIndex); + assert_equals_float (confLvl, 0.1f); + + /* Verify the relation meta contain the correct number of relatable metadata */ + fail_unless (gst_analytics_relation_get_length (rmeta) == 2); + + /* Verify first relatable metadata has the correct id. */ + assert_equals_int (gst_analytics_mtd_get_id ((GstAnalyticsMtd *) & cls_mtd), + 0); + + /* Verify second relatable metadata has the correct id. */ + assert_equals_int (gst_analytics_mtd_get_id ( + (GstAnalyticsMtd *) & cls_mtd2), 1); + + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_add_relation_meta) +{ + /* Verify we set a relation between relatable metadata. */ + + GstBuffer *buf; + GstAnalyticsClsMtd cls_mtd[3]; + GstAnalyticsRelationMetaInitParams init_params = { 5, 150 }; + GstAnalyticsRelationMeta *relations; + GQuark class_quarks[2]; + gboolean ret; + + buf = gst_buffer_new (); + relations = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + gfloat conf_lvl[] = { 0.6f, 0.4f }; + class_quarks[0] = g_quark_from_string ("dog"); + class_quarks[1] = g_quark_from_string ("cat"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[0]); + fail_unless (ret == TRUE); + gst_analytics_mtd_get_id ((GstAnalyticsMtd *) + & cls_mtd[0]); + + conf_lvl[0] = 0.6f; + conf_lvl[1] = 0.4f; + + class_quarks[0] = g_quark_from_string ("plant"); + class_quarks[1] = g_quark_from_string ("animal"); + + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[1]); + gst_analytics_mtd_get_id ((GstAnalyticsMtd *) + & cls_mtd[1]); + + fail_unless (gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd[0].id, + cls_mtd[1].id) == TRUE); + + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_add_relation_inefficiency_reporting_cases) +{ + /* + * Verify inefficiency of relation order is reported. + */ + GstBuffer *buf; + GstAnalyticsClsMtd cls_mtd[3]; + GstAnalyticsRelationMetaInitParams init_params = { 2, 10 }; + GstAnalyticsRelationMeta *relations; + gboolean ret; + GQuark class_quarks[2]; + + + buf = gst_buffer_new (); + relations = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + gfloat conf_lvl[] = { 0.6f, 0.4f }; + + class_quarks[0] = g_quark_from_string ("dog"); + class_quarks[1] = g_quark_from_string ("cat"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[0]); + fail_unless (gst_analytics_relation_get_length (relations) == 1); + fail_unless (ret == TRUE); + + conf_lvl[0] = 0.6f; + conf_lvl[1] = 0.4f; + class_quarks[0] = g_quark_from_string ("plant"); + class_quarks[1] = g_quark_from_string ("animal"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[1]); + fail_unless (gst_analytics_relation_get_length (relations) == 2); + fail_unless (ret == TRUE); + + conf_lvl[0] = 0.6f; + conf_lvl[1] = 0.4f; + class_quarks[0] = g_quark_from_string ("male"); + class_quarks[1] = g_quark_from_string ("female"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[2]); + fail_unless (gst_analytics_relation_get_length (relations) == 3); + fail_unless (ret == TRUE); + + fail_unless (gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd[0].id, cls_mtd[1].id) + == TRUE); + fail_unless (gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd[0].id, cls_mtd[2].id) + == TRUE); + + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_query_relation_meta_cases) +{ + /* Verify we can query existence of direct and indirect relation */ + GstBuffer *buf; + GstAnalyticsClsMtd cls_mtd[3]; + GstAnalyticsRelationMetaInitParams init_params = { 2, 150 }; + GstAnalyticsRelationMeta *relations; + gboolean ret; + GQuark class_quarks[2]; + + buf = gst_buffer_new (); + relations = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + gfloat conf_lvl[] = { 0.6f, 0.4f }; + + class_quarks[0] = g_quark_from_string ("dog"); + class_quarks[1] = g_quark_from_string ("cat"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[0]); + fail_unless (ret == TRUE); + + conf_lvl[0] = 0.6f; + conf_lvl[1] = 0.4f; + class_quarks[0] = g_quark_from_string ("plant"); + class_quarks[1] = g_quark_from_string ("animal"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[1]); + fail_unless (ret == TRUE); + + conf_lvl[0] = 0.6f; + conf_lvl[1] = 0.4f; + class_quarks[0] = g_quark_from_string ("male"); + class_quarks[1] = g_quark_from_string ("female"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[2]); + fail_unless (ret == TRUE); + + // Pet is part of kingdom + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd[0].id, cls_mtd[1].id); + + // Kingdom contain pet + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_CONTAIN, cls_mtd[1].id, cls_mtd[0].id); + + // Pet contain gender + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_CONTAIN, cls_mtd[0].id, cls_mtd[2].id); + + /* Query if pet relate kingdom through a IS_PART relation with a maximum + * relation span of 1. Max relation span of 1 mean they directly related.*/ + gboolean exist = gst_analytics_relation_meta_exist (relations, cls_mtd[0].id, + cls_mtd[1].id, 1, GST_ANALYTICS_REL_TYPE_IS_PART_OF, NULL); + fail_unless (exist == TRUE); + + /* Query if pet relate to gender through a IS_PART relation. */ + exist = gst_analytics_relation_meta_exist (relations, cls_mtd[0].id, + cls_mtd[2].id, 1, GST_ANALYTICS_REL_TYPE_IS_PART_OF, NULL); + fail_unless (exist == FALSE); + + /* Query if pet relate to kingdom through a CONTAIN relation. */ + exist = gst_analytics_relation_meta_exist (relations, cls_mtd[0].id, + cls_mtd[1].id, 1, GST_ANALYTICS_REL_TYPE_CONTAIN, NULL); + fail_unless (exist == FALSE); + + GstAnalyticsRelTypes cond = + GST_ANALYTICS_REL_TYPE_IS_PART_OF | GST_ANALYTICS_REL_TYPE_CONTAIN | + GST_ANALYTICS_REL_TYPE_RELATE_TO; + + /* Query if pet relate to gender through IS_PART or CONTAIN or + * RELATE_TO relation. */ + exist = gst_analytics_relation_meta_exist (relations, cls_mtd[0].id, + cls_mtd[2].id, 1, cond, NULL); + fail_unless (exist == TRUE); + + /* Query if pet relate to kindom through CONTAIN or RELATE_TO relation */ + cond = GST_ANALYTICS_REL_TYPE_CONTAIN | GST_ANALYTICS_REL_TYPE_RELATE_TO; + exist = gst_analytics_relation_meta_exist (relations, cls_mtd[0].id, + cls_mtd[1].id, 1, cond, NULL); + fail_unless (exist == FALSE); + + /* Query if kingdom relate to gender through a CONTAIN relation with a maximum + * relation span of 1. */ + exist = gst_analytics_relation_meta_exist (relations, cls_mtd[1].id, + cls_mtd[2].id, 1, GST_ANALYTICS_REL_TYPE_CONTAIN, NULL); + + /* We expect this to fail because kingdom relate to gender CONTAIN relations + * but indirectly (via pet) and we set the max relation span to 1*/ + fail_unless (exist == FALSE); + + /* Same has previous check but using INFINIT relation span */ + exist = gst_analytics_relation_meta_exist (relations, cls_mtd[1].id, + cls_mtd[2].id, GST_INF_RELATION_SPAN, GST_ANALYTICS_REL_TYPE_CONTAIN, + NULL); + fail_unless (exist == TRUE); + + exist = gst_analytics_relation_meta_exist (relations, cls_mtd[2].id, + cls_mtd[1].id, GST_INF_RELATION_SPAN, GST_ANALYTICS_REL_TYPE_CONTAIN, + NULL); + fail_unless (exist == FALSE); + + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_path_relation_meta) +{ + /* Verify we can retrieve relation path */ + GstBuffer *buf; + GstAnalyticsClsMtd cls_mtd[3]; + GstAnalyticsRelationMetaInitParams init_params = { 2, 150 }; + GstAnalyticsRelationMeta *relations; + gboolean ret; + GQuark class_quarks[2]; + + buf = gst_buffer_new (); + relations = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + gfloat conf_lvl[] = { 0.6f, 0.4f }; + class_quarks[0] = g_quark_from_string ("dog"); + class_quarks[1] = g_quark_from_string ("cat"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[0]); + fail_unless (ret == TRUE); + fail_unless (cls_mtd[0].id == 0); + + conf_lvl[0] = 0.6f; + conf_lvl[1] = 0.4f; + class_quarks[0] = g_quark_from_string ("plant"); + class_quarks[1] = g_quark_from_string ("animal"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[1]); + fail_unless (ret == TRUE); + fail_unless (cls_mtd[1].id == 1); + + conf_lvl[0] = 0.6f; + conf_lvl[1] = 0.4f; + class_quarks[0] = g_quark_from_string ("male"); + class_quarks[1] = g_quark_from_string ("female"); + + ret = gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[2]); + fail_unless (ret == TRUE); + fail_unless (cls_mtd[2].id == 2); + + // Pet is part of kingdom + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd[0].id, cls_mtd[1].id); + + // Kingdom contain pet + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_CONTAIN, cls_mtd[1].id, cls_mtd[0].id); + + // Pet contain gender + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_CONTAIN, cls_mtd[0].id, cls_mtd[2].id); + + GArray *path = NULL; + GstAnalyticsRelTypes cond = GST_ANALYTICS_REL_TYPE_CONTAIN; + gboolean exist = gst_analytics_relation_meta_exist (relations, cls_mtd[0].id, + cls_mtd[2].id, GST_INF_RELATION_SPAN, cond, &path); + if (exist) { + fail_unless (path != NULL); + gint i; + gint path_check_ids[] = { 0, 2 }; + fail_unless (path->len == 2); + for (i = 0; i < path->len; i++) { + fail_unless (path_check_ids[i] == g_array_index (path, gint, i)); + } + g_array_free (g_steal_pointer (&path), TRUE); + fail_unless (i == 2); + } + + cond = GST_ANALYTICS_REL_TYPE_CONTAIN; + exist = gst_analytics_relation_meta_exist (relations, cls_mtd[1].id, + cls_mtd[2].id, GST_INF_RELATION_SPAN, cond, &path); + if (exist) { + gint i; + gint path_check_ids[] = { 1, 0, 2 }; + fail_unless (path != NULL); + fail_unless (path->len == 3); + for (i = 0; i < path->len; i++) { + fail_unless (path_check_ids[i] == g_array_index (path, gint, i)); + } + g_array_free (g_steal_pointer (&path), TRUE); + fail_unless (i == 3); + } + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_cyclic_relation_meta) +{ + /* Verify we can discover cycle in relation and not report multiple time + * the same node and get into an infinit exploration */ + + GstBuffer *buf; + GstAnalyticsClsMtd cls_mtd[3]; + GstAnalyticsRelationMetaInitParams init_params = { 2, 150 }; + GstAnalyticsRelationMeta *relations; + gfloat conf_lvl[2]; + GQuark class_quarks[2]; + + class_quarks[0] = g_quark_from_string ("attr1"); + class_quarks[1] = g_quark_from_string ("attr2"); + + buf = gst_buffer_new (); + relations = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + conf_lvl[0] = 0.5f; + conf_lvl[1] = 0.5f; + gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[0]); + + gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[1]); + + gst_analytics_relation_meta_add_cls_mtd (relations, 2, conf_lvl, + class_quarks, &cls_mtd[2]); + + // (0) -> (1) + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd[0].id, cls_mtd[1].id); + + // (1)->(2) + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd[1].id, cls_mtd[2].id); + + // (2) -> (0) + gst_analytics_relation_meta_set_relation (relations, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd[2].id, cls_mtd[0].id); + + GArray *path = NULL; + GstAnalyticsRelTypes cond = GST_ANALYTICS_REL_TYPE_CONTAIN; + gboolean exist = gst_analytics_relation_meta_exist (relations, cls_mtd[0].id, + cls_mtd[2].id, GST_INF_RELATION_SPAN, cond, &path); + fail_unless (exist == FALSE); + + cond = GST_ANALYTICS_REL_TYPE_IS_PART_OF; + exist = + gst_analytics_relation_meta_exist (relations, cls_mtd[0].id, + cls_mtd[2].id, GST_INF_RELATION_SPAN, cond, &path); + fail_unless (exist == TRUE); + if (exist) { + gint i; + gint path_ids[] = { 0, 1, 2 }; + fail_unless (path->len == 3); + for (i = 0; i < path->len; i++) { + fail_unless (path_ids[i] == g_array_index (path, gint, i)); + } + g_array_free (g_steal_pointer (&path), TRUE); + fail_unless (i == 3); + } + + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_add_od_meta) +{ + /* Verity we can add Object Detection relatable metadata to a relation + * metadata */ + GstBuffer *buf; + GstAnalyticsRelationMetaInitParams init_params = { 5, 150 }; + GstAnalyticsRelationMeta *rmeta; + GstAnalyticsODMtd od_mtd; + gboolean ret; + buf = gst_buffer_new (); + + rmeta = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + GQuark type = g_quark_from_string ("dog"); + gint x = 20; + gint y = 20; + gint w = 10; + gint h = 15; + gfloat loc_conf_lvl = 0.6f; + ret = gst_analytics_relation_meta_add_od_mtd (rmeta, type, x, y, + w, h, loc_conf_lvl, &od_mtd); + fail_unless (ret == TRUE); + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_od_meta_fields) +{ + /* Verify we can readback fields of object detection metadata */ + GstBuffer *buf; + GstAnalyticsRelationMetaInitParams init_params = { 5, 150 }; + GstAnalyticsRelationMeta *rmeta; + GstAnalyticsODMtd od_mtd; + gboolean ret; + buf = gst_buffer_new (); + + rmeta = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + GQuark type = g_quark_from_string ("dog"); + gint x = 21; + gint y = 20; + gint w = 10; + gint h = 15; + gfloat loc_conf_lvl = 0.6f; + ret = gst_analytics_relation_meta_add_od_mtd (rmeta, type, x, y, + w, h, loc_conf_lvl, &od_mtd); + + fail_unless (ret == TRUE); + + gint _x, _y, _w, _h; + gfloat _loc_conf_lvl; + gst_analytics_od_mtd_get_location (&od_mtd, &_x, &_y, &_w, &_h, + &_loc_conf_lvl); + + fail_unless (_x == x); + fail_unless (_y == y); + fail_unless (_w == w); + fail_unless (_h == h); + fail_unless (_loc_conf_lvl == loc_conf_lvl); + + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_od_cls_relation) +{ + /* Verify we can add a object detection and classification metadata to + * a relation metadata */ + + GstBuffer *buf; + GstAnalyticsClsMtd cls_mtd; + GstAnalyticsODMtd od_mtd; + /* We intentionally set buffer small than required to verify sanity + * with re-allocation */ + GstAnalyticsRelationMetaInitParams init_params = { 5, 150 }; + GstAnalyticsRelationMeta *rmeta; + gboolean ret; + GArray *path = NULL; + gboolean exist; + gint _x, _y, _w, _h; + gfloat _loc_conf_lvl; + GQuark class_quarks[2]; + + buf = gst_buffer_new (); + rmeta = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + gfloat conf_lvl[] = { 0.7f, 0.3f }; + class_quarks[0] = g_quark_from_string ("dog"); + class_quarks[1] = g_quark_from_string ("cat"); + + gst_analytics_relation_meta_add_cls_mtd (rmeta, 2, conf_lvl, + class_quarks, &cls_mtd); + + GQuark type = g_quark_from_string ("dog"); + gint x = 21; + gint y = 20; + gint w = 10; + gint h = 15; + gfloat loc_conf_lvl = 0.6f; + gst_analytics_relation_meta_add_od_mtd (rmeta, type, x, y, w, h, + loc_conf_lvl, &od_mtd); + + ret = gst_analytics_relation_meta_set_relation (rmeta, + GST_ANALYTICS_REL_TYPE_CONTAIN, od_mtd.id, cls_mtd.id); + fail_unless (ret == TRUE); + + ret = gst_analytics_relation_meta_set_relation (rmeta, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, cls_mtd.id, od_mtd.id); + fail_unless (ret == TRUE); + + /* Verify OD relate to CLS only through a CONTAIN relation */ + exist = gst_analytics_relation_meta_exist (rmeta, + od_mtd.id, cls_mtd.id, GST_INF_RELATION_SPAN, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, NULL); + fail_unless (exist == FALSE); + + exist = gst_analytics_relation_meta_exist (rmeta, + od_mtd.id, cls_mtd.id, GST_INF_RELATION_SPAN, + GST_ANALYTICS_REL_TYPE_CONTAIN, &path); + fail_unless (exist == TRUE); + + /* Query the relation path and verify it is correct */ + gint ids[2]; + gint i; + fail_unless (path->len == 2); + for (i = 0; i < path->len; i++) { + ids[i] = g_array_index (path, gint, i); + GST_LOG ("id=%u", ids[i]); + } + g_array_free (g_steal_pointer (&path), TRUE); + fail_unless (ids[0] == 1); + fail_unless (ids[1] == 0); + + GstAnalyticsMtd rlt_mtd; + exist = gst_analytics_relation_meta_get_mtd (rmeta, ids[0], 0, &rlt_mtd); + fail_unless (exist == TRUE); + + GQuark mtd_type = gst_analytics_mtd_get_type_quark (&rlt_mtd); + + /* Verify relatable meta with id == 1 is of type Object Detection */ + fail_unless (mtd_type == gst_analytics_od_mtd_get_type_quark ()); + + gst_analytics_od_mtd_get_location ((GstAnalyticsODMtd *) & rlt_mtd, &_x, &_y, + &_w, &_h, &_loc_conf_lvl); + fail_unless (_x == x); + fail_unless (_y == y); + fail_unless (_w == w); + fail_unless (_h == h); + fail_unless (_loc_conf_lvl == loc_conf_lvl); + + GST_LOG ("mtd_type:%s", g_quark_to_string (mtd_type)); + + exist = gst_analytics_relation_meta_get_mtd (rmeta, ids[1], 0, &rlt_mtd); + fail_unless (exist == TRUE); + mtd_type = gst_analytics_mtd_get_type_quark (&rlt_mtd); + + /* Verify relatable meta with id == 0 is of type classification */ + fail_unless (mtd_type == gst_analytics_cls_mtd_get_type_quark ()); + gint index = + gst_analytics_cls_mtd_get_index_by_quark ((GstAnalyticsClsMtd *) & + rlt_mtd, + g_quark_from_string ("dog")); + gfloat lvl = + gst_analytics_cls_mtd_get_level ((GstAnalyticsClsMtd *) & rlt_mtd, index); + GST_LOG ("dog %f [%d, %d %d, %d", lvl, _x, _y, _w, _h); + + fail_unless (lvl == 0.7f); + index = + gst_analytics_cls_mtd_get_index_by_quark ((GstAnalyticsClsMtd *) & + rlt_mtd, g_quark_from_string ("cat")); + lvl = + gst_analytics_cls_mtd_get_level ((GstAnalyticsClsMtd *) & rlt_mtd, index); + fail_unless (lvl == 0.3f); + + GST_LOG ("mtd_type:%s", g_quark_to_string (mtd_type)); + GST_LOG ("cat %f [%d, %d %d, %d", lvl, _x, _y, _w, _h); + + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_multi_od_cls_relation) +{ + GstBuffer *buf; + GstAnalyticsClsMtd cls_mtd[3]; + GstAnalyticsODMtd od_mtd[2]; + GstAnalyticsRelationMetaInitParams init_params = { 5, 150 }; + GstAnalyticsRelationMeta *rmeta; + gint cls_id, ids[2], i; + gboolean ret; + const gint dog_cls_index = 0; + const gint cat_cls_index = 1; + gfloat cls_conf_lvl[2], lvl; + GArray *path = NULL; + gfloat _loc_conf_lvl; + gint x, _x, y, _y, w, _w, h, _h; + GQuark mtd_type, cls_type; + GstAnalyticsMtd mtd; + gpointer state = NULL; + GQuark class_quarks[2]; + class_quarks[0] = g_quark_from_string ("dog"); + class_quarks[1] = g_quark_from_string ("cat"); + + + buf = gst_buffer_new (); + rmeta = gst_buffer_add_analytics_relation_meta_full (buf, &init_params); + + /* Define first relation ObjectDetection -contain-> Classification */ + cls_conf_lvl[dog_cls_index] = 0.7f; + cls_conf_lvl[cat_cls_index] = 0.3f; + + gst_analytics_relation_meta_add_cls_mtd (rmeta, 2, cls_conf_lvl, + class_quarks, &cls_mtd[0]); + + cls_type = g_quark_from_string ("dog"); + x = 21; + y = 20; + w = 10; + h = 15; + gfloat loc_conf_lvl = 0.6f; + gst_analytics_relation_meta_add_od_mtd (rmeta, cls_type, x, y, w, + h, loc_conf_lvl, &od_mtd[0]); + + ret = gst_analytics_relation_meta_set_relation (rmeta, + GST_ANALYTICS_REL_TYPE_CONTAIN, od_mtd[0].id, cls_mtd[0].id); + fail_unless (ret == TRUE); + GST_LOG ("Set rel Obj:%d -c-> Cls:%d", od_mtd[0].id, cls_mtd[0].id); + + /* Define second relation ObjectDetection -contain-> Classification */ + cls_conf_lvl[dog_cls_index] = 0.1f; + cls_conf_lvl[cat_cls_index] = 0.9f; + gst_analytics_relation_meta_add_cls_mtd (rmeta, 2, cls_conf_lvl, + class_quarks, &cls_mtd[1]); + + cls_type = g_quark_from_string ("cat"); + x = 50; + y = 21; + w = 11; + h = 16; + loc_conf_lvl = 0.7f; + gst_analytics_relation_meta_add_od_mtd (rmeta, cls_type, x, y, w, + h, loc_conf_lvl, &od_mtd[1]); + + ret = gst_analytics_relation_meta_set_relation (rmeta, + GST_ANALYTICS_REL_TYPE_CONTAIN, od_mtd[1].id, cls_mtd[1].id); + + GST_LOG ("Set rel Obj:%d -c-> Cls:%d", od_mtd[1].id, cls_mtd[1].id); + fail_unless (ret == TRUE); + + /* Query relations */ + + /* Query relation between first object detection and first classification + * and verify they are only related by CONTAIN relation OD relate to + * CLASSIFICATION through a CONTAIN relation. */ + gboolean exist = + gst_analytics_relation_meta_exist (rmeta, od_mtd[0].id, cls_mtd[0].id, + GST_INF_RELATION_SPAN, + GST_ANALYTICS_REL_TYPE_IS_PART_OF, NULL); + fail_unless (exist == FALSE); + + exist = + gst_analytics_relation_meta_exist (rmeta, od_mtd[0].id, cls_mtd[0].id, + GST_INF_RELATION_SPAN, GST_ANALYTICS_REL_TYPE_CONTAIN, NULL); + fail_unless (exist == TRUE); + + + /* Query relation between second object detection and second classification + * and verify they are only related by CONTAIN relation OD relate to + * CLASSIFICATION through a CONTAIN relation. + */ + exist = + gst_analytics_relation_meta_exist (rmeta, od_mtd[1].id, cls_mtd[1].id, + GST_INF_RELATION_SPAN, GST_ANALYTICS_REL_TYPE_CONTAIN, &path); + fail_unless (exist == TRUE); + + /* Verify relation path between OD second (id=3) and Cls second (id=2) + * is correct + */ + fail_unless (path->len == 2); + for (i = 0; i < path->len; i++) { + ids[i] = g_array_index (path, gint, i); + GST_LOG ("id=%u", ids[i]); + } + g_array_free (g_steal_pointer (&path), TRUE); + fail_unless (ids[0] == 3); + fail_unless (ids[1] == 2); + + /* Verify the relatable metadata 3 is of correct type + * (ObjectDetection). Verify it describe the correct + * the correct data. + */ + gst_analytics_relation_meta_get_mtd (rmeta, ids[0], 0, &mtd); + mtd_type = gst_analytics_mtd_get_type_quark (&mtd); + fail_unless (mtd_type == gst_analytics_od_mtd_get_type_quark ()); + + gst_analytics_od_mtd_get_location ((GstAnalyticsODMtd *) & mtd, &_x, &_y, &_w, + &_h, &_loc_conf_lvl); + fail_unless (_x == 50); + fail_unless (_y == 21); + fail_unless (_w == 11); + fail_unless (_h == 16); + fail_unless (_loc_conf_lvl == 0.7f); + + GST_LOG ("mtd_type:%s", g_quark_to_string (mtd_type)); + + /* Verify the relatable metadata 2 is of correct type + * (ObjectDetection). + */ + gst_analytics_relation_meta_get_mtd (rmeta, ids[1], 0, &mtd); + mtd_type = gst_analytics_mtd_get_type_quark (&mtd); + fail_unless (mtd_type == gst_analytics_cls_mtd_get_type_quark ()); + + /* Verify data of the CLASSIFICATION retrieved */ + gint index = + gst_analytics_cls_mtd_get_index_by_quark ((GstAnalyticsClsMtd *) & mtd, + g_quark_from_string ("dog")); + lvl = gst_analytics_cls_mtd_get_level ((GstAnalyticsClsMtd *) & mtd, index); + GST_LOG ("dog %f [%d, %d %d, %d", lvl, _x, _y, _w, _h); + fail_unless (lvl == 0.1f); + + /* Verify data of the CLASSIFICATION retrieved */ + index = + gst_analytics_cls_mtd_get_index_by_quark ((GstAnalyticsClsMtd *) & mtd, + g_quark_from_string ("cat")); + lvl = gst_analytics_cls_mtd_get_level ((GstAnalyticsClsMtd *) & mtd, index); + GST_LOG ("mtd_type:%s", g_quark_to_string (mtd_type)); + GST_LOG ("cat %f [%d, %d %d, %d", lvl, _x, _y, _w, _h); + fail_unless (lvl == 0.9f); + + /* Retrieve relatable metadata related to the first object detection + * through a CONTAIN relation of type CLASSIFICATION + * Verify it's the first classification metadata + */ + gst_analytics_relation_meta_get_direct_related (rmeta, od_mtd[0].id, + GST_ANALYTICS_REL_TYPE_CONTAIN, gst_analytics_cls_mtd_get_type_quark (), + &state, &mtd); + + cls_id = gst_analytics_mtd_get_id (&mtd); + GST_LOG ("Obj:%d -> Cls:%d", od_mtd[0].id, cls_id); + fail_unless (cls_id == cls_mtd[0].id); + + state = NULL; + /* Retrieve relatable metadata related to the second object detection + * through a CONTAIN relation of type CLASSIFICATION + * Verify it's the first classification metadata + */ + gst_analytics_relation_meta_get_direct_related (rmeta, od_mtd[1].id, + GST_ANALYTICS_REL_TYPE_CONTAIN, gst_analytics_cls_mtd_get_type_quark (), + &state, &mtd); + cls_id = gst_analytics_mtd_get_id (&mtd); + + GST_LOG ("Obj:%d -> Cls:%d", od_mtd[1].id, cls_id); + fail_unless (cls_id == cls_mtd[1].id); + + cls_conf_lvl[dog_cls_index] = 0.2f; + cls_conf_lvl[cat_cls_index] = 0.8f; + class_quarks[0] = g_quark_from_string ("canine"); + class_quarks[1] = g_quark_from_string ("feline"); + gst_analytics_relation_meta_add_cls_mtd (rmeta, 2, cls_conf_lvl, + class_quarks, &cls_mtd[2]); + + ret = gst_analytics_relation_meta_set_relation (rmeta, + GST_ANALYTICS_REL_TYPE_CONTAIN, od_mtd[1].id, cls_mtd[2].id); + + state = NULL; + ret = gst_analytics_relation_meta_get_direct_related (rmeta, od_mtd[1].id, + GST_ANALYTICS_REL_TYPE_CONTAIN, gst_analytics_od_mtd_get_type_quark (), + &state, &mtd); + + fail_unless (ret == FALSE); + + gst_buffer_unref (buf); +} + +GST_END_TEST; + +GST_START_TEST (test_add_tracking_meta) +{ + /* Verify we can add tracking relatable meta to relation meta */ + GstBuffer *buf1, *buf2; + GstAnalyticsRelationMetaInitParams init_params = { 5, 150 }; + GstAnalyticsRelationMeta *rmeta; + GstAnalyticsTrackingMtd tracking_mtd; + guint tracking_id; + GstClockTime tracking_observation_time_1; + gboolean ret; + + /* Verify we can add multiple trackings to relation metadata + */ + + buf1 = gst_buffer_new (); + rmeta = gst_buffer_add_analytics_relation_meta_full (buf1, &init_params); + tracking_id = 1; + tracking_observation_time_1 = GST_BUFFER_TIMESTAMP (buf1); + ret = gst_analytics_relation_meta_add_tracking_mtd (rmeta, tracking_id, + tracking_observation_time_1, &tracking_mtd); + fail_unless (ret == TRUE); + + gst_buffer_unref (buf1); + + buf2 = gst_buffer_new (); + rmeta = gst_buffer_add_analytics_relation_meta_full (buf2, &init_params); + tracking_id = 1; + ret = gst_analytics_relation_meta_add_tracking_mtd (rmeta, tracking_id, + tracking_observation_time_1, &tracking_mtd); + fail_unless (ret == TRUE); + + gst_buffer_unref (buf2); +} + +GST_END_TEST; + +static Suite * +analyticmeta_suite (void) +{ + + Suite *s; + TCase *tc_chain_cls; + TCase *tc_chain_relation; + TCase *tc_chain_od; + TCase *tc_chain_od_cls; + TCase *tc_chain_tracking; + + s = suite_create ("Analytic Meta Library"); + + tc_chain_cls = tcase_create ("Classification Mtd"); + suite_add_tcase (s, tc_chain_cls); + tcase_add_test (tc_chain_cls, test_add_classification_meta); + tcase_add_test (tc_chain_cls, test_classification_meta_classes); + + tcase_add_test (tc_chain_cls, test_meta_pooled); + + tc_chain_relation = tcase_create ("Relation Meta"); + suite_add_tcase (s, tc_chain_relation); + tcase_add_test (tc_chain_relation, test_add_relation_meta); + tcase_add_test (tc_chain_relation, + test_add_relation_inefficiency_reporting_cases); + tcase_add_test (tc_chain_relation, test_query_relation_meta_cases); + tcase_add_test (tc_chain_relation, test_path_relation_meta); + tcase_add_test (tc_chain_relation, test_cyclic_relation_meta); + + tc_chain_od = tcase_create ("Object Detection Mtd"); + suite_add_tcase (s, tc_chain_od); + tcase_add_test (tc_chain_od, test_add_od_meta); + tcase_add_test (tc_chain_od, test_od_meta_fields); + + tc_chain_od_cls = tcase_create ("Object Detection <-> Classification Mtd"); + suite_add_tcase (s, tc_chain_od_cls); + tcase_add_test (tc_chain_od_cls, test_od_cls_relation); + tcase_add_test (tc_chain_od_cls, test_multi_od_cls_relation); + + tc_chain_tracking = tcase_create ("Tracking Mtd"); + suite_add_tcase (s, tc_chain_tracking); + tcase_add_test (tc_chain_tracking, test_add_tracking_meta); + return s; +} + +GST_CHECK_MAIN (analyticmeta); diff --git a/subprojects/gst-plugins-bad/tests/check/meson.build b/subprojects/gst-plugins-bad/tests/check/meson.build index e507105430..de602acaba 100644 --- a/subprojects/gst-plugins-bad/tests/check/meson.build +++ b/subprojects/gst-plugins-bad/tests/check/meson.build @@ -96,6 +96,7 @@ base_tests = [ [['libs/h265bitwriter.c'], false, [gstcodecparsers_dep]], [['libs/vkformat.c'], not gstvulkan_dep.found(), [gstvulkan_dep]], [['libs/vkmemory.c'], not gstvulkan_dep.found(), [gstvulkan_dep]], + [['libs/analyticsmeta.c'], false, [gstanalytics_dep]], [['elements/vkcolorconvert.c'], not gstvulkan_dep.found(), [gstvulkan_dep]], [['libs/vkwindow.c'], not gstvulkan_dep.found(), [gstvulkan_dep]], [['libs/vkdevice.c'], not gstvulkan_dep.found(), [gstvulkan_dep]],