mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-23 02:01:12 +00:00
Code Issues 1 Projects Releases Wiki Activity

onnx: Port SSD detector to C

Browse source 
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6001>
This commit is contained in:
Olivier Crête 2024-01-25 00:02:30 -05:00
parent 5e1291fd86
commit 3325a10f57
8 changed files with 262 additions and 378 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

228
subprojects/gst-plugins-bad/ext/onnx/decoders/gstobjectdetectorutils.cpp
View file

@ -1,228 +0,0 @@
/*
* GStreamer gstreamer-objectdetectorutils
* Copyright (C) 2023 Collabora Ltd
*
* gstobjectdetectorutils.cpp
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include "gstobjectdetectorutils.h"
#include <gio/gio.h>
char **
read_labels (const char * labels_file)
{
GPtrArray *array;
GFile *file = g_file_new_for_path (labels_file);
GFileInputStream *file_stream;
GDataInputStream *data_stream;
GError *error = NULL;
gchar *line;
file_stream = g_file_read (file, NULL, &error);
g_object_unref (file);
if (!file_stream) {
GST_WARNING ("Could not open file %s: %s\n", labels_file,
error->message);
g_clear_error (&error);
return NULL;
}
data_stream = g_data_input_stream_new (G_INPUT_STREAM (file_stream));
g_object_unref (file_stream);
array = g_ptr_array_new();
while ((line = g_data_input_stream_read_line (data_stream, NULL, NULL,
&error)))
g_ptr_array_add (array, line);
g_object_unref (data_stream);
if (error) {
GST_WARNING ("Could not open file %s: %s", labels_file, error->message);
g_ptr_array_free (array, TRUE);
g_clear_error (&error);
return NULL;
}
if (array->len == 0) {
g_ptr_array_free (array, TRUE);
return NULL;
}
g_ptr_array_add (array, NULL);
return (char **) g_ptr_array_free (array, FALSE);
}
GstMlBoundingBox::GstMlBoundingBox (std::string lbl, float score, float _x0,
float _y0, float _width, float _height):
label (lbl),
score (score),
x0 (_x0),
y0 (_y0),
width (_width),
height (_height)
{
}
GstMlBoundingBox::GstMlBoundingBox ():
GstMlBoundingBox ("", 0.0f, 0.0f, 0.0f, 0.0f, 0.0f)
{
}
namespace GstObjectDetectorUtils
{
GstObjectDetectorUtils::GstObjectDetectorUtils ()
{
}
std::vector < GstMlBoundingBox > GstObjectDetectorUtils::run (int32_t w,
int32_t h, GstTensorMeta * tmeta, gchar **labels,
float scoreThreshold)
{
auto classIndex = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES));
if (classIndex == GST_TENSOR_MISSING_ID) {
GST_ERROR ("Missing class tensor id");
return std::vector < GstMlBoundingBox > ();
}
auto type = tmeta->tensor[classIndex].type;
return (type == GST_TENSOR_TYPE_FLOAT32) ?
doRun < float >(w, h, tmeta, labels, scoreThreshold)
: doRun < int >(w, h, tmeta, labels, scoreThreshold);
}
template < typename T > std::vector < GstMlBoundingBox >
GstObjectDetectorUtils::doRun (int32_t w, int32_t h,
GstTensorMeta * tmeta, char **labels, float scoreThreshold)
{
std::vector < GstMlBoundingBox > boundingBoxes;
GstMapInfo map_info[GstObjectDetectorMaxNodes];
GstMemory *memory[GstObjectDetectorMaxNodes] = { NULL };
gint index;
T *numDetections = nullptr, *bboxes = nullptr, *scores =
nullptr, *labelIndex = nullptr;
// number of detections
index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS));
if (index == GST_TENSOR_MISSING_ID) {
GST_WARNING ("Missing tensor data for tensor index %d", index);
goto cleanup;
}
memory[index] = gst_buffer_peek_memory (tmeta->tensor[index].data, 0);
if (!memory[index]) {
GST_WARNING ("Missing tensor data for tensor index %d", index);
goto cleanup;
}
if (!gst_memory_map (memory[index], map_info + index, GST_MAP_READ)) {
GST_WARNING ("Failed to map tensor memory for index %d", index);
goto cleanup;
}
numDetections = (T *) map_info[index].data;
// bounding boxes
index =
gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string(GST_MODEL_OBJECT_DETECTOR_BOXES));
if (index == GST_TENSOR_MISSING_ID) {
GST_WARNING ("Missing tensor data for tensor index %d", index);
goto cleanup;
}
memory[index] = gst_buffer_peek_memory (tmeta->tensor[index].data, 0);
if (!memory[index]) {
GST_WARNING ("Failed to map tensor memory for index %d", index);
goto cleanup;
}
if (!gst_memory_map (memory[index], map_info + index, GST_MAP_READ)) {
GST_ERROR ("Failed to map GstMemory");
goto cleanup;
}
bboxes = (T *) map_info[index].data;
// scores
index =
gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_SCORES));
if (index == GST_TENSOR_MISSING_ID) {
GST_ERROR ("Missing scores tensor id");
goto cleanup;
}
memory[index] = gst_buffer_peek_memory (tmeta->tensor[index].data, 0);
if (!memory[index]) {
GST_WARNING ("Missing tensor data for tensor index %d", index);
goto cleanup;
}
if (!gst_memory_map (memory[index], map_info + index, GST_MAP_READ)) {
GST_ERROR ("Failed to map GstMemory");
goto cleanup;
}
scores = (T *) map_info[index].data;
// optional label
labelIndex = nullptr;
index =
gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES));
if (index != GST_TENSOR_MISSING_ID) {
memory[index] = gst_buffer_peek_memory (tmeta->tensor[index].data, 0);
if (!memory[index]) {
GST_WARNING ("Missing tensor data for tensor index %d", index);
goto cleanup;
}
if (!gst_memory_map (memory[index], map_info + index, GST_MAP_READ)) {
GST_ERROR ("Failed to map GstMemory");
goto cleanup;
}
labelIndex = (T *) map_info[index].data;
}
for (int i = 0; i < numDetections[0]; ++i) {
if (scores[i] > scoreThreshold) {
std::string label = "";
if (labels && labelIndex)
label = labels[(int)labelIndex[i] - 1];
auto score = scores[i];
auto y0 = bboxes[i * 4] * h;
auto x0 = bboxes[i * 4 + 1] * w;
auto bheight = bboxes[i * 4 + 2] * h - y0;
auto bwidth = bboxes[i * 4 + 3] * w - x0;
boundingBoxes.push_back (GstMlBoundingBox (label, score, x0, y0, bwidth,
bheight));
}
}
cleanup:
for (int i = 0; i < GstObjectDetectorMaxNodes; ++i) {
if (memory[i])
gst_memory_unmap (memory[i], map_info + i);
}
return boundingBoxes;
}
}

83
subprojects/gst-plugins-bad/ext/onnx/decoders/gstobjectdetectorutils.h
View file

@ -1,83 +0,0 @@
/*
* GStreamer gstreamer-objectdetectorutils
* Copyright (C) 2023 Collabora Ltd
*
* gstobjectdetectorutils.h
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_OBJECT_DETECTOR_UTILS_H__
#define __GST_OBJECT_DETECTOR_UTILS_H__
#include <gst/gst.h>
#include <string>
#include <vector>
#include "gstml.h"
#include "tensor/gsttensormeta.h"
char ** read_labels (const char * labels_file);
/* Object detection tensor id strings */
#define GST_MODEL_OBJECT_DETECTOR_BOXES "Gst.Model.ObjectDetector.Boxes"
#define GST_MODEL_OBJECT_DETECTOR_SCORES "Gst.Model.ObjectDetector.Scores"
#define GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS "Gst.Model.ObjectDetector.NumDetections"
#define GST_MODEL_OBJECT_DETECTOR_CLASSES "Gst.Model.ObjectDetector.Classes"
/**
* GstMlBoundingBox:
*
* @label label
* @score detection confidence
* @x0 top left hand x coordinate
* @y0 top left hand y coordinate
* @width width
* @height height
*
* Since: 1.20
*/
struct GstMlBoundingBox {
GstMlBoundingBox(std::string lbl, float score, float _x0, float _y0,
float _width, float _height);
GstMlBoundingBox();
std::string label;
float score;
float x0;
float y0;
float width;
float height;
};
namespace GstObjectDetectorUtils {
const int GstObjectDetectorMaxNodes = 4;
class GstObjectDetectorUtils {
public:
GstObjectDetectorUtils(void);
~GstObjectDetectorUtils(void) = default;
std::vector < GstMlBoundingBox > run(int32_t w, int32_t h,
GstTensorMeta *tmeta,
char **labels,
float scoreThreshold);
private:
template < typename T > std::vector < GstMlBoundingBox >
doRun(int32_t w, int32_t h,
GstTensorMeta *tmeta, char **labels,
float scoreThreshold);
};
}
#endif /* __GST_OBJECT_DETECTOR_UTILS_H__ */

286
subprojects/gst-plugins-bad/ext/onnx/decoders/gstssdobjectdetector.cpp → subprojects/gst-plugins-bad/ext/onnx/decoders/gstssdobjectdetector.c
View file

@ -44,19 +44,23 @@
#include "config.h"
#endif
#include "gstssdobjectdetector.h"
#include "gstobjectdetectorutils.h"
#include <gio/gio.h>
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/analytics/analytics.h>
#include "tensor/gsttensormeta.h"
/* Object detection tensor id strings */
#define GST_MODEL_OBJECT_DETECTOR_BOXES "Gst.Model.ObjectDetector.Boxes"
#define GST_MODEL_OBJECT_DETECTOR_SCORES "Gst.Model.ObjectDetector.Scores"
#define GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS "Gst.Model.ObjectDetector.NumDetections"
#define GST_MODEL_OBJECT_DETECTOR_CLASSES "Gst.Model.ObjectDetector.Classes"
GST_DEBUG_CATEGORY_STATIC (ssd_object_detector_debug);
#define GST_CAT_DEFAULT ssd_object_detector_debug
#define GST_ODUTILS_MEMBER( self ) ((GstObjectDetectorUtils::GstObjectDetectorUtils *) (self->odutils))
GST_ELEMENT_REGISTER_DEFINE (ssd_object_detector, "ssdobjectdetector",
GST_RANK_PRIMARY, GST_TYPE_SSD_OBJECT_DETECTOR);
@ -68,7 +72,7 @@ enum
PROP_SCORE_THRESHOLD,
};
#define GST_SSD_OBJECT_DETECTOR_DEFAULT_SCORE_THRESHOLD 0.3f /* 0 to 1 */
#define GST_SSD_OBJECT_DETECTOR_DEFAULT_SCORE_THRESHOLD 0.3f /* 0 to 1 */
static GstStaticPadTemplate gst_ssd_object_detector_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
@ -97,7 +101,8 @@ static gboolean
gst_ssd_object_detector_set_caps (GstBaseTransform * trans, GstCaps * incaps,
GstCaps * outcaps);
G_DEFINE_TYPE (GstSsdObjectDetector, gst_ssd_object_detector, GST_TYPE_BASE_TRANSFORM);
G_DEFINE_TYPE (GstSsdObjectDetector, gst_ssd_object_detector,
GST_TYPE_BASE_TRANSFORM);
static void
gst_ssd_object_detector_class_init (GstSsdObjectDetectorClass * klass)
@ -135,7 +140,8 @@ gst_ssd_object_detector_class_init (GstSsdObjectDetectorClass * klass)
g_param_spec_float ("score-threshold",
"Score threshold",
"Threshold for deciding when to remove boxes based on score",
0.0, 1.0, GST_SSD_OBJECT_DETECTOR_DEFAULT_SCORE_THRESHOLD, (GParamFlags)
0.0, 1.0, GST_SSD_OBJECT_DETECTOR_DEFAULT_SCORE_THRESHOLD,
(GParamFlags)
(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
gst_element_class_set_static_metadata (element_class, "objectdetector",
@ -155,7 +161,6 @@ gst_ssd_object_detector_class_init (GstSsdObjectDetectorClass * klass)
static void
gst_ssd_object_detector_init (GstSsdObjectDetector * self)
{
self->odutils = new GstObjectDetectorUtils::GstObjectDetectorUtils ();
}
static void
@ -164,12 +169,58 @@ gst_ssd_object_detector_finalize (GObject * object)
GstSsdObjectDetector *self = GST_SSD_OBJECT_DETECTOR (object);
g_free (self->label_file);
g_strfreev (self->labels);
delete GST_ODUTILS_MEMBER (self);
g_clear_pointer (&self->labels, g_array_unref);
G_OBJECT_CLASS (gst_ssd_object_detector_parent_class)->finalize (object);
}
static GArray *
read_labels (const char *labels_file)
{
GArray *array;
GFile *file = g_file_new_for_path (labels_file);
GFileInputStream *file_stream;
GDataInputStream *data_stream;
GError *error = NULL;
gchar *line;
file_stream = g_file_read (file, NULL, &error);
g_object_unref (file);
if (!file_stream) {
GST_WARNING ("Could not open file %s: %s\n", labels_file, error->message);
g_clear_error (&error);
return NULL;
}
data_stream = g_data_input_stream_new (G_INPUT_STREAM (file_stream));
g_object_unref (file_stream);
array = g_array_new (FALSE, FALSE, sizeof (GQuark));
while ((line = g_data_input_stream_read_line (data_stream, NULL, NULL,
&error))) {
GQuark label = g_quark_from_string (line);
g_array_append_val (array, label);
g_free (line);
}
g_object_unref (data_stream);
if (error) {
GST_WARNING ("Could not open file %s: %s", labels_file, error->message);
g_array_free (array, TRUE);
g_clear_error (&error);
return NULL;
}
if (array->len == 0) {
g_array_free (array, TRUE);
return NULL;
}
return array;
}
static void
gst_ssd_object_detector_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
@ -179,21 +230,21 @@ gst_ssd_object_detector_set_property (GObject * object, guint prop_id,
switch (prop_id) {
case PROP_LABEL_FILE:
{
gchar **labels;
{
GArray *labels;
filename = g_value_get_string (value);
labels = read_labels (filename);
filename = g_value_get_string (value);
labels = read_labels (filename);
if (labels) {
g_free (self->label_file);
self->label_file = g_strdup (filename);
g_strfreev (self->labels);
self->labels = labels;
} else {
GST_WARNING_OBJECT (self, "Label file '%s' not found!", filename);
}
if (labels) {
g_free (self->label_file);
self->label_file = g_strdup (filename);
g_clear_pointer (&self->labels, g_array_unref);
self->labels = labels;
} else {
GST_WARNING_OBJECT (self, "Label file '%s' not found!", filename);
}
}
break;
case PROP_SCORE_THRESHOLD:
GST_OBJECT_LOCK (self);
@ -259,7 +310,8 @@ gst_ssd_object_detector_get_tensor_meta (GstSsdObjectDetector * object_detector,
gint clasesIndex = gst_tensor_meta_get_index_from_id (tensor_meta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES));
if (boxesIndex == GST_TENSOR_MISSING_ID || scoresIndex == GST_TENSOR_MISSING_ID
if (boxesIndex == GST_TENSOR_MISSING_ID
|| scoresIndex == GST_TENSOR_MISSING_ID
|| numDetectionsIndex == GST_TENSOR_MISSING_ID)
continue;
@ -300,13 +352,175 @@ gst_ssd_object_detector_transform_ip (GstBaseTransform * trans, GstBuffer * buf)
return GST_FLOW_OK;
}
#define DEFINE_GET_FUNC(TYPE, MAX) \
static gboolean \
get_ ## TYPE ## _at_index (GstTensor *tensor, GstMapInfo *map, \
guint index, TYPE * out) \
{ \
switch (tensor->type) { \
case GST_TENSOR_TYPE_FLOAT32: { \
float *f = (float *) map->data; \
if (sizeof(*f) * (index + 1) > map->size) \
return FALSE; \
*out = f[index]; \
break; \
} \
case GST_TENSOR_TYPE_UINT32: { \
guint32 *u = (guint32 *) map->data; \
if (sizeof(*u) * (index + 1) > map->size) \
return FALSE; \
*out = u[index]; \
break; \
} \
default: \
GST_ERROR ("Only float32 and int32 tensors are understood"); \
return FALSE; \
} \
return TRUE; \
}
DEFINE_GET_FUNC (guint32, UINT32_MAX)
DEFINE_GET_FUNC (float, FLOAT_MAX)
#undef DEFINE_GET_FUNC
static void
extract_bounding_boxes (GstSsdObjectDetector * self, gsize w, gsize h,
GstAnalyticsRelationMeta * rmeta, GstTensorMeta * tmeta)
{
gint classes_index;
gint boxes_index;
gint scores_index;
gint numdetect_index;
GstMapInfo boxes_map = GST_MAP_INFO_INIT;
GstMapInfo numdetect_map = GST_MAP_INFO_INIT;
GstMapInfo scores_map = GST_MAP_INFO_INIT;
GstMapInfo classes_map = GST_MAP_INFO_INIT;
guint num_detections = 0;
classes_index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES));
numdetect_index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS));
scores_index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_SCORES));
boxes_index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_BOXES));
if (numdetect_index == GST_TENSOR_MISSING_ID
|| scores_index == GST_TENSOR_MISSING_ID
|| numdetect_index == GST_TENSOR_MISSING_ID) {
GST_WARNING ("Missing tensor data expected for SSD model");
return;
}
if (!gst_buffer_map (tmeta->tensor[numdetect_index].data, &numdetect_map,
GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map tensor memory for index %d",
numdetect_index);
goto cleanup;
}
if (!gst_buffer_map (tmeta->tensor[boxes_index].data, &boxes_map,
GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map tensor memory for index %d",
boxes_index);
goto cleanup;
}
if (!gst_buffer_map (tmeta->tensor[scores_index].data, &scores_map,
GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map tensor memory for index %d",
scores_index);
goto cleanup;
}
if (classes_index != GST_TENSOR_MISSING_ID &&
!gst_buffer_map (tmeta->tensor[classes_index].data, &classes_map,
GST_MAP_READ)) {
GST_DEBUG_OBJECT (self, "Failed to map tensor memory for index %d",
classes_index);
}
if (!get_guint32_at_index (&tmeta->tensor[numdetect_index], &numdetect_map,
0, &num_detections)) {
GST_ERROR_OBJECT (self, "Failed to get the number of detections");
goto cleanup;
}
GST_LOG_OBJECT (self, "Model claims %d detections", num_detections);
for (int i = 0; i < num_detections; i++) {
float score;
float x, y, bwidth, bheight;
gint x_i, y_i, bwidth_i, bheight_i;
guint32 bclass;
GQuark label = 0;
GstAnalyticsODMtd odmtd;
if (!get_float_at_index (&tmeta->tensor[numdetect_index], &scores_map,
i, &score))
continue;
GST_LOG_OBJECT (self, "Detection %u score is %f", i, score);
if (score < self->score_threshold)
continue;
if (!get_float_at_index (&tmeta->tensor[boxes_index], &boxes_map,
i * 4, &y))
continue;
if (!get_float_at_index (&tmeta->tensor[boxes_index], &boxes_map,
i * 4 + 1, &x))
continue;
if (!get_float_at_index (&tmeta->tensor[boxes_index], &boxes_map,
i * 4 + 2, &bheight))
continue;
if (!get_float_at_index (&tmeta->tensor[boxes_index], &boxes_map,
i * 4 + 3, &bwidth))
continue;
if (self->labels && classes_map.memory &&
get_guint32_at_index (&tmeta->tensor[classes_index], &classes_map,
i, &bclass)) {
if (bclass < self->labels->len)
label = g_array_index (self->labels, GQuark, bclass);
}
x_i = x * w;
y_i = y * h;
bheight_i = (bheight * h) - y_i;
bwidth_i = (bwidth * w) - x_i;
if (gst_analytics_relation_meta_add_od_mtd (rmeta, label,
x_i, y_i, bwidth_i, bheight_i, score, &odmtd))
GST_DEBUG_OBJECT (self,
"Object detected with label : %s, score: %f, bound box: %dx%d at (%d,%d)",
g_quark_to_string (label), score, bwidth_i, bheight_i, x_i, y_i);
else
GST_WARNING_OBJECT (self, "Could not add detection to meta");
}
cleanup:
if (numdetect_map.memory)
gst_buffer_unmap (tmeta->tensor[numdetect_index].data, &numdetect_map);
if (classes_map.memory)
gst_buffer_unmap (tmeta->tensor[classes_index].data, &classes_map);
if (scores_map.memory)
gst_buffer_unmap (tmeta->tensor[scores_index].data, &scores_map);
if (boxes_map.memory)
gst_buffer_unmap (tmeta->tensor[boxes_index].data, &boxes_map);
}
static gboolean
gst_ssd_object_detector_process (GstBaseTransform * trans, GstBuffer * buf)
{
GstTensorMeta *tmeta = NULL;
GstAnalyticsODMtd odmtd;
GstAnalyticsRelationMeta *rmeta;
GstSsdObjectDetector *self = GST_SSD_OBJECT_DETECTOR (trans);
GstTensorMeta *tmeta;
GstAnalyticsRelationMeta *rmeta;
// get all tensor metas
tmeta = gst_ssd_object_detector_get_tensor_meta (self, buf);
@ -315,25 +529,11 @@ gst_ssd_object_detector_process (GstBaseTransform * trans, GstBuffer * buf)
return TRUE;
} else {
rmeta = gst_buffer_add_analytics_relation_meta (buf);
g_return_val_if_fail (rmeta != NULL, FALSE);
g_assert (rmeta);
}
std::vector < GstMlBoundingBox > boxes =
GST_ODUTILS_MEMBER (self)->run (self->video_info.width,
self->video_info.height, tmeta, self->labels,
self->score_threshold);
for (auto & b:boxes) {
if (gst_analytics_relation_meta_add_od_mtd (rmeta,
g_quark_from_string(b.label.c_str ()), b.x0, b.y0, b.width, b.height,
b.score, &odmtd)) {
GST_DEBUG_OBJECT (self,
"Object detected with label : %s, score: %f, bound box: (%f,%f,%f,%f)",
b.label.c_str (), b.score, b.x0, b.y0, b.x0 + b.width, b.y0 + b.height);
} else {
GST_ERROR_OBJECT (self, "Failed to add object detection analytics-meta");
}
}
extract_bounding_boxes (self, self->video_info.width,
self->video_info.height, rmeta, tmeta);
return TRUE;
}

10
subprojects/gst-plugins-bad/ext/onnx/decoders/gstssdobjectdetector.h
View file

@ -37,14 +37,11 @@ G_DECLARE_FINAL_TYPE (GstSsdObjectDetector, gst_ssd_object_detector, GST, SSD_OB
#define GST_SSD_OBJECT_DETECTOR_META_FIELD_LABEL "label"
#define GST_SSD_OBJECT_DETECTOR_META_FIELD_SCORE "score"
/**
/*
* GstSsdObjectDetector:
*
* @label_file label file
* @score_threshold score threshold
* @confidence_threshold confidence threshold
* @iou_threhsold iou threshold
* @od_ptr opaque pointer to GstOd object detection implementation
*
* Since: 1.20
*/
@ -52,11 +49,8 @@ struct _GstSsdObjectDetector
{
GstBaseTransform basetransform;
gchar *label_file;
gchar **labels;
GArray *labels;
gfloat score_threshold;
gfloat confidence_threshold;
gfloat iou_threshold;
gpointer odutils;
GstVideoInfo video_info;
};

4
subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.cpp
View file

@ -96,7 +96,7 @@ GstOnnxClient::GstOnnxClient (GstElement *debug_parent):debug_parent(debug_paren
return inputImageFormat;
}
void GstOnnxClient::setInputImageDatatype(GstTensorType datatype)
void GstOnnxClient::setInputImageDatatype(GstTensorDataType datatype)
{
inputDatatype = datatype;
switch (inputDatatype) {
@ -144,7 +144,7 @@ GstOnnxClient::GstOnnxClient (GstElement *debug_parent):debug_parent(debug_paren
return inputTensorScale;
}
GstTensorType GstOnnxClient::getInputImageDatatype(void)
GstTensorDataType GstOnnxClient::getInputImageDatatype(void)
{
return inputDatatype;
}

6
subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.h
View file

@ -56,7 +56,7 @@ namespace GstOnnxNamespace {
bool hasSession(void);
void setInputImageFormat(GstMlInputImageFormat format);
GstMlInputImageFormat getInputImageFormat(void);
GstTensorType getInputImageDatatype(void);
GstTensorDataType getInputImageDatatype(void);
void setInputImageOffset (float offset);
float getInputImageOffset ();
void setInputImageScale (float offset);
@ -73,7 +73,7 @@ namespace GstOnnxNamespace {
private:
GstElement *debug_parent;
void setInputImageDatatype (GstTensorType datatype);
void setInputImageDatatype (GstTensorDataType datatype);
template < typename T>
void convert_image_remove_alpha (T *dest, GstMlInputImageFormat hwc,
uint8_t **srcPtr, uint32_t srcSamplesPerPixel, uint32_t stride, T offset, T div);
@ -91,7 +91,7 @@ namespace GstOnnxNamespace {
std::vector < Ort::AllocatedStringPtr > outputNames;
std::vector < GQuark > outputIds;
GstMlInputImageFormat inputImageFormat;
GstTensorType inputDatatype;
GstTensorDataType inputDatatype;
size_t inputDatatypeSize;
bool fixedInputImageSize;
float inputTensorOffset;

3
subprojects/gst-plugins-bad/ext/onnx/meson.build
View file

@ -15,8 +15,7 @@ endif
if onnxrt_dep.found()
gstonnx = library('gstonnx',
'gstonnx.c',
'decoders/gstobjectdetectorutils.cpp',
'decoders/gstssdobjectdetector.cpp',
'decoders/gstssdobjectdetector.c',
'gstonnxinference.cpp',
'gstonnxclient.cpp',
'tensor/gsttensormeta.c',

20
subprojects/gst-plugins-bad/ext/onnx/tensor/gsttensormeta.h
View file

@ -25,7 +25,7 @@
#include <gst/gst.h>
/**
* GstTensorType:
* GstTensorDataType:
*
* @GST_TENSOR_TYPE_INT4 signed 4 bit integer tensor data
* @GST_TENSOR_TYPE_INT8 signed 8 bit integer tensor data
@ -42,9 +42,11 @@
* @GST_TENSOR_TYPE_FLOAT64 64 bit floating point tensor data
* @GST_TENSOR_TYPE_BFLOAT16 "brain" 16 bit floating point tensor data
*
* Describe the type of data contain in the tensor.
*
* Since: 1.24
*/
typedef enum _GstTensorType
typedef enum _GstTensorDataType
{
GST_TENSOR_TYPE_INT4,
GST_TENSOR_TYPE_INT8,
@ -60,17 +62,17 @@ typedef enum _GstTensorType
GST_TENSOR_TYPE_FLOAT32,
GST_TENSOR_TYPE_FLOAT64,
GST_TENSOR_TYPE_BFLOAT16,
} GstTensorType;
} GstTensorDataType;
/**
* GstTensor:
*
* @id unique tensor identifier
* @num_dims number of tensor dimensions
* @dims tensor dimensions
* @type @ref GstTensorType of tensor data
* @data @ref GstBuffer holding tensor data
* @id: semantically identify the contents of the tensor
* @num_dims: number of tensor dimensions
* @dims: tensor dimensions
* @type: #GstTensorDataType of tensor data
* @data: #GstBuffer holding tensor data
*
* Since: 1.24
*/
@ -79,7 +81,7 @@ typedef struct _GstTensor
GQuark id;
gint num_dims;
int64_t *dims;
GstTensorType type;
GstTensorDataType data_type;
GstBuffer *data;
} GstTensor;