From 15e5866e51ea74208f06255ee53b4955daddd381 Mon Sep 17 00:00:00 2001 From: Daniel Morin Date: Mon, 4 Dec 2023 19:57:39 -0500 Subject: [PATCH] onnx: add offset and scale properties - Offset each datapoint by the value set on offset property. - Scale each datapoint by the value set on scale property. Part-of: --- .../ext/onnx/gstonnxclient.cpp | 41 ++++++++++++++++--- .../gst-plugins-bad/ext/onnx/gstonnxclient.h | 8 +++- .../ext/onnx/gstonnxinference.cpp | 33 ++++++++++++++- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.cpp b/subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.cpp index 95071d0eea..6afbba8eba 100644 --- a/subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.cpp +++ b/subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.cpp @@ -52,7 +52,9 @@ GstOnnxClient::GstOnnxClient ():session (nullptr), inputImageFormat (GST_ML_INPUT_IMAGE_FORMAT_HWC), inputDatatype (GST_TENSOR_TYPE_INT8), inputDatatypeSize (sizeof (uint8_t)), - fixedInputImageSize (false) { + fixedInputImageSize (false), + inputTensorOffset (0.0), + inputTensorScale (1.0) { } GstOnnxClient::~GstOnnxClient () { @@ -107,6 +109,26 @@ GstOnnxClient::GstOnnxClient ():session (nullptr), }; } + void GstOnnxClient::setInputImageOffset (float offset) + { + inputTensorOffset = offset; + } + + float GstOnnxClient::getInputImageOffset () + { + return inputTensorOffset; + } + + void GstOnnxClient::setInputImageScale (float scale) + { + inputTensorScale = scale; + } + + float GstOnnxClient::getInputImageScale () + { + return inputTensorScale; + } + GstTensorType GstOnnxClient::getInputImageDatatype(void) { return inputDatatype; @@ -422,14 +444,16 @@ GstOnnxClient::GstOnnxClient ():session (nullptr), switch (inputDatatype) { case GST_TENSOR_TYPE_INT8: convert_image_remove_alpha (dest, inputImageFormat , srcPtr, - srcSamplesPerPixel, stride); + srcSamplesPerPixel, stride, (uint8_t)inputTensorOffset, + (uint8_t)inputTensorScale); inputTensors.push_back (Ort::Value::CreateTensor < uint8_t > ( memoryInfo, dest, inputTensorSize, inputDims.data (), inputDims.size ())); break; case GST_TENSOR_TYPE_FLOAT32: { convert_image_remove_alpha ((float*)dest, inputImageFormat , srcPtr, - srcSamplesPerPixel, stride); + srcSamplesPerPixel, stride, (float)inputTensorOffset, (float) + inputTensorScale); inputTensors.push_back (Ort::Value::CreateTensor < float > ( memoryInfo, (float*)dest, inputTensorSize, inputDims.data (), inputDims.size ())); @@ -451,14 +475,17 @@ GstOnnxClient::GstOnnxClient ():session (nullptr), template < typename T> void GstOnnxClient::convert_image_remove_alpha (T *dst, GstMlInputImageFormat hwc, uint8_t **srcPtr, uint32_t srcSamplesPerPixel, - uint32_t stride) { + uint32_t stride, T offset, T div) { size_t destIndex = 0; + T tmp; if (inputImageFormat == GST_ML_INPUT_IMAGE_FORMAT_HWC) { for (int32_t j = 0; j < height; ++j) { for (int32_t i = 0; i < width; ++i) { for (int32_t k = 0; k < channels; ++k) { - dst[destIndex++] = (T)*srcPtr[k]; + tmp = *srcPtr[k]; + tmp += offset; + dst[destIndex++] = (T)(tmp / div); srcPtr[k] += srcSamplesPerPixel; } } @@ -472,7 +499,9 @@ GstOnnxClient::GstOnnxClient ():session (nullptr), for (int32_t j = 0; j < height; ++j) { for (int32_t i = 0; i < width; ++i) { for (int32_t k = 0; k < channels; ++k) { - destPtr[k][destIndex] = (T)*srcPtr[k]; + tmp = *srcPtr[k]; + tmp += offset; + destPtr[k][destIndex] = (T)(tmp / div); srcPtr[k] += srcSamplesPerPixel; } destIndex++; diff --git a/subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.h b/subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.h index 3721b8968a..f0a9b8bb89 100644 --- a/subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.h +++ b/subprojects/gst-plugins-bad/ext/onnx/gstonnxclient.h @@ -57,6 +57,10 @@ namespace GstOnnxNamespace { GstMlInputImageFormat getInputImageFormat(void); void setInputImageDatatype(GstTensorType datatype); GstTensorType getInputImageDatatype(void); + void setInputImageOffset (float offset); + float getInputImageOffset (); + void setInputImageScale (float offset); + float getInputImageScale (); std::vector < Ort::Value > run (uint8_t * img_data, GstVideoInfo vinfo); std::vector < const char *> genOutputNamesRaw(void); bool isFixedInputImageSize(void); @@ -68,7 +72,7 @@ namespace GstOnnxNamespace { template < typename T> void convert_image_remove_alpha (T *dest, GstMlInputImageFormat hwc, - uint8_t **srcPtr, uint32_t srcSamplesPerPixel, uint32_t stride); + uint8_t **srcPtr, uint32_t srcSamplesPerPixel, uint32_t stride, T offset, T div); bool doRun(uint8_t * img_data, GstVideoInfo vinfo, std::vector < Ort::Value > &modelOutput); Ort::Env env; Ort::Session * session; @@ -86,6 +90,8 @@ namespace GstOnnxNamespace { GstTensorType inputDatatype; size_t inputDatatypeSize; bool fixedInputImageSize; + float inputTensorOffset; + float inputTensorScale; }; } diff --git a/subprojects/gst-plugins-bad/ext/onnx/gstonnxinference.cpp b/subprojects/gst-plugins-bad/ext/onnx/gstonnxinference.cpp index 3256ea31bd..8f4c6828c3 100644 --- a/subprojects/gst-plugins-bad/ext/onnx/gstonnxinference.cpp +++ b/subprojects/gst-plugins-bad/ext/onnx/gstonnxinference.cpp @@ -123,7 +123,9 @@ enum PROP_INPUT_IMAGE_FORMAT, PROP_OPTIMIZATION_LEVEL, PROP_EXECUTION_PROVIDER, - PROP_INPUT_IMAGE_DATATYPE + PROP_INPUT_IMAGE_DATATYPE, + PROP_INPUT_OFFSET, + PROP_INPUT_SCALE }; #define GST_ONNX_INFERENCE_DEFAULT_EXECUTION_PROVIDER GST_ONNX_EXECUTION_PROVIDER_CPU @@ -362,6 +364,23 @@ gst_onnx_inference_class_init (GstOnnxInferenceClass * klass) GST_TENSOR_TYPE_INT8, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property (G_OBJECT_CLASS (klass), + PROP_INPUT_OFFSET, + g_param_spec_float ("input-tensor-offset", + "Input tensor offset", + "offset each tensor value by this value", + -G_MAXFLOAT, G_MAXFLOAT, 0.0, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (G_OBJECT_CLASS (klass), + PROP_INPUT_SCALE, + g_param_spec_float ("input-tensor-scale", + "Input tensor scale", + "Divide each tensor value by this value", + G_MINFLOAT, G_MAXFLOAT, 1.0, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + gst_element_class_set_static_metadata (element_class, "onnxinference", "Filter/Effect/Video", "Apply neural network to video frames and create tensor output", @@ -433,6 +452,12 @@ gst_onnx_inference_set_property (GObject * object, guint prop_id, onnxClient->setInputImageDatatype ((GstTensorType) g_value_get_enum (value)); break; + case PROP_INPUT_OFFSET: + onnxClient->setInputImageOffset (g_value_get_float (value)); + break; + case PROP_INPUT_SCALE: + onnxClient->setInputImageScale (g_value_get_float (value)); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -462,6 +487,12 @@ gst_onnx_inference_get_property (GObject * object, guint prop_id, case PROP_INPUT_IMAGE_DATATYPE: g_value_set_enum (value, onnxClient->getInputImageDatatype ()); break; + case PROP_INPUT_OFFSET: + g_value_set_float (value, onnxClient->getInputImageOffset ()); + break; + case PROP_INPUT_SCALE: + g_value_set_float (value, onnxClient->getInputImageScale ()); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break;