gstreamer/ext/opencv/gsttemplatematch.cpp

387 lines
13 KiB
C++
Raw Normal View History

2009-05-13 08:55:31 +00:00
/*
* GStreamer
* Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
* Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
* Copyright (C) 2008 Michael Sheldon <mike@mikeasoft.com>
* Copyright (C) 2009 Noam Lewis <jones.noamle@gmail.com>
*
2009-05-13 08:55:31 +00:00
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Alternatively, the contents of this file may be used under the
* GNU Lesser General Public License Version 2.1 (the "LGPL"), in
* which case the following provisions apply instead of the ones
* mentioned above:
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
2009-05-13 08:55:31 +00:00
*/
/**
* SECTION:element-templatematch
*
2011-11-18 18:19:32 +00:00
* Performs template matching on videos and images, providing detected positions via bus messages.
2009-05-13 08:55:31 +00:00
*
2019-05-29 20:58:08 +00:00
* ## Example launch line
*
2009-05-13 08:55:31 +00:00
* |[
2018-12-01 21:48:53 +00:00
* gst-launch-1.0 videotestsrc ! videoconvert ! templatematch template=/path/to/file.jpg ! videoconvert ! xvimagesink
2009-05-13 08:55:31 +00:00
* ]|
*/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include <gst/gst-i18n-plugin.h>
2009-05-13 08:55:31 +00:00
#include "gsttemplatematch.h"
#include <opencv2/imgproc.hpp>
#include <opencv2/imgcodecs.hpp>
2009-05-13 08:55:31 +00:00
GST_DEBUG_CATEGORY_STATIC (gst_template_match_debug);
#define GST_CAT_DEFAULT gst_template_match_debug
2009-05-13 08:55:31 +00:00
2009-05-13 09:19:43 +00:00
#define DEFAULT_METHOD (3)
2009-05-13 08:55:31 +00:00
/* Filter signals and args */
enum
{
/* FILL ME */
LAST_SIGNAL
2009-05-13 08:55:31 +00:00
};
enum
{
PROP_0,
PROP_METHOD,
PROP_TEMPLATE,
PROP_DISPLAY,
2009-05-13 08:55:31 +00:00
};
/* the capabilities of the inputs and outputs.
*/
static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("BGR"))
2009-05-13 08:55:31 +00:00
);
static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("BGR"))
2009-05-13 08:55:31 +00:00
);
G_DEFINE_TYPE_WITH_CODE (GstTemplateMatch, gst_template_match,
GST_TYPE_OPENCV_VIDEO_FILTER,
GST_DEBUG_CATEGORY_INIT (gst_template_match_debug, "templatematch", 0,
"Performs template matching on videos and images, providing detected positions via bus messages");
);
GST_ELEMENT_REGISTER_DEFINE (templatematch, "templatematch",
GST_RANK_NONE, GST_TYPE_TEMPLATE_MATCH);
2009-05-13 08:55:31 +00:00
static void gst_template_match_finalize (GObject * object);
static void gst_template_match_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_template_match_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
2009-05-13 08:55:31 +00:00
static GstFlowReturn gst_template_match_transform_ip (GstOpencvVideoFilter *
2018-12-01 21:48:53 +00:00
filter, GstBuffer * buf, cv::Mat img);
2009-05-13 08:55:31 +00:00
/* initialize the templatematch's class */
static void
gst_template_match_class_init (GstTemplateMatchClass * klass)
2009-05-13 08:55:31 +00:00
{
GObjectClass *gobject_class;
GstOpencvVideoFilterClass *gstopencvbasefilter_class;
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
gobject_class = (GObjectClass *) klass;
gstopencvbasefilter_class = (GstOpencvVideoFilterClass *) klass;
gobject_class->finalize = gst_template_match_finalize;
gobject_class->set_property = gst_template_match_set_property;
gobject_class->get_property = gst_template_match_get_property;
gstopencvbasefilter_class->cv_trans_ip_func = gst_template_match_transform_ip;
g_object_class_install_property (gobject_class, PROP_METHOD,
g_param_spec_int ("method", "Method",
"Specifies the way the template must be compared with image regions. 0=SQDIFF, 1=SQDIFF_NORMED, 2=CCOR, 3=CCOR_NORMED, 4=CCOEFF, 5=CCOEFF_NORMED.",
0, 5, DEFAULT_METHOD,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_TEMPLATE,
g_param_spec_string ("template", "Template", "Filename of template image",
NULL, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_DISPLAY,
g_param_spec_boolean ("display", "Display",
"Sets whether the detected template should be highlighted in the output",
TRUE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
gst_element_class_set_static_metadata (element_class,
"templatematch",
"Filter/Effect/Video",
"Performs template matching on videos and images, providing detected positions via bus messages.",
"Noam Lewis <jones.noamle@gmail.com>");
gst_element_class_add_static_pad_template (element_class, &src_factory);
gst_element_class_add_static_pad_template (element_class, &sink_factory);
2009-05-13 08:55:31 +00:00
}
/* initialize the new element
* instantiate pads and add them to element
* set pad callback functions
2009-05-13 08:55:31 +00:00
* initialize instance structure
*/
static void
gst_template_match_init (GstTemplateMatch * filter)
2009-05-13 08:55:31 +00:00
{
filter->templ = NULL;
filter->display = TRUE;
2018-12-01 21:48:53 +00:00
filter->reload_dist_image = TRUE;
filter->method = DEFAULT_METHOD;
gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER_CAST (filter),
TRUE);
2009-05-13 08:55:31 +00:00
}
/* We take ownership of template here */
static void
gst_template_match_load_template (GstTemplateMatch * filter, gchar * templ)
{
2018-12-01 21:48:53 +00:00
cv::Mat newTemplateImage;
if (templ) {
2018-12-01 21:48:53 +00:00
newTemplateImage = cv::imread (templ);
if (newTemplateImage.empty ()) {
/* Unfortunately OpenCV doesn't seem to provide any way of finding out
why the image load failed, so we can't be more specific than FAILED: */
GST_ELEMENT_WARNING (filter, RESOURCE, FAILED,
(_("OpenCV failed to load template image")),
("While attempting to load template '%s'", templ));
g_free (templ);
templ = NULL;
}
}
GST_OBJECT_LOCK (filter);
2018-12-01 21:48:53 +00:00
g_free (filter->templ);
filter->templ = templ;
2018-12-01 21:48:53 +00:00
filter->cvTemplateImage = cv::Mat (newTemplateImage);
filter->reload_dist_image = TRUE;
GST_OBJECT_UNLOCK (filter);
}
2009-05-13 08:55:31 +00:00
static void
gst_template_match_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
2009-05-13 08:55:31 +00:00
{
GstTemplateMatch *filter = GST_TEMPLATE_MATCH (object);
2009-05-13 08:55:31 +00:00
switch (prop_id) {
2009-05-13 09:19:43 +00:00
case PROP_METHOD:
GST_OBJECT_LOCK (filter);
switch (g_value_get_int (value)) {
2009-05-13 09:19:43 +00:00
case 0:
2018-12-01 21:48:53 +00:00
filter->method = cv::TM_SQDIFF;
break;
2009-05-13 09:19:43 +00:00
case 1:
2018-12-01 21:48:53 +00:00
filter->method = cv::TM_SQDIFF_NORMED;
break;
2009-05-13 09:19:43 +00:00
case 2:
2018-12-01 21:48:53 +00:00
filter->method = cv::TM_CCORR;
break;
2009-05-13 09:19:43 +00:00
case 3:
2018-12-01 21:48:53 +00:00
filter->method = cv::TM_CCORR_NORMED;
break;
2009-05-13 09:19:43 +00:00
case 4:
2018-12-01 21:48:53 +00:00
filter->method = cv::TM_CCOEFF;
break;
2009-05-13 09:19:43 +00:00
case 5:
2018-12-01 21:48:53 +00:00
filter->method = cv::TM_CCOEFF_NORMED;
break;
}
GST_OBJECT_UNLOCK (filter);
break;
2009-05-13 08:55:31 +00:00
case PROP_TEMPLATE:
gst_template_match_load_template (filter, g_value_dup_string (value));
break;
2009-05-13 08:55:31 +00:00
case PROP_DISPLAY:
GST_OBJECT_LOCK (filter);
filter->display = g_value_get_boolean (value);
GST_OBJECT_UNLOCK (filter);
break;
2009-05-13 08:55:31 +00:00
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
2009-05-13 08:55:31 +00:00
}
static void
gst_template_match_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
2009-05-13 08:55:31 +00:00
{
GstTemplateMatch *filter = GST_TEMPLATE_MATCH (object);
2009-05-13 08:55:31 +00:00
switch (prop_id) {
2009-05-13 09:19:43 +00:00
case PROP_METHOD:
g_value_set_int (value, filter->method);
break;
2009-05-13 08:55:31 +00:00
case PROP_TEMPLATE:
g_value_set_string (value, filter->templ);
break;
2009-05-13 08:55:31 +00:00
case PROP_DISPLAY:
g_value_set_boolean (value, filter->display);
break;
2009-05-13 08:55:31 +00:00
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
2009-05-13 08:55:31 +00:00
}
/* GstElement vmethod implementations */
static void
gst_template_match_finalize (GObject * object)
2009-05-13 08:55:31 +00:00
{
GstTemplateMatch *filter;
filter = GST_TEMPLATE_MATCH (object);
g_free (filter->templ);
2018-12-01 21:48:53 +00:00
filter->cvDistImage.release ();
filter->cvTemplateImage.release ();
G_OBJECT_CLASS (gst_template_match_parent_class)->finalize (object);
2009-05-13 08:55:31 +00:00
}
static void
2018-12-01 21:48:53 +00:00
gst_template_match_match (cv::Mat input, cv::Mat templ,
cv::Mat dist_image, double *best_res, cv::Point * best_pos, int method)
{
double dist_min = 0, dist_max = 0;
2018-12-01 21:48:53 +00:00
cv::Point min_pos, max_pos;
matchTemplate (input, templ, dist_image, method);
minMaxLoc (dist_image, &dist_min, &dist_max, &min_pos, &max_pos);
if ((cv::TM_SQDIFF_NORMED == method) || (cv::TM_SQDIFF == method)) {
*best_res = dist_min;
*best_pos = min_pos;
2018-12-01 21:48:53 +00:00
if (cv::TM_SQDIFF_NORMED == method) {
*best_res = 1 - *best_res;
}
} else {
*best_res = dist_max;
*best_pos = max_pos;
}
}
2009-05-13 08:55:31 +00:00
/* chain function
* this function does the actual processing
*/
static GstFlowReturn
gst_template_match_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf,
2018-12-01 21:48:53 +00:00
cv::Mat img)
2009-05-13 08:55:31 +00:00
{
GstTemplateMatch *filter;
2018-12-01 21:48:53 +00:00
cv::Point best_pos;
double best_res;
GstMessage *m = NULL;
filter = GST_TEMPLATE_MATCH (base);
GST_LOG_OBJECT (filter, "Buffer size %u", (guint) gst_buffer_get_size (buf));
GST_OBJECT_LOCK (filter);
2018-12-01 21:48:53 +00:00
if (!filter->cvTemplateImage.empty () && filter->reload_dist_image) {
if (filter->cvTemplateImage.size ().width > img.size ().width) {
GST_WARNING ("Template Image is wider than input image");
2018-12-01 21:48:53 +00:00
} else if (filter->cvTemplateImage.size ().height > img.size ().height) {
GST_WARNING ("Template Image is taller than input image");
} else {
2018-12-01 21:48:53 +00:00
GST_DEBUG_OBJECT (filter, "cv create (Size(%d-%d+1,%d) %d)",
img.size ().width, filter->cvTemplateImage.size ().width,
img.size ().height - filter->cvTemplateImage.size ().height + 1,
CV_32FC1);
filter->cvDistImage.create (cv::Size (img.size ().width -
filter->cvTemplateImage.size ().width + 1,
img.size ().height - filter->cvTemplateImage.size ().height + 1),
CV_32FC1);
filter->reload_dist_image = FALSE;
2009-05-13 08:55:31 +00:00
}
}
2018-12-01 21:48:53 +00:00
if (!filter->cvTemplateImage.empty () && !filter->reload_dist_image) {
GstStructure *s;
gst_template_match_match (img, filter->cvTemplateImage,
filter->cvDistImage, &best_res, &best_pos, filter->method);
s = gst_structure_new ("template_match",
"x", G_TYPE_UINT, best_pos.x,
"y", G_TYPE_UINT, best_pos.y,
2018-12-01 21:48:53 +00:00
"width", G_TYPE_UINT, filter->cvTemplateImage.size ().width,
"height", G_TYPE_UINT, filter->cvTemplateImage.size ().height,
"result", G_TYPE_DOUBLE, best_res, NULL);
m = gst_message_new_element (GST_OBJECT (filter), s);
if (filter->display) {
2018-12-01 21:48:53 +00:00
cv::Point corner = best_pos;
cv::Scalar color;
if (filter->method == cv::TM_SQDIFF_NORMED
|| filter->method == cv::TM_CCORR_NORMED
|| filter->method == cv::TM_CCOEFF_NORMED) {
/* Yellow growing redder as match certainty approaches 1.0. This can
only be applied with method == *_NORMED as the other match methods
aren't normalized to be in range 0.0 - 1.0 */
color = CV_RGB (255, 255 - pow (255, best_res), 32);
} else {
color = CV_RGB (255, 32, 32);
}
buf = gst_buffer_make_writable (buf);
2018-12-01 21:48:53 +00:00
corner.x += filter->cvTemplateImage.size ().width;
corner.y += filter->cvTemplateImage.size ().height;
cv::rectangle (img, best_pos, corner, color, 3, 8, 0);
2009-05-13 08:55:31 +00:00
}
}
GST_OBJECT_UNLOCK (filter);
if (m) {
gst_element_post_message (GST_ELEMENT (filter), m);
}
return GST_FLOW_OK;
2009-05-13 08:55:31 +00:00
}