diff --git a/ext/opencv/Makefile.am b/ext/opencv/Makefile.am index 02a77ea628..da815a3d8b 100644 --- a/ext/opencv/Makefile.am +++ b/ext/opencv/Makefile.am @@ -22,7 +22,8 @@ libgstopencv_la_SOURCES = gstopencv.cpp \ gstgrabcut.cpp \ gstdisparity.cpp \ motioncells_wrapper.cpp \ - MotionCells.cpp + MotionCells.cpp \ + gstdewarp.cpp libgstopencv_la_CXXFLAGS = \ -I$(top_srcdir)/gst-libs \ @@ -70,7 +71,8 @@ noinst_HEADERS = \ gstdisparity.h \ gstmotioncells.h \ motioncells_wrapper.h \ - MotionCells.h + MotionCells.h \ + gstdewarp.h opencv_haarcascadesdir = $(pkgdatadir)/$(GST_API_VERSION)/opencv_haarcascades opencv_haarcascades_DATA = fist.xml palm.xml diff --git a/ext/opencv/gstdewarp.cpp b/ext/opencv/gstdewarp.cpp new file mode 100644 index 0000000000..1acece2906 --- /dev/null +++ b/ext/opencv/gstdewarp.cpp @@ -0,0 +1,723 @@ +/* + * GStreamer + * Copyright (C) 2016 Prassel S.r.l + * Author: Nicola Murino + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Alternatively, the contents of this file may be used under the + * GNU Lesser General Public License Version 2.1 (the "LGPL"), in + * which case the following provisions apply instead of the ones + * mentioned above: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-dewarp + * + * Dewarp fisheye images + * + * + * Example launch line + * |[ + * gst-launch-1.0 videotestsrc ! videoconvert ! circle radius=0.1 height=80 ! dewarp outer-radius=0.35 inner-radius=0.1 ! videoconvert ! xvimagesink + * ]| + * + */ + + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "gstdewarp.h" +#include + +GST_DEBUG_CATEGORY_STATIC (gst_dewarp_debug); +#define GST_CAT_DEFAULT gst_dewarp_debug + +enum +{ + PROP_0, + PROP_X_CENTER, + PROP_Y_CENTER, + PROP_INNER_RADIUS, + PROP_OUTER_RADIUS, + PROP_REMAP_X_CORRECTION, + PROP_REMAP_Y_CORRECTION, + PROP_DISPLAY_MODE, + PROP_INTERPOLATION_MODE +}; + +#define DEFAULT_CENTER 0.5 +#define DEFAULT_RADIUS 0.0 +#define DEFAULT_REMAP_CORRECTION 1.0 + +#define GST_TYPE_DEWARP_DISPLAY_MODE (dewarp_display_mode_get_type ()) + +static GType +dewarp_display_mode_get_type (void) +{ + static GType dewarp_display_mode_type = 0; + static const GEnumValue dewarp_display_mode[] = { + {GST_DEWARP_DISPLAY_PANORAMA, "Single panorama image", "single-panorama"}, + {GST_DEWARP_DISPLAY_DOUBLE_PANORAMA, "Dewarped image is splitted in two " + "images displayed one below the other", "double-panorama"}, + {GST_DEWARP_DISPLAY_QUAD_VIEW, "Dewarped image is splitted in four images " + "dysplayed as a quad view", + "quad-view"}, + {0, NULL, NULL}, + }; + + if (!dewarp_display_mode_type) { + dewarp_display_mode_type = + g_enum_register_static ("GstDewarpDisplayMode", dewarp_display_mode); + } + return dewarp_display_mode_type; +} + +#define GST_TYPE_DEWARP_INTERPOLATION_MODE (dewarp_interpolation_mode_get_type ()) + +static GType +dewarp_interpolation_mode_get_type (void) +{ + static GType dewarp_interpolation_mode_type = 0; + static const GEnumValue dewarp_interpolation_mode[] = { + {GST_DEWARP_INTER_NEAREST, "A nearest-neighbor interpolation", "nearest"}, + {GST_DEWARP_INTER_LINEAR, "A bilinear interpolation", "bilinear"}, + {GST_DEWARP_INTER_CUBIC, + "A bicubic interpolation over 4x4 pixel neighborhood", "bicubic"}, + {GST_DEWARP_INTER_LANCZOS4, + "A Lanczos interpolation over 8x8 pixel neighborhood", "Lanczos"}, + {0, NULL, NULL}, + }; + + if (!dewarp_interpolation_mode_type) { + dewarp_interpolation_mode_type = + g_enum_register_static ("GstDewarpInterpolationMode", + dewarp_interpolation_mode); + } + return dewarp_interpolation_mode_type; +} + +G_DEFINE_TYPE (GstDewarp, gst_dewarp, GST_TYPE_OPENCV_VIDEO_FILTER); + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA"))); + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA"))); + +static void gst_dewarp_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_dewarp_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstCaps *gst_dewarp_transform_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps); + +static GstFlowReturn gst_dewarp_transform_frame (GstOpencvVideoFilter * btrans, + GstBuffer * buffer, IplImage * img, GstBuffer * outbuf, IplImage * outimg); + +static gboolean gst_dewarp_set_caps (GstOpencvVideoFilter * filter, + gint in_width, gint in_height, gint in_depth, gint in_channels, + gint out_width, gint out_height, gint out_depth, gint out_channels); + +static void +gst_dewarp_finalize (GObject * obj) +{ + GstDewarp *filter = GST_DEWARP (obj); + filter->map_x.release (); + filter->map_y.release (); + + G_OBJECT_CLASS (gst_dewarp_parent_class)->finalize (obj); +} + +static void +gst_dewarp_class_init (GstDewarpClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseTransformClass *basesrc_class = GST_BASE_TRANSFORM_CLASS (klass); + GstOpencvVideoFilterClass *cvfilter_class = + (GstOpencvVideoFilterClass *) klass; + + gobject_class = (GObjectClass *) klass; + + gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_dewarp_finalize); + gobject_class->set_property = gst_dewarp_set_property; + gobject_class->get_property = gst_dewarp_get_property; + + basesrc_class->transform_caps = GST_DEBUG_FUNCPTR (gst_dewarp_transform_caps); + basesrc_class->transform_ip_on_passthrough = FALSE; + basesrc_class->passthrough_on_same_caps = TRUE; + + cvfilter_class->cv_trans_func = + GST_DEBUG_FUNCPTR (gst_dewarp_transform_frame); + cvfilter_class->cv_set_caps = GST_DEBUG_FUNCPTR (gst_dewarp_set_caps); + + g_object_class_install_property (gobject_class, PROP_X_CENTER, + g_param_spec_double ("x-center", "x center", + "X axis center of the fisheye image", + 0.0, 1.0, DEFAULT_CENTER, + (GParamFlags) (GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_Y_CENTER, + g_param_spec_double ("y-center", "y center", + "Y axis center of the fisheye image", + 0.0, 1.0, DEFAULT_CENTER, + (GParamFlags) (GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_INNER_RADIUS, + g_param_spec_double ("inner-radius", "inner radius", + "Inner radius of the fisheye image donut. If outer radius <= inner " + "radius the element will work in passthrough mode", + 0.0, 1.0, DEFAULT_RADIUS, + (GParamFlags) (GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_OUTER_RADIUS, + g_param_spec_double ("outer-radius", "outer radius", + "Outer radius of the fisheye image donut. If outer radius <= inner " + "radius the element will work in passthrough mode", + 0.0, 1.0, DEFAULT_RADIUS, + (GParamFlags) (GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_REMAP_X_CORRECTION, + g_param_spec_double ("x-remap-correction", "x remap correction", + "Correction factor for remapping on x axis. A correction is needed if " + "the fisheye image is not inside a circle", + 0.1, 10.0, DEFAULT_REMAP_CORRECTION, + (GParamFlags) (GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_REMAP_Y_CORRECTION, + g_param_spec_double ("y-remap-correction", "y remap correction", + "Correction factor for remapping on y axis. A correction is needed if " + "the fisheye image is not inside a circle", + 0.1, 10.0, DEFAULT_REMAP_CORRECTION, + (GParamFlags) (GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_INTERPOLATION_MODE, + g_param_spec_enum ("interpolation-method", "Interpolation method", + "Interpolation method to use", + GST_TYPE_DEWARP_INTERPOLATION_MODE, GST_DEWARP_INTER_LINEAR, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_DISPLAY_MODE, + g_param_spec_enum ("display-mode", "Display mode", + "How to display the dewarped image", + GST_TYPE_DEWARP_DISPLAY_MODE, GST_DEWARP_DISPLAY_PANORAMA, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + gst_element_class_set_static_metadata (element_class, + "Dewarp fisheye images", + "Filter/Effect/Video", + "Dewarp fisheye images", "Nicola Murino "); + + gst_element_class_add_static_pad_template (element_class, &src_factory); + gst_element_class_add_static_pad_template (element_class, &sink_factory); + +} + +static void +gst_dewarp_init (GstDewarp * filter) +{ + filter->x_center = DEFAULT_CENTER; + filter->y_center = DEFAULT_CENTER; + filter->inner_radius = DEFAULT_RADIUS; + filter->outer_radius = DEFAULT_RADIUS; + filter->remap_correction_x = DEFAULT_REMAP_CORRECTION; + filter->remap_correction_y = DEFAULT_REMAP_CORRECTION; + filter->display_mode = GST_DEWARP_DISPLAY_PANORAMA; + filter->interpolation_mode = GST_DEWARP_INTER_LINEAR; + filter->pad_sink_width = 0; + filter->pad_sink_height = 0; + filter->in_width = 0; + filter->in_height = 0; + filter->out_width = 0; + filter->out_height = 0; + filter->need_map_update = TRUE; + gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER_CAST (filter), + FALSE); +} + +static void +gst_dewarp_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + gdouble v; + gboolean need_reconfigure; + int disp_mode; + GstDewarp *filter = GST_DEWARP (object); + + need_reconfigure = FALSE; + + GST_OBJECT_LOCK (filter); + + switch (prop_id) { + case PROP_X_CENTER: + v = g_value_get_double (value); + if (v != filter->x_center) { + filter->x_center = v; + filter->need_map_update = TRUE; + need_reconfigure = TRUE; + GST_LOG_OBJECT (filter, "x center setted to %f", filter->x_center); + } + break; + case PROP_Y_CENTER: + v = g_value_get_double (value); + if (v != filter->y_center) { + filter->y_center = v; + filter->need_map_update = TRUE; + need_reconfigure = TRUE; + GST_LOG_OBJECT (filter, "y center setted to %f", filter->y_center); + } + break; + case PROP_INNER_RADIUS: + v = g_value_get_double (value); + if (v != filter->inner_radius) { + filter->inner_radius = v; + filter->need_map_update = TRUE; + need_reconfigure = TRUE; + GST_LOG_OBJECT (filter, "inner radius setted to %f", + filter->inner_radius); + } + break; + case PROP_OUTER_RADIUS: + v = g_value_get_double (value); + if (v != filter->outer_radius) { + filter->outer_radius = v; + filter->need_map_update = TRUE; + need_reconfigure = TRUE; + GST_LOG_OBJECT (filter, "outer radius setted to %f", + filter->outer_radius); + } + break; + case PROP_REMAP_X_CORRECTION: + v = g_value_get_double (value); + if (v != filter->remap_correction_x) { + filter->remap_correction_x = v; + filter->need_map_update = TRUE; + need_reconfigure = TRUE; + GST_LOG_OBJECT (filter, "x remap correction setted to %f", + filter->remap_correction_x); + } + break; + case PROP_REMAP_Y_CORRECTION: + v = g_value_get_double (value); + if (v != filter->remap_correction_y) { + filter->remap_correction_y = v; + filter->need_map_update = TRUE; + need_reconfigure = TRUE; + GST_LOG_OBJECT (filter, "y remap correction setted to %f", + filter->remap_correction_y); + } + break; + case PROP_INTERPOLATION_MODE: + filter->interpolation_mode = g_value_get_enum (value); + GST_LOG_OBJECT (filter, "interpolation mode setted to %" G_GINT32_FORMAT, + filter->interpolation_mode); + break; + case PROP_DISPLAY_MODE: + disp_mode = g_value_get_enum (value); + if (disp_mode != filter->display_mode) { + filter->display_mode = disp_mode; + need_reconfigure = TRUE; + GST_LOG_OBJECT (filter, "display mode setted to %" G_GINT32_FORMAT, + filter->display_mode); + } + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + + if (filter->need_map_update) + GST_LOG_OBJECT (filter, "need map update after property change"); + + GST_OBJECT_UNLOCK (filter); + + if (need_reconfigure) { + GST_DEBUG_OBJECT (filter, "Reconfigure src after property change"); + gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (filter)); + } else { + GST_DEBUG_OBJECT (filter, + "No property value changed, reconfigure src is not" " needed"); + } +} + +static void +gst_dewarp_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstDewarp *filter = GST_DEWARP (object); + + GST_OBJECT_LOCK (filter); + + switch (prop_id) { + case PROP_X_CENTER: + g_value_set_double (value, filter->x_center); + break; + case PROP_Y_CENTER: + g_value_set_double (value, filter->y_center); + break; + case PROP_INNER_RADIUS: + g_value_set_double (value, filter->inner_radius); + break; + case PROP_OUTER_RADIUS: + g_value_set_double (value, filter->outer_radius); + break; + case PROP_REMAP_X_CORRECTION: + g_value_set_double (value, filter->remap_correction_x); + break; + case PROP_REMAP_Y_CORRECTION: + g_value_set_double (value, filter->remap_correction_y); + break; + case PROP_INTERPOLATION_MODE: + g_value_set_enum (value, filter->interpolation_mode); + break; + case PROP_DISPLAY_MODE: + g_value_set_enum (value, filter->display_mode); + break; + + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + + GST_OBJECT_UNLOCK (filter); +} + +static void +gst_dewarp_update_map (GstDewarp * filter) +{ + gdouble r1, r2, cx, cy; + gint x, y; + gint out_width, out_height; + + if (filter->display_mode == GST_DEWARP_DISPLAY_PANORAMA) { + out_width = filter->out_width; + out_height = filter->out_height; + } else { + out_width = filter->out_width * 2; + out_height = filter->out_height / 2; + } + + GST_DEBUG_OBJECT (filter, + "start update map out_width: %" G_GINT32_FORMAT " out height: %" + G_GINT32_FORMAT, out_width, out_height); + + r1 = filter->in_width * filter->inner_radius; + r2 = filter->in_width * filter->outer_radius; + cx = filter->x_center * filter->in_width; + cy = filter->y_center * filter->in_height; + cv::Size destSize (out_width, out_height); + filter->map_x.create (destSize, CV_32FC1); + filter->map_y.create (destSize, CV_32FC1); + + for (y = 0; y < out_height; y++) { + for (x = 0; x < out_width; x++) { + float r = ((float) (y) / (float) (out_height)) * (r2 - r1) + r1; + float theta = ((float) (x) / (float) (out_width)) * 2.0 * G_PI; + float xs = cx + r * sin (theta) * filter->remap_correction_x; + float ys = cy + r * cos (theta) * filter->remap_correction_y; + filter->map_x.at < float >(y, x) = xs; + filter->map_y.at < float >(y, x) = ys; + } + } + + filter->need_map_update = FALSE; + + GST_DEBUG_OBJECT (filter, "update map done"); +} + +static void +gst_dewarp_calculate_dimensions (GstDewarp * filter, GstPadDirection direction, + gint in_width, gint in_height, gint * out_width, gint * out_height) +{ + if (filter->outer_radius <= filter->inner_radius) { + GST_LOG_OBJECT (filter, + "No dimensions conversion required, in width: %" G_GINT32_FORMAT + " in height: %" G_GINT32_FORMAT, in_width, in_height); + *out_width = in_width; + *out_height = in_height; + } else { + gdouble r1, r2; + + GST_LOG_OBJECT (filter, + "Calculate dimensions, in_width: %" G_GINT32_FORMAT + " in_height: %" G_GINT32_FORMAT " pad sink width: %" G_GINT32_FORMAT + " pad sink height: %" G_GINT32_FORMAT + " inner radius: %f, outer radius: %f, direction: %d", in_width, + in_height, filter->pad_sink_width, filter->pad_sink_height, + filter->inner_radius, filter->outer_radius, direction); + + r1 = in_width * filter->inner_radius; + r2 = in_width * filter->outer_radius; + + if (direction == GST_PAD_SINK) { + /* roundup is required to have integer results when we divide width, height + * in display mode different from GST_DEWARP_PANORAMA. + * Additionally some elements such as xvimagesink have problems with arbitrary + * dimensions, a roundup solves this issue too + */ + *out_width = GST_ROUND_UP_8 ((gint) ((2.0 * G_PI) * ((r2 + r1) / 2.0))); + *out_height = GST_ROUND_UP_8 ((gint) (r2 - r1)); + + if (filter->display_mode != GST_DEWARP_DISPLAY_PANORAMA) { + *out_width = *out_width / 2; + *out_height = *out_height * 2; + } + + /* if outer_radius and inner radius are very close then width and height + could be 0, we assume passtrough in this case + */ + if (G_UNLIKELY (*out_width == 0) || G_UNLIKELY (*out_height == 0)) { + GST_WARNING_OBJECT (filter, + "Invalid calculated dimensions, width: %" G_GINT32_FORMAT + " height: %" G_GINT32_FORMAT, *out_width, *out_height); + *out_width = in_width; + *out_height = in_height; + } + filter->pad_sink_width = in_width; + filter->pad_sink_height = in_height; + } else { + if (filter->pad_sink_width > 0) { + *out_width = filter->pad_sink_width; + } else { + *out_width = in_width; + } + if (filter->pad_sink_height > 0) { + *out_height = filter->pad_sink_height; + } else { + *out_height = in_height; + } + } + } + + GST_LOG_OBJECT (filter, + "Calculated dimensions: width %" G_GINT32_FORMAT " => %" G_GINT32_FORMAT + ", height %" G_GINT32_FORMAT " => %" G_GINT32_FORMAT " direction: %d", + in_width, *out_width, in_height, *out_height, direction); +} + +static GstCaps * +gst_dewarp_transform_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps) +{ + GstDewarp *dewarp = GST_DEWARP (trans); + + GstCaps *ret; + gint width, height; + guint i; + + ret = gst_caps_copy (caps); + + GST_OBJECT_LOCK (dewarp); + + for (i = 0; i < gst_caps_get_size (ret); i++) { + GstStructure *structure = gst_caps_get_structure (ret, i); + + if (gst_structure_get_int (structure, "width", &width) && + gst_structure_get_int (structure, "height", &height)) { + gint out_width, out_height; + gst_dewarp_calculate_dimensions (dewarp, direction, width, height, + &out_width, &out_height); + gst_structure_set (structure, "width", G_TYPE_INT, out_width, "height", + G_TYPE_INT, out_height, NULL); + } + } + + GST_OBJECT_UNLOCK (dewarp); + + if (filter_caps) { + GstCaps *intersection; + + GST_DEBUG_OBJECT (dewarp, "Using filter caps %" GST_PTR_FORMAT, + filter_caps); + + intersection = + gst_caps_intersect_full (filter_caps, ret, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (ret); + ret = intersection; + + GST_DEBUG_OBJECT (dewarp, "Intersection %" GST_PTR_FORMAT, ret); + } + + return ret; +} + +static gboolean +gst_dewarp_set_caps (GstOpencvVideoFilter * filter, + gint in_width, gint in_height, gint in_depth, gint in_channels, + gint out_width, gint out_height, gint out_depth, gint out_channels) +{ + GstDewarp *dewarp = GST_DEWARP (filter); + + GST_DEBUG_OBJECT (dewarp, + "Set new caps, in width: %" G_GINT32_FORMAT " in height: %" + G_GINT32_FORMAT " out width: %" G_GINT32_FORMAT " out height: %" + G_GINT32_FORMAT, in_width, in_height, out_width, out_height); + + GST_OBJECT_LOCK (dewarp); + + dewarp->in_width = in_width; + dewarp->in_height = in_height; + dewarp->out_width = out_width; + dewarp->out_height = out_height; + gst_dewarp_update_map (dewarp); + + GST_OBJECT_UNLOCK (dewarp); + + return TRUE; +} + +static GstFlowReturn +gst_dewarp_transform_frame (GstOpencvVideoFilter * btrans, GstBuffer * buffer, + IplImage * img, GstBuffer * outbuf, IplImage * outimg) +{ + GstDewarp *filter = GST_DEWARP (btrans); + GstFlowReturn ret; + + GST_OBJECT_LOCK (filter); + + if (img->width == filter->in_width + && img->height == filter->in_height + && outimg->width == filter->out_width + && outimg->height == filter->out_height) { + cv::Mat fisheye_image, dewarped_image; + int inter_mode; + + if (filter->need_map_update) { + GST_LOG_OBJECT (filter, "map update is needed"); + gst_dewarp_update_map (filter); + } + + switch (filter->interpolation_mode) { + case GST_DEWARP_INTER_NEAREST: + inter_mode = cv::INTER_NEAREST; + break; + case GST_DEWARP_INTER_LINEAR: + inter_mode = cv::INTER_LINEAR; + break; + case GST_DEWARP_INTER_CUBIC: + inter_mode = cv::INTER_CUBIC; + break; + case GST_DEWARP_INTER_LANCZOS4: + inter_mode = cv::INTER_LANCZOS4; + break; + default: + inter_mode = cv::INTER_LINEAR; + break; + } + + fisheye_image = cv::cvarrToMat (img, false); + dewarped_image = cv::cvarrToMat (outimg, false); + + if (filter->display_mode == GST_DEWARP_DISPLAY_PANORAMA) { + cv::remap (fisheye_image, dewarped_image, filter->map_x, filter->map_y, + inter_mode); + } else if (filter->display_mode == GST_DEWARP_DISPLAY_DOUBLE_PANORAMA) { + cv::Mat view1, view2, panorama_image, concatenated; + gint panorama_width, panorama_height; + panorama_width = filter->out_width * 2; + panorama_height = filter->out_height / 2; + cv::Size panoramaSize (panorama_width, panorama_height); + panorama_image.create (panoramaSize, fisheye_image.type ()); + cv::remap (fisheye_image, panorama_image, filter->map_x, filter->map_y, + inter_mode); + view1 = + panorama_image (cv::Rect (0, 0, filter->out_width, panorama_height)); + view2 = + panorama_image (cv::Rect (filter->out_width, 0, filter->out_width, + panorama_height)); + cv::vconcat (view1, view2, concatenated); + concatenated.copyTo (dewarped_image); + } else if (filter->display_mode == GST_DEWARP_DISPLAY_QUAD_VIEW) { + cv::Mat view1, view2, view3, view4, concat1, concat2, panorama_image, + concatenated; + gint panorama_width, panorama_height; + gint view_width, view_height; + panorama_width = filter->out_width * 2; + panorama_height = filter->out_height / 2; + view_width = filter->out_width / 2; + view_height = filter->out_height / 2; + cv::Size panoramaSize (panorama_width, panorama_height); + panorama_image.create (panoramaSize, fisheye_image.type ()); + cv::remap (fisheye_image, panorama_image, filter->map_x, filter->map_y, + inter_mode); + view1 = panorama_image (cv::Rect (0, 0, view_width, view_height)); + view2 = + panorama_image (cv::Rect (view_width, 0, view_width, view_height)); + view3 = + panorama_image (cv::Rect ((view_width * 2), 0, view_width, + view_height)); + view4 = + panorama_image (cv::Rect ((view_width * 3), 0, view_width, + view_height)); + cv::vconcat (view1, view2, concat1); + cv::vconcat (view3, view4, concat2); + cv::hconcat (concat1, concat2, concatenated); + concatenated.copyTo (dewarped_image); + } + + ret = GST_FLOW_OK; + } else { + GST_WARNING_OBJECT (filter, "Frame dropped, dimensions do not match"); + + ret = GST_BASE_TRANSFORM_FLOW_DROPPED; + } + + GST_OBJECT_UNLOCK (filter); + + return ret; +} + +gboolean +gst_dewarp_plugin_init (GstPlugin * plugin) +{ + GST_DEBUG_CATEGORY_INIT (gst_dewarp_debug, "dewarp", + 0, "Dewarp fisheye images"); + + return gst_element_register (plugin, "dewarp", GST_RANK_NONE, + GST_TYPE_DEWARP); +} diff --git a/ext/opencv/gstdewarp.h b/ext/opencv/gstdewarp.h new file mode 100644 index 0000000000..545a2800bc --- /dev/null +++ b/ext/opencv/gstdewarp.h @@ -0,0 +1,114 @@ +/* + * GStreamer + * Copyright (C) 2016 Prassel S.r.l + * Author: Nicola Murino + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Alternatively, the contents of this file may be used under the + * GNU Lesser General Public License Version 2.1 (the "LGPL"), in + * which case the following provisions apply instead of the ones + * mentioned above: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_DEWARP_H__ +#define __GST_DEWARP_H__ + +#include +#include +#include +#include + +G_BEGIN_DECLS +/* #defines don't like whitespacey bits */ +#define GST_TYPE_DEWARP \ + (gst_dewarp_get_type()) +#define GST_DEWARP(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_DEWARP,GstDewarp)) +#define GST_DEWARP_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_DEWARP,GstDewarpClass)) +#define GST_IS_DEWARP(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_DEWARP)) +#define GST_IS_DEWARP_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_DEWARP)) +typedef struct _GstDewarp GstDewarp; +typedef struct _GstDewarpClass GstDewarpClass; + +enum _GstDewarpDisplayMode { + GST_DEWARP_DISPLAY_PANORAMA = 0, + GST_DEWARP_DISPLAY_DOUBLE_PANORAMA = 1, + GST_DEWARP_DISPLAY_QUAD_VIEW = 2 +}; + +enum _GstDewarpInterpolationMode { + GST_DEWARP_INTER_NEAREST = 0, + GST_DEWARP_INTER_LINEAR = 1, + GST_DEWARP_INTER_CUBIC = 2, + GST_DEWARP_INTER_LANCZOS4 = 3 +}; + +struct _GstDewarp +{ + GstOpencvVideoFilter element; + cv::Mat map_x; + cv::Mat map_y; + gdouble x_center; + gdouble y_center; + gdouble inner_radius; + gdouble outer_radius; + gdouble remap_correction_x; + gdouble remap_correction_y; + gboolean need_map_update; + gint pad_sink_width; + gint pad_sink_height; + gint in_width; + gint in_height; + gint out_width; + gint out_height; + gint display_mode; + gint interpolation_mode; +}; + +struct _GstDewarpClass +{ + GstOpencvVideoFilterClass parent_class; +}; + +GType gst_dewarp_get_type (void); + +gboolean gst_dewarp_plugin_init (GstPlugin * plugin); + +G_END_DECLS +#endif /* __GST_DEWARP_H__ */ + diff --git a/ext/opencv/gstopencv.cpp b/ext/opencv/gstopencv.cpp index 8f50cd88a5..522d36e5c6 100644 --- a/ext/opencv/gstopencv.cpp +++ b/ext/opencv/gstopencv.cpp @@ -41,6 +41,7 @@ #include "gstsegmentation.h" #include "gstgrabcut.h" #include "gstdisparity.h" +#include "gstdewarp.h" static gboolean plugin_init (GstPlugin * plugin) @@ -99,6 +100,9 @@ plugin_init (GstPlugin * plugin) if (!gst_disparity_plugin_init (plugin)) return FALSE; + if (!gst_dewarp_plugin_init (plugin)) + return FALSE; + return TRUE; } diff --git a/ext/opencv/meson.build b/ext/opencv/meson.build index 1ccd38d28b..59002c7d6f 100644 --- a/ext/opencv/meson.build +++ b/ext/opencv/meson.build @@ -20,7 +20,8 @@ gstopencv_sources = [ 'gsttemplatematch.cpp', 'gsttextoverlay.cpp', 'MotionCells.cpp', - 'motioncells_wrapper.cpp' + 'motioncells_wrapper.cpp', + 'gstdewarp.cpp' ] libopencv2_headers = [