mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-26 19:51:11 +00:00
Code Issues 1 Projects Releases Wiki Activity

opencv: new cameracalibrate and cameraundistort elements

Browse source 
https://bugzilla.gnome.org/show_bug.cgi?id=789075
This commit is contained in:
Philippe Renon 2017-10-16 22:29:01 +02:00 committed by Nicolas Dufresne
parent 8ede252a8a
commit f460448198
11 changed files with 1584 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
ext/opencv/Makefile.am
View file

@ -23,7 +23,11 @@ libgstopencv_la_SOURCES = gstopencv.cpp \
gstdisparity.cpp \
motioncells_wrapper.cpp \
MotionCells.cpp \
gstdewarp.cpp
gstdewarp.cpp \
camerautils.cpp \
cameraevent.cpp \
gstcameracalibrate.cpp \
gstcameraundistort.cpp
libgstopencv_la_CXXFLAGS = \
-I$(top_srcdir)/gst-libs \
@ -72,6 +76,10 @@ noinst_HEADERS = \
motioncells_wrapper.h \
MotionCells.h \
gstdewarp.h
camerautils.hpp \
cameraevent.hpp \
gstcameracalibrate.hpp \
gstcameraundistort.hpp
opencv_haarcascadesdir = $(pkgdatadir)/$(GST_API_VERSION)/opencv_haarcascades
opencv_haarcascades_DATA = fist.xml palm.xml

83
ext/opencv/cameraevent.cpp Normal file
View file

@ -0,0 +1,83 @@
/* GStreamer
* Copyright (C) <2017> Philippe Renon <philippe_renon@yahoo.fr>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include "cameraevent.hpp"
#include <opencv2/opencv.hpp>
/**
* gst_camera_event_new_calibrated:
* @settings: .
*
* Creates a new calibrated event.
*
* To parse an event created by gst_camera_event_new_calibrated() use
* gst_camera_event_parse_calibrated().
*
* Returns: The new GstEvent
*/
GstEvent *
gst_camera_event_new_calibrated (gchar * settings)
{
GstEvent *calibrated_event;
GstStructure *s;
s = gst_structure_new (GST_CAMERA_EVENT_CALIBRATED_NAME,
"undistort-settings", G_TYPE_STRING, g_strdup(settings), NULL);
calibrated_event = gst_event_new_custom (GST_EVENT_CUSTOM_BOTH, s);
return calibrated_event;
}
/**
* gst_camera_event_parse_calibrated:
* @event: A #GstEvent to parse
* @in_still: A boolean to receive the still-frame status from the event, or NULL
*
* Parse a #GstEvent, identify if it is a calibrated event, and
* return the s.
*
* Create a calibrated event using gst_camera_event_new_calibrated()
*
* Returns: %TRUE if the event is a valid calibrated event. %FALSE if not
*/
gboolean
gst_camera_event_parse_calibrated (GstEvent * event, gchar ** settings)
{
const GstStructure *s;
g_return_val_if_fail (event != NULL, FALSE);
if (GST_EVENT_TYPE (event) != GST_EVENT_CUSTOM_BOTH)
return FALSE; /* Not a calibrated event */
s = gst_event_get_structure (event);
if (s == NULL
|| !gst_structure_has_name (s, GST_CAMERA_EVENT_CALIBRATED_NAME))
return FALSE; /* Not a calibrated event */
const gchar *str = gst_structure_get_string(s, "undistort-settings");
if (!str)
return FALSE; /* Not calibrated frame event */
*settings = g_strdup (str);
return TRUE;
}

37
ext/opencv/cameraevent.hpp Normal file
View file

@ -0,0 +1,37 @@
/* GStreamer
* Copyright (C) <2017> Philippe Renon <philippe_renon@yahoo.fr>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_CAMERA_EVENT_H__
#define __GST_CAMERA_EVENT_H__
#include <gst/gst.h>
G_BEGIN_DECLS
#define GST_CAMERA_EVENT_CALIBRATED_NAME "GstEventCalibrated"
/* camera calibration event creation and parsing */
GstEvent *gst_camera_event_new_calibrated (gchar * settings);
gboolean gst_camera_event_parse_calibrated (GstEvent * event, gchar ** settings);
G_END_DECLS
#endif /* __GST_CAMERA_EVENT_H__ */

43
ext/opencv/camerautils.cpp Normal file
View file

@ -0,0 +1,43 @@
/* GStreamer
* Copyright (C) <2017> Philippe Renon <philippe_renon@yahoo.fr>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include "camerautils.hpp"
#include <opencv2/opencv.hpp>
gchar *
camera_serialize_undistort_settings (cv::Mat &cameraMatrix, cv::Mat &distCoeffs)
{
cv::FileStorage fs(".xml", cv::FileStorage::WRITE + cv::FileStorage::MEMORY);
fs << "cameraMatrix" << cameraMatrix;
fs << "distCoeffs" << distCoeffs;
std::string buf = fs.releaseAndGetString();
return g_strdup(buf.c_str());
}
gboolean
camera_deserialize_undistort_settings (gchar * str, cv::Mat &cameraMatrix, cv::Mat &distCoeffs)
{
cv::FileStorage fs(str, cv::FileStorage::READ + cv::FileStorage::MEMORY);
fs["cameraMatrix"] >> cameraMatrix;
fs["distCoeffs"] >> distCoeffs;
return TRUE;
}

34
ext/opencv/camerautils.hpp Normal file
View file

@ -0,0 +1,34 @@
/* GStreamer
* Copyright (C) <2017> Philippe Renon <philippe_renon@yahoo.fr>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_CAMERA_UTILS_H__
#define __GST_CAMERA_UTILS_H__
#include <gst/gst.h>
#include <cv.h>
G_BEGIN_DECLS
gchar *camera_serialize_undistort_settings (cv::Mat &cameraMatrix, cv::Mat &distCoeffs);
gboolean camera_deserialize_undistort_settings (gchar *str, cv::Mat &cameraMatrix, cv::Mat &distCoeffs);
G_END_DECLS
#endif /* __GST_CAMERA_UTILS_H__ */

730
ext/opencv/gstcameracalibrate.cpp Normal file
View file

@ -0,0 +1,730 @@
/*
* GStreamer
* Copyright (C) <2017> Philippe Renon <philippe_renon@yahoo.fr>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Alternatively, the contents of this file may be used under the
* GNU Lesser General Public License Version 2.1 (the "LGPL"), in
* which case the following provisions apply instead of the ones
* mentioned above:
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-cameraundistort
*
* This element performs camera calibration.
*
* Once the calibration procedure is done:
* - An event, containing the camera correction parameters, is sent upstream
* and downstream to be consumed by cameraundistort elements.
* - The _settings_ property is set to the camera correction parameters (as
* an opaque string of serialized OpenCV objects).
* The value of this property can later be used to configure a
* cameraundistort element.
* - The element becomes idle and can later be restarted [TODO].
*
* Based on this tutorial: https://docs.opencv.org/2.4/doc/tutorials/calib3d/camera_calibration/camera_calibration.html
*
* <refsect2>
* <title>Example pipelines</title>
* |[
* gst-launch-1.0 -v v4l2src ! videoconvert ! cameraundistort ! cameracalibrate | autovideosink
* ]| will correct camera distortion once camera calibration is done.
* </refsect2>
*/
/*
* TODO
* - signal when calibration is done
* - action signal to start calibration
* - do pattern detection asynchronously
* - do final calibration computation asynchronously
* - use cairo for drawing overlay
* - use overlay
* - implement settings query
* - validate user settings (see validate() in tutorial)
* - save complete state (see saveCameraParams() in tutorial)
*/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include "gstcameracalibrate.h"
#if (CV_MAJOR_VERSION >= 3)
#include <opencv2/imgproc.hpp>
#endif
#include <opencv2/calib3d.hpp>
#include <gst/opencv/gstopencvutils.h>
#include "camerautils.hpp"
#include "cameraevent.hpp"
#include <vector>
GST_DEBUG_CATEGORY_STATIC (gst_camera_calibrate_debug);
#define GST_CAT_DEFAULT gst_camera_calibrate_debug
#define DEFAULT_CALIBRATON_PATTERN GST_CAMERA_CALIBRATION_PATTERN_CHESSBOARD
#define DEFAULT_BOARD_WIDTH 9
#define DEFAULT_BOARD_HEIGHT 6
#define DEFAULT_SQUARE_SIZE 50
#define DEFAULT_ASPECT_RATIO 1.0
#define DEFAULT_CORNER_SUB_PIXEL true
#define DEFAULT_ZERO_TANGENT_DISTORTION FALSE
#define DEFAULT_CENTER_PRINCIPAL_POINT FALSE
#define DEFAULT_USE_FISHEYE FALSE
#define DEFAULT_FRAME_COUNT 25
#define DEFAULT_DELAY 350
#define DEFAULT_SHOW_CORNERS true
enum
{
PROP_0,
PROP_CALIBRATON_PATTERN,
PROP_BOARD_WIDTH,
PROP_BOARD_HEIGHT,
PROP_SQUARE_SIZE,
PROP_ASPECT_RATIO,
PROP_CORNER_SUB_PIXEL,
PROP_ZERO_TANGENT_DISTORTION,
PROP_CENTER_PRINCIPAL_POINT,
PROP_USE_FISHEYE,
PROP_FRAME_COUNT,
PROP_DELAY,
PROP_SHOW_CORNERS,
PROP_SETTINGS
};
enum {
DETECTION = 0,
CAPTURING = 1,
CALIBRATED = 2
};
#define GST_TYPE_CAMERA_CALIBRATION_PATTERN (camera_calibration_pattern_get_type ())
static GType
camera_calibration_pattern_get_type (void)
{
static GType camera_calibration_pattern_type = 0;
static const GEnumValue camera_calibration_pattern[] = {
{GST_CAMERA_CALIBRATION_PATTERN_CHESSBOARD, "Chessboard", "chessboard"},
{GST_CAMERA_CALIBRATION_PATTERN_CIRCLES_GRID, "Circle Grids", "circle_grids"},
{GST_CAMERA_CALIBRATION_PATTERN_ASYMMETRIC_CIRCLES_GRID, "Asymmetric Circle Grids", "asymmetric_circle_grids"},
{0, NULL, NULL},
};
if (!camera_calibration_pattern_type) {
camera_calibration_pattern_type =
g_enum_register_static ("GstCameraCalibrationPattern", camera_calibration_pattern);
}
return camera_calibration_pattern_type;
}
G_DEFINE_TYPE (GstCameraCalibrate, gst_camera_calibrate, GST_TYPE_OPENCV_VIDEO_FILTER);
static void gst_camera_calibrate_dispose (GObject * object);
static void gst_camera_calibrate_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_camera_calibrate_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static GstFlowReturn gst_camera_calibrate_transform_frame_ip (
GstOpencvVideoFilter * cvfilter, GstBuffer * frame, IplImage * img);
/* clean up */
static void
gst_camera_calibrate_finalize (GObject * obj)
{
G_OBJECT_CLASS (gst_camera_calibrate_parent_class)->finalize (obj);
}
/* initialize the cameracalibration's class */
static void
gst_camera_calibrate_class_init (GstCameraCalibrateClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstOpencvVideoFilterClass *opencvfilter_class = GST_OPENCV_VIDEO_FILTER_CLASS (klass);
GstCaps *caps;
GstPadTemplate *templ;
gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_camera_calibrate_finalize);
gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_camera_calibrate_dispose);
gobject_class->set_property = gst_camera_calibrate_set_property;
gobject_class->get_property = gst_camera_calibrate_get_property;
opencvfilter_class->cv_trans_ip_func =
gst_camera_calibrate_transform_frame_ip;
g_object_class_install_property (gobject_class, PROP_CALIBRATON_PATTERN,
g_param_spec_enum ("pattern", "Calibration Pattern",
"One of the chessboard, circles, or asymmetric circle pattern",
GST_TYPE_CAMERA_CALIBRATION_PATTERN, DEFAULT_CALIBRATON_PATTERN,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_BOARD_WIDTH,
g_param_spec_int ("board-width", "Board Width",
"The board width in number of items",
1, G_MAXINT, DEFAULT_BOARD_WIDTH,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_BOARD_HEIGHT,
g_param_spec_int ("board-height", "Board Height",
"The board height in number of items",
1, G_MAXINT, DEFAULT_BOARD_WIDTH,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_SQUARE_SIZE,
g_param_spec_float ("square-size", "Square Size",
"The size of a square in your defined unit (point, millimeter, etc.)",
0.0, G_MAXFLOAT, DEFAULT_SQUARE_SIZE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_ASPECT_RATIO,
g_param_spec_float ("aspect-ratio", "Aspect Ratio",
"The aspect ratio",
0.0, G_MAXFLOAT, DEFAULT_ASPECT_RATIO,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_CORNER_SUB_PIXEL,
g_param_spec_boolean ("corner-sub-pixel", "Corner Sub Pixel",
"Improve corner detection accuracy for chessboard",
DEFAULT_CORNER_SUB_PIXEL, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_ZERO_TANGENT_DISTORTION,
g_param_spec_boolean ("zero-tangent-distorsion", "Zero Tangent Distorsion",
"Assume zero tangential distortion",
DEFAULT_ZERO_TANGENT_DISTORTION, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_CENTER_PRINCIPAL_POINT,
g_param_spec_boolean ("center-principal-point", "Center Principal Point",
"Fix the principal point at the center",
DEFAULT_CENTER_PRINCIPAL_POINT, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_USE_FISHEYE,
g_param_spec_boolean ("use-fisheye", "Use Fisheye",
"Use fisheye camera model for calibration",
DEFAULT_USE_FISHEYE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_DELAY,
g_param_spec_int ("delay", "Delay",
"Sampling periodicity in ms", 0, G_MAXINT,
DEFAULT_DELAY,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_FRAME_COUNT,
g_param_spec_int ("frame-count", "Frame Count",
"The number of frames to use from the input for calibration", 1, G_MAXINT,
DEFAULT_FRAME_COUNT,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_SHOW_CORNERS,
g_param_spec_boolean ("show-corners", "Show Corners",
"Show corners",
DEFAULT_SHOW_CORNERS, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_SETTINGS,
g_param_spec_string ("settings", "Settings",
"Camera correction parameters (opaque string of serialized OpenCV objects)",
NULL, (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
gst_element_class_set_static_metadata (element_class,
"cameracalibrate",
"Filter/Effect/Video",
"Performs camera calibration",
"Philippe Renon <philippe_renon@yahoo.fr>");
/* add sink and source pad templates */
caps = gst_opencv_caps_from_cv_image_type (CV_8UC4);
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC3));
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC1));
templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
gst_caps_ref (caps));
gst_element_class_add_pad_template (element_class, templ);
templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps);
gst_element_class_add_pad_template (element_class, templ);
}
/* initialize the new element
* initialize instance structure
*/
static void
gst_camera_calibrate_init (GstCameraCalibrate * calib)
{
calib->calibrationPattern = DEFAULT_CALIBRATON_PATTERN;
calib->boardSize.width = DEFAULT_BOARD_WIDTH;
calib->boardSize.height = DEFAULT_BOARD_HEIGHT;
calib->squareSize = DEFAULT_SQUARE_SIZE;
calib->aspectRatio = DEFAULT_ASPECT_RATIO;
calib->cornerSubPix = DEFAULT_CORNER_SUB_PIXEL;
calib->calibZeroTangentDist = DEFAULT_ZERO_TANGENT_DISTORTION;
calib->calibFixPrincipalPoint = DEFAULT_CENTER_PRINCIPAL_POINT;
calib->useFisheye = DEFAULT_USE_FISHEYE;
calib->nrFrames = DEFAULT_FRAME_COUNT;
calib->delay = DEFAULT_DELAY;
calib->showCorners = DEFAULT_SHOW_CORNERS;
calib->flags = cv::CALIB_FIX_K4 | cv::CALIB_FIX_K5;
if (calib->calibFixPrincipalPoint) calib->flags |= cv::CALIB_FIX_PRINCIPAL_POINT;
if (calib->calibZeroTangentDist) calib->flags |= cv::CALIB_ZERO_TANGENT_DIST;
if (calib->aspectRatio) calib->flags |= cv::CALIB_FIX_ASPECT_RATIO;
if (calib->useFisheye) {
/* the fisheye model has its own enum, so overwrite the flags */
calib->flags = cv::fisheye::CALIB_FIX_SKEW | cv::fisheye::CALIB_RECOMPUTE_EXTRINSIC |
/*cv::fisheye::CALIB_FIX_K1 |*/
cv::fisheye::CALIB_FIX_K2 | cv::fisheye::CALIB_FIX_K3 | cv::fisheye::CALIB_FIX_K4;
}
calib->mode = CAPTURING; //DETECTION;
calib->prevTimestamp = 0;
calib->imagePoints.clear();
calib->cameraMatrix = 0;
calib->distCoeffs = 0;
calib->settings = NULL;
gst_opencv_video_filter_set_in_place (
GST_OPENCV_VIDEO_FILTER_CAST (calib), TRUE);
}
static void
gst_camera_calibrate_dispose (GObject * object)
{
GstCameraCalibrate *calib = GST_CAMERA_CALIBRATE (object);
g_free (calib->settings);
calib->settings = NULL;
G_OBJECT_CLASS (gst_camera_calibrate_parent_class)->dispose (object);
}
static void
gst_camera_calibrate_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstCameraCalibrate *calib = GST_CAMERA_CALIBRATE (object);
switch (prop_id) {
case PROP_CALIBRATON_PATTERN:
calib->calibrationPattern = g_value_get_enum (value);
break;
case PROP_BOARD_WIDTH:
calib->boardSize.width = g_value_get_int (value);
break;
case PROP_BOARD_HEIGHT:
calib->boardSize.height = g_value_get_int (value);
break;
case PROP_SQUARE_SIZE:
calib->squareSize = g_value_get_float (value);
break;
case PROP_ASPECT_RATIO:
calib->aspectRatio = g_value_get_float (value);
break;
case PROP_CORNER_SUB_PIXEL:
calib->cornerSubPix = g_value_get_boolean (value);
break;
case PROP_ZERO_TANGENT_DISTORTION:
calib->calibZeroTangentDist = g_value_get_boolean (value);
break;
case PROP_CENTER_PRINCIPAL_POINT:
calib->calibFixPrincipalPoint = g_value_get_boolean (value);
break;
case PROP_USE_FISHEYE:
calib->useFisheye = g_value_get_boolean (value);
break;
case PROP_FRAME_COUNT:
calib->nrFrames = g_value_get_int (value);
break;
case PROP_DELAY:
calib->delay = g_value_get_int (value);
break;
case PROP_SHOW_CORNERS:
calib->showCorners = g_value_get_boolean (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_camera_calibrate_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstCameraCalibrate *calib = GST_CAMERA_CALIBRATE (object);
switch (prop_id) {
case PROP_CALIBRATON_PATTERN:
g_value_set_enum (value, calib->calibrationPattern);
break;
case PROP_BOARD_WIDTH:
g_value_set_int (value, calib->boardSize.width);
break;
case PROP_BOARD_HEIGHT:
g_value_set_int (value, calib->boardSize.height);
break;
case PROP_SQUARE_SIZE:
g_value_set_float (value, calib->squareSize);
break;
case PROP_ASPECT_RATIO:
g_value_set_float (value, calib->aspectRatio);
break;
case PROP_CORNER_SUB_PIXEL:
g_value_set_boolean (value, calib->cornerSubPix);
break;
case PROP_ZERO_TANGENT_DISTORTION:
g_value_set_boolean (value, calib->calibZeroTangentDist);
break;
case PROP_CENTER_PRINCIPAL_POINT:
g_value_set_boolean (value, calib->calibFixPrincipalPoint);
break;
case PROP_USE_FISHEYE:
g_value_set_boolean (value, calib->useFisheye);
break;
case PROP_FRAME_COUNT:
g_value_set_int (value, calib->nrFrames);
break;
case PROP_DELAY:
g_value_set_int (value, calib->delay);
break;
case PROP_SHOW_CORNERS:
g_value_set_boolean (value, calib->showCorners);
break;
case PROP_SETTINGS:
g_value_set_string (value, calib->settings);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
void camera_calibrate_run(GstCameraCalibrate *calib, IplImage *img);
/*
* Performs the camera calibration
*/
static GstFlowReturn
gst_camera_calibrate_transform_frame_ip (GstOpencvVideoFilter * cvfilter,
G_GNUC_UNUSED GstBuffer * frame, IplImage * img)
{
GstCameraCalibrate *calib = GST_CAMERA_CALIBRATE (cvfilter);
camera_calibrate_run(calib, img);
return GST_FLOW_OK;
}
bool camera_calibrate_calibrate(GstCameraCalibrate *calib,
cv::Size imageSize, cv::Mat& cameraMatrix, cv::Mat& distCoeffs,
std::vector<std::vector<cv::Point2f> > imagePoints );
void camera_calibrate_run(GstCameraCalibrate *calib, IplImage *img)
{
cv::Mat view = cv::cvarrToMat(img);
// For camera only take new samples after delay time
if (calib->mode == CAPTURING) {
// get_input
cv::Size imageSize = view.size();
/* find_pattern
* FIXME find ways to reduce CPU usage
* don't do it on all frames ? will it help ? corner display will be affected.
* in a separate frame?
* in a separate element that gets composited back into the main stream
* (video is tee-d into it and can then be decimated, scaled, etc..) */
std::vector<cv::Point2f> pointBuf;
bool found;
int chessBoardFlags = cv::CALIB_CB_ADAPTIVE_THRESH | cv::CALIB_CB_NORMALIZE_IMAGE;
if (!calib->useFisheye) {
/* fast check erroneously fails with high distortions like fisheye */
chessBoardFlags |= cv::CALIB_CB_FAST_CHECK;
}
/* Find feature points on the input format */
switch(calib->calibrationPattern) {
case GST_CAMERA_CALIBRATION_PATTERN_CHESSBOARD:
found = cv::findChessboardCorners(view, calib->boardSize, pointBuf, chessBoardFlags);
break;
case GST_CAMERA_CALIBRATION_PATTERN_CIRCLES_GRID:
found = cv::findCirclesGrid(view, calib->boardSize, pointBuf);
break;
case GST_CAMERA_CALIBRATION_PATTERN_ASYMMETRIC_CIRCLES_GRID:
found = cv::findCirclesGrid(view, calib->boardSize, pointBuf, cv::CALIB_CB_ASYMMETRIC_GRID );
break;
default:
found = FALSE;
break;
}
bool blinkOutput = FALSE;
if (found) {
/* improve the found corners' coordinate accuracy for chessboard */
if (calib->calibrationPattern == GST_CAMERA_CALIBRATION_PATTERN_CHESSBOARD && calib->cornerSubPix) {
/* FIXME findChessboardCorners and alike do a cv::COLOR_BGR2GRAY (and a histogram balance)
* the color convert should be done once (if needed) and shared
* FIXME keep viewGray around to avoid reallocating it each time... */
cv::Mat viewGray;
cv::cvtColor(view, viewGray, cv::COLOR_BGR2GRAY);
cv::cornerSubPix(viewGray, pointBuf, cv::Size(11, 11), cv::Size(-1, -1),
cv::TermCriteria(cv::TermCriteria::EPS + cv::TermCriteria::COUNT, 30, 0.1));
}
/* take new samples after delay time */
if ((calib->mode == CAPTURING) && ((clock() - calib->prevTimestamp) > calib->delay * 1e-3 * CLOCKS_PER_SEC)) {
calib->imagePoints.push_back(pointBuf);
calib->prevTimestamp = clock();
blinkOutput = true;
}
/* draw the corners */
if (calib->showCorners) {
cv::drawChessboardCorners(view, calib->boardSize, cv::Mat(pointBuf), found);
}
}
/* if got enough frames then stop calibration and show result */
if (calib->mode == CAPTURING && calib->imagePoints.size() >= (size_t)calib->nrFrames) {
if (camera_calibrate_calibrate(calib, imageSize, calib->cameraMatrix, calib->distCoeffs, calib->imagePoints)) {
calib->mode = CALIBRATED;
GstPad *sink_pad = GST_BASE_TRANSFORM_SINK_PAD (calib);
GstPad *src_pad = GST_BASE_TRANSFORM_SRC_PAD (calib);
GstEvent *sink_event;
GstEvent *src_event;
/* set settings property */
g_free (calib->settings);
calib->settings = camera_serialize_undistort_settings(calib->cameraMatrix, calib->distCoeffs);
/* create calibrated event and send upstream and downstream */
sink_event = gst_camera_event_new_calibrated (calib->settings);
GST_LOG_OBJECT (sink_pad, "Sending upstream event %s.", GST_EVENT_TYPE_NAME (sink_event));
if (!gst_pad_push_event (sink_pad, sink_event)) {
GST_WARNING_OBJECT (sink_pad, "Sending upstream event %p (%s) failed.",
sink_event, GST_EVENT_TYPE_NAME (sink_event));
}
src_event = gst_camera_event_new_calibrated (calib->settings);
GST_LOG_OBJECT (src_pad, "Sending downstream event %s.", GST_EVENT_TYPE_NAME (src_event));
if (!gst_pad_push_event (src_pad, src_event)) {
GST_WARNING_OBJECT (src_pad, "Sending downstream event %p (%s) failed.",
src_event, GST_EVENT_TYPE_NAME (src_event));
}
} else {
/* failed to calibrate, go back to detection mode */
calib->mode = DETECTION;
}
}
if (calib->mode == CAPTURING && blinkOutput) {
bitwise_not(view, view);
}
}
/* output text */
/* FIXME ll additional rendering (text, corners, ...) should be done with
* cairo or another gst framework.
* this will relax the conditions on the input format (RBG only at the moment).
* the calibration itself accepts more formats... */
std::string msg = (calib->mode == CAPTURING) ? "100/100" :
(calib->mode == CALIBRATED) ? "Calibrated" : "Waiting...";
int baseLine = 0;
cv::Size textSize = cv::getTextSize(msg, 1, 1, 1, &baseLine);
cv::Point textOrigin(view.cols - 2 * textSize.width - 10, view.rows - 2 * baseLine - 10);
if (calib->mode == CAPTURING) {
msg = cv::format("%d/%d", (int)calib->imagePoints.size(), calib->nrFrames);
}
const cv::Scalar RED(0,0,255);
const cv::Scalar GREEN(0,255,0);
cv::putText(view, msg, textOrigin, 1, 1, calib->mode == CALIBRATED ? GREEN : RED);
}
static double camera_calibrate_calc_reprojection_errors (
const std::vector<std::vector<cv::Point3f> >& objectPoints,
const std::vector<std::vector<cv::Point2f> >& imagePoints,
const std::vector<cv::Mat>& rvecs, const std::vector<cv::Mat>& tvecs,
const cv::Mat& cameraMatrix , const cv::Mat& distCoeffs,
std::vector<float>& perViewErrors, bool fisheye)
{
std::vector<cv::Point2f> imagePoints2;
size_t totalPoints = 0;
double totalErr = 0, err;
perViewErrors.resize(objectPoints.size());
for(size_t i = 0; i < objectPoints.size(); ++i)
{
if (fisheye)
{
cv::fisheye::projectPoints(objectPoints[i], imagePoints2,
rvecs[i], tvecs[i], cameraMatrix, distCoeffs);
}
else
{
cv::projectPoints(objectPoints[i], rvecs[i], tvecs[i],
cameraMatrix, distCoeffs, imagePoints2);
}
err = cv::norm(imagePoints[i], imagePoints2, cv::NORM_L2);
size_t n = objectPoints[i].size();
perViewErrors[i] = (float) std::sqrt(err*err/n);
totalErr += err*err;
totalPoints += n;
}
return std::sqrt(totalErr/totalPoints);
}
static void camera_calibrate_calc_corners (cv::Size boardSize, float squareSize,
std::vector<cv::Point3f>& corners, gint patternType /*= CHESSBOARD*/)
{
corners.clear();
switch(patternType) {
case GST_CAMERA_CALIBRATION_PATTERN_CHESSBOARD:
case GST_CAMERA_CALIBRATION_PATTERN_CIRCLES_GRID:
for( int i = 0; i < boardSize.height; ++i)
for( int j = 0; j < boardSize.width; ++j)
corners.push_back(cv::Point3f(j * squareSize, i * squareSize, 0));
break;
case GST_CAMERA_CALIBRATION_PATTERN_ASYMMETRIC_CIRCLES_GRID:
for( int i = 0; i < boardSize.height; i++)
for( int j = 0; j < boardSize.width; j++)
corners.push_back(cv::Point3f((2 * j + i % 2) * squareSize, i * squareSize, 0));
break;
default:
break;
}
}
static bool camera_calibrate_calibrate_full(GstCameraCalibrate *calib,
cv::Size& imageSize, cv::Mat& cameraMatrix, cv::Mat& distCoeffs,
std::vector<std::vector<cv::Point2f> > imagePoints,
std::vector<cv::Mat>& rvecs, std::vector<cv::Mat>& tvecs,
std::vector<float>& reprojErrs, double& totalAvgErr)
{
cameraMatrix = cv::Mat::eye(3, 3, CV_64F);
if (calib->flags & cv::CALIB_FIX_ASPECT_RATIO) {
cameraMatrix.at<double>(0,0) = calib->aspectRatio;
}
if (calib->useFisheye) {
distCoeffs = cv::Mat::zeros(4, 1, CV_64F);
} else {
distCoeffs = cv::Mat::zeros(8, 1, CV_64F);
}
std::vector<std::vector<cv::Point3f> > objectPoints(1);
camera_calibrate_calc_corners (calib->boardSize, calib->squareSize,
objectPoints[0], calib->calibrationPattern);
objectPoints.resize(imagePoints.size(), objectPoints[0]);
/* Find intrinsic and extrinsic camera parameters */
double rms;
if (calib->useFisheye) {
cv::Mat _rvecs, _tvecs;
rms = cv::fisheye::calibrate(objectPoints, imagePoints, imageSize,
cameraMatrix, distCoeffs, _rvecs, _tvecs, calib->flags);
rvecs.reserve(_rvecs.rows);
tvecs.reserve(_tvecs.rows);
for(int i = 0; i < int(objectPoints.size()); i++){
rvecs.push_back(_rvecs.row(i));
tvecs.push_back(_tvecs.row(i));
}
} else {
rms = cv::calibrateCamera(objectPoints, imagePoints, imageSize,
cameraMatrix, distCoeffs, rvecs, tvecs, calib->flags);
}
GST_LOG_OBJECT (calib,
"Re-projection error reported by calibrateCamera: %f", rms);
bool ok = checkRange(cameraMatrix) && checkRange(distCoeffs);
totalAvgErr = camera_calibrate_calc_reprojection_errors (objectPoints, imagePoints,
rvecs, tvecs, cameraMatrix, distCoeffs, reprojErrs, calib->useFisheye);
return ok;
}
bool camera_calibrate_calibrate(GstCameraCalibrate *calib,
cv::Size imageSize, cv::Mat& cameraMatrix, cv::Mat& distCoeffs,
std::vector<std::vector<cv::Point2f> > imagePoints)
{
std::vector<cv::Mat> rvecs, tvecs;
std::vector<float> reprojErrs;
double totalAvgErr = 0;
bool ok = camera_calibrate_calibrate_full(calib,
imageSize, cameraMatrix, distCoeffs, imagePoints,
rvecs, tvecs, reprojErrs, totalAvgErr);
GST_LOG_OBJECT (calib, (ok ? "Calibration succeeded" : "Calibration failed"));
/* + ". avg re projection error = " + totalAvgErr);*/
return ok;
}
/* entry point to initialize the plug-in
* initialize the plug-in itself
* register the element factories and other features
*/
gboolean
gst_camera_calibrate_plugin_init (GstPlugin * plugin)
{
/* debug category for filtering log messages */
GST_DEBUG_CATEGORY_INIT (gst_camera_calibrate_debug, "cameracalibrate",
0,
"Performs camera calibration");
return gst_element_register (plugin, "cameracalibrate", GST_RANK_NONE,
GST_TYPE_CAMERA_CALIBRATE);
}

113
ext/opencv/gstcameracalibrate.h Normal file
View file

@ -0,0 +1,113 @@
/*
* GStreamer
* Copyright (C) <2017> Philippe Renon <philippe_renon@yahoo.fr>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Alternatively, the contents of this file may be used under the
* GNU Lesser General Public License Version 2.1 (the "LGPL"), in
* which case the following provisions apply instead of the ones
* mentioned above:
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_CAMERA_CALIBRATE_H__
#define __GST_CAMERA_CALIBRATE_H__
#include <gst/gst.h>
#include <gst/opencv/gstopencvvideofilter.h>
#include <opencv2/core.hpp>
G_BEGIN_DECLS
#define GST_TYPE_CAMERA_CALIBRATE \
(gst_camera_calibrate_get_type())
#define GST_CAMERA_CALIBRATE(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_CAMERA_CALIBRATE,GstCameraCalibrate))
#define GST_CAMERA_CALIBRATE_CLASS(klass) \
(G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_CAMERA_CALIBRATE,GstCameraCalibrateClass))
#define GST_IS_CAMERA_CALIBRATE(obj) \
(G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_CAMERA_CALIBRATE))
#define GST_IS_CAMERA_CALIBRATE_CLASS(klass) \
(G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_CAMERA_CALIBRATE))
typedef struct _GstCameraCalibrate GstCameraCalibrate;
typedef struct _GstCameraCalibrateClass GstCameraCalibrateClass;
enum _GstCameraCalibrationPattern {
GST_CAMERA_CALIBRATION_PATTERN_CHESSBOARD,
GST_CAMERA_CALIBRATION_PATTERN_CIRCLES_GRID,
GST_CAMERA_CALIBRATION_PATTERN_ASYMMETRIC_CIRCLES_GRID
};
struct _GstCameraCalibrate
{
GstOpencvVideoFilter cvfilter;
// settings
gint calibrationPattern; // One of the chessboard, circles, or asymmetric circle pattern
cv::Size boardSize; // The size of the board -> Number of items by width and height
float squareSize; // The size of a square in your defined unit (point, millimeter,etc).
float aspectRatio; // The aspect ratio
bool cornerSubPix; //
bool calibZeroTangentDist; // Assume zero tangential distortion
bool calibFixPrincipalPoint; // Fix the principal point at the center
bool useFisheye; // use fisheye camera model for calibration
int nrFrames; // The number of frames to use from the input for calibration
int delay; // In case of a video input
bool showUndistorsed; // Show undistorted images after calibration
bool showCorners; // Show corners
// state
int flags;
int mode;
clock_t prevTimestamp;
std::vector<std::vector<cv::Point2f>> imagePoints;
cv::Mat cameraMatrix, distCoeffs;
// opaque string containing opencv calibration settings
gchar *settings;
};
struct _GstCameraCalibrateClass
{
GstOpencvVideoFilterClass parent_class;
};
GType gst_camera_calibrate_get_type (void);
gboolean gst_camera_calibrate_plugin_init (GstPlugin * plugin);
G_END_DECLS
#endif /* __GST_CAMERA_CALIBRATE_H__ */

418
ext/opencv/gstcameraundistort.cpp Normal file
View file

@ -0,0 +1,418 @@
/*
* GStreamer
* Copyright (C) <2017> Philippe Renon <philippe_renon@yahoo.fr>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Alternatively, the contents of this file may be used under the
* GNU Lesser General Public License Version 2.1 (the "LGPL"), in
* which case the following provisions apply instead of the ones
* mentioned above:
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-cameraundistort
*
* This element performs camera distortion correction.
*
* Camera correction settings are obtained by running through
* the camera calibration process with the cameracalibrate element.
*
* It is possible to do live correction and calibration by chaining
* a cameraundistort and a cameracalibrate element. The cameracalibrate
* will send an event with the correction parameters to the the cameraundistort.
*
* Based on this tutorial: https://docs.opencv.org/2.4/doc/tutorials/calib3d/camera_calibration/camera_calibration.html
*
* <refsect2>
* <title>Example pipelines</title>
* |[
* gst-launch-1.0 -v v4l2src ! videoconvert ! cameraundistort settings="???" ! autovideosink
* ]| will correct camera distortion based on provided settings.
* |[
* gst-launch-1.0 -v v4l2src ! videoconvert ! cameraundistort ! cameracalibrate | autovideosink
* ]| will correct camera distortion once camera calibration is done.
* </refsect2>
*/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include <vector>
#include "camerautils.hpp"
#include "cameraevent.hpp"
#include "gstcameraundistort.h"
#if (CV_MAJOR_VERSION >= 3)
#include <opencv2/imgproc.hpp>
#endif
#include <opencv2/calib3d.hpp>
#include <gst/opencv/gstopencvutils.h>
GST_DEBUG_CATEGORY_STATIC (gst_camera_undistort_debug);
#define GST_CAT_DEFAULT gst_camera_undistort_debug
#define DEFAULT_SHOW_UNDISTORTED TRUE
#define DEFAULT_ALPHA 0.0
#define DEFAULT_CROP FALSE
enum
{
PROP_0,
PROP_SHOW_UNDISTORTED,
PROP_ALPHA,
PROP_CROP,
PROP_SETTINGS
};
G_DEFINE_TYPE (GstCameraUndistort, gst_camera_undistort, GST_TYPE_OPENCV_VIDEO_FILTER);
static void gst_camera_undistort_dispose (GObject * object);
static void gst_camera_undistort_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_camera_undistort_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static gboolean gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter,
gint in_width, gint in_height, gint in_depth, gint in_channels,
gint out_width, gint out_height, gint out_depth, gint out_channels);
static GstFlowReturn gst_camera_undistort_transform_frame (
GstOpencvVideoFilter * cvfilter,
GstBuffer * frame, IplImage * img,
GstBuffer * outframe, IplImage * outimg);
static gboolean gst_camera_undistort_sink_event (GstBaseTransform *trans, GstEvent *event);
static gboolean gst_camera_undistort_src_event (GstBaseTransform *trans, GstEvent *event);
static void camera_undistort_run(GstCameraUndistort *undist, IplImage *img, IplImage *outimg);
static gboolean camera_undistort_init_undistort_rectify_map(GstCameraUndistort *undist);
/* initialize the cameraundistort's class */
static void
gst_camera_undistort_class_init (GstCameraUndistortClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass);
GstOpencvVideoFilterClass *opencvfilter_class = GST_OPENCV_VIDEO_FILTER_CLASS (klass);
GstCaps *caps;
GstPadTemplate *templ;
gobject_class->dispose = gst_camera_undistort_dispose;
gobject_class->set_property = gst_camera_undistort_set_property;
gobject_class->get_property = gst_camera_undistort_get_property;
trans_class->sink_event =
GST_DEBUG_FUNCPTR (gst_camera_undistort_sink_event);
trans_class->src_event =
GST_DEBUG_FUNCPTR (gst_camera_undistort_src_event);
opencvfilter_class->cv_set_caps = gst_camera_undistort_set_info;
opencvfilter_class->cv_trans_func =
gst_camera_undistort_transform_frame;
g_object_class_install_property (gobject_class, PROP_SHOW_UNDISTORTED,
g_param_spec_boolean ("undistort", "Apply camera corrections",
"Apply camera corrections",
DEFAULT_SHOW_UNDISTORTED, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_ALPHA,
g_param_spec_float ("alpha", "Pixels",
"Show all pixels (1), only valid ones (0) or something in between",
0.0, 1.0, DEFAULT_ALPHA,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_SETTINGS,
g_param_spec_string ("settings", "Settings",
"Camera correction parameters (opaque string of serialized OpenCV objects)",
NULL, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
gst_element_class_set_static_metadata (element_class,
"cameraundistort",
"Filter/Effect/Video",
"Performs camera undistort",
"Philippe Renon <philippe_renon@yahoo.fr>");
/* add sink and source pad templates */
caps = gst_opencv_caps_from_cv_image_type (CV_16UC1);
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC4));
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC3));
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC1));
templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
gst_caps_ref (caps));
gst_element_class_add_pad_template (element_class, templ);
templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps);
gst_element_class_add_pad_template (element_class, templ);
}
/* initialize the new element
* initialize instance structure
*/
static void
gst_camera_undistort_init (GstCameraUndistort * undist)
{
undist->showUndistorted = DEFAULT_SHOW_UNDISTORTED;
undist->alpha = DEFAULT_ALPHA;
undist->crop = DEFAULT_CROP;
undist->doUndistort = FALSE;
undist->settingsChanged = FALSE;
undist->cameraMatrix = 0;
undist->distCoeffs = 0;
undist->map1 = 0;
undist->map2 = 0;
undist->settings = NULL;
}
static void
gst_camera_undistort_dispose (GObject * object)
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
g_free (undist->settings);
undist->settings = NULL;
G_OBJECT_CLASS (gst_camera_undistort_parent_class)->dispose (object);
}
static void
gst_camera_undistort_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
const char *str;
switch (prop_id) {
case PROP_SHOW_UNDISTORTED:
undist->showUndistorted = g_value_get_boolean (value);
undist->settingsChanged = TRUE;
break;
case PROP_ALPHA:
undist->alpha = g_value_get_float (value);
undist->settingsChanged = TRUE;
break;
case PROP_CROP:
undist->crop = g_value_get_boolean (value);
break;
case PROP_SETTINGS:
if (undist->settings) {
g_free (undist->settings);
undist->settings = NULL;
}
str = g_value_get_string (value);
if (str)
undist->settings = g_strdup (str);
undist->settingsChanged = TRUE;
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_camera_undistort_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
switch (prop_id) {
case PROP_SHOW_UNDISTORTED:
g_value_set_boolean (value, undist->showUndistorted);
break;
case PROP_ALPHA:
g_value_set_float (value, undist->alpha);
break;
case PROP_CROP:
g_value_set_boolean (value, undist->crop);
break;
case PROP_SETTINGS:
g_value_set_string (value, undist->settings);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
gboolean
gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter,
gint in_width, gint in_height,
__attribute__((unused)) gint in_depth, __attribute__((unused)) gint in_channels,
__attribute__((unused)) gint out_width, __attribute__((unused)) gint out_height,
__attribute__((unused)) gint out_depth, __attribute__((unused)) gint out_channels)
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter);
undist->imageSize = cv::Size (in_width, in_height);
return TRUE;
}
/*
* Performs the camera undistort
*/
static GstFlowReturn
gst_camera_undistort_transform_frame (GstOpencvVideoFilter * cvfilter,
G_GNUC_UNUSED GstBuffer * frame, IplImage * img,
G_GNUC_UNUSED GstBuffer * outframe, IplImage * outimg)
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter);
camera_undistort_run (undist, img, outimg);
return GST_FLOW_OK;
}
static void
camera_undistort_run (GstCameraUndistort *undist, IplImage *img, IplImage *outimg)
{
const cv::Mat view = cv::cvarrToMat (img);
cv::Mat outview = cv::cvarrToMat (outimg);
/* TODO is settingsChanged handling thread safe ? */
if (undist->settingsChanged) {
/* settings have changed, need to recompute undistort */
undist->settingsChanged = FALSE;
undist->doUndistort = FALSE;
if (undist->showUndistorted && undist->settings) {
if (camera_deserialize_undistort_settings (
undist->settings, undist->cameraMatrix, undist->distCoeffs)) {
undist->doUndistort = camera_undistort_init_undistort_rectify_map (undist);
}
}
}
if (undist->showUndistorted && undist->doUndistort) {
/* do the undistort */
cv::remap (view, outview, undist->map1, undist->map2, cv::INTER_LINEAR);
if (undist->crop) {
/* TODO do the cropping */
const cv::Scalar CROP_COLOR (0, 255, 0);
cv::rectangle (outview, undist->validPixROI, CROP_COLOR);
}
}
else {
/* FIXME should use pass through to avoid this copy when not undistorting */
view.copyTo (outview);
}
}
/* compute undistort */
static gboolean
camera_undistort_init_undistort_rectify_map (GstCameraUndistort *undist)
{
cv::Size newImageSize;
cv::Rect validPixROI;
cv::Mat newCameraMatrix = cv::getOptimalNewCameraMatrix (
undist->cameraMatrix, undist->distCoeffs, undist->imageSize,
undist->alpha, newImageSize, &validPixROI);
undist->validPixROI = validPixROI;
cv::initUndistortRectifyMap (undist->cameraMatrix, undist->distCoeffs, cv::Mat(),
newCameraMatrix, undist->imageSize, CV_16SC2, undist->map1, undist->map2);
return TRUE;
}
static gboolean
camera_undistort_calibration_event (GstCameraUndistort *undist, GstEvent *event)
{
g_free (undist->settings);
if (!gst_camera_event_parse_calibrated (event, &(undist->settings))) {
return FALSE;
}
undist->settingsChanged = TRUE;
return TRUE;
}
static gboolean
gst_camera_undistort_sink_event (GstBaseTransform *trans, GstEvent *event)
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans);
const GstStructure *structure = gst_event_get_structure (event);
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
if (strcmp (gst_structure_get_name (structure), GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) {
return camera_undistort_calibration_event (undist, event);
}
}
return GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->sink_event (trans, event);
}
static gboolean
gst_camera_undistort_src_event (GstBaseTransform *trans, GstEvent *event)
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans);
const GstStructure *structure = gst_event_get_structure (event);
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
if (strcmp (gst_structure_get_name (structure), GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) {
return camera_undistort_calibration_event (undist, event);
}
}
return GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->src_event (trans, event);
}
/* entry point to initialize the plug-in
* initialize the plug-in itself
* register the element factories and other features
*/
gboolean
gst_camera_undistort_plugin_init (GstPlugin * plugin)
{
/* debug category for filtering log messages */
GST_DEBUG_CATEGORY_INIT (gst_camera_undistort_debug, "cameraundistort",
0,
"Performs camera undistortion");
return gst_element_register (plugin, "cameraundistort", GST_RANK_NONE,
GST_TYPE_CAMERA_UNDISTORT);
}

104
ext/opencv/gstcameraundistort.h Normal file
View file

@ -0,0 +1,104 @@
/*
* GStreamer
* Copyright (C) <2017> Philippe Renon <philippe_renon@yahoo.fr>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Alternatively, the contents of this file may be used under the
* GNU Lesser General Public License Version 2.1 (the "LGPL"), in
* which case the following provisions apply instead of the ones
* mentioned above:
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_CAMERA_UNDISTORT_H__
#define __GST_CAMERA_UNDISTORT_H__
#include <gst/gst.h>
#include <gst/opencv/gstopencvvideofilter.h>
#include <opencv2/core.hpp>
G_BEGIN_DECLS
#define GST_TYPE_CAMERA_UNDISTORT \
(gst_camera_undistort_get_type())
#define GST_CAMERA_UNDISTORT(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_CAMERA_UNDISTORT,GstCameraUndistort))
#define GST_CAMERA_UNDISTORT_CLASS(klass) \
(G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_CAMERA_UNDISTORT,GstCameraUndistortClass))
#define GST_IS_CAMERA_UNDISTORT(obj) \
(G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_CAMERA_UNDISTORT))
#define GST_IS_CAMERA_UNDISTORT_CLASS(klass) \
(G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_CAMERA_UNDISTORT))
typedef struct _GstCameraUndistort GstCameraUndistort;
typedef struct _GstCameraUndistortClass GstCameraUndistortClass;
struct _GstCameraUndistort
{
GstOpencvVideoFilter cvfilter;
// settings
bool showUndistorted;
float alpha;
bool crop;
// opaque string containing opencv calibration settings
gchar *settings;
// opencv calibration settings
cv::Mat cameraMatrix, distCoeffs;
// state
bool doUndistort;
bool settingsChanged;
// undistort
cv::Size imageSize;
cv::Mat map1, map2;
cv::Rect validPixROI;
};
struct _GstCameraUndistortClass
{
GstOpencvVideoFilterClass parent_class;
};
GType gst_camera_undistort_get_type (void);
gboolean gst_camera_undistort_plugin_init (GstPlugin * plugin);
G_END_DECLS
#endif /* __GST_CAMERA_UNDISTORT_H__ */

8
ext/opencv/gstopencv.cpp
View file

@ -42,6 +42,8 @@
#include "gstgrabcut.h"
#include "gstdisparity.h"
#include "gstdewarp.h"
#include "gstcameracalibrate.h"
#include "gstcameraundistort.h"
static gboolean
plugin_init (GstPlugin * plugin)
@ -103,6 +105,12 @@ plugin_init (GstPlugin * plugin)
if (!gst_dewarp_plugin_init (plugin))
return FALSE;
if (!gst_camera_calibrate_plugin_init (plugin))
return FALSE;
if (!gst_camera_undistort_plugin_init (plugin))
return FALSE;
return TRUE;
}

6
ext/opencv/meson.build
View file

@ -21,7 +21,11 @@ gstopencv_sources = [
'gsttextoverlay.cpp',
'MotionCells.cpp',
'motioncells_wrapper.cpp',
'gstdewarp.cpp'
'gstdewarp.cpp',
'camerautils.cpp',
'cameraevent.cpp',
'gstcameracalibrate.cpp',
'gstcameraundistort.cpp'
]
libopencv2_headers = [