From c313e1d3b848a9450225525923122d3b16f5d474 Mon Sep 17 00:00:00 2001 From: Miguel Casas-Sanchez Date: Tue, 11 Jun 2013 14:32:43 +0200 Subject: [PATCH] opencv: add foreground/background segmentation element Add an element to the opencv plugin for foregroung/background image sequence segmentation, using one out of 3 algorithms. https://bugzilla.gnome.org/show_bug.cgi?id=701421 --- ext/opencv/Makefile.am | 4 + ext/opencv/gstopencv.c | 4 + ext/opencv/gstsegmentation.cpp | 848 +++++++++++++++++++++++++++++++++ ext/opencv/gstsegmentation.h | 127 +++++ 4 files changed, 983 insertions(+) create mode 100644 ext/opencv/gstsegmentation.cpp create mode 100644 ext/opencv/gstsegmentation.h diff --git a/ext/opencv/Makefile.am b/ext/opencv/Makefile.am index ad47916bbb..05e0077ab5 100644 --- a/ext/opencv/Makefile.am +++ b/ext/opencv/Makefile.am @@ -21,6 +21,7 @@ libgstopencv_la_SOURCES = gstopencv.c \ gstmotioncells.c \ gstskindetect.c \ gstretinex.c \ + gstsegmentation.cpp \ motioncells_wrapper.cpp \ MotionCells.cpp @@ -59,6 +60,9 @@ noinst_HEADERS = gstopencvvideofilter.h gstopencvutils.h \ gstpyramidsegment.h \ gsttemplatematch.h \ gsttextoverlay.h \ + gstskindetect.h \ + gstretinex.h \ + gstsegmentation.h \ gstmotioncells.h \ motioncells_wrapper.h \ MotionCells.h diff --git a/ext/opencv/gstopencv.c b/ext/opencv/gstopencv.c index ff9ead6331..9347991bc2 100644 --- a/ext/opencv/gstopencv.c +++ b/ext/opencv/gstopencv.c @@ -39,6 +39,7 @@ #include "gsthanddetect.h" #include "gstskindetect.h" #include "gstretinex.h" +#include "gstsegmentation.h" static gboolean plugin_init (GstPlugin * plugin) @@ -91,6 +92,9 @@ plugin_init (GstPlugin * plugin) if (!gst_retinex_plugin_init (plugin)) return FALSE; + if (!gst_segmentation_plugin_init (plugin)) + return FALSE; + return TRUE; } diff --git a/ext/opencv/gstsegmentation.cpp b/ext/opencv/gstsegmentation.cpp new file mode 100644 index 0000000000..880313e12a --- /dev/null +++ b/ext/opencv/gstsegmentation.cpp @@ -0,0 +1,848 @@ +/* + * GStreamer + * Copyright (C) 2013 Miguel Casas-Sanchez + * Except: Parts of code inside the preprocessor define CODE_FROM_OREILLY_BOOK, + * which are downloaded from O'Reilly website + * [http://examples.oreilly.com/9780596516130/] + * and adapted. Its license reads: + * "Oct. 3, 2008 + * Right to use this code in any way you want without warrenty, support or + * any guarentee of it working. " + * + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Alternatively, the contents of this file may be used under the + * GNU Lesser General Public License Version 2.1 (the "LGPL"), in + * which case the following provisions apply instead of the ones + * mentioned above: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#define CODE_FROM_OREILLY_BOOK + +/** + * SECTION:element-segmentation + * + * This element creates and updates a fg/bg model using one of several approaches. + * The one called "codebook" refers to the codebook approach following the opencv + * O'Reilly book [1] implementation of the algorithm described in K. Kim, + * T. H. Chalidabhongse, D. Harwood and L. Davis [2]. BackgroundSubtractorMOG [3], + * or MOG for shorts, refers to a Gaussian Mixture-based Background/Foreground + * Segmentation Algorithm. OpenCV MOG implements the algorithm described in [4]. + * BackgroundSubtractorMOG2 [5], refers to another Gaussian Mixture-based + * Background/Foreground segmentation algorithm. OpenCV MOG2 implements the + * algorithm described in [6] and [7]. + * + * [1] Learning OpenCV: Computer Vision with the OpenCV Library by Gary Bradski + * and Adrian Kaehler, Published by O'Reilly Media, October 3, 2008 + * [2] "Real-time Foreground-Background Segmentation using Codebook Model", + * Real-time Imaging, Volume 11, Issue 3, Pages 167-256, June 2005. + * [3] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog + * [4] P. KadewTraKuPong and R. Bowden, "An improved adaptive background + * mixture model for real-time tracking with shadow detection", Proc. 2nd + * European Workshop on Advanced Video-Based Surveillance Systems, 2001 + * [5] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog2 + * [6] Z.Zivkovic, "Improved adaptive Gausian mixture model for background + * subtraction", International Conference Pattern Recognition, UK, August, 2004. + * [7] Z.Zivkovic, F. van der Heijden, "Efficient Adaptive Density Estimation + * per Image Pixel for the Task of Background Subtraction", Pattern Recognition + * Letters, vol. 27, no. 7, pages 773-780, 2006. + * + * + * Example launch line + * |[ + * gst-launch-1.0 v4l2src device=/dev/video0 ! videoconvert ! video/x-raw,width=320,height=240 ! videoconvert ! segmentation test-mode=true method=2 ! videoconvert ! ximagesink + * ]| + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "gstsegmentation.h" + +GST_DEBUG_CATEGORY_STATIC (gst_segmentation_debug); +#define GST_CAT_DEFAULT gst_segmentation_debug + +/* Filter signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +enum +{ + PROP_0, + PROP_TEST_MODE, + PROP_METHOD, + PROP_LEARNING_RATE +}; +typedef enum +{ + METHOD_BOOK, + METHOD_MOG, + METHOD_MOG2 +} GstSegmentationMethod; + +#define DEFAULT_TEST_MODE FALSE +#define DEFAULT_METHOD METHOD_MOG2 +#define DEFAULT_LEARNING_RATE 0.01 + +#define GST_TYPE_SEGMENTATION_METHOD (gst_segmentation_method_get_type ()) +static GType +gst_segmentation_method_get_type (void) +{ + static GType etype = 0; + if (etype == 0) { + static const GEnumValue values[] = { + {METHOD_BOOK, "Codebook-based segmentation (Bradski2008)", "codebook"}, + {METHOD_MOG, "Mixture-of-Gaussians segmentation (Bowden2001)", "mog"}, + {METHOD_MOG2, "Mixture-of-Gaussians segmentation (Zivkovic2004)", "mog2"}, + {0, NULL, NULL}, + }; + etype = g_enum_register_static ("GstSegmentationMethod", values); + } + return etype; +} + +G_DEFINE_TYPE (GstSegmentation, gst_segmentation, GST_TYPE_VIDEO_FILTER); +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA"))); + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA"))); + + +static void gst_segmentation_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_segmentation_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstFlowReturn gst_segmentation_transform_ip (GstVideoFilter * btrans, + GstVideoFrame *frame); + +static gboolean gst_segmentation_stop (GstBaseTransform * basesrc); +static gboolean gst_segmentation_set_info(GstVideoFilter *filter, + GstCaps *incaps, GstVideoInfo *in_info, + GstCaps *outcaps, GstVideoInfo *out_info); +static void gst_segmentation_release_all_pointers (GstSegmentation * filter); + +/* Codebook algorithm + connected components functions*/ +static int update_codebook (unsigned char *p, codeBook * c, + unsigned *cbBounds, int numChannels); +static int clear_stale_entries (codeBook * c); +static unsigned char background_diff (unsigned char *p, codeBook * c, + int numChannels, int *minMod, int *maxMod); +static void find_connected_components (IplImage * mask, int poly1_hull0, + float perimScale, CvMemStorage * mem_storage, CvSeq * contours); + +/* MOG (Mixture-of-Gaussians functions */ +static int initialise_mog (GstSegmentation * filter); +static int run_mog_iteration (GstSegmentation * filter); +static int run_mog2_iteration (GstSegmentation * filter); +static int finalise_mog (GstSegmentation * filter); + +/* initialize the segmentation's class */ +static void +gst_segmentation_class_init (GstSegmentationClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseTransformClass *basesrc_class = GST_BASE_TRANSFORM_CLASS (klass); + GstVideoFilterClass *video_class = (GstVideoFilterClass*) klass; + + gobject_class = (GObjectClass *) klass; + + gobject_class->set_property = gst_segmentation_set_property; + gobject_class->get_property = gst_segmentation_get_property; + + basesrc_class->stop = gst_segmentation_stop; + + video_class->transform_frame_ip = gst_segmentation_transform_ip; + video_class->set_info = gst_segmentation_set_info; + + g_object_class_install_property (gobject_class, PROP_METHOD, + g_param_spec_enum ("method", + "Segmentation method to use", + "Segmentation method to use", + GST_TYPE_SEGMENTATION_METHOD, DEFAULT_METHOD, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_TEST_MODE, + g_param_spec_boolean ("test-mode", "test-mode", + "If true, the output RGB is overwritten with the calculated foreground (white color)", + DEFAULT_TEST_MODE, (GParamFlags) + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_LEARNING_RATE, + g_param_spec_float ("learning-rate", "learning-rate", + "Speed with which a motionless foreground pixel would become background (inverse of number of frames)", + 0, 1, DEFAULT_LEARNING_RATE, (GParamFlags) (G_PARAM_READWRITE))); + + gst_element_class_set_static_metadata (element_class, + "Foreground/background video sequence segmentation", + "Filter/Effect/Video", + "Create a Foregound/Background mask applying a particular algorithm", + "Miguel Casas-Sanchez "); + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_factory)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_factory)); + +} + +/* initialize the new element + * instantiate pads and add them to element + * set pad calback functions + * initialize instance structure + */ +static void +gst_segmentation_init (GstSegmentation * filter) +{ + filter->method = DEFAULT_METHOD; + filter->test_mode = DEFAULT_TEST_MODE; + filter->framecount = 0; + filter->learning_rate = DEFAULT_LEARNING_RATE; + gst_base_transform_set_in_place (GST_BASE_TRANSFORM (filter), TRUE); +} + + +static void +gst_segmentation_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstSegmentation *filter = GST_SEGMENTATION (object); + + switch (prop_id) { + case PROP_METHOD: + filter->method = g_value_get_enum (value); + break; + case PROP_TEST_MODE: + filter->test_mode = g_value_get_boolean (value); + break; + case PROP_LEARNING_RATE: + filter->learning_rate = g_value_get_float (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_segmentation_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstSegmentation *filter = GST_SEGMENTATION (object); + + switch (prop_id) { + case PROP_METHOD: + g_value_set_enum (value, filter->method); + break; + case PROP_TEST_MODE: + g_value_set_boolean (value, filter->test_mode); + break; + case PROP_LEARNING_RATE: + g_value_set_float (value, filter->learning_rate); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/* GstElement vmethod implementations */ +/* this function handles the link with other elements */ +static gboolean +gst_segmentation_set_info(GstVideoFilter *filter, + GstCaps *incaps, GstVideoInfo *in_info, + GstCaps *outcaps, GstVideoInfo *out_info) +{ + GstSegmentation *segmentation = GST_SEGMENTATION (filter); + CvSize size; + + size = cvSize (in_info->width, in_info->height); + segmentation->width = in_info->width; + segmentation->height = in_info->height; + /* If cvRGB is already allocated, it means there's a cap modification, */ + /* so release first all the images. */ + if (NULL != segmentation->cvRGBA) + gst_segmentation_release_all_pointers (segmentation); + + segmentation->cvRGBA = cvCreateImageHeader (size, IPL_DEPTH_8U, 4); + + segmentation->cvRGB = cvCreateImage (size, IPL_DEPTH_8U, 3); + segmentation->cvYUV = cvCreateImage (size, IPL_DEPTH_8U, 3); + + segmentation->cvFG = cvCreateImage (size, IPL_DEPTH_8U, 1); + cvZero (segmentation->cvFG); + + segmentation->ch1 = cvCreateImage (size, IPL_DEPTH_8U, 1); + segmentation->ch2 = cvCreateImage (size, IPL_DEPTH_8U, 1); + segmentation->ch3 = cvCreateImage (size, IPL_DEPTH_8U, 1); + + /* Codebook method */ + segmentation->TcodeBook = (codeBook *) + g_malloc (sizeof (codeBook) * + (segmentation->width * segmentation->height + 1)); + for (int j = 0; j < segmentation->width * segmentation->height; j++) { + segmentation->TcodeBook[j].numEntries = 0; + segmentation->TcodeBook[j].t = 0; + } + segmentation->learning_interval = (int) (1.0 / segmentation->learning_rate); + + /* Mixture-of-Gaussians (mog) methods */ + initialise_mog (segmentation); + + return TRUE; +} + +/* Clean up */ +static gboolean +gst_segmentation_stop (GstBaseTransform * basesrc) +{ + GstSegmentation *filter = GST_SEGMENTATION (basesrc); + + if (filter->cvRGBA != NULL) + gst_segmentation_release_all_pointers (filter); + + return TRUE; +} + +static void +gst_segmentation_release_all_pointers (GstSegmentation * filter) +{ + cvReleaseImage (&filter->cvRGBA); + cvReleaseImage (&filter->cvRGB); + cvReleaseImage (&filter->cvYUV); + cvReleaseImage (&filter->cvFG); + cvReleaseImage (&filter->ch1); + cvReleaseImage (&filter->ch2); + cvReleaseImage (&filter->ch3); + + g_free (filter->TcodeBook); + finalise_mog (filter); +} + +static GstFlowReturn +gst_segmentation_transform_ip (GstVideoFilter * btrans, GstVideoFrame * frame) +{ + GstSegmentation *filter = GST_SEGMENTATION (btrans); + int j; + + /* get image data from the input, which is RGBA */ + filter->cvRGBA->imageData = (char *) GST_VIDEO_FRAME_COMP_DATA (frame, 0); + filter->cvRGBA->widthStep = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); + filter->framecount++; + + /* Image preprocessing: color space conversion etc */ + cvCvtColor (filter->cvRGBA, filter->cvRGB, CV_RGBA2RGB); + cvCvtColor (filter->cvRGB, filter->cvYUV, CV_RGB2YCrCb); + + /* Create and update a fg/bg model using a codebook approach following the + * opencv O'Reilly book [1] implementation of the algo described in [2]. + * + * [1] Learning OpenCV: Computer Vision with the OpenCV Library by Gary + * Bradski and Adrian Kaehler, Published by O'Reilly Media, October 3, 2008 + * [2] "Real-time Foreground-Background Segmentation using Codebook Model", + * Real-time Imaging, Volume 11, Issue 3, Pages 167-256, June 2005. */ + if (METHOD_BOOK == filter->method) { + unsigned cbBounds[3] = { 10, 5, 5 }; + int minMod[3] = { 20, 20, 20 }, maxMod[3] = { + 20, 20, 20}; + + if (filter->framecount < 30) { + /* Learning background phase: update_codebook on every frame */ + for (j = 0; j < filter->width * filter->height; j++) { + update_codebook ((unsigned char *) filter->cvYUV->imageData + j * 3, + (codeBook *) & (filter->TcodeBook[j]), cbBounds, 3); + } + } else { + /* this updating is responsible for FG becoming BG again */ + if (filter->framecount % filter->learning_interval == 0) { + for (j = 0; j < filter->width * filter->height; j++) { + update_codebook ((uchar *) filter->cvYUV->imageData + j * 3, + (codeBook *) & (filter->TcodeBook[j]), cbBounds, 3); + } + } + if (filter->framecount % 60 == 0) { + for (j = 0; j < filter->width * filter->height; j++) + clear_stale_entries ((codeBook *) & (filter->TcodeBook[j])); + } + + for (j = 0; j < filter->width * filter->height; j++) { + if (background_diff + ((uchar *) filter->cvYUV->imageData + j * 3, + (codeBook *) & (filter->TcodeBook[j]), 3, minMod, maxMod)) { + filter->cvFG->imageData[j] = 255; + } else { + filter->cvFG->imageData[j] = 0; + } + } + } + + /* 3rd param is the smallest area to show: (w+h)/param , in pixels */ + find_connected_components (filter->cvFG, 1, 10000, + filter->mem_storage, filter->contours); + + } + /* Create the foreground and background masks using BackgroundSubtractorMOG [1], + * Gaussian Mixture-based Background/Foreground segmentation algorithm. OpenCV + * MOG implements the algorithm described in [2]. + * + * [1] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog + * [2] P. KadewTraKuPong and R. Bowden, "An improved adaptive background + * mixture model for real-time tracking with shadow detection", Proc. 2nd + * European Workshop on Advanced Video-Based Surveillance Systems, 2001 + */ + else if (METHOD_MOG == filter->method) { + run_mog_iteration (filter); + } + /* Create the foreground and background masks using BackgroundSubtractorMOG2 + * [1], Gaussian Mixture-based Background/Foreground segmentation algorithm. + * OpenCV MOG2 implements the algorithm described in [2] and [3]. + * + * [1] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog2 + * [2] Z.Zivkovic, "Improved adaptive Gausian mixture model for background + * subtraction", International Conference Pattern Recognition, UK, Aug 2004. + * [3] Z.Zivkovic, F. van der Heijden, "Efficient Adaptive Density Estimation + * per Image Pixel for the Task of Background Subtraction", Pattern + * Recognition Letters, vol. 27, no. 7, pages 773-780, 2006. */ + else if (METHOD_MOG2 == filter->method) { + run_mog2_iteration (filter); + } + + /* if we want to test_mode, just overwrite the output */ + if (filter->test_mode) { + cvCvtColor (filter->cvFG, filter->cvRGB, CV_GRAY2RGB); + + cvSplit (filter->cvRGB, filter->ch1, filter->ch2, filter->ch3, NULL); + } else + cvSplit (filter->cvRGBA, filter->ch1, filter->ch2, filter->ch3, NULL); + + /* copy anyhow the fg/bg to the alpha channel in the output image */ + cvMerge (filter->ch1, filter->ch2, filter->ch3, filter->cvFG, filter->cvRGBA); + + + return GST_FLOW_OK; +} + +/* entry point to initialize the plug-in + * initialize the plug-in itself + * register the element factories and other features + */ +gboolean +gst_segmentation_plugin_init (GstPlugin * plugin) +{ + GST_DEBUG_CATEGORY_INIT (gst_segmentation_debug, "segmentation", + 0, "Performs Foreground/Background segmentation in video sequences"); + + return gst_element_register (plugin, "segmentation", GST_RANK_NONE, + GST_TYPE_SEGMENTATION); +} + + + +#ifdef CODE_FROM_OREILLY_BOOK /* See license at the beginning of the page */ +/* + int update_codebook(uchar *p, codeBook &c, unsigned cbBounds) + Updates the codebook entry with a new data point + + p Pointer to a YUV or HSI pixel + c Codebook for this pixel + cbBounds Learning bounds for codebook (Rule of thumb: 10) + numChannels Number of color channels we¡¯re learning + + NOTES: + cvBounds must be of length equal to numChannels + + RETURN + codebook index +*/ +int +update_codebook (unsigned char *p, codeBook * c, unsigned *cbBounds, + int numChannels) +{ +/* c->t+=1; */ + unsigned int high[3], low[3]; + int n, i; + int matchChannel; + + for (n = 0; n < numChannels; n++) { + high[n] = *(p + n) + *(cbBounds + n); + if (high[n] > 255) + high[n] = 255; + low[n] = *(p + n) - *(cbBounds + n); + if (low[n] < 0) + low[n] = 0; + } + +/* SEE IF THIS FITS AN EXISTING CODEWORD */ + for (i = 0; i < c->numEntries; i++) { + matchChannel = 0; + for (n = 0; n < numChannels; n++) { + if ((c->cb[i]->learnLow[n] <= *(p + n)) && +/* Found an entry for this channel */ + (*(p + n) <= c->cb[i]->learnHigh[n])) { + matchChannel++; + } + } + if (matchChannel == numChannels) { /* If an entry was found */ + c->cb[i]->t_last_update = c->t; +/* adjust this codeword for the first channel */ + for (n = 0; n < numChannels; n++) { + if (c->cb[i]->max[n] < *(p + n)) { + c->cb[i]->max[n] = *(p + n); + } else if (c->cb[i]->min[n] > *(p + n)) { + c->cb[i]->min[n] = *(p + n); + } + } + break; + } + } +/* OVERHEAD TO TRACK POTENTIAL STALE ENTRIES */ + for (int s = 0; s < c->numEntries; s++) { +/* Track which codebook entries are going stale: */ + int negRun = c->t - c->cb[s]->t_last_update; + if (c->cb[s]->stale < negRun) + c->cb[s]->stale = negRun; + } +/* ENTER A NEW CODEWORD IF NEEDED */ + if (i == c->numEntries) { /* if no existing codeword found, make one */ + code_element **foo = + (code_element **) g_malloc (sizeof (code_element *) * + (c->numEntries + 1)); + for (int ii = 0; ii < c->numEntries; ii++) { + foo[ii] = c->cb[ii]; /* copy all pointers */ + } + foo[c->numEntries] = (code_element *) g_malloc (sizeof (code_element)); + if (c->numEntries) + g_free (c->cb); + c->cb = foo; + for (n = 0; n < numChannels; n++) { + c->cb[c->numEntries]->learnHigh[n] = high[n]; + c->cb[c->numEntries]->learnLow[n] = low[n]; + c->cb[c->numEntries]->max[n] = *(p + n); + c->cb[c->numEntries]->min[n] = *(p + n); + } + c->cb[c->numEntries]->t_last_update = c->t; + c->cb[c->numEntries]->stale = 0; + c->numEntries += 1; + } +/* SLOWLY ADJUST LEARNING BOUNDS */ + for (n = 0; n < numChannels; n++) { + if (c->cb[i]->learnHigh[n] < high[n]) + c->cb[i]->learnHigh[n] += 1; + if (c->cb[i]->learnLow[n] > low[n]) + c->cb[i]->learnLow[n] -= 1; + } + return (i); +} + + + + + +/* + int clear_stale_entries(codeBook &c) + During learning, after you've learned for some period of time, + periodically call this to clear out stale codebook entries + + c Codebook to clean up + + Return + number of entries cleared +*/ +int +clear_stale_entries (codeBook * c) +{ + int staleThresh = c->t >> 1; + int *keep = (int *) g_malloc (sizeof (int) * (c->numEntries)); + int keepCnt = 0; + code_element **foo; + int k; + int numCleared; +/* SEE WHICH CODEBOOK ENTRIES ARE TOO STALE */ + for (int i = 0; i < c->numEntries; i++) { + if (c->cb[i]->stale > staleThresh) + keep[i] = 0; /* Mark for destruction */ + else { + keep[i] = 1; /* Mark to keep */ + keepCnt += 1; + } + } + /* KEEP ONLY THE GOOD */ + c->t = 0; /* Full reset on stale tracking */ + foo = (code_element **) g_malloc (sizeof (code_element *) * keepCnt); + k = 0; + for (int ii = 0; ii < c->numEntries; ii++) { + if (keep[ii]) { + foo[k] = c->cb[ii]; + /* We have to refresh these entries for next clearStale */ + foo[k]->t_last_update = 0; + k++; + } + } + /* CLEAN UP */ + g_free (keep); + g_free (c->cb); + c->cb = foo; + numCleared = c->numEntries - keepCnt; + c->numEntries = keepCnt; + return (numCleared); +} + + + +/* + uchar background_diff( uchar *p, codeBook &c, + int minMod, int maxMod) + Given a pixel and a codebook, determine if the pixel is + covered by the codebook + + p Pixel pointer (YUV interleaved) + c Codebook reference + numChannels Number of channels we are testing + maxMod Add this (possibly negative) number onto + + max level when determining if new pixel is foreground + minMod Subract this (possibly negative) number from + min level when determining if new pixel is foreground + + NOTES: + minMod and maxMod must have length numChannels, + e.g. 3 channels => minMod[3], maxMod[3]. There is one min and + one max threshold per channel. + + Return + 0 => background, 255 => foreground +*/ +unsigned char +background_diff (unsigned char *p, codeBook * c, int numChannels, + int *minMod, int *maxMod) +{ + int matchChannel; +/* SEE IF THIS FITS AN EXISTING CODEWORD */ + int i; + for (i = 0; i < c->numEntries; i++) { + matchChannel = 0; + for (int n = 0; n < numChannels; n++) { + if ((c->cb[i]->min[n] - minMod[n] <= *(p + n)) && + (*(p + n) <= c->cb[i]->max[n] + maxMod[n])) { + matchChannel++; /* Found an entry for this channel */ + } else { + break; + } + } + if (matchChannel == numChannels) { + break; /* Found an entry that matched all channels */ + } + } + if (i >= c->numEntries) + return (255); + return (0); +} + + + + +/* + void find_connected_components(IplImage *mask, int poly1_hull0, + float perimScale, int *num, + CvRect *bbs, CvPoint *centers) + This cleans up the foreground segmentation mask derived from calls + to backgroundDiff + + mask Is a grayscale (8-bit depth) “raw” mask image that + will be cleaned up + + OPTIONAL PARAMETERS: + poly1_hull0 If set, approximate connected component by + (DEFAULT) polygon, or else convex hull (0) + perimScale Len = image (width+height)/perimScale. If contour + len < this, delete that contour (DEFAULT: 4) + num Maximum number of rectangles and/or centers to + return; on return, will contain number filled + (DEFAULT: NULL) + bbs Pointer to bounding box rectangle vector of + length num. (DEFAULT SETTING: NULL) + centers Pointer to contour centers vector of length + num (DEFAULT: NULL) +*/ + +/* Approx.threshold - the bigger it is, the simpler is the boundary */ +#define CVCONTOUR_APPROX_LEVEL 1 +/* How many iterations of erosion and/or dilation there should be */ +#define CVCLOSE_ITR 1 +static void +find_connected_components (IplImage * mask, int poly1_hull0, float perimScale, + CvMemStorage * mem_storage, CvSeq * contours) +{ + CvContourScanner scanner; + CvSeq *c; + int numCont = 0; + /* Just some convenience variables */ + const CvScalar CVX_WHITE = CV_RGB (0xff, 0xff, 0xff); + const CvScalar CVX_BLACK = CV_RGB (0x00, 0x00, 0x00); + + /* CLEAN UP RAW MASK */ + cvMorphologyEx (mask, mask, 0, 0, CV_MOP_OPEN, CVCLOSE_ITR); + cvMorphologyEx (mask, mask, 0, 0, CV_MOP_CLOSE, CVCLOSE_ITR); + /* FIND CONTOURS AROUND ONLY BIGGER REGIONS */ + if (mem_storage == NULL) { + mem_storage = cvCreateMemStorage (0); + } else { + cvClearMemStorage (mem_storage); + } + + scanner = cvStartFindContours (mask, mem_storage, sizeof (CvContour), + CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, cvPoint (0, 0)); + + while ((c = cvFindNextContour (scanner)) != NULL) { + double len = cvContourArea (c, CV_WHOLE_SEQ, 0); + /* calculate perimeter len threshold: */ + double q = (mask->height + mask->width) / perimScale; + /* Get rid of blob if its perimeter is too small: */ + if (len < q) { + cvSubstituteContour (scanner, NULL); + } else { + /* Smooth its edges if its large enough */ + CvSeq *c_new; + if (poly1_hull0) { + /* Polygonal approximation */ + c_new = + cvApproxPoly (c, sizeof (CvContour), mem_storage, CV_POLY_APPROX_DP, + CVCONTOUR_APPROX_LEVEL, 0); + } else { + /* Convex Hull of the segmentation */ + c_new = cvConvexHull2 (c, mem_storage, CV_CLOCKWISE, 1); + } + cvSubstituteContour (scanner, c_new); + numCont++; + } + } + contours = cvEndFindContours (&scanner); + + /* PAINT THE FOUND REGIONS BACK INTO THE IMAGE */ + cvZero (mask); + /* DRAW PROCESSED CONTOURS INTO THE MASK */ + for (c = contours; c != NULL; c = c->h_next) + cvDrawContours (mask, c, CVX_WHITE, CVX_BLACK, -1, CV_FILLED, 8, cvPoint (0, + 0)); +} +#endif /*ifdef CODE_FROM_OREILLY_BOOK */ + + +int +initialise_mog (GstSegmentation * filter) +{ + filter->img_input_as_cvMat = (void *) new cv::Mat (filter->cvYUV, false); + filter->img_fg_as_cvMat = (void *) new cv::Mat (filter->cvFG, false); + + filter->mog = (void *) new cv::BackgroundSubtractorMOG (); + filter->mog2 = (void *) new cv::BackgroundSubtractorMOG2 (); + + return (0); +} + +int +run_mog_iteration (GstSegmentation * filter) +{ + ((cv::Mat *) filter->img_input_as_cvMat)->data = + (uchar *) filter->cvYUV->imageData; + ((cv::Mat *) filter->img_fg_as_cvMat)->data = + (uchar *) filter->cvFG->imageData; + + /* + BackgroundSubtractorMOG [1], Gaussian Mixture-based Background/Foreground + Segmentation Algorithm. OpenCV MOG implements the algorithm described in [2]. + + [1] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog + [2] P. KadewTraKuPong and R. Bowden, "An improved adaptive background + mixture model for real-time tracking with shadow detection", Proc. 2nd + European Workshop on Advanced Video-Based Surveillance Systems, 2001 + */ + + (*((cv::BackgroundSubtractorMOG *) filter->mog)) (*((cv::Mat *) filter-> + img_input_as_cvMat), *((cv::Mat *) filter->img_fg_as_cvMat), + filter->learning_rate); + + return (0); +} + +int +run_mog2_iteration (GstSegmentation * filter) +{ + ((cv::Mat *) filter->img_input_as_cvMat)->data = + (uchar *) filter->cvYUV->imageData; + ((cv::Mat *) filter->img_fg_as_cvMat)->data = + (uchar *) filter->cvFG->imageData; + + /* + BackgroundSubtractorMOG2 [1], Gaussian Mixture-based Background/Foreground + segmentation algorithm. OpenCV MOG2 implements the algorithm described in + [2] and [3]. + + [1] http://opencv.itseez.com/modules/video/doc/motion_analysis_and_object_tracking.html#backgroundsubtractormog2 + [2] Z.Zivkovic, "Improved adaptive Gausian mixture model for background + subtraction", International Conference Pattern Recognition, UK, August, 2004. + [3] Z.Zivkovic, F. van der Heijden, "Efficient Adaptive Density Estimation per + Image Pixel for the Task of Background Subtraction", Pattern Recognition + Letters, vol. 27, no. 7, pages 773-780, 2006. + */ + + (*((cv::BackgroundSubtractorMOG *) filter->mog2)) (*((cv::Mat *) filter-> + img_input_as_cvMat), *((cv::Mat *) filter->img_fg_as_cvMat), + filter->learning_rate); + + return (0); +} + +int +finalise_mog (GstSegmentation * filter) +{ + delete (cv::Mat *) filter->img_input_as_cvMat; + delete (cv::Mat *) filter->img_fg_as_cvMat; + delete (cv::BackgroundSubtractorMOG *) filter->mog; + delete (cv::BackgroundSubtractorMOG2 *) filter->mog2; + return (0); +} diff --git a/ext/opencv/gstsegmentation.h b/ext/opencv/gstsegmentation.h new file mode 100644 index 0000000000..146ae6cc4f --- /dev/null +++ b/ext/opencv/gstsegmentation.h @@ -0,0 +1,127 @@ +/* + * GStreamer + * Copyright (C) 2013 Miguel Casas-Sanchez + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Alternatively, the contents of this file may be used under the + * GNU Lesser General Public License Version 2.1 (the "LGPL"), in + * which case the following provisions apply instead of the ones + * mentioned above: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_SEGMENTATION_H__ +#define __GST_SEGMENTATION_H__ + +#include +#include + +#include +#include + +G_BEGIN_DECLS +/* #defines don't like whitespacey bits */ +#define GST_TYPE_SEGMENTATION \ + (gst_segmentation_get_type()) +#define GST_SEGMENTATION(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_SEGMENTATION,GstSegmentation)) +#define GST_SEGMENTATION_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_SEGMENTATION,GstSegmentationClass)) +#define GST_IS_SEGMENTATION(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_SEGMENTATION)) +#define GST_IS_SEGMENTATION_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_SEGMENTATION)) +typedef struct _GstSegmentation GstSegmentation; +typedef struct _GstSegmentationClass GstSegmentationClass; + +#define CHANNELS 3 +typedef struct ce +{ + unsigned char learnHigh[CHANNELS]; /* High side threshold for learning */ + unsigned char learnLow[CHANNELS]; /* Low side threshold for learning */ + unsigned char max[CHANNELS]; /* High side of box boundary */ + unsigned char min[CHANNELS]; /* Low side of box boundary */ + int t_last_update; /* Allow us to kill stale entries */ + int stale; /* max negative run (longest period of inactivity) */ +} code_element; + + +typedef struct code_book +{ + code_element **cb; + int numEntries; + int t; /*count every access */ +} codeBook; + +struct _GstSegmentation +{ + GstVideoFilter element; + gint method; + + gboolean test_mode; + gint width, height; + + IplImage *cvRGBA; + IplImage *cvRGB; + IplImage *cvYUV; + + IplImage *cvFG; /* used for the alpha BW 1ch image composition */ + IplImage *ch1, *ch2, *ch3; + int framecount; + + /* for codebook approach */ + codeBook *TcodeBook; + int learning_interval; + CvMemStorage *mem_storage; + CvSeq *contours; + + /* for MOG methods */ + void *mog; /* cv::BackgroundSubtractorMOG */ + void *mog2; /* cv::BackgroundSubtractorMOG2 */ + void *img_input_as_cvMat; /* cv::Mat */ + void *img_fg_as_cvMat; /* cv::Mat */ + double learning_rate; +}; + +struct _GstSegmentationClass +{ + GstVideoFilterClass parent_class; +}; + +GType gst_segmentation_get_type (void); + +gboolean gst_segmentation_plugin_init (GstPlugin * plugin); + +G_END_DECLS +#endif /* __GST_SEGMENTATION_H__ */