mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-06-07 07:58:51 +00:00
Code Issues 1 Projects Releases Wiki Activity

handdetect: need to migrate to C++

Browse source 
The cascade classifier changes its structure on new version of OpenCV 2.4.11.
It is need to migrate to C++ to utilize the new load method of OpenCV which
allows to load the old and new classifiers.

https://bugzilla.gnome.org/show_bug.cgi?id=752528
This commit is contained in:
Luis de Bethencourt 2015-10-02 16:22:36 +01:00
parent dab249a445
commit 75e38f915f
2 changed files with 153 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

289
ext/opencv/gsthanddetect.cpp
View file

@ -72,6 +72,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_handdetect_debug);
#define HAAR_FILE_FIST GST_HAAR_CASCADES_DIR G_DIR_SEPARATOR_S "fist.xml" #define HAAR_FILE_FIST GST_HAAR_CASCADES_DIR G_DIR_SEPARATOR_S "fist.xml"
#define HAAR_FILE_PALM GST_HAAR_CASCADES_DIR G_DIR_SEPARATOR_S "palm.xml" #define HAAR_FILE_PALM GST_HAAR_CASCADES_DIR G_DIR_SEPARATOR_S "palm.xml"
using namespace cv;
/* Filter signals and args */ /* Filter signals and args */
enum enum
{ {
@ -113,8 +114,8 @@ static gboolean gst_handdetect_set_caps (GstOpencvVideoFilter * transform,
static GstFlowReturn gst_handdetect_transform_ip (GstOpencvVideoFilter * static GstFlowReturn gst_handdetect_transform_ip (GstOpencvVideoFilter *
transform, GstBuffer * buffer, IplImage * img); transform, GstBuffer * buffer, IplImage * img);
static CvHaarClassifierCascade *gst_handdetect_load_profile (GstHanddetect * static CascadeClassifier *gst_handdetect_load_profile (GstHanddetect * filter,
filter, gchar * profile); gchar * profile);
static void gst_handdetect_navigation_interface_init (GstNavigationInterface * static void gst_handdetect_navigation_interface_init (GstNavigationInterface *
iface); iface);
@ -164,6 +165,7 @@ gst_handdetect_finalize (GObject * obj)
cvReleaseMemStorage (&filter->cvStorage); cvReleaseMemStorage (&filter->cvStorage);
g_free (filter->profile_fist); g_free (filter->profile_fist);
g_free (filter->profile_palm); g_free (filter->profile_palm);
g_free (filter->best_r);
G_OBJECT_CLASS (gst_handdetect_parent_class)->finalize (obj); G_OBJECT_CLASS (gst_handdetect_parent_class)->finalize (obj);
} }
@ -297,7 +299,7 @@ gst_handdetect_set_property (GObject * object, guint prop_id,
case PROP_PROFILE_FIST: case PROP_PROFILE_FIST:
g_free (filter->profile_fist); g_free (filter->profile_fist);
if (filter->cvCascade_fist) if (filter->cvCascade_fist)
cvReleaseHaarClassifierCascade (&filter->cvCascade_fist); delete filter->cvCascade_fist;
filter->profile_fist = g_value_dup_string (value); filter->profile_fist = g_value_dup_string (value);
filter->cvCascade_fist = filter->cvCascade_fist =
gst_handdetect_load_profile (filter, filter->profile_fist); gst_handdetect_load_profile (filter, filter->profile_fist);
@ -305,7 +307,7 @@ gst_handdetect_set_property (GObject * object, guint prop_id,
case PROP_PROFILE_PALM: case PROP_PROFILE_PALM:
g_free (filter->profile_palm); g_free (filter->profile_palm);
if (filter->cvCascade_palm) if (filter->cvCascade_palm)
cvReleaseHaarClassifierCascade (&filter->cvCascade_palm); delete filter->cvCascade_palm;
filter->profile_palm = g_value_dup_string (value); filter->profile_palm = g_value_dup_string (value);
filter->cvCascade_palm = filter->cvCascade_palm =
gst_handdetect_load_profile (filter, filter->profile_palm); gst_handdetect_load_profile (filter, filter->profile_palm);
@ -400,147 +402,55 @@ gst_handdetect_transform_ip (GstOpencvVideoFilter * transform,
GstBuffer * buffer, IplImage * img) GstBuffer * buffer, IplImage * img)
{ {
GstHanddetect *filter = GST_HANDDETECT (transform); GstHanddetect *filter = GST_HANDDETECT (transform);
CvSeq *hands; Rect *r;
CvRect *r;
GstStructure *s; GstStructure *s;
GstMessage *m; GstMessage *m;
int i; unsigned int i;
vector < Rect > hands;
/* check detection cascades */ /* check detection cascades */
if (!filter->cvCascade_fist || !filter->cvCascade_palm) if (filter->cvCascade_fist && filter->cvCascade_palm) {
return GST_FLOW_OK;
/* cvt to gray colour space for hand detect */ /* cvt to gray colour space for hand detect */
cvCvtColor (img, filter->cvGray, CV_RGB2GRAY); cvCvtColor (img, filter->cvGray, CV_RGB2GRAY);
cvClearMemStorage (filter->cvStorage); cvClearMemStorage (filter->cvStorage);
/* detect FIST gesture fist */ /* detect FIST gesture fist */
hands = Mat roi (filter->cvGray, Rect (filter->cvGray->origin,
cvHaarDetectObjects (filter->cvGray, filter->cvCascade_fist, filter->cvGray->origin, filter->cvGray->width,
filter->cvStorage, 1.1, 2, CV_HAAR_DO_CANNY_PRUNING, cvSize (24, 24), filter->cvGray->height));
cvSize (0, 0)); filter->cvCascade_fist->detectMultiScale (roi, hands, 1.1, 2,
CV_HAAR_DO_CANNY_PRUNING, cvSize (24, 24), cvSize (0, 0));
/* if FIST gesture detected */ /* if FIST gesture detected */
if (hands && hands->total > 0) { if (!hands.empty ()) {
int min_distance, distance;
CvRect temp_r;
CvPoint c;
GST_DEBUG_OBJECT (filter, "%d FIST gestures detected\n",
(int) hands->total);
/* Go through all detected FIST gestures to get the best one
* prev_r => previous hand
* best_r => best hand in this frame
*/
/* set min_distance for init comparison */
min_distance = img->width + img->height;
/* Init filter->prev_r */
temp_r = cvRect (0, 0, 0, 0);
if (filter->prev_r == NULL)
filter->prev_r = &temp_r;
/* Get the best FIST gesture */
for (i = 0; i < hands->total; i++) {
r = (CvRect *) cvGetSeqElem (hands, i);
distance = (int) sqrt (pow ((r->x - filter->prev_r->x),
2) + pow ((r->y - filter->prev_r->y), 2));
if (distance <= min_distance) {
min_distance = distance;
filter->best_r = r;
}
}
/* Save best_r as prev_r for next frame comparison */
filter->prev_r = (CvRect *) filter->best_r;
/* send msg to app/bus if the detected gesture falls in the region of interest */
/* get center point of gesture */
c = cvPoint (filter->best_r->x + filter->best_r->width / 2,
filter->best_r->y + filter->best_r->height / 2);
/* send message:
* if the center point is in the region of interest, OR,
* if the region of interest remains default as (0,0,0,0)*/
if ((c.x >= filter->roi_x && c.x <= (filter->roi_x + filter->roi_width)
&& c.y >= filter->roi_y
&& c.y <= (filter->roi_y + filter->roi_height))
|| (filter->roi_x == 0
&& filter->roi_y == 0
&& filter->roi_width == 0 && filter->roi_height == 0)) {
/* Define structure for message post */
s = gst_structure_new ("hand-gesture",
"gesture", G_TYPE_STRING, "fist",
"x", G_TYPE_UINT,
(guint) (filter->best_r->x + filter->best_r->width * 0.5), "y",
G_TYPE_UINT,
(guint) (filter->best_r->y + filter->best_r->height * 0.5), "width",
G_TYPE_UINT, (guint) filter->best_r->width, "height", G_TYPE_UINT,
(guint) filter->best_r->height, NULL);
/* Init message element */
m = gst_message_new_element (GST_OBJECT (filter), s);
/* Send message */
gst_element_post_message (GST_ELEMENT (filter), m);
#if 0
/* send event
* here we use mouse-move event instead of fist-move or palm-move event
* !!! this will CHANGE in the future !!!
* !!! by adding gst_navigation_send_hand_detect_event() in navigation.c !!!
*/
gst_navigation_send_mouse_event (GST_NAVIGATION (filter),
"mouse-move",
0,
(double) (filter->best_r->x + filter->best_r->width * 0.5),
(double) (filter->best_r->y + filter->best_r->height * 0.5));
#endif
}
/* Check filter->display,
* If TRUE, displaying red circle marker in the out frame */
if (filter->display) {
CvPoint center;
int radius;
center.x = cvRound ((filter->best_r->x + filter->best_r->width * 0.5));
center.y = cvRound ((filter->best_r->y + filter->best_r->height * 0.5));
radius =
cvRound ((filter->best_r->width + filter->best_r->height) * 0.25);
cvCircle (img, center, radius, CV_RGB (0, 0, 200), 1, 8, 0);
}
} else {
/* if NO FIST gesture, detecting PALM gesture */
hands =
cvHaarDetectObjects (filter->cvGray, filter->cvCascade_palm,
filter->cvStorage, 1.1, 3, CV_HAAR_DO_CANNY_PRUNING, cvSize (24, 24),
cvSize (0, 0));
/* if PALM detected */
if (hands && hands->total > 0) {
int min_distance, distance; int min_distance, distance;
CvRect temp_r; Rect temp_r;
CvPoint c; CvPoint c;
/* set frame buffer writable */
if (filter->display) { /* Go through all detected FIST gestures to get the best one
buffer = gst_buffer_make_writable (buffer);
GST_DEBUG_OBJECT (filter, "%d PALM gestures detected\n",
(int) hands->total);
}
/* Go through all detected PALM gestures to get the best one
* prev_r => previous hand * prev_r => previous hand
* best_r => best hand in this frame * best_r => best hand in this frame
*/ */
/* suppose a min_distance for init comparison */ /* set min_distance for init comparison */
min_distance = img->width + img->height; min_distance = img->width + img->height;
/* Init filter->prev_r */ /* Init filter->prev_r */
temp_r = cvRect (0, 0, 0, 0); temp_r = Rect (0, 0, 0, 0);
if (filter->prev_r == NULL) if (filter->prev_r == NULL)
filter->prev_r = &temp_r; filter->prev_r = &temp_r;
/* Get the best PALM gesture */ /* Get the best FIST gesture */
for (i = 0; i < hands->total; i++) { for (i = 0; i < hands.size(); i++) {
r = (CvRect *) cvGetSeqElem (hands, i); r = &hands[i];
distance = (int) sqrt (pow ((r->x - filter->prev_r->x), distance = (int) sqrt (pow ((r->x - filter->prev_r->x),
2) + pow ((r->y - filter->prev_r->y), 2)); 2) + pow ((r->y - filter->prev_r->y), 2));
if (distance <= min_distance) { if (distance <= min_distance) {
min_distance = distance; min_distance = distance;
filter->best_r = r; g_free (filter->best_r);
filter->best_r = new Rect (*r);
} }
} }
/* Save best_r as prev_r for next frame comparison */ /* Save best_r as prev_r for next frame comparison */
filter->prev_r = (CvRect *) filter->best_r; filter->prev_r = filter->best_r;
/* send msg to app/bus if the detected gesture falls in the region of interest */ /* send msg to app/bus if the detected gesture falls in the region of interest */
/* get center point of gesture */ /* get center point of gesture */
@ -557,7 +467,7 @@ gst_handdetect_transform_ip (GstOpencvVideoFilter * transform,
&& filter->roi_width == 0 && filter->roi_height == 0)) { && filter->roi_width == 0 && filter->roi_height == 0)) {
/* Define structure for message post */ /* Define structure for message post */
s = gst_structure_new ("hand-gesture", s = gst_structure_new ("hand-gesture",
"gesture", G_TYPE_STRING, "palm", "gesture", G_TYPE_STRING, "fist",
"x", G_TYPE_UINT, "x", G_TYPE_UINT,
(guint) (filter->best_r->x + filter->best_r->width * 0.5), "y", (guint) (filter->best_r->x + filter->best_r->width * 0.5), "y",
G_TYPE_UINT, G_TYPE_UINT,
@ -581,20 +491,6 @@ gst_handdetect_transform_ip (GstOpencvVideoFilter * transform,
(double) (filter->best_r->x + filter->best_r->width * 0.5), (double) (filter->best_r->x + filter->best_r->width * 0.5),
(double) (filter->best_r->y + filter->best_r->height * 0.5)); (double) (filter->best_r->y + filter->best_r->height * 0.5));
/* or use another way to send upstream navigation event for debug
*
* GstEvent *event =
* gst_event_new_navigation (gst_structure_new
* ("application/x-gst-navigation", "event", G_TYPE_STRING,
* "mouse-move",
* "button", G_TYPE_INT, 0,
* "pointer_x", G_TYPE_DOUBLE,
* (double) (filter->best_r->x + filter->best_r->width * 0.5),
* "pointer_y", G_TYPE_DOUBLE,
* (double) (filter->best_r->y + filter->best_r->height * 0.5),
* NULL));
* gst_pad_send_event (GST_BASE_TRANSFORM_CAST (filter)->srcpad, event);
*/
#endif #endif
} }
/* Check filter->display, /* Check filter->display,
@ -608,6 +504,114 @@ gst_handdetect_transform_ip (GstOpencvVideoFilter * transform,
cvRound ((filter->best_r->width + filter->best_r->height) * 0.25); cvRound ((filter->best_r->width + filter->best_r->height) * 0.25);
cvCircle (img, center, radius, CV_RGB (0, 0, 200), 1, 8, 0); cvCircle (img, center, radius, CV_RGB (0, 0, 200), 1, 8, 0);
} }
} else {
/* if NO FIST gesture, detecting PALM gesture */
filter->cvCascade_palm->detectMultiScale (roi, hands, 1.1, 2,
CV_HAAR_DO_CANNY_PRUNING, cvSize (24, 24), cvSize (0, 0));
/* if PALM detected */
if (!hands.empty ()) {
int min_distance, distance;
Rect temp_r;
CvPoint c;
/* set frame buffer writable */
if (filter->display) {
buffer = gst_buffer_make_writable (buffer);
GST_DEBUG_OBJECT (filter, "%d PALM gestures detected\n",
(int) hands.size ());
}
/* Go through all detected PALM gestures to get the best one
* prev_r => previous hand
* best_r => best hand in this frame
*/
/* suppose a min_distance for init comparison */
min_distance = img->width + img->height;
/* Init filter->prev_r */
temp_r = Rect (0, 0, 0, 0);
if (filter->prev_r == NULL)
filter->prev_r = &temp_r;
/* Get the best PALM gesture */
for (i = 0; i < hands.size (); ++i) {
r = &hands[i];
distance = (int) sqrt (pow ((r->x - filter->prev_r->x),
2) + pow ((r->y - filter->prev_r->y), 2));
if (distance <= min_distance) {
min_distance = distance;
g_free (filter->best_r);
filter->best_r = new Rect (*r);
}
}
/* Save best_r as prev_r for next frame comparison */
filter->prev_r = filter->best_r;
/* send msg to app/bus if the detected gesture falls in the region of interest */
/* get center point of gesture */
c = cvPoint (filter->best_r->x + filter->best_r->width / 2,
filter->best_r->y + filter->best_r->height / 2);
/* send message:
* if the center point is in the region of interest, OR,
* if the region of interest remains default as (0,0,0,0)*/
if (((guint) c.x >= filter->roi_x
&& (guint) c.x <= (filter->roi_x + filter->roi_width)
&& (guint) c.y >= filter->roi_y
&& (guint) c.y <= (filter->roi_y + filter->roi_height))
|| (filter->roi_x == 0 && filter->roi_y == 0
&& filter->roi_width == 0 && filter->roi_height == 0)) {
/* Define structure for message post */
s = gst_structure_new ("hand-gesture",
"gesture", G_TYPE_STRING, "palm",
"x", G_TYPE_UINT,
(guint) (filter->best_r->x + filter->best_r->width * 0.5), "y",
G_TYPE_UINT,
(guint) (filter->best_r->y + filter->best_r->height * 0.5),
"width", G_TYPE_UINT, (guint) filter->best_r->width, "height",
G_TYPE_UINT, (guint) filter->best_r->height, NULL);
/* Init message element */
m = gst_message_new_element (GST_OBJECT (filter), s);
/* Send message */
gst_element_post_message (GST_ELEMENT (filter), m);
#if 0
/* send event
* here we use mouse-move event instead of fist-move or palm-move event
* !!! this will CHANGE in the future !!!
* !!! by adding gst_navigation_send_hand_detect_event() in navigation.c !!!
*/
gst_navigation_send_mouse_event (GST_NAVIGATION (filter),
"mouse-move",
0,
(double) (filter->best_r->x + filter->best_r->width * 0.5),
(double) (filter->best_r->y + filter->best_r->height * 0.5));
/* or use another way to send upstream navigation event for debug
*
* GstEvent *event =
* gst_event_new_navigation (gst_structure_new
* ("application/x-gst-navigation", "event", G_TYPE_STRING,
* "mouse-move",
* "button", G_TYPE_INT, 0,
* "pointer_x", G_TYPE_DOUBLE,
* (double) (filter->best_r->x + filter->best_r->width * 0.5),
* "pointer_y", G_TYPE_DOUBLE,
* (double) (filter->best_r->y + filter->best_r->height * 0.5),
* NULL));
* gst_pad_send_event (GST_BASE_TRANSFORM_CAST (filter)->srcpad, event);
*/
#endif
}
/* Check filter->display,
* If TRUE, displaying red circle marker in the out frame */
if (filter->display) {
CvPoint center;
int radius;
center.x =
cvRound ((filter->best_r->x + filter->best_r->width * 0.5));
center.y =
cvRound ((filter->best_r->y + filter->best_r->height * 0.5));
radius =
cvRound ((filter->best_r->width + filter->best_r->height) * 0.25);
cvCircle (img, center, radius, CV_RGB (0, 0, 200), 1, 8, 0);
}
}
} }
} }
@ -615,17 +619,18 @@ gst_handdetect_transform_ip (GstOpencvVideoFilter * transform,
return GST_FLOW_OK; return GST_FLOW_OK;
} }
static CvHaarClassifierCascade * static CascadeClassifier *
gst_handdetect_load_profile (GstHanddetect * filter, gchar * profile) gst_handdetect_load_profile (GstHanddetect * filter, gchar * profile)
{ {
CvHaarClassifierCascade *cascade; CascadeClassifier *cascade;
if (profile == NULL) cascade = new CascadeClassifier (profile);
if (cascade->empty ()) {
GST_ERROR_OBJECT (filter, "Invalid profile file: %s", profile);
delete cascade;
return NULL; return NULL;
if (!(cascade = (CvHaarClassifierCascade *) cvLoad (profile, 0, 0, 0))) {
GST_WARNING_OBJECT (filter, "Couldn't load Haar classifier cascade: %s.",
profile);
} }
return cascade; return cascade;
} }

10
ext/opencv/gsthanddetect.h
View file

@ -49,6 +49,8 @@
#include <gst/video/video.h> #include <gst/video/video.h>
#include <gst/video/navigation.h> #include <gst/video/navigation.h>
#include <cv.h>
#include "gstopencvvideofilter.h" #include "gstopencvvideofilter.h"
/* opencv */ /* opencv */
#include <opencv2/core/version.hpp> #include <opencv2/core/version.hpp>
@ -92,11 +94,11 @@ struct _GstHanddetect
* cvGray - image to gray colour * cvGray - image to gray colour
*/ */
IplImage *cvGray; IplImage *cvGray;
CvHaarClassifierCascade *cvCascade_fist; cv::CascadeClassifier *cvCascade_fist;
CvHaarClassifierCascade *cvCascade_palm; cv::CascadeClassifier *cvCascade_palm;
CvMemStorage *cvStorage; CvMemStorage *cvStorage;
CvRect *prev_r; cv::Rect *prev_r;
CvRect *best_r; cv::Rect *best_r;
}; };
struct _GstHanddetectClass struct _GstHanddetectClass