mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-02-17 03:35:21 +00:00
Code Issues 1 Projects Releases Wiki Activity

facedetect: detect face features

Browse source 
Also detect mouth, nose and eyes. Drop faces that don't have them. Fixes leaking the
cascades. Adds more docs.
This commit is contained in:
Stefan Sauer 2011-11-16 20:53:13 +01:00
parent a857c90590
commit fefa1df8b9
2 changed files with 287 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

331
ext/opencv/gstfacedetect.c
View file

@ -3,6 +3,7 @@
* Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
* Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
* Copyright (C) 2008 Michael Sheldon <mike@mikeasoft.com>
* Copyright (C) 2011 Stefan Sauer <ensonic@users.sf.net>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@ -48,14 +49,27 @@
*
* Performs face detection on videos and images.
*
* The image is scaled down multiple times using the GstFacedetect::scale-factor
* until the size is &lt;= GstFacedetect::min-size-width or
* GstFacedetect::min-size-height.
*
* <refsect2>
* <title>Example launch line</title>
* |[
* gst-launch-0.10 autovideosrc ! decodebin2 ! colorspace ! facedetect ! colorspace ! xvimagesink
* ]|
* ]| Detect and show faces
* |[
* gst-launch-0.10 autovideosrc ! video/x-raw-yuv,width=320,height=240 ! colorspace ! facedetect min-size-width=60 min-size-height=60 ! colorspace ! xvimagesink
* ]| Detect large faces on a smaller image
*
* </refsect2>
*/
/* FIXME: development version of OpenCV has CV_HAAR_FIND_BIGGEST_OBJECT which
* we might want to use if available
* see https://code.ros.org/svn/opencv/trunk/opencv/modules/objdetect/src/haar.cpp
*/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
@ -67,7 +81,10 @@
GST_DEBUG_CATEGORY_STATIC (gst_facedetect_debug);
#define GST_CAT_DEFAULT gst_facedetect_debug
#define DEFAULT_PROFILE "/usr/share/opencv/haarcascades/haarcascade_frontalface_default.xml"
#define DEFAULT_FACE_PROFILE "/usr/share/opencv/haarcascades/haarcascade_frontalface_default.xml"
#define DEFAULT_NOSE_PROFILE "/usr/share/opencv/haarcascades/haarcascade_mcs_nose.xml"
#define DEFAULT_MOUTH_PROFILE "/usr/share/opencv/haarcascades/haarcascade_mcs_mouth.xml"
#define DEFAULT_EYES_PROFILE "/usr/share/opencv/haarcascades/haarcascade_mcs_eyepair_small.xml"
#define DEFAULT_SCALE_FACTOR 1.1
#define DEFAULT_FLAGS 0
#define DEFAULT_MIN_NEIGHBORS 3
@ -85,7 +102,10 @@ enum
{
PROP_0,
PROP_DISPLAY,
PROP_PROFILE,
PROP_FACE_PROFILE,
PROP_NOSE_PROFILE,
PROP_MOUTH_PROFILE,
PROP_EYES_PROFILE,
PROP_SCALE_FACTOR,
PROP_MIN_NEIGHBORS,
PROP_FLAGS,
@ -155,7 +175,8 @@ static gboolean gst_facedetect_set_caps (GstOpencvVideoFilter * transform,
static GstFlowReturn gst_facedetect_transform_ip (GstOpencvVideoFilter * base,
GstBuffer * buf, IplImage * img);
static void gst_facedetect_load_profile (GstFacedetect * filter);
static CvHaarClassifierCascade *gst_facedetect_load_profile (GstFacedetect *
filter, gchar * profile);
/* Clean up */
static void
@ -163,14 +184,24 @@ gst_facedetect_finalize (GObject * obj)
{
GstFacedetect *filter = GST_FACEDETECT (obj);
if (filter->cvGray) {
if (filter->cvGray)
cvReleaseImage (&filter->cvGray);
}
if (filter->cvStorage) {
if (filter->cvStorage)
cvReleaseMemStorage (&filter->cvStorage);
}
g_free (filter->profile);
g_free (filter->face_profile);
g_free (filter->nose_profile);
g_free (filter->mouth_profile);
g_free (filter->eyes_profile);
if (filter->cvFaceDetect)
cvReleaseHaarClassifierCascade (&filter->cvFaceDetect);
if (filter->cvNoseDetect)
cvReleaseHaarClassifierCascade (&filter->cvNoseDetect);
if (filter->cvMouthDetect)
cvReleaseHaarClassifierCascade (&filter->cvMouthDetect);
if (filter->cvEyesDetect)
cvReleaseHaarClassifierCascade (&filter->cvEyesDetect);
G_OBJECT_CLASS (parent_class)->finalize (obj);
}
@ -215,10 +246,24 @@ gst_facedetect_class_init (GstFacedetectClass * klass)
g_param_spec_boolean ("display", "Display",
"Sets whether the detected faces should be highlighted in the output",
TRUE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PROFILE,
g_param_spec_string ("profile", "Profile",
g_object_class_install_property (gobject_class, PROP_FACE_PROFILE,
g_param_spec_string ("profile", "Face profile",
"Location of Haar cascade file to use for face detection",
DEFAULT_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
DEFAULT_FACE_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_NOSE_PROFILE,
g_param_spec_string ("nose-profile", "Nose profile",
"Location of Haar cascade file to use for nose detection",
DEFAULT_NOSE_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_MOUTH_PROFILE,
g_param_spec_string ("mouth-profile", "Mouth profile",
"Location of Haar cascade file to use for mouth detection",
DEFAULT_MOUTH_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_EYES_PROFILE,
g_param_spec_string ("eyes-profile", "Eyes profile",
"Location of Haar cascade file to use for eye-pair detection",
DEFAULT_EYES_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_FLAGS,
g_param_spec_flags ("flags", "Flags", "Flags to cvHaarDetectObjects",
GST_TYPE_OPENCV_FACE_DETECT_FLAGS, DEFAULT_FLAGS,
@ -244,21 +289,29 @@ gst_facedetect_class_init (GstFacedetectClass * klass)
}
/* initialize the new element
* instantiate pads and add them to element
* set pad calback functions
* initialize instance structure
*/
static void
gst_facedetect_init (GstFacedetect * filter, GstFacedetectClass * gclass)
{
filter->profile = g_strdup (DEFAULT_PROFILE);
filter->face_profile = g_strdup (DEFAULT_FACE_PROFILE);
filter->nose_profile = g_strdup (DEFAULT_NOSE_PROFILE);
filter->mouth_profile = g_strdup (DEFAULT_MOUTH_PROFILE);
filter->eyes_profile = g_strdup (DEFAULT_EYES_PROFILE);
filter->display = TRUE;
filter->scale_factor = DEFAULT_SCALE_FACTOR;
filter->min_neighbors = DEFAULT_MIN_NEIGHBORS;
filter->flags = DEFAULT_FLAGS;
filter->min_size_width = DEFAULT_MIN_SIZE_WIDTH;
filter->min_size_height = DEFAULT_MIN_SIZE_HEIGHT;
gst_facedetect_load_profile (filter);
filter->cvFaceDetect =
gst_facedetect_load_profile (filter, filter->face_profile);
filter->cvNoseDetect =
gst_facedetect_load_profile (filter, filter->nose_profile);
filter->cvMouthDetect =
gst_facedetect_load_profile (filter, filter->mouth_profile);
filter->cvEyesDetect =
gst_facedetect_load_profile (filter, filter->eyes_profile);
gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER_CAST (filter),
TRUE);
@ -271,10 +324,37 @@ gst_facedetect_set_property (GObject * object, guint prop_id,
GstFacedetect *filter = GST_FACEDETECT (object);
switch (prop_id) {
case PROP_PROFILE:
g_free (filter->profile);
filter->profile = g_value_dup_string (value);
gst_facedetect_load_profile (filter);
case PROP_FACE_PROFILE:
g_free (filter->face_profile);
if (filter->cvFaceDetect)
cvReleaseHaarClassifierCascade (&filter->cvFaceDetect);
filter->face_profile = g_value_dup_string (value);
filter->cvFaceDetect =
gst_facedetect_load_profile (filter, filter->face_profile);
break;
case PROP_NOSE_PROFILE:
g_free (filter->nose_profile);
if (filter->cvNoseDetect)
cvReleaseHaarClassifierCascade (&filter->cvNoseDetect);
filter->nose_profile = g_value_dup_string (value);
filter->cvNoseDetect =
gst_facedetect_load_profile (filter, filter->nose_profile);
break;
case PROP_MOUTH_PROFILE:
g_free (filter->mouth_profile);
if (filter->cvMouthDetect)
cvReleaseHaarClassifierCascade (&filter->cvMouthDetect);
filter->mouth_profile = g_value_dup_string (value);
filter->cvMouthDetect =
gst_facedetect_load_profile (filter, filter->mouth_profile);
break;
case PROP_EYES_PROFILE:
g_free (filter->eyes_profile);
if (filter->cvEyesDetect)
cvReleaseHaarClassifierCascade (&filter->cvEyesDetect);
filter->eyes_profile = g_value_dup_string (value);
filter->cvEyesDetect =
gst_facedetect_load_profile (filter, filter->eyes_profile);
break;
case PROP_DISPLAY:
filter->display = g_value_get_boolean (value);
@ -307,8 +387,17 @@ gst_facedetect_get_property (GObject * object, guint prop_id,
GstFacedetect *filter = GST_FACEDETECT (object);
switch (prop_id) {
case PROP_PROFILE:
g_value_set_string (value, filter->profile);
case PROP_FACE_PROFILE:
g_value_set_string (value, filter->face_profile);
break;
case PROP_NOSE_PROFILE:
g_value_set_string (value, filter->nose_profile);
break;
case PROP_MOUTH_PROFILE:
g_value_set_string (value, filter->mouth_profile);
break;
case PROP_EYES_PROFILE:
g_value_set_string (value, filter->eyes_profile);
break;
case PROP_DISPLAY:
g_value_set_boolean (value, filter->display);
@ -391,17 +480,27 @@ gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf,
{
GstFacedetect *filter = GST_FACEDETECT (base);
if (filter->cvCascade) {
if (filter->cvFaceDetect) {
GstMessage *msg = NULL;
GValue facelist = { 0 };
CvSeq *faces;
CvSeq *mouth, *nose, *eyes;
gint i;
gboolean do_display = FALSE;
if (filter->display) {
if (gst_buffer_is_writable (buf)) {
do_display = TRUE;
} else {
GST_LOG_OBJECT (filter, "Buffer is not writable, not drawing faces.");
}
}
cvCvtColor (img, filter->cvGray, CV_RGB2GRAY);
cvClearMemStorage (filter->cvStorage);
faces =
cvHaarDetectObjects (filter->cvGray, filter->cvCascade,
cvHaarDetectObjects (filter->cvGray, filter->cvFaceDetect,
filter->cvStorage, filter->scale_factor, filter->min_neighbors,
filter->flags, cvSize (filter->min_size_width, filter->min_size_height)
#if (CV_MAJOR_VERSION >= 2) && (CV_MINOR_VERSION >= 2)
@ -417,43 +516,167 @@ gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf,
for (i = 0; i < (faces ? faces->total : 0); i++) {
CvRect *r = (CvRect *) cvGetSeqElem (faces, i);
GValue value = { 0 };
GstStructure *s = gst_structure_new ("face",
GstStructure *s;
guint mw = filter->min_size_width / 8;
guint mh = filter->min_size_height / 8;
guint rnx, rny, rnw, rnh;
guint rmx, rmy, rmw, rmh;
guint rex, rey, rew, reh;
gboolean have_nose, have_mouth, have_eyes;
/* detect face features */
rnx = r->x + r->width / 4;
rny = r->y + r->height / 4;
rnw = r->width / 2;
rnh = r->height / 2;
cvSetImageROI (filter->cvGray, cvRect (rnx, rny, rnw, rnh));
nose =
cvHaarDetectObjects (filter->cvGray, filter->cvNoseDetect,
filter->cvStorage, filter->scale_factor, filter->min_neighbors,
filter->flags, cvSize (mw, mh)
#if (CV_MAJOR_VERSION >= 2) && (CV_MINOR_VERSION >= 2)
, cvSize (mw + 2, mh + 2)
#endif
);
have_nose = (nose && nose->total);
cvResetImageROI (filter->cvGray);
rmx = r->x;
rmy = r->y + r->height / 2;
rmw = r->width;
rmh = r->height / 2;
cvSetImageROI (filter->cvGray, cvRect (rmx, rmy, rmw, rmh));
mouth =
cvHaarDetectObjects (filter->cvGray, filter->cvMouthDetect,
filter->cvStorage, filter->scale_factor, filter->min_neighbors,
filter->flags, cvSize (mw, mh)
#if (CV_MAJOR_VERSION >= 2) && (CV_MINOR_VERSION >= 2)
, cvSize (mw + 2, mh + 2)
#endif
);
have_mouth = (mouth && mouth->total);
cvResetImageROI (filter->cvGray);
rex = r->x;
rey = r->y;
rew = r->width;
reh = r->height / 2;
cvSetImageROI (filter->cvGray, cvRect (rex, rey, rew, reh));
eyes =
cvHaarDetectObjects (filter->cvGray, filter->cvEyesDetect,
filter->cvStorage, filter->scale_factor, filter->min_neighbors,
filter->flags, cvSize (mw, mh)
#if (CV_MAJOR_VERSION >= 2) && (CV_MINOR_VERSION >= 2)
, cvSize (mw + 2, mh + 2)
#endif
);
have_eyes = (eyes && eyes->total);
cvResetImageROI (filter->cvGray);
GST_LOG_OBJECT (filter,
"%2d/%2d: x,y = %4u,%4u: w.h = %4u,%4u : features(e,n,m) = %d,%d,%d",
i, faces->total, r->x, r->y, r->width, r->height,
have_eyes, have_nose, have_mouth);
/* ignore 'face' where we don't fix mount/nose/eyes ? */
if (!(have_eyes && have_nose && have_mouth))
continue;
s = gst_structure_new ("face",
"x", G_TYPE_UINT, r->x,
"y", G_TYPE_UINT, r->y,
"width", G_TYPE_UINT, r->width,
"height", G_TYPE_UINT, r->height, NULL);
GST_LOG_OBJECT (filter, "%2d/%2d: x,y = %4u,%4u: w.h = %4u,%4u", i,
faces->total, r->x, r->y, r->width, r->height);
if (nose && nose->total) {
CvRect *sr = (CvRect *) cvGetSeqElem (nose, 0);
GST_LOG_OBJECT (filter, "nose/%d: x,y = %4u,%4u: w.h = %4u,%4u",
nose->total, rnx + sr->x, rny + sr->y, sr->width, sr->height);
gst_structure_set (s,
"nose->x", G_TYPE_UINT, rnx + sr->x,
"nose->y", G_TYPE_UINT, rny + sr->y,
"nose->width", G_TYPE_UINT, sr->width,
"nose->height", G_TYPE_UINT, sr->height, NULL);
}
if (mouth && mouth->total) {
CvRect *sr = (CvRect *) cvGetSeqElem (mouth, 0);
GST_LOG_OBJECT (filter, "mouth/%d: x,y = %4u,%4u: w.h = %4u,%4u",
mouth->total, rmx + sr->x, rmy + sr->y, sr->width, sr->height);
gst_structure_set (s,
"mouth->x", G_TYPE_UINT, rmx + sr->x,
"mouth->y", G_TYPE_UINT, rmy + sr->y,
"mouth->width", G_TYPE_UINT, sr->width,
"mouth->height", G_TYPE_UINT, sr->height, NULL);
}
if (eyes && eyes->total) {
CvRect *sr = (CvRect *) cvGetSeqElem (eyes, 0);
GST_LOG_OBJECT (filter, "eyes/%d: x,y = %4u,%4u: w.h = %4u,%4u",
eyes->total, rex + sr->x, rey + sr->y, sr->width, sr->height);
gst_structure_set (s,
"eyes->x", G_TYPE_UINT, rex + sr->x,
"eyes->y", G_TYPE_UINT, rey + sr->y,
"eyes->width", G_TYPE_UINT, sr->width,
"eyes->height", G_TYPE_UINT, sr->height, NULL);
}
g_value_init (&value, GST_TYPE_STRUCTURE);
gst_value_set_structure (&value, s);
gst_value_list_append_value (&facelist, &value);
g_value_unset (&value);
}
if (filter->display) {
if (gst_buffer_is_writable (buf)) {
/* draw colored circles for each face */
for (i = 0; i < (faces ? faces->total : 0); i++) {
CvRect *r = (CvRect *) cvGetSeqElem (faces, i);
CvPoint center;
CvSize axes;
gdouble w = r->width * 0.5;
gdouble h = r->height * 0.6; /* tweak for face form */
gint cb = 255 - ((i & 3) << 7);
gint cg = 255 - ((i & 12) << 5);
gint cr = 255 - ((i & 48) << 3);
center.x = cvRound ((r->x + w));
center.y = cvRound ((r->y + h));
if (do_display) {
CvPoint center;
CvSize axes;
gdouble w, h;
gint cb = 255 - ((i & 3) << 7);
gint cg = 255 - ((i & 12) << 5);
gint cr = 255 - ((i & 48) << 3);
w = r->width / 2;
h = r->height / 2;
center.x = cvRound ((r->x + w));
center.y = cvRound ((r->y + h));
axes.width = w;
axes.height = h * 1.25; /* tweak for face form */
cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb),
3, 8, 0);
if (nose && nose->total) {
CvRect *sr = (CvRect *) cvGetSeqElem (nose, 0);
w = sr->width / 2;
h = sr->height / 2;
center.x = cvRound ((rnx + sr->x + w));
center.y = cvRound ((rny + sr->y + h));
axes.width = w;
axes.height = h * 1.25; /* tweak for nose form */
cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb),
1, 8, 0);
}
if (mouth && mouth->total) {
CvRect *sr = (CvRect *) cvGetSeqElem (mouth, 0);
w = sr->width / 2;
h = sr->height / 2;
center.x = cvRound ((rmx + sr->x + w));
center.y = cvRound ((rmy + sr->y + h));
axes.width = w * 1.5; /* tweak for mouth form */
axes.height = h;
cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb),
3, 8, 0);
1, 8, 0);
}
if (eyes && eyes->total) {
CvRect *sr = (CvRect *) cvGetSeqElem (eyes, 0);
w = sr->width / 2;
h = sr->height / 2;
center.x = cvRound ((rex + sr->x + w));
center.y = cvRound ((rey + sr->y + h));
axes.width = w * 1.5; /* tweak for eyes form */
axes.height = h;
cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb),
1, 8, 0);
}
} else {
GST_LOG_OBJECT (filter, "Buffer is not writable, not drawing "
"circles for faces");
}
}
@ -468,14 +691,16 @@ gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf,
}
static void
gst_facedetect_load_profile (GstFacedetect * filter)
static CvHaarClassifierCascade *
gst_facedetect_load_profile (GstFacedetect * filter, gchar * profile)
{
filter->cvCascade =
(CvHaarClassifierCascade *) cvLoad (filter->profile, 0, 0, 0);
if (!filter->cvCascade) {
GST_WARNING ("Couldn't load Haar classifier cascade: %s.", filter->profile);
CvHaarClassifierCascade *cascade;
if (!(cascade = (CvHaarClassifierCascade *) cvLoad (profile, 0, 0, 0))) {
GST_WARNING_OBJECT (filter, "Couldn't load Haar classifier cascade: %s.",
profile);
}
return cascade;
}

11
ext/opencv/gstfacedetect.h
View file

@ -3,6 +3,7 @@
* Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
* Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
* Copyright (C) 2008 Michael Sheldon <mike@mikeasoft.com>
* Copyright (C) 2011 Stefan Sauer <ensonic@users.sf.net>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@ -75,7 +76,10 @@ struct _GstFacedetect
gboolean display;
gchar *profile;
gchar *face_profile;
gchar *nose_profile;
gchar *mouth_profile;
gchar *eyes_profile;
gdouble scale_factor;
gint min_neighbors;
gint flags;
@ -83,7 +87,10 @@ struct _GstFacedetect
gint min_size_height;
IplImage *cvGray;
CvHaarClassifierCascade *cvCascade;
CvHaarClassifierCascade *cvFaceDetect;
CvHaarClassifierCascade *cvNoseDetect;
CvHaarClassifierCascade *cvMouthDetect;
CvHaarClassifierCascade *cvEyesDetect;
CvMemStorage *cvStorage;
};