faceoverlay: Add element that displays an SVG image over a detected face

Fixes bug #642759.
This commit is contained in:
Laura Lucas Alday 2011-05-30 12:24:31 +02:00 committed by Sebastian Dröge
parent 8f7183dc85
commit d11178aac0
4 changed files with 606 additions and 0 deletions

View file

@ -310,6 +310,7 @@ AG_GST_CHECK_PLUGIN(debugutils)
AG_GST_CHECK_PLUGIN(dtmf)
AG_GST_CHECK_PLUGIN(dvbsuboverlay)
AG_GST_CHECK_PLUGIN(dvdspu)
AG_GST_CHECK_PLUGIN(faceoverlay)
AG_GST_CHECK_PLUGIN(festival)
AG_GST_CHECK_PLUGIN(fieldanalysis)
AG_GST_CHECK_PLUGIN(freeze)
@ -1822,6 +1823,7 @@ gst/debugutils/Makefile
gst/dtmf/Makefile
gst/dvbsuboverlay/Makefile
gst/dvdspu/Makefile
gst/faceoverlay/Makefile
gst/festival/Makefile
gst/fieldanalysis/Makefile
gst/freeze/Makefile

View file

@ -0,0 +1,13 @@
plugin_LTLIBRARIES = libgstfaceoverlay.la
# sources used to compile this plug-in
libgstfaceoverlay_la_SOURCES = gstfaceoverlay.c
# compiler and linker flags used to compile this plugin, set in configure.ac
libgstfaceoverlay_la_CFLAGS = $(GST_CFLAGS)
libgstfaceoverlay_la_LIBADD = $(GST_LIBS)
libgstfaceoverlay_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS)
libgstfaceoverlay_la_LIBTOOLFLAGS = --tag=disable-static
# headers we need but don't want installed
noinst_HEADERS = gstfaceoverlay.h

View file

@ -0,0 +1,500 @@
/*
* GStreamer faceoverlay plugin
* Copyright (C) 2011 Laura Lucas Alday <lauralucas@gmail.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Alternatively, the contents of this file may be used under the
* GNU Lesser General Public License Version 2.1 (the "LGPL"), in
* which case the following provisions apply instead of the ones
* mentioned above:
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
/**
* SECTION:element-faceoverlay
*
* Overlays a SVG image over a detected face in a video stream.
* x, y, w, and h properties are optional, and change the image position and
* size relative to the detected face position and size.
*
* <refsect2>
* <title>Example launch line</title>
* |[
* gst-launch autovideosrc ! ffmpegcolorspace ! faceoverlay location=/path/to/gnome-video-effects/pixmaps/bow.svg x=-5 y=-15 w=0.3 h=0.1 ! ffmpegcolorspace ! autovideosink
* ]|
* </refsect2>
*/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include <gst/gst.h>
#include <gst/video/video.h>
#include <string.h>
#include "gstfaceoverlay.h"
GST_DEBUG_CATEGORY_STATIC (gst_face_overlay_debug);
#define GST_CAT_DEFAULT gst_face_overlay_debug
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
#define GST_STR_VIDEO_CAPS GST_VIDEO_CAPS_BGRA
#else
#define GST_STR_VIDEO_CAPS GST_VIDEO_CAPS_ARGB
#endif
/* Filter signals and args */
enum
{
/* FILL ME */
LAST_SIGNAL
};
enum
{
PROP_0,
PROP_LOCATION,
PROP_X,
PROP_Y,
PROP_W,
PROP_H
};
/* the capabilities of the inputs and outputs. */
static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_STR_VIDEO_CAPS)
);
static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_STR_VIDEO_CAPS)
);
GST_BOILERPLATE (GstFaceOverlay, gst_face_overlay, GstBin, GST_TYPE_BIN);
static void gst_face_overlay_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_face_overlay_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static void gst_face_overlay_message_handler (GstBin * bin,
GstMessage * message);
static GstStateChangeReturn gst_face_overlay_change_state (GstElement * element,
GstStateChange transition);
static gboolean gst_face_overlay_create_children (GstFaceOverlay * filter);
static gboolean gst_face_overlay_reset (GstFaceOverlay * filter);
static gboolean gst_face_overlay_create_pad (GstFaceOverlay * filter,
GstPad * filter_pad, const char *pad_name, GstElement * child_element);
static gboolean toggle_pads_link_state (GstPad * pad1, GstPad * pad2);
static gboolean
toggle_pads_link_state (GstPad * pad1, GstPad * pad2)
{
gboolean ok = TRUE;
if (gst_pad_is_linked (pad1)) {
if (gst_pad_get_direction (pad1) == GST_PAD_SINK)
gst_pad_unlink (pad2, pad1);
else
gst_pad_unlink (pad1, pad2);
} else {
if (gst_pad_get_direction (pad1) == GST_PAD_SINK)
ok &= (gst_pad_link (pad2, pad1) == 0);
else
ok &= (gst_pad_link (pad1, pad2) == 0);
}
return ok;
}
/* Unlinks and removes the pad that was created in gst_face_overlay_init ()
* and adds the internal element ghost pad instead */
static gboolean
gst_face_overlay_create_pad (GstFaceOverlay * filter, GstPad * filter_pad,
const char *pad_name, GstElement * child_element)
{
GstPad *peer = NULL;
GstPad *pad = NULL;
gboolean ok = TRUE;
/* get the outside world pad connected to faceoverlay src/sink pad */
peer = gst_pad_get_peer (filter_pad);
/* unlink and remove the faceoverlay src/sink pad */
toggle_pads_link_state (peer, filter_pad);
gst_element_remove_pad (GST_ELEMENT (filter), filter_pad);
/* add a ghost pad pointing to the child element pad (facedetect sink or
* svg_overlay src depending on filter_pad direction) and add it to
* faceoverlay bin */
pad = gst_element_get_static_pad (child_element, pad_name);
filter_pad = gst_ghost_pad_new (pad_name, pad);
gst_object_unref (GST_OBJECT (pad));
gst_element_add_pad (GST_ELEMENT (filter), filter_pad);
/* link the child element pad to the outside world thru the ghost pad */
toggle_pads_link_state (peer, filter_pad);
g_object_unref (peer);
return ok;
}
static gboolean
gst_face_overlay_reset (GstFaceOverlay * filter)
{
gst_element_set_state (filter->face_detect, GST_STATE_NULL);
gst_bin_remove (GST_BIN (filter), filter->face_detect);
filter->face_detect = NULL;
gst_element_set_state (filter->svg_overlay, GST_STATE_NULL);
gst_bin_remove (GST_BIN (filter), filter->svg_overlay);
filter->svg_overlay = NULL;
gst_element_set_state (filter->colorspace, GST_STATE_NULL);
gst_bin_remove (GST_BIN (filter), filter->colorspace);
filter->colorspace = NULL;
return TRUE;
}
static gboolean
gst_face_overlay_create_children (GstFaceOverlay * filter)
{
gboolean ret = TRUE;
if ((filter->colorspace = gst_element_factory_make ("ffmpegcolorspace",
NULL)) == NULL) {
return FALSE;
}
if ((filter->face_detect = gst_element_factory_make ("facedetect",
NULL)) == NULL) {
return FALSE;
}
g_object_set (filter->face_detect, "display", 0, NULL);
if ((filter->svg_overlay = gst_element_factory_make ("rsvgoverlay",
NULL)) == NULL) {
return FALSE;
}
gst_bin_add_many (GST_BIN (filter),
filter->face_detect, filter->colorspace, filter->svg_overlay, NULL);
ret &= gst_element_link_pads (filter->face_detect, "src",
filter->colorspace, "sink");
ret &= gst_element_link_pads (filter->colorspace, "src",
filter->svg_overlay, "sink");
ret &= gst_face_overlay_create_pad (filter, filter->sinkpad, "sink",
filter->face_detect);
ret &= gst_face_overlay_create_pad (filter, filter->srcpad, "src",
filter->svg_overlay);
return ret;
}
static GstStateChangeReturn
gst_face_overlay_change_state (GstElement * element, GstStateChange transition)
{
GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
GstFaceOverlay *filter = GST_FACEOVERLAY (element);
switch (transition) {
case GST_STATE_CHANGE_NULL_TO_READY:
if (!gst_face_overlay_create_children (filter))
return GST_STATE_CHANGE_FAILURE;
break;
default:
break;
}
ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
switch (transition) {
case GST_STATE_CHANGE_READY_TO_NULL:
gst_face_overlay_reset (filter);
break;
default:
break;
}
return ret;
}
static void
gst_face_overlay_message_handler (GstBin * bin, GstMessage * message)
{
if (GST_MESSAGE_TYPE (message) == GST_MESSAGE_ELEMENT &&
strcmp (gst_structure_get_name (message->structure), "facedetect") == 0) {
GstFaceOverlay *filter = GST_FACEOVERLAY (bin);
/* optionally draw the image once every two messages for better performance
* filter->process_message = !filter->process_message;
* if(!filter->process_message)
* return;
*/
guint x, y, width, height;
int delta_x, delta_y, svg_x, svg_y, svg_width, svg_height;
const GstStructure *face;
int face_count;
face_count =
gst_value_list_get_size (gst_structure_get_value (message->structure,
"faces"));
/* The last face in the list seems to be the right one, objects mistakenly
* detected as faces for a couple of frames seem to be in the list
* beginning. TODO: needs confirmation. */
face =
gst_value_get_structure (gst_value_list_get_value
(gst_structure_get_value (message->structure, "faces"),
face_count - 1));
gst_structure_get_uint (face, "x", &x);
gst_structure_get_uint (face, "y", &y);
gst_structure_get_uint (face, "width", &width);
gst_structure_get_uint (face, "height", &height);
/* Apply x and y offsets relative to face position and size.
* Set image width and height as a fraction of face width and height.
* Cast to int since face position and size will never be bigger than
* G_MAX_INT and we may have negative values as svg_x or svg_y */
delta_x = (int) (filter->x * (int) width);
svg_x = (int) x + delta_x;
delta_y = (int) (filter->y * (int) height);
svg_y = (int) y + delta_y;
svg_width = (int) width *filter->w;
svg_height = (int) height *filter->h;
g_object_set (filter->svg_overlay,
"location", filter->location,
"x", svg_x, "y", svg_y, "width", svg_width, "height", svg_height, NULL);
}
GST_BIN_CLASS (parent_class)->handle_message (bin, message);
}
/* GObject vmethod implementations */
/* the _base_init() function is meant to initialize class and child class
* properties during each new child class creation */
static void
gst_face_overlay_base_init (gpointer gclass)
{
GstElementClass *element_class = GST_ELEMENT_CLASS (gclass);
gst_element_class_set_details_simple (element_class,
"faceoverlay",
"Filter/Editor/Video",
"Overlays SVG graphics over a detected face in a video stream",
"Laura Lucas Alday <lauralucas@gmail.com>");
gst_element_class_add_pad_template (element_class,
gst_static_pad_template_get (&src_factory));
gst_element_class_add_pad_template (element_class,
gst_static_pad_template_get (&sink_factory));
}
/* initialize the faceoverlay's class */
/* the _class_init() function is used to initialise the class only once
* (specifying what signals, arguments and virtual functions the class has and
* setting up global state) */
static void
gst_face_overlay_class_init (GstFaceOverlayClass * klass)
{
GObjectClass *gobject_class;
GstBinClass *gstbin_class;
GstElementClass *gstelement_class;
gobject_class = G_OBJECT_CLASS (klass);
gstbin_class = GST_BIN_CLASS (klass);
gstelement_class = GST_ELEMENT_CLASS (klass);
gobject_class->set_property = gst_face_overlay_set_property;
gobject_class->get_property = gst_face_overlay_get_property;
g_object_class_install_property (gobject_class, PROP_LOCATION,
g_param_spec_string ("location", "Location",
"Location of SVG file to use for face overlay",
"", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_X,
g_param_spec_float ("x", "face x offset",
"Specify image x relative to detected face x.", -G_MAXFLOAT,
G_MAXFLOAT, 0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_Y,
g_param_spec_float ("y", "face y offset",
"Specify image y relative to detected face y.", -G_MAXFLOAT,
G_MAXFLOAT, 0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_W,
g_param_spec_float ("w", "face width percent",
"Specify image width relative to face width.", 0, G_MAXFLOAT, 1,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_H,
g_param_spec_float ("h", "face height percent",
"Specify image height relative to face height.", 0, G_MAXFLOAT, 1,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
gstbin_class->handle_message =
GST_DEBUG_FUNCPTR (gst_face_overlay_message_handler);
gstelement_class->change_state =
GST_DEBUG_FUNCPTR (gst_face_overlay_change_state);
}
/* initialize the new element
* instantiate pads and add them to element
* set pad calback functions
* initialize instance structure
* the _init() function is used to initialise a specific instance of this type.
*/
static void
gst_face_overlay_init (GstFaceOverlay * filter, GstFaceOverlayClass * gclass)
{
filter->x = 0;
filter->y = 0;
filter->w = 1;
filter->h = 1;
filter->colorspace = NULL;
filter->svg_overlay = NULL;
filter->face_detect = NULL;
filter->location = NULL;
filter->process_message = TRUE;
filter->sinkpad = gst_pad_new_from_static_template (&sink_factory, "sink");
gst_element_add_pad (GST_ELEMENT (filter), filter->sinkpad);
filter->srcpad = gst_pad_new_from_static_template (&src_factory, "src");
gst_element_add_pad (GST_ELEMENT (filter), filter->srcpad);
}
static void
gst_face_overlay_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstFaceOverlay *filter = GST_FACEOVERLAY (object);
switch (prop_id) {
case PROP_LOCATION:
filter->location = g_value_dup_string (value);
break;
case PROP_X:
filter->x = g_value_get_float (value);
break;
case PROP_Y:
filter->y = g_value_get_float (value);
break;
case PROP_W:
filter->w = g_value_get_float (value);
break;
case PROP_H:
filter->h = g_value_get_float (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_face_overlay_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstFaceOverlay *filter = GST_FACEOVERLAY (object);
switch (prop_id) {
case PROP_LOCATION:
g_value_set_string (value, filter->location);
break;
case PROP_X:
g_value_set_float (value, filter->x);
break;
case PROP_Y:
g_value_set_float (value, filter->y);
break;
case PROP_W:
g_value_set_float (value, filter->w);
break;
case PROP_H:
g_value_set_float (value, filter->h);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
/* entry point to initialize the plug-in
* initialize the plug-in itself
* register the element factories and other features
*/
static gboolean
faceoverlay_init (GstPlugin * faceoverlay)
{
/* debug category for fltering log messages */
GST_DEBUG_CATEGORY_INIT (gst_face_overlay_debug, "faceoverlay",
0, "SVG Face Overlay");
return gst_element_register (faceoverlay, "faceoverlay", GST_RANK_NONE,
GST_TYPE_FACEOVERLAY);
}
/* PACKAGE: this is usually set by autotools depending on some _INIT macro
* in configure.ac and then written into and defined in config.h, but we can
* just set it ourselves here in case someone doesn't use autotools to
* compile this code. GST_PLUGIN_DEFINE needs PACKAGE to be defined.
*/
#ifndef PACKAGE
#define PACKAGE "faceoverlay"
#endif
/* gstreamer looks for this structure to register plugins */
GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
GST_VERSION_MINOR,
"faceoverlay",
"SVG Face Overlay",
faceoverlay_init, VERSION, "LGPL", "GStreamer", "http://gstreamer.net/")

View file

@ -0,0 +1,91 @@
/*
* GStreamer faceoverlay plugin
* Copyright (C) 2011 Laura Lucas Alday <lauralucas@gmail.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Alternatively, the contents of this file may be used under the
* GNU Lesser General Public License Version 2.1 (the "LGPL"), in
* which case the following provisions apply instead of the ones
* mentioned above:
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#ifndef __GST_FACEOVERLAY_H__
#define __GST_FACEOVERLAY_H__
#include <gst/gst.h>
G_BEGIN_DECLS
/* #defines don't like whitespacey bits */
#define GST_TYPE_FACEOVERLAY \
(gst_face_overlay_get_type())
#define GST_FACEOVERLAY(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_FACEOVERLAY,GstFaceOverlay))
#define GST_FACEOVERLAY_CLASS(klass) \
(G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_FACEOVERLAY,GstFaceOverlayClass))
#define GST_IS_FACEOVERLAY(obj) \
(G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_FACEOVERLAY))
#define GST_IS_FACEOVERLAY_CLASS(klass) \
(G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_FACEOVERLAY))
typedef struct _GstFaceOverlay GstFaceOverlay;
typedef struct _GstFaceOverlayClass GstFaceOverlayClass;
struct _GstFaceOverlay
{
GstBin parent;
GstPad *sinkpad, *srcpad;
GstElement *face_detect;
GstElement *colorspace;
GstElement *svg_overlay;
gboolean process_message;
gchar *location;
gfloat x;
gfloat y;
gfloat w;
gfloat h;
};
struct _GstFaceOverlayClass
{
GstBinClass parent_class;
};
GType gst_face_overlay_get_type (void);
G_END_DECLS
#endif /* __GST_FACEOVERLAY_H__ */