mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-27 20:21:24 +00:00
Code Issues 1 Projects Releases Wiki Activity

scenechange: new scene change detection element

Browse source
This commit is contained in:
David Schleef 2011-04-08 12:11:07 -07:00
parent 02b15b4e8b
commit 38c6f6366c
5 changed files with 519 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
gst/videofilters/Makefile.am
View file

@ -7,6 +7,7 @@ libgstvideofiltersbad_la_SOURCES = \
gstvideofilter2.c \
gstvideofilter2.h \
gstzebrastripe.c \
gstscenechange.c \
gstvideofiltersbad.c
#nodist_libgstvideofiltersbad_la_SOURCES = $(ORC_NODIST_SOURCES)
libgstvideofiltersbad_la_CFLAGS = \
@ -23,6 +24,7 @@ libgstvideofiltersbad_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS)
libgstvideofiltersbad_la_LIBTOOLFLAGS = --tag=disable-static
noinst_HEADERS = \
gstzebrastripe.h
gstzebrastripe.h \
gstscenechange.h

454
gst/videofilters/gstscenechange.c Normal file
View file

@ -0,0 +1,454 @@
/* GStreamer
* Copyright (C) 2011 Entropy Wave Inc <ds@entropywave.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
* Boston, MA 02110-1335, USA.
*/
/**
* SECTION:element-gstscenechange
*
* The scenechange element detects scene changes (also known as shot
* changes) in a video stream, and sends a signal when this occurs.
* Applications can listen to this signal and make changes to the
* pipeline such as cutting the stream. In addition, whenever a
* scene change is detected, a custom downstream "GstForceKeyUnit"
* event is sent to downstream elements. Most video encoder elements
* will insert synchronization points into the stream when this event
* is received. When used with a tee element, the scenechange element
* can be used to align the synchronization points among multiple
* video encoders, which is useful for segmented streaming.
*
* The scenechange element does not work with compressed video.
*
* <refsect2>
* <title>Example launch line</title>
* |[
* gst-launch -v filesrc location=some_file.ogv ! decodebin !
* scenechange ! theoraenc ! fakesink
* ]|
* </refsect2>
*/
/*
* The algorithm used for scene change detection is a modification
* of Jim Easterbrook's shot change detector. I'm not aware of a
* research paper, but the code I got the idea from is here:
* http://sourceforge.net/projects/shot-change/
*
* The method is relatively simple. Calculate the sum of absolute
* differences of a picture and the previous picture, and compare this
* picture difference value with neighboring pictures. In the original
* algorithm, the value is compared to a configurable number of past
* and future pictures. However, comparing to future frames requires
* introducing latency into the stream, which I did not want. So this
* implementation only compared to previous frames.
*
* This code is more directly derived from the scene change detection
* implementation in Schroedinger. Schro's implementation is closer
* to the Easterbrook algorithm, comparing to future pictures. In
* terms of accuracy, schro's implementation has about 2-3 false positives
* or false negatives per 100 scene changes. This implementation has
* about 5 per 100. The threshold is tuned for minimum total false
* positives or negatives, on the assumption that the badness of a
* false negative is the same as a false positive.
*
* This algorithm is pretty much at its limit for error rate. I
* recommend any future work in this area to increase the complexity
* of detection, and then write an automatic tuning system as opposed
* to the manual tuning I did here.
*
* Inside the TESTING define are some hard-coded (mostly hand-written)
* scene change frame numbers for some easily available sequences.
*
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gst/gst.h>
#include <gst/base/gstbasetransform.h>
#include <gst/video/video.h>
#include "gstvideofilter2.h"
#include "gstscenechange.h"
#include <string.h>
GST_DEBUG_CATEGORY_STATIC (gst_scene_change_debug_category);
#define GST_CAT_DEFAULT gst_scene_change_debug_category
/* prototypes */
static void gst_scene_change_set_property (GObject * object,
guint property_id, const GValue * value, GParamSpec * pspec);
static void gst_scene_change_get_property (GObject * object,
guint property_id, GValue * value, GParamSpec * pspec);
static void gst_scene_change_dispose (GObject * object);
static void gst_scene_change_finalize (GObject * object);
static gboolean gst_scene_change_start (GstBaseTransform * trans);
static gboolean gst_scene_change_stop (GstBaseTransform * trans);
static GstFlowReturn
gst_scene_change_prefilter (GstVideoFilter2 * videofilter2, GstBuffer * buf);
static GstVideoFilter2Functions gst_scene_change_filter_functions[];
#undef TESTING
#ifdef TESTING
static gboolean is_shot_change (int frame_number);
#endif
enum
{
PROP_0
};
/* pad templates */
/* class initialization */
#define DEBUG_INIT(bla) \
GST_DEBUG_CATEGORY_INIT (gst_scene_change_debug_category, "scenechange", 0, \
"debug category for scenechange element");
GST_BOILERPLATE_FULL (GstSceneChange, gst_scene_change, GstVideoFilter2,
GST_TYPE_VIDEO_FILTER2, DEBUG_INIT);
static void
gst_scene_change_base_init (gpointer g_class)
{
GstElementClass *element_class = GST_ELEMENT_CLASS (g_class);
gst_element_class_set_details_simple (element_class, "Scene change detector",
"Video/Filter", "Detects scene changes in video",
"David Schleef <ds@entropywave.com>");
}
static void
gst_scene_change_class_init (GstSceneChangeClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstBaseTransformClass *base_transform_class =
GST_BASE_TRANSFORM_CLASS (klass);
GstVideoFilter2Class *video_filter2_class = GST_VIDEO_FILTER2_CLASS (klass);
gobject_class->set_property = gst_scene_change_set_property;
gobject_class->get_property = gst_scene_change_get_property;
gobject_class->dispose = gst_scene_change_dispose;
gobject_class->finalize = gst_scene_change_finalize;
base_transform_class->start = GST_DEBUG_FUNCPTR (gst_scene_change_start);
base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_scene_change_stop);
video_filter2_class->prefilter =
GST_DEBUG_FUNCPTR (gst_scene_change_prefilter);
gst_video_filter2_class_add_functions (video_filter2_class,
gst_scene_change_filter_functions);
}
static void
gst_scene_change_init (GstSceneChange * scenechange,
GstSceneChangeClass * scenechange_class)
{
}
void
gst_scene_change_set_property (GObject * object, guint property_id,
const GValue * value, GParamSpec * pspec)
{
GstSceneChange *scenechange;
g_return_if_fail (GST_IS_SCENE_CHANGE (object));
scenechange = GST_SCENE_CHANGE (object);
switch (property_id) {
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
break;
}
}
void
gst_scene_change_get_property (GObject * object, guint property_id,
GValue * value, GParamSpec * pspec)
{
GstSceneChange *scenechange;
g_return_if_fail (GST_IS_SCENE_CHANGE (object));
scenechange = GST_SCENE_CHANGE (object);
switch (property_id) {
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
break;
}
}
void
gst_scene_change_dispose (GObject * object)
{
GstSceneChange *scenechange;
g_return_if_fail (GST_IS_SCENE_CHANGE (object));
scenechange = GST_SCENE_CHANGE (object);
/* clean up as possible. may be called multiple times */
G_OBJECT_CLASS (parent_class)->dispose (object);
}
void
gst_scene_change_finalize (GObject * object)
{
GstSceneChange *scenechange;
g_return_if_fail (GST_IS_SCENE_CHANGE (object));
scenechange = GST_SCENE_CHANGE (object);
if (scenechange->oldbuf)
gst_buffer_unref (scenechange->oldbuf);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static gboolean
gst_scene_change_start (GstBaseTransform * trans)
{
return TRUE;
}
static gboolean
gst_scene_change_stop (GstBaseTransform * trans)
{
return TRUE;
}
static GstFlowReturn
gst_scene_change_prefilter (GstVideoFilter2 * video_filter2, GstBuffer * buf)
{
return GST_FLOW_OK;
}
static double
get_frame_score (guint8 * s1, guint8 * s2, int width, int height)
{
int i;
int j;
int score = 0;
for (j = 0; j < height; j++) {
for (i = 0; i < width; i++) {
score += ABS (s1[i] - s2[i]);
}
s1 += width;
s2 += width;
}
return ((double) score) / (width * height);
}
static GstFlowReturn
gst_scene_change_filter_ip_I420 (GstVideoFilter2 * videofilter2,
GstBuffer * buf, int start, int end)
{
GstSceneChange *scenechange;
double score_min;
double score_max;
double threshold;
double score;
gboolean change;
int i;
int width;
int height;
g_return_val_if_fail (GST_IS_SCENE_CHANGE (videofilter2), GST_FLOW_ERROR);
scenechange = GST_SCENE_CHANGE (videofilter2);
width = GST_VIDEO_FILTER2_WIDTH (videofilter2);
height = GST_VIDEO_FILTER2_HEIGHT (videofilter2);
if (!scenechange->oldbuf) {
scenechange->n_diffs = 0;
memset (scenechange->diffs, 0, sizeof (double) * SC_N_DIFFS);
scenechange->oldbuf = gst_buffer_ref (buf);
return GST_FLOW_OK;
}
score = get_frame_score (GST_BUFFER_DATA (scenechange->oldbuf),
GST_BUFFER_DATA (buf), width, height);
memmove (scenechange->diffs, scenechange->diffs + 1,
sizeof (double) * (SC_N_DIFFS - 1));
scenechange->diffs[SC_N_DIFFS - 1] = score;
scenechange->n_diffs++;
gst_buffer_unref (scenechange->oldbuf);
scenechange->oldbuf = gst_buffer_ref (buf);
score_min = scenechange->diffs[0];
score_max = scenechange->diffs[0];
for (i = 1; i < SC_N_DIFFS - 1; i++) {
score_min = MIN (score_min, scenechange->diffs[i]);
score_max = MAX (score_max, scenechange->diffs[i]);
}
threshold = 1.8 * score_max - 0.8 * score_min;
if (scenechange->n_diffs > 2) {
if (score < 5) {
change = FALSE;
} else if (score / threshold < 1.0) {
change = FALSE;
} else if (score / threshold > 2.5) {
change = TRUE;
} else if (score > 50) {
change = TRUE;
} else {
change = FALSE;
}
} else {
change = FALSE;
}
#ifdef TESTING
if (change != is_shot_change (scenechange->n_diffs)) {
g_print ("%d %g %g %g %d\n", scenechange->n_diffs, score / threshold,
score, threshold, change);
}
#endif
if (change) {
GstEvent *event;
GST_DEBUG_OBJECT (scenechange, "%d %g %g %g %d",
scenechange->n_diffs, score / threshold, score, threshold, change);
event = gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM,
gst_structure_new ("GstForceKeyUnit", NULL));
gst_pad_push_event (GST_BASE_TRANSFORM_SRC_PAD (scenechange), event);
}
return GST_FLOW_OK;
}
static GstVideoFilter2Functions gst_scene_change_filter_functions[] = {
{GST_VIDEO_FORMAT_I420, NULL, gst_scene_change_filter_ip_I420},
{GST_VIDEO_FORMAT_UNKNOWN}
};
#ifdef TESTING
/* This is from ds's personal collection. No, you can't have it. */
int showreel_changes[] = {
242, 483, 510, 550, 579, 603, 609, 1056, 1067, 1074, 1079, 1096,
1106, 1113, 1127, 1145, 1156, 1170, 1212, 1228, 1243, 1269, 1274,
1322, 1349, 1370, 1378, 1423, 1456, 1458, 1508, 1519, 1542, 1679,
1767, 1837, 1895, 1962, 2006, 2035, 2102, 2139, 2196, 2561, 2664,
2837, 2895, 2985, 3035, 3077, 3128, 3176, 3218, 3306, 3351, 3388,
3421, 3470, 3711, 3832, 4029, 4184, 4444, 4686, 4719, 4825, 4941,
5009, 5091, 5194, 5254, 5286, 5287, 5343, 5431, 5501, 5634, 5695, 5788,
5839, 5861, 5930, 6030, 6168, 6193, 6237, 6336, 6376, 6421, 6495,
6550, 6611, 6669, 6733, 6819, 6852, 6944, 7087, 7148, 7189, 7431,
7540, 7599, 7632, 7661, 7693, 7930, 7963, 8003, 8076, 8109, 8147,
8177, 8192, 8219, 8278, 8322, 8370, 8409, 8566, 8603, 8747, 8775,
8873, 8907, 8955, 8969, 8983, 8997, 9026, 9079, 9140, 9165, 9206,
9276, 9378, 9449, 9523, 9647, 9703, 9749, 9790, 9929, 10056, 10216,
10307, 10411, 10487, 10557, 10695, 10770, 10854, 11095, 11265, 11517, 11589,
11686, 11825, 11940, 12004, 12047, 12113, 12179, 12233, 12532, 12586, 12708,
12793, 12877, 12954, 13030, 13105, 13177, 13279, 13396, 13486, 13538, 13561,
13591, 13627, 13656, 13709, 13763, 13815, 13842, 13876, 13906, 13929, 13955,
14003, 14070, 14097, 14127, 14153, 14198, 14269, 14348, 14367, 14440, 14488,
14548, 14573, 14599, 14630, 14665, 14907, 14962, 15013, 15089, 15148, 15227,
15314, 15355, 15369, 15451, 15470, 15542, 15570, 15640, 15684, 15781, 15869,
15938, 16172, 16266, 16429, 16479, 16521, 16563, 16612, 16671, 16692, 16704,
16720, 16756, 16789, 16802, 16815, 16867, 16908, 16939, 16953, 16977, 17006,
17014, 17026, 17040, 17062, 17121, 17176, 17226, 17322, 17444, 17496, 17641,
17698, 17744, 17826, 17913, 17993, 18073, 18219, 18279, 18359, 18475, 18544,
18587, 18649, 18698, 18756, 18826, 18853, 18866, 19108, 19336, 19481, 19544,
19720, 19816, 19908, 19982, 20069, 20310, 20355, 20374, 20409, 20469, 20599,
20607, 20652, 20805, 20822, 20882, 20982, 21029, 21433, 21468, 21561, 21602,
21661, 21720, 21909, 22045, 22166, 22225, 22323, 22362, 22433, 22477, 22529,
22571, 22617, 22642, 22676, 22918, 22978, 23084, 23161, 23288, 23409, 23490,
23613, 23721, 23815, 24131, 24372, 24468, 24507, 24555, 24568, 24616, 24634,
24829, 24843, 24919, 24992, 25040, 25160, 25288, 25607, 25684, 25717, 25764,
25821, 25866, 25901, 25925, 25941, 25978, 25998, 26011, 26030, 26055, 26118,
26133, 26145, 26159, 26175, 26182, 26195, 26205, 26238, 26258, 26316, 26340,
26581, 26725, 26834, 26874, 26995, 27065, 27178, 27238, 27365, 27607, 27669,
27694,
27774, 27800, 27841, 27930, 27985, 28057, 28091, 28132, 28189, 28270, 28545,
28653, 28711, 28770, 28886, 28966, 29139, 29241, 29356, 29415, 29490, 29576,
29659, 29776, 29842, 29910, 30029, 30056, 30100, 30129, 30175, 30316, 30376,
30441, 30551, 30666, 30784, 30843, 30948, 31045, 31286, 31315, 31534, 31607,
31742,
31817, 31853, 31984, 32009, 32112, 32162, 32210, 32264
};
/* Sintel */
int sintel_changes[] = {
752, 1018, 1036, 1056, 1078, 1100, 1169, 1319, 1339, 1370,
1425, 1455, 1494, 1552, 1572, 1637, 1663, 1777, 1955, 2060,
2125, 2429, 2624, 2780, 2835, 2881, 2955, 3032, 3144, 3217,
3315, 3384, 3740, 3890, 4234, 4261, 4322, 4368, 4425, 4481,
4555, 4605, 4671, 4714, 4743, 4875, 4920, 5082, 5158, 5267,
5379, 5956, 6021, 6071, 6112, 6139, 6221, 6318, 6374, 6519,
6558, 6615, 6691, 6803, 6900, 6944, 7134, 7266, 7351, 7414,
7467, 7503, 7559, 7573, 7656, 7733, 7876, 7929, 7971, 7985,
8047, 8099, 8144, 8215, 8394, 8435, 8480, 9133, 9190, 9525,
9962,
};
/* Breathe Out video, http://media.xiph.org/video/misc/ */
int breatheout_changes[] = {
143, 263, 334, 426, 462, 563, 583, 618, 655, 707,
818, 823, 858, 913, 956, 977, 999, 1073, 1124, 1144,
1166, 1187, 1206, 1227, 1240, 1264, 1289, 1312, 1477, 1535,
1646, 1692, 1739, 1757, 1798, 1855, 1974, 2048, 2129, 2212,
2369, 2412, 2463, 2578, 2649, 2699, 2778, 2857, 2923, 3014,
3107, 3246, 3321, 3350, 3459, 3498, 3541, 3567, 3613, 3636,
3673, 3709, 3747, 3834, 3862, 3902, 3922, 4022, 4117, 4262,
4303, 4357, 4556, 4578, 4617, 4716, 4792, 4873, 4895, 4917,
4932, 4972, 5015, 5034, 5058, 5090, 5162, 5180, 5202, 5222,
5239, 5258, 5281, 5298, 5397, 5430,
485, 507, 534, 665, 685, 755, 1023, 1379, 1441, 1503,
1584, 1621, 1903, 2081, 2281, 2511, 2958, 3071, 3185, 3214,
3271, 3424, 3479, 3588, 3879, 3979, 4043, 4062, 4143, 4207,
4237, 4336, 4461, 4476, 4533, 4647, 4815, 4853, 4949, 5075,
5142, 5316, 5376,
3514, 3952, 4384, 5337
};
#define changes showreel_changes
static gboolean
is_shot_change (int frame_number)
{
int i;
for (i = 0; i < sizeof (changes) / sizeof (changes[0]); i++) {
if (changes[i] == frame_number)
return TRUE;
}
return FALSE;
}
#endif

58
gst/videofilters/gstscenechange.h Normal file
View file

@ -0,0 +1,58 @@
/* GStreamer
* Copyright (C) 2011 FIXME <fixme@example.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#ifndef _GST_SCENE_CHANGE_H_
#define _GST_SCENE_CHANGE_H_
#include <gst/base/gstbasetransform.h>
#include <gst/video/video.h>
#include "gstvideofilter2.h"
G_BEGIN_DECLS
#define GST_TYPE_SCENE_CHANGE (gst_scene_change_get_type())
#define GST_SCENE_CHANGE(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_SCENE_CHANGE,GstSceneChange))
#define GST_SCENE_CHANGE_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_SCENE_CHANGE,GstSceneChangeClass))
#define GST_IS_SCENE_CHANGE(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_SCENE_CHANGE))
#define GST_IS_SCENE_CHANGE_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_SCENE_CHANGE))
typedef struct _GstSceneChange GstSceneChange;
typedef struct _GstSceneChangeClass GstSceneChangeClass;
#define SC_N_DIFFS 5
struct _GstSceneChange
{
GstVideoFilter2 base_scenechange;
int n_diffs;
double diffs[SC_N_DIFFS];
GstBuffer *oldbuf;
};
struct _GstSceneChangeClass
{
GstVideoFilter2Class base_scenechange_class;
};
GType gst_scene_change_get_type (void);
G_END_DECLS
#endif

2
gst/videofilters/gstvideofilter2.c
View file

@ -90,7 +90,7 @@ gst_video_filter2_base_init (gpointer g_class)
GstCaps *caps = NULL;
caps = gst_caps_new_empty ();
for (i = GST_VIDEO_FORMAT_I420; i <= GST_VIDEO_FORMAT_AYUV; i++) {
for (i = GST_VIDEO_FORMAT_I420; i <= GST_VIDEO_FORMAT_I420; i++) {
gst_caps_append (caps, gst_video_format_new_template_caps (i));
}

3
gst/videofilters/gstvideofiltersbad.c
View file

@ -23,6 +23,7 @@
#include <gst/gst.h>
#include <gst/base/gstbasetransform.h>
#include "gstscenechange.h"
#include "gstzebrastripe.h"
@ -30,6 +31,8 @@ static gboolean
plugin_init (GstPlugin * plugin)
{
gst_element_register (plugin, "scenechange", GST_RANK_NONE,
gst_scene_change_get_type ());
gst_element_register (plugin, "zebrastripe", GST_RANK_NONE,
gst_zebra_stripe_get_type ());