/* GStreamer Element * * Copyright 2011 Collabora Ltd. * @author: Mark Nauwelaerts * Copyright 2011 Nokia Corp. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 59 Temple Place - Suite 330, * Boston, MA 02111-1307, USA. */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include #include #include #include #include "gstcompare.h" GST_DEBUG_CATEGORY_STATIC (compare_debug); #define GST_CAT_DEFAULT compare_debug static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); static GstStaticPadTemplate check_sink_factory = GST_STATIC_PAD_TEMPLATE ("check", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); enum GstCompareMethod { GST_COMPARE_METHOD_MEM, GST_COMPARE_METHOD_MAX, GST_COMPARE_METHOD_SSIM }; #define GST_COMPARE_METHOD_TYPE (gst_compare_method_get_type()) static GType gst_compare_method_get_type (void) { static GType method_type = 0; static const GEnumValue method_types[] = { {GST_COMPARE_METHOD_MEM, "Memory", "mem"}, {GST_COMPARE_METHOD_MAX, "Maximum metric", "max"}, {GST_COMPARE_METHOD_SSIM, "SSIM (raw video)", "ssim"}, {0, NULL, NULL} }; if (!method_type) { method_type = g_enum_register_static ("GstCompareMethod", method_types); } return method_type; } /* Filter signals and args */ enum { /* FILL ME */ LAST_SIGNAL }; enum { PROP_0, PROP_META, PROP_OFFSET_TS, PROP_METHOD, PROP_THRESHOLD, PROP_UPPER, PROP_LAST }; #define DEFAULT_META GST_BUFFER_COPY_ALL #define DEFAULT_OFFSET_TS FALSE #define DEFAULT_METHOD GST_COMPARE_METHOD_MEM #define DEFAULT_THRESHOLD 0 #define DEFAULT_UPPER TRUE static void gst_compare_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_compare_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_compare_reset (GstCompare * overlay); static GstCaps *gst_compare_getcaps (GstPad * pad); static GstFlowReturn gst_compare_collect_pads (GstCollectPads * cpads, GstCompare * comp); static GstStateChangeReturn gst_compare_change_state (GstElement * element, GstStateChange transition); GST_BOILERPLATE (GstCompare, gst_compare, GstElement, GST_TYPE_ELEMENT); static void gst_compare_base_init (gpointer g_class) { GstElementClass *element_class = GST_ELEMENT_CLASS (g_class); gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&src_factory)); gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&sink_factory)); gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&check_sink_factory)); gst_element_class_set_details_simple (element_class, "Compare buffers", "Filter/Debug", "Compares incoming buffers", "Mark Nauwelaerts "); } static void gst_compare_finalize (GObject * object) { GstCompare *comp = GST_COMPARE (object); gst_object_unref (comp->cpads); G_OBJECT_CLASS (parent_class)->finalize (object); } static void gst_compare_class_init (GstCompareClass * klass) { GObjectClass *gobject_class; GstElementClass *gstelement_class; gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; GST_DEBUG_CATEGORY_INIT (compare_debug, "compare", 0, "Compare buffers"); gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_compare_change_state); gobject_class->set_property = gst_compare_set_property; gobject_class->get_property = gst_compare_get_property; gobject_class->finalize = gst_compare_finalize; g_object_class_install_property (gobject_class, PROP_META, g_param_spec_flags ("meta", "Compare Meta", "Indicates which metadata should be compared", gst_buffer_copy_flags_get_type (), DEFAULT_META, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_OFFSET_TS, g_param_spec_boolean ("offset-ts", "Offsets Timestamps", "Consider OFFSET and OFFSET_END part of timestamp metadata", DEFAULT_OFFSET_TS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_METHOD, g_param_spec_enum ("method", "Content Compare Method", "Method to compare buffer content", GST_COMPARE_METHOD_TYPE, DEFAULT_METHOD, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_THRESHOLD, g_param_spec_double ("threshold", "Content Threshold", "Threshold beyond which to consider content different as determined by content-method", 0, G_MAXDOUBLE, DEFAULT_THRESHOLD, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_UPPER, g_param_spec_boolean ("upper", "Threshold Upper Bound", "Whether threshold value is upper bound or lower bound for difference measure", DEFAULT_UPPER, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } static void gst_compare_init (GstCompare * comp, GstCompareClass * klass) { comp->cpads = gst_collect_pads_new (); gst_collect_pads_set_function (comp->cpads, (GstCollectPadsFunction) GST_DEBUG_FUNCPTR (gst_compare_collect_pads), comp); comp->sinkpad = gst_pad_new_from_static_template (&sink_factory, "sink"); gst_pad_set_getcaps_function (comp->sinkpad, gst_compare_getcaps); gst_element_add_pad (GST_ELEMENT (comp), comp->sinkpad); comp->checkpad = gst_pad_new_from_static_template (&check_sink_factory, "check"); gst_pad_set_getcaps_function (comp->checkpad, gst_compare_getcaps); gst_element_add_pad (GST_ELEMENT (comp), comp->checkpad); gst_collect_pads_add_pad (comp->cpads, comp->sinkpad, sizeof (GstCollectData), NULL); gst_collect_pads_add_pad (comp->cpads, comp->checkpad, sizeof (GstCollectData), NULL); comp->srcpad = gst_pad_new_from_static_template (&src_factory, "src"); gst_pad_set_getcaps_function (comp->srcpad, gst_compare_getcaps); gst_element_add_pad (GST_ELEMENT (comp), comp->srcpad); /* init properties */ comp->meta = DEFAULT_META; comp->offset_ts = DEFAULT_OFFSET_TS; comp->method = DEFAULT_METHOD; comp->threshold = DEFAULT_THRESHOLD; comp->upper = DEFAULT_UPPER; gst_compare_reset (comp); } static void gst_compare_reset (GstCompare * comp) { } static GstCaps * gst_compare_getcaps (GstPad * pad) { GstCompare *comp; GstPad *otherpad; GstCaps *result; comp = GST_COMPARE (gst_pad_get_parent (pad)); if (G_UNLIKELY (comp == NULL)) return gst_caps_new_any (); otherpad = (pad == comp->srcpad ? comp->sinkpad : comp->srcpad); result = gst_pad_peer_get_caps (otherpad); if (result == NULL) result = gst_caps_new_any (); gst_object_unref (comp); return result; } static void gst_compare_meta (GstCompare * comp, GstBuffer * buf1, GstBuffer * buf2) { gint flags = 0; if (comp->meta & GST_BUFFER_COPY_FLAGS) { if (GST_BUFFER_FLAGS (buf1) != GST_BUFFER_FLAGS (buf2)) { flags |= GST_BUFFER_COPY_FLAGS; GST_DEBUG_OBJECT (comp, "flags %d != flags %d", GST_BUFFER_FLAGS (buf1), GST_BUFFER_FLAGS (buf2)); } } if (comp->meta & GST_BUFFER_COPY_TIMESTAMPS) { if (GST_BUFFER_TIMESTAMP (buf1) != GST_BUFFER_TIMESTAMP (buf2)) { flags |= GST_BUFFER_COPY_TIMESTAMPS; GST_DEBUG_OBJECT (comp, "ts %" GST_TIME_FORMAT " != ts %" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf1)), GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf2))); } if (GST_BUFFER_DURATION (buf1) != GST_BUFFER_DURATION (buf2)) { flags |= GST_BUFFER_COPY_TIMESTAMPS; GST_DEBUG_OBJECT (comp, "dur %" GST_TIME_FORMAT " != dur %" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_DURATION (buf1)), GST_TIME_ARGS (GST_BUFFER_DURATION (buf2))); } if (comp->offset_ts) { if (GST_BUFFER_OFFSET (buf1) != GST_BUFFER_OFFSET (buf2)) { flags |= GST_BUFFER_COPY_TIMESTAMPS; GST_DEBUG_OBJECT (comp, "offset %" G_GINT64_FORMAT " != offset %" G_GINT64_FORMAT, GST_BUFFER_OFFSET (buf1), GST_BUFFER_OFFSET (buf2)); } if (GST_BUFFER_OFFSET_END (buf1) != GST_BUFFER_OFFSET_END (buf2)) { flags |= GST_BUFFER_COPY_TIMESTAMPS; GST_DEBUG_OBJECT (comp, "offset_end %" G_GINT64_FORMAT " != offset_end %" G_GINT64_FORMAT, GST_BUFFER_OFFSET_END (buf1), GST_BUFFER_OFFSET_END (buf2)); } } } if (comp->meta & GST_BUFFER_COPY_CAPS) { if (!gst_caps_is_equal (GST_BUFFER_CAPS (buf1), GST_BUFFER_CAPS (buf2))) { flags |= GST_BUFFER_COPY_CAPS; GST_DEBUG_OBJECT (comp, "caps %" GST_PTR_FORMAT " != caps %" GST_PTR_FORMAT, GST_BUFFER_CAPS (buf1), GST_BUFFER_CAPS (buf2)); } } /* signal mismatch by debug and message */ if (flags) { GST_WARNING_OBJECT (comp, "buffers %p and %p failed metadata match %d", buf1, buf2, flags); gst_element_post_message (GST_ELEMENT (comp), gst_message_new_element (GST_OBJECT (comp), gst_structure_new ("delta", "meta", G_TYPE_INT, flags, NULL))); } } /* when comparing contents, it is already ensured sizes are equal */ static gint gst_compare_mem (GstCompare * comp, GstBuffer * buf1, GstBuffer * buf2) { return memcmp (GST_BUFFER_DATA (buf1), GST_BUFFER_DATA (buf2), GST_BUFFER_SIZE (buf1)) ? 1 : 0; } static gint gst_compare_max (GstCompare * comp, GstBuffer * buf1, GstBuffer * buf2) { gint i, delta = 0; gint8 *data1, *data2; data1 = (gint8 *) GST_BUFFER_DATA (buf1); data2 = (gint8 *) GST_BUFFER_DATA (buf2); /* primitive loop */ for (i = 0; i < GST_BUFFER_SIZE (buf1); i++) { gint diff = ABS (*data1 - *data2); if (diff > 0) GST_LOG_OBJECT (comp, "diff at %d = %d", i, diff); delta = MAX (delta, ABS (*data1 - *data2)); data1++; data2++; } return delta; } static double gst_compare_ssim_window (GstCompare * comp, guint8 * data1, guint8 * data2, gint width, gint height, gint step, gint stride) { gint count = 0, i, j; gint sum1 = 0, sum2 = 0, ssum1 = 0, ssum2 = 0, acov = 0; gdouble avg1, avg2, var1, var2, cov; const gdouble k1 = 0.01; const gdouble k2 = 0.03; const gdouble L = 255.0; const gdouble c1 = (k1 * L) * (k1 * L); const gdouble c2 = (k2 * L) * (k2 * L); /* plain and simple; no fancy optimizations */ for (i = 0; i < height; i++) { for (j = 0; j < width; j++) { sum1 += *data1; sum2 += *data2; ssum1 += *data1 * *data1; ssum2 += *data2 * *data2; acov += *data1 * *data2; count++; data1 += step; data2 += step; } data1 -= j * step; data2 -= j * step; data1 += stride; data2 += stride; } avg1 = sum1 / count; avg2 = sum2 / count; var1 = ssum1 / count - avg1 * avg1; var2 = ssum2 / count - avg2 * avg2; cov = acov / count - avg1 * avg2; return (2 * avg1 * avg2 + c1) * (2 * cov + c2) / ((avg1 * avg1 + avg2 * avg2 + c1) * (var1 + var2 + c2)); } /* @width etc are for the particular component */ static gdouble gst_compare_ssim_component (GstCompare * comp, guint8 * data1, guint8 * data2, gint width, gint height, gint step, gint stride) { const gint window = 16; gdouble ssim_sum = 0; gint count = 0, i, j; for (j = 0; j + (window / 2) < height; j += (window / 2)) { for (i = 0; i + (window / 2) < width; i += (window / 2)) { gdouble ssim; ssim = gst_compare_ssim_window (comp, data1 + step * i + j * stride, data2 + step * i + j * stride, MIN (window, width - i), MIN (window, height - j), step, stride); GST_LOG_OBJECT (comp, "ssim for %dx%d at (%d, %d) = %f", window, window, i, j, ssim); ssim_sum += ssim; count++; } } return (ssim_sum / count); } static gdouble gst_compare_ssim (GstCompare * comp, GstBuffer * buf1, GstBuffer * buf2) { GstCaps *caps; GstVideoFormat format, f; gint width, height, w, h, i, comps; gdouble cssim[4], ssim, c[4] = { 1.0, 0.0, 0.0, 0.0 }; guint8 *data1, *data2; caps = GST_BUFFER_CAPS (buf1); if (!caps) goto invalid_input; if (!gst_video_format_parse_caps (caps, &format, &width, &height)) goto invalid_input; caps = GST_BUFFER_CAPS (buf2); if (!caps) goto invalid_input; if (!gst_video_format_parse_caps (caps, &f, &w, &h)) goto invalid_input; if (f != format || w != width || h != height) return comp->threshold + 1; comps = gst_video_format_is_gray (format) ? 1 : 3; if (gst_video_format_has_alpha (format)) comps += 1; /* note that some are reported both yuv and gray */ for (i = 0; i < comps; ++i) c[i] = 1.0; /* increase luma weight if yuv */ if (gst_video_format_is_yuv (format) && (comps > 1)) c[0] = comps - 1; for (i = 0; i < comps; ++i) c[i] /= (gst_video_format_is_yuv (format) && (comps > 1)) ? 2 * (comps - 1) : comps; data1 = GST_BUFFER_DATA (buf1); data2 = GST_BUFFER_DATA (buf2); for (i = 0; i < comps; i++) { gint offset, cw, ch, step, stride; /* only support most common formats */ if (gst_video_format_get_component_depth (format, i) != 8) goto unsupported_input; offset = gst_video_format_get_component_offset (format, i, width, height); cw = gst_video_format_get_component_width (format, i, width); ch = gst_video_format_get_component_height (format, i, height); step = gst_video_format_get_pixel_stride (format, i); stride = gst_video_format_get_row_stride (format, i, width); GST_LOG_OBJECT (comp, "component %d", i); cssim[i] = gst_compare_ssim_component (comp, data1 + offset, data2 + offset, cw, ch, step, stride); GST_LOG_OBJECT (comp, "ssim[%d] = %f", i, cssim[i]); } #ifndef GST_DISABLE_GST_DEBUG for (i = 0; i < 4; i++) { GST_DEBUG_OBJECT (comp, "ssim[%d] = %f, c[%d] = %f", i, cssim[i], i, c[i]); } #endif ssim = cssim[0] * c[0] + cssim[1] * c[1] + cssim[2] * c[2] + cssim[3] * c[3]; return ssim; /* ERRORS */ invalid_input: { GST_ERROR_OBJECT (comp, "ssim method needs raw video input"); return 0; } unsupported_input: { GST_ERROR_OBJECT (comp, "raw video format not supported %" GST_PTR_FORMAT, caps); return 0; } } static void gst_compare_buffers (GstCompare * comp, GstBuffer * buf1, GstBuffer * buf2) { gdouble delta = 0; /* first check metadata */ gst_compare_meta (comp, buf1, buf2); /* check content according to method */ /* but at least size should match */ if (GST_BUFFER_SIZE (buf1) != GST_BUFFER_SIZE (buf2)) { delta = comp->threshold + 1; } else { GST_MEMDUMP_OBJECT (comp, "buffer 1", GST_BUFFER_DATA (buf1), GST_BUFFER_SIZE (buf1)); GST_MEMDUMP_OBJECT (comp, "buffer 2", GST_BUFFER_DATA (buf2), GST_BUFFER_SIZE (buf2)); switch (comp->method) { case GST_COMPARE_METHOD_MEM: delta = gst_compare_mem (comp, buf1, buf2); break; case GST_COMPARE_METHOD_MAX: delta = gst_compare_max (comp, buf1, buf2); break; case GST_COMPARE_METHOD_SSIM: delta = gst_compare_ssim (comp, buf1, buf2); break; default: g_assert_not_reached (); break; } } if ((comp->upper && delta > comp->threshold) || (!comp->upper && delta < comp->threshold)) { GST_WARNING_OBJECT (comp, "buffers %p and %p failed content match %f", buf1, buf2, delta); gst_element_post_message (GST_ELEMENT (comp), gst_message_new_element (GST_OBJECT (comp), gst_structure_new ("delta", "content", G_TYPE_DOUBLE, delta, NULL))); } } static GstFlowReturn gst_compare_collect_pads (GstCollectPads * cpads, GstCompare * comp) { GstBuffer *buf1, *buf2; buf1 = gst_collect_pads_pop (comp->cpads, gst_pad_get_element_private (comp->sinkpad)); buf2 = gst_collect_pads_pop (comp->cpads, gst_pad_get_element_private (comp->checkpad)); if (!buf1 && !buf2) { gst_pad_push_event (comp->srcpad, gst_event_new_eos ()); return GST_FLOW_UNEXPECTED; } else if (buf1 && buf2) { gst_compare_buffers (comp, buf1, buf2); } else { GST_WARNING_OBJECT (comp, "buffer %p != NULL", buf1 ? buf1 : buf2); comp->count++; gst_element_post_message (GST_ELEMENT (comp), gst_message_new_element (GST_OBJECT (comp), gst_structure_new ("delta", "count", G_TYPE_INT, comp->count, NULL))); } if (buf1) gst_pad_push (comp->srcpad, buf1); if (buf2) gst_buffer_unref (buf2); return GST_FLOW_OK; } static void gst_compare_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { GstCompare *comp = GST_COMPARE (object); switch (prop_id) { case PROP_META: comp->meta = g_value_get_flags (value); break; case PROP_OFFSET_TS: comp->offset_ts = g_value_get_boolean (value); break; case PROP_METHOD: comp->method = g_value_get_enum (value); break; case PROP_THRESHOLD: comp->threshold = g_value_get_double (value); break; case PROP_UPPER: comp->upper = g_value_get_boolean (value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } } static void gst_compare_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstCompare *comp = GST_COMPARE (object); switch (prop_id) { case PROP_META: g_value_set_flags (value, comp->meta); break; case PROP_OFFSET_TS: g_value_set_boolean (value, comp->offset_ts); break; case PROP_METHOD: g_value_set_enum (value, comp->method); break; case PROP_THRESHOLD: g_value_set_double (value, comp->threshold); break; case PROP_UPPER: g_value_set_boolean (value, comp->upper); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } } static GstStateChangeReturn gst_compare_change_state (GstElement * element, GstStateChange transition) { GstCompare *comp = GST_COMPARE (element); GstStateChangeReturn ret; switch (transition) { case GST_STATE_CHANGE_NULL_TO_READY: case GST_STATE_CHANGE_READY_TO_PAUSED: gst_collect_pads_start (comp->cpads); break; case GST_STATE_CHANGE_PAUSED_TO_READY: gst_collect_pads_stop (comp->cpads); break; default: break; } ret = GST_CALL_PARENT_WITH_DEFAULT (GST_ELEMENT_CLASS, change_state, (element, transition), GST_STATE_CHANGE_SUCCESS); if (ret != GST_STATE_CHANGE_SUCCESS) return ret; switch (transition) { case GST_STATE_CHANGE_PAUSED_TO_READY: gst_compare_reset (comp); break; default: break; } return GST_STATE_CHANGE_SUCCESS; }