Adds a new Image Quality Assessment plugin.

It only offers one metric for now, "dssim", available if
https://github.com/pornel/dssim was installed on the system
at the time the plugin was compiled.

The spearman correlation for dssim against the TID2008 dataset
is 0.81, against 0.70 for the old ssim implementation, and
it runs 15 times faster.

https://bugzilla.gnome.org/show_bug.cgi?id=751324

configure.ac

@@ -358,6 +358,23 @@ AC_SUBST(EXIF_LIBS)
 AC_SUBST(EXIF_CFLAGS)
 AM_CONDITIONAL(USE_EXIF, test "x$HAVE_EXIF" = "xyes")
 
+AG_GST_CHECK_FEATURE(IQA, [iqa], iqa , [
+  PKG_CHECK_MODULES(DSSIM, dssim, [
+    HAVE_DSSIM="yes"
+    HAVE_IQA="yes"
+  ], [
+    HAVE_DSSIM="no"
+    HAVE_IQA="no"
+  ])
+
+  AM_CONDITIONAL(HAVE_DSSIM, test "x$HAVE_DSSIM" = "xyes")
+  if test "x$HAVE_DSSIM" = "xyes"; then
+    AC_DEFINE(HAVE_DSSIM, 1, [Define if you have dssim library])
+  fi
+  AC_SUBST(DSSIM_LIBS)
+  AC_SUBST(DSSIM_CFLAGS)
+])
+
 dnl Orc
 ORC_CHECK([0.4.17])
 
@@ -3525,6 +3542,7 @@ AM_CONDITIONAL(USE_GSM, false)
 AM_CONDITIONAL(USE_GTK3, false)
 AM_CONDITIONAL(USE_GTK3_GL, false)
 AM_CONDITIONAL(USE_HLS, false)
+AM_CONDITIONAL(USE_IQA, false)
 AM_CONDITIONAL(USE_KATE, false)
 AM_CONDITIONAL(USE_KMS, false)
 AM_CONDITIONAL(USE_TIGER, false)
@@ -3846,6 +3864,7 @@ ext/flite/Makefile
 ext/fluidsynth/Makefile
 ext/gsm/Makefile
 ext/hls/Makefile
+ext/iqa/Makefile
 ext/kate/Makefile
 ext/ladspa/Makefile
 ext/lv2/Makefile

docs/plugins/Makefile.am

@@ -75,6 +75,7 @@ EXTRA_HFILES = \
 	$(top_srcdir)/ext/dts/gstdtsdec.h \
 	$(top_srcdir)/ext/faac/gstfaac.h \
 	$(top_srcdir)/ext/faad/gstfaad.h \
+	$(top_srcdir)/ext/iqa/iqa.h \
 	$(top_srcdir)/ext/kate/gstkateenc.h \
 	$(top_srcdir)/ext/kate/gstkatedec.h \
 	$(top_srcdir)/ext/kate/gstkateparse.h \

docs/plugins/gst-plugins-bad-plugins-docs.sgml

@@ -104,6 +104,7 @@
     <xi:include href="xml/element-glvideomixerelement.xml" />
     <xi:include href="xml/element-glvideomixer.xml" />
     <xi:include href="xml/element-glviewconvert.xml" />
+    <xi:include href="xml/element-iqa.xml" />
     <xi:include href="xml/element-jpegparse.xml" />
     <xi:include href="xml/element-kaleidoscope.xml" />
     <xi:include href="xml/element-liveadder.xml" />

docs/plugins/gst-plugins-bad-plugins-sections.txt

@@ -2281,6 +2281,18 @@ GST_TYPE_INTERLACE
 gst_interlace_get_type
 </SECTION>
 
+<SECTION>
+<FILE>element-iqa</FILE>
+<TITLE>IQA</TITLE>
+Iqa
+<SUBSECTION Standard>
+IqaClass
+IQA
+GST_TYPE_IQA
+iqa_get_type
+gst_iqa_plugin_init
+</SECTION>
+
 <SECTION>
 <FILE>element-ivfparse</FILE>
 <TITLE>ivfparse</TITLE>

ext/Makefile.am

@@ -142,6 +142,12 @@ else
 GSM_DIR=
 endif
 
+if USE_IQA
+IQA_DIR = iqa
+else
+IQA_DIR =
+endif
+
 if USE_KATE
 KATE_DIR=kate
 else
@@ -457,6 +463,7 @@ SUBDIRS=\
 	$(FLUIDSYNTH_DIR) \
 	$(GSM_DIR) \
 	$(G729_DIR) \
+	$(IQA_DIR) \
 	$(KATE_DIR) \
 	$(LADSPA_DIR) \
 	$(LV2_DIR) \

ext/iqa/Makefile.am

@@ -0,0 +1,27 @@
+plugin_LTLIBRARIES = libgstiqa.la
+
+libgstiqa_la_SOURCES = \
+	iqa.c
+
+libgstiqa_la_CFLAGS =  \
+	-I$(top_srcdir)/gst-libs \
+	-I$(top_builddir)/gst-libs \
+	$(GST_PLUGINS_BASE_CFLAGS) \
+	$(GST_BASE_CFLAGS) $(GST_CFLAGS)
+
+libgstiqa_la_CFLAGS += $(DSSIM_CFLAGS)
+
+libgstiqa_la_LIBADD =  \
+	$(top_builddir)/gst-libs/gst/base/libgstbadbase-$(GST_API_VERSION).la \
+	$(top_builddir)/gst-libs/gst/video/libgstbadvideo-$(GST_API_VERSION).la \
+	$(GST_PLUGINS_BASE_LIBS) \
+	$(GST_BASE_LIBS) $(GST_LIBS)
+
+libgstiqa_la_LIBADD += $(DSSIM_LIBS)
+
+libgstiqa_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS)
+libgstiqa_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS)
+
+noinst_HEADERS = \
+	iqa.h
+

ext/iqa/iqa.c

@@ -0,0 +1,343 @@
+/* Image Quality Assessment plugin
+ * Copyright (C) 2015 Mathieu Duponchelle <mathieu.duponchelle@collabora.co.uk>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * SECTION:element-iqa
+ * @short_description: Image Quality Assessment plugin.
+ *
+ * IQA will perform full reference image quality assessment, with the
+ * first added pad being the reference.
+ *
+ * It will perform comparisons on video streams with the same geometry.
+ *
+ * The image output will be the heat map of differences, between
+ * the two pads with the highest measured difference.
+ *
+ * For each reference frame, IQA will post a message containing
+ * a structure named IQA.
+ *
+ * The only metric supported for now is "dssim", which will be available
+ * if https://github.com/pornel/dssim was installed on the system
+ * at the time that plugin was compiled.
+ *
+ * For each metric activated, this structure will contain another
+ * structure, named after the metric.
+ *
+ * The message will also contain a "time" field.
+ *
+ * For example, if do-dssim is set to true, and there are
+ * two compared streams, the emitted structure will look like this:
+ *
+ * IQA, dssim=(structure)"dssim\,\ sink_1\=\(double\)0.053621271267184856\,\
+ * sink_2\=\(double\)0.0082939683976297474\;",
+ * time=(guint64)0;
+ *
+ * <refsect2>
+ * <title>Example launch line</title>
+ * |[
+ * gst-launch-1.0 -m uridecodebin uri=file:///test/file/1 ! iqa name=iqa do-dssim=true \
+ * ! videoconvert ! autovideosink uridecodebin uri=file:///test/file/2 ! iqa.
+ * ]| This pipeline will output messages to the console for each set of compared frames.
+ * </refsect2>
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "iqa.h"
+
+#ifdef HAVE_DSSIM
+#include "dssim.h"
+#endif
+
+GST_DEBUG_CATEGORY_STATIC (gst_iqa_debug);
+#define GST_CAT_DEFAULT gst_iqa_debug
+
+#define SINK_FORMATS " { AYUV, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, "\
+                "   YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, "\
+                "   RGBx, BGRx } "
+
+#define SRC_FORMAT " { RGBA } "
+
+static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
+    GST_PAD_SRC,
+    GST_PAD_ALWAYS,
+    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SRC_FORMAT))
+    );
+
+enum
+{
+  PROP_0,
+  PROP_DO_SSIM,
+  PROP_LAST,
+};
+
+static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
+    GST_PAD_SINK,
+    GST_PAD_REQUEST,
+    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SINK_FORMATS))
+    );
+
+
+/* GstIqa */
+
+#define gst_iqa_parent_class parent_class
+G_DEFINE_TYPE (GstIqa, gst_iqa, GST_TYPE_VIDEO_AGGREGATOR);
+
+#ifdef HAVE_DSSIM
+
+inline static unsigned char
+to_byte (float in)
+{
+  if (in <= 0)
+    return 0;
+  if (in >= 255.f / 256.f)
+    return 255;
+  return in * 256.f;
+}
+
+static void
+do_dssim (GstIqa * self, GstVideoFrame * ref, GstVideoFrame * cmp,
+    GstBuffer * outbuf, GstStructure * msg_structure, gchar * padname)
+{
+  dssim_attr *attr = dssim_create_attr ();
+  gint y;
+  unsigned char **ptrs, **ptrs2;
+  GstMapInfo ref_info;
+  GstMapInfo cmp_info;
+  GstMapInfo out_info;
+  dssim_image *ref_image;
+  dssim_image *cmp_image;
+  double dssim;
+  dssim_ssim_map map_meta;
+  float *map;
+  gint i;
+  dssim_rgba *out;
+  GstStructure *dssim_structure;
+
+  gst_structure_get (msg_structure, "dssim", GST_TYPE_STRUCTURE,
+      &dssim_structure, NULL);
+
+  dssim_set_save_ssim_maps (attr, 1, 1);
+  if (ref->info.width != cmp->info.width ||
+      ref->info.height != cmp->info.height) {
+    GST_WARNING_OBJECT (self,
+        "Cannot compare two images with a different geometry yet");
+    return;
+  }
+
+  gst_buffer_map (ref->buffer, &ref_info, GST_MAP_READ);
+  gst_buffer_map (cmp->buffer, &cmp_info, GST_MAP_READ);
+  gst_buffer_map (outbuf, &out_info, GST_MAP_WRITE);
+  out = (dssim_rgba *) out_info.data;
+
+  ptrs = g_malloc (sizeof (char **) * ref->info.height);
+  ptrs2 = g_malloc (sizeof (char **) * cmp->info.height);
+
+  for (y = 0; y < ref->info.height; y++) {
+    ptrs[y] = ref_info.data + (ref->info.width * 4 * y);
+  }
+
+  ref_image =
+      dssim_create_image (attr, ptrs, DSSIM_RGBA, ref->info.width,
+      ref->info.height, 0.45455);
+
+  ptrs2 = g_malloc (sizeof (char **) * cmp->info.height);
+
+  for (y = 0; y < cmp->info.height; y++) {
+    ptrs2[y] = cmp_info.data + (cmp->info.width * 4 * y);
+  }
+
+  cmp_image =
+      dssim_create_image (attr, ptrs2, DSSIM_RGBA, cmp->info.width,
+      cmp->info.height, 0.45455);
+  dssim = dssim_compare (attr, ref_image, cmp_image);
+
+  map_meta = dssim_pop_ssim_map (attr, 0, 0);
+
+  if (dssim > self->max_dssim) {
+    map = map_meta.data;
+
+    for (i = 0; i < map_meta.width * map_meta.height; i++) {
+      const float max = 1.0 - map[i];
+      const float maxsq = max * max;
+      out[i] = (dssim_rgba) {
+      .r = to_byte (max * 3.0),.g = to_byte (maxsq * 6.0),.b =
+            to_byte (max / ((1.0 - map_meta.dssim) * 4.0)),.a = 255,};
+    }
+    self->max_dssim = dssim;
+  }
+
+  gst_structure_set (dssim_structure, padname, G_TYPE_DOUBLE, dssim, NULL);
+  gst_structure_set (msg_structure, "dssim", GST_TYPE_STRUCTURE,
+      dssim_structure, NULL);
+  gst_structure_free (dssim_structure);
+
+  g_free (ptrs);
+  g_free (ptrs2);
+  gst_buffer_unmap (ref->buffer, &ref_info);
+  gst_buffer_unmap (cmp->buffer, &cmp_info);
+  gst_buffer_unmap (outbuf, &out_info);
+  dssim_dealloc_image (ref_image);
+  dssim_dealloc_image (cmp_image);
+  dssim_dealloc_attr (attr);
+}
+#else
+static void
+do_dssim (GstIqa * self, GstVideoFrame * ref, GstVideoFrame * cmp,
+    GstBuffer * outbuf, GstStructure * msg_structure, gchar * padname)
+{
+}
+#endif
+
+static void
+compare_frames (GstIqa * self, GstVideoFrame * ref, GstVideoFrame * cmp,
+    GstBuffer * outbuf, GstStructure * msg_structure, gchar * padname)
+{
+  if (self->do_dssim)
+    do_dssim (self, ref, cmp, outbuf, msg_structure, padname);
+}
+
+static GstFlowReturn
+gst_iqa_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf)
+{
+  GList *l;
+  GstVideoFrame *ref_frame = NULL;
+  GstIqa *self = GST_IQA (vagg);
+  GstStructure *msg_structure = gst_structure_new_empty ("IQA");
+  GstMessage *m = gst_message_new_element (GST_OBJECT (self), msg_structure);
+  GstAggregator *agg = GST_AGGREGATOR (vagg);
+
+  if (self->do_dssim) {
+    gst_structure_set (msg_structure, "dssim", GST_TYPE_STRUCTURE,
+        gst_structure_new_empty ("dssim"), NULL);
+    self->max_dssim = 0.0;
+  }
+
+  GST_OBJECT_LOCK (vagg);
+  for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
+    GstVideoAggregatorPad *pad = l->data;
+
+    if (pad->aggregated_frame != NULL) {
+      if (!ref_frame) {
+        ref_frame = pad->aggregated_frame;
+      } else {
+        gchar *padname = gst_pad_get_name (pad);
+        GstVideoFrame *cmp_frame = pad->aggregated_frame;
+
+        compare_frames (self, ref_frame, cmp_frame, outbuf, msg_structure,
+            padname);
+        g_free (padname);
+      }
+    }
+  }
+
+  GST_OBJECT_UNLOCK (vagg);
+
+  /* We only post the message here, because we can't post it while the object
+   * is locked.
+   */
+  gst_structure_set (msg_structure, "time", GST_TYPE_CLOCK_TIME,
+      agg->segment.position, NULL);
+  gst_element_post_message (GST_ELEMENT (self), m);
+  return GST_FLOW_OK;
+}
+
+static void
+_set_property (GObject * object, guint prop_id, const GValue * value,
+    GParamSpec * pspec)
+{
+  GstIqa *self = GST_IQA (object);
+
+  switch (prop_id) {
+    case PROP_DO_SSIM:
+      self->do_dssim = g_value_get_boolean (value);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+_get_property (GObject * object,
+    guint prop_id, GValue * value, GParamSpec * pspec)
+{
+  GstIqa *self = GST_IQA (object);
+
+  switch (prop_id) {
+    case PROP_DO_SSIM:
+      g_value_set_boolean (value, self->do_dssim);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+/* GObject boilerplate */
+static void
+gst_iqa_class_init (GstIqaClass * klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+  GstElementClass *gstelement_class = (GstElementClass *) klass;
+  GstVideoAggregatorClass *videoaggregator_class =
+      (GstVideoAggregatorClass *) klass;
+
+  videoaggregator_class->aggregate_frames = gst_iqa_aggregate_frames;
+
+  gst_element_class_add_pad_template (gstelement_class,
+      gst_static_pad_template_get (&src_factory));
+  gst_element_class_add_pad_template (gstelement_class,
+      gst_static_pad_template_get (&sink_factory));
+
+  gobject_class->set_property = _set_property;
+  gobject_class->get_property = _get_property;
+
+#ifdef HAVE_DSSIM
+  g_object_class_install_property (gobject_class, PROP_DO_SSIM,
+      g_param_spec_boolean ("do-dssim", "do-dssim",
+          "Run structural similarity checks", FALSE, G_PARAM_READWRITE));
+#endif
+
+  gst_element_class_set_static_metadata (gstelement_class, "Iqa",
+      "Filter/Analyzer/Video",
+      "Provides various Image Quality Assessment metrics",
+      "Mathieu Duponchelle <mathieu.duponchelle@collabora.co.uk>");
+}
+
+static void
+gst_iqa_init (GstIqa * self)
+{
+}
+
+static gboolean
+plugin_init (GstPlugin * plugin)
+{
+  GST_DEBUG_CATEGORY_INIT (gst_iqa_debug, "iqa", 0, "iqa");
+
+  return gst_element_register (plugin, "iqa", GST_RANK_PRIMARY, GST_TYPE_IQA);
+}
+
+GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
+    GST_VERSION_MINOR,
+    iqa,
+    "Iqa", plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME,
+    GST_PACKAGE_ORIGIN)

ext/iqa/iqa.h

@@ -0,0 +1,64 @@
+/* Image Quality Assessment plugin
+ * Copyright (C) 2015 Mathieu Duponchelle <mathieu.duponchelle@collabora.co.uk>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_IQA_H__
+#define __GST_IQA_H__
+
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/video/gstvideoaggregator.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_IQA (gst_iqa_get_type())
+#define GST_IQA(obj) \
+        (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_IQA, GstIqa))
+#define GST_IQA_CLASS(klass) \
+        (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_IQA, GstIqaClass))
+#define GST_IS_IQA(obj) \
+        (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_IQA))
+#define GST_IS_IQA_CLASS(klass) \
+        (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_IQA))
+
+typedef struct _GstIqa GstIqa;
+typedef struct _GstIqaClass GstIqaClass;
+
+/**
+ * GstIqa:
+ *
+ * The opaque #GstIqa structure.
+ */
+struct _GstIqa
+{
+  GstVideoAggregator videoaggregator;
+
+  gboolean do_dssim;
+  double max_dssim;
+};
+
+struct _GstIqaClass
+{
+  GstVideoAggregatorClass parent_class;
+};
+
+GType gst_iqa_get_type (void);
+
+G_END_DECLS
+#endif /* __GST_IQA_H__ */
+