/* GStreamer * Copyright (C) <2009> Руслан Ижбулатов * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301 USA */ /** * SECTION:element-ssim * * The ssim calculates SSIM (Structural SIMilarity) index for two or more * streams, for each frame. * First stream is the original, other streams are modified (compressed) ones. * ssim will calculate SSIM index of each frame of each modified stream, using * original stream as a reference. * * The ssim accepts only YUV planar top-first data and calculates only Y-SSIM. * All streams must have the same width, height and colorspace. * Output streams are greyscale video streams, where bright pixels indicate * high SSIM values, dark pixels - low SSIM values. * The ssim also calculates mean SSIM index for each frame and emits is as a * message. * ssim is intended to be used with videomeasure_collector element to catch the * events (such as mean SSIM index values) and save them into a file. * * * Example launch line * |[ * gst-launch ssim name=ssim ssim.src0 ! ffmpegcolorspace ! glimagesink filesrc * location=orig.avi ! decodebin2 ! ssim.original filesrc location=compr.avi ! * decodebin2 ! ssim.modified0 * ]| This pipeline produces a video stream that consists of SSIM frames. * * * Last reviewed on 2009-09-06 (0.10.?) */ /* Element-Checklist-Version: 5 */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include "gstvideomeasure.h" #include "gstvideomeasure_ssim.h" #include #include #include #include #define GST_CAT_DEFAULT gst_ssim_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); /* elementfactory information */ #define SINK_CAPS \ "video/x-raw-yuv, " \ "format = (fourcc) { I420, YV12, Y41B, Y42B } " #define SRC_CAPS \ "video/x-raw-gray, " \ "width = (int) [ 1, MAX ], " \ "height = (int) [ 1, MAX ], " \ "framerate = (fraction) [ 0/1, MAX ], " \ "bpp = (int) 8, " \ "depth = (int) 8 " static GstStaticPadTemplate gst_ssim_src_template = GST_STATIC_PAD_TEMPLATE ("src%d", GST_PAD_SRC, GST_PAD_SOMETIMES, GST_STATIC_CAPS (SRC_CAPS) ); static GstStaticPadTemplate gst_ssim_sink_original_template = GST_STATIC_PAD_TEMPLATE ("original", GST_PAD_SINK, GST_PAD_REQUEST, GST_STATIC_CAPS (SINK_CAPS) ); static GstStaticPadTemplate gst_ssim_sink_modified_template = GST_STATIC_PAD_TEMPLATE ("modified%d", GST_PAD_SINK, GST_PAD_REQUEST, GST_STATIC_CAPS (SINK_CAPS) ); static void gst_ssim_class_init (GstSSimClass * klass); static void gst_ssim_init (GstSSim * ssim); static void gst_ssim_finalize (GObject * object); static gboolean gst_ssim_setcaps (GstPad * pad, GstCaps * caps); static gboolean gst_ssim_query (GstPad * pad, GstQuery * query); static gboolean gst_ssim_src_event (GstPad * pad, GstEvent * event); static gboolean gst_ssim_sink_event (GstPad * pad, GstEvent * event); static GstPad *gst_ssim_request_new_pad (GstElement * element, GstPadTemplate * temp, const gchar * unused); static void gst_ssim_release_pad (GstElement * element, GstPad * pad); static GstStateChangeReturn gst_ssim_change_state (GstElement * element, GstStateChange transition); static GstFlowReturn gst_ssim_collected (GstCollectPads2 * pads, gpointer user_data); static GstElementClass *parent_class = NULL; GType gst_ssim_get_type (void) { static GType ssim_type = 0; if (G_UNLIKELY (ssim_type == 0)) { static const GTypeInfo ssim_info = { sizeof (GstSSimClass), NULL, NULL, (GClassInitFunc) gst_ssim_class_init, NULL, NULL, sizeof (GstSSim), 0, (GInstanceInitFunc) gst_ssim_init, }; ssim_type = g_type_register_static (GST_TYPE_ELEMENT, "GstSSim", &ssim_info, 0); GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "ssim", 0, "SSIM calculator"); } return ssim_type; } static void gst_ssim_post_message (GstSSim * ssim, GstBuffer * buffer, gfloat mssim, gfloat lowest, gfloat highest) { GstMessage *m; guint64 offset; offset = GST_BUFFER_OFFSET (buffer); m = gst_message_new_element (GST_OBJECT_CAST (ssim), gst_structure_new ("SSIM", "offset", G_TYPE_UINT64, offset, "timestamp", GST_TYPE_CLOCK_TIME, GST_BUFFER_TIMESTAMP (buffer), "mean", G_TYPE_FLOAT, mssim, "lowest", G_TYPE_FLOAT, lowest, "highest", G_TYPE_FLOAT, highest, NULL)); GST_DEBUG_OBJECT (GST_OBJECT (ssim), "Frame %" G_GINT64_FORMAT " @ %" GST_TIME_FORMAT " mean SSIM is %f, l-h is %f-%f", offset, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer)), mssim, lowest, highest); gst_element_post_message (GST_ELEMENT_CAST (ssim), m); } static GstCaps * gst_ssim_src_getcaps (GstPad * pad) { GstCaps *result; gchar *capstr; result = gst_caps_copy (gst_pad_get_pad_template_caps (pad)); capstr = gst_caps_to_string (result); GST_DEBUG ("getsrccaps - return static caps: %s", capstr); g_free (capstr); return result; } static GstCaps * gst_ssim_sink_getcaps (GstPad * pad) { GstCaps *result = NULL; GstSSim *ssim; gchar *capstr; ssim = GST_SSIM (GST_PAD_PARENT (pad)); GST_OBJECT_LOCK (ssim); result = gst_pad_get_fixed_caps_func (pad); capstr = gst_caps_to_string (result); GST_DEBUG ("getsinkcaps - return caps: %s", capstr); g_free (capstr); GST_OBJECT_UNLOCK (ssim); return result; } static void calculate_mu (GstSSim * ssim, gfloat * outmu, guint8 * buf) { gint oy, ox, iy, ix; for (oy = 0; oy < ssim->height; oy++) { for (ox = 0; ox < ssim->width; ox++) { gfloat mu = 0; gfloat elsumm; gint weight_y_base, weight_x_base; gint weight_offset; gint pixel_offset; gint winstart_y; gint wghstart_y; gint winend_y; gint winstart_x; gint wghstart_x; gint winend_x; gfloat weight; gint source_offset; source_offset = oy * ssim->width + ox; winstart_x = ssim->windows[source_offset].x_window_start; wghstart_x = ssim->windows[source_offset].x_weight_start; winend_x = ssim->windows[source_offset].x_window_end; winstart_y = ssim->windows[source_offset].y_window_start; wghstart_y = ssim->windows[source_offset].y_weight_start; winend_y = ssim->windows[source_offset].y_window_end; elsumm = ssim->windows[source_offset].element_summ; switch (ssim->windowtype) { case 0: for (iy = winstart_y; iy <= winend_y; iy++) { pixel_offset = iy * ssim->width; for (ix = winstart_x; ix <= winend_x; ix++) mu += buf[pixel_offset + ix]; } mu = mu / elsumm; break; case 1: weight_y_base = wghstart_y - winstart_y; weight_x_base = wghstart_x - winstart_x; for (iy = winstart_y; iy <= winend_y; iy++) { pixel_offset = iy * ssim->width; weight_offset = (weight_y_base + iy) * ssim->windowsize + weight_x_base; for (ix = winstart_x; ix <= winend_x; ix++) { weight = ssim->weights[weight_offset + ix]; mu += weight * buf[pixel_offset + ix]; } } mu = mu / elsumm; break; } outmu[oy * ssim->width + ox] = mu; } } } static void calcssim_without_mu (GstSSim * ssim, guint8 * org, gfloat * orgmu, guint8 * mod, guint8 * out, gfloat * mean, gfloat * lowest, gfloat * highest) { gint oy, ox, iy, ix; gfloat cumulative_ssim = 0; *lowest = G_MAXFLOAT; *highest = -G_MAXFLOAT; for (oy = 0; oy < ssim->height; oy++) { for (ox = 0; ox < ssim->width; ox++) { gfloat mu_o = 128, mu_m = 128; gdouble sigma_o = 0, sigma_m = 0, sigma_om = 0; gfloat tmp1 = 0, tmp2 = 0; gfloat elsumm = 0; gint weight_y_base, weight_x_base; gint weight_offset; gint pixel_offset; gint winstart_y; gint wghstart_y; gint winend_y; gint winstart_x; gint wghstart_x; gint winend_x; gfloat weight; gint source_offset; source_offset = oy * ssim->width + ox; winstart_x = ssim->windows[source_offset].x_window_start; wghstart_x = ssim->windows[source_offset].x_weight_start; winend_x = ssim->windows[source_offset].x_window_end; winstart_y = ssim->windows[source_offset].y_window_start; wghstart_y = ssim->windows[source_offset].y_weight_start; winend_y = ssim->windows[source_offset].y_window_end; elsumm = ssim->windows[source_offset].element_summ; weight_y_base = wghstart_y - winstart_y; weight_x_base = wghstart_x - winstart_x; switch (ssim->windowtype) { case 0: for (iy = winstart_y; iy <= winend_y; iy++) { guint8 *org_with_offset, *mod_with_offset; pixel_offset = iy * ssim->width; org_with_offset = &org[pixel_offset]; mod_with_offset = &mod[pixel_offset]; for (ix = winstart_x; ix <= winend_x; ix++) { tmp1 = org_with_offset[ix] - mu_o; sigma_o += tmp1 * tmp1; tmp2 = mod_with_offset[ix] - mu_m; sigma_m += tmp2 * tmp2; sigma_om += tmp1 * tmp2; } } break; case 1: weight_y_base = wghstart_y - winstart_y; weight_x_base = wghstart_x - winstart_x; for (iy = winstart_y; iy <= winend_y; iy++) { guint8 *org_with_offset, *mod_with_offset; gfloat *weights_with_offset; gfloat wt1, wt2; pixel_offset = iy * ssim->width; weight_offset = (weight_y_base + iy) * ssim->windowsize + weight_x_base; org_with_offset = &org[pixel_offset]; mod_with_offset = &mod[pixel_offset]; weights_with_offset = &ssim->weights[weight_offset]; for (ix = winstart_x; ix <= winend_x; ix++) { weight = weights_with_offset[ix]; tmp1 = org_with_offset[ix] - mu_o; tmp2 = mod_with_offset[ix] - mu_m; wt1 = weight * tmp1; wt2 = weight * tmp2; sigma_o += wt1 * tmp1; sigma_m += wt2 * tmp2; sigma_om += wt1 * tmp2; } } break; } sigma_o = sqrt (sigma_o / elsumm); sigma_m = sqrt (sigma_m / elsumm); sigma_om = sigma_om / elsumm; tmp1 = (2 * mu_o * mu_m + ssim->const1) * (2 * sigma_om + ssim->const2) / ((mu_o * mu_o + mu_m * mu_m + ssim->const1) * (sigma_o * sigma_o + sigma_m * sigma_m + ssim->const2)); /* SSIM can go negative, that's why it is 127 + index * 128 instead of index * 255 */ out[oy * ssim->width + ox] = 127 + tmp1 * 128; *lowest = MIN (*lowest, tmp1); *highest = MAX (*highest, tmp1); cumulative_ssim += tmp1; } } *mean = cumulative_ssim / (ssim->width * ssim->height); } static void calcssim_canonical (GstSSim * ssim, guint8 * org, gfloat * orgmu, guint8 * mod, guint8 * out, gfloat * mean, gfloat * lowest, gfloat * highest) { gint oy, ox, iy, ix; gfloat cumulative_ssim = 0; *lowest = G_MAXFLOAT; *highest = -G_MAXFLOAT; for (oy = 0; oy < ssim->height; oy++) { for (ox = 0; ox < ssim->width; ox++) { gfloat mu_o = 0, mu_m = 0; gdouble sigma_o = 0, sigma_m = 0, sigma_om = 0; gfloat tmp1, tmp2; gfloat elsumm = 0; gint weight_y_base, weight_x_base; gint weight_offset; gint pixel_offset; gint winstart_y; gint wghstart_y; gint winend_y; gint winstart_x; gint wghstart_x; gint winend_x; gfloat weight; gint source_offset; source_offset = oy * ssim->width + ox; winstart_x = ssim->windows[source_offset].x_window_start; wghstart_x = ssim->windows[source_offset].x_weight_start; winend_x = ssim->windows[source_offset].x_window_end; winstart_y = ssim->windows[source_offset].y_window_start; wghstart_y = ssim->windows[source_offset].y_weight_start; winend_y = ssim->windows[source_offset].y_window_end; elsumm = ssim->windows[source_offset].element_summ; switch (ssim->windowtype) { case 0: for (iy = winstart_y; iy <= winend_y; iy++) { pixel_offset = iy * ssim->width; for (ix = winstart_x; ix <= winend_x; ix++) { mu_m += mod[pixel_offset + ix]; } } mu_m = mu_m / elsumm; mu_o = orgmu[oy * ssim->width + ox]; for (iy = winstart_y; iy <= winend_y; iy++) { pixel_offset = iy * ssim->width; for (ix = winstart_x; ix <= winend_x; ix++) { tmp1 = org[pixel_offset + ix] - mu_o; tmp2 = mod[pixel_offset + ix] - mu_m; sigma_o += tmp1 * tmp1; sigma_m += tmp2 * tmp2; sigma_om += tmp1 * tmp2; } } break; case 1: weight_y_base = wghstart_y - winstart_y; weight_x_base = wghstart_x - winstart_x; for (iy = winstart_y; iy <= winend_y; iy++) { pixel_offset = iy * ssim->width; weight_offset = (weight_y_base + iy) * ssim->windowsize + weight_x_base; for (ix = winstart_x; ix <= winend_x; ix++) { weight = ssim->weights[weight_offset + ix]; mu_o += weight * org[pixel_offset + ix]; mu_m += weight * mod[pixel_offset + ix]; } } mu_m = mu_m / elsumm; mu_o = orgmu[oy * ssim->width + ox]; for (iy = winstart_y; iy <= winend_y; iy++) { gfloat *weights_with_offset; guint8 *org_with_offset, *mod_with_offset; gfloat wt1, wt2; pixel_offset = iy * ssim->width; weight_offset = (weight_y_base + iy) * ssim->windowsize + weight_x_base; weights_with_offset = &ssim->weights[weight_offset]; org_with_offset = &org[pixel_offset]; mod_with_offset = &mod[pixel_offset]; for (ix = winstart_x; ix <= winend_x; ix++) { weight = weights_with_offset[ix]; tmp1 = org_with_offset[ix] - mu_o; tmp2 = mod_with_offset[ix] - mu_m; wt1 = weight * tmp1; wt2 = weight * tmp2; sigma_o += wt1 * tmp1; sigma_m += wt2 * tmp2; sigma_om += wt1 * tmp2; } } break; } sigma_o = sqrt (sigma_o / elsumm); sigma_m = sqrt (sigma_m / elsumm); sigma_om = sigma_om / elsumm; tmp1 = (2 * mu_o * mu_m + ssim->const1) * (2 * sigma_om + ssim->const2) / ((mu_o * mu_o + mu_m * mu_m + ssim->const1) * (sigma_o * sigma_o + sigma_m * sigma_m + ssim->const2)); /* SSIM can go negative, that's why it is 127 + index * 128 instead of index * 255 */ out[oy * ssim->width + ox] = 127 + tmp1 * 128; *lowest = MIN (*lowest, tmp1); *highest = MAX (*highest, tmp1); cumulative_ssim += tmp1; } } *mean = cumulative_ssim / (ssim->width * ssim->height); } /* the first caps we receive on any of the sinkpads will define the caps for all * the other sinkpads because we can only measure streams with the same caps. */ static gboolean gst_ssim_setcaps (GstPad * pad, GstCaps * caps) { GstSSim *ssim; GList *pads; const char *media_type; GstStructure *capsstr; gint width, height, fps_n, fps_d; guint32 fourcc; ssim = GST_SSIM (GST_PAD_PARENT (pad)); GST_DEBUG_OBJECT (ssim, "setting caps on pad %p,%s to %" GST_PTR_FORMAT, pad, GST_PAD_NAME (pad), caps); capsstr = gst_caps_get_structure (caps, 0); gst_structure_get_int (capsstr, "width", &width); gst_structure_get_int (capsstr, "height", &height); gst_structure_get_fraction (capsstr, "framerate", &fps_n, &fps_d); gst_structure_get_fourcc (capsstr, "format", &fourcc); GST_OBJECT_LOCK (ssim); /* Sink caps are stored only once. At the moment it doesn't feel * right to measure streams with variable caps. */ if (G_UNLIKELY (!ssim->sinkcaps)) { GstStructure *newstr; GValue list = { 0, } , fourcc = { 0,}; g_value_init (&list, GST_TYPE_LIST); g_value_init (&fourcc, GST_TYPE_FOURCC); gst_value_set_fourcc (&fourcc, GST_MAKE_FOURCC ('I', '4', '2', '0')); gst_value_list_append_value (&list, &fourcc); gst_value_set_fourcc (&fourcc, GST_MAKE_FOURCC ('Y', 'V', '1', '2')); gst_value_list_append_value (&list, &fourcc); gst_value_set_fourcc (&fourcc, GST_MAKE_FOURCC ('Y', '4', '1', 'B')); gst_value_list_append_value (&list, &fourcc); gst_value_set_fourcc (&fourcc, GST_MAKE_FOURCC ('Y', '4', '2', 'B')); gst_value_list_append_value (&list, &fourcc); newstr = gst_structure_new ("video/x-raw-yuv", NULL); gst_structure_set (newstr, "width", G_TYPE_INT, width, NULL); gst_structure_set (newstr, "height", G_TYPE_INT, height, NULL); gst_structure_set_value (newstr, "format", &list); ssim->sinkcaps = gst_caps_new_full (newstr, NULL); g_value_unset (&list); g_value_unset (&fourcc); } if (G_UNLIKELY (!ssim->srccaps)) { GstStructure *newstr; newstr = gst_structure_new ("video/x-raw-gray", NULL); gst_structure_set (newstr, "width", G_TYPE_INT, width, NULL); gst_structure_set (newstr, "height", G_TYPE_INT, height, NULL); gst_structure_set (newstr, "framerate", GST_TYPE_FRACTION, fps_n, fps_d, NULL); /* Calculates SSIM only for Y channel, hence the output is monochrome. * TODO: an option (a mask?) to calculate SSIM for more than one channel, * will probably output RGB, one metric per channel...that would * look kinda funny :) */ gst_structure_set (newstr, "bpp", G_TYPE_INT, 8, "depth", G_TYPE_INT, 8, NULL); ssim->srccaps = gst_caps_new_full (newstr, NULL); } pads = GST_ELEMENT (ssim)->pads; while (pads) { GstPadDirection direction; GstPad *otherpad = GST_PAD (pads->data); direction = gst_pad_get_direction (otherpad); GST_DEBUG_OBJECT (ssim, "checking caps on pad %p", otherpad); if (direction == GST_PAD_SINK) { gchar *capstr; capstr = gst_caps_to_string (GST_PAD_CAPS (otherpad)); GST_DEBUG_OBJECT (ssim, "old caps on pad %p,%s were %s", otherpad, GST_PAD_NAME (otherpad), capstr); g_free (capstr); gst_caps_replace (&GST_PAD_CAPS (otherpad), ssim->sinkcaps); capstr = gst_caps_to_string (ssim->sinkcaps); GST_DEBUG_OBJECT (ssim, "new caps on pad %p,%s are %s", otherpad, GST_PAD_NAME (otherpad), capstr); g_free (capstr); } else if (direction == GST_PAD_SRC) { gst_caps_replace (&GST_PAD_CAPS (otherpad), ssim->srccaps); } pads = g_list_next (pads); } /* parse caps now */ media_type = gst_structure_get_name (capsstr); GST_DEBUG_OBJECT (ssim, "media type is %s", media_type); if (strcmp (media_type, "video/x-raw-yuv") == 0) { ssim->width = width; ssim->height = height; ssim->frame_rate = fps_n; ssim->frame_rate_base = fps_d; GST_INFO_OBJECT (ssim, "parse_caps sets ssim to yuv format " "%d, %dx%d, %d/%d fps", fourcc, ssim->width, ssim->height, ssim->frame_rate, ssim->frame_rate_base); /* Only planar formats are supported. * TODO: implement support for interleaved formats * Only YUV formats are supported. There's no sense in calculating the * index for R, G or B channels separately. */ switch (fourcc) { case GST_MAKE_FOURCC ('I', '4', '2', '0'): case GST_MAKE_FOURCC ('Y', 'V', '1', '2'): case GST_MAKE_FOURCC ('Y', '4', '1', 'B'): case GST_MAKE_FOURCC ('Y', '4', '2', 'B'): break; default: goto not_supported; } } else { goto not_supported; } GST_OBJECT_UNLOCK (ssim); return TRUE; /* ERRORS */ not_supported: { GST_OBJECT_UNLOCK (ssim); GST_DEBUG_OBJECT (ssim, "unsupported format set as caps"); return FALSE; } } static gboolean gst_ssim_query_latency (GstSSim * ssim, GstQuery * query) { GstClockTime min, max; gboolean live; gboolean res; GstIterator *it; gboolean done; res = TRUE; done = FALSE; live = FALSE; min = 0; max = GST_CLOCK_TIME_NONE; /* Take maximum of all latency values */ it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (ssim)); while (!done) { GstIteratorResult ires; gpointer item; ires = gst_iterator_next (it, &item); switch (ires) { case GST_ITERATOR_DONE: done = TRUE; break; case GST_ITERATOR_OK: { GstPad *pad = GST_PAD_CAST (item); GstQuery *peerquery; GstClockTime min_cur, max_cur; gboolean live_cur; peerquery = gst_query_new_latency (); /* Ask peer for latency */ res &= gst_pad_peer_query (pad, peerquery); /* take max from all valid return values */ if (res) { gst_query_parse_latency (peerquery, &live_cur, &min_cur, &max_cur); if (min_cur > min) min = min_cur; if (max_cur != GST_CLOCK_TIME_NONE && ((max != GST_CLOCK_TIME_NONE && max_cur > max) || (max == GST_CLOCK_TIME_NONE))) max = max_cur; live = live || live_cur; } gst_query_unref (peerquery); gst_object_unref (pad); break; } case GST_ITERATOR_RESYNC: live = FALSE; min = 0; max = GST_CLOCK_TIME_NONE; res = TRUE; gst_iterator_resync (it); break; default: res = FALSE; done = TRUE; break; } } gst_iterator_free (it); if (res) { /* store the results */ GST_DEBUG_OBJECT (ssim, "Calculated total latency: live %s, min %" GST_TIME_FORMAT ", max %" GST_TIME_FORMAT, (live ? "yes" : "no"), GST_TIME_ARGS (min), GST_TIME_ARGS (max)); gst_query_set_latency (query, live, min, max); } return res; } static gboolean gst_ssim_query_duration (GstSSim * ssim, GstQuery * query) { gint64 max, min; gboolean res; GstFormat format; GstIterator *it; gboolean done; /* parse format */ gst_query_parse_duration (query, &format, NULL); max = -1; min = G_MAXINT64; res = TRUE; done = FALSE; it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (ssim)); while (!done) { GstIteratorResult ires; gpointer item; ires = gst_iterator_next (it, &item); switch (ires) { case GST_ITERATOR_DONE: done = TRUE; break; case GST_ITERATOR_OK: { GstPad *pad = GST_PAD_CAST (item); gint64 duration; /* ask sink peer for duration */ res &= gst_pad_query_peer_duration (pad, &format, &duration); /* take min&max from all valid return values */ if (res) { /* valid unknown length, stop searching */ if (duration == -1) { max = duration; done = TRUE; } /* else see if bigger than current max */ else { if (duration > max) max = duration; if (duration < min) min = duration; } } gst_object_unref (pad); break; } case GST_ITERATOR_RESYNC: max = -1; min = G_MAXINT64; res = TRUE; gst_iterator_resync (it); break; default: res = FALSE; done = TRUE; break; } } gst_iterator_free (it); if (res) { /* and store the max */ GST_DEBUG_OBJECT (ssim, "Total duration in format %s: %" GST_TIME_FORMAT, gst_format_get_name (format), GST_TIME_ARGS (min)); gst_query_set_duration (query, format, min); } return res; } static gboolean gst_ssim_query (GstPad * pad, GstQuery * query) { GstSSim *ssim = GST_SSIM (gst_pad_get_parent (pad)); gboolean res = FALSE; switch (GST_QUERY_TYPE (query)) { case GST_QUERY_POSITION: { GstFormat format; gst_query_parse_position (query, &format, NULL); switch (format) { case GST_FORMAT_TIME: /* FIXME, bring to stream time, might be tricky */ gst_query_set_position (query, format, ssim->timestamp); res = TRUE; break; case GST_FORMAT_DEFAULT: gst_query_set_position (query, format, ssim->offset); res = TRUE; break; default: break; } break; } case GST_QUERY_DURATION: res = gst_ssim_query_duration (ssim, query); break; case GST_QUERY_LATENCY: res = gst_ssim_query_latency (ssim, query); break; default: /* FIXME, needs a custom query handler because we have multiple * sinkpads */ res = gst_pad_query_default (pad, query); break; } gst_object_unref (ssim); return res; } static gboolean forward_event_func (GstPad * pad, GValue * ret, GstEvent * event) { gst_event_ref (event); GST_LOG_OBJECT (pad, "About to send event %s", GST_EVENT_TYPE_NAME (event)); if (!gst_pad_push_event (pad, event)) { g_value_set_boolean (ret, FALSE); GST_LOG_OBJECT (pad, "Sending event %p (%s) failed.", event, GST_EVENT_TYPE_NAME (event)); } else { GST_LOG_OBJECT (pad, "Sent event %p (%s).", event, GST_EVENT_TYPE_NAME (event)); } gst_object_unref (pad); return TRUE; } /* forwards the event to all sinkpads, takes ownership of the * event * * Returns: TRUE if the event could be forwarded on all * sinkpads. */ static gboolean forward_event (GstSSim * ssim, GstEvent * event) { GstIterator *it; GValue vret = { 0 }; GST_LOG_OBJECT (ssim, "Forwarding event %p (%s)", event, GST_EVENT_TYPE_NAME (event)); g_value_init (&vret, G_TYPE_BOOLEAN); g_value_set_boolean (&vret, TRUE); it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (ssim)); gst_iterator_fold (it, (GstIteratorFoldFunction) forward_event_func, &vret, event); gst_iterator_free (it); gst_event_unref (event); return g_value_get_boolean (&vret); } static gboolean gst_ssim_src_event (GstPad * pad, GstEvent * event) { GstSSim *ssim; gboolean result; ssim = GST_SSIM (gst_pad_get_parent (pad)); switch (GST_EVENT_TYPE (event)) { case GST_EVENT_QOS: /* QoS might be tricky */ result = FALSE; break; case GST_EVENT_SEEK: { GstSeekFlags flags; GstSeekType curtype; gint64 cur; /* parse the seek parameters */ gst_event_parse_seek (event, &ssim->segment_rate, NULL, &flags, &curtype, &cur, NULL, NULL); /* check if we are flushing */ if (flags & GST_SEEK_FLAG_FLUSH) { /* make sure we accept nothing anymore and return WRONG_STATE */ gst_collect_pads2_set_flushing (ssim->collect, TRUE); /* flushing seek, start flush downstream, the flush will be done * when all pads received a FLUSH_STOP. */ gst_pad_push_event (pad, gst_event_new_flush_start ()); } /* now wait for the collected to be finished and mark a new * segment */ GST_OBJECT_LOCK (ssim->collect); if (curtype == GST_SEEK_TYPE_SET) ssim->segment_position = cur; else ssim->segment_position = 0; { GstSSimOutputContext *c; gint i = 0; for (i = 0; i < ssim->src->len; i++) { c = (GstSSimOutputContext *) g_ptr_array_index (ssim->src, i); c->segment_pending = TRUE; } } GST_OBJECT_UNLOCK (ssim->collect); result = forward_event (ssim, event); break; } case GST_EVENT_NAVIGATION: /* navigation is rather pointless. */ result = FALSE; break; default: /* just forward the rest for now */ result = forward_event (ssim, event); break; } gst_object_unref (ssim); return result; } static gboolean gst_ssim_sink_event (GstPad * pad, GstEvent * event) { GstSSim *ssim; gboolean ret; ssim = GST_SSIM (gst_pad_get_parent (pad)); GST_DEBUG ("Got %s event on pad %s:%s", GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad)); switch (GST_EVENT_TYPE (event)) { case GST_EVENT_NEWSEGMENT: { gboolean update; gdouble rate; gdouble applied_rate; GstFormat format; gint64 start; gint64 stop; gint64 position; gst_event_parse_new_segment_full (event, &update, &rate, &applied_rate, &format, &start, &stop, &position); GST_DEBUG ("NEWSEGMENTEVENT: update(%d), rate(%f), app_rate(%f), " "format(%d), start(%" GST_TIME_FORMAT ") stop(%" GST_TIME_FORMAT ") " "position(%" GST_TIME_FORMAT ")", update, rate, applied_rate, format, GST_TIME_ARGS (start), GST_TIME_ARGS (stop), GST_TIME_ARGS (position)); break; } case GST_EVENT_FLUSH_STOP: /* mark a pending new segment. This event is synchronized * with the streaming thread so we can safely update the * variable without races. It's somewhat weird because we * assume the collectpads forwarded the FLUSH_STOP past us * and downstream (using our source pad, the bastard!). */ { GstSSimOutputContext *c; gint i = 0; for (i = 0; i < ssim->src->len; i++) { c = (GstSSimOutputContext *) g_ptr_array_index (ssim->src, i); c->segment_pending = TRUE; } } break; default: break; } /* now GstCollectPads2 can take care of the rest, e.g. EOS */ GST_DEBUG ("Dispatching %s event on pad %s:%s", GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad)); ret = ssim->collect_event (pad, event); GST_DEBUG ("Event %s on pad %s:%s is dispatched", GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad)); gst_object_unref (ssim); return ret; } static void gst_ssim_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { GstSSim *ssim; ssim = GST_SSIM (object); switch (prop_id) { case PROP_SSIM_TYPE: ssim->ssimtype = g_value_get_int (value); break; case PROP_WINDOW_TYPE: ssim->windowtype = g_value_get_int (value); g_free (ssim->windows); ssim->windows = NULL; break; case PROP_WINDOW_SIZE: ssim->windowsize = g_value_get_int (value); g_free (ssim->windows); ssim->windows = NULL; break; case PROP_GAUSS_SIGMA: ssim->sigma = g_value_get_float (value); g_free (ssim->windows); ssim->windows = NULL; break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } } static void gst_ssim_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstSSim *ssim; ssim = GST_SSIM (object); switch (prop_id) { case PROP_SSIM_TYPE: g_value_set_int (value, ssim->ssimtype); break; case PROP_WINDOW_TYPE: g_value_set_int (value, ssim->windowtype); break; case PROP_WINDOW_SIZE: g_value_set_int (value, ssim->windowsize); break; case PROP_GAUSS_SIGMA: g_value_set_float (value, ssim->sigma); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } } static void gst_ssim_class_init (GstSSimClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; GstElementClass *gstelement_class = (GstElementClass *) klass; gobject_class->set_property = gst_ssim_set_property; gobject_class->get_property = gst_ssim_get_property; gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_ssim_finalize); g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_SSIM_TYPE, g_param_spec_int ("ssim-type", "SSIM type", "Type of the SSIM metric. 0 - canonical. 1 - with fixed mu " "(almost the same results, but roughly 20% faster)", 0, 1, 0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_WINDOW_TYPE, g_param_spec_int ("window-type", "Window type", "Type of the weighting in the window. " "0 - no weighting. 1 - Gaussian weighting (controlled by \"sigma\")", 0, 1, 1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_WINDOW_SIZE, g_param_spec_int ("window-size", "Window size", "Size of a window.", 1, 22, 11, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_GAUSS_SIGMA, g_param_spec_float ("gauss-sigma", "Deviation (for Gauss function)", "Used to calculate Gussian weights " "(only when using Gaussian window).", G_MINFLOAT, 10, 1.5, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); gst_element_class_add_static_pad_template (gstelement_class, &gst_ssim_src_template); gst_element_class_add_static_pad_template (gstelement_class, &gst_ssim_sink_original_template); gst_element_class_add_static_pad_template (gstelement_class, &gst_ssim_sink_modified_template); gst_element_class_set_details_simple (gstelement_class, "SSim", "Filter/Analyzer/Video", "Calculate Y-SSIM for n+2 YUV video streams", "Руслан Ижбулатов "); parent_class = g_type_class_peek_parent (klass); gstelement_class->request_new_pad = GST_DEBUG_FUNCPTR (gst_ssim_request_new_pad); gstelement_class->release_pad = GST_DEBUG_FUNCPTR (gst_ssim_release_pad); gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_ssim_change_state); } static GstPad * gst_ssim_request_new_pad (GstElement * element, GstPadTemplate * templ, const gchar * padname) { gchar *name; GstSSim *ssim; GstPad *newpad; GstPad *newsrc; gint padcount; GstPadTemplate *template; gint num = -1; if (templ->direction != GST_PAD_SINK) goto not_sink; ssim = GST_SSIM (element); padcount = ssim->padcount; GST_DEBUG_OBJECT (ssim, "number of pads = %d", padcount); if (padname) GST_DEBUG_OBJECT (ssim, "reqested pad %s", padname); else goto unnamed_pad; if (strcmp (padname, "original") == 0) { newpad = gst_pad_new_from_template (templ, "original"); GST_DEBUG_OBJECT (ssim, "request new sink pad original"); ssim->orig = newpad; } else if (strncmp (padname, "modified", 8) == 0) { const gchar *numstr = &padname[8]; num = strtol (numstr, NULL, 10); if (errno == EINVAL || errno == ERANGE) goto bad_name; newpad = gst_pad_new_from_template (templ, padname); GST_DEBUG_OBJECT (ssim, "request new sink pad %s", padname); } else goto bad_name; gst_pad_set_getcaps_function (newpad, GST_DEBUG_FUNCPTR (gst_ssim_sink_getcaps)); gst_pad_set_setcaps_function (newpad, GST_DEBUG_FUNCPTR (gst_ssim_setcaps)); gst_collect_pads2_add_pad (ssim->collect, newpad, sizeof (GstCollectData2)); /* FIXME: hacked way to override/extend the event function of * GstCollectPads2; because it sets its own event function giving the * element no access to events */ GST_DEBUG_OBJECT (ssim, "Current collect_event is %p, changing to %p", ssim->collect_event, GST_PAD_EVENTFUNC (newpad)); ssim->collect_event = (GstPadEventFunction) GST_PAD_EVENTFUNC (newpad); gst_pad_set_event_function (newpad, GST_DEBUG_FUNCPTR (gst_ssim_sink_event)); GST_DEBUG_OBJECT (ssim, "Adding a pad..."); /* takes ownership of the pad */ if (!gst_element_add_pad (GST_ELEMENT (ssim), newpad)) goto could_not_add_sink; else /* increment pad counter */ #if GLIB_CHECK_VERSION(2,29,5) padcount = g_atomic_int_add (&ssim->padcount, 1); #else padcount = g_atomic_int_exchange_and_add (&ssim->padcount, 1); #endif if (num >= 0) { GstSSimOutputContext *c; template = gst_static_pad_template_get (&gst_ssim_src_template); name = g_strdup_printf ("src%d", num); newsrc = gst_pad_new_from_template (template, name); GST_DEBUG_OBJECT (ssim, "creating src pad %s", name); g_free (name); gst_object_unref (template); gst_pad_set_getcaps_function (newsrc, GST_DEBUG_FUNCPTR (gst_ssim_src_getcaps)); gst_pad_set_query_function (newsrc, GST_DEBUG_FUNCPTR (gst_ssim_query)); gst_pad_set_event_function (newsrc, GST_DEBUG_FUNCPTR (gst_ssim_src_event)); if (!gst_element_add_pad (GST_ELEMENT (ssim), newsrc)) goto could_not_add_src; c = g_new (GstSSimOutputContext, 1); c->pad = newsrc; g_object_set_data (G_OBJECT (newpad), "ssim-match-output-context", c); g_ptr_array_add (ssim->src, (gpointer) c); } return newpad; /* errors */ bad_name: { g_warning ("gstssim: request new pad with bad name %s (must be " "'modified')\n", padname); return NULL; } unnamed_pad: { g_warning ("gstssim: request new pad without a name (must be " "'modified')\n"); return NULL; } not_sink: { g_warning ("gstssim: request new pad that is not a SINK pad\n"); return NULL; } could_not_add_src: { GST_DEBUG_OBJECT (ssim, "could not add src pad"); gst_object_unref (newsrc); } could_not_add_sink: { GST_DEBUG_OBJECT (ssim, "could not add sink pad"); gst_collect_pads2_remove_pad (ssim->collect, newpad); gst_object_unref (newpad); return NULL; } } static void gst_ssim_release_pad (GstElement * element, GstPad * pad) { GstSSim *ssim; ssim = GST_SSIM (element); GST_DEBUG_OBJECT (ssim, "release pad %s:%s", GST_DEBUG_PAD_NAME (pad)); gst_collect_pads2_remove_pad (ssim->collect, pad); gst_element_remove_pad (element, pad); } static void gst_ssim_init (GstSSim * ssim) { ssim->windowsize = 11; ssim->windowtype = 1; ssim->windows = NULL; ssim->sigma = 1.5; ssim->ssimtype = 0; ssim->src = g_ptr_array_new (); ssim->padcount = 0; ssim->collect_event = NULL; ssim->sinkcaps = NULL; /* keep track of the sinkpads requested */ ssim->collect = gst_collect_pads2_new (); gst_collect_pads2_set_function (ssim->collect, GST_DEBUG_FUNCPTR (gst_ssim_collected), ssim); } static void gst_ssim_finalize (GObject * object) { GstSSim *ssim = GST_SSIM (object); gst_object_unref (ssim->collect); ssim->collect = NULL; g_free (ssim->windows); ssim->windows = NULL; g_free (ssim->weights); ssim->weights = NULL; if (ssim->sinkcaps) gst_caps_unref (ssim->sinkcaps); if (ssim->srccaps) gst_caps_unref (ssim->srccaps); g_ptr_array_free (ssim->src, TRUE); G_OBJECT_CLASS (parent_class)->finalize (object); } typedef gfloat (*GstSSimWeightFunc) (GstSSim * ssim, gint y, gint x); static gfloat gst_ssim_weight_func_none (GstSSim * ssim, gint y, gint x) { return 1; } static gfloat gst_ssim_weight_func_gauss (GstSSim * ssim, gint y, gint x) { gfloat coord = sqrt (x * x + y * y); return exp (-1 * (coord * coord) / (2 * ssim->sigma * ssim->sigma)) / (ssim->sigma * sqrt (2 * G_PI)); } static gboolean gst_ssim_regenerate_windows (GstSSim * ssim) { gint windowiseven; gint y, x, y2, x2; GstSSimWeightFunc func; gfloat normal_summ = 0; gint normal_count = 0; g_free (ssim->weights); ssim->weights = g_new (gfloat, ssim->windowsize * ssim->windowsize); windowiseven = ((gint) ssim->windowsize / 2) * 2 == ssim->windowsize ? 1 : 0; g_free (ssim->windows); ssim->windows = g_new (GstSSimWindowCache, ssim->height * ssim->width); switch (ssim->windowtype) { case 0: func = gst_ssim_weight_func_none; break; case 1: func = gst_ssim_weight_func_gauss; break; default: GST_WARNING_OBJECT (ssim, "unknown window type - %d. Defaulting to %d", ssim->windowtype, 1); ssim->windowtype = 1; func = gst_ssim_weight_func_gauss; } for (y = 0; y < ssim->windowsize; y++) { gint yoffset = y * ssim->windowsize; for (x = 0; x < ssim->windowsize; x++) { ssim->weights[yoffset + x] = func (ssim, x - ssim->windowsize / 2 + windowiseven, y - ssim->windowsize / 2 + windowiseven); normal_summ += ssim->weights[yoffset + x]; normal_count++; } } for (y = 0; y < ssim->height; y++) { for (x = 0; x < ssim->width; x++) { GstSSimWindowCache win; gint element_count = 0; win.x_window_start = x - ssim->windowsize / 2 + windowiseven; win.x_weight_start = 0; if (win.x_window_start < 0) { win.x_weight_start = -win.x_window_start; win.x_window_start = 0; } win.x_window_end = x + ssim->windowsize / 2; if (win.x_window_end >= ssim->width) win.x_window_end = ssim->width - 1; win.y_window_start = y - ssim->windowsize / 2 + windowiseven; win.y_weight_start = 0; if (win.y_window_start < 0) { win.y_weight_start = -win.y_window_start; win.y_window_start = 0; } win.y_window_end = y + ssim->windowsize / 2; if (win.y_window_end >= ssim->height) win.y_window_end = ssim->height - 1; win.element_summ = 0; element_count = (win.y_window_end - win.y_window_start + 1) * (win.x_window_end - win.x_window_start + 1); if (element_count == normal_count) win.element_summ = normal_summ; else { for (y2 = win.y_weight_start; y2 < ssim->windowsize; y2++) { for (x2 = win.x_weight_start; x2 < ssim->windowsize; x2++) { win.element_summ += ssim->weights[y2 * ssim->windowsize + x2]; } } } ssim->windows[(y * ssim->width + x)] = win; } } /* FIXME: while 0.01 and 0.03 are pretty much static, the 255 implies that * we're working with 8-bit-per-color-component format, which may not be true */ ssim->const1 = 0.01 * 255 * 0.01 * 255; ssim->const2 = 0.03 * 255 * 0.03 * 255; return TRUE; } static GstFlowReturn gst_ssim_collected (GstCollectPads2 * pads, gpointer user_data) { GstSSim *ssim; GSList *collected; GstFlowReturn ret = GST_FLOW_OK; GstBuffer *orgbuf = NULL; gfloat *orgmu = NULL; GstBuffer *outbuf = NULL; gpointer outdata = NULL; guint outsize = 0; gfloat mssim = 0, lowest = 1, highest = -1; gboolean ready = TRUE; gint padnumber = 0; ssim = GST_SSIM (user_data); if (G_UNLIKELY (ssim->windows == NULL)) { GST_DEBUG_OBJECT (ssim, "Regenerating windows"); gst_ssim_regenerate_windows (ssim); } switch (ssim->ssimtype) { case 0: ssim->func = (GstSSimFunction) calcssim_canonical; break; case 1: ssim->func = (GstSSimFunction) calcssim_without_mu; break; default: return GST_FLOW_ERROR; } for (collected = pads->data; collected; collected = g_slist_next (collected)) { GstCollectData2 *collect_data; GstBuffer *inbuf; collect_data = (GstCollectData2 *) collected->data; inbuf = gst_collect_pads2_peek (pads, collect_data); if (inbuf == NULL) { GST_LOG_OBJECT (ssim, "channel %p: no bytes available", collect_data); ready = FALSE; } else gst_buffer_unref (inbuf); } /* if _collected() was called, all pads should have data, but if * one of them doesn't, it means that it is EOS and we can't go any further * * FIXME, shouldn't we do something about pads that DO have data? * Flush them or something? */ if (G_UNLIKELY (!ready)) goto eos; /* Mu is just a blur, we can calculate it once */ if (ssim->ssimtype == 0) { orgmu = g_new (gfloat, ssim->width * ssim->height); for (collected = pads->data; collected; collected = g_slist_next (collected)) { GstCollectData2 *collect_data; collect_data = (GstCollectData2 *) collected->data; if (collect_data->pad == ssim->orig) { orgbuf = gst_collect_pads2_pop (pads, collect_data);; GST_DEBUG_OBJECT (ssim, "Original stream - flags(0x%x), timestamp(%" GST_TIME_FORMAT "), duration(%" GST_TIME_FORMAT ")", GST_BUFFER_FLAGS (orgbuf), GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (orgbuf)), GST_TIME_ARGS (GST_BUFFER_DURATION (orgbuf))); calculate_mu (ssim, orgmu, GST_BUFFER_DATA (orgbuf)); break; } } } GST_LOG_OBJECT (ssim, "starting to cycle through streams"); for (collected = pads->data; collected; collected = g_slist_next (collected)) { GstCollectData2 *collect_data; GstBuffer *inbuf; guint8 *indata; collect_data = (GstCollectData2 *) collected->data; if (collect_data->pad != ssim->orig) { inbuf = gst_collect_pads2_pop (pads, collect_data); indata = GST_BUFFER_DATA (inbuf); GST_DEBUG_OBJECT (ssim, "Modified stream - flags(0x%x), timestamp(%" GST_TIME_FORMAT "), duration(%" GST_TIME_FORMAT ")", GST_BUFFER_FLAGS (inbuf), GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (inbuf)), GST_TIME_ARGS (GST_BUFFER_DURATION (inbuf))); if (!GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_GAP)) { GstSSimOutputContext *c; GstEvent *measured; guint64 offset; GValue vmean = { 0 } , vlowest = { 0} , vhighest = { 0}; c = (GstSSimOutputContext *) g_object_get_data (G_OBJECT (collect_data->pad), "ssim-match-output-context"); GST_DEBUG_OBJECT (ssim, "Output context is %" GST_PTR_FORMAT ", pad will be %" GST_PTR_FORMAT, c, c->pad); outsize = GST_ROUND_UP_4 (ssim->width) * ssim->height; GST_LOG_OBJECT (ssim, "channel %p: making output buffer of %d bytes", collect_data, outsize); /* first buffer, alloc outsize. * FIXME: we can easily subbuffer and _make_writable. * FIXME: only create empty buffer for first non-gap buffer, so that we * only use ssim function when really calculating */ outbuf = gst_buffer_new_and_alloc (GST_ROUND_UP_4 (ssim->width) * ssim->height); outdata = GST_BUFFER_DATA (outbuf); gst_buffer_set_caps (outbuf, gst_pad_get_fixed_caps_func (c->pad)); /* Videos should match, so the output video has the same characteristics * as the input video */ /* set timestamps on the output buffer */ gst_buffer_copy_metadata (outbuf, inbuf, (GstBufferCopyFlags) GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS); g_value_init (&vmean, G_TYPE_FLOAT); g_value_init (&vlowest, G_TYPE_FLOAT); g_value_init (&vhighest, G_TYPE_FLOAT); GST_LOG_OBJECT (ssim, "channel %p: calculating SSIM", collect_data); ssim->func (ssim, GST_BUFFER_DATA (orgbuf), orgmu, indata, outdata, &mssim, &lowest, &highest); GST_DEBUG_OBJECT (GST_OBJECT (ssim), "MSSIM is %f, l-h is %f - %f", mssim, lowest, highest); gst_ssim_post_message (ssim, outbuf, mssim, lowest, highest); g_value_set_float (&vmean, mssim); g_value_set_float (&vlowest, lowest); g_value_set_float (&vhighest, highest); offset = GST_BUFFER_OFFSET (inbuf); /* our timestamping is very simple, just an ever incrementing * counter, the new segment time will take care of their respective * stream time. */ if (c->segment_pending) { GstEvent *event; /* FIXME, use rate/applied_rate as set on all sinkpads. * - currently we just set rate as received from last seek-event * We could potentially figure out the duration as well using * the current segment positions and the stated stop positions. * Also we just start from stream time 0 which is rather * weird. For non-synchronized mixing, the time should be * the min of the stream times of all received segments, * rationale being that the duration is at least going to * be as long as the earliest stream we start mixing. This * would also be correct for synchronized mixing but then * the later streams would be delayed until the stream times` * match. */ event = gst_event_new_new_segment_full (FALSE, ssim->segment_rate, 1.0, GST_FORMAT_TIME, ssim->timestamp, -1, ssim->segment_position); gst_pad_push_event (c->pad, event); c->segment_pending = FALSE; } measured = gst_event_new_measured (offset, GST_BUFFER_TIMESTAMP (inbuf), "SSIM", &vmean, &vlowest, &vhighest); gst_pad_push_event (c->pad, measured); /* send it out */ GST_DEBUG_OBJECT (ssim, "pushing outbuf, timestamp %" GST_TIME_FORMAT ", size %d", GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), GST_BUFFER_SIZE (outbuf)); ret &= gst_pad_push (c->pad, outbuf); } else { GST_LOG_OBJECT (ssim, "channel %p: skipping", collect_data); } gst_buffer_unref (inbuf); padnumber++; } } gst_buffer_unref (orgbuf); if (ssim->ssimtype == 0) g_free (orgmu); ssim->segment_position = 0; return ret; /* ERRORS */ eos: { gint i; GST_DEBUG_OBJECT (ssim, "no data available, must be EOS"); for (i = 0; i < ssim->src->len; i++) { GstSSimOutputContext *c = (GstSSimOutputContext *) g_ptr_array_index (ssim->src, i); gst_pad_push_event (c->pad, gst_event_new_eos ()); } return GST_FLOW_UNEXPECTED; } } static GstStateChangeReturn gst_ssim_change_state (GstElement * element, GstStateChange transition) { GstSSim *ssim; GstStateChangeReturn ret; ssim = GST_SSIM (element); switch (transition) { case GST_STATE_CHANGE_NULL_TO_READY: break; case GST_STATE_CHANGE_READY_TO_PAUSED: ssim->timestamp = 0; ssim->offset = 0; { GstSSimOutputContext *c; gint i = 0; for (i = 0; i < ssim->src->len; i++) { c = (GstSSimOutputContext *) g_ptr_array_index (ssim->src, i); c->segment_pending = TRUE; } } ssim->segment_position = 0; ssim->segment_rate = 1.0; gst_segment_init (&ssim->segment, GST_FORMAT_UNDEFINED); gst_collect_pads2_start (ssim->collect); break; case GST_STATE_CHANGE_PAUSED_TO_PLAYING: break; case GST_STATE_CHANGE_PAUSED_TO_READY: /* need to unblock the collectpads before calling the * parent change_state so that streaming can finish */ gst_collect_pads2_stop (ssim->collect); break; default: break; } ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); switch (transition) { default: break; } return ret; }