mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-12-30 04:00:37 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/gst/audiovisualizers/gstaudiovisualizer.c
Tim-Philipp Müller a84e2ccbb8 audiovisualizer: shaders assume 32bpp 
Backport fix for crashes and invalid writes in totem from libvisual
in -base, to minimise differences to version in -base and to make
sure the bug doesn't sneak back in later when the base class is
made public.

The shader code looks like it makes assumptions that are not
necessarily always true, even if they're true for now for the
existing elements, namly that pixel stride is 4, for example.

See https://bugzilla.gnome.org/show_bug.cgi?id=683527
2013-04-05 00:53:54 +01:00

1393 lines
39 KiB
C
Raw Blame History

/* GStreamer
* Copyright (C) <2011> Stefan Kost <ensonic@users.sf.net>
*
* gstaudiovisualizer.h: base class for audio visualisation elements
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
*/
/**
* SECTION:gstaudiovisualizer
*
* A baseclass for scopes (visualizers). It takes care of re-fitting the
* audio-rate to video-rate and handles renegotiation (downstream video size
* changes).
*
* It also provides several background shading effects. These effects are
* applied to a previous picture before the render() implementation can draw a
* new frame.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
/* FIXME 0.11: suppress warnings for deprecated API such as GStaticRecMutex
* with newer GLib versions (>= 2.31.0) */
#define GLIB_DISABLE_DEPRECATION_WARNINGS
#include <string.h>
#include <gst/video/video.h>
#include <gst/video/gstvideometa.h>
#include <gst/video/gstvideopool.h>
#include "gstaudiovisualizer.h"
GST_DEBUG_CATEGORY_STATIC (audio_visualizer_debug);
#define GST_CAT_DEFAULT (audio_visualizer_debug)
#define DEFAULT_SHADER GST_AUDIO_VISUALIZER_SHADER_FADE
#define DEFAULT_SHADE_AMOUNT 0x000a0a0a
enum
{
PROP_0,
PROP_SHADER,
PROP_SHADE_AMOUNT
};
static GstBaseTransformClass *parent_class = NULL;
static void gst_audio_visualizer_class_init (GstAudioVisualizerClass * klass);
static void gst_audio_visualizer_init (GstAudioVisualizer * scope,
GstAudioVisualizerClass * g_class);
static void gst_audio_visualizer_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_audio_visualizer_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec);
static void gst_audio_visualizer_dispose (GObject * object);
static gboolean gst_audio_visualizer_src_negotiate (GstAudioVisualizer * scope);
static gboolean gst_audio_visualizer_src_setcaps (GstAudioVisualizer *
scope, GstCaps * caps);
static gboolean gst_audio_visualizer_sink_setcaps (GstAudioVisualizer *
scope, GstCaps * caps);
static GstFlowReturn gst_audio_visualizer_chain (GstPad * pad,
GstObject * parent, GstBuffer * buffer);
static gboolean gst_audio_visualizer_src_event (GstPad * pad,
GstObject * parent, GstEvent * event);
static gboolean gst_audio_visualizer_sink_event (GstPad * pad,
GstObject * parent, GstEvent * event);
static gboolean gst_audio_visualizer_src_query (GstPad * pad,
GstObject * parent, GstQuery * query);
static gboolean gst_audio_visualizer_sink_query (GstPad * pad,
GstObject * parent, GstQuery * query);
static GstStateChangeReturn gst_audio_visualizer_change_state (GstElement *
element, GstStateChange transition);
static gboolean gst_audio_visualizer_do_bufferpool (GstAudioVisualizer * scope,
GstCaps * outcaps);
static gboolean
default_decide_allocation (GstAudioVisualizer * scope, GstQuery * query);
/* shading functions */
#define GST_TYPE_AUDIO_VISUALIZER_SHADER (gst_audio_visualizer_shader_get_type())
static GType
gst_audio_visualizer_shader_get_type (void)
{
static GType shader_type = 0;
static const GEnumValue shaders[] = {
{GST_AUDIO_VISUALIZER_SHADER_NONE, "None", "none"},
{GST_AUDIO_VISUALIZER_SHADER_FADE, "Fade", "fade"},
{GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_UP, "Fade and move up",
"fade-and-move-up"},
{GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_DOWN, "Fade and move down",
"fade-and-move-down"},
{GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_LEFT, "Fade and move left",
"fade-and-move-left"},
{GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_RIGHT,
"Fade and move right",
"fade-and-move-right"},
{GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_HORIZ_OUT,
"Fade and move horizontally out", "fade-and-move-horiz-out"},
{GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_HORIZ_IN,
"Fade and move horizontally in", "fade-and-move-horiz-in"},
{GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_VERT_OUT,
"Fade and move vertically out", "fade-and-move-vert-out"},
{GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_VERT_IN,
"Fade and move vertically in", "fade-and-move-vert-in"},
{0, NULL, NULL},
};
if (G_UNLIKELY (shader_type == 0)) {
/* TODO: rename when exporting it as a library */
shader_type =
g_enum_register_static
("GstAudioVisualizerShader-BadGstAudioVisualizers", shaders);
}
return shader_type;
}
/* we're only supporting GST_VIDEO_FORMAT_xRGB right now) */
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
#define SHADE(_d, _s, _i, _r, _g, _b) \
G_STMT_START { \
_d[_i * 4 + 0] = (_s[_i * 4 + 0] > _b) ? _s[_i * 4 + 0] - _b : 0; \
_d[_i * 4 + 1] = (_s[_i * 4 + 1] > _g) ? _s[_i * 4 + 1] - _g : 0; \
_d[_i * 4 + 2] = (_s[_i * 4 + 2] > _r) ? _s[_i * 4 + 2] - _r : 0; \
_d[_i * 4 + 3] = 0; \
} G_STMT_END
#else /* G_BYTE_ORDER == G_LITTLE_ENDIAN */
#define SHADE(_d, _s, _i, _r, _g, _b) \
G_STMT_START { \
_d[_i * 4 + 0] = 0; \
_d[_i * 4 + 1] = (_s[_i * 4 + 1] > _r) ? _s[_i * 4 + 1] - _r : 0; \
_d[_i * 4 + 2] = (_s[_i * 4 + 2] > _g) ? _s[_i * 4 + 2] - _g : 0; \
_d[_i * 4 + 3] = (_s[_i * 4 + 3] > _b) ? _s[_i * 4 + 3] - _b : 0; \
} G_STMT_END
#endif
static void
shader_fade (GstAudioVisualizer * scope, const GstVideoFrame * sframe,
GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *d;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
for (j = 0; j < height; j++) {
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
s += ss;
d += ds;
}
}
static void
shader_fade_and_move_up (GstAudioVisualizer * scope,
const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *d;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
for (j = 1; j < height; j++) {
s += ss;
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
d += ds;
}
}
static void
shader_fade_and_move_down (GstAudioVisualizer * scope,
const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *d;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
for (j = 1; j < height; j++) {
d += ds;
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
s += ss;
}
}
static void
shader_fade_and_move_left (GstAudioVisualizer * scope,
const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *d;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
width -= 1;
s += 4;
/* move to the left */
for (j = 0; j < height; j++) {
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
d += ds;
s += ss;
}
}
static void
shader_fade_and_move_right (GstAudioVisualizer * scope,
const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *d;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
width -= 1;
d += 4;
/* move to the right */
for (j = 0; j < height; j++) {
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
d += ds;
s += ss;
}
}
static void
shader_fade_and_move_horiz_out (GstAudioVisualizer * scope,
const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *d;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
/* move upper half up */
for (j = 0; j < height / 2; j++) {
s += ss;
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
d += ds;
}
/* move lower half down */
for (j = 0; j < height / 2; j++) {
d += ds;
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
s += ss;
}
}
static void
shader_fade_and_move_horiz_in (GstAudioVisualizer * scope,
const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *d;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
/* move upper half down */
for (j = 0; j < height / 2; j++) {
d += ds;
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
s += ss;
}
/* move lower half up */
for (j = 0; j < height / 2; j++) {
s += ss;
for (i = 0; i < width; i++) {
SHADE (d, s, i, r, g, b);
}
d += ds;
}
}
static void
shader_fade_and_move_vert_out (GstAudioVisualizer * scope,
const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *s1, *d, *d1;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
for (j = 0; j < height; j++) {
/* move left half to the left */
s1 = s + 1;
for (i = 0; i < width / 2; i++) {
SHADE (d, s1, i, r, g, b);
}
/* move right half to the right */
d1 = d + 1;
for (; i < width - 1; i++) {
SHADE (d1, s, i, r, g, b);
}
s += ss;
d += ds;
}
}
static void
shader_fade_and_move_vert_in (GstAudioVisualizer * scope,
const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
guint i, j;
guint r = (scope->shade_amount >> 16) & 0xff;
guint g = (scope->shade_amount >> 8) & 0xff;
guint b = (scope->shade_amount >> 0) & 0xff;
guint8 *s, *s1, *d, *d1;
gint ss, ds, width, height;
s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
width = GST_VIDEO_FRAME_WIDTH (sframe);
height = GST_VIDEO_FRAME_HEIGHT (sframe);
for (j = 0; j < height; j++) {
/* move left half to the right */
d1 = d + 1;
for (i = 0; i < width / 2; i++) {
SHADE (d1, s, i, r, g, b);
}
/* move right half to the left */
s1 = s + 1;
for (; i < width - 1; i++) {
SHADE (d, s1, i, r, g, b);
}
s += ss;
d += ds;
}
}
static void
gst_audio_visualizer_change_shader (GstAudioVisualizer * scope)
{
switch (scope->shader_type) {
case GST_AUDIO_VISUALIZER_SHADER_NONE:
scope->shader = NULL;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE:
scope->shader = shader_fade;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_UP:
scope->shader = shader_fade_and_move_up;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_DOWN:
scope->shader = shader_fade_and_move_down;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_LEFT:
scope->shader = shader_fade_and_move_left;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_RIGHT:
scope->shader = shader_fade_and_move_right;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_HORIZ_OUT:
scope->shader = shader_fade_and_move_horiz_out;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_HORIZ_IN:
scope->shader = shader_fade_and_move_horiz_in;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_VERT_OUT:
scope->shader = shader_fade_and_move_vert_out;
break;
case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_VERT_IN:
scope->shader = shader_fade_and_move_vert_in;
break;
default:
GST_ERROR ("invalid shader function");
scope->shader = NULL;
break;
}
}
/* base class */
#define GST_AUDIO_VISUALIZER_GET_PRIVATE(obj) \
(G_TYPE_INSTANCE_GET_PRIVATE ((obj), GST_TYPE_AUDIO_VISUALIZER, GstAudioVisualizerPrivate))
struct _GstAudioVisualizerPrivate
{
gboolean negotiated;
GstBufferPool *pool;
gboolean pool_active;
GstAllocator *allocator;
GstAllocationParams params;
GstQuery *query;
};
GType
gst_audio_visualizer_get_type (void)
{
static volatile gsize audio_visualizer_type = 0;
if (g_once_init_enter (&audio_visualizer_type)) {
static const GTypeInfo audio_visualizer_info = {
sizeof (GstAudioVisualizerClass),
NULL,
NULL,
(GClassInitFunc) gst_audio_visualizer_class_init,
NULL,
NULL,
sizeof (GstAudioVisualizer),
0,
(GInstanceInitFunc) gst_audio_visualizer_init,
};
GType _type;
/* TODO: rename when exporting it as a library */
_type = g_type_register_static (GST_TYPE_ELEMENT,
"GstAudioVisualizer-BadGstAudioVisualizers", &audio_visualizer_info,
G_TYPE_FLAG_ABSTRACT);
g_once_init_leave (&audio_visualizer_type, _type);
}
return (GType) audio_visualizer_type;
}
static void
gst_audio_visualizer_class_init (GstAudioVisualizerClass * klass)
{
GObjectClass *gobject_class = (GObjectClass *) klass;
GstElementClass *element_class = (GstElementClass *) klass;
g_type_class_add_private (klass, sizeof (GstAudioVisualizerPrivate));
parent_class = g_type_class_peek_parent (klass);
GST_DEBUG_CATEGORY_INIT (audio_visualizer_debug, "baseaudiovisualizer",
0, "scope audio visualisation base class");
gobject_class->set_property = gst_audio_visualizer_set_property;
gobject_class->get_property = gst_audio_visualizer_get_property;
gobject_class->dispose = gst_audio_visualizer_dispose;
element_class->change_state =
GST_DEBUG_FUNCPTR (gst_audio_visualizer_change_state);
klass->decide_allocation = GST_DEBUG_FUNCPTR (default_decide_allocation);
g_object_class_install_property (gobject_class, PROP_SHADER,
g_param_spec_enum ("shader", "shader type",
"Shader function to apply on each frame",
GST_TYPE_AUDIO_VISUALIZER_SHADER, DEFAULT_SHADER,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_SHADE_AMOUNT,
g_param_spec_uint ("shade-amount", "shade amount",
"Shading color to use (big-endian ARGB)", 0, G_MAXUINT32,
DEFAULT_SHADE_AMOUNT,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
}
static void
gst_audio_visualizer_init (GstAudioVisualizer * scope,
GstAudioVisualizerClass * g_class)
{
GstPadTemplate *pad_template;
scope->priv = GST_AUDIO_VISUALIZER_GET_PRIVATE (scope);
/* create the sink and src pads */
pad_template =
gst_element_class_get_pad_template (GST_ELEMENT_CLASS (g_class), "sink");
g_return_if_fail (pad_template != NULL);
scope->sinkpad = gst_pad_new_from_template (pad_template, "sink");
gst_pad_set_chain_function (scope->sinkpad,
GST_DEBUG_FUNCPTR (gst_audio_visualizer_chain));
gst_pad_set_event_function (scope->sinkpad,
GST_DEBUG_FUNCPTR (gst_audio_visualizer_sink_event));
gst_pad_set_query_function (scope->sinkpad,
GST_DEBUG_FUNCPTR (gst_audio_visualizer_sink_query));
gst_element_add_pad (GST_ELEMENT (scope), scope->sinkpad);
pad_template =
gst_element_class_get_pad_template (GST_ELEMENT_CLASS (g_class), "src");
g_return_if_fail (pad_template != NULL);
scope->srcpad = gst_pad_new_from_template (pad_template, "src");
gst_pad_set_event_function (scope->srcpad,
GST_DEBUG_FUNCPTR (gst_audio_visualizer_src_event));
gst_pad_set_query_function (scope->srcpad,
GST_DEBUG_FUNCPTR (gst_audio_visualizer_src_query));
gst_element_add_pad (GST_ELEMENT (scope), scope->srcpad);
scope->adapter = gst_adapter_new ();
scope->inbuf = gst_buffer_new ();
/* properties */
scope->shader_type = DEFAULT_SHADER;
gst_audio_visualizer_change_shader (scope);
scope->shade_amount = DEFAULT_SHADE_AMOUNT;
/* reset the initial video state */
gst_video_info_init (&scope->vinfo);
scope->frame_duration = GST_CLOCK_TIME_NONE;
/* reset the initial state */
gst_audio_info_init (&scope->ainfo);
gst_video_info_init (&scope->vinfo);
g_mutex_init (&scope->config_lock);
}
static void
gst_audio_visualizer_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);
switch (prop_id) {
case PROP_SHADER:
scope->shader_type = g_value_get_enum (value);
gst_audio_visualizer_change_shader (scope);
break;
case PROP_SHADE_AMOUNT:
scope->shade_amount = g_value_get_uint (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_audio_visualizer_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);
switch (prop_id) {
case PROP_SHADER:
g_value_set_enum (value, scope->shader_type);
break;
case PROP_SHADE_AMOUNT:
g_value_set_uint (value, scope->shade_amount);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_audio_visualizer_dispose (GObject * object)
{
GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);
if (scope->adapter) {
g_object_unref (scope->adapter);
scope->adapter = NULL;
}
if (scope->inbuf) {
gst_buffer_unref (scope->inbuf);
scope->inbuf = NULL;
}
if (scope->tempbuf) {
gst_video_frame_unmap (&scope->tempframe);
gst_buffer_unref (scope->tempbuf);
scope->tempbuf = NULL;
}
if (scope->config_lock.p) {
g_mutex_clear (&scope->config_lock);
scope->config_lock.p = NULL;
}
G_OBJECT_CLASS (parent_class)->dispose (object);
}
static void
gst_audio_visualizer_reset (GstAudioVisualizer * scope)
{
gst_adapter_clear (scope->adapter);
gst_segment_init (&scope->segment, GST_FORMAT_UNDEFINED);
GST_OBJECT_LOCK (scope);
scope->proportion = 1.0;
scope->earliest_time = -1;
GST_OBJECT_UNLOCK (scope);
}
static gboolean
gst_audio_visualizer_sink_setcaps (GstAudioVisualizer * scope, GstCaps * caps)
{
GstAudioInfo info;
if (!gst_audio_info_from_caps (&info, caps))
goto wrong_caps;
scope->ainfo = info;
GST_DEBUG_OBJECT (scope, "audio: channels %d, rate %d",
GST_AUDIO_INFO_CHANNELS (&info), GST_AUDIO_INFO_RATE (&info));
gst_pad_mark_reconfigure (scope->srcpad);
return TRUE;
/* Errors */
wrong_caps:
{
GST_WARNING_OBJECT (scope, "could not parse caps");
return FALSE;
}
}
static gboolean
gst_audio_visualizer_src_setcaps (GstAudioVisualizer * scope, GstCaps * caps)
{
GstVideoInfo info;
GstAudioVisualizerClass *klass;
gboolean res;
if (!gst_video_info_from_caps (&info, caps))
goto wrong_caps;
klass = GST_AUDIO_VISUALIZER_CLASS (G_OBJECT_GET_CLASS (scope));
scope->vinfo = info;
scope->frame_duration = gst_util_uint64_scale_int (GST_SECOND,
GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info));
scope->spf = gst_util_uint64_scale_int (GST_AUDIO_INFO_RATE (&scope->ainfo),
GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info));
scope->req_spf = scope->spf;
if (scope->tempbuf) {
gst_video_frame_unmap (&scope->tempframe);
gst_buffer_unref (scope->tempbuf);
}
scope->tempbuf = gst_buffer_new_wrapped (g_malloc0 (scope->vinfo.size),
scope->vinfo.size);
gst_video_frame_map (&scope->tempframe, &scope->vinfo, scope->tempbuf,
GST_MAP_READWRITE);
if (klass->setup)
res = klass->setup (scope);
GST_DEBUG_OBJECT (scope, "video: dimension %dx%d, framerate %d/%d",
GST_VIDEO_INFO_WIDTH (&info), GST_VIDEO_INFO_HEIGHT (&info),
GST_VIDEO_INFO_FPS_N (&info), GST_VIDEO_INFO_FPS_D (&info));
GST_DEBUG_OBJECT (scope, "blocks: spf %u, req_spf %u",
scope->spf, scope->req_spf);
gst_pad_set_caps (scope->srcpad, caps);
/* find a pool for the negotiated caps now */
res = gst_audio_visualizer_do_bufferpool (scope, caps);
return res;
/* ERRORS */
wrong_caps:
{
GST_DEBUG_OBJECT (scope, "error parsing caps");
return FALSE;
}
}
static gboolean
gst_audio_visualizer_src_negotiate (GstAudioVisualizer * scope)
{
GstCaps *othercaps, *target;
GstStructure *structure;
GstCaps *templ;
gboolean ret;
templ = gst_pad_get_pad_template_caps (scope->srcpad);
GST_DEBUG_OBJECT (scope, "performing negotiation");
/* see what the peer can do */
othercaps = gst_pad_peer_query_caps (scope->srcpad, NULL);
if (othercaps) {
target = gst_caps_intersect (othercaps, templ);
gst_caps_unref (othercaps);
gst_caps_unref (templ);
if (gst_caps_is_empty (target))
goto no_format;
target = gst_caps_truncate (target);
} else {
target = templ;
}
target = gst_caps_make_writable (target);
structure = gst_caps_get_structure (target, 0);
gst_structure_fixate_field_nearest_int (structure, "width", 320);
gst_structure_fixate_field_nearest_int (structure, "height", 200);
gst_structure_fixate_field_nearest_fraction (structure, "framerate", 25, 1);
target = gst_caps_fixate (target);
GST_DEBUG_OBJECT (scope, "final caps are %" GST_PTR_FORMAT, target);
ret = gst_audio_visualizer_src_setcaps (scope, target);
return ret;
no_format:
{
gst_caps_unref (target);
return FALSE;
}
}
/* takes ownership of the pool, allocator and query */
static gboolean
gst_audio_visualizer_set_allocation (GstAudioVisualizer * scope,
GstBufferPool * pool, GstAllocator * allocator,
GstAllocationParams * params, GstQuery * query)
{
GstAllocator *oldalloc;
GstBufferPool *oldpool;
GstQuery *oldquery;
GstAudioVisualizerPrivate *priv = scope->priv;
GST_OBJECT_LOCK (scope);
oldpool = priv->pool;
priv->pool = pool;
priv->pool_active = FALSE;
oldalloc = priv->allocator;
priv->allocator = allocator;
oldquery = priv->query;
priv->query = query;
if (params)
priv->params = *params;
else
gst_allocation_params_init (&priv->params);
GST_OBJECT_UNLOCK (scope);
if (oldpool) {
GST_DEBUG_OBJECT (scope, "deactivating old pool %p", oldpool);
gst_buffer_pool_set_active (oldpool, FALSE);
gst_object_unref (oldpool);
}
if (oldalloc) {
gst_object_unref (oldalloc);
}
if (oldquery) {
gst_query_unref (oldquery);
}
return TRUE;
}
static gboolean
gst_audio_visualizer_do_bufferpool (GstAudioVisualizer * scope,
GstCaps * outcaps)
{
GstQuery *query;
gboolean result = TRUE;
GstBufferPool *pool = NULL;
GstAudioVisualizerClass *klass;
GstAllocator *allocator;
GstAllocationParams params;
/* not passthrough, we need to allocate */
/* find a pool for the negotiated caps now */
GST_DEBUG_OBJECT (scope, "doing allocation query");
query = gst_query_new_allocation (outcaps, TRUE);
if (!gst_pad_peer_query (scope->srcpad, query)) {
/* not a problem, we use the query defaults */
GST_DEBUG_OBJECT (scope, "allocation query failed");
}
klass = GST_AUDIO_VISUALIZER_GET_CLASS (scope);
GST_DEBUG_OBJECT (scope, "calling decide_allocation");
g_assert (klass->decide_allocation != NULL);
result = klass->decide_allocation (scope, query);
GST_DEBUG_OBJECT (scope, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result,
query);
if (!result)
goto no_decide_allocation;
/* we got configuration from our peer or the decide_allocation method,
* parse them */
if (gst_query_get_n_allocation_params (query) > 0) {
gst_query_parse_nth_allocation_param (query, 0, &allocator, &params);
} else {
allocator = NULL;
gst_allocation_params_init (&params);
}
if (gst_query_get_n_allocation_pools (query) > 0)
gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL);
/* now store */
result =
gst_audio_visualizer_set_allocation (scope, pool, allocator, &params,
query);
return result;
/* Errors */
no_decide_allocation:
{
GST_WARNING_OBJECT (scope, "Subclass failed to decide allocation");
gst_query_unref (query);
return result;
}
}
static gboolean
default_decide_allocation (GstAudioVisualizer * scope, GstQuery * query)
{
GstCaps *outcaps;
GstBufferPool *pool;
guint size, min, max;
GstAllocator *allocator;
GstAllocationParams params;
GstStructure *config;
gboolean update_allocator;
gboolean update_pool;
gst_query_parse_allocation (query, &outcaps, NULL);
/* we got configuration from our peer or the decide_allocation method,
* parse them */
if (gst_query_get_n_allocation_params (query) > 0) {
/* try the allocator */
gst_query_parse_nth_allocation_param (query, 0, &allocator, &params);
update_allocator = TRUE;
} else {
allocator = NULL;
gst_allocation_params_init (&params);
update_allocator = FALSE;
}
if (gst_query_get_n_allocation_pools (query) > 0) {
gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
update_pool = TRUE;
} else {
pool = NULL;
size = GST_VIDEO_INFO_SIZE (&scope->vinfo);
min = max = 0;
update_pool = FALSE;
}
if (pool == NULL) {
/* we did not get a pool, make one ourselves then */
pool = gst_video_buffer_pool_new ();
}
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, outcaps, size, min, max);
gst_buffer_pool_config_set_allocator (config, allocator, &params);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
gst_buffer_pool_set_config (pool, config);
if (update_allocator)
gst_query_set_nth_allocation_param (query, 0, allocator, &params);
else
gst_query_add_allocation_param (query, allocator, &params);
if (allocator)
gst_object_unref (allocator);
if (update_pool)
gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
else
gst_query_add_allocation_pool (query, pool, size, min, max);
if (pool)
gst_object_unref (pool);
return TRUE;
}
static GstFlowReturn
default_prepare_output_buffer (GstAudioVisualizer * scope, GstBuffer ** outbuf)
{
GstAudioVisualizerPrivate *priv;
priv = scope->priv;
g_assert (priv->pool != NULL);
/* we can't reuse the input buffer */
if (!priv->pool_active) {
GST_DEBUG_OBJECT (scope, "setting pool %p active", priv->pool);
if (!gst_buffer_pool_set_active (priv->pool, TRUE))
goto activate_failed;
priv->pool_active = TRUE;
}
GST_DEBUG_OBJECT (scope, "using pool alloc");
return gst_buffer_pool_acquire_buffer (priv->pool, outbuf, NULL);
/* ERRORS */
activate_failed:
{
GST_ELEMENT_ERROR (scope, RESOURCE, SETTINGS,
("failed to activate bufferpool"), ("failed to activate bufferpool"));
return GST_FLOW_ERROR;
}
}
static GstFlowReturn
gst_audio_visualizer_chain (GstPad * pad, GstObject * parent,
GstBuffer * buffer)
{
GstFlowReturn ret = GST_FLOW_OK;
GstAudioVisualizer *scope;
GstAudioVisualizerClass *klass;
GstBuffer *inbuf;
guint64 dist, ts;
guint avail, sbpf;
gpointer adata;
gint bps, channels, rate;
scope = GST_AUDIO_VISUALIZER (parent);
klass = GST_AUDIO_VISUALIZER_CLASS (G_OBJECT_GET_CLASS (scope));
GST_LOG_OBJECT (scope, "chainfunc called");
/* resync on DISCONT */
if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT)) {
gst_adapter_clear (scope->adapter);
}
/* Make sure have an output format */
if (gst_pad_check_reconfigure (scope->srcpad)) {
if (!gst_audio_visualizer_src_negotiate (scope)) {
gst_pad_mark_reconfigure (scope->srcpad);
goto not_negotiated;
}
}
channels = GST_AUDIO_INFO_CHANNELS (&scope->ainfo);
rate = GST_AUDIO_INFO_RATE (&scope->ainfo);
bps = GST_AUDIO_INFO_BPS (&scope->ainfo);
if (bps == 0) {
ret = GST_FLOW_NOT_NEGOTIATED;
goto beach;
}
gst_adapter_push (scope->adapter, buffer);
g_mutex_lock (&scope->config_lock);
/* this is what we want */
sbpf = scope->req_spf * channels * sizeof (gint16);
inbuf = scope->inbuf;
/* FIXME: the timestamp in the adapter would be different */
gst_buffer_copy_into (inbuf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
/* this is what we have */
avail = gst_adapter_available (scope->adapter);
GST_LOG_OBJECT (scope, "avail: %u, bpf: %u", avail, sbpf);
while (avail >= sbpf) {
GstBuffer *outbuf;
GstVideoFrame outframe;
/* get timestamp of the current adapter content */
ts = gst_adapter_prev_pts (scope->adapter, &dist);
if (GST_CLOCK_TIME_IS_VALID (ts)) {
/* convert bytes to time */
dist /= bps;
ts += gst_util_uint64_scale_int (dist, GST_SECOND, rate);
}
if (GST_CLOCK_TIME_IS_VALID (ts)) {
gint64 qostime;
gboolean need_skip;
qostime =
gst_segment_to_running_time (&scope->segment, GST_FORMAT_TIME, ts) +
scope->frame_duration;
GST_OBJECT_LOCK (scope);
/* check for QoS, don't compute buffers that are known to be late */
need_skip = scope->earliest_time != -1 && qostime <= scope->earliest_time;
GST_OBJECT_UNLOCK (scope);
if (need_skip) {
GST_WARNING_OBJECT (scope,
"QoS: skip ts: %" GST_TIME_FORMAT ", earliest: %" GST_TIME_FORMAT,
GST_TIME_ARGS (qostime), GST_TIME_ARGS (scope->earliest_time));
goto skip;
}
}
g_mutex_unlock (&scope->config_lock);
ret = default_prepare_output_buffer (scope, &outbuf);
g_mutex_lock (&scope->config_lock);
/* recheck as the value could have changed */
sbpf = scope->req_spf * channels * sizeof (gint16);
/* no buffer allocated, we don't care why. */
if (ret != GST_FLOW_OK)
break;
/* sync controlled properties */
if (GST_CLOCK_TIME_IS_VALID (ts))
gst_object_sync_values (GST_OBJECT (scope), ts);
GST_BUFFER_TIMESTAMP (outbuf) = ts;
GST_BUFFER_DURATION (outbuf) = scope->frame_duration;
/* this can fail as the data size we need could have changed */
if (!(adata = (gpointer) gst_adapter_map (scope->adapter, sbpf)))
break;
gst_video_frame_map (&outframe, &scope->vinfo, outbuf, GST_MAP_READWRITE);
if (scope->shader) {
gst_video_frame_copy (&outframe, &scope->tempframe);
} else {
/* gst_video_frame_clear() or is output frame already cleared */
gint i;
for (i = 0; i < scope->vinfo.finfo->n_planes; i++) {
memset (outframe.data[i], 0, outframe.map[i].size);
}
}
gst_buffer_replace_all_memory (inbuf,
gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, adata, sbpf, 0,
sbpf, NULL, NULL));
/* call class->render() vmethod */
if (klass->render) {
if (!klass->render (scope, inbuf, &outframe)) {
ret = GST_FLOW_ERROR;
} else {
/* run various post processing (shading and geometric transformation) */
/* FIXME: SHADER assumes 32bpp */
if (scope->shader &&
GST_VIDEO_INFO_COMP_PSTRIDE (&scope->vinfo, 0) == 4) {
scope->shader (scope, &outframe, &scope->tempframe);
}
}
}
gst_video_frame_unmap (&outframe);
g_mutex_unlock (&scope->config_lock);
ret = gst_pad_push (scope->srcpad, outbuf);
outbuf = NULL;
g_mutex_lock (&scope->config_lock);
skip:
/* recheck as the value could have changed */
sbpf = scope->req_spf * channels * sizeof (gint16);
GST_LOG_OBJECT (scope, "avail: %u, bpf: %u", avail, sbpf);
/* we want to take less or more, depending on spf : req_spf */
if (avail - sbpf >= sbpf) {
gst_adapter_flush (scope->adapter, sbpf);
gst_adapter_unmap (scope->adapter);
} else if (avail >= sbpf) {
/* just flush a bit and stop */
gst_adapter_flush (scope->adapter, (avail - sbpf));
gst_adapter_unmap (scope->adapter);
break;
}
avail = gst_adapter_available (scope->adapter);
if (ret != GST_FLOW_OK)
break;
}
g_mutex_unlock (&scope->config_lock);
beach:
return ret;
/* ERRORS */
not_negotiated:
{
GST_DEBUG_OBJECT (scope, "Failed to renegotiate");
return GST_FLOW_NOT_NEGOTIATED;
}
}
static gboolean
gst_audio_visualizer_src_event (GstPad * pad, GstObject * parent,
GstEvent * event)
{
gboolean res;
GstAudioVisualizer *scope;
scope = GST_AUDIO_VISUALIZER (parent);
switch (GST_EVENT_TYPE (event)) {
case GST_EVENT_QOS:
{
gdouble proportion;
GstClockTimeDiff diff;
GstClockTime timestamp;
gst_event_parse_qos (event, NULL, &proportion, &diff, &timestamp);
/* save stuff for the _chain() function */
GST_OBJECT_LOCK (scope);
scope->proportion = proportion;
if (diff >= 0)
/* we're late, this is a good estimate for next displayable
* frame (see part-qos.txt) */
scope->earliest_time = timestamp + 2 * diff + scope->frame_duration;
else
scope->earliest_time = timestamp + diff;
GST_OBJECT_UNLOCK (scope);
res = gst_pad_push_event (scope->sinkpad, event);
break;
}
case GST_EVENT_RECONFIGURE:
/* dont't forward */
gst_event_unref (event);
res = TRUE;
break;
default:
res = gst_pad_push_event (scope->sinkpad, event);
break;
}
return res;
}
static gboolean
gst_audio_visualizer_sink_event (GstPad * pad, GstObject * parent,
GstEvent * event)
{
gboolean res;
GstAudioVisualizer *scope;
scope = GST_AUDIO_VISUALIZER (parent);
switch (GST_EVENT_TYPE (event)) {
case GST_EVENT_CAPS:
{
GstCaps *caps;
gst_event_parse_caps (event, &caps);
res = gst_audio_visualizer_sink_setcaps (scope, caps);
break;
}
case GST_EVENT_FLUSH_START:
res = gst_pad_push_event (scope->srcpad, event);
break;
case GST_EVENT_FLUSH_STOP:
gst_audio_visualizer_reset (scope);
res = gst_pad_push_event (scope->srcpad, event);
break;
case GST_EVENT_SEGMENT:
{
/* the newsegment values are used to clip the input samples
* and to convert the incomming timestamps to running time so
* we can do QoS */
gst_event_copy_segment (event, &scope->segment);
res = gst_pad_push_event (scope->srcpad, event);
break;
}
default:
res = gst_pad_push_event (scope->srcpad, event);
break;
}
return res;
}
static gboolean
gst_audio_visualizer_src_query (GstPad * pad, GstObject * parent,
GstQuery * query)
{
gboolean res = FALSE;
GstAudioVisualizer *scope;
scope = GST_AUDIO_VISUALIZER (parent);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_LATENCY:
{
/* We need to send the query upstream and add the returned latency to our
* own */
GstClockTime min_latency, max_latency;
gboolean us_live;
GstClockTime our_latency;
guint max_samples;
gint rate = GST_AUDIO_INFO_RATE (&scope->ainfo);
if (rate == 0)
break;
if ((res = gst_pad_peer_query (scope->sinkpad, query))) {
gst_query_parse_latency (query, &us_live, &min_latency, &max_latency);
GST_DEBUG_OBJECT (scope, "Peer latency: min %"
GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));
/* the max samples we must buffer buffer */
max_samples = MAX (scope->req_spf, scope->spf);
our_latency = gst_util_uint64_scale_int (max_samples, GST_SECOND, rate);
GST_DEBUG_OBJECT (scope, "Our latency: %" GST_TIME_FORMAT,
GST_TIME_ARGS (our_latency));
/* we add some latency but only if we need to buffer more than what
* upstream gives us */
min_latency += our_latency;
if (max_latency != -1)
max_latency += our_latency;
GST_DEBUG_OBJECT (scope, "Calculated total latency : min %"
GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));
gst_query_set_latency (query, TRUE, min_latency, max_latency);
}
break;
}
default:
res = gst_pad_query_default (pad, parent, query);
break;
}
return res;
}
static gboolean
gst_audio_visualizer_sink_query (GstPad * pad, GstObject * parent,
GstQuery * query)
{
gboolean res = FALSE;
switch (GST_QUERY_TYPE (query)) {
default:
res = gst_pad_query_default (pad, parent, query);
break;
}
return res;
}
static GstStateChangeReturn
gst_audio_visualizer_change_state (GstElement * element,
GstStateChange transition)
{
GstStateChangeReturn ret;
GstAudioVisualizer *scope;
scope = GST_AUDIO_VISUALIZER (element);
switch (transition) {
case GST_STATE_CHANGE_READY_TO_PAUSED:
gst_audio_visualizer_reset (scope);
break;
default:
break;
}
ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
switch (transition) {
case GST_STATE_CHANGE_PAUSED_TO_READY:
gst_audio_visualizer_set_allocation (scope, NULL, NULL, NULL, NULL);
break;
case GST_STATE_CHANGE_READY_TO_NULL:
break;
default:
break;
}
return ret;
}
Reference in a new issue View git blame Copy permalink