/* GStreamer * Copyright (C) 2004 Benjamin Otte * Copyright (C) 2009 Jonathan Matthew * Copyright (C) 2011 Julien Isorce * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 59 Temple Place - Suite 330, * Boston, MA 02111-1307, USA. */ /** * SECTION:element-libvisualgl * * Wrapper for libvisual plugins that use OpenGL * * * Examples * |[ * gst-launch -v audiotestsrc ! libvisual_gl_lv_flower ! glimagesink * ]| * */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include #include #include #include #include #include #include #define GST_TYPE_VISUAL_GL (gst_visual_gl_get_type()) #define GST_IS_VISUAL_GL(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VISUAL_GL)) #define GST_VISUAL_GL(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VISUAL_GL,GstVisualGL)) #define GST_IS_VISUAL_GL_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VISUAL_GL)) #define GST_VISUAL_GL_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VISUAL_GL,GstVisualGLClass)) #define GST_VISUAL_GL_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_VISUAL_GL, GstVisualGLClass)) typedef struct _GstVisualGL GstVisualGL; typedef struct _GstVisualGLClass GstVisualGLClass; /* XXX use same category as libvisual plugin in -base? */ GST_DEBUG_CATEGORY_STATIC (libvisual_debug); #define GST_CAT_DEFAULT (libvisual_debug) /* amounf of samples before we can feed libvisual */ #define VISUAL_SAMPLES 512 #define DEFAULT_WIDTH 320 #define DEFAULT_HEIGHT 240 #define DEFAULT_FPS_N 25 #define DEFAULT_FPS_D 1 struct _GstVisualGL { GstElement element; /* pads */ GstPad *sinkpad; GstPad *srcpad; GstSegment segment; /* GL stuff */ GstGLDisplay *display; GLuint fbo; GLuint depthbuffer; GLdouble actor_projection_matrix[16]; GLdouble actor_modelview_matrix[16]; GLboolean is_enabled_gl_depth_test; GLint gl_depth_func; GLboolean is_enabled_gl_blend; GLint gl_blend_src_alpha; /* libvisual stuff */ VisAudio *audio; VisVideo *video; VisActor *actor; int actor_setup_result; /* audio/video state */ gint channels; gint rate; /* Input samplerate */ gint bps; VisAudioSampleRateType libvisual_rate; /* framerate numerator & denominator */ gint fps_n; gint fps_d; gint width; gint height; GstClockTime duration; guint outsize; /* samples per frame based on caps */ guint spf; /* state stuff */ GstAdapter *adapter; guint count; /* QoS stuff *//* with LOCK */ gdouble proportion; GstClockTime earliest_time; }; struct _GstVisualGLClass { GstElementClass parent_class; VisPluginRef *plugin; }; GType gst_visual_gl_get_type (void); static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_GL_VIDEO_CAPS) ); static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS ("audio/x-raw-int, " "width = (int) 16, " "depth = (int) 16, " "endianness = (int) BYTE_ORDER, " "signed = (boolean) TRUE, " "channels = (int) { 1, 2 }, " "rate = (int) { 8000, 11250, 22500, 32000, 44100, 48000, 96000 }") ); static void gst_visual_gl_class_init (gpointer g_class, gpointer class_data); static void gst_visual_gl_init (GstVisualGL * visual); static void gst_visual_gl_dispose (GObject * object); static GstStateChangeReturn gst_visual_gl_change_state (GstElement * element, GstStateChange transition); static GstFlowReturn gst_visual_gl_chain (GstPad * pad, GstBuffer * buffer); static gboolean gst_visual_gl_sink_event (GstPad * pad, GstEvent * event); static gboolean gst_visual_gl_src_event (GstPad * pad, GstEvent * event); static gboolean gst_visual_gl_src_query (GstPad * pad, GstQuery * query); static gboolean gst_visual_gl_sink_setcaps (GstPad * pad, GstCaps * caps); static gboolean gst_visual_gl_src_setcaps (GstPad * pad, GstCaps * caps); static GstCaps *gst_visual_gl_getcaps (GstPad * pad); static void libvisual_log_handler (const char *message, const char *funcname, void *priv); static GstElementClass *parent_class = NULL; GType gst_visual_gl_get_type (void) { static GType type = 0; if (G_UNLIKELY (type == 0)) { static const GTypeInfo info = { sizeof (GstVisualGLClass), NULL, NULL, gst_visual_gl_class_init, NULL, NULL, sizeof (GstVisualGL), 0, (GInstanceInitFunc) gst_visual_gl_init, }; type = g_type_register_static (GST_TYPE_ELEMENT, "GstVisualGL", &info, 0); } return type; } static void libvisual_log_handler (const char *message, const char *funcname, void *priv) { GST_CAT_LEVEL_LOG (libvisual_debug, (GstDebugLevel) (priv), NULL, "%s - %s", funcname, message); } static void gst_visual_gl_class_init (gpointer g_class, gpointer class_data) { GstVisualGLClass *klass = GST_VISUAL_GL_CLASS (g_class); GstElementClass *element_class = GST_ELEMENT_CLASS (g_class); GObjectClass *object = G_OBJECT_CLASS (g_class); klass->plugin = class_data; element_class->change_state = gst_visual_gl_change_state; if (class_data == NULL) { parent_class = g_type_class_peek_parent (g_class); } else { char *longname = g_strdup_printf ("libvisual %s plugin v.%s", klass->plugin->info->name, klass->plugin->info->version); /* FIXME: improve to only register what plugin supports? */ gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&src_template)); gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&sink_template)); gst_element_class_set_details_simple (element_class, longname, "Visualization", klass->plugin->info->about, "Benjamin Otte "); g_free (longname); } object->dispose = gst_visual_gl_dispose; } static void gst_visual_gl_init (GstVisualGL * visual) { /* create the sink and src pads */ visual->sinkpad = gst_pad_new_from_static_template (&sink_template, "sink"); gst_pad_set_setcaps_function (visual->sinkpad, gst_visual_gl_sink_setcaps); gst_pad_set_chain_function (visual->sinkpad, gst_visual_gl_chain); gst_pad_set_event_function (visual->sinkpad, gst_visual_gl_sink_event); gst_element_add_pad (GST_ELEMENT (visual), visual->sinkpad); visual->srcpad = gst_pad_new_from_static_template (&src_template, "src"); gst_pad_set_setcaps_function (visual->srcpad, gst_visual_gl_src_setcaps); gst_pad_set_getcaps_function (visual->srcpad, gst_visual_gl_getcaps); gst_pad_set_event_function (visual->srcpad, gst_visual_gl_src_event); gst_pad_set_query_function (visual->srcpad, gst_visual_gl_src_query); gst_element_add_pad (GST_ELEMENT (visual), visual->srcpad); visual->adapter = gst_adapter_new (); visual->actor = NULL; visual->display = NULL; visual->fbo = 0; visual->depthbuffer = 0; visual->is_enabled_gl_depth_test = GL_FALSE; visual->gl_depth_func = GL_LESS; visual->is_enabled_gl_blend = GL_FALSE; visual->gl_blend_src_alpha = GL_ONE; } static void gst_visual_gl_clear_actors (GstVisualGL * visual) { if (visual->actor) { visual_object_unref (VISUAL_OBJECT (visual->actor)); visual->actor = NULL; } if (visual->video) { visual_object_unref (VISUAL_OBJECT (visual->video)); visual->video = NULL; } if (visual->audio) { visual_object_unref (VISUAL_OBJECT (visual->audio)); visual->audio = NULL; } } static void gst_visual_gl_dispose (GObject * object) { GstVisualGL *visual = GST_VISUAL_GL (object); if (visual->adapter) { g_object_unref (visual->adapter); visual->adapter = NULL; } GST_CALL_PARENT (G_OBJECT_CLASS, dispose, (object)); } static void gst_visual_gl_reset (GstVisualGL * visual) { gst_adapter_clear (visual->adapter); gst_segment_init (&visual->segment, GST_FORMAT_UNDEFINED); GST_OBJECT_LOCK (visual); visual->proportion = 1.0; visual->earliest_time = -1; GST_OBJECT_UNLOCK (visual); } static GstCaps * gst_visual_gl_getcaps (GstPad * pad) { GstCaps *ret; GstVisualGL *visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); int depths; if (!visual->actor) { ret = gst_caps_copy (gst_pad_get_pad_template_caps (visual->srcpad)); goto beach; } ret = gst_caps_new_empty (); depths = visual_actor_get_supported_depth (visual->actor); if (depths < 0) { /* FIXME: set an error */ goto beach; } if ((depths & VISUAL_VIDEO_DEPTH_GL) == 0) { /* We don't handle non-GL plugins */ goto beach; } GST_DEBUG_OBJECT (visual, "libvisual-gl plugin supports depths %u (0x%04x)", depths, depths); /* only do GL output */ gst_caps_append (ret, gst_caps_from_string (GST_GL_VIDEO_CAPS)); beach: GST_DEBUG_OBJECT (visual, "returning caps %" GST_PTR_FORMAT, ret); gst_object_unref (visual); return ret; } static gboolean gst_visual_gl_src_setcaps (GstPad * pad, GstCaps * caps) { GstVisualGL *visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); GstStructure *structure; structure = gst_caps_get_structure (caps, 0); GST_DEBUG_OBJECT (visual, "src pad got caps %" GST_PTR_FORMAT, caps); if (!gst_structure_get_int (structure, "width", &visual->width)) goto error; if (!gst_structure_get_int (structure, "height", &visual->height)) goto error; if (!gst_structure_get_fraction (structure, "framerate", &visual->fps_n, &visual->fps_d)) goto error; /* precalc some values */ visual->spf = gst_util_uint64_scale_int (visual->rate, visual->fps_d, visual->fps_n); visual->duration = gst_util_uint64_scale_int (GST_SECOND, visual->fps_d, visual->fps_n); gst_gl_display_gen_fbo (visual->display, visual->width, visual->height, &visual->fbo, &visual->depthbuffer); gst_object_unref (visual); return TRUE; /* ERRORS */ error: { GST_DEBUG_OBJECT (visual, "error parsing caps"); gst_object_unref (visual); return FALSE; } } static gboolean gst_visual_gl_sink_setcaps (GstPad * pad, GstCaps * caps) { GstVisualGL *visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); GstStructure *structure; structure = gst_caps_get_structure (caps, 0); gst_structure_get_int (structure, "channels", &visual->channels); gst_structure_get_int (structure, "rate", &visual->rate); switch (visual->rate) { case 8000: visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_8000; break; case 11250: visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_11250; break; case 22500: visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_22500; break; case 32000: visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_32000; break; case 44100: visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_44100; break; case 48000: visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_48000; break; case 96000: visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_96000; break; default: gst_object_unref (visual); return FALSE; } /* this is how many samples we need to fill one frame at the requested * framerate. */ if (visual->fps_n != 0) { visual->spf = gst_util_uint64_scale_int (visual->rate, visual->fps_d, visual->fps_n); } visual->bps = visual->channels * sizeof (gint16); gst_object_unref (visual); return TRUE; } static gboolean gst_vis_gl_src_negotiate (GstVisualGL * visual) { GstCaps *othercaps, *target; GstStructure *structure; GstCaps *caps; caps = gst_pad_get_caps (visual->srcpad); /* see what the peer can do */ othercaps = gst_pad_peer_get_caps (visual->srcpad); if (othercaps) { target = gst_caps_intersect (othercaps, caps); gst_caps_unref (othercaps); gst_caps_unref (caps); if (gst_caps_is_empty (target)) goto no_format; gst_caps_truncate (target); } else { /* need a copy, we'll be modifying it when fixating */ target = gst_caps_copy (caps); gst_caps_unref (caps); } /* fixate in case something is not fixed. This does nothing if the value is * already fixed. For video we always try to fixate to something like * 320x240x25 by convention. */ structure = gst_caps_get_structure (target, 0); gst_structure_fixate_field_nearest_int (structure, "width", DEFAULT_WIDTH); gst_structure_fixate_field_nearest_int (structure, "height", DEFAULT_HEIGHT); gst_structure_fixate_field_nearest_fraction (structure, "framerate", DEFAULT_FPS_N, DEFAULT_FPS_D); gst_pad_set_caps (visual->srcpad, target); gst_caps_unref (target); return TRUE; /* ERRORS */ no_format: { GST_ELEMENT_ERROR (visual, STREAM, FORMAT, (NULL), ("could not negotiate output format")); gst_caps_unref (target); return FALSE; } } static gboolean gst_visual_gl_sink_event (GstPad * pad, GstEvent * event) { GstVisualGL *visual; gboolean res; visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); switch (GST_EVENT_TYPE (event)) { case GST_EVENT_FLUSH_START: res = gst_pad_push_event (visual->srcpad, event); break; case GST_EVENT_FLUSH_STOP: /* reset QoS and adapter. */ gst_visual_gl_reset (visual); res = gst_pad_push_event (visual->srcpad, event); break; case GST_EVENT_NEWSEGMENT: { GstFormat format; gdouble rate, arate; gint64 start, stop, time; gboolean update; /* the newsegment values are used to clip the input samples * and to convert the incomming timestamps to running time so * we can do QoS */ gst_event_parse_new_segment_full (event, &update, &rate, &arate, &format, &start, &stop, &time); /* now configure the values */ gst_segment_set_newsegment_full (&visual->segment, update, rate, arate, format, start, stop, time); /* and forward */ res = gst_pad_push_event (visual->srcpad, event); break; } default: res = gst_pad_push_event (visual->srcpad, event); break; } gst_object_unref (visual); return res; } static gboolean gst_visual_gl_src_event (GstPad * pad, GstEvent * event) { GstVisualGL *visual; gboolean res; visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); switch (GST_EVENT_TYPE (event)) { case GST_EVENT_QOS: { gdouble proportion; GstClockTimeDiff diff; GstClockTime timestamp; gst_event_parse_qos (event, &proportion, &diff, ×tamp); /* save stuff for the _chain function */ GST_OBJECT_LOCK (visual); visual->proportion = proportion; if (diff >= 0) /* we're late, this is a good estimate for next displayable * frame (see part-qos.txt) */ visual->earliest_time = timestamp + 2 * diff + visual->duration; else visual->earliest_time = timestamp + diff; GST_OBJECT_UNLOCK (visual); res = gst_pad_push_event (visual->sinkpad, event); break; } default: res = gst_pad_push_event (visual->sinkpad, event); break; } gst_object_unref (visual); return res; } static gboolean gst_visual_gl_src_query (GstPad * pad, GstQuery * query) { gboolean res; GstVisualGL *visual; visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); switch (GST_QUERY_TYPE (query)) { case GST_QUERY_LATENCY: { /* We need to send the query upstream and add the returned latency to our * own */ GstClockTime min_latency, max_latency; gboolean us_live; GstClockTime our_latency; guint max_samples; if ((res = gst_pad_peer_query (visual->sinkpad, query))) { gst_query_parse_latency (query, &us_live, &min_latency, &max_latency); GST_DEBUG_OBJECT (visual, "Peer latency: min %" GST_TIME_FORMAT " max %" GST_TIME_FORMAT, GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency)); /* the max samples we must buffer buffer */ max_samples = MAX (VISUAL_SAMPLES, visual->spf); our_latency = gst_util_uint64_scale_int (max_samples, GST_SECOND, visual->rate); GST_DEBUG_OBJECT (visual, "Our latency: %" GST_TIME_FORMAT, GST_TIME_ARGS (our_latency)); /* we add some latency but only if we need to buffer more than what * upstream gives us */ min_latency += our_latency; if (max_latency != -1) max_latency += our_latency; GST_DEBUG_OBJECT (visual, "Calculated total latency : min %" GST_TIME_FORMAT " max %" GST_TIME_FORMAT, GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency)); gst_query_set_latency (query, TRUE, min_latency, max_latency); } break; } case GST_QUERY_CUSTOM: { GstStructure *structure = gst_query_get_structure (query); res = g_strcmp0 (gst_element_get_name (visual), gst_structure_get_name (structure)) == 0; if (!res) res = gst_pad_query_default (pad, query); break; } default: res = gst_pad_peer_query (visual->sinkpad, query); break; } gst_object_unref (visual); return res; } /* allocate and output buffer, if no format was negotiated, this * function will negotiate one. After calling this function, a * reverse negotiation could have happened. */ static GstFlowReturn get_buffer (GstVisualGL * visual, GstGLBuffer ** outbuf) { /* we don't know an output format yet, pick one */ if (GST_PAD_CAPS (visual->srcpad) == NULL) { if (!gst_vis_gl_src_negotiate (visual)) return GST_FLOW_NOT_NEGOTIATED; } GST_DEBUG_OBJECT (visual, "allocating output buffer with caps %" GST_PTR_FORMAT, GST_PAD_CAPS (visual->srcpad)); *outbuf = gst_gl_buffer_new (visual->display, visual->width, visual->height); if (*outbuf == NULL) return GST_FLOW_ERROR; gst_buffer_set_caps (GST_BUFFER (*outbuf), GST_PAD_CAPS (visual->srcpad)); return GST_FLOW_OK; } static void actor_setup (GstGLDisplay * display, GstVisualGL * visual) { /* save and clear top of the stack */ glPushAttrib (GL_ALL_ATTRIB_BITS); glMatrixMode (GL_PROJECTION); glPushMatrix (); glLoadIdentity (); glMatrixMode (GL_MODELVIEW); glPushMatrix (); glLoadIdentity (); visual->actor_setup_result = visual_actor_realize (visual->actor); if (visual->actor_setup_result == 0) { /* store the actor's matrices for rendering the first frame */ glGetDoublev (GL_MODELVIEW_MATRIX, visual->actor_modelview_matrix); glGetDoublev (GL_PROJECTION_MATRIX, visual->actor_projection_matrix); visual->is_enabled_gl_depth_test = glIsEnabled (GL_DEPTH_TEST); glGetIntegerv (GL_DEPTH_FUNC, &visual->gl_depth_func); visual->is_enabled_gl_blend = glIsEnabled (GL_BLEND); glGetIntegerv (GL_BLEND_SRC_ALPHA, &visual->gl_blend_src_alpha); /* retore matrix */ glMatrixMode (GL_PROJECTION); glPopMatrix (); glMatrixMode (GL_MODELVIEW); glPopMatrix (); glPopAttrib (); } } static void actor_negotiate (GstGLDisplay * display, GstVisualGL * visual) { gint err = VISUAL_OK; err = visual_video_set_depth (visual->video, VISUAL_VIDEO_DEPTH_GL); if (err != VISUAL_OK) g_warning ("failed to visual_video_set_depth\n"); err = visual_video_set_dimension (visual->video, visual->width, visual->height); if (err != VISUAL_OK) g_warning ("failed to visual_video_set_dimension\n"); err = visual_actor_video_negotiate (visual->actor, 0, FALSE, FALSE); if (err != VISUAL_OK) g_warning ("failed to visual_actor_video_negotiate\n"); } static void check_gl_matrix (void) { GLdouble projection_matrix[16]; GLdouble modelview_matrix[16]; gint i = 0; gint j = 0; glGetDoublev (GL_PROJECTION_MATRIX, projection_matrix); glGetDoublev (GL_MODELVIEW_MATRIX, modelview_matrix); for (j = 0; j < 4; ++j) { for (i = 0; i < 4; ++i) { if (projection_matrix[i + 4 * j] != projection_matrix[i + 4 * j]) g_warning ("invalid projection matrix at coordiante %dx%d: %f\n", i, j, projection_matrix[i + 4 * j]); if (modelview_matrix[i + 4 * j] != modelview_matrix[i + 4 * j]) g_warning ("invalid modelview_matrix matrix at coordiante %dx%d: %f\n", i, j, modelview_matrix[i + 4 * j]); } } } static void render_frame (gint width, gint height, guint texture, GstVisualGL * visual) { const guint16 *data; VisBuffer *lbuf, *rbuf; guint16 ldata[VISUAL_SAMPLES], rdata[VISUAL_SAMPLES]; guint i; //GLint current_fbo = 0; /* Read VISUAL_SAMPLES samples per channel */ data = (const guint16 *) gst_adapter_peek (visual->adapter, VISUAL_SAMPLES * visual->bps); lbuf = visual_buffer_new_with_buffer (ldata, sizeof (ldata), NULL); rbuf = visual_buffer_new_with_buffer (rdata, sizeof (rdata), NULL); if (visual->channels == 2) { for (i = 0; i < VISUAL_SAMPLES; i++) { ldata[i] = *data++; rdata[i] = *data++; } } else { for (i = 0; i < VISUAL_SAMPLES; i++) { ldata[i] = *data; rdata[i] = *data++; } } visual_audio_samplepool_input_channel (visual->audio->samplepool, lbuf, visual->libvisual_rate, VISUAL_AUDIO_SAMPLE_FORMAT_S16, VISUAL_AUDIO_CHANNEL_LEFT); visual_audio_samplepool_input_channel (visual->audio->samplepool, rbuf, visual->libvisual_rate, VISUAL_AUDIO_SAMPLE_FORMAT_S16, VISUAL_AUDIO_CHANNEL_RIGHT); visual_object_unref (VISUAL_OBJECT (lbuf)); visual_object_unref (VISUAL_OBJECT (rbuf)); visual_audio_analyze (visual->audio); /* apply the matrices that the actor set up */ glPushAttrib (GL_ALL_ATTRIB_BITS); glMatrixMode (GL_PROJECTION); glPushMatrix (); glLoadMatrixd (visual->actor_projection_matrix); glMatrixMode (GL_MODELVIEW); glPushMatrix (); glLoadMatrixd (visual->actor_modelview_matrix); /* This line try to hacks compatiblity with libprojectM * You have to do that before calling glDrawBuffer(GL_BACK) * Actually, at this point the current fbo is attached. * then the folowing line unbind it. * TODO: We have to rebind it just before final drawing * if we want to append other glfilters after it. */ //glGetIntegerv(GL_FRAMEBUFFER_BINDING, ¤t_fbo); if (g_ascii_strncasecmp (gst_element_get_name (GST_ELEMENT (visual)), "visualglprojectm", 16) == 0) glBindFramebufferEXT (GL_FRAMEBUFFER_EXT, 0); actor_negotiate (visual->display, visual); if (visual->is_enabled_gl_depth_test) { glEnable (GL_DEPTH_TEST); glDepthFunc (visual->gl_depth_func); } if (visual->is_enabled_gl_blend) { glEnable (GL_BLEND); glBlendFunc (visual->gl_blend_src_alpha, GL_ZERO); } glMatrixMode (GL_MODELVIEW); glScaled (1.0, -1.0, 1.0); /* TODO: It should be possible to split libvisual rendering: * framebuffer pass1,2,3 ... and final rendering * This way we could rebind our fbo just before the * final libvisual rendering */ //glBindFramebufferEXT (GL_FRAMEBUFFER_EXT, current_fbo); visual_actor_run (visual->actor, visual->audio); check_gl_matrix (); glMatrixMode (GL_PROJECTION); glPopMatrix (); glMatrixMode (GL_MODELVIEW); glPopMatrix (); glPopAttrib (); glDisable (GL_DEPTH_TEST); glDisable (GL_BLEND); /*glDisable (GL_LIGHT0); glDisable (GL_LIGHTING); glDisable (GL_POLYGON_OFFSET_FILL); glDisable (GL_COLOR_MATERIAL); glDisable (GL_CULL_FACE); */ GST_DEBUG_OBJECT (visual, "rendered one frame"); } static GstFlowReturn gst_visual_gl_chain (GstPad * pad, GstBuffer * buffer) { GstGLBuffer *outbuf = NULL; GstVisualGL *visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); GstFlowReturn ret = GST_FLOW_OK; guint avail; GST_DEBUG_OBJECT (visual, "chain function called"); /* If we don't have an output format yet, preallocate a buffer to try and * set one */ if (GST_PAD_CAPS (visual->srcpad) == NULL) { ret = get_buffer (visual, &outbuf); if (ret != GST_FLOW_OK) { gst_buffer_unref (buffer); goto beach; } } /* resync on DISCONT */ if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT)) { gst_adapter_clear (visual->adapter); } GST_DEBUG_OBJECT (visual, "Input buffer has %d samples, time=%" G_GUINT64_FORMAT, GST_BUFFER_SIZE (buffer) / visual->bps, GST_BUFFER_TIMESTAMP (buffer)); gst_adapter_push (visual->adapter, buffer); while (TRUE) { gboolean need_skip; guint64 dist, timestamp; GST_DEBUG_OBJECT (visual, "processing buffer"); avail = gst_adapter_available (visual->adapter); GST_DEBUG_OBJECT (visual, "avail now %u", avail); /* we need at least VISUAL_SAMPLES samples */ if (avail < VISUAL_SAMPLES * visual->bps) break; /* we need at least enough samples to make one frame */ if (avail < visual->spf * visual->bps) break; /* get timestamp of the current adapter byte */ timestamp = gst_adapter_prev_timestamp (visual->adapter, &dist); if (GST_CLOCK_TIME_IS_VALID (timestamp)) { /* convert bytes to time */ dist /= visual->bps; timestamp += gst_util_uint64_scale_int (dist, GST_SECOND, visual->rate); } if (timestamp != -1) { gint64 qostime; /* QoS is done on running time */ qostime = gst_segment_to_running_time (&visual->segment, GST_FORMAT_TIME, timestamp); qostime += visual->duration; GST_OBJECT_LOCK (visual); /* check for QoS, don't compute buffers that are known to be late */ need_skip = visual->earliest_time != -1 && qostime <= visual->earliest_time; GST_OBJECT_UNLOCK (visual); if (need_skip) { GST_WARNING_OBJECT (visual, "QoS: skip ts: %" GST_TIME_FORMAT ", earliest: %" GST_TIME_FORMAT, GST_TIME_ARGS (qostime), GST_TIME_ARGS (visual->earliest_time)); goto skip; } } /* alloc a buffer if we don't have one yet, this happens * when we pushed a buffer in this while loop before */ if (outbuf == NULL) { ret = get_buffer (visual, &outbuf); if (ret != GST_FLOW_OK) { goto beach; } } /* this is a pretty weird API */ gst_gl_display_use_fbo (visual->display, visual->width, visual->height, visual->fbo, visual->depthbuffer, outbuf->texture, (GLCB) render_frame, 0, 0, 0, 0, visual->width, 0, visual->height, GST_GL_DISPLAY_PROJECTION_ORTHO2D, (gpointer *) visual); GST_BUFFER_TIMESTAMP (outbuf) = timestamp; GST_BUFFER_DURATION (outbuf) = visual->duration; ret = gst_pad_push (visual->srcpad, GST_BUFFER (outbuf)); outbuf = NULL; skip: GST_DEBUG_OBJECT (visual, "finished frame, flushing %u samples from input", visual->spf); /* Flush out the number of samples per frame */ gst_adapter_flush (visual->adapter, visual->spf * visual->bps); /* quit the loop if something was wrong */ if (ret != GST_FLOW_OK) break; } beach: if (outbuf != NULL) gst_gl_buffer_unref (outbuf); gst_object_unref (visual); return ret; } static GstStateChangeReturn gst_visual_gl_change_state (GstElement * element, GstStateChange transition) { GstVisualGL *visual = GST_VISUAL_GL (element); GstStateChangeReturn ret; switch (transition) { case GST_STATE_CHANGE_NULL_TO_READY: break; case GST_STATE_CHANGE_READY_TO_PAUSED: { GstElement *parent = GST_ELEMENT (gst_element_get_parent (visual)); GstStructure *structure = NULL; GstQuery *query = NULL; gboolean isPerformed = FALSE; if (!parent) { GST_ELEMENT_ERROR (visual, CORE, STATE_CHANGE, (NULL), ("A parent bin is required")); return FALSE; } structure = gst_structure_new (gst_element_get_name (visual), NULL); query = gst_query_new_application (GST_QUERY_CUSTOM, structure); isPerformed = gst_element_query (parent, query); if (isPerformed) { const GValue *id_value = gst_structure_get_value (structure, "gstgldisplay"); if (G_VALUE_HOLDS_POINTER (id_value)) /* at least one gl element is after in our gl chain */ visual->display = g_object_ref (GST_GL_DISPLAY (g_value_get_pointer (id_value))); else { /* this gl filter is a sink in terms of the gl chain */ visual->display = gst_gl_display_new (); gst_gl_display_create_context (visual->display, 0); //TODO visual->external_gl_context); } gst_visual_gl_reset (visual); visual->actor = visual_actor_new (GST_VISUAL_GL_GET_CLASS (visual)->plugin->info-> plugname); visual->video = visual_video_new (); visual->audio = visual_audio_new (); if (!visual->actor || !visual->video) goto actor_setup_failed; gst_gl_display_thread_add (visual->display, (GstGLDisplayThreadFunc) actor_setup, visual); if (visual->actor_setup_result != 0) goto actor_setup_failed; else visual_actor_set_video (visual->actor, visual->video); } gst_query_unref (query); gst_object_unref (GST_OBJECT (parent)); if (!isPerformed) return GST_STATE_CHANGE_FAILURE; } break; case GST_STATE_CHANGE_PAUSED_TO_PLAYING: break; default: break; } ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); switch (transition) { case GST_STATE_CHANGE_PLAYING_TO_PAUSED: break; case GST_STATE_CHANGE_PAUSED_TO_READY: { if (visual->fbo) { gst_gl_display_del_fbo (visual->display, visual->fbo, visual->depthbuffer); visual->fbo = 0; visual->depthbuffer = 0; } if (visual->display) { g_object_unref (visual->display); visual->display = NULL; } gst_visual_gl_clear_actors (visual); } break; case GST_STATE_CHANGE_READY_TO_NULL: break; default: break; } return ret; /* ERRORS */ actor_setup_failed: { GST_ELEMENT_ERROR (visual, LIBRARY, INIT, (NULL), ("could not set up actor")); gst_visual_gl_clear_actors (visual); return GST_STATE_CHANGE_FAILURE; } } static void make_valid_name (char *name) { /* * Replace invalid chars with _ in the type name */ static const gchar extra_chars[] = "-_+"; gchar *p = name; for (; *p; p++) { int valid = ((p[0] >= 'A' && p[0] <= 'Z') || (p[0] >= 'a' && p[0] <= 'z') || (p[0] >= '0' && p[0] <= '9') || strchr (extra_chars, p[0])); if (!valid) *p = '_'; } } static gboolean gst_visual_gl_actor_plugin_is_gl (VisObject * plugin, const gchar * name) { gboolean is_gl; gint depth; depth = VISUAL_ACTOR_PLUGIN (plugin)->vidoptions.depth; is_gl = (depth & VISUAL_VIDEO_DEPTH_GL) != 0; if (!is_gl) { GST_DEBUG ("plugin %s is not a GL plugin (%d), ignoring", name, depth); } else { GST_DEBUG ("plugin %s is a GL plugin (%d), registering", name, depth); } return is_gl; } static gboolean plugin_init (GstPlugin * plugin) { guint i, count; VisList *list; GST_DEBUG_CATEGORY_INIT (libvisual_debug, "libvisual", 0, "libvisual audio visualisations"); #ifdef LIBVISUAL_PLUGINSBASEDIR gst_plugin_add_dependency_simple (plugin, "HOME/.libvisual/actor", LIBVISUAL_PLUGINSBASEDIR "/actor", NULL, GST_PLUGIN_DEPENDENCY_FLAG_NONE); #endif visual_log_set_verboseness (VISUAL_LOG_VERBOSENESS_LOW); visual_log_set_info_handler (libvisual_log_handler, (void *) GST_LEVEL_INFO); visual_log_set_warning_handler (libvisual_log_handler, (void *) GST_LEVEL_WARNING); visual_log_set_critical_handler (libvisual_log_handler, (void *) GST_LEVEL_ERROR); visual_log_set_error_handler (libvisual_log_handler, (void *) GST_LEVEL_ERROR); if (!visual_is_initialized ()) if (visual_init (NULL, NULL) != 0) return FALSE; list = visual_actor_get_list (); count = visual_collection_size (VISUAL_COLLECTION (list)); for (i = 0; i < count; i++) { VisPluginRef *ref = visual_list_get (list, i); VisPluginData *visplugin = NULL; gboolean skip = FALSE; GType type; gchar *name; GTypeInfo info = { sizeof (GstVisualGLClass), NULL, NULL, gst_visual_gl_class_init, NULL, ref, sizeof (GstVisualGL), 0, NULL }; visplugin = visual_plugin_load (ref); if (ref->info->plugname == NULL) continue; /* Blacklist some plugins */ if (strcmp (ref->info->plugname, "gstreamer") == 0 || strcmp (ref->info->plugname, "gdkpixbuf") == 0) { skip = TRUE; } else { /* only register plugins that support GL */ skip = !(gst_visual_gl_actor_plugin_is_gl (visplugin->info->plugin, visplugin->info->plugname)); } visual_plugin_unload (visplugin); if (!skip) { name = g_strdup_printf ("GstVisualGL%s", ref->info->plugname); make_valid_name (name); type = g_type_register_static (GST_TYPE_VISUAL_GL, name, &info, 0); g_free (name); name = g_strdup_printf ("libvisual_gl_%s", ref->info->plugname); make_valid_name (name); if (!gst_element_register (plugin, name, GST_RANK_NONE, type)) { g_free (name); return FALSE; } g_free (name); } } return TRUE; } GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, "libvisual-gl", "libvisual-gl visualization plugins", plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)