diff --git a/sys/androidmedia/Makefile.am b/sys/androidmedia/Makefile.am index 5fcfb8f24c..95d31cf4a1 100644 --- a/sys/androidmedia/Makefile.am +++ b/sys/androidmedia/Makefile.am @@ -5,6 +5,9 @@ libgstandroidmedia_la_SOURCES = \ gstamcaudiodec.c \ gstamcvideodec.c \ gstamcvideoenc.c \ + gstamcsurface.c \ + gstamcsurfacetexture.c \ + gstamc2dtexturerenderer.c \ gstjniutils.c noinst_HEADERS = \ @@ -13,14 +16,20 @@ noinst_HEADERS = \ gstamcaudiodec.h \ gstamcvideodec.h \ gstamcvideoenc.h \ + gstamcsurface.h \ + gstamcsurfacetexture.h \ + gstamc2dtexturerenderer.h \ gstjniutils.h libgstandroidmedia_la_CFLAGS = \ + -I$(top_srcdir)/gst-libs \ + -I$(top_builddir)/gst-libs \ $(GST_PLUGINS_BASE_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_CFLAGS) \ $(ORC_CFLAGS) libgstandroidmedia_la_LIBADD = \ + $(top_builddir)/gst-libs/gst/gl/libgstgl-$(GST_API_VERSION).la \ $(GST_PLUGINS_BASE_LIBS) \ -lgstaudio-@GST_API_VERSION@ \ -lgstpbutils-@GST_API_VERSION@ \ @@ -30,3 +39,6 @@ libgstandroidmedia_la_LIBADD = \ $(ORC_LIBS) libgstandroidmedia_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) libgstandroidmedia_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) + +androidmedia_java_classesdir = $(datadir)/gst-android/ndk-build/androidmedia/ +androidmedia_java_classes_DATA = org/freedesktop/gstreamer/androidmedia/GstAmcOnFrameAvailableListener.java diff --git a/sys/androidmedia/gstamc.c b/sys/androidmedia/gstamc.c index 0ca7e2db87..da1063a319 100644 --- a/sys/androidmedia/gstamc.c +++ b/sys/androidmedia/gstamc.c @@ -185,8 +185,8 @@ gst_amc_codec_free (GstAmcCodec * codec) } gboolean -gst_amc_codec_configure (GstAmcCodec * codec, GstAmcFormat * format, gint flags, - GError ** err) +gst_amc_codec_configure (GstAmcCodec * codec, GstAmcFormat * format, + jobject surface, gint flags, GError ** err) { JNIEnv *env; @@ -195,7 +195,7 @@ gst_amc_codec_configure (GstAmcCodec * codec, GstAmcFormat * format, gint flags, env = gst_amc_jni_get_env (); return gst_amc_jni_call_void_method (env, err, codec->object, - media_codec.configure, format->object, NULL, NULL, flags); + media_codec.configure, format->object, surface, NULL, flags); } GstAmcFormat * @@ -591,7 +591,7 @@ gst_amc_codec_queue_input_buffer (GstAmcCodec * codec, gint index, gboolean gst_amc_codec_release_output_buffer (GstAmcCodec * codec, gint index, - GError ** err) + gboolean render, GError ** err) { JNIEnv *env; @@ -599,7 +599,7 @@ gst_amc_codec_release_output_buffer (GstAmcCodec * codec, gint index, env = gst_amc_jni_get_env (); return gst_amc_jni_call_void_method (env, err, codec->object, - media_codec.release_output_buffer, index, JNI_FALSE); + media_codec.release_output_buffer, index, render); } GstAmcFormat * diff --git a/sys/androidmedia/gstamc.h b/sys/androidmedia/gstamc.h index a3fb9bc085..7b0cd9b3ad 100644 --- a/sys/androidmedia/gstamc.h +++ b/sys/androidmedia/gstamc.h @@ -82,7 +82,7 @@ extern GQuark gst_amc_codec_info_quark; GstAmcCodec * gst_amc_codec_new (const gchar *name, GError **err); void gst_amc_codec_free (GstAmcCodec * codec); -gboolean gst_amc_codec_configure (GstAmcCodec * codec, GstAmcFormat * format, gint flags, GError **err); +gboolean gst_amc_codec_configure (GstAmcCodec * codec, GstAmcFormat * format, jobject surface, gint flags, GError **err); GstAmcFormat * gst_amc_codec_get_output_format (GstAmcCodec * codec, GError **err); gboolean gst_amc_codec_start (GstAmcCodec * codec, GError **err); @@ -97,7 +97,7 @@ gint gst_amc_codec_dequeue_input_buffer (GstAmcCodec * codec, gint64 timeoutUs, gint gst_amc_codec_dequeue_output_buffer (GstAmcCodec * codec, GstAmcBufferInfo *info, gint64 timeoutUs, GError **err); gboolean gst_amc_codec_queue_input_buffer (GstAmcCodec * codec, gint index, const GstAmcBufferInfo *info, GError **err); -gboolean gst_amc_codec_release_output_buffer (GstAmcCodec * codec, gint index, GError **err); +gboolean gst_amc_codec_release_output_buffer (GstAmcCodec * codec, gint index, gboolean render, GError **err); GstAmcFormat * gst_amc_format_new_audio (const gchar *mime, gint sample_rate, gint channels, GError **err); @@ -184,6 +184,8 @@ void gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, GstCaps **s g_clear_error (&err); \ } G_STMT_END +GST_DEBUG_CATEGORY_EXTERN (gst_amc_debug); + G_END_DECLS #endif /* __GST_AMC_H__ */ diff --git a/sys/androidmedia/gstamc2dtexturerenderer.c b/sys/androidmedia/gstamc2dtexturerenderer.c new file mode 100644 index 0000000000..b8361ac5f3 --- /dev/null +++ b/sys/androidmedia/gstamc2dtexturerenderer.c @@ -0,0 +1,438 @@ +/* + * Copyright (C) 2014, Collabora Ltd. + * Author: Matthieu Bouron + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstamc2dtexturerenderer.h" + +/* *INDENT-OFF* */ + +static const gchar frag_COPY_OES[] = + "#extension GL_OES_EGL_image_external : require \n" + "precision mediump float; \n" + "varying vec2 v_texcoord; \n" + "uniform samplerExternalOES u_tex; \n" + "void main (void) \n" + "{ \n" + " vec4 t = texture2D(u_tex, v_texcoord); \n" + " gl_FragColor = vec4(t.rgb, 1.0); \n" + "}"; + +static const gchar vert_COPY_OES[] = + "attribute vec4 a_position; \n" + "attribute vec2 a_texcoord; \n" + "varying vec2 v_texcoord; \n" + "uniform mat4 u_transformation; \n" + "void main() \n" + "{ \n" + " gl_Position = a_position; \n" + " v_texcoord = (u_transformation * vec4(a_texcoord, 0, 1)).xy; \n" + "}"; + +/* *INDENT-ON* */ +static void +_surface_texture_detach_from_gl_context (GstGLContext * context, + GstAmc2DTextureRenderer * renderer) +{ + renderer->gl_context_result = + gst_amc_surface_texture_detach_from_gl_context (renderer->surface_texture, + &renderer->gl_context_error); +} + +static gboolean +_surface_texture_detach_from_gl_context_perform (GstAmc2DTextureRenderer * + renderer, GError ** error) +{ + renderer->gl_context_result = FALSE; + renderer->gl_context_error = NULL; + + gst_gl_context_thread_add (renderer->context, + (GstGLContextThreadFunc) _surface_texture_detach_from_gl_context, + renderer); + + *error = renderer->gl_context_error; + renderer->gl_context_error = NULL; + + return renderer->gl_context_result; +} + +static void +_gen_oes_texture (GstGLContext * context, guint * tex_id) +{ + const GstGLFuncs *gl = context->gl_vtable; + + GST_TRACE ("Generating OES texture"); + + gl->GenTextures (1, tex_id); + gl->BindTexture (GL_TEXTURE_EXTERNAL_OES, *tex_id); + + gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, + GL_CLAMP_TO_EDGE); + gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, + GL_CLAMP_TO_EDGE); + + gl->BindTexture (GL_TEXTURE_EXTERNAL_OES, 0); + + GST_LOG ("generated OES texture id:%d", *tex_id); +} + +GstAmc2DTextureRenderer * +gst_amc_2d_texture_renderer_new (GstGLContext * context, + GstAmcSurfaceTexture * surface_texture, guint width, guint height) +{ + GstAmc2DTextureRenderer *renderer; + + g_return_val_if_fail (surface_texture != NULL, NULL); + + renderer = g_new0 (GstAmc2DTextureRenderer, 1); + if (!renderer) + return NULL; + + renderer->context = gst_object_ref (context); + renderer->surface_texture = g_object_ref (surface_texture); + + gst_video_info_init (&renderer->info); + gst_video_info_set_format (&renderer->info, + GST_VIDEO_FORMAT_RGBA, width, height); + + return renderer; +} + +void +gst_amc_2d_texture_renderer_free (GstAmc2DTextureRenderer * renderer) +{ + GError *error = NULL; + + if (renderer->surface_texture) { + _surface_texture_detach_from_gl_context_perform (renderer, &error); + g_object_unref (renderer->surface_texture); + } + + if (renderer->fbo || renderer->depth_buffer) { + gst_gl_context_del_fbo (renderer->context, renderer->fbo, + renderer->depth_buffer); + } + + if (renderer->shader) { + gst_object_unref (renderer->shader); + } + + if (renderer->oes_tex_id) { + gst_gl_context_del_texture (renderer->context, &renderer->oes_tex_id); + } + + if (renderer->context) { + gst_object_unref (renderer->context); + } + + g_free (renderer); +} + +static gboolean +_2d_texture_renderer_init_fbo (GstAmc2DTextureRenderer * renderer) +{ + const GstGLFuncs *gl; + GLuint fake_texture = 0; + guint out_width, out_height; + guint internal_format; + + out_width = GST_VIDEO_INFO_WIDTH (&renderer->info); + out_height = GST_VIDEO_INFO_HEIGHT (&renderer->info); + internal_format = + gst_gl_sized_gl_format_from_gl_format_type (renderer->context, GL_RGBA, + GL_UNSIGNED_BYTE); + + gl = renderer->context->gl_vtable; + + if (!gl->GenFramebuffers) { + /* turn off the pipeline because Frame buffer object is a not present */ + gst_gl_context_set_error (renderer->context, + "Context, EXT_framebuffer_object supported: no"); + return FALSE; + } + + GST_INFO ("Context, EXT_framebuffer_object supported: yes"); + + /* setup FBO */ + gl->GenFramebuffers (1, &renderer->fbo); + gl->BindFramebuffer (GL_FRAMEBUFFER, renderer->fbo); + + /* setup the render buffer for depth */ + gl->GenRenderbuffers (1, &renderer->depth_buffer); + gl->BindRenderbuffer (GL_RENDERBUFFER, renderer->depth_buffer); + gl->RenderbufferStorage (GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, + out_width, out_height); + + /* a fake texture is attached to the render FBO (cannot init without it) */ + gl->GenTextures (1, &fake_texture); + gl->BindTexture (GL_TEXTURE_2D, fake_texture); + gl->TexImage2D (GL_TEXTURE_2D, 0, internal_format, out_width, out_height, + 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL); + gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + /* attach the texture to the FBO to renderer to */ + gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, fake_texture, 0); + + /* attach the depth render buffer to the FBO */ + gl->FramebufferRenderbuffer (GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, + GL_RENDERBUFFER, renderer->depth_buffer); + + if (!gst_gl_context_check_framebuffer_status (renderer->context)) { + gst_gl_context_set_error (renderer->context, + "GL framebuffer status incomplete"); + return FALSE; + } + + /* unbind the FBO */ + gl->BindFramebuffer (GL_FRAMEBUFFER, 0); + gl->DeleteTextures (1, &fake_texture); + + return TRUE; +} + +static gboolean +_2d_texture_renderer_init (GstAmc2DTextureRenderer * renderer) +{ + GstGLFuncs *gl; + gboolean res; + + gl = renderer->context->gl_vtable; + + if (renderer->initialized) + return TRUE; + + if (!gl->CreateProgramObject && !gl->CreateProgram) { + gst_gl_context_set_error (renderer->context, + "Cannot perform conversion without OpenGL shaders"); + return FALSE; + } + + _gen_oes_texture (renderer->context, &renderer->oes_tex_id); + + res = + gst_gl_context_gen_shader (renderer->context, vert_COPY_OES, + frag_COPY_OES, &renderer->shader); + if (!res) + return FALSE; + + renderer->shader_attr_position_loc = + gst_gl_shader_get_attribute_location (renderer->shader, "a_position"); + renderer->shader_attr_texture_loc = + gst_gl_shader_get_attribute_location (renderer->shader, "a_texcoord"); + + gst_gl_shader_use (renderer->shader); + + gst_gl_shader_set_uniform_1i (renderer->shader, "u_tex", 0); + + gst_gl_context_clear_shader (renderer->context); + + if (!_2d_texture_renderer_init_fbo (renderer)) + return FALSE; + + gl->BindTexture (GL_TEXTURE_2D, 0); + + renderer->initialized = TRUE; + + return TRUE; +} + +static gboolean +_2d_texture_renderer_draw (GstAmc2DTextureRenderer * renderer) +{ + GstGLFuncs *gl; + guint out_width, out_height; + + GLint viewport_dim[4]; + + /* *INDENT-OFF* */ + const GLfloat vertices[] = { + 1.0f, -1.0f, 0.0f, 1.0f, 0.0f, + -1.0f, -1.0f, 0.0f, 0.0f, 0.0f, + -1.0f, 1.0f, 0.0f, 0.0f, 1.0f, + 1.0f, 1.0f, 0.0f, 1.0f, 1.0f + }; + /* *INDENT-ON* */ + + GLushort indices[] = { 0, 1, 2, 0, 2, 3 }; + + GLenum multipleRT[] = { + GL_COLOR_ATTACHMENT0, + }; + + gl = renderer->context->gl_vtable; + + out_width = GST_VIDEO_INFO_WIDTH (&renderer->info); + out_height = GST_VIDEO_INFO_HEIGHT (&renderer->info); + + gl->BindFramebuffer (GL_FRAMEBUFFER, renderer->fbo); + + /* attach the texture to the FBO to rendererer to */ + gl->BindTexture (GL_TEXTURE_2D, renderer->tex_id); + gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, renderer->tex_id, 0); + + if (gl->DrawBuffers) + gl->DrawBuffers (1, multipleRT); + else if (gl->DrawBuffer) + gl->DrawBuffer (GL_COLOR_ATTACHMENT0); + + gl->GetIntegerv (GL_VIEWPORT, viewport_dim); + + gl->Viewport (0, 0, out_width, out_height); + + gl->ClearColor (0.0, 0.0, 0.0, 0.0); + gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + + gst_gl_shader_use (renderer->shader); + gst_gl_shader_set_uniform_matrix_4fv (renderer->shader, "u_transformation", 1, + FALSE, renderer->transformation_matrix); + + gl->VertexAttribPointer (renderer->shader_attr_position_loc, 3, + GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), vertices); + gl->VertexAttribPointer (renderer->shader_attr_texture_loc, 2, + GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), &vertices[3]); + + gl->EnableVertexAttribArray (renderer->shader_attr_position_loc); + gl->EnableVertexAttribArray (renderer->shader_attr_texture_loc); + + gl->ActiveTexture (GL_TEXTURE0); + gl->BindTexture (GL_TEXTURE_EXTERNAL_OES, renderer->oes_tex_id); + + gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, + GL_CLAMP_TO_EDGE); + gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, + GL_CLAMP_TO_EDGE); + + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); + + gl->DisableVertexAttribArray (renderer->shader_attr_position_loc); + gl->DisableVertexAttribArray (renderer->shader_attr_texture_loc); + + if (gl->DrawBuffer) + gl->DrawBuffer (GL_NONE); + + /* we are done with the shader */ + gst_gl_context_clear_shader (renderer->context); + + gl->Viewport (viewport_dim[0], viewport_dim[1], viewport_dim[2], + viewport_dim[3]); + + gst_gl_context_check_framebuffer_status (renderer->context); + + gl->BindFramebuffer (GL_FRAMEBUFFER, 0); + + return TRUE; +} + +static void +_2d_texture_renderer_render (GstGLContext * context, + GstAmc2DTextureRenderer * renderer) +{ + gfloat identify_matrix[16] = { 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f + }; + + gfloat yflip_matrix[16] = { 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 1.0f + }; + + gfloat transformation_matrix[16] = { 0 }; + + if (!renderer->initialized) { + if (!_2d_texture_renderer_init (renderer)) { + renderer->result = FALSE; + return; + } + + if (!gst_amc_surface_texture_attach_to_gl_context + (renderer->surface_texture, renderer->oes_tex_id, + &renderer->gl_context_error)) { + renderer->result = FALSE; + return; + } + } + + if (!gst_amc_surface_texture_update_tex_image (renderer->surface_texture, + &renderer->gl_context_error)) { + renderer->result = FALSE; + return; + } + + if (gst_amc_surface_texture_get_transform_matrix (renderer->surface_texture, + transformation_matrix, &renderer->gl_context_error)) { + int i, j; + + for (i = 0; i < 16; i += 4) { + renderer->transformation_matrix[i + j] = 0.0f; + for (j = 0; j < 4; ++j) { + renderer->transformation_matrix[i + j] = + (transformation_matrix[i + 0] * yflip_matrix[j + 0]) + + (transformation_matrix[i + 1] * yflip_matrix[j + 4]) + + (transformation_matrix[i + 2] * yflip_matrix[j + 8]) + + (transformation_matrix[i + 3] * yflip_matrix[j + 12]); + } + } + } else { + memcpy (renderer->transformation_matrix, identify_matrix, + sizeof (identify_matrix[0] * 16)); + } + + if (!_2d_texture_renderer_draw (renderer)) { + renderer->result = FALSE; + return; + } + + renderer->result = TRUE; +} + +gboolean +gst_amc_2d_texture_renderer_render (GstAmc2DTextureRenderer * + renderer, guint tex_id, GError ** error) +{ + g_return_val_if_fail (renderer != NULL, FALSE); + g_return_val_if_fail (tex_id, FALSE); + + renderer->tex_id = tex_id; + + renderer->result = FALSE; + renderer->gl_context_error = NULL; + + gst_gl_context_thread_add (renderer->context, + (GstGLContextThreadFunc) _2d_texture_renderer_render, renderer); + + *error = renderer->gl_context_error; + renderer->gl_context_error = NULL; + + return renderer->result; +} diff --git a/sys/androidmedia/gstamc2dtexturerenderer.h b/sys/androidmedia/gstamc2dtexturerenderer.h new file mode 100644 index 0000000000..0e1d89453d --- /dev/null +++ b/sys/androidmedia/gstamc2dtexturerenderer.h @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2014, Collabora Ltd. + * Author: Matthieu Bouron + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifndef __GST_AMC_2D_TEXTURE_RENDER_H__ +#define __GST_AMC_2D_TEXTURE_RENDER_H__ + +#include +#include + +#include "gstamcsurfacetexture.h" + +G_BEGIN_DECLS + +typedef struct _GstAmc2DTextureRenderer GstAmc2DTextureRenderer; + +struct _GstAmc2DTextureRenderer { + + /* private */ + gboolean initialized; + + GstGLContext *context; + + GstVideoInfo info; + + GLuint fbo; + GLuint depth_buffer; + + GstGLShader *shader; + + GLint shader_attr_position_loc; + GLint shader_attr_texture_loc; + + GError *gl_context_error; + gboolean gl_context_result; + + GstAmcSurfaceTexture *surface_texture; + + guint tex_id; + guint oes_tex_id; + + gfloat transformation_matrix[16]; + + /* out fields */ + gboolean result; +}; + +GstAmc2DTextureRenderer * gst_amc_2d_texture_renderer_new (GstGLContext * context, + GstAmcSurfaceTexture *surface_texture, + guint width, + guint height); + +void gst_amc_2d_texture_renderer_free (GstAmc2DTextureRenderer * render); + +gboolean gst_amc_2d_texture_renderer_render (GstAmc2DTextureRenderer * render, + guint tex_id, + GError ** error); + +G_END_DECLS +#endif diff --git a/sys/androidmedia/gstamcaudiodec.c b/sys/androidmedia/gstamcaudiodec.c index bc8190740f..ce86a96862 100644 --- a/sys/androidmedia/gstamcaudiodec.c +++ b/sys/androidmedia/gstamcaudiodec.c @@ -573,7 +573,7 @@ retry: } } - if (!gst_amc_codec_release_output_buffer (self->codec, idx, &err)) { + if (!gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err)) { if (self->flushing) { g_clear_error (&err); goto flushing; @@ -703,7 +703,7 @@ invalid_buffer_size: { GST_ELEMENT_ERROR (self, LIBRARY, FAILED, (NULL), ("Invalid buffer size %u (bfp %d)", buffer_info.size, self->info.bpf)); - gst_amc_codec_release_output_buffer (self->codec, idx, &err); + gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err); if (err && !self->flushing) GST_ELEMENT_WARNING_FROM_ERROR (self, err); g_clear_error (&err); @@ -722,7 +722,7 @@ failed_allocate: { GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL), ("Failed to allocate output buffer")); - gst_amc_codec_release_output_buffer (self->codec, idx, &err); + gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err); if (err && !self->flushing) GST_ELEMENT_WARNING_FROM_ERROR (self, err); g_clear_error (&err); @@ -927,7 +927,7 @@ gst_amc_audio_dec_set_format (GstAudioDecoder * decoder, GstCaps * caps) GST_STR_NULL (format_string)); g_free (format_string); - if (!gst_amc_codec_configure (self->codec, format, 0, &err)) { + if (!gst_amc_codec_configure (self->codec, format, NULL, 0, &err)) { GST_ERROR_OBJECT (self, "Failed to configure codec"); GST_ELEMENT_ERROR_FROM_ERROR (self, err); return FALSE; diff --git a/sys/androidmedia/gstamcsurface.c b/sys/androidmedia/gstamcsurface.c new file mode 100644 index 0000000000..33dc6a7ce7 --- /dev/null +++ b/sys/androidmedia/gstamcsurface.c @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2013, Fluendo S.A. + * Author: Andoni Morales + * + * Copyright (C) 2015, Collabora Ltd. + * Author: Matthieu Bouron + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstjniutils.h" +#include "gstamcsurface.h" + +G_DEFINE_TYPE (GstAmcSurface, gst_amc_surface, G_TYPE_OBJECT); + +static gpointer parent_class = NULL; +static void gst_amc_surface_dispose (GObject * object); + +static gboolean +_cache_java_class (GstAmcSurfaceClass * klass, GError ** err) +{ + JNIEnv *env; + + gst_amc_jni_initialize (); + + env = gst_amc_jni_get_env (); + + klass->jklass = gst_amc_jni_get_class (env, err, "android/view/Surface"); + if (!klass->jklass) { + return FALSE; + } + + klass->constructor = + gst_amc_jni_get_method_id (env, err, klass->jklass, "", + "(Landroid/graphics/SurfaceTexture;)V"); + if (!klass->constructor) { + goto error; + } + + klass->is_valid = + gst_amc_jni_get_method_id (env, err, klass->jklass, "isValid", "()Z"); + if (!klass->is_valid) { + goto error; + } + + klass->release = + gst_amc_jni_get_method_id (env, err, klass->jklass, "release", "()V"); + if (!klass->release) { + goto error; + } + + klass->describe_contents = + gst_amc_jni_get_method_id (env, err, klass->jklass, "describeContents", + "()I"); + if (!klass->describe_contents) { + goto error; + } + + return TRUE; + +error: + gst_amc_jni_object_unref (env, klass->jklass); + return FALSE; +} + +static void +gst_amc_surface_class_init (GstAmcSurfaceClass * klass) +{ + GError *err = NULL; + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + parent_class = g_type_class_peek_parent (klass); + gobject_class->dispose = gst_amc_surface_dispose; + + if (!_cache_java_class (klass, &err)) { + g_log (G_LOG_DOMAIN, G_LOG_LEVEL_CRITICAL, + "Could not cache java class android/view/Surface: %s", err->message); + g_clear_error (&err); + } +} + +static void +gst_amc_surface_init (GstAmcSurface * self) +{ + /* initialize the object */ +} + +static void +gst_amc_surface_dispose (GObject * object) +{ + GstAmcSurface *self; + JNIEnv *env; + GError *err = NULL; + + self = GST_AMC_SURFACE (object); + env = gst_amc_jni_get_env (); + + if (!gst_amc_surface_release (self, &err)) { + if (err) { + GST_ERROR ("Error: %s", err->message); + } + } + + if (self->jobject) { + gst_amc_jni_object_unref (env, self->jobject); + } + + if (self->texture != NULL) { + g_object_unref (self->texture); + self->texture = NULL; + } + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +GstAmcSurface * +gst_amc_surface_new (GstAmcSurfaceTexture * texture, GError ** err) +{ + GstAmcSurface *surface; + GstAmcSurfaceClass *klass; + JNIEnv *env; + + surface = g_object_new (GST_TYPE_AMC_SURFACE, NULL); + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_GET_CLASS (surface); + + surface->jobject = gst_amc_jni_new_object (env, err, TRUE, klass->jklass, + klass->constructor, texture->jobject); + if (surface->jobject == NULL) { + g_object_unref (surface); + return NULL; + } + + surface->texture = g_object_ref (texture); + return surface; +} + +gboolean +gst_amc_surface_is_valid (GstAmcSurface * self, gboolean * result, + GError ** err) +{ + JNIEnv *env; + GstAmcSurfaceClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_GET_CLASS (self); + + return gst_amc_jni_call_boolean_method (env, err, self->jobject, + klass->is_valid, result); +} + +gboolean +gst_amc_surface_release (GstAmcSurface * self, GError ** err) +{ + JNIEnv *env; + GstAmcSurfaceClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_GET_CLASS (self); + + return gst_amc_jni_call_void_method (env, err, self->jobject, klass->release); +} + +gboolean +gst_amc_surface_describe_contents (GstAmcSurface * self, gint * result, + GError ** err) +{ + JNIEnv *env; + GstAmcSurfaceClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_GET_CLASS (self); + + return gst_amc_jni_call_int_method (env, err, self->jobject, + klass->describe_contents, result); +} diff --git a/sys/androidmedia/gstamcsurface.h b/sys/androidmedia/gstamcsurface.h new file mode 100644 index 0000000000..21f9969b25 --- /dev/null +++ b/sys/androidmedia/gstamcsurface.h @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2013, Fluendo S.A. + * Author: Andoni Morales + * + * Copyright (C) 2015, Collabora Ltd. + * Author: Matthieu Bouron + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifndef __GST_AMC_SURFACE_H__ +#define __GST_AMC_SURFACE_H__ + +#include +#include +#include "gstamcsurfacetexture.h" + +G_BEGIN_DECLS + +#define GST_TYPE_AMC_SURFACE (gst_amc_surface_get_type ()) +#define GST_AMC_SURFACE(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_AMC_SURFACE, GstAmcSurface)) +#define GST_IS_AMC_SURFACE(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_AMC_SURFACE)) +#define GST_AMC_SURFACE_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_AMC_SURFACE, GstAmcSurfaceClass)) +#define GST_IS_AMC_SURFACE_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_AMC_SURFACE)) +#define GST_AMC_SURFACE_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_AMC_SURFACE, GstAmcSurfaceClass)) + +typedef struct _GstAmcSurface GstAmcSurface; +typedef struct _GstAmcSurfaceClass GstAmcSurfaceClass; + +struct _GstAmcSurface +{ + GObject parent_instance; + + /* instance members */ + jobject jobject; + GstAmcSurfaceTexture *texture; +}; + +struct _GstAmcSurfaceClass +{ + GObjectClass parent_class; + + /* class members */ + jclass jklass; + jmethodID constructor; + jmethodID is_valid; + jmethodID release; + jmethodID describe_contents; +}; + +GType gst_amc_surface_get_type (void); + +GstAmcSurface * gst_amc_surface_new (GstAmcSurfaceTexture *texture, + GError ** err); + +gboolean gst_amc_surface_is_valid (GstAmcSurface *surface, + gboolean * result, + GError ** err); + +gboolean gst_amc_surface_release (GstAmcSurface *surface, + GError ** err); + +gboolean gst_amc_surface_describe_contents (GstAmcSurface *surface, + gint * result, + GError ** err); + +G_END_DECLS +#endif diff --git a/sys/androidmedia/gstamcsurfacetexture.c b/sys/androidmedia/gstamcsurfacetexture.c new file mode 100644 index 0000000000..43c3142a96 --- /dev/null +++ b/sys/androidmedia/gstamcsurfacetexture.c @@ -0,0 +1,318 @@ +/* + * Copyright (C) 2013, Fluendo S.A. + * Author: Andoni Morales + * + * Copyright (C) 2014, Collabora Ltd. + * Author: Matthieu Bouron + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstjniutils.h" +#include "gstamcsurfacetexture.h" + +G_DEFINE_TYPE (GstAmcSurfaceTexture, gst_amc_surface_texture, G_TYPE_OBJECT); + +static gpointer parent_class = NULL; +static void gst_amc_surface_texture_dispose (GObject * object); + +static gboolean +_cache_java_class (GstAmcSurfaceTextureClass * klass, GError ** err) +{ + JNIEnv *env; + + gst_amc_jni_initialize (); + env = gst_amc_jni_get_env (); + + klass->jklass = + gst_amc_jni_get_class (env, err, "android/graphics/SurfaceTexture"); + if (!klass->jklass) { + return FALSE; + } + + klass->constructor = + gst_amc_jni_get_method_id (env, err, klass->jklass, "", "(I)V"); + if (!klass->constructor) { + goto error; + } + + klass->set_on_frame_available_listener = + gst_amc_jni_get_method_id (env, err, klass->jklass, + "setOnFrameAvailableListener", + "(Landroid/graphics/SurfaceTexture$OnFrameAvailableListener;)V"); + + klass->set_default_buffer_size = + gst_amc_jni_get_method_id (env, err, klass->jklass, + "setDefaultBufferSize", "(II)V"); + if (!klass->set_default_buffer_size) { + goto error; + } + + klass->update_tex_image = + gst_amc_jni_get_method_id (env, err, klass->jklass, "updateTexImage", + "()V"); + if (!klass->update_tex_image) { + goto error; + } + + klass->detach_from_gl_context = + gst_amc_jni_get_method_id (env, err, klass->jklass, "detachFromGLContext", + "()V"); + if (!klass->detach_from_gl_context) { + goto error; + } + + klass->attach_to_gl_context = + gst_amc_jni_get_method_id (env, err, klass->jklass, "attachToGLContext", + "(I)V"); + if (!klass->attach_to_gl_context) { + goto error; + } + + klass->get_transform_matrix = + gst_amc_jni_get_method_id (env, err, klass->jklass, "getTransformMatrix", + "([F)V"); + if (!klass->get_transform_matrix) { + goto error; + } + + klass->get_timestamp = + gst_amc_jni_get_method_id (env, err, klass->jklass, "getTimestamp", + "()J"); + if (!klass->get_timestamp) { + goto error; + } + + klass->release = + gst_amc_jni_get_method_id (env, err, klass->jklass, "release", "()V"); + if (!klass->release) { + goto error; + } + + return TRUE; + +error: + gst_amc_jni_object_unref (env, klass->constructor); + return FALSE; +} + +static void +gst_amc_surface_texture_init (GstAmcSurfaceTexture * self) +{ +} + +static void +gst_amc_surface_texture_class_init (GstAmcSurfaceTextureClass * klass) +{ + GError *err = NULL; + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + parent_class = g_type_class_peek_parent (klass); + gobject_class->dispose = gst_amc_surface_texture_dispose; + + if (!_cache_java_class (klass, &err)) { + g_log (G_LOG_DOMAIN, G_LOG_LEVEL_CRITICAL, + "Could not cache java class android/graphics/SurfaceTexture: %s", + err->message); + g_clear_error (&err); + } +} + +static void +gst_amc_surface_texture_dispose (GObject * object) +{ + GstAmcSurfaceTexture *self; + JNIEnv *env; + GError *err = NULL; + + self = GST_AMC_SURFACE_TEXTURE (object); + env = gst_amc_jni_get_env (); + + if (!gst_amc_surface_texture_release (self, &err)) { + GST_ERROR ("Could not release surface texture: %s", err->message); + g_clear_error (&err); + } + + if (self->jobject) { + gst_amc_jni_object_unref (env, self->jobject); + } + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +GstAmcSurfaceTexture * +gst_amc_surface_texture_new (GError ** err) +{ + GstAmcSurfaceTexture *texture = NULL; + GstAmcSurfaceTextureClass *klass; + JNIEnv *env; + + texture = g_object_new (GST_TYPE_AMC_SURFACE_TEXTURE, NULL); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (texture); + env = gst_amc_jni_get_env (); + + texture->texture_id = 0; + + texture->jobject = gst_amc_jni_new_object (env, err, TRUE, klass->jklass, + klass->constructor, texture->texture_id); + if (texture->jobject == NULL) { + goto error; + } + + if (!gst_amc_surface_texture_detach_from_gl_context (texture, err)) { + goto error; + } + + return texture; + +error: + if (texture) + g_object_unref (texture); + return NULL; +} + +gboolean +gst_amc_surface_texture_set_default_buffer_size (GstAmcSurfaceTexture * self, + gint width, gint height, GError ** err) +{ + JNIEnv *env; + GstAmcSurfaceTextureClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (self); + + return gst_amc_jni_call_void_method (env, err, self->jobject, + klass->set_default_buffer_size, width, height); +} + +gboolean +gst_amc_surface_texture_update_tex_image (GstAmcSurfaceTexture * self, + GError ** err) +{ + JNIEnv *env; + GstAmcSurfaceTextureClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (self); + + return gst_amc_jni_call_void_method (env, err, self->jobject, + klass->update_tex_image); +} + +gboolean +gst_amc_surface_texture_detach_from_gl_context (GstAmcSurfaceTexture * self, + GError ** err) +{ + JNIEnv *env; + gboolean ret; + GstAmcSurfaceTextureClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (self); + + ret = + gst_amc_jni_call_void_method (env, err, self->jobject, + klass->detach_from_gl_context); + self->texture_id = 0; + return ret; +} + +gboolean +gst_amc_surface_texture_attach_to_gl_context (GstAmcSurfaceTexture * self, + gint texture_id, GError ** err) +{ + JNIEnv *env; + gboolean ret; + GstAmcSurfaceTextureClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (self); + + ret = + gst_amc_jni_call_void_method (env, err, self->jobject, + klass->attach_to_gl_context, texture_id); + self->texture_id = texture_id; + return ret; +} + +gboolean +gst_amc_surface_texture_get_transform_matrix (GstAmcSurfaceTexture * self, + const gfloat * matrix, GError ** err) +{ + JNIEnv *env; + gboolean ret; + GstAmcSurfaceTextureClass *klass; + /* 4x4 Matrix */ + jsize size = 16; + jfloatArray floatarray; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (self); + + floatarray = (*env)->NewFloatArray (env, size); + ret = + gst_amc_jni_call_void_method (env, err, self->jobject, + klass->get_transform_matrix, floatarray); + if (ret) { + (*env)->GetFloatArrayRegion (env, floatarray, 0, size, (jfloat *) matrix); + (*env)->DeleteLocalRef (env, floatarray); + } + + return ret; +} + +gboolean +gst_amc_surface_texture_get_timestamp (GstAmcSurfaceTexture * self, + gint64 * result, GError ** err) +{ + JNIEnv *env; + GstAmcSurfaceTextureClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (self); + + return gst_amc_jni_call_long_method (env, err, self->jobject, + klass->get_timestamp, result); +} + +gboolean +gst_amc_surface_texture_release (GstAmcSurfaceTexture * self, GError ** err) +{ + JNIEnv *env; + GstAmcSurfaceTextureClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (self); + + return gst_amc_jni_call_void_method (env, err, self->jobject, klass->release); +} + +gboolean +gst_amc_surface_texture_set_on_frame_available_listener (GstAmcSurfaceTexture * + self, jobject listener, GError ** err) +{ + JNIEnv *env; + GstAmcSurfaceTextureClass *klass; + + env = gst_amc_jni_get_env (); + klass = GST_AMC_SURFACE_TEXTURE_GET_CLASS (self); + + return gst_amc_jni_call_void_method (env, err, self->jobject, + klass->set_on_frame_available_listener, listener); +} diff --git a/sys/androidmedia/gstamcsurfacetexture.h b/sys/androidmedia/gstamcsurfacetexture.h new file mode 100644 index 0000000000..0d6ebf14eb --- /dev/null +++ b/sys/androidmedia/gstamcsurfacetexture.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2013, Fluendo S.A. + * Author: Andoni Morales + * + * Copyright (C) 2014, Collabora Ltd. + * Author: Matthieu Bouron + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifndef __GST_AMC_SURFACE_TEXTURE_H__ +#define __GST_AMC_SURFACE_TEXTURE_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_AMC_SURFACE_TEXTURE (gst_amc_surface_texture_get_type ()) +#define GST_AMC_SURFACE_TEXTURE(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_AMC_SURFACE_TEXTURE, GstAmcSurfaceTexture)) +#define GST_IS_AMC_SURFACE_TEXTURE(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_AMC_SURFACE_TEXTURE)) +#define GST_AMC_SURFACE_TEXTURE_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_AMC_SURFACE_TEXTURE, GstAmcSurfaceTextureClass)) +#define GST_IS_AMC_SURFACE_TEXTURE_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_AMC_SURFACE_TEXTURE)) +#define GST_AMC_SURFACE_TEXTURE_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_AMC_SURFACE_TEXTURE, GstAmcSurfaceTextureClass)) + +typedef struct _GstAmcSurfaceTexture GstAmcSurfaceTexture; +typedef struct _GstAmcSurfaceTextureClass GstAmcSurfaceTextureClass; + +struct _GstAmcSurfaceTexture +{ + GObject parent_instance; + + /* instance members */ + gint texture_id; + jobject jobject; +}; + +struct _GstAmcSurfaceTextureClass +{ + GObjectClass parent_class; + + /* class members */ + gint texture_id; + + jclass jklass; + jmethodID constructor; + jmethodID set_on_frame_available_listener; + jmethodID set_default_buffer_size; + jmethodID update_tex_image; + jmethodID detach_from_gl_context; + jmethodID attach_to_gl_context; + jmethodID get_transform_matrix; + jmethodID get_timestamp; + jmethodID release; +}; + +GType gst_amc_surface_texture_get_type (void); + +GstAmcSurfaceTexture * gst_amc_surface_texture_new (GError ** err); + +gboolean gst_amc_surface_texture_set_default_buffer_size (GstAmcSurfaceTexture *texture, + gint width, + gint height, + GError ** err); + +gboolean gst_amc_surface_texture_update_tex_image (GstAmcSurfaceTexture *texture, + GError ** err); + +gboolean gst_amc_surface_texture_detach_from_gl_context (GstAmcSurfaceTexture *texture, + GError ** err); + +gboolean gst_amc_surface_texture_attach_to_gl_context (GstAmcSurfaceTexture *texture, + gint index, + GError ** err); + +gboolean gst_amc_surface_texture_get_transform_matrix (GstAmcSurfaceTexture *texture, + const gfloat *matrix, + GError ** err); + +gboolean gst_amc_surface_texture_get_timestamp (GstAmcSurfaceTexture *texture, + gint64 * result, + GError ** err); + +gboolean gst_amc_surface_texture_release (GstAmcSurfaceTexture *texture, + GError ** err); + +gboolean gst_amc_surface_texture_set_on_frame_available_listener (GstAmcSurfaceTexture * self, + jobject listener, + GError ** err); + +G_END_DECLS +#endif diff --git a/sys/androidmedia/gstamcvideodec.c b/sys/androidmedia/gstamcvideodec.c index 4049c488f4..70371ddbba 100644 --- a/sys/androidmedia/gstamcvideodec.c +++ b/sys/androidmedia/gstamcvideodec.c @@ -11,6 +11,12 @@ * * Copyright (C) 2015, Sebastian Dröge * + * Copyright (C) 2014-2015, Collabora Ltd. + * Author: Matthieu Bouron + * + * Copyright (C) 2015, Edward Hervey + * Author: Edward Hervey + * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation @@ -32,6 +38,7 @@ #endif #include +#include #include #include #include @@ -44,6 +51,7 @@ #include "gstamcvideodec.h" #include "gstamc-constants.h" +#include "gstamc2dtexturerenderer.h" GST_DEBUG_CATEGORY_STATIC (gst_amc_video_dec_debug_category); #define GST_CAT_DEFAULT gst_amc_video_dec_debug_category @@ -58,6 +66,17 @@ GST_DEBUG_CATEGORY_STATIC (gst_amc_video_dec_debug_category); g_clear_error (&err); \ } G_STMT_END +/* Assume the device is able to decode at 30fps by default */ +#define GST_AMC_VIDEO_DEC_ON_FRAME_AVAILABLE_DEFAULT_TIMEOUT (33 * G_TIME_SPAN_MILLISECOND) + +#if GLIB_SIZEOF_VOID_P == 8 +#define JLONG_TO_GST_AMC_VIDEO_DEC(value) (GstAmcVideoDec *)(value) +#define GST_AMC_VIDEO_DEC_TO_JLONG(value) (jlong)(value) +#else +#define JLONG_TO_GST_AMC_VIDEO_DEC(value) (GstAmcVideoDec *)(jint)(value) +#define GST_AMC_VIDEO_DEC_TO_JLONG(value) (jlong)(jint)(value) +#endif + typedef struct _BufferIdentification BufferIdentification; struct _BufferIdentification { @@ -101,6 +120,10 @@ static gboolean gst_amc_video_dec_decide_allocation (GstVideoDecoder * bdec, GstQuery * query); static GstFlowReturn gst_amc_video_dec_drain (GstAmcVideoDec * self); +static gboolean gst_amc_video_dec_check_codec_config (GstAmcVideoDec * self); +static void +gst_amc_video_dec_on_frame_available (JNIEnv * env, jobject thiz, + long long context, jobject surfaceTexture); enum { @@ -190,7 +213,7 @@ gst_amc_video_dec_base_init (gpointer g_class) GstAmcVideoDecClass *amcvideodec_class = GST_AMC_VIDEO_DEC_CLASS (g_class); const GstAmcCodecInfo *codec_info; GstPadTemplate *templ; - GstCaps *sink_caps, *src_caps; + GstCaps *sink_caps, *src_caps, *all_src_caps; gchar *longname; codec_info = @@ -202,15 +225,23 @@ gst_amc_video_dec_base_init (gpointer g_class) amcvideodec_class->codec_info = codec_info; gst_amc_codec_info_to_caps (codec_info, &sink_caps, &src_caps); + + all_src_caps = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA")); + + gst_caps_append (all_src_caps, src_caps); + /* Add pad templates */ templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, sink_caps); gst_element_class_add_pad_template (element_class, templ); gst_caps_unref (sink_caps); - templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps); + templ = + gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, all_src_caps); gst_element_class_add_pad_template (element_class, templ); - gst_caps_unref (src_caps); + gst_caps_unref (all_src_caps); longname = g_strdup_printf ("Android MediaCodec %s", codec_info->name); gst_element_class_set_metadata (element_class, @@ -255,6 +286,10 @@ gst_amc_video_dec_init (GstAmcVideoDec * self) g_mutex_init (&self->drain_lock); g_cond_init (&self->drain_cond); + + g_mutex_init (&self->on_frame_available_lock); + g_cond_init (&self->on_frame_available_cond); + self->on_frame_available = FALSE; } static gboolean @@ -271,6 +306,8 @@ gst_amc_video_dec_open (GstVideoDecoder * decoder) GST_ELEMENT_ERROR_FROM_ERROR (self, err); return FALSE; } + self->codec_config = AMC_CODEC_CONFIG_NONE; + self->started = FALSE; self->flushing = TRUE; @@ -296,9 +333,16 @@ gst_amc_video_dec_close (GstVideoDecoder * decoder) gst_amc_codec_free (self->codec); } self->codec = NULL; + self->codec_config = AMC_CODEC_CONFIG_NONE; + + if (self->surface) { + gst_object_unref (self->surface); + self->surface = NULL; + } self->started = FALSE; self->flushing = TRUE; + self->downstream_supports_gl = FALSE; GST_DEBUG_OBJECT (self, "Closed decoder"); @@ -327,6 +371,10 @@ gst_amc_video_dec_change_state (GstElement * element, GstStateChange transition) GST_STATE_CHANGE_FAILURE); self = GST_AMC_VIDEO_DEC (element); + GST_DEBUG_OBJECT (element, "changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + switch (transition) { case GST_STATE_CHANGE_NULL_TO_READY: break; @@ -367,8 +415,20 @@ gst_amc_video_dec_change_state (GstElement * element, GstStateChange transition) case GST_STATE_CHANGE_PAUSED_TO_READY: self->downstream_flow_ret = GST_FLOW_FLUSHING; self->started = FALSE; - break; - case GST_STATE_CHANGE_READY_TO_NULL: + + GST_DEBUG_OBJECT (element, "Freeing GL context: %" GST_PTR_FORMAT, + self->gl_context); + if (self->gl_context) { + gst_object_unref (self->gl_context); + self->gl_context = NULL; + } + + GST_DEBUG_OBJECT (element, "Freeing GL renderer: %p", self->renderer); + if (self->renderer) { + gst_amc_2d_texture_renderer_free (self->renderer); + self->renderer = NULL; + } + break; default: break; @@ -465,11 +525,31 @@ _find_nearest_frame (GstAmcVideoDec * self, GstClockTime reference_timestamp) return best; } +static gboolean +gst_amc_video_dec_check_codec_config (GstAmcVideoDec * self) +{ + gboolean ret = (self->codec_config == AMC_CODEC_CONFIG_NONE + || (self->codec_config == AMC_CODEC_CONFIG_WITH_SURFACE + && self->downstream_supports_gl) + || (self->codec_config == AMC_CODEC_CONFIG_WITHOUT_SURFACE + && !self->downstream_supports_gl)); + + if (!ret) { + GST_ERROR_OBJECT + (self, + "Codec configuration (%d) is not compatible with downstream which %s support GL output", + self->codec_config, self->downstream_supports_gl ? "does" : "does not"); + } + + return ret; +} + static gboolean gst_amc_video_dec_set_src_caps (GstAmcVideoDec * self, GstAmcFormat * format) { GstVideoCodecState *output_state; const gchar *mime; + gint ret; gint color_format, width, height; gint stride, slice_height; gint crop_left, crop_right; @@ -530,6 +610,10 @@ gst_amc_video_dec_set_src_caps (GstAmcVideoDec * self, GstAmcFormat * format) gst_amc_color_format_to_video_format (klass->codec_info, mime, color_format); + if (self->codec_config == AMC_CODEC_CONFIG_WITH_SURFACE) { + gst_format = GST_VIDEO_FORMAT_RGBA; + } + if (gst_format == GST_VIDEO_FORMAT_UNKNOWN) { GST_ERROR_OBJECT (self, "Unknown color format 0x%08x", color_format); return FALSE; @@ -544,7 +628,17 @@ gst_amc_video_dec_set_src_caps (GstAmcVideoDec * self, GstAmcFormat * format) GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM, GST_VIDEO_MULTIVIEW_FLAGS_NONE); } + if (self->codec_config == AMC_CODEC_CONFIG_WITH_SURFACE) { + if (output_state->caps) + gst_caps_unref (output_state->caps); + output_state->caps = gst_video_info_to_caps (&output_state->info); + gst_caps_set_features (output_state->caps, 0, + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL)); + } + self->format = gst_format; + self->width = width; + self->height = height; if (!gst_amc_color_format_info_set (&self->color_format_info, klass->codec_info, mime, color_format, width, height, stride, slice_height, crop_left, crop_right, crop_top, crop_bottom)) { @@ -564,6 +658,7 @@ gst_amc_video_dec_set_src_caps (GstAmcVideoDec * self, GstAmcFormat * format) self->color_format_info.crop_bottom, self->color_format_info.frame_size); ret = gst_video_decoder_negotiate (GST_VIDEO_DECODER (self)); + gst_video_codec_state_unref (output_state); self->input_state_changed = FALSE; @@ -598,6 +693,7 @@ gst_amc_video_dec_loop (GstAmcVideoDec * self) GstAmcBufferInfo buffer_info; gint idx; GError *err = NULL; + gboolean release_buffer = TRUE; GST_VIDEO_DECODER_STREAM_LOCK (self); @@ -615,6 +711,8 @@ retry: GST_VIDEO_DECODER_STREAM_LOCK (self); /*} */ + GST_DEBUG_OBJECT (self, "dequeueOutputBuffer() returned %d (0x%x)", idx, idx); + if (idx < 0) { if (self->flushing) { g_clear_error (&err); @@ -689,7 +787,104 @@ retry: "Frame is too late, dropping (deadline %" GST_TIME_FORMAT ")", GST_TIME_ARGS (-deadline)); flow_ret = gst_video_decoder_drop_frame (GST_VIDEO_DECODER (self), frame); - } else if (!frame && buffer_info.size > 0) { + } else if (self->codec_config == AMC_CODEC_CONFIG_WITH_SURFACE + && buffer_info.size > 0) { + GstMemory *mem; + GstBuffer *outbuf; + gint64 timeout; + gint64 end_time; + GstVideoCodecState *state; + + outbuf = + gst_video_decoder_allocate_output_buffer (GST_VIDEO_DECODER (self)); + + state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (self)); + if (state && state->info.fps_n > 0 && state->info.fps_d > 0) { + timeout = + gst_util_uint64_scale_int (G_TIME_SPAN_SECOND, state->info.fps_d, + state->info.fps_n); + } else { + timeout = GST_AMC_VIDEO_DEC_ON_FRAME_AVAILABLE_DEFAULT_TIMEOUT; + } + gst_video_codec_state_unref (state); + + mem = gst_buffer_peek_memory (outbuf, 0); + if (gst_is_gl_memory (mem)) { + GstGLMemory *gl_mem = (GstGLMemory *) mem; + + if (!self->renderer) { + self->renderer = + gst_amc_2d_texture_renderer_new (self->gl_context, + self->surface->texture, self->width, self->height); + } + + release_buffer = FALSE; + self->on_frame_available = FALSE; + g_mutex_lock (&self->on_frame_available_lock); + + /* Render the frame into the surface */ + if (!gst_amc_codec_release_output_buffer (self->codec, idx, TRUE, &err)) { + gst_buffer_unref (outbuf); + GST_ERROR_OBJECT (self, "Failed to render buffer, index %d", idx); + GST_ELEMENT_ERROR_FROM_ERROR (self, err); + + goto gl_output_error; + } + + /* Wait for the frame to become available */ + end_time = g_get_monotonic_time () + timeout; + g_cond_wait_until (&self->on_frame_available_cond, + &self->on_frame_available_lock, end_time); + + g_mutex_unlock (&self->on_frame_available_lock); + + /* Now that the frame is available, we can render it to a 2D texture. + * + * Calling updateTexImage seems necessary even if no frame is available + * otherwise it could happen that the onFrameAvailable callback is not + * executed anymore. */ + if (!gst_amc_2d_texture_renderer_render (self->renderer, + gl_mem->tex_id, &err)) { + gst_buffer_unref (outbuf); + GST_ERROR_OBJECT (self, "Failed to render to a 2D texture"); + GST_ELEMENT_ERROR_FROM_ERROR (self, err); + + goto gl_output_error; + } + + } else { + GST_ERROR_OBJECT (self, "Wrong memory type for GL output mode"); + goto format_error; + } + + if (self->on_frame_available) { + if (frame) { + frame->output_buffer = outbuf; + flow_ret = + gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame); + } else { + /* This sometimes happens at EOS or if the input is not properly framed, + * let's handle it gracefully by allocating a new buffer for the current + * caps and filling it + */ + GST_BUFFER_PTS (outbuf) = + gst_util_uint64_scale (buffer_info.presentation_time_us, + GST_USECOND, 1); + + flow_ret = gst_pad_push (GST_VIDEO_DECODER_SRC_PAD (self), outbuf); + } + } else { + GST_WARNING_OBJECT (self, "No frame available after %lldms", + timeout / G_TIME_SPAN_MILLISECOND); + + if (frame) { + flow_ret = + gst_video_decoder_drop_frame (GST_VIDEO_DECODER (self), frame); + } + gst_buffer_unref (outbuf); + } + } else if (self->codec_config == AMC_CODEC_CONFIG_WITHOUT_SURFACE && !frame + && buffer_info.size > 0) { GstBuffer *outbuf; /* This sometimes happens at EOS or if the input is not properly framed, @@ -703,7 +898,7 @@ retry: if (!gst_amc_video_dec_fill_buffer (self, buf, &buffer_info, outbuf)) { gst_buffer_unref (outbuf); - if (!gst_amc_codec_release_output_buffer (self->codec, idx, &err)) + if (!gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err)) GST_ERROR_OBJECT (self, "Failed to release output buffer index %d", idx); if (err && !self->flushing) @@ -718,11 +913,13 @@ retry: gst_util_uint64_scale (buffer_info.presentation_time_us, GST_USECOND, 1); flow_ret = gst_pad_push (GST_VIDEO_DECODER_SRC_PAD (self), outbuf); - } else if (buffer_info.size > 0) { - if ((flow_ret = gst_video_decoder_allocate_output_frame (GST_VIDEO_DECODER - (self), frame)) != GST_FLOW_OK) { + } else if (self->codec_config == AMC_CODEC_CONFIG_WITHOUT_SURFACE && frame + && buffer_info.size > 0) { + if ((flow_ret = + gst_video_decoder_allocate_output_frame (GST_VIDEO_DECODER (self), + frame)) != GST_FLOW_OK) { GST_ERROR_OBJECT (self, "Failed to allocate buffer"); - if (!gst_amc_codec_release_output_buffer (self->codec, idx, &err)) + if (!gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err)) GST_ERROR_OBJECT (self, "Failed to release output buffer index %d", idx); if (err && !self->flushing) @@ -737,7 +934,7 @@ retry: frame->output_buffer)) { gst_buffer_replace (&frame->output_buffer, NULL); gst_video_decoder_drop_frame (GST_VIDEO_DECODER (self), frame); - if (!gst_amc_codec_release_output_buffer (self->codec, idx, &err)) + if (!gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err)) GST_ERROR_OBJECT (self, "Failed to release output buffer index %d", idx); if (err && !self->flushing) @@ -756,12 +953,14 @@ retry: gst_amc_buffer_free (buf); buf = NULL; - if (!gst_amc_codec_release_output_buffer (self->codec, idx, &err)) { - if (self->flushing) { - g_clear_error (&err); - goto flushing; + if (release_buffer) { + if (!gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err)) { + if (self->flushing) { + g_clear_error (&err); + goto flushing; + } + goto failed_release; } - goto failed_release; } if (is_eos || flow_ret == GST_FLOW_EOS) { @@ -897,6 +1096,18 @@ invalid_buffer: g_mutex_unlock (&self->drain_lock); return; } +gl_output_error: + { + gst_pad_push_event (GST_VIDEO_DECODER_SRC_PAD (self), gst_event_new_eos ()); + gst_pad_pause_task (GST_VIDEO_DECODER_SRC_PAD (self)); + self->downstream_flow_ret = GST_FLOW_NOT_NEGOTIATED; + GST_VIDEO_DECODER_STREAM_UNLOCK (self); + g_mutex_lock (&self->drain_lock); + self->draining = FALSE; + g_cond_broadcast (&self->drain_cond); + g_mutex_unlock (&self->drain_lock); + return; + } } static gboolean @@ -949,6 +1160,65 @@ gst_amc_video_dec_stop (GstVideoDecoder * decoder) return TRUE; } +static jobject +gst_amc_video_dec_new_on_frame_available_listener (GstAmcVideoDec * decoder, + JNIEnv * env, GError ** err) +{ + jobject listener = NULL; + jclass listener_cls = NULL; + jmethodID constructor_id = 0; + jmethodID set_context_id = 0; + + JNINativeMethod amcOnFrameAvailableListener = { + "native_onFrameAvailable", + "(JLandroid/graphics/SurfaceTexture;)V", + (void *) gst_amc_video_dec_on_frame_available, + }; + + listener_cls = + gst_amc_jni_get_application_class (env, + "org/freedesktop/gstreamer/androidmedia/GstAmcOnFrameAvailableListener", + err); + if (!listener_cls) { + return FALSE; + } + + (*env)->RegisterNatives (env, listener_cls, &amcOnFrameAvailableListener, 1); + if ((*env)->ExceptionCheck (env)) { + (*env)->ExceptionClear (env); + goto done; + } + + constructor_id = + gst_amc_jni_get_method_id (env, err, listener_cls, "", "()V"); + if (!constructor_id) { + goto done; + } + + set_context_id = + gst_amc_jni_get_method_id (env, err, listener_cls, "setContext", "(J)V"); + if (!set_context_id) { + goto done; + } + + listener = + gst_amc_jni_new_object (env, err, TRUE, listener_cls, constructor_id); + if (!listener) { + goto done; + } + + if (!gst_amc_jni_call_void_method (env, err, listener, + set_context_id, GST_AMC_VIDEO_DEC_TO_JLONG (decoder))) { + gst_amc_jni_object_unref (env, listener); + listener = NULL; + } + +done: + gst_amc_jni_object_unref (env, listener_cls); + + return listener; +} + static gboolean gst_amc_video_dec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) @@ -962,6 +1232,7 @@ gst_amc_video_dec_set_format (GstVideoDecoder * decoder, guint8 *codec_data = NULL; gsize codec_data_size = 0; GError *err = NULL; + jobject jsurface = NULL; self = GST_AMC_VIDEO_DEC (decoder); @@ -1055,6 +1326,134 @@ gst_amc_video_dec_set_format (GstVideoDecoder * decoder, GST_ELEMENT_WARNING_FROM_ERROR (self, err); } + { + gboolean downstream_supports_gl = FALSE; + GstVideoDecoder *decoder = GST_VIDEO_DECODER (self); + GstPad *src_pad = GST_VIDEO_DECODER_SRC_PAD (decoder); + GstCaps *templ_caps = gst_pad_get_pad_template_caps (src_pad); + GstCaps *downstream_caps = gst_pad_peer_query_caps (src_pad, templ_caps); + + gst_caps_unref (templ_caps); + + if (downstream_caps) { + guint i, n; + GstStaticCaps static_caps = + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA")); + GstCaps *gl_memory_caps = gst_static_caps_get (&static_caps); + + GST_DEBUG_OBJECT (self, "Available downstream caps: %" GST_PTR_FORMAT, + downstream_caps); + + /* Check if downstream caps supports + * video/x-raw(memory:GLMemory),format=RGBA */ + n = gst_caps_get_size (downstream_caps); + for (i = 0; i < n; i++) { + GstCaps *caps = NULL; + GstStructure *structure = gst_caps_get_structure (downstream_caps, i); + GstCapsFeatures *features = gst_caps_get_features (downstream_caps, i); + + caps = gst_caps_new_full (gst_structure_copy (structure), NULL); + if (!caps) + continue; + + gst_caps_set_features (caps, 0, gst_caps_features_copy (features)); + + if (gst_caps_can_intersect (caps, gl_memory_caps)) { + downstream_supports_gl = TRUE; + } + + gst_caps_unref (caps); + if (downstream_supports_gl) + break; + } + + gst_caps_unref (gl_memory_caps); + + /* If video/x-raw(memory:GLMemory),format=RGBA is supported, + * update the video decoder output state accordingly and negotiate */ + if (downstream_supports_gl) { + GstVideoCodecState *output_state = NULL; + GstVideoCodecState *prev_output_state = NULL; + + prev_output_state = gst_video_decoder_get_output_state (decoder); + + output_state = + gst_video_decoder_set_output_state (decoder, GST_VIDEO_FORMAT_RGBA, + state->info.width, state->info.height, state); + + if (output_state->caps) { + gst_caps_unref (output_state->caps); + } + + output_state->caps = gst_video_info_to_caps (&output_state->info); + gst_caps_set_features (output_state->caps, 0, + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL)); + + /* gst_amc_video_dec_decide_allocation will update + * self->downstream_supports_gl */ + if (!gst_video_decoder_negotiate (decoder)) { + GST_ERROR_OBJECT (self, "Failed to negotiate"); + + /* Rollback output state changes */ + if (prev_output_state) { + output_state->info = prev_output_state->info; + gst_caps_replace (&output_state->caps, prev_output_state->caps); + } else { + gst_video_info_init (&output_state->info); + gst_caps_replace (&output_state->caps, NULL); + } + } + if (prev_output_state) { + gst_video_codec_state_unref (prev_output_state); + } + } + } + } + + GST_INFO_OBJECT (self, "GL output: %s", + self->downstream_supports_gl ? "enabled" : "disabled"); + + if (self->downstream_supports_gl && self->surface) { + jsurface = self->surface->jobject; + } else if (self->downstream_supports_gl && !self->surface) { + int ret = TRUE; + JNIEnv *env = NULL; + jobject listener = NULL; + GstAmcSurfaceTexture *surface_texture = NULL; + + env = gst_amc_jni_get_env (); + surface_texture = gst_amc_surface_texture_new (&err); + if (!surface_texture) { + GST_ELEMENT_ERROR_FROM_ERROR (self, err); + return FALSE; + } + + listener = + gst_amc_video_dec_new_on_frame_available_listener (self, env, &err); + if (!listener) { + ret = FALSE; + goto done; + } + + if (!gst_amc_surface_texture_set_on_frame_available_listener + (surface_texture, listener, &err)) { + ret = FALSE; + goto done; + } + + self->surface = gst_amc_surface_new (surface_texture, &err); + jsurface = self->surface->jobject; + + done: + g_object_unref (surface_texture); + gst_amc_jni_object_unref (env, listener); + if (!ret) { + GST_ELEMENT_ERROR_FROM_ERROR (self, err); + return FALSE; + } + } + format_string = gst_amc_format_to_string (format, &err); if (err) GST_ELEMENT_WARNING_FROM_ERROR (self, err); @@ -1062,11 +1461,16 @@ gst_amc_video_dec_set_format (GstVideoDecoder * decoder, GST_STR_NULL (format_string)); g_free (format_string); - if (!gst_amc_codec_configure (self->codec, format, 0, &err)) { + if (!gst_amc_codec_configure (self->codec, format, jsurface, 0, &err)) { GST_ERROR_OBJECT (self, "Failed to configure codec"); GST_ELEMENT_ERROR_FROM_ERROR (self, err); return FALSE; } + if (jsurface) { + self->codec_config = AMC_CODEC_CONFIG_WITH_SURFACE; + } else { + self->codec_config = AMC_CODEC_CONFIG_WITHOUT_SURFACE; + } gst_amc_format_free (format); @@ -1256,9 +1660,9 @@ gst_amc_video_dec_handle_frame (GstVideoDecoder * decoder, offset += buffer_info.size; GST_DEBUG_OBJECT (self, - "Queueing buffer %d: size %d time %" G_GINT64_FORMAT " flags 0x%08x", - idx, buffer_info.size, buffer_info.presentation_time_us, - buffer_info.flags); + "Queueing buffer %d: size %d time %" G_GINT64_FORMAT + " flags 0x%08x", idx, buffer_info.size, + buffer_info.presentation_time_us, buffer_info.flags); if (!gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, &err)) { if (self->flushing) { @@ -1415,26 +1819,113 @@ gst_amc_video_dec_drain (GstAmcVideoDec * self) return ret; } +static gboolean +_caps_are_rgba_with_gl_memory (GstCaps * caps) +{ + GstVideoInfo info; + GstCapsFeatures *features; + + if (!caps) + return FALSE; + + if (!gst_video_info_from_caps (&info, caps)) + return FALSE; + + if (info.finfo->format != GST_VIDEO_FORMAT_RGBA) + return FALSE; + + if (!(features = gst_caps_get_features (caps, 0))) + return FALSE; + + return gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); +} + static gboolean gst_amc_video_dec_decide_allocation (GstVideoDecoder * bdec, GstQuery * query) { - GstBufferPool *pool; - GstStructure *config; + GstCaps *caps = NULL; + gboolean need_pool = FALSE; + GstAmcVideoDec *self = GST_AMC_VIDEO_DEC (bdec); if (!GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation (bdec, query)) return FALSE; - g_assert (gst_query_get_n_allocation_pools (query) > 0); - gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL); - g_assert (pool != NULL); + self->downstream_supports_gl = FALSE; + gst_query_parse_allocation (query, &caps, &need_pool); + if (_caps_are_rgba_with_gl_memory (caps)) { + guint i, n_allocation_pools; + GstGLBufferPool *gl_pool = NULL; + GstGLContext *gl_context = NULL; + GstGLDisplay *gl_display = NULL; - config = gst_buffer_pool_get_config (pool); - if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) { - gst_buffer_pool_config_add_option (config, - GST_BUFFER_POOL_OPTION_VIDEO_META); + n_allocation_pools = MAX (gst_query_get_n_allocation_pools (query), 1); + for (i = 0; i < n_allocation_pools; i++) { + GstBufferPool *pool = NULL; + GstStructure *config = NULL; + guint min, max, size; + + gst_query_parse_nth_allocation_pool (query, i, &pool, &size, &min, &max); + config = gst_buffer_pool_get_config (pool); + if (!config) { + gst_object_unref (pool); + continue; + } + + if (!GST_IS_GL_BUFFER_POOL (pool)) { + gst_object_unref (pool); + continue; + } + + gl_pool = GST_GL_BUFFER_POOL (pool); + break; + } + + if (!gl_pool) { + GST_WARNING_OBJECT (bdec, "Failed to get gl pool from downstream"); + gst_object_unref (gl_pool); + return FALSE; + } + + gl_context = gl_pool->context; + if (!gl_context) { + GST_WARNING_OBJECT (bdec, "Failed to get gl context from downstream"); + gst_object_unref (gl_pool); + return FALSE; + } + + if (self->gl_context) { + gst_object_unref (self->gl_context); + } + + if (self->renderer) { + gst_amc_2d_texture_renderer_free (self->renderer); + self->renderer = NULL; + } + +// gl_display = gst_gl_context_get_display (gl_context); +// self->gl_context = gst_gl_context_new (gl_display); +// gst_object_unref (gl_display); + self->gl_context = gst_object_ref (gl_context); + +// gst_gl_context_create (self->gl_context, gl_context, NULL); + + gst_object_unref (gl_pool); + + self->downstream_supports_gl = TRUE; } - gst_buffer_pool_set_config (pool, config); - gst_object_unref (pool); - return TRUE; + return gst_amc_video_dec_check_codec_config (self); +} + +static void +gst_amc_video_dec_on_frame_available (JNIEnv * env, jobject thiz, + long long context, jobject surfaceTexture) +{ + GstAmcVideoDec *dec = JLONG_TO_GST_AMC_VIDEO_DEC (context); + + g_mutex_lock (&dec->on_frame_available_lock); + dec->on_frame_available = TRUE; + g_cond_signal (&dec->on_frame_available_cond); + g_mutex_unlock (&dec->on_frame_available_lock); } diff --git a/sys/androidmedia/gstamcvideodec.h b/sys/androidmedia/gstamcvideodec.h index c6c2b4f49a..07a2f72048 100644 --- a/sys/androidmedia/gstamcvideodec.h +++ b/sys/androidmedia/gstamcvideodec.h @@ -22,10 +22,13 @@ #define __GST_AMC_VIDEO_DEC_H__ #include +#include #include #include "gstamc.h" +#include "gstamcsurface.h" +#include "gstamc2dtexturerenderer.h" G_BEGIN_DECLS @@ -44,6 +47,14 @@ G_BEGIN_DECLS typedef struct _GstAmcVideoDec GstAmcVideoDec; typedef struct _GstAmcVideoDecClass GstAmcVideoDecClass; +typedef enum _GstAmcCodecConfig GstAmcCodecConfig; + +enum _GstAmcCodecConfig +{ + AMC_CODEC_CONFIG_NONE, + AMC_CODEC_CONFIG_WITH_SURFACE, + AMC_CODEC_CONFIG_WITHOUT_SURFACE, +}; struct _GstAmcVideoDec { @@ -51,6 +62,7 @@ struct _GstAmcVideoDec /* < private > */ GstAmcCodec *codec; + GstAmcCodecConfig codec_config; GstVideoCodecState *input_state; gboolean input_state_changed; @@ -59,6 +71,10 @@ struct _GstAmcVideoDec GstVideoFormat format; GstAmcColorFormatInfo color_format_info; + /* Output dimensions */ + guint width; + guint height; + guint8 *codec_data; gsize codec_data_size; /* TRUE if the component is configured and saw @@ -76,7 +92,17 @@ struct _GstAmcVideoDec /* TRUE if the component is drained currently */ gboolean drained; + GstAmcSurface *surface; + + GstGLContext *gl_context; + GstAmc2DTextureRenderer *renderer; + + gboolean downstream_supports_gl; GstFlowReturn downstream_flow_ret; + + GMutex on_frame_available_lock; + GCond on_frame_available_cond; + gboolean on_frame_available; }; struct _GstAmcVideoDecClass diff --git a/sys/androidmedia/gstamcvideoenc.c b/sys/androidmedia/gstamcvideoenc.c index 0d9f7de4fe..5198fe67bc 100644 --- a/sys/androidmedia/gstamcvideoenc.c +++ b/sys/androidmedia/gstamcvideoenc.c @@ -1032,7 +1032,7 @@ process_buffer: gst_amc_buffer_free (buf); buf = NULL; - if (!gst_amc_codec_release_output_buffer (self->codec, idx, &err)) { + if (!gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err)) { if (self->flushing) { g_clear_error (&err); goto flushing; @@ -1291,7 +1291,7 @@ gst_amc_video_enc_set_format (GstVideoEncoder * encoder, GST_STR_NULL (format_string)); g_free (format_string); - if (!gst_amc_codec_configure (self->codec, format, 1, &err)) { + if (!gst_amc_codec_configure (self->codec, format, NULL, 1, &err)) { GST_ERROR_OBJECT (self, "Failed to configure codec"); GST_ELEMENT_ERROR_FROM_ERROR (self, err); goto quit; diff --git a/sys/androidmedia/gstjniutils.c b/sys/androidmedia/gstjniutils.c index 74d476afda..7861002765 100644 --- a/sys/androidmedia/gstjniutils.c +++ b/sys/androidmedia/gstjniutils.c @@ -41,6 +41,7 @@ static jint (*create_java_vm) (JavaVM ** p_vm, JNIEnv ** p_env, void *vm_args); static JavaVM *java_vm; static gboolean started_java_vm = FALSE; static pthread_key_t current_jni_env; +static jobject (*get_class_loader) (void); static struct { @@ -563,6 +564,26 @@ symbol_error: } } +static gboolean +check_application_class_loader (void) +{ + gboolean ret = TRUE; + GModule *module = NULL; + + module = g_module_open (NULL, G_MODULE_BIND_LOCAL); + if (!module) { + return FALSE; + } + if (!g_module_symbol (module, "gst_android_get_application_class_loader", + (gpointer *) & get_class_loader)) { + ret = FALSE; + } + + g_module_close (module); + + return ret; +} + static gboolean initialize_classes (void) { @@ -623,6 +644,11 @@ initialize_classes (void) return FALSE; } + if (!check_application_class_loader ()) { + GST_ERROR ("Could not find application class loader provider"); + return FALSE; + } + return TRUE; } @@ -802,6 +828,57 @@ gst_amc_jni_is_vm_started (void) return started_java_vm; } +jclass +gst_amc_jni_get_application_class (JNIEnv * env, const gchar * name, + GError ** err) +{ + jclass tmp = NULL; + jclass class = NULL; + jstring name_jstr = NULL; + + jobject class_loader = NULL; + jclass class_loader_cls = NULL; + jmethodID load_class_id = 0; + + + class_loader = get_class_loader (); + if (!class_loader) { + g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED, + "Could not retreive application class loader"); + goto done; + } + + class_loader_cls = (*env)->GetObjectClass (env, class_loader); + if (!class_loader_cls) { + g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED, + "Could not retreive application class loader java class"); + goto done; + } + + load_class_id = + gst_amc_jni_get_method_id (env, err, class_loader_cls, "loadClass", + "(Ljava/lang/String;)Ljava/lang/Class;"); + if (!class_loader_cls) { + goto done; + } + + name_jstr = gst_amc_jni_string_from_gchar (env, err, FALSE, name); + if (!name_jstr) { + goto done; + } + + if (gst_amc_jni_call_object_method (env, err, class_loader, + load_class_id, &tmp, name_jstr)) { + class = gst_amc_jni_object_make_global (env, tmp); + } + +done: + gst_amc_jni_object_local_unref (env, name_jstr); + gst_amc_jni_object_local_unref (env, class_loader_cls); + + return class; +} + #define CALL_STATIC_TYPE_METHOD(_type, _name, _jname) \ gboolean gst_amc_jni_call_static_##_name##_method (JNIEnv *env, GError ** err, jclass klass, jmethodID methodID, _type * value, ...) \ { \ diff --git a/sys/androidmedia/gstjniutils.h b/sys/androidmedia/gstjniutils.h index a6bc68ac22..c323cf8e17 100644 --- a/sys/androidmedia/gstjniutils.h +++ b/sys/androidmedia/gstjniutils.h @@ -107,6 +107,10 @@ gboolean gst_amc_jni_is_vm_started (void); JNIEnv *gst_amc_jni_get_env (void); +jclass gst_amc_jni_get_application_class (JNIEnv * env, + const gchar * name, + GError ** err); + #define DEF_CALL_STATIC_TYPE_METHOD(_type, _name, _jname, _retval) \ gboolean gst_amc_jni_call_static_##_name##_method (JNIEnv *env, GError ** err, jclass klass, jmethodID methodID, _type * value, ...) diff --git a/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmcOnFrameAvailableListener.java b/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmcOnFrameAvailableListener.java new file mode 100644 index 0000000000..9a4df2396e --- /dev/null +++ b/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmcOnFrameAvailableListener.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2015, Collabora Ltd. + * Author: Matthieu Bouron + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +package org.freedesktop.gstreamer.androidmedia; + +import android.graphics.SurfaceTexture; +import android.graphics.SurfaceTexture.OnFrameAvailableListener; + +public class GstAmcOnFrameAvailableListener implements OnFrameAvailableListener +{ + private long context = 0; + + public synchronized void onFrameAvailable (SurfaceTexture surfaceTexture) { + native_onFrameAvailable(context, surfaceTexture); + } + + public long getContext () { + return context; + } + + public void setContext (long c) { + context = c; + } + + private native void native_onFrameAvailable (long context, SurfaceTexture surfaceTexture); +}