mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-01-31 19:42:26 +00:00
Code Issues 1 Projects Releases Wiki Activity

amcviddec: output external-oes textures

Browse source 
This provides a performance and power usage improvement by removing
the texture copy from an OES texture to 2D texture.

The flow is as follows
1. Generate the output buffer with the required sync meta with the incrementing
   push counter and OES GL memory
1.1 release_output_buffer (buf, render=true) and push downstream
2. Downstream waits for on the sync meta (timed wait) or drops the frame (no wait)
2.1 Timed wait for the frame number to reach the number of frame callbacks fired
2.2 Unconditionally update the image when the wait completes (success or fail).
    Sets the affine transformation matrix meta on the buffer.
3. Downstream renders as usual.

At *some* point through this the on_frame_callback may or may not fire.  If it
does fire, we can finish waiting early and render. Otherwise we have to
wait for a timeout to occur which may cause more buffers to be pused into the
internal GL queue which siginificantly decreases the chances of the
on_frame_callback to fire again.  This is because the frame callback only occurs
when the internal GL queue changes state from empty to non-empty.

Because there is no way to reliably correlate between the number of buffers
pushed and the number of frame callbacks received, there are a number of
workarounds in place.
1. We self-increment the ready counter when it falls behind the push counter
2. Time based waits as the frame callback may not be fired for a certain frame.
3. It is assumed that the device can render at speed or performs some QoS of
   the interal GL queue (which may not match the GStreamer QoS).

It holds that we call SurfaceTexture::updateTexImage for each buffer pushed
downstream however there's no guarentee that updateTexImage will result in
the exact next frame (it could skip or duplicate) so synchronization is not
guaranteed to be accurate although it seems to be close enough to be unable
to discern visually.  This has not changed from before this patch.  The current
requirement for synchronization is that updateTexImage is called at the point in
time when the buffers is to be rendered.

https://bugzilla.gnome.org/show_bug.cgi?id=757285
This commit is contained in:
Matthew Waters 2015-11-03 13:19:41 +11:00
parent 67327615df
commit f2ca0eaf27
5 changed files with 508 additions and 631 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
sys/androidmedia/Makefile.am
View file

@ -7,7 +7,6 @@ libgstandroidmedia_la_SOURCES = \
gstamcvideoenc.c \
gstamcsurface.c \
gstamcsurfacetexture.c \
gstamc2dtexturerenderer.c \
gstjniutils.c
noinst_HEADERS = \
@ -18,7 +17,6 @@ noinst_HEADERS = \
gstamcvideoenc.h \
gstamcsurface.h \
gstamcsurfacetexture.h \
gstamc2dtexturerenderer.h \
gstjniutils.h
libgstandroidmedia_la_CFLAGS = \

438
sys/androidmedia/gstamc2dtexturerenderer.c
View file

@ -1,438 +0,0 @@
/*
* Copyright (C) 2014, Collabora Ltd.
* Author: Matthieu Bouron <matthieu.bouron@collabora.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation
* version 2.1 of the License.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstamc2dtexturerenderer.h"
/* *INDENT-OFF* */
static const gchar frag_COPY_OES[] =
"#extension GL_OES_EGL_image_external : require \n"
"precision mediump float; \n"
"varying vec2 v_texcoord; \n"
"uniform samplerExternalOES u_tex; \n"
"void main (void) \n"
"{ \n"
" vec4 t = texture2D(u_tex, v_texcoord); \n"
" gl_FragColor = vec4(t.rgb, 1.0); \n"
"}";
static const gchar vert_COPY_OES[] =
"attribute vec4 a_position; \n"
"attribute vec2 a_texcoord; \n"
"varying vec2 v_texcoord; \n"
"uniform mat4 u_transformation; \n"
"void main() \n"
"{ \n"
" gl_Position = a_position; \n"
" v_texcoord = (u_transformation * vec4(a_texcoord, 0, 1)).xy; \n"
"}";
/* *INDENT-ON* */
static void
_surface_texture_detach_from_gl_context (GstGLContext * context,
GstAmc2DTextureRenderer * renderer)
{
renderer->gl_context_result =
gst_amc_surface_texture_detach_from_gl_context (renderer->surface_texture,
&renderer->gl_context_error);
}
static gboolean
_surface_texture_detach_from_gl_context_perform (GstAmc2DTextureRenderer *
renderer, GError ** error)
{
renderer->gl_context_result = FALSE;
renderer->gl_context_error = NULL;
gst_gl_context_thread_add (renderer->context,
(GstGLContextThreadFunc) _surface_texture_detach_from_gl_context,
renderer);
*error = renderer->gl_context_error;
renderer->gl_context_error = NULL;
return renderer->gl_context_result;
}
static void
_gen_oes_texture (GstGLContext * context, guint * tex_id)
{
const GstGLFuncs *gl = context->gl_vtable;
GST_TRACE ("Generating OES texture");
gl->GenTextures (1, tex_id);
gl->BindTexture (GL_TEXTURE_EXTERNAL_OES, *tex_id);
gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S,
GL_CLAMP_TO_EDGE);
gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T,
GL_CLAMP_TO_EDGE);
gl->BindTexture (GL_TEXTURE_EXTERNAL_OES, 0);
GST_LOG ("generated OES texture id:%d", *tex_id);
}
GstAmc2DTextureRenderer *
gst_amc_2d_texture_renderer_new (GstGLContext * context,
GstAmcSurfaceTexture * surface_texture, guint width, guint height)
{
GstAmc2DTextureRenderer *renderer;
g_return_val_if_fail (surface_texture != NULL, NULL);
renderer = g_new0 (GstAmc2DTextureRenderer, 1);
if (!renderer)
return NULL;
renderer->context = gst_object_ref (context);
renderer->surface_texture = g_object_ref (surface_texture);
gst_video_info_init (&renderer->info);
gst_video_info_set_format (&renderer->info,
GST_VIDEO_FORMAT_RGBA, width, height);
return renderer;
}
void
gst_amc_2d_texture_renderer_free (GstAmc2DTextureRenderer * renderer)
{
GError *error = NULL;
if (renderer->surface_texture) {
_surface_texture_detach_from_gl_context_perform (renderer, &error);
g_object_unref (renderer->surface_texture);
}
if (renderer->fbo || renderer->depth_buffer) {
gst_gl_context_del_fbo (renderer->context, renderer->fbo,
renderer->depth_buffer);
}
if (renderer->shader) {
gst_object_unref (renderer->shader);
}
if (renderer->oes_tex_id) {
gst_gl_context_del_texture (renderer->context, &renderer->oes_tex_id);
}
if (renderer->context) {
gst_object_unref (renderer->context);
}
g_free (renderer);
}
static gboolean
_2d_texture_renderer_init_fbo (GstAmc2DTextureRenderer * renderer)
{
const GstGLFuncs *gl;
GLuint fake_texture = 0;
guint out_width, out_height;
guint internal_format;
out_width = GST_VIDEO_INFO_WIDTH (&renderer->info);
out_height = GST_VIDEO_INFO_HEIGHT (&renderer->info);
internal_format =
gst_gl_sized_gl_format_from_gl_format_type (renderer->context, GL_RGBA,
GL_UNSIGNED_BYTE);
gl = renderer->context->gl_vtable;
if (!gl->GenFramebuffers) {
/* turn off the pipeline because Frame buffer object is a not present */
gst_gl_context_set_error (renderer->context,
"Context, EXT_framebuffer_object supported: no");
return FALSE;
}
GST_INFO ("Context, EXT_framebuffer_object supported: yes");
/* setup FBO */
gl->GenFramebuffers (1, &renderer->fbo);
gl->BindFramebuffer (GL_FRAMEBUFFER, renderer->fbo);
/* setup the render buffer for depth */
gl->GenRenderbuffers (1, &renderer->depth_buffer);
gl->BindRenderbuffer (GL_RENDERBUFFER, renderer->depth_buffer);
gl->RenderbufferStorage (GL_RENDERBUFFER, GL_DEPTH_COMPONENT16,
out_width, out_height);
/* a fake texture is attached to the render FBO (cannot init without it) */
gl->GenTextures (1, &fake_texture);
gl->BindTexture (GL_TEXTURE_2D, fake_texture);
gl->TexImage2D (GL_TEXTURE_2D, 0, internal_format, out_width, out_height,
0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
/* attach the texture to the FBO to renderer to */
gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_TEXTURE_2D, fake_texture, 0);
/* attach the depth render buffer to the FBO */
gl->FramebufferRenderbuffer (GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
GL_RENDERBUFFER, renderer->depth_buffer);
if (!gst_gl_context_check_framebuffer_status (renderer->context)) {
gst_gl_context_set_error (renderer->context,
"GL framebuffer status incomplete");
return FALSE;
}
/* unbind the FBO */
gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
gl->DeleteTextures (1, &fake_texture);
return TRUE;
}
static gboolean
_2d_texture_renderer_init (GstAmc2DTextureRenderer * renderer)
{
GstGLFuncs *gl;
gboolean res;
gl = renderer->context->gl_vtable;
if (renderer->initialized)
return TRUE;
if (!gl->CreateProgramObject && !gl->CreateProgram) {
gst_gl_context_set_error (renderer->context,
"Cannot perform conversion without OpenGL shaders");
return FALSE;
}
_gen_oes_texture (renderer->context, &renderer->oes_tex_id);
res =
gst_gl_context_gen_shader (renderer->context, vert_COPY_OES,
frag_COPY_OES, &renderer->shader);
if (!res)
return FALSE;
renderer->shader_attr_position_loc =
gst_gl_shader_get_attribute_location (renderer->shader, "a_position");
renderer->shader_attr_texture_loc =
gst_gl_shader_get_attribute_location (renderer->shader, "a_texcoord");
gst_gl_shader_use (renderer->shader);
gst_gl_shader_set_uniform_1i (renderer->shader, "u_tex", 0);
gst_gl_context_clear_shader (renderer->context);
if (!_2d_texture_renderer_init_fbo (renderer))
return FALSE;
gl->BindTexture (GL_TEXTURE_2D, 0);
renderer->initialized = TRUE;
return TRUE;
}
static gboolean
_2d_texture_renderer_draw (GstAmc2DTextureRenderer * renderer)
{
GstGLFuncs *gl;
guint out_width, out_height;
GLint viewport_dim[4];
/* *INDENT-OFF* */
const GLfloat vertices[] = {
1.0f, -1.0f, 0.0f, 1.0f, 0.0f,
-1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
-1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f, 1.0f
};
/* *INDENT-ON* */
GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
GLenum multipleRT[] = {
GL_COLOR_ATTACHMENT0,
};
gl = renderer->context->gl_vtable;
out_width = GST_VIDEO_INFO_WIDTH (&renderer->info);
out_height = GST_VIDEO_INFO_HEIGHT (&renderer->info);
gl->BindFramebuffer (GL_FRAMEBUFFER, renderer->fbo);
/* attach the texture to the FBO to rendererer to */
gl->BindTexture (GL_TEXTURE_2D, renderer->tex_id);
gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_TEXTURE_2D, renderer->tex_id, 0);
if (gl->DrawBuffers)
gl->DrawBuffers (1, multipleRT);
else if (gl->DrawBuffer)
gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
gl->GetIntegerv (GL_VIEWPORT, viewport_dim);
gl->Viewport (0, 0, out_width, out_height);
gl->ClearColor (0.0, 0.0, 0.0, 0.0);
gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
gst_gl_shader_use (renderer->shader);
gst_gl_shader_set_uniform_matrix_4fv (renderer->shader, "u_transformation", 1,
FALSE, renderer->transformation_matrix);
gl->VertexAttribPointer (renderer->shader_attr_position_loc, 3,
GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), vertices);
gl->VertexAttribPointer (renderer->shader_attr_texture_loc, 2,
GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), &vertices[3]);
gl->EnableVertexAttribArray (renderer->shader_attr_position_loc);
gl->EnableVertexAttribArray (renderer->shader_attr_texture_loc);
gl->ActiveTexture (GL_TEXTURE0);
gl->BindTexture (GL_TEXTURE_EXTERNAL_OES, renderer->oes_tex_id);
gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S,
GL_CLAMP_TO_EDGE);
gl->TexParameteri (GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T,
GL_CLAMP_TO_EDGE);
gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices);
gl->DisableVertexAttribArray (renderer->shader_attr_position_loc);
gl->DisableVertexAttribArray (renderer->shader_attr_texture_loc);
if (gl->DrawBuffer)
gl->DrawBuffer (GL_NONE);
/* we are done with the shader */
gst_gl_context_clear_shader (renderer->context);
gl->Viewport (viewport_dim[0], viewport_dim[1], viewport_dim[2],
viewport_dim[3]);
gst_gl_context_check_framebuffer_status (renderer->context);
gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
return TRUE;
}
static void
_2d_texture_renderer_render (GstGLContext * context,
GstAmc2DTextureRenderer * renderer)
{
gfloat identify_matrix[16] = { 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f
};
gfloat yflip_matrix[16] = { 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, -1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 1.0f, 0.0f, 1.0f
};
gfloat transformation_matrix[16] = { 0 };
if (!renderer->initialized) {
if (!_2d_texture_renderer_init (renderer)) {
renderer->result = FALSE;
return;
}
if (!gst_amc_surface_texture_attach_to_gl_context
(renderer->surface_texture, renderer->oes_tex_id,
&renderer->gl_context_error)) {
renderer->result = FALSE;
return;
}
}
if (!gst_amc_surface_texture_update_tex_image (renderer->surface_texture,
&renderer->gl_context_error)) {
renderer->result = FALSE;
return;
}
if (gst_amc_surface_texture_get_transform_matrix (renderer->surface_texture,
transformation_matrix, &renderer->gl_context_error)) {
int i, j;
for (i = 0; i < 16; i += 4) {
renderer->transformation_matrix[i + j] = 0.0f;
for (j = 0; j < 4; ++j) {
renderer->transformation_matrix[i + j] =
(transformation_matrix[i + 0] * yflip_matrix[j + 0])
+ (transformation_matrix[i + 1] * yflip_matrix[j + 4])
+ (transformation_matrix[i + 2] * yflip_matrix[j + 8])
+ (transformation_matrix[i + 3] * yflip_matrix[j + 12]);
}
}
} else {
memcpy (renderer->transformation_matrix, identify_matrix,
sizeof (identify_matrix[0] * 16));
}
if (!_2d_texture_renderer_draw (renderer)) {
renderer->result = FALSE;
return;
}
renderer->result = TRUE;
}
gboolean
gst_amc_2d_texture_renderer_render (GstAmc2DTextureRenderer *
renderer, guint tex_id, GError ** error)
{
g_return_val_if_fail (renderer != NULL, FALSE);
g_return_val_if_fail (tex_id, FALSE);
renderer->tex_id = tex_id;
renderer->result = FALSE;
renderer->gl_context_error = NULL;
gst_gl_context_thread_add (renderer->context,
(GstGLContextThreadFunc) _2d_texture_renderer_render, renderer);
*error = renderer->gl_context_error;
renderer->gl_context_error = NULL;
return renderer->result;
}

76
sys/androidmedia/gstamc2dtexturerenderer.h
View file

@ -1,76 +0,0 @@
/*
* Copyright (C) 2014, Collabora Ltd.
* Author: Matthieu Bouron <matthieu.bouron@collabora.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation
* version 2.1 of the License.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
#ifndef __GST_AMC_2D_TEXTURE_RENDER_H__
#define __GST_AMC_2D_TEXTURE_RENDER_H__
#include <gst/gst.h>
#include <gst/gl/gl.h>
#include "gstamcsurfacetexture.h"
G_BEGIN_DECLS
typedef struct _GstAmc2DTextureRenderer GstAmc2DTextureRenderer;
struct _GstAmc2DTextureRenderer {
/* private */
gboolean initialized;
GstGLContext *context;
GstVideoInfo info;
GLuint fbo;
GLuint depth_buffer;
GstGLShader *shader;
GLint shader_attr_position_loc;
GLint shader_attr_texture_loc;
GError *gl_context_error;
gboolean gl_context_result;
GstAmcSurfaceTexture *surface_texture;
guint tex_id;
guint oes_tex_id;
gfloat transformation_matrix[16];
/* out fields */
gboolean result;
};
GstAmc2DTextureRenderer * gst_amc_2d_texture_renderer_new (GstGLContext * context,
GstAmcSurfaceTexture *surface_texture,
guint width,
guint height);
void gst_amc_2d_texture_renderer_free (GstAmc2DTextureRenderer * render);
gboolean gst_amc_2d_texture_renderer_render (GstAmc2DTextureRenderer * render,
guint tex_id,
GError ** error);
G_END_DECLS
#endif

608
sys/androidmedia/gstamcvideodec.c
View file

@ -17,6 +17,8 @@
* Copyright (C) 2015, Edward Hervey
* Author: Edward Hervey <bilboed@gmail.com>
*
* Copyright (C) 2015, Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation
@ -40,6 +42,7 @@
#include <gst/gst.h>
#include <gst/gl/gl.h>
#include <gst/video/gstvideometa.h>
#include <gst/video/gstvideoaffinetransformationmeta.h>
#include <gst/video/gstvideopool.h>
#include <string.h>
@ -51,7 +54,6 @@
#include "gstamcvideodec.h"
#include "gstamc-constants.h"
#include "gstamc2dtexturerenderer.h"
GST_DEBUG_CATEGORY_STATIC (gst_amc_video_dec_debug_category);
#define GST_CAT_DEFAULT gst_amc_video_dec_debug_category
@ -66,9 +68,6 @@ GST_DEBUG_CATEGORY_STATIC (gst_amc_video_dec_debug_category);
g_clear_error (&err); \
} G_STMT_END
/* Assume the device is able to decode at 30fps by default */
#define GST_AMC_VIDEO_DEC_ON_FRAME_AVAILABLE_DEFAULT_TIMEOUT (33 * G_TIME_SPAN_MILLISECOND)
#if GLIB_SIZEOF_VOID_P == 8
#define JLONG_TO_GST_AMC_VIDEO_DEC(value) (GstAmcVideoDec *)(value)
#define GST_AMC_VIDEO_DEC_TO_JLONG(value) (jlong)(value)
@ -83,6 +82,113 @@ struct _BufferIdentification
guint64 timestamp;
};
struct gl_sync_result
{
gint refcount;
gint64 frame_available_ts;
gboolean updated;
};
static struct gl_sync_result *
_gl_sync_result_ref (struct gl_sync_result *result)
{
g_assert (result != NULL);
g_atomic_int_inc (&result->refcount);
GST_TRACE ("gl_sync result %p ref", result);
return result;
}
static void
_gl_sync_result_unref (struct gl_sync_result *result)
{
g_assert (result != NULL);
GST_TRACE ("gl_sync result %p unref", result);
if (g_atomic_int_dec_and_test (&result->refcount)) {
GST_TRACE ("freeing gl_sync result %p", result);
g_free (result);
}
}
struct gl_sync
{
gint refcount;
GstAmcVideoDec *sink; /* back reference for statistics, lock, cond, etc */
GstBuffer *buffer; /* back reference to the buffer */
GstGLMemory *oes_mem; /* where amc is rendering into. The same for every gl_sync */
GstAmcSurface *surface; /* java wrapper for where amc is rendering into */
guint gl_frame_no; /* effectively the frame id */
gint64 released_ts; /* microseconds from g_get_monotonic_time() */
struct gl_sync_result *result;
};
static struct gl_sync *
_gl_sync_ref (struct gl_sync *sync)
{
g_assert (sync != NULL);
g_atomic_int_inc (&sync->refcount);
GST_TRACE ("gl_sync %p ref", sync);
return sync;
}
static void
_gl_sync_unref (struct gl_sync *sync)
{
g_assert (sync != NULL);
GST_TRACE ("gl_sync %p unref", sync);
if (g_atomic_int_dec_and_test (&sync->refcount)) {
GST_TRACE ("freeing gl_sync %p", sync);
_gl_sync_result_unref (sync->result);
g_object_unref (sync->surface);
gst_memory_unref ((GstMemory *) sync->oes_mem);
g_free (sync);
}
}
static void
_attach_mem_to_context (GstGLContext * context, GstAmcVideoDec * self)
{
GST_TRACE_OBJECT (self, "attaching texture %p id %u to current context",
self->surface->texture, self->oes_mem->tex_id);
if (!gst_amc_surface_texture_attach_to_gl_context (self->surface->texture,
self->oes_mem->tex_id, &self->gl_error)) {
GST_ERROR_OBJECT (self, "Failed to attach texture to the GL context");
GST_ELEMENT_ERROR_FROM_ERROR (self, self->gl_error);
} else {
self->gl_mem_attached = TRUE;
}
}
static void
_dettach_mem_from_context (GstGLContext * context, GstAmcVideoDec * self)
{
if (self->surface) {
guint tex_id = self->oes_mem ? self->oes_mem->tex_id : 0;
GST_TRACE_OBJECT (self, "detaching texture %p id %u from current context",
self->surface->texture, tex_id);
if (!gst_amc_surface_texture_detach_from_gl_context (self->surface->texture,
&self->gl_error)) {
GST_ERROR_OBJECT (self, "Failed to attach texture to the GL context");
GST_ELEMENT_ERROR_FROM_ERROR (self, self->gl_error);
}
}
self->gl_mem_attached = FALSE;
}
static BufferIdentification *
buffer_identification_new (GstClockTime timestamp)
{
@ -231,8 +337,8 @@ gst_amc_video_dec_base_init (gpointer g_class)
gst_amc_codec_info_to_caps (codec_info, &sink_caps, &src_caps);
all_src_caps =
gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA"));
gst_caps_from_string ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY
"), format = (string) RGBA, texture-target = (string) external-oes");
if (codec_info->gl_output_only) {
gst_caps_unref (src_caps);
@ -298,9 +404,10 @@ gst_amc_video_dec_init (GstAmcVideoDec * self)
g_mutex_init (&self->drain_lock);
g_cond_init (&self->drain_cond);
g_mutex_init (&self->on_frame_available_lock);
g_cond_init (&self->on_frame_available_cond);
self->on_frame_available = FALSE;
g_mutex_init (&self->gl_lock);
g_cond_init (&self->gl_cond);
self->gl_queue = g_queue_new ();
}
static gboolean
@ -343,8 +450,26 @@ gst_amc_video_dec_close (GstVideoDecoder * decoder)
gst_amc_codec_free (self->codec);
}
self->codec = NULL;
self->codec_config = AMC_CODEC_CONFIG_NONE;
if (self->downstream_supports_gl
&& self->codec_config == AMC_CODEC_CONFIG_WITH_SURFACE) {
g_mutex_lock (&self->gl_lock);
GST_INFO_OBJECT (self, "shutting down gl queue pushed %u ready %u "
"rendered %u", self->gl_pushed_frame_count, self->gl_ready_frame_count,
self->gl_rendered_frame_count);
g_queue_free_full (self->gl_queue, (GDestroyNotify) _gl_sync_unref);
self->gl_queue = g_queue_new ();
g_mutex_unlock (&self->gl_lock);
if (self->gl_mem_attached)
gst_gl_context_thread_add (self->gl_context,
(GstGLContextThreadFunc) _dettach_mem_from_context, self);
}
self->gl_pushed_frame_count = 0;
self->gl_ready_frame_count = 0;
self->gl_rendered_frame_count = 0;
self->gl_last_rendered_frame = 0;
if (self->surface) {
gst_object_unref (self->surface);
@ -355,6 +480,31 @@ gst_amc_video_dec_close (GstVideoDecoder * decoder)
self->flushing = TRUE;
self->downstream_supports_gl = FALSE;
self->codec = NULL;
self->codec_config = AMC_CODEC_CONFIG_NONE;
GST_DEBUG_OBJECT (self, "Freeing GL context: %" GST_PTR_FORMAT,
self->gl_context);
if (self->gl_context) {
gst_object_unref (self->gl_context);
self->gl_context = NULL;
}
if (self->oes_mem) {
gst_memory_unref ((GstMemory *) self->oes_mem);
self->oes_mem = NULL;
}
if (self->gl_display) {
gst_object_unref (self->gl_display);
self->gl_display = NULL;
}
if (self->other_gl_context) {
gst_object_unref (self->other_gl_context);
self->other_gl_context = NULL;
}
GST_DEBUG_OBJECT (self, "Closed decoder");
return TRUE;
@ -368,6 +518,14 @@ gst_amc_video_dec_finalize (GObject * object)
g_mutex_clear (&self->drain_lock);
g_cond_clear (&self->drain_cond);
g_mutex_clear (&self->gl_lock);
g_cond_clear (&self->gl_cond);
if (self->gl_queue) {
g_queue_free_full (self->gl_queue, (GDestroyNotify) _gl_sync_unref);
self->gl_queue = NULL;
}
G_OBJECT_CLASS (parent_class)->finalize (object);
}
@ -437,20 +595,6 @@ gst_amc_video_dec_change_state (GstElement * element, GstStateChange transition)
case GST_STATE_CHANGE_PAUSED_TO_READY:
self->downstream_flow_ret = GST_FLOW_FLUSHING;
self->started = FALSE;
GST_DEBUG_OBJECT (element, "Freeing GL context: %" GST_PTR_FORMAT,
self->gl_context);
if (self->gl_context) {
gst_object_unref (self->gl_context);
self->gl_context = NULL;
}
GST_DEBUG_OBJECT (element, "Freeing GL renderer: %p", self->renderer);
if (self->renderer) {
gst_amc_2d_texture_renderer_free (self->renderer);
self->renderer = NULL;
}
break;
default:
break;
@ -655,6 +799,8 @@ gst_amc_video_dec_set_src_caps (GstAmcVideoDec * self, GstAmcFormat * format)
output_state->caps = gst_video_info_to_caps (&output_state->info);
gst_caps_set_features (output_state->caps, 0,
gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL));
gst_caps_set_simple (output_state->caps, "texture-target", G_TYPE_STRING,
"external-oes", NULL);
}
self->format = gst_format;
@ -703,6 +849,270 @@ gst_amc_video_dec_fill_buffer (GstAmcVideoDec * self, GstAmcBuffer * buf,
return ret;
}
static const gfloat yflip_matrix[16] = {
1.0f, 0.0f, 0.0f, 0.0f,
0.0f, -1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 1.0f, 0.0f, 1.0f
};
static void
_amc_gl_set_sync (GstGLSyncMeta * sync_meta, GstGLContext * context)
{
}
/* caller should remove from the gl_queue after calling this function */
static void
_gl_sync_render_unlocked (struct gl_sync *sync)
{
GstVideoAffineTransformationMeta *af_meta;
GError *error = NULL;
gfloat matrix[16];
gint64 ts = 0;
GST_TRACE ("gl_sync %p result %p render (updated:%u)", sync, sync->result,
sync->result->updated);
if (sync->result->updated)
return;
/* FIXME: if this ever starts returning valid values we should attempt
* to use it */
if (!gst_amc_surface_texture_get_timestamp (sync->surface->texture, &ts, &error)) {
GST_ERROR_OBJECT (sync->sink, "Failed to update texture image");
GST_ELEMENT_ERROR_FROM_ERROR (sync->sink, error);
goto out;
}
GST_TRACE ("gl_sync %p rendering timestamp before update %" G_GINT64_FORMAT, sync, ts);
GST_TRACE ("gl_sync %p update_tex_image", sync);
if (!gst_amc_surface_texture_update_tex_image (sync->surface->texture,
&error)) {
GST_ERROR_OBJECT (sync->sink, "Failed to update texture image");
GST_ELEMENT_ERROR_FROM_ERROR (sync->sink, error);
goto out;
}
GST_TRACE ("gl_sync result %p updated", sync->result);
sync->result->updated = TRUE;
sync->sink->gl_rendered_frame_count++;
sync->sink->gl_last_rendered_frame = sync->gl_frame_no;
if (!gst_amc_surface_texture_get_timestamp (sync->surface->texture, &ts, &error)) {
GST_ERROR_OBJECT (sync->sink, "Failed to update texture image");
GST_ELEMENT_ERROR_FROM_ERROR (sync->sink, error);
goto out;
}
GST_TRACE ("gl_sync %p rendering timestamp after update %" G_GINT64_FORMAT, sync, ts);
af_meta = gst_buffer_add_video_affine_transformation_meta (sync->buffer);
if (!af_meta) {
GST_WARNING ("Failed to retreive the transformation meta from the "
"gl_sync %p buffer %p", sync, sync->buffer);
} else if (gst_amc_surface_texture_get_transform_matrix (sync->surface->
texture, matrix, &error)) {
gst_video_affine_transformation_meta_apply_matrix (af_meta, matrix);
gst_video_affine_transformation_meta_apply_matrix (af_meta, yflip_matrix);
}
GST_LOG ("gl_sync %p successfully updated SurfaceTexture %p into "
"OES texture %u", sync, sync->surface->texture, sync->oes_mem->tex_id);
out:
if (error) {
if (sync->sink->gl_error == NULL)
sync->sink->gl_error = error;
else
g_clear_error (&error);
}
}
static gboolean
_amc_gl_possibly_wait_for_gl_sync (struct gl_sync *sync, gint64 timeout)
{
gint64 end_time;
/* FIXME: remove this somehow */
if (timeout != -1)
end_time = g_get_monotonic_time () + timeout;
GST_TRACE ("gl_sync %p waiting for frame %u current %u updated %u ", sync,
sync->gl_frame_no, sync->sink->gl_ready_frame_count,
sync->result->updated);
if ((gint) (sync->sink->gl_last_rendered_frame - sync->gl_frame_no) > 0) {
GST_ERROR ("gl_sync %p unsuccessfully waited for frame %u. out of order "
"wait detected", sync, sync->gl_frame_no);
return FALSE;
}
/* The number of frame callbacks (gl_ready_frame_count) is not a direct
* relationship with the number of pushed buffers (gl_pushed_frame_count)
* as, from the frameworks/native/include/gui/ConsumerBase.h file,
*
* "...frames that are queued while in asynchronous mode only trigger the
* callback if no previous frames are pending."
*
* As a result, we need to advance the ready counter somehow ourselves when
* such events happen. There is no reliable way of knowing when/if the frame
* listener is going to fire. The only uniqueu identifier,
* SurfaceTexture::get_timestamp seems to always return 0.
*
* The maximum queue size as defined in
* frameworks/native/include/gui/BufferQueue.h
* is 32 of which a maximum of 30 can be acquired at a time so we picked a
* number less than that to wait for before updating the ready frame count.
*/
while (!sync->result->updated && (gint)(sync->sink->gl_ready_frame_count - sync->gl_frame_no) < 0) {
/* The time limit is need otherwise when amc decides to not emit the
* frame listener (say, on orientation changes) we don't wait foreever */
if (timeout == -1 || !g_cond_wait_until (&sync->sink->gl_cond,
&sync->sink->gl_lock, end_time)) {
GST_LOG ("gl_sync %p unsuccessfully waited for frame %u", sync,
sync->gl_frame_no);
/* Assume that the decoder<->renderer can ultimately keep up */
if ((gint)(sync->sink->gl_pushed_frame_count - sync->sink->gl_ready_frame_count) > 0) {
guint diff = sync->sink->gl_pushed_frame_count - sync->sink->gl_ready_frame_count - 1u;
sync->sink->gl_ready_frame_count += diff;
GST_LOG ("gl_sync %p possible \'on_frame_available\' listener miss "
"detected, attempting to work around. Jumping forward %u "
"frames for frame %u", sync, diff, sync->gl_frame_no);
}
return FALSE;
}
}
GST_LOG ("gl_sync %p successfully waited for frame %u", sync,
sync->gl_frame_no);
return TRUE;
}
static gboolean
_amc_gl_iterate_queue_unlocked (GstGLSyncMeta * sync_meta, gint64 timeout)
{
struct gl_sync *sync = sync_meta->data;
struct gl_sync *tmp;
gboolean ret = TRUE;
while ((tmp = g_queue_peek_head (sync->sink->gl_queue))) {
/* skip frames that are ahead of the current wait frame */
if ((gint) (sync->gl_frame_no - tmp->gl_frame_no) < 0) {
GST_TRACE ("gl_sync %p frame %u is ahead of gl_sync %p frame %u", tmp,
tmp->gl_frame_no, sync, sync->gl_frame_no);
break;
}
/* Frames are currently pushed in order and waits need to be performed
* in the same order */
if (!_amc_gl_possibly_wait_for_gl_sync (tmp, timeout)) {
ret = FALSE;
break;
}
_gl_sync_render_unlocked (tmp);
g_queue_pop_head (sync->sink->gl_queue);
_gl_sync_unref (tmp);
}
return ret;
}
struct gl_wait
{
GstGLSyncMeta *sync_meta;
gboolean ret;
};
static void
_amc_gl_wait_gl (GstGLContext * context, struct gl_wait * wait)
{
struct gl_sync *sync = wait->sync_meta->data;
gint64 current_time, wait_time;
g_mutex_lock (&sync->sink->gl_lock);
current_time = g_get_monotonic_time ();
/* Assume that the device can do 20fps. See the comment in
* _amc_gl_possibly_wait_for_gl_sync() as to why this is ultimately needed
* even though it is ultimately a HACK */
wait_time = 50 * G_TIME_SPAN_MILLISECOND - (current_time - sync->released_ts);
if (wait_time < 0)
wait_time = -1;
wait->ret = _amc_gl_iterate_queue_unlocked (wait->sync_meta, wait_time);
g_mutex_unlock (&sync->sink->gl_lock);
}
static void
_amc_gl_wait (GstGLSyncMeta * sync_meta, GstGLContext * context)
{
struct gl_sync *sync = sync_meta->data;
struct gl_wait wait;
wait.sync_meta = sync_meta;
wait.ret = FALSE;
gst_gl_context_thread_add (context,
(GstGLContextThreadFunc) _amc_gl_wait_gl, &wait);
if (!wait.ret)
GST_WARNING ("gl_sync %p could not wait for frame, took too long", sync);
}
static void
_amc_gl_copy (GstGLSyncMeta * src, GstBuffer * sbuffer, GstGLSyncMeta * dest,
GstBuffer * dbuffer)
{
struct gl_sync *sync = src->data;
struct gl_sync *tmp;
tmp = g_new0 (struct gl_sync, 1);
GST_TRACE ("copying gl_sync %p to %p", sync, tmp);
g_mutex_lock (&sync->sink->gl_lock);
tmp->refcount = 1;
tmp->sink = sync->sink;
tmp->buffer = dbuffer;
tmp->oes_mem = (GstGLMemory *) gst_memory_ref ((GstMemory *) sync->oes_mem);
tmp->surface = g_object_ref (sync->surface);
tmp->gl_frame_no = sync->gl_frame_no;
tmp->released_ts = sync->released_ts;
tmp->result = sync->result;
_gl_sync_result_ref (tmp->result);
dest->data = tmp;
g_mutex_unlock (&sync->sink->gl_lock);
}
static void
_amc_gl_render_on_free (GstGLContext * context, GstGLSyncMeta * sync_meta)
{
struct gl_sync *sync = sync_meta->data;
g_mutex_lock (&sync->sink->gl_lock);
/* just render as many frames as we have */
_amc_gl_iterate_queue_unlocked (sync_meta, -1);
g_mutex_unlock (&sync->sink->gl_lock);
}
static void
_amc_gl_free (GstGLSyncMeta * sync_meta, GstGLContext *context)
{
struct gl_sync *sync = sync_meta->data;
/* The wait render queue inside android is not very deep so when we drop
* frames we need to signal that we have rendered them if we have any chance
* of keeping up between the decoder, the android GL queue and downstream
* OpenGL. If we don't do this, once we start dropping frames downstream,
* it is very near to impossible for the pipeline to catch up. */
gst_gl_context_thread_add (context,
(GstGLContextThreadFunc) _amc_gl_render_on_free, sync_meta);
_gl_sync_unref (sync);
}
static void
gst_amc_video_dec_loop (GstAmcVideoDec * self)
{
@ -810,100 +1220,81 @@ retry:
flow_ret = gst_video_decoder_drop_frame (GST_VIDEO_DECODER (self), frame);
} else if (self->codec_config == AMC_CODEC_CONFIG_WITH_SURFACE
&& buffer_info.size > 0) {
GstMemory *mem;
GstBuffer *outbuf;
gint64 timeout;
gint64 end_time;
GstGLSyncMeta *sync_meta;
GstVideoCodecState *state;
struct gl_sync *sync;
outbuf =
gst_video_decoder_allocate_output_buffer (GST_VIDEO_DECODER (self));
outbuf = gst_buffer_new ();
state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (self));
if (state && state->info.fps_n > 0 && state->info.fps_d > 0) {
timeout =
gst_util_uint64_scale_int (G_TIME_SPAN_SECOND, state->info.fps_d,
state->info.fps_n);
} else {
timeout = GST_AMC_VIDEO_DEC_ON_FRAME_AVAILABLE_DEFAULT_TIMEOUT;
if (!self->oes_mem) {
self->oes_mem = (GstGLMemory *) gst_gl_memory_alloc (self->gl_context,
GST_GL_TEXTURE_TARGET_EXTERNAL_OES, NULL, &state->info, 0, NULL);
gst_gl_context_thread_add (self->gl_context,
(GstGLContextThreadFunc) _attach_mem_to_context, self);
if (self->gl_error) {
GST_ELEMENT_ERROR_FROM_ERROR (self, self->gl_error);
gst_video_codec_state_unref (state);
goto gl_output_error;
}
}
gst_buffer_append_memory (outbuf,
gst_memory_ref ((GstMemory *) self->oes_mem));
gst_video_codec_state_unref (state);
mem = gst_buffer_peek_memory (outbuf, 0);
if (gst_is_gl_memory (mem)) {
GstGLMemory *gl_mem = (GstGLMemory *) mem;
sync = g_new0 (struct gl_sync, 1);
sync->refcount = 1;
sync->sink = self;
sync->buffer = outbuf;
sync->surface = g_object_ref (self->surface);
sync->oes_mem =
(GstGLMemory *) gst_memory_ref ((GstMemory *) self->oes_mem);
sync->result = g_new0 (struct gl_sync_result, 1);
sync->result->refcount = 1;
sync->result->updated = FALSE;
if (!self->renderer) {
self->renderer =
gst_amc_2d_texture_renderer_new (self->gl_context,
self->surface->texture, self->width, self->height);
}
GST_TRACE ("new gl_sync %p result %p", sync, sync->result);
release_buffer = FALSE;
self->on_frame_available = FALSE;
g_mutex_lock (&self->on_frame_available_lock);
sync_meta = gst_buffer_add_gl_sync_meta_full (self->gl_context, outbuf,
sync);
sync_meta->set_sync = _amc_gl_set_sync;
sync_meta->wait = _amc_gl_wait;
sync_meta->copy = _amc_gl_copy;
sync_meta->free = _amc_gl_free;
/* Render the frame into the surface */
if (!gst_amc_codec_release_output_buffer (self->codec, idx, TRUE, &err)) {
gst_buffer_unref (outbuf);
GST_ERROR_OBJECT (self, "Failed to render buffer, index %d", idx);
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
gst_buffer_add_video_affine_transformation_meta (outbuf);
goto gl_output_error;
}
g_mutex_lock (&self->gl_lock);
/* Wait for the frame to become available */
end_time = g_get_monotonic_time () + timeout;
g_cond_wait_until (&self->on_frame_available_cond,
&self->on_frame_available_lock, end_time);
self->gl_pushed_frame_count++;
sync->gl_frame_no = self->gl_pushed_frame_count;
g_queue_push_tail (self->gl_queue, _gl_sync_ref (sync));
g_mutex_unlock (&self->on_frame_available_lock);
GST_DEBUG_OBJECT (self, "render GL frame %u %" GST_PTR_FORMAT,
sync->gl_frame_no, outbuf);
/* Now that the frame is available, we can render it to a 2D texture.
*
* Calling updateTexImage seems necessary even if no frame is available
* otherwise it could happen that the onFrameAvailable callback is not
* executed anymore. */
if (!gst_amc_2d_texture_renderer_render (self->renderer,
gl_mem->tex_id, &err)) {
gst_buffer_unref (outbuf);
GST_ERROR_OBJECT (self, "Failed to render to a 2D texture");
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
sync->released_ts = g_get_monotonic_time ();
g_mutex_unlock (&self->gl_lock);
goto gl_output_error;
}
} else {
GST_ERROR_OBJECT (self, "Wrong memory type for GL output mode");
goto format_error;
}
if (self->on_frame_available) {
if (frame) {
frame->output_buffer = outbuf;
flow_ret =
gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame);
} else {
/* This sometimes happens at EOS or if the input is not properly framed,
* let's handle it gracefully by allocating a new buffer for the current
* caps and filling it
*/
GST_BUFFER_PTS (outbuf) =
gst_util_uint64_scale (buffer_info.presentation_time_us,
GST_USECOND, 1);
flow_ret = gst_pad_push (GST_VIDEO_DECODER_SRC_PAD (self), outbuf);
}
} else {
GST_WARNING_OBJECT (self, "No frame available after "
"%" G_GINT64_FORMAT "ms", timeout / G_TIME_SPAN_MILLISECOND);
if (frame) {
flow_ret =
gst_video_decoder_drop_frame (GST_VIDEO_DECODER (self), frame);
}
/* Render the frame into the surface */
if (!gst_amc_codec_release_output_buffer (self->codec, idx, TRUE, &err)) {
gst_buffer_unref (outbuf);
GST_ERROR_OBJECT (self, "Failed to render buffer, index %d", idx);
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
goto gl_output_error;
}
frame->output_buffer = outbuf;
flow_ret = gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame);
release_buffer = FALSE;
} else if (self->codec_config == AMC_CODEC_CONFIG_WITHOUT_SURFACE && !frame
&& buffer_info.size > 0) {
GstBuffer *outbuf;
@ -1993,10 +2384,6 @@ gst_amc_video_dec_decide_allocation (GstVideoDecoder * bdec, GstQuery * query)
GST_OBJECT_UNLOCK (self->gl_display);
}
#endif
if (self->renderer) {
gst_amc_2d_texture_renderer_free (self->renderer);
self->renderer = NULL;
}
self->downstream_supports_gl = TRUE;
}
@ -2018,10 +2405,11 @@ static void
gst_amc_video_dec_on_frame_available (JNIEnv * env, jobject thiz,
long long context, jobject surfaceTexture)
{
GstAmcVideoDec *dec = JLONG_TO_GST_AMC_VIDEO_DEC (context);
GstAmcVideoDec *self = JLONG_TO_GST_AMC_VIDEO_DEC (context);
g_mutex_lock (&dec->on_frame_available_lock);
dec->on_frame_available = TRUE;
g_cond_signal (&dec->on_frame_available_cond);
g_mutex_unlock (&dec->on_frame_available_lock);
g_mutex_lock (&self->gl_lock);
self->gl_ready_frame_count++;
GST_LOG_OBJECT (self, "frame %u available", self->gl_ready_frame_count);
g_cond_broadcast (&self->gl_cond);
g_mutex_unlock (&self->gl_lock);
}

15
sys/androidmedia/gstamcvideodec.h
View file

@ -28,7 +28,6 @@
#include "gstamc.h"
#include "gstamcsurface.h"
#include "gstamc2dtexturerenderer.h"
G_BEGIN_DECLS
@ -97,14 +96,20 @@ struct _GstAmcVideoDec
GstGLDisplay *gl_display;
GstGLContext *gl_context;
GstGLContext *other_gl_context;
GstAmc2DTextureRenderer *renderer;
gboolean downstream_supports_gl;
GstFlowReturn downstream_flow_ret;
GMutex on_frame_available_lock;
GCond on_frame_available_cond;
gboolean on_frame_available;
gboolean gl_mem_attached;
GstGLMemory *oes_mem;
GError *gl_error;
GMutex gl_lock;
GCond gl_cond;
guint gl_last_rendered_frame;
guint gl_pushed_frame_count; /* n buffers pushed */
guint gl_ready_frame_count; /* n buffers ready for GL access */
guint gl_rendered_frame_count; /* n buffers rendered */
GQueue *gl_queue;
};
struct _GstAmcVideoDecClass