From 6141d3fc7e915e4ab5dea03e10f427bcd95e283f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 29 Oct 2015 13:04:31 +1100 Subject: [PATCH] glcolorconvert: add support for converting texture targets Solved with a simple shader templating mechanism and string replacements of the necessary sampler types/texture accesses and texture coordinate mangling for rectangular and external-oes textures. --- ext/gl/gstglcolorconvertelement.c | 14 +- gst-libs/gst/gl/gstglcolorconvert.c | 906 +++++++++++++++++----------- gst-libs/gst/gl/gstglcolorconvert.h | 8 +- 3 files changed, 573 insertions(+), 355 deletions(-) diff --git a/ext/gl/gstglcolorconvertelement.c b/ext/gl/gstglcolorconvertelement.c index d19ccb75cd..0f52b4c29f 100644 --- a/ext/gl/gstglcolorconvertelement.c +++ b/ext/gl/gstglcolorconvertelement.c @@ -238,18 +238,8 @@ static GstCaps * gst_gl_color_convert_element_fixate_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * othercaps) { + GstGLContext *context = GST_GL_BASE_FILTER (bt)->context; GstCaps *ret; - ret = - GST_BASE_TRANSFORM_CLASS - (gst_gl_color_convert_element_parent_class)->fixate_caps (bt, direction, - caps, othercaps); - - if (direction == GST_PAD_SINK) { - if (gst_caps_is_subset (caps, ret)) { - gst_caps_replace (&ret, caps); - } - } - - return ret; + return gst_gl_color_convert_fixate_caps (context, direction, caps, othercaps); } diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c index 5f3452ec8e..ebdc55bf99 100644 --- a/gst-libs/gst/gl/gstglcolorconvert.c +++ b/gst-libs/gst/gl/gstglcolorconvert.c @@ -110,7 +110,6 @@ static const gfloat from_rgb_bt709_ycoeff[] = { 0.182604, 0.614526, 0.061976 }; static const gfloat from_rgb_bt709_ucoeff[] = { -0.100640, -0.338688, 0.439327 }; static const gfloat from_rgb_bt709_vcoeff[] = { 0.440654, -0.400285, -0.040370 }; - /* GRAY16 to RGB conversion * data transfered as GL_LUMINANCE_ALPHA then convert back to GRAY16 * high byte weight as : 255*256/65535 @@ -120,39 +119,60 @@ static const gfloat from_rgb_bt709_vcoeff[] = { 0.440654, -0.400285, -0.040370 } #define COMPOSE_WEIGHT \ "const vec2 compose_weight = vec2(0.996109, 0.003891);\n" -/* Channel reordering for XYZ <-> ZYX conversion */ -static const gchar frag_REORDER[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D tex;\n" - "uniform vec2 tex_scale0;\n" - "uniform vec2 tex_scale1;\n" - "uniform vec2 tex_scale2;\n" - "void main(void)\n" - "{\n" - " vec4 t = texture2D(tex, v_texcoord * tex_scale0);\n" - " %s\n" /* clobber alpha channel? */ - " gl_FragColor = vec4(t.%c, t.%c, t.%c, t.%c);\n" - "}"; +#define DEFAULT_UNIFORMS \ + "#ifdef GL_ES\n" \ + "precision mediump float;\n" \ + "#endif\n" \ + "uniform vec2 tex_scale0;\n" \ + "uniform vec2 tex_scale1;\n" \ + "uniform vec2 tex_scale2;\n" \ + "uniform float width;\n" \ + "uniform float height;\n" -static const gchar frag_APPLE_YUV_TO_RGB[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform float width;\n" - "uniform float height;\n" - "uniform sampler2DRect tex;\n" - "uniform vec2 tex_scale0;\n" - "uniform vec2 tex_scale1;\n" - "uniform vec2 tex_scale2;\n" - "void main(void)\n" - "{\n" - " vec4 t = texture2DRect(tex, v_texcoord * vec2(width, height) * tex_scale0);\n" - " gl_FragColor = vec4(t.%c, t.%c, t.%c, t.%c);\n" - "}"; +#define MAX_FUNCTIONS 4 + +#define glsl_OES_extension_string "#extension GL_OES_EGL_image_external : require \n" + +struct shader_templ +{ + const gchar *extensions; + const gchar *uniforms; + const gchar *functions[MAX_FUNCTIONS]; + const gchar *body; + + GstGLTextureTarget target; +}; + +#define glsl_func_yuv_to_rgb \ + "vec3 yuv_to_rgb (vec3 val, vec3 offset, vec3 ycoeff, vec3 ucoeff, vec3 vcoeff) {\n" \ + " vec3 rgb;\n" \ + " val += offset;\n" \ + " rgb.r = dot(val, ycoeff);\n" \ + " rgb.g = dot(val, ucoeff);\n" \ + " rgb.b = dot(val, vcoeff);\n" \ + " return rgb;\n" \ + "}\n" + +#define glsl_func_rgb_to_yuv \ + "vec3 rgb_to_yuv (vec3 val, vec3 offset, vec3 rcoeff, vec3 gcoeff, vec3 bcoeff) {\n" \ + " vec3 yuv;\n" \ + " yuv.r = dot(val.rgb, rcoeff);\n" \ + " yuv.g = dot(val.rgb, gcoeff);\n" \ + " yuv.b = dot(val.rgb, bcoeff);\n" \ + " yuv += offset;\n" \ + " return yuv;\n" \ + "}\n" + +/* Channel reordering for XYZ <-> ZYX conversion */ +static const struct shader_templ templ_REORDER = + { NULL, + DEFAULT_UNIFORMS "uniform sampler2D tex;\n", + { NULL, }, + "vec4 t = texture2D(tex, texcoord * tex_scale0);\n" + "%s\n" /* clobber alpha channel? */ + "gl_FragColor = vec4(t.%c, t.%c, t.%c, t.%c);\n", + GST_GL_TEXTURE_TARGET_2D + }; /* GRAY16 to RGB conversion * data transfered as GL_LUMINANCE_ALPHA then convert back to GRAY16 @@ -160,274 +180,185 @@ static const gchar frag_APPLE_YUV_TO_RGB[] = * ([0~1] denormalize to [0~255],shift to high byte,normalize to [0~1]) * low byte weight as : 255/65535 (similar) * */ -static const gchar frag_COMPOSE[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D tex;\n" - "uniform vec2 tex_scale0;\n" - "uniform vec2 tex_scale1;\n" - "uniform vec2 tex_scale2;\n" - COMPOSE_WEIGHT - "void main(void)\n" - "{\n" - " float r, g, b, a;\n" - " vec4 t = texture2D(tex, v_texcoord * tex_scale0);\n" - " r = dot(t.%c%c, compose_weight);" - " g = r;\n" - " b = r;\n" - " a = 1.0;\n" - " gl_FragColor = vec4(%c, %c, %c, %c);\n" - "}"; +static const struct shader_templ templ_COMPOSE = + { NULL, + DEFAULT_UNIFORMS COMPOSE_WEIGHT "uniform sampler2D tex;\n", + { NULL, }, + "vec4 rgba;\n" + "vec4 t = texture2D(tex, texcoord * tex_scale0);\n" + "rgba.rgb = dot(t.%c%c, compose_weight);" + "rgba.a = 1.0;\n" + "gl_FragColor = vec4(rgba.%c, rgba.%c, rgba.%c, rgba.%c);\n", + GST_GL_TEXTURE_TARGET_2D + }; -static const char frag_AYUV_to_RGB[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D tex;\n" - "uniform vec2 tex_scale0;\n" - "uniform vec2 tex_scale1;\n" - "uniform vec2 tex_scale2;\n" - YUV_TO_RGB_COEFFICIENTS - "void main(void) {\n" - " float r,g,b,a;\n" - " vec4 texel;\n" - " texel = texture2D(tex, v_texcoord * tex_scale0);\n" - " texel.gba += offset;\n" - " r = dot(texel.gba, coeff1);\n" - " g = dot(texel.gba, coeff2);\n" - " b = dot(texel.gba, coeff3);\n" - " a = texel.r;\n" - " gl_FragColor=vec4(%c,%c,%c,%c);\n" - "}"; +static const struct shader_templ templ_AYUV_to_RGB = + { NULL, + DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D tex;\n", + { glsl_func_yuv_to_rgb, NULL, }, + "vec4 texel, rgba;\n" + "texel = texture2D(tex, texcoord * tex_scale0);\n" + "rgba.rgb = yuv_to_rgb (texel.yzw, offset, coeff1, coeff2, coeff3);\n" + "rgba.a = texel.r;\n" + "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n", + GST_GL_TEXTURE_TARGET_2D + }; -static const gchar frag_RGB_to_AYUV[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D tex;\n" - RGB_TO_YUV_COEFFICIENTS - "void main(void) {\n" - " vec4 texel;\n" - " float y, u, v, a;\n" - " texel = texture2D(tex, v_texcoord).%c%c%c%c;\n" - " y = dot(texel.rgb, coeff1);\n" - " u = dot(texel.rgb, coeff2);\n" - " v = dot(texel.rgb, coeff3);\n" - " y += offset.x;\n" - " u += offset.y;\n" - " v += offset.z;\n" - " a = %s;\n" - " gl_FragColor = vec4(a,y,u,v);\n" - "}\n"; +static const struct shader_templ templ_RGB_to_AYUV = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n", + { glsl_func_rgb_to_yuv, NULL, }, + "vec4 texel, ayuv;\n" + "texel = texture2D(tex, texcoord).%c%c%c%c;\n" + "ayuv.yzw = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3);\n" + "ayuv.x = %s;\n" + "gl_FragColor = ayuv;\n", + GST_GL_TEXTURE_TARGET_2D + }; /* YUV to RGB conversion */ -static const char frag_PLANAR_YUV_to_RGB[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D Ytex, Utex, Vtex;\n" - "uniform vec2 tex_scale0;\n" - "uniform vec2 tex_scale1;\n" - "uniform vec2 tex_scale2;\n" - YUV_TO_RGB_COEFFICIENTS - "void main(void) {\n" - " float r, g, b, a;\n" - " vec3 yuv;\n" - " yuv.x = texture2D(Ytex,v_texcoord * tex_scale0).r;\n" - " yuv.y = texture2D(Utex,v_texcoord * tex_scale1).r;\n" - " yuv.z = texture2D(Vtex,v_texcoord * tex_scale2).r;\n" - " yuv += offset;\n" - " r = dot(yuv, coeff1);\n" - " g = dot(yuv, coeff2);\n" - " b = dot(yuv, coeff3);\n" - " a = 1.0;\n" - " gl_FragColor = vec4(%c, %c, %c, %c);\n" - "}\n"; +static const struct shader_templ templ_PLANAR_YUV_to_RGB = + { NULL, + DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex;\n", + { glsl_func_yuv_to_rgb, NULL, }, + "vec4 texel, rgba;\n" + /* FIXME: should get the sampling right... */ + "texel.x = texture2D(Ytex, texcoord * tex_scale0).r;\n" + "texel.y = texture2D(Utex, texcoord * tex_scale1).r;\n" + "texel.z = texture2D(Vtex, texcoord * tex_scale2).r;\n" + "rgba.rgb = yuv_to_rgb (texel.xyz, offset, coeff1, coeff2, coeff3);\n" + "rgba.a = 1.0;\n" + "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n", + GST_GL_TEXTURE_TARGET_2D + }; -static const gchar frag_RGB_to_PLANAR_YUV[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D tex;\n" - "uniform float width;\n" - "uniform float height;\n" - "uniform vec2 chroma_sampling;\n" - RGB_TO_YUV_COEFFICIENTS - "void main(void) {\n" - " float y, u, v;\n" - " vec4 uv_texel;\n" - " vec4 texel = texture2D(tex, v_texcoord).%c%c%c%c;\n" +static const struct shader_templ templ_RGB_to_PLANAR_YUV = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n" + "uniform vec2 chroma_sampling;\n", + { glsl_func_rgb_to_yuv, NULL, }, + "vec4 texel;\n" + "vec3 yuv;\n" + "texel = texture2D(tex, texcoord).%c%c%c%c;\n" + /* FIXME: this is not quite correct yet */ + "vec4 uv_texel = vec4(0.0);\n" /* One u and v sample can be generated by a nxm sized block given by * @chroma_sampling. The result is the average of all the values in the * block computed with a rolling average. */ - " vec2 size = vec2(width, height);\n" - " vec2 pos = v_texcoord * size;\n" - /* scale for chroma size */ - " vec2 chroma_pos = v_texcoord * chroma_sampling * size;\n" - /* offset chroma to the center of the first texel in the block */ - " chroma_pos -= clamp(chroma_sampling * 0.5 - 0.5, vec2(0.0), chroma_sampling);\n" - " if (chroma_pos.x < width && chroma_pos.y < height) {\n" - " for (int i = 0; i < int(chroma_sampling.x); i++) {\n" - " vec2 delta = vec2 (float(i), 0.0);\n" - " for (int j = 0; j < int(chroma_sampling.y); j++) {\n" - " int n = (i+1)*(j+1);\n" - " delta.y = float(j);\n" - " vec4 sample = texture2D(tex, (chroma_pos + delta) / size).%c%c%c%c;\n" - /* rolling average */ - " uv_texel = (float(n-1) * uv_texel + sample) / float(n);\n" - " }\n" + "vec2 size = vec2(width, height);\n" + "vec2 pos = texcoord * size;\n" + /* scale for chroma size */ + "vec2 chroma_pos = texcoord * chroma_sampling * size;\n" + /* offset chroma to the center of the first texel in the block */ + "chroma_pos -= clamp(chroma_sampling * 0.5 - 0.5, vec2(0.0), chroma_sampling);\n" + "if (chroma_pos.x < width && chroma_pos.y < height) {\n" + " for (int i = 0; i < int(chroma_sampling.x); i++) {\n" + " vec2 delta = vec2 (float(i), 0.0);\n" + " for (int j = 0; j < int(chroma_sampling.y); j++) {\n" + " int n = (i+1)*(j+1);\n" + " delta.y = float(j);\n" + " vec4 sample = texture2D(tex, (chroma_pos + delta) / size).%c%c%c%c;\n" + /* rolling average */ + " uv_texel = (float(n-1) * uv_texel + sample) / float(n);\n" " }\n" " }\n" - " y = dot(texel.rgb, coeff1);\n" - " u = dot(uv_texel.rgb, coeff2);\n" - " v = dot(uv_texel.rgb, coeff3);\n" - " y += offset.x;\n" - " u += offset.y;\n" - " v += offset.z;\n" - " gl_FragData[0] = vec4(y, 0.0, 0.0, 1.0);\n" - " gl_FragData[1] = vec4(u, 0.0, 0.0, 1.0);\n" - " gl_FragData[2] = vec4(v, 0.0, 0.0, 1.0);\n" - "}\n"; + "}\n" + + "yuv.x = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3).x;\n" + "yuv.yz = rgb_to_yuv (uv_texel.rgb, offset, coeff1, coeff2, coeff3).yz;\n" + "gl_FragData[0] = vec4(yuv.x, 0.0, 0.0, 1.0);\n" + "gl_FragData[1] = vec4(yuv.y, 0.0, 0.0, 1.0);\n" + "gl_FragData[2] = vec4(yuv.z, 0.0, 0.0, 1.0);\n", + GST_GL_TEXTURE_TARGET_2D + }; /* NV12/NV21 to RGB conversion */ -static const char frag_NV12_NV21_to_RGB[] = { - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D Ytex,UVtex;\n" - "uniform vec2 tex_scale0;\n" - "uniform vec2 tex_scale1;\n" - "uniform vec2 tex_scale2;\n" - YUV_TO_RGB_COEFFICIENTS - "void main(void) {\n" - " float r, g, b, a;\n" - " vec3 yuv;\n" - " yuv.x=texture2D(Ytex, v_texcoord * tex_scale0).r;\n" - " yuv.yz=texture2D(UVtex, v_texcoord * tex_scale1).%c%c;\n" - " yuv += offset;\n" - " r = dot(yuv, coeff1);\n" - " g = dot(yuv, coeff2);\n" - " b = dot(yuv, coeff3);\n" - " a = 1.0;\n" - " gl_FragColor=vec4(%c, %c, %c, %c);\n" - "}" -}; +static const struct shader_templ templ_NV12_NV21_to_RGB = + { NULL, + DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, UVtex;\n", + { glsl_func_yuv_to_rgb, NULL, }, + "vec4 rgba;\n" + "vec3 yuv;\n" + /* FIXME: should get the sampling right... */ + "yuv.x=texture2D(Ytex, texcoord * tex_scale0).r;\n" + "yuv.yz=texture2D(UVtex, texcoord * tex_scale1).%c%c;\n" + "rgba.rgb = yuv_to_rgb (yuv, offset, coeff1, coeff2, coeff3);\n" + "rgba.a = 1.0;\n" + "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n", + GST_GL_TEXTURE_TARGET_2D + }; /* RGB to NV12/NV21 conversion */ /* NV12: u, v NV21: v, u */ -static const char frag_RGB_to_NV12_NV21[] = { - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D tex;\n" - "uniform vec2 tex_scale0;\n" - "uniform vec2 tex_scale1;\n" - "uniform vec2 tex_scale2;\n" - RGB_TO_YUV_COEFFICIENTS - "void main(void) {\n" - " float y, u, v;\n" - " vec4 texel, uv_texel;\n" - " texel = texture2D(tex, v_texcoord * tex_scale0).%c%c%c%c;\n" - " uv_texel = texture2D(tex, v_texcoord * tex_scale0 * 2.0).%c%c%c%c;\n" - " y = dot(texel.rgb, coeff1);\n" - " u = dot(uv_texel.rgb, coeff2);\n" - " v = dot(uv_texel.rgb, coeff3);\n" - " y += offset.x;\n" - " u += offset.y;\n" - " v += offset.z;\n" - " gl_FragData[0] = vec4(y, 0.0, 0.0, 1.0);\n" - " gl_FragData[1] = vec4(%c, %c, 0.0, 1.0);\n" - "}" -}; +static const struct shader_templ templ_RGB_to_NV12_NV21 = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n", + { glsl_func_rgb_to_yuv, NULL, }, + "vec4 texel, uv_texel;\n" + "vec3 yuv;\n" + "texel = texture2D(tex, texcoord).%c%c%c%c;\n" + "uv_texel = texture2D(tex, texcoord * tex_scale0 * 2.0).%c%c%c%c;\n" + "yuv.x = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3).x;\n" + "yuv.yz = rgb_to_yuv (uv_texel.rgb, offset, coeff1, coeff2, coeff3).yz;\n" + "gl_FragData[0] = vec4(yuv.x, 0.0, 0.0, 1.0);\n" + "gl_FragData[1] = vec4(yuv.%c, yuv.%c, 0.0, 1.0);\n", + GST_GL_TEXTURE_TARGET_2D + }; /* YUY2:r,g,a UYVY:a,b,r */ -static const gchar frag_YUY2_UYVY_to_RGB[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D Ytex, UVtex;\n" - "uniform vec2 tex_scale0;\n" - "uniform vec2 tex_scale1;\n" - "uniform vec2 tex_scale2;\n" - "uniform float width;\n" - YUV_TO_RGB_COEFFICIENTS - "void main(void) {\n" - " vec3 yuv;\n" - " vec4 uv_texel;\n" - " float r, g, b, a;\n" - " float dx1 = -1.0 / width;\n" - " float dx2 = 0.0;\n" - " yuv.x = texture2D(Ytex, v_texcoord * tex_scale0).%c;\n" - " float inorder = mod (v_texcoord.x * width, 2.0);\n" - " if (inorder < 1.0) {\n" - " dx2 = -dx1;\n" - " dx1 = 0.0;\n" - " }\n" - " uv_texel.rg = texture2D(Ytex, v_texcoord * tex_scale0 + dx1).r%c;\n" - " uv_texel.ba = texture2D(Ytex, v_texcoord * tex_scale0 + dx2).r%c;\n" - " yuv.yz = uv_texel.%c%c;\n" - " yuv += offset;\n" - " r = dot(yuv, coeff1);\n" - " g = dot(yuv, coeff2);\n" - " b = dot(yuv, coeff3);\n" - " a = 1.0;\n" - " gl_FragColor = vec4(%c, %c, %c, %c);\n" - "}\n"; +static const struct shader_templ templ_YUY2_UYVY_to_RGB = + { NULL, + DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex;\n", + { glsl_func_yuv_to_rgb, NULL, }, + "vec4 rgba, uv_texel;\n" + "vec3 yuv;\n" + /* FIXME: should get the sampling right... */ + "float dx1 = -1.0 / width;\n" + "float dx2 = 0.0;\n" + "yuv.x = texture2D(Ytex, texcoord * tex_scale0).%c;\n" + "float inorder = mod (texcoord.x * width, 2.0);\n" + "if (inorder < 1.0) {\n" + " dx2 = -dx1;\n" + " dx1 = 0.0;\n" + "}\n" + "uv_texel.rg = texture2D(Ytex, texcoord * tex_scale0 + dx1).r%c;\n" + "uv_texel.ba = texture2D(Ytex, texcoord * tex_scale0 + dx2).r%c;\n" + "yuv.yz = uv_texel.%c%c;\n" + "rgba.rgb = yuv_to_rgb (yuv, offset, coeff1, coeff2, coeff3);\n" + "rgba.a = 1.0;\n" + "gl_FragColor = vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n", + GST_GL_TEXTURE_TARGET_2D + }; -static const gchar frag_RGB_to_YUY2_UYVY[] = - "#ifdef GL_ES\n" - "precision mediump float;\n" - "#endif\n" - "varying vec2 v_texcoord;\n" - "uniform sampler2D tex;\n" - "uniform float width;\n" - RGB_TO_YUV_COEFFICIENTS - "void main(void) {\n" - " vec4 texel1, texel2;\n" - " vec2 texel3;\n" - " float fx, dx, fy, y, u, u1, u2, v, v1, v2;\n" - " float inorder = mod (v_texcoord.x * width, 2.0);\n" - " fx = v_texcoord.x;\n" - " dx = 1.0 / width;\n" - " if (v_texcoord.x >= (1.0 - 0.5 * dx) || (v_texcoord.x > 0.5 * dx && inorder < 1.0)) {\n" - " dx = -dx;\n" - " }\n" - " fy = v_texcoord.y;\n" - " texel1 = texture2D(tex, vec2(fx, fy)).%c%c%c%c;\n" - " texel2 = texture2D(tex, vec2(fx + dx, fy)).%c%c%c%c;\n" - " y = dot(texel1.rgb, coeff1);\n" - " u1 = dot(texel1.rgb, coeff2);\n" - " u2 = dot(texel2.rgb, coeff2);\n" - " v1 = dot(texel1.rgb, coeff3);\n" - " v2 = dot(texel2.rgb, coeff3);\n" - " y += offset.x;\n" - " u1 += offset.y;\n" - " u2 += offset.y;\n" - " v1 += offset.z;\n" - " v2 += offset.z;\n" - " u = (u1 + u2) / 2.0;\n" - " v = (v1 + v2) / 2.0;\n" - " if (inorder < 1.0) {\n" - " texel3.r =%s;\n" - " texel3.g = %s;\n" - " } else {\n" - " texel3.r =%s;\n" - " texel3.g = %s;\n" - " }\n" - " gl_FragColor = vec4(texel3.r, texel3.g, 0.0, 0.0);\n" - "}\n"; +static const struct shader_templ templ_RGB_to_YUY2_UYVY = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n", + { glsl_func_rgb_to_yuv, NULL, }, + "vec4 texel1, texel2;\n" + "vec3 yuv, yuv1, yuv2;\n" + "float fx, dx, fy;\n" + "float inorder = mod (texcoord.x * width, 2.0);\n" + "fx = texcoord.x;\n" + "dx = 1.0 / width;\n" + "if (texcoord.x >= (1.0 - 0.5 * dx) || (texcoord.x > 0.5 * dx && inorder < 1.0)) {\n" + " dx = -dx;\n" + "}\n" + "fy = texcoord.y;\n" + "texel1 = texture2D(tex, vec2(fx, fy)).%c%c%c%c;\n" + "texel2 = texture2D(tex, vec2(fx + dx, fy)).%c%c%c%c;\n" + "yuv1 = rgb_to_yuv (texel1.rgb, offset, coeff1, coeff2, coeff3);" + "yuv2 = rgb_to_yuv (texel2.rgb, offset, coeff1, coeff2, coeff3);" + "yuv = (yuv1 + yuv2) / 2.0;\n" + "if (inorder < 1.0) {\n" + " gl_FragColor = vec4(yuv.%c, yuv.%c, 0.0, 0.0);\n" + "} else {\n" + " gl_FragColor = vec4(yuv.%c, yuv.%c, 0.0, 0.0);\n" + "}\n", + GST_GL_TEXTURE_TARGET_2D + }; static const gchar text_vertex_shader[] = "attribute vec4 a_position; \n" @@ -435,8 +366,8 @@ static const gchar text_vertex_shader[] = "varying vec2 v_texcoord; \n" "void main() \n" "{ \n" - " gl_Position = a_position; \n" - " v_texcoord = a_texcoord; \n" + " gl_Position = a_position; \n" + " v_texcoord = a_texcoord; \n" "} \n"; static const GLfloat vertices[] = { @@ -454,6 +385,8 @@ struct ConvertInfo { gint in_n_textures; gint out_n_textures; + const struct shader_templ *templ; + gchar *frag_body; gchar *frag_prog; const gchar *shader_tex_names[GST_VIDEO_MAX_PLANES]; gfloat *cms_offset; @@ -469,6 +402,9 @@ struct _GstGLColorConvertPrivate struct ConvertInfo convert_info; + GstGLTextureTarget from_texture_target; + GstGLTextureTarget to_texture_target; + GstGLMemory *in_tex[GST_VIDEO_MAX_PLANES]; GstGLMemory *out_tex[GST_VIDEO_MAX_PLANES]; @@ -605,7 +541,8 @@ gst_gl_color_convert_reset (GstGLColorConvert * convert) } static gboolean -_gst_gl_color_convert_can_passthrough (GstVideoInfo * in, GstVideoInfo * out) +_gst_gl_color_convert_can_passthrough_info (GstVideoInfo * in, + GstVideoInfo * out) { gint i; @@ -639,6 +576,8 @@ _gst_gl_color_convert_set_caps_unlocked (GstGLColorConvert * convert, { GstVideoInfo in_info, out_info; GstCapsFeatures *in_features, *out_features; + GstGLTextureTarget from_target, to_target; + gboolean passthrough; g_return_val_if_fail (convert != NULL, FALSE); g_return_val_if_fail (in_caps, FALSE); @@ -672,18 +611,53 @@ _gst_gl_color_convert_set_caps_unlocked (GstGLColorConvert * convert, return FALSE; } + { + GstStructure *in_s = gst_caps_get_structure (in_caps, 0); + GstStructure *out_s = gst_caps_get_structure (out_caps, 0); + + if (gst_structure_has_field_typed (in_s, "texture-target", G_TYPE_STRING)) + from_target = + gst_gl_texture_target_from_string (gst_structure_get_string (in_s, + "texture-target")); + else + from_target = GST_GL_TEXTURE_TARGET_2D; + + if (gst_structure_has_field_typed (out_s, "texture-target", G_TYPE_STRING)) + to_target = + gst_gl_texture_target_from_string (gst_structure_get_string (out_s, + "texture-target")); + else + to_target = GST_GL_TEXTURE_TARGET_2D; + + if (to_target == GST_GL_TEXTURE_TARGET_NONE + || from_target == GST_GL_TEXTURE_TARGET_NONE) + /* invalid caps */ + return FALSE; + } + if (gst_video_info_is_equal (&convert->in_info, &in_info) && - gst_video_info_is_equal (&convert->out_info, &out_info)) + gst_video_info_is_equal (&convert->out_info, &out_info) && + convert->priv->from_texture_target == from_target && + convert->priv->to_texture_target == to_target) return TRUE; + /* If input and output are identical, pass through directly */ + passthrough = + _gst_gl_color_convert_can_passthrough_info (&in_info, &out_info) && + from_target == to_target; + + if (!passthrough && to_target != GST_GL_TEXTURE_TARGET_2D + && to_target != GST_GL_TEXTURE_TARGET_RECTANGLE) + return FALSE; + gst_gl_color_convert_reset (convert); convert->in_info = in_info; convert->out_info = out_info; + convert->priv->from_texture_target = from_target; + convert->priv->to_texture_target = to_target; convert->initted = FALSE; - /* If input and output are identical, pass through directly */ - convert->passthrough = - _gst_gl_color_convert_can_passthrough (&in_info, &out_info); + convert->passthrough = passthrough; #ifndef GST_DISABLE_GST_DEBUG if (G_UNLIKELY (convert->passthrough)) GST_DEBUG_OBJECT (convert, @@ -718,6 +692,41 @@ gst_gl_color_convert_set_caps (GstGLColorConvert * convert, return ret; } +static guint +_get_target_bitmask_from_g_value (const GValue * targets) +{ + guint new_targets = 0; + + if (targets == NULL) { + new_targets = 1 << GST_GL_TEXTURE_TARGET_2D; + } else if (G_TYPE_CHECK_VALUE_TYPE (targets, G_TYPE_STRING)) { + GstGLTextureTarget target; + const gchar *str; + + str = g_value_get_string (targets); + target = gst_gl_texture_target_from_string (str); + + if (target) + new_targets |= 1 << target; + } else if (G_TYPE_CHECK_VALUE_TYPE (targets, GST_TYPE_LIST)) { + gint j, m; + + m = gst_value_list_get_size (targets); + for (j = 0; j < m; j++) { + const GValue *val = gst_value_list_get_value (targets, j); + GstGLTextureTarget target; + const gchar *str; + + str = g_value_get_string (val); + target = gst_gl_texture_target_from_string (str); + if (target) + new_targets |= 1 << target; + } + } + + return new_targets; +} + /* copies the given caps */ static GstCaps * gst_gl_color_convert_caps_remove_format_info (GstCaps * caps) @@ -741,10 +750,9 @@ gst_gl_color_convert_caps_remove_format_info (GstCaps * caps) st = gst_structure_copy (st); gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", - NULL); + "texture-target", NULL); - gst_caps_append_structure_full (res, st, - gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL)); + gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); } return res; @@ -756,9 +764,7 @@ gst_gl_color_convert_transform_caps (GstGLContext * convert, { GstCaps *templ, *result; - templ = - gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, GST_GL_COLOR_CONVERT_FORMATS)); + templ = gst_caps_from_string (GST_GL_COLOR_CONVERT_VIDEO_CAPS); caps = gst_gl_color_convert_caps_remove_format_info (caps); @@ -766,8 +772,6 @@ gst_gl_color_convert_transform_caps (GstGLContext * convert, gst_caps_unref (caps); gst_caps_unref (templ); - result = gst_gl_overlay_compositor_add_caps (result); - if (filter) { GstCaps *tmp; @@ -779,6 +783,59 @@ gst_gl_color_convert_transform_caps (GstGLContext * convert, return result; } +GstCaps * +gst_gl_color_convert_fixate_caps (GstGLContext * convert, + GstPadDirection direction, GstCaps * caps, GstCaps * other) +{ + GValue item = G_VALUE_INIT; + const GValue *targets, *other_targets; + guint targets_mask = 0, other_targets_mask = 0, result_mask; + GstStructure *s; + + other = gst_caps_make_writable (other); + s = gst_caps_get_structure (other, 0); + + other_targets = gst_structure_get_value (s, "texture-target"); + targets = gst_structure_get_value (s, "texture-target"); + + targets_mask = _get_target_bitmask_from_g_value (targets); + other_targets_mask = _get_target_bitmask_from_g_value (other_targets); + + result_mask = targets_mask & other_targets_mask; + if (result_mask == 0) { + /* nothing we can do here */ + return gst_caps_fixate (other); + } + + if (direction == GST_PAD_SINK) { + result_mask &= + (1 << GST_GL_TEXTURE_TARGET_2D | 1 << GST_GL_TEXTURE_TARGET_RECTANGLE); + } else { + /* if the src caps has 2D support we can 'convert' to anything */ + if (targets_mask & (1 << GST_GL_TEXTURE_TARGET_2D)) + result_mask = -1; + else + result_mask = other_targets_mask; + } + + g_value_init (&item, G_TYPE_STRING); + if (result_mask & (1 << GST_GL_TEXTURE_TARGET_2D)) { + g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_2D_STR); + } else if (result_mask & (1 << GST_GL_TEXTURE_TARGET_RECTANGLE)) { + g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_RECTANGLE_STR); + } else if (result_mask & (1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES)) { + g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_EXTERNAL_OES_STR); + } + + gst_structure_set_value (s, "texture-target", &item); + + g_value_unset (&item); + + other = gst_caps_fixate (other); + + return other; +} + /** * gst_gl_color_convert_perform: * @convert: a #GstGLColorConvert @@ -962,7 +1019,8 @@ _RGB_to_RGB (GstGLColorConvert * convert) } alpha = g_strdup_printf ("t.%c = 1.0;", input_alpha_channel); } - info->frag_prog = g_strdup_printf (frag_REORDER, alpha ? alpha : "", + info->templ = &templ_REORDER; + info->frag_body = g_strdup_printf (info->templ->body, alpha ? alpha : "", pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->shader_tex_names[0] = "tex"; @@ -993,7 +1051,7 @@ _YUV_to_RGB (GstGLColorConvert * convert) if (gst_is_gl_memory (memory) && (USING_OPENGL (convert->context) || USING_OPENGL3 (convert->context))) { in_tex_rectangular = - ((GstGLMemory *) memory)->tex_target == GL_TEXTURE_RECTANGLE; + convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE; } #endif @@ -1005,15 +1063,19 @@ _YUV_to_RGB (GstGLColorConvert * convert) * the Apple YCbCr422 extension. It could also be a normal UYVY texture * with RB or Lum/Alpha */ - info->frag_prog = - g_strdup_printf (frag_APPLE_YUV_TO_RGB, pixel_order[0], pixel_order[1], + /* The mangling will change this to the correct texture2DRect, sampler2DRect + * for us */ + info->templ = &templ_REORDER; + info->frag_body = + g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->in_n_textures = 1; info->shader_tex_names[0] = "tex"; } else { switch (GST_VIDEO_INFO_FORMAT (&convert->in_info)) { case GST_VIDEO_FORMAT_AYUV: - info->frag_prog = g_strdup_printf (frag_AYUV_to_RGB, pixel_order[0], + info->templ = &templ_AYUV_to_RGB; + info->frag_body = g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->in_n_textures = 1; info->shader_tex_names[0] = "tex"; @@ -1022,8 +1084,9 @@ _YUV_to_RGB (GstGLColorConvert * convert) case GST_VIDEO_FORMAT_Y444: case GST_VIDEO_FORMAT_Y42B: case GST_VIDEO_FORMAT_Y41B: - info->frag_prog = - g_strdup_printf (frag_PLANAR_YUV_to_RGB, pixel_order[0], + info->templ = &templ_PLANAR_YUV_to_RGB; + info->frag_body = + g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->in_n_textures = 3; info->shader_tex_names[0] = "Ytex"; @@ -1031,8 +1094,9 @@ _YUV_to_RGB (GstGLColorConvert * convert) info->shader_tex_names[2] = "Vtex"; break; case GST_VIDEO_FORMAT_YV12: - info->frag_prog = - g_strdup_printf (frag_PLANAR_YUV_to_RGB, pixel_order[0], + info->templ = &templ_PLANAR_YUV_to_RGB; + info->frag_body = + g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->in_n_textures = 3; info->shader_tex_names[0] = "Ytex"; @@ -1042,17 +1106,30 @@ _YUV_to_RGB (GstGLColorConvert * convert) case GST_VIDEO_FORMAT_YUY2: { char uv_val = texture_rg ? 'g' : 'a'; - info->frag_prog = g_strdup_printf (frag_YUY2_UYVY_to_RGB, 'r', uv_val, + info->templ = &templ_YUY2_UYVY_to_RGB; + info->frag_body = g_strdup_printf (info->templ->body, 'r', uv_val, uv_val, 'g', 'a', pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->in_n_textures = 1; info->shader_tex_names[0] = "Ytex"; break; } + case GST_VIDEO_FORMAT_UYVY: + { + char y_val = texture_rg ? 'g' : 'a'; + info->templ = &templ_YUY2_UYVY_to_RGB; + info->frag_body = g_strdup_printf (info->templ->body, y_val, 'g', + 'g', 'r', 'b', pixel_order[0], pixel_order[1], pixel_order[2], + pixel_order[3]); + info->in_n_textures = 1; + info->shader_tex_names[0] = "Ytex"; + break; + } case GST_VIDEO_FORMAT_NV12: { char val2 = texture_rg ? 'g' : 'a'; - info->frag_prog = g_strdup_printf (frag_NV12_NV21_to_RGB, 'r', val2, + info->templ = &templ_NV12_NV21_to_RGB; + info->frag_body = g_strdup_printf (info->templ->body, 'r', val2, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->in_n_textures = 2; info->shader_tex_names[0] = "Ytex"; @@ -1062,23 +1139,14 @@ _YUV_to_RGB (GstGLColorConvert * convert) case GST_VIDEO_FORMAT_NV21: { char val2 = texture_rg ? 'g' : 'a'; - info->frag_prog = g_strdup_printf (frag_NV12_NV21_to_RGB, val2, 'r', + info->templ = &templ_NV12_NV21_to_RGB; + info->frag_body = g_strdup_printf (info->templ->body, val2, 'r', pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->in_n_textures = 2; info->shader_tex_names[0] = "Ytex"; info->shader_tex_names[1] = "UVtex"; break; } - case GST_VIDEO_FORMAT_UYVY: - { - char y_val = texture_rg ? 'g' : 'a'; - info->frag_prog = g_strdup_printf (frag_YUY2_UYVY_to_RGB, y_val, 'g', - 'g', 'r', 'b', pixel_order[0], pixel_order[1], pixel_order[2], - pixel_order[3]); - info->in_n_textures = 1; - info->shader_tex_names[0] = "Ytex"; - break; - } default: break; } @@ -1119,7 +1187,8 @@ _RGB_to_YUV (GstGLColorConvert * convert) switch (out_format) { case GST_VIDEO_FORMAT_AYUV: alpha = _is_RGBx (in_format) ? "1.0" : "texel.a"; - info->frag_prog = g_strdup_printf (frag_RGB_to_AYUV, pixel_order[0], + info->templ = &templ_RGB_to_AYUV; + info->frag_body = g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], alpha); info->out_n_textures = 1; break; @@ -1128,7 +1197,8 @@ _RGB_to_YUV (GstGLColorConvert * convert) case GST_VIDEO_FORMAT_Y444: case GST_VIDEO_FORMAT_Y42B: case GST_VIDEO_FORMAT_Y41B: - info->frag_prog = g_strdup_printf (frag_RGB_to_PLANAR_YUV, + info->templ = &templ_RGB_to_PLANAR_YUV; + info->frag_body = g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); info->out_n_textures = 3; @@ -1145,31 +1215,35 @@ _RGB_to_YUV (GstGLColorConvert * convert) } break; case GST_VIDEO_FORMAT_YUY2: - info->frag_prog = g_strdup_printf (frag_RGB_to_YUY2_UYVY, + info->templ = &templ_RGB_to_YUY2_UYVY; + info->frag_body = g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], - "y", "u", "y", "v"); + 'x', 'y', 'x', 'z'); info->out_n_textures = 1; break; case GST_VIDEO_FORMAT_UYVY: - info->frag_prog = g_strdup_printf (frag_RGB_to_YUY2_UYVY, + info->templ = &templ_RGB_to_YUY2_UYVY, + info->frag_body = g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], - "u", "y", "v", "y"); + 'y', 'x', 'z', 'x'); info->out_n_textures = 1; break; case GST_VIDEO_FORMAT_NV12: - info->frag_prog = g_strdup_printf (frag_RGB_to_NV12_NV21, + info->templ = &templ_RGB_to_NV12_NV21, + info->frag_body = g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], - 'u', 'v'); + 'y', 'z'); info->out_n_textures = 2; break; case GST_VIDEO_FORMAT_NV21: - info->frag_prog = g_strdup_printf (frag_RGB_to_NV12_NV21, + info->templ = &templ_RGB_to_NV12_NV21, + info->frag_body = g_strdup_printf (info->templ->body, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], - 'v', 'u'); + 'z', 'y'); info->out_n_textures = 2; break; default: @@ -1211,7 +1285,8 @@ _RGB_to_GRAY (GstGLColorConvert * convert) switch (GST_VIDEO_INFO_FORMAT (&convert->out_info)) { case GST_VIDEO_FORMAT_GRAY8: - info->frag_prog = g_strdup_printf (frag_REORDER, alpha ? alpha : "", + info->templ = &templ_REORDER; + info->frag_body = g_strdup_printf (info->templ->body, alpha ? alpha : "", pixel_order[0], pixel_order[0], pixel_order[0], pixel_order[3]); break; default: @@ -1243,20 +1318,23 @@ _GRAY_to_RGB (GstGLColorConvert * convert) switch (GST_VIDEO_INFO_FORMAT (&convert->in_info)) { case GST_VIDEO_FORMAT_GRAY8: - info->frag_prog = g_strdup_printf (frag_REORDER, "", pixel_order[0], + info->templ = &templ_REORDER; + info->frag_body = g_strdup_printf (info->templ->body, "", pixel_order[0], pixel_order[0], pixel_order[0], pixel_order[3]); break; case GST_VIDEO_FORMAT_GRAY16_LE: { char val2 = texture_rg ? 'g' : 'a'; - info->frag_prog = g_strdup_printf (frag_COMPOSE, val2, 'r', + info->templ = &templ_COMPOSE; + info->frag_body = g_strdup_printf (info->templ->body, val2, 'r', pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); break; } case GST_VIDEO_FORMAT_GRAY16_BE: { char val2 = texture_rg ? 'g' : 'a'; - info->frag_prog = g_strdup_printf (frag_COMPOSE, 'r', val2, + info->templ = &templ_COMPOSE; + info->frag_body = g_strdup_printf (info->templ->body, 'r', val2, pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); break; } @@ -1299,6 +1377,88 @@ _unbind_buffer (GstGLColorConvert * convert) gl->DisableVertexAttribArray (convert->priv->attr_texture); } +static gchar * +_mangle_texture_access (const gchar * str, GstGLTextureTarget from, + GstGLTextureTarget to) +{ + const gchar *from_str = NULL, *to_str = NULL; + gchar *ret, *tmp; + gchar *regex_find; + GRegex *regex; + + if (from == GST_GL_TEXTURE_TARGET_2D) + from_str = "texture2D"; + if (from == GST_GL_TEXTURE_TARGET_RECTANGLE) + from_str = "texture2DRect"; + if (from == GST_GL_TEXTURE_TARGET_OES) + from_str = "texture2D"; + + if (to == GST_GL_TEXTURE_TARGET_2D) + to_str = "texture2D"; + if (to == GST_GL_TEXTURE_TARGET_RECTANGLE) + to_str = "texture2DRect"; + if (to == GST_GL_TEXTURE_TARGET_OES) + to_str = "texture2D"; + + /* followed by any amount of whitespace then a bracket */ + regex_find = g_strdup_printf ("%s(?=\\s*\\()", from_str); + regex = g_regex_new (regex_find, 0, 0, NULL); + tmp = g_regex_replace_literal (regex, str, -1, 0, to_str, 0, NULL); + g_free (regex_find); + g_regex_unref (regex); + + if (tmp) { + ret = tmp; + } else { + GST_FIXME ("Couldn't mangle texture access successfully from %s to %s", + from_str, to_str); + ret = g_strdup (str); + } + + return ret; +} + +static gchar * +_mangle_sampler_type (const gchar * str, GstGLTextureTarget from, + GstGLTextureTarget to) +{ + const gchar *from_str = NULL, *to_str = NULL; + gchar *ret, *tmp; + gchar *regex_find; + GRegex *regex; + + if (from == GST_GL_TEXTURE_TARGET_2D) + from_str = "sampler2D"; + if (from == GST_GL_TEXTURE_TARGET_RECTANGLE) + from_str = "sampler2DRect"; + if (from == GST_GL_TEXTURE_TARGET_OES) + from_str = "samplerExternalOES"; + + if (to == GST_GL_TEXTURE_TARGET_2D) + to_str = "sampler2D"; + if (to == GST_GL_TEXTURE_TARGET_RECTANGLE) + to_str = "sampler2DRect"; + if (to == GST_GL_TEXTURE_TARGET_OES) + to_str = "samplerExternalOES"; + + /* followed by some whitespace */ + regex_find = g_strdup_printf ("%s(?=\\s)", from_str); + regex = g_regex_new (regex_find, 0, 0, NULL); + tmp = g_regex_replace_literal (regex, str, -1, 0, to_str, 0, NULL); + g_free (regex_find); + g_regex_unref (regex); + + if (tmp) { + ret = tmp; + } else { + GST_FIXME ("Couldn't mangle sampler type successfully from %s to %s", + from_str, to_str); + ret = g_strdup (str); + } + + return ret; +} + /* Called in the gl thread */ static gboolean _init_convert (GstGLColorConvert * convert) @@ -1353,7 +1513,68 @@ _init_convert (GstGLColorConvert * convert) } } - if (!info->frag_prog || info->in_n_textures == 0 || info->out_n_textures == 0) + if (!info->frag_body || info->in_n_textures == 0 || info->out_n_textures == 0) + goto unhandled_format; + + /* XXX: poor mans shader templating */ + { + GString *str = g_string_new (NULL); + int i; + + if (info->templ->extensions) + g_string_append (str, info->templ->extensions); + + if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_OES + && info->templ->target != GST_GL_TEXTURE_TARGET_OES) + g_string_append (str, glsl_OES_extension_string); + + if (info->templ->uniforms) { + gchar *uniforms = + _mangle_sampler_type (info->templ->uniforms, info->templ->target, + convert->priv->from_texture_target); + g_string_append (str, uniforms); + g_free (uniforms); + } + + for (i = 0; i < MAX_FUNCTIONS; i++) { + gchar *function; + + if (info->templ->functions[i] == NULL) + break; + + function = + _mangle_texture_access (info->templ->functions[i], + info->templ->target, convert->priv->from_texture_target); + g_string_append_c (str, '\n'); + g_string_append (str, function); + g_string_append_c (str, '\n'); + g_free (function); + } + + g_string_append (str, "\nvarying vec2 v_texcoord;\nvoid main (void) {\n"); + if (info->frag_body) { + gchar *body; + + g_string_append (str, "vec2 texcoord;\n"); + if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE + && info->templ->target != GST_GL_TEXTURE_TARGET_RECTANGLE) { + g_string_append (str, + "texcoord = v_texcoord * vec2 (width, height);\n"); + } else { + g_string_append (str, "texcoord = v_texcoord;\n"); + } + + body = + _mangle_texture_access (info->frag_body, info->templ->target, + convert->priv->from_texture_target); + g_string_append (str, body); + g_free (body); + } + g_string_append (str, "\n}"); + info->frag_prog = g_string_free (str, FALSE); + } + + if (!info->frag_prog) goto unhandled_format; /* multiple draw targets not supported on GLES2... */ @@ -1792,11 +2013,14 @@ _do_convert_draw (GstGLContext * context, GstGLColorConvert * convert) /* attach the texture to the FBO to renderer to */ for (i = 0; i < c_info->out_n_textures; i++) { + guint gl_target = + gst_gl_texture_target_to_gl (convert->priv->to_texture_target); + /* needed? */ - gl->BindTexture (GL_TEXTURE_2D, convert->priv->out_tex[i]->tex_id); + gl->BindTexture (gl_target, convert->priv->out_tex[i]->tex_id); gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0 + i, - GL_TEXTURE_2D, convert->priv->out_tex[i]->tex_id, 0); + gl_target, convert->priv->out_tex[i]->tex_id, 0); } if (gl->DrawBuffers) @@ -1817,15 +2041,15 @@ _do_convert_draw (GstGLContext * context, GstGLColorConvert * convert) for (i = c_info->in_n_textures - 1; i >= 0; i--) { gchar *scale_name = g_strdup_printf ("tex_scale%u", i); - GstGLMemory *m = convert->priv->in_tex[i]; - guint tex_target = m->tex_target; + guint gl_target = + gst_gl_texture_target_to_gl (convert->priv->from_texture_target); gl->ActiveTexture (GL_TEXTURE0 + i); - gl->BindTexture (tex_target, convert->priv->in_tex[i]->tex_id); - gl->TexParameteri (tex_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - gl->TexParameteri (tex_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - gl->TexParameteri (tex_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - gl->TexParameteri (tex_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + gl->BindTexture (gl_target, convert->priv->in_tex[i]->tex_id); + gl->TexParameteri (gl_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->TexParameteri (gl_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->TexParameteri (gl_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->TexParameteri (gl_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); gst_gl_shader_set_uniform_2fv (convert->shader, scale_name, 1, convert->priv->in_tex[i]->tex_scaling); diff --git a/gst-libs/gst/gl/gstglcolorconvert.h b/gst-libs/gst/gl/gstglcolorconvert.h index 3ec3b5203c..89e49fc9d5 100644 --- a/gst-libs/gst/gl/gstglcolorconvert.h +++ b/gst-libs/gst/gl/gstglcolorconvert.h @@ -100,7 +100,7 @@ struct _GstGLColorConvertClass "width = " GST_VIDEO_SIZE_RANGE ", " \ "height = " GST_VIDEO_SIZE_RANGE ", " \ "framerate = " GST_VIDEO_FPS_RANGE ", " \ - "texture-target = (string) 2D " \ + "texture-target = (string) { 2D, rectangle, oes } " \ " ; " \ "video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "," \ GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION "), " \ @@ -108,7 +108,7 @@ struct _GstGLColorConvertClass "width = " GST_VIDEO_SIZE_RANGE ", " \ "height = " GST_VIDEO_SIZE_RANGE ", " \ "framerate = " GST_VIDEO_FPS_RANGE ", " \ - "texture-target = (string) 2D" + "texture-target = (string) { 2D, rectangle, oes }" GstGLColorConvert * gst_gl_color_convert_new (GstGLContext * context); @@ -116,6 +116,10 @@ GstCaps * gst_gl_color_convert_transform_caps (GstGLContext * convert, GstPadDirection direction, GstCaps * caps, GstCaps * filter); +GstCaps * gst_gl_color_convert_fixate_caps (GstGLContext * convert, + GstPadDirection direction, + GstCaps * caps, + GstCaps * other); gboolean gst_gl_color_convert_set_caps (GstGLColorConvert * convert, GstCaps * in_caps, GstCaps * out_caps);