videobox: Correctly clamp frame/background alphas to [0,255] before writing them

This commit is contained in:
Sebastian Dröge 2010-04-12 11:25:59 +02:00
parent d11998247c
commit 9da4f2906f

View file

@ -135,6 +135,8 @@ fill_ayuv (GstVideoBoxFill fill_type, guint b_alpha, GstVideoFormat format,
{ {
guint32 empty_pixel; guint32 empty_pixel;
b_alpha = CLAMP (b_alpha, 0, 255);
if (sdtv) if (sdtv)
empty_pixel = GUINT32_FROM_BE ((b_alpha << 24) | empty_pixel = GUINT32_FROM_BE ((b_alpha << 24) |
(yuv_sdtv_colors_Y[fill_type] << 16) | (yuv_sdtv_colors_Y[fill_type] << 16) |
@ -602,6 +604,9 @@ copy_i420_ayuv (guint i_alpha, GstVideoFormat dest_format, guint8 * dest,
hY = h; hY = h;
hUV = (h + 1) / 2; hUV = (h + 1) / 2;
i_alpha = CLAMP (i_alpha, 0, 255);
if (src_sdtv != dest_sdtv) { if (src_sdtv != dest_sdtv) {
gint matrix[12]; gint matrix[12];
gint y1, y2, y3, y4; gint y1, y2, y3, y4;
@ -788,6 +793,8 @@ fill_rgb32 (GstVideoBoxFill fill_type, guint b_alpha, GstVideoFormat format,
_argb_order (format, p, &alpha); _argb_order (format, p, &alpha);
b_alpha = CLAMP (b_alpha, 0, 255);
empty_pixel = GUINT32_FROM_BE ((b_alpha << (p[0] * 8)) | empty_pixel = GUINT32_FROM_BE ((b_alpha << (p[0] * 8)) |
(rgb_colors_R[fill_type] << (p[1] * 8)) | (rgb_colors_R[fill_type] << (p[1] * 8)) |
(rgb_colors_G[fill_type] << (p[2] * 8)) | (rgb_colors_G[fill_type] << (p[2] * 8)) |
@ -860,9 +867,11 @@ copy_rgb32 (guint i_alpha, GstVideoFormat dest_format, guint8 * dest,
} }
} else if (out_alpha && !packed_in) { } else if (out_alpha && !packed_in) {
w *= 4; w *= 4;
i_alpha = CLAMP (i_alpha, 0, 255);
for (i = 0; i < h; i++) { for (i = 0; i < h; i++) {
for (j = 0; j < w; j += 4) { for (j = 0; j < w; j += 4) {
dest[j + p_out[0]] = i_alpha & 0xff; dest[j + p_out[0]] = i_alpha;
dest[j + p_out[1]] = src[j + p_in[1]]; dest[j + p_out[1]] = src[j + p_in[1]];
dest[j + p_out[2]] = src[j + p_in[2]]; dest[j + p_out[2]] = src[j + p_in[2]];
dest[j + p_out[3]] = src[j + p_in[3]]; dest[j + p_out[3]] = src[j + p_in[3]];
@ -871,9 +880,11 @@ copy_rgb32 (guint i_alpha, GstVideoFormat dest_format, guint8 * dest,
src += src_stride; src += src_stride;
} }
} else if (out_alpha && packed_in) { } else if (out_alpha && packed_in) {
i_alpha = CLAMP (i_alpha, 0, 255);
for (i = 0; i < h; i++) { for (i = 0; i < h; i++) {
for (j = 0; j < w; j++) { for (j = 0; j < w; j++) {
dest[4 * j + p_out[0]] = i_alpha & 0xff; dest[4 * j + p_out[0]] = i_alpha;
dest[4 * j + p_out[1]] = src[in_bpp * j + p_in[1]]; dest[4 * j + p_out[1]] = src[in_bpp * j + p_in[1]];
dest[4 * j + p_out[2]] = src[in_bpp * j + p_in[2]]; dest[4 * j + p_out[2]] = src[in_bpp * j + p_in[2]];
dest[4 * j + p_out[3]] = src[in_bpp * j + p_in[3]]; dest[4 * j + p_out[3]] = src[in_bpp * j + p_in[3]];
@ -960,9 +971,11 @@ copy_rgb32_ayuv (guint i_alpha, GstVideoFormat dest_format, guint8 * dest,
} }
} else if (!packed_in) { } else if (!packed_in) {
w *= 4; w *= 4;
i_alpha = CLAMP (i_alpha, 0, 255);
for (i = 0; i < h; i++) { for (i = 0; i < h; i++) {
for (j = 0; j < w; j += 4) { for (j = 0; j < w; j += 4) {
a = i_alpha & 0xff; a = i_alpha;
r = src[j + p_in[1]]; r = src[j + p_in[1]];
g = src[j + p_in[2]]; g = src[j + p_in[2]];
b = src[j + p_in[3]]; b = src[j + p_in[3]];
@ -980,9 +993,11 @@ copy_rgb32_ayuv (guint i_alpha, GstVideoFormat dest_format, guint8 * dest,
src += src_stride; src += src_stride;
} }
} else { } else {
i_alpha = CLAMP (i_alpha, 0, 255);
for (i = 0; i < h; i++) { for (i = 0; i < h; i++) {
for (j = 0; j < w; j++) { for (j = 0; j < w; j++) {
a = i_alpha & 0xff; a = i_alpha;
r = src[in_bpp * j + p_in[1]]; r = src[in_bpp * j + p_in[1]];
g = src[in_bpp * j + p_in[2]]; g = src[in_bpp * j + p_in[2]];
b = src[in_bpp * j + p_in[3]]; b = src[in_bpp * j + p_in[3]];