From 9bf1ae04ccdc3de04a3c694b573c90266a59bea6 Mon Sep 17 00:00:00 2001 From: Holger Kaelberer Date: Tue, 26 Feb 2013 00:38:24 +0100 Subject: [PATCH] vaapiupload: fix illegal write in ensure_image(). Fix ensure_image() to only zero-initialize the first line of each plane. Properly initializing each plane to their full vertical resolution would require to actually compute it based on the image format. In particular, for NV12 images, the UV plane has half vertical resolution vs. the Y plane. So using the full image height to initialize the UV plane will obviously lead to a buffer overflow. Likewise for other YUV format. Since ensure_image() is only a helper function to initialize something, and not necessarily the whole thing, it is fine to initializ the first line only. Besides, the target surface is not rendered either. Signed-off-by: Gwenole Beauchesne --- gst/vaapi/gstvaapiuploader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst/vaapi/gstvaapiuploader.c b/gst/vaapi/gstvaapiuploader.c index 460435e54a..437f6af489 100644 --- a/gst/vaapi/gstvaapiuploader.c +++ b/gst/vaapi/gstvaapiuploader.c @@ -108,7 +108,7 @@ ensure_image(GstVaapiImage *image) for (i = 0; i < num_planes; i++) { guchar * const plane = gst_vaapi_image_get_plane(image, i); if (plane) - memset(plane, 0, height * gst_vaapi_image_get_pitch(image, i)); + memset(plane, 0, gst_vaapi_image_get_pitch(image, i)); } if (!gst_vaapi_image_unmap(image))