smpte: Fix integer overflow with possible heap corruption in GstMask creation.

Check that width*height*sizeof(guint32) doesn't overflow when
allocated user_data for mask, potential for heap overwrite when
inverting.

Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1231

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2603>
This commit is contained in:
Adam Doupe 2022-05-19 04:16:25 +00:00 committed by GStreamer Marge Bot
parent 87230db55c
commit be11a6e26b

View file

@ -85,6 +85,13 @@ gst_mask_factory_new (gint type, gboolean invert, gint bpp, gint width,
mask->height = height;
mask->destroy_func = definition->destroy_func;
mask->user_data = definition->user_data;
if (((guint64) width * (guint64) height * sizeof (guint32)) > G_MAXUINT) {
GST_WARNING ("width x height overflows");
g_free (mask);
return NULL;
}
mask->data = g_malloc (width * height * sizeof (guint32));
definition->draw_func (mask);