From 10ed707b6085e3edc14e897ba53dbd321e4f557b Mon Sep 17 00:00:00 2001 From: Vineeth TM Date: Thu, 17 Dec 2015 08:51:48 +0900 Subject: [PATCH] pnmenc: Fix wrong logic leading to memory mishandling While encoding the frame in ASCII mode, per component four bytes are needed and after every 20 bytes, a \n will be added. So the calculation should be size = size * (4 + 1 / 20). This should exclude the header being written. Since header is also being included in the calculations, memory mishandlings are happening. https://bugzilla.gnome.org/show_bug.cgi?id=759520 --- gst/pnm/gstpnmenc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst/pnm/gstpnmenc.c b/gst/pnm/gstpnmenc.c index d886273593..32fd6f68ed 100644 --- a/gst/pnm/gstpnmenc.c +++ b/gst/pnm/gstpnmenc.c @@ -202,10 +202,10 @@ gst_pnmenc_handle_frame (GstVideoEncoder * encoder, GstVideoCodecFrame * frame) if (pnmenc->info.encoding == GST_PNM_ENCODING_ASCII) { /* Per component 4 bytes are used in case of ASCII encoding */ - size = size * 4; + size = size * 4 + size / 20; size += strlen (header); frame->output_buffer = - gst_video_encoder_allocate_output_buffer (encoder, (size + size / 20)); + gst_video_encoder_allocate_output_buffer (encoder, (size)); } else { size += strlen (header); frame->output_buffer =