From be9b6969cb7d945170b3a05da1a643aad5870f1e Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Mon, 2 Jul 2018 04:05:19 +0200 Subject: [PATCH] avviddec: fix invalid alignment calculations. avcodec_align_dimensions2 uses context->pix_fmt to make its calculations, we thus need to make sure it is adequately set when calling it. Fixes: gst-launch-1.0 videotestsrc ! video/x-raw, width=1920, height=1080 \ ! avenc_mpeg4 ! avdec_mpeg4 ! xvimagesink This showed invalid writes under valgrind, then segfault. https://bugzilla.gnome.org/show_bug.cgi?id=792900 --- ext/libav/gstavviddec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ext/libav/gstavviddec.c b/ext/libav/gstavviddec.c index 4fc6dc20d2..e3ae04f57d 100644 --- a/ext/libav/gstavviddec.c +++ b/ext/libav/gstavviddec.c @@ -678,6 +678,12 @@ gst_ffmpegviddec_ensure_internal_pool (GstFFMpegVidDec * ffmpegdec, format = gst_ffmpeg_pixfmt_to_videoformat (picture->format); gst_video_info_set_format (&info, format, picture->width, picture->height); + /* If we have not yet been negotiated, a NONE format here would + * result in invalid initial dimension alignments, and potential + * out of bounds writes. + */ + ffmpegdec->context->pix_fmt = picture->format; + for (i = 0; i < G_N_ELEMENTS (ffmpegdec->stride); i++) ffmpegdec->stride[i] = -1;