From 1ce66e2ed5eeecf0b6c28118dccdd83fdc4f6afb Mon Sep 17 00:00:00 2001 From: "U. Artie Eoff" Date: Thu, 19 Sep 2019 11:17:24 -0700 Subject: [PATCH] libs: encoder: h264_fei: fix potential overflow before widen Found by static analysis. encoder->mb_width * encoder->mb_height is evaluated using 32-bit arithmetic before widen. Thus, cast at least one of these to guint64 to avoid overflow. --- gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c index ed392a034e..ea5c495261 100644 --- a/gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c +++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c @@ -2600,7 +2600,7 @@ ensure_bitrate (GstVaapiEncoderH264Fei * encoder) if (!encoder->use_dct8x8) bits_per_mb += (bits_per_mb * 10) / 100; - factor = encoder->mb_width * encoder->mb_height * bits_per_mb; + factor = (guint64) encoder->mb_width * encoder->mb_height * bits_per_mb; base_encoder->bitrate = gst_util_uint64_scale (factor, GST_VAAPI_ENCODER_FPS_N (encoder), GST_VAAPI_ENCODER_FPS_D (encoder)) / 1000;