From 0d2d695eff56655e4f7cb50c59fe3a6456f17e11 Mon Sep 17 00:00:00 2001
From: Julien Isorce <jisorce@oblong.com>
Date: Tue, 12 Dec 2017 14:45:30 +0000
Subject: [PATCH] gstomxvideodec: fix framerate overflow

Some live streams can set the framerate to 50000/1677 (=29.81).

GstVideoInfo.fps_n << 16 is wrong if the fps_n is 50000
(i.e. greater than 32767).

https://bugzilla.gnome.org/show_bug.cgi?id=759043
---
 omx/gstomxvideo.c    | 14 ++++++++++----
 omx/gstomxvideo.h    |  2 ++
 omx/gstomxvideodec.c |  9 +++------
 omx/gstomxvideoenc.c | 14 ++++++--------
 4 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/omx/gstomxvideo.c b/omx/gstomxvideo.c
index 8a42c710a7..fca0fd589d 100644
--- a/omx/gstomxvideo.c
+++ b/omx/gstomxvideo.c
@@ -102,10 +102,8 @@ gst_omx_video_get_supported_colorformats (GstOMXPort * port,
   GST_OMX_INIT_STRUCT (&param);
   param.nPortIndex = port->index;
   param.nIndex = 0;
-  if (!state || state->info.fps_n == 0)
-    param.xFramerate = 0;
-  else
-    param.xFramerate = (state->info.fps_n << 16) / (state->info.fps_d);
+  param.xFramerate =
+      state ? gst_omx_video_calculate_framerate_q16 (&state->info) : 0;
 
   old_index = -1;
   do {
@@ -200,3 +198,11 @@ gst_omx_video_find_nearest_frame (GstOMXBuffer * buf, GList * frames)
 
   return best;
 }
+
+OMX_U32
+gst_omx_video_calculate_framerate_q16 (GstVideoInfo * info)
+{
+  g_assert (info);
+
+  return gst_util_uint64_scale_int (1 << 16, info->fps_n, info->fps_d);
+}
diff --git a/omx/gstomxvideo.h b/omx/gstomxvideo.h
index f146df7599..87de9739df 100644
--- a/omx/gstomxvideo.h
+++ b/omx/gstomxvideo.h
@@ -55,6 +55,8 @@ gst_omx_video_negotiation_map_free (GstOMXVideoNegotiationMap * m);
 GstVideoCodecFrame *
 gst_omx_video_find_nearest_frame (GstOMXBuffer * buf, GList * frames);
 
+OMX_U32 gst_omx_video_calculate_framerate_q16 (GstVideoInfo * info);
+
 G_END_DECLS
 
 #endif /* __GST_OMX_VIDEO_H__ */
diff --git a/omx/gstomxvideodec.c b/omx/gstomxvideodec.c
index 38b4c1bf96..550d57e0ba 100644
--- a/omx/gstomxvideodec.c
+++ b/omx/gstomxvideodec.c
@@ -2233,6 +2233,7 @@ gst_omx_video_dec_set_format (GstVideoDecoder * decoder,
   gboolean is_format_change = FALSE;
   gboolean needs_disable = FALSE;
   OMX_PARAM_PORTDEFINITIONTYPE port_def;
+  OMX_U32 framerate_q16 = gst_omx_video_calculate_framerate_q16 (info);
 
   self = GST_OMX_VIDEO_DEC (decoder);
   klass = GST_OMX_VIDEO_DEC_GET_CLASS (decoder);
@@ -2256,8 +2257,7 @@ gst_omx_video_dec_set_format (GstVideoDecoder * decoder,
   is_format_change |= port_def.format.video.nFrameHeight != info->height;
   is_format_change |= (port_def.format.video.xFramerate == 0
       && info->fps_n != 0)
-      || (port_def.format.video.xFramerate !=
-      (info->fps_n << 16) / (info->fps_d));
+      || (port_def.format.video.xFramerate != framerate_q16);
   is_format_change |= (self->codec_data != state->codec_data);
   if (klass->is_format_change)
     is_format_change |=
@@ -2291,10 +2291,7 @@ gst_omx_video_dec_set_format (GstVideoDecoder * decoder,
 
   port_def.format.video.nFrameWidth = info->width;
   port_def.format.video.nFrameHeight = info->height;
-  if (info->fps_n == 0)
-    port_def.format.video.xFramerate = 0;
-  else
-    port_def.format.video.xFramerate = (info->fps_n << 16) / (info->fps_d);
+  port_def.format.video.xFramerate = framerate_q16;
 
   GST_DEBUG_OBJECT (self, "Setting inport port definition");
 
diff --git a/omx/gstomxvideoenc.c b/omx/gstomxvideoenc.c
index dfd67041b3..f60bbaa5b0 100644
--- a/omx/gstomxvideoenc.c
+++ b/omx/gstomxvideoenc.c
@@ -1287,14 +1287,12 @@ gst_omx_video_enc_set_format (GstVideoEncoder * encoder,
   port_def.format.video.nFrameWidth = info->width;
   port_def.format.video.nFrameHeight = info->height;
 
-  if (info->fps_n == 0) {
-    port_def.format.video.xFramerate = 0;
-  } else {
-    if (!(klass->cdata.hacks & GST_OMX_HACK_VIDEO_FRAMERATE_INTEGER))
-      port_def.format.video.xFramerate = (info->fps_n << 16) / (info->fps_d);
-    else
-      port_def.format.video.xFramerate = (info->fps_n) / (info->fps_d);
-  }
+  if (G_UNLIKELY (klass->cdata.hacks & GST_OMX_HACK_VIDEO_FRAMERATE_INTEGER))
+    port_def.format.video.xFramerate =
+        info->fps_n ? (info->fps_n) / (info->fps_d) : 0;
+  else
+    port_def.format.video.xFramerate =
+        gst_omx_video_calculate_framerate_q16 (info);
 
   GST_DEBUG_OBJECT (self, "Setting inport port definition");
   if (gst_omx_port_update_port_definition (self->enc_in_port,