diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index 692b2ff797..7942a58a03 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -2853,7 +2853,8 @@ gst_video_decoder_get_max_decode_time (GstVideoDecoder *
 
   GST_OBJECT_LOCK (decoder);
   earliest_time = decoder->priv->earliest_time;
-  if (GST_CLOCK_TIME_IS_VALID (earliest_time))
+  if (GST_CLOCK_TIME_IS_VALID (earliest_time)
+      && GST_CLOCK_TIME_IS_VALID (frame->deadline))
     deadline = GST_CLOCK_DIFF (earliest_time, frame->deadline);
   else
     deadline = G_MAXINT64;