diff --git a/gst-libs/gst/video/gstbasevideocodec.h b/gst-libs/gst/video/gstbasevideocodec.h
index 41c2282a6e..7a76f9d12a 100644
--- a/gst-libs/gst/video/gstbasevideocodec.h
+++ b/gst-libs/gst/video/gstbasevideocodec.h
@@ -124,6 +124,10 @@ struct _GstVideoFrame
   gboolean is_sync_point;
   gboolean is_eos;
 
+  /* Frames that should not be pushed downstream and are
+   * not meant for display */
+  gboolean decode_only;
+
   GstBuffer *sink_buffer;
   GstBuffer *src_buffer;
 
diff --git a/gst-libs/gst/video/gstbasevideodecoder.c b/gst-libs/gst/video/gstbasevideodecoder.c
index 6a5554ea31..83255f4141 100644
--- a/gst-libs/gst/video/gstbasevideodecoder.c
+++ b/gst-libs/gst/video/gstbasevideodecoder.c
@@ -1411,6 +1411,11 @@ gst_base_video_decoder_prepare_finish_frame (GstBaseVideoDecoder *
   }
   g_list_free (events);
 
+  /* Check if the data should not be displayed. For example altref/invisible
+   * frame in vp8. In this case we should not update the timestamps. */
+  if (frame->decode_only)
+    return;
+
   if (GST_CLOCK_TIME_IS_VALID (frame->presentation_timestamp)) {
     if (frame->presentation_timestamp != base_video_decoder->timestamp_offset) {
       GST_DEBUG_OBJECT (base_video_decoder,
@@ -1564,7 +1569,7 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
   base_video_decoder->processed++;
 
   /* no buffer data means this frame is skipped */
-  if (!frame->src_buffer) {
+  if (!frame->src_buffer || frame->decode_only) {
     GST_DEBUG_OBJECT (base_video_decoder, "skipping frame %" GST_TIME_FORMAT,
         GST_TIME_ARGS (frame->presentation_timestamp));
     goto done;