diff --git a/sys/vdpau/gstvdpmpegdec.c b/sys/vdpau/gstvdpmpegdec.c
index 7eb05afafd..e85b1ec338 100644
--- a/sys/vdpau/gstvdpmpegdec.c
+++ b/sys/vdpau/gstvdpmpegdec.c
@@ -344,24 +344,6 @@ gst_vdp_mpeg_dec_decode (GstVdpMpegDec * mpeg_dec,
   buffer = gst_adapter_take_buffer (mpeg_dec->adapter,
       gst_adapter_available (mpeg_dec->adapter));
 
-  if (info->picture_coding_type != B_FRAME) {
-    if (info->backward_reference != VDP_INVALID_HANDLE) {
-      gst_buffer_ref (mpeg_dec->b_buffer);
-      gst_vdp_mpeg_dec_push_video_buffer (mpeg_dec,
-          GST_VDP_VIDEO_BUFFER (mpeg_dec->b_buffer));
-    }
-
-    if (info->forward_reference != VDP_INVALID_HANDLE) {
-      gst_buffer_unref (mpeg_dec->f_buffer);
-      info->forward_reference = VDP_INVALID_HANDLE;
-    }
-
-    info->forward_reference = info->backward_reference;
-    mpeg_dec->f_buffer = mpeg_dec->b_buffer;
-
-    info->backward_reference = VDP_INVALID_HANDLE;
-  }
-
   outbuf = gst_vdp_video_buffer_new (mpeg_dec->device, VDP_CHROMA_TYPE_420,
       mpeg_dec->width, mpeg_dec->height);
   GST_BUFFER_TIMESTAMP (outbuf) = timestamp;
@@ -380,12 +362,31 @@ gst_vdp_mpeg_dec_decode (GstVdpMpegDec * mpeg_dec,
     GST_BUFFER_FLAG_UNSET (outbuf, GST_VIDEO_BUFFER_TFF);
 
 
+  if (info->picture_coding_type != B_FRAME) {
+    if (info->backward_reference != VDP_INVALID_HANDLE) {
+      gst_buffer_ref (mpeg_dec->b_buffer);
+      gst_vdp_mpeg_dec_push_video_buffer (mpeg_dec,
+          GST_VDP_VIDEO_BUFFER (mpeg_dec->b_buffer));
+    }
+
+    if (info->forward_reference != VDP_INVALID_HANDLE) {
+      gst_buffer_unref (mpeg_dec->f_buffer);
+      info->forward_reference = VDP_INVALID_HANDLE;
+    }
+
+    info->forward_reference = info->backward_reference;
+    mpeg_dec->f_buffer = mpeg_dec->b_buffer;
+
+    info->backward_reference = VDP_INVALID_HANDLE;
+  }
+
   if (info->forward_reference != VDP_INVALID_HANDLE &&
       info->picture_coding_type != I_FRAME)
     gst_vdp_video_buffer_add_reference (outbuf,
         GST_VDP_VIDEO_BUFFER (mpeg_dec->f_buffer));
 
-  if (info->backward_reference != VDP_INVALID_HANDLE)
+  if (info->backward_reference != VDP_INVALID_HANDLE
+      && info->picture_coding_type == B_FRAME)
     gst_vdp_video_buffer_add_reference (outbuf,
         GST_VDP_VIDEO_BUFFER (mpeg_dec->b_buffer));
 
diff --git a/sys/vdpau/mpegutil.c b/sys/vdpau/mpegutil.c
index 5348bd6a4a..b52ab6f8ee 100644
--- a/sys/vdpau/mpegutil.c
+++ b/sys/vdpau/mpegutil.c
@@ -256,20 +256,22 @@ mpeg_util_parse_picture_hdr (MPEGPictureHdr * hdr, GstBuffer * buffer)
     if (!gst_bit_reader_get_bits_uint8 (&reader, &hdr->f_code[0][0], 3))
       return FALSE;
     hdr->f_code[0][1] = hdr->f_code[0][0];
-
-    if (hdr->pic_type == B_FRAME) {
-      if (!gst_bit_reader_get_bits_uint8 (&reader,
-              &hdr->full_pel_backward_vector, 1))
-        return FALSE;
-
-      if (!gst_bit_reader_get_bits_uint8 (&reader, &hdr->f_code[1][0], 3))
-        return FALSE;
-      hdr->f_code[1][1] = hdr->f_code[1][0];
-    } else
-      hdr->full_pel_backward_vector = 0;
   } else {
     hdr->full_pel_forward_vector = 0;
+    hdr->f_code[0][0] = hdr->f_code[0][1] = 0;
+  }
+
+  if (hdr->pic_type == B_FRAME) {
+    if (!gst_bit_reader_get_bits_uint8 (&reader,
+            &hdr->full_pel_backward_vector, 1))
+      return FALSE;
+
+    if (!gst_bit_reader_get_bits_uint8 (&reader, &hdr->f_code[1][0], 3))
+      return FALSE;
+    hdr->f_code[1][1] = hdr->f_code[1][0];
+  } else {
     hdr->full_pel_backward_vector = 0;
+    hdr->f_code[1][0] = hdr->f_code[1][1] = 0;
   }
 
   return TRUE;