diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
index cf556cab2c..4664085b40 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
@@ -483,7 +483,6 @@ struct _GstVaapiDecoderH264Private
   guint decoder_state;
   GstVaapiStreamAlignH264 stream_alignment;
   GstVaapiPictureH264 *current_picture;
-  GstVaapiPictureH264 *missing_picture;
   GstVaapiParserInfoH264 *sps[GST_H264_MAX_SPS_COUNT];
   GstVaapiParserInfoH264 *active_sps;
   GstVaapiParserInfoH264 *pps[GST_H264_MAX_PPS_COUNT];
@@ -1248,7 +1247,6 @@ gst_vaapi_decoder_h264_close (GstVaapiDecoderH264 * decoder)
   GstVaapiDecoderH264Private *const priv = &decoder->priv;
 
   gst_vaapi_picture_replace (&priv->current_picture, NULL);
-  gst_vaapi_picture_replace (&priv->missing_picture, NULL);
   gst_vaapi_parser_info_h264_replace (&priv->prev_slice_pi, NULL);
   gst_vaapi_parser_info_h264_replace (&priv->prev_pi, NULL);
 
@@ -3079,85 +3077,6 @@ init_picture_refs (GstVaapiDecoderH264 * decoder,
   mark_picture_refs (decoder, picture);
 }
 
-static GstVaapiPictureH264 *
-fill_picture_first_field_gap (GstVaapiDecoderH264 * decoder,
-    GstVaapiPictureH264 * f0)
-{
-  GstVaapiDecoderH264Private *const priv = &decoder->priv;
-  GstVaapiPictureH264 *f1;
-
-  f1 = gst_vaapi_picture_h264_new_clone (f0);
-  if (!f1)
-    goto error_allocate_field;
-
-  gst_vaapi_picture_replace (&priv->missing_picture, f1);
-  gst_vaapi_picture_unref (f1);
-
-  GST_VAAPI_PICTURE_FLAG_SET (f1,
-      (GST_VAAPI_PICTURE_FLAG_ONEFIELD |
-          GST_VAAPI_PICTURE_FLAG_SKIPPED | GST_VAAPI_PICTURE_FLAG_GHOST));
-
-  gst_vaapi_picture_h264_set_reference (f1, 0, FALSE);
-  return f1;
-
-  /* ERRORS */
-error_allocate_field:
-  {
-    GST_ERROR ("failed to allocate missing field for current frame store");
-    return NULL;
-  }
-}
-
-static gboolean
-fill_picture_first_field_gap_done (GstVaapiDecoderH264 * decoder,
-    GstH264SliceHdr * slice_hdr)
-{
-  GstVaapiDecoderH264Private *const priv = &decoder->priv;
-  GstVaapiPictureH264 *const lost_field = priv->missing_picture;
-  GstH264SliceHdr lost_slice_hdr;
-  gboolean success = FALSE;
-
-  g_return_val_if_fail (priv->current_picture != NULL, FALSE);
-
-  if (!lost_field)
-    return TRUE;
-
-  lost_field->frame_num = slice_hdr->frame_num;
-  lost_field->frame_num_wrap = slice_hdr->frame_num;
-
-  gst_vaapi_picture_h264_set_reference (lost_field,
-      (GST_VAAPI_PICTURE_FLAGS (priv->current_picture) &
-          GST_VAAPI_PICTURE_FLAGS_REFERENCE), FALSE);
-
-  lost_slice_hdr = *slice_hdr;
-  lost_slice_hdr.bottom_field_flag = !lost_slice_hdr.bottom_field_flag;
-
-  init_picture_poc (decoder, lost_field, &lost_slice_hdr);
-  init_picture_ref_lists (decoder, lost_field);
-  init_picture_refs_pic_num (decoder, lost_field, &lost_slice_hdr);
-  if (!exec_ref_pic_marking_sliding_window (decoder))
-    goto error_exec_ref_pic_marking;
-  if (!dpb_add (decoder, lost_field))
-    goto error_dpb_add;
-  success = TRUE;
-
-cleanup:
-  gst_vaapi_picture_replace (&priv->missing_picture, NULL);
-  return success;
-
-  /* ERRORS */
-error_exec_ref_pic_marking:
-  {
-    GST_ERROR ("failed to execute reference picture marking process");
-    goto cleanup;
-  }
-error_dpb_add:
-  {
-    GST_ERROR ("failed to store lost picture into the DPB");
-    goto cleanup;
-  }
-}
-
 static GstVaapiPictureH264 *
 fill_picture_other_field_gap (GstVaapiDecoderH264 * decoder,
     GstVaapiPictureH264 * f0)
@@ -3458,7 +3377,6 @@ init_picture (GstVaapiDecoderH264 * decoder,
           GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
   }
 
-  fill_picture_first_field_gap_done (decoder, slice_hdr);
   init_picture_poc (decoder, picture, slice_hdr);
   return TRUE;
 }
@@ -3993,8 +3911,7 @@ same_field_parity (GstVaapiPictureH264 * field, GstH264SliceHdr * slice_hdr)
 
 /* Finds the first field picture corresponding to the supplied picture */
 static GstVaapiPictureH264 *
-find_first_field (GstVaapiDecoderH264 * decoder, GstVaapiParserInfoH264 * pi,
-    gboolean fill_gaps)
+find_first_field (GstVaapiDecoderH264 * decoder, GstVaapiParserInfoH264 * pi)
 {
   GstVaapiDecoderH264Private *const priv = &decoder->priv;
   GstH264SliceHdr *const slice_hdr = &pi->data.slice_hdr;
@@ -4007,37 +3924,29 @@ find_first_field (GstVaapiDecoderH264 * decoder, GstVaapiParserInfoH264 * pi,
 
   f0 = fs->buffers[0];
   if (!slice_hdr->field_pic_flag) {
-    if (fill_gaps && !gst_vaapi_frame_store_has_frame (fs))
+    if (!gst_vaapi_frame_store_has_frame (fs))
       fill_picture_other_field_gap (decoder, f0);
     return NULL;
   }
 
   /* At this point, the current frame is known to be interlaced */
   if (gst_vaapi_frame_store_has_frame (fs)) {
-    f1 = NULL;
-    if (fill_gaps && !same_field_parity (f0, slice_hdr))
-      f1 = fill_picture_first_field_gap (decoder, f0);
-    return f1;
+    return NULL;
   }
 
   /* At this point, the previous frame is interlaced and contains a
      single field */
   if (f0->frame_num == slice_hdr->frame_num) {
     f1 = f0;
-    if (fill_gaps && same_field_parity (f0, slice_hdr)) {
+    if (same_field_parity (f0, slice_hdr)) {
       fill_picture_other_field_gap (decoder, f0);
       f1 = NULL;
     }
     return f1;
   }
 
-  f1 = NULL;
-  if (fill_gaps) {
-    fill_picture_other_field_gap (decoder, f0);
-    if (!same_field_parity (f0, slice_hdr))
-      f1 = fill_picture_first_field_gap (decoder, f0);
-  }
-  return f1;
+  fill_picture_other_field_gap (decoder, f0);
+  return NULL;
 }
 
 static GstVaapiDecoderStatus
@@ -4059,9 +3968,8 @@ decode_picture (GstVaapiDecoderH264 * decoder, GstVaapiDecoderUnit * unit)
     return status;
 
   priv->decoder_state = 0;
-  gst_vaapi_picture_replace (&priv->missing_picture, NULL);
 
-  first_field = find_first_field (decoder, pi, TRUE);
+  first_field = find_first_field (decoder, pi);
   if (first_field) {
     /* Re-use current picture where the first field was decoded */
     picture = gst_vaapi_picture_h264_new_field (first_field);