diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gsth264parser.h b/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gsth264parser.h
index b05267ecd3..3f84d0b928 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gsth264parser.h
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gsth264parser.h
@@ -141,8 +141,6 @@ typedef enum {
  * @GST_H264_NAL_PREFIX_UNIT: Prefix NAL unit
  * @GST_H264_NAL_SUBSET_SPS: Subset sequence parameter set (SSPS) NAL unit
  * @GST_H264_NAL_DEPTH_SPS: Depth parameter set (DPS) NAL unit
- * @GST_H264_NAL_RSV_1: First reserved parameter
- * @GST_H264_NAL_RSV_2: Second reserved parameter
  * @GST_H264_NAL_SLICE_AUX: Auxiliary coded picture without partitioning NAL unit
  * @GST_H264_NAL_SLICE_EXT: Coded slice extension NAL unit
  * @GST_H264_NAL_SLICE_DEPTH: Coded slice extension for depth or 3D-AVC texture view
@@ -168,24 +166,6 @@ typedef enum
   GST_H264_NAL_PREFIX_UNIT  = 14,
   GST_H264_NAL_SUBSET_SPS   = 15,
   GST_H264_NAL_DEPTH_SPS    = 16,
-
-  /**
-   * GST_H264_NAL_RSV_1:
-   *
-   * First reserved parameter
-   *
-   * Since: 1.24
-   */
-  GST_H264_NAL_RSV_1        = 17,
-
-  /**
-   * GST_H264_NAL_RSV_2:
-   *
-   * Second reserved parameter
-   *
-   * Since: 1.24
-   */
-  GST_H264_NAL_RSV_2        = 18,
   GST_H264_NAL_SLICE_AUX    = 19,
   GST_H264_NAL_SLICE_EXT    = 20,
   GST_H264_NAL_SLICE_DEPTH  = 21
diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c
index b4f361e228..d7fea7c026 100644
--- a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c
+++ b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c
@@ -40,9 +40,6 @@ GST_DEBUG_CATEGORY (h264_parse_debug);
 #define DEFAULT_CONFIG_INTERVAL      (0)
 #define DEFAULT_UPDATE_TIMECODE       FALSE
 
-#define HIST_IDX_CURR                 0
-#define HIST_IDX_PREV                 1
-
 enum
 {
   PROP_0,
@@ -85,14 +82,6 @@ enum
   GST_H264_PARSE_SEI_PARSED = 2,
 };
 
-typedef enum
-{
-  GST_H264_PARSE_BACKLOG_STATUS_AU_INCOMPLETE = 0,
-  GST_H264_PARSE_BACKLOG_STATUS_AU_COMPLETE,
-  GST_H264_PARSE_BACKLOG_STATUS_UPD_FAILED,
-  GST_H264_PARSE_BACKLOG_STATUS_NOT_SUPPORTED
-} GstH264ParseBacklogStatus;
-
 #define GST_H264_PARSE_STATE_VALID(parse, expected_state) \
   (((parse)->state & (expected_state)) == (expected_state))
 
@@ -139,9 +128,6 @@ static gboolean gst_h264_parse_src_event (GstBaseParse * parse,
 static void gst_h264_parse_update_src_caps (GstH264Parse * h264parse,
     GstCaps * caps);
 
-static GstH264ParseBacklogStatus gst_h264_parse_update_backlog (GstH264Parse *
-    h264parse, GstH264NalUnit * nalu);
-
 static void
 gst_h264_parse_class_init (GstH264ParseClass * klass)
 {
@@ -219,14 +205,6 @@ gst_h264_parse_init (GstH264Parse * h264parse)
   h264parse->aud_needed = TRUE;
   h264parse->aud_insert = TRUE;
   h264parse->update_timecode = DEFAULT_UPDATE_TIMECODE;
-  h264parse->nal_backlog = g_array_new (FALSE, FALSE, sizeof (GstH264NalUnit));
-  h264parse->bl_curr_au_last_vcl = -1;
-  h264parse->bl_next_au_first_vcl = 1;
-  h264parse->bl_next_au_first_nal = 1;
-  h264parse->bl_next_nal = 0;
-
-  h264parse->history_slice[HIST_IDX_CURR].valid = FALSE;
-  h264parse->history_slice[HIST_IDX_PREV].valid = FALSE;
 }
 
 static void
@@ -238,7 +216,6 @@ gst_h264_parse_finalize (GObject * object)
       TRUE);
 
   g_object_unref (h264parse->frame_out);
-  g_array_unref (h264parse->nal_backlog);
 
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
@@ -248,6 +225,9 @@ gst_h264_parse_reset_frame (GstH264Parse * h264parse)
 {
   GST_DEBUG_OBJECT (h264parse, "reset frame");
 
+  /* done parsing; reset state */
+  h264parse->current_off = -1;
+
   h264parse->update_caps = FALSE;
   h264parse->idr_pos = -1;
   h264parse->sei_pos = -1;
@@ -335,7 +315,6 @@ gst_h264_parse_reset (GstH264Parse * h264parse)
   h264parse->discard_bidirectional = FALSE;
   h264parse->marker = FALSE;
 
-  g_array_set_size (h264parse->nal_backlog, 0);
   gst_h264_parse_reset_stream_info (h264parse);
 }
 
@@ -357,7 +336,6 @@ gst_h264_parse_start (GstBaseParse * parse)
   h264parse->field_pic_flag = 0;
   h264parse->aud_needed = TRUE;
   h264parse->aud_insert = FALSE;
-  h264parse->current_off = -1;
 
   gst_base_parse_set_min_frame_size (parse, 4);
 
@@ -974,66 +952,6 @@ gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
   g_array_free (messages, TRUE);
 }
 
-static void
-gst_h264_parse_update_vcl_nal_history_sps (GstH264Parse * h264parse,
-    GstH264SPS * sps)
-{
-  h264parse->history_sps[HIST_IDX_PREV] = h264parse->history_sps[HIST_IDX_CURR];
-  h264parse->history_sps[HIST_IDX_CURR].pic_order_cnt_type =
-      sps->pic_order_cnt_type;
-  h264parse->history_sps[HIST_IDX_CURR].profile_idc = sps->profile_idc;
-}
-
-static void
-gst_h264_parse_update_vcl_nal_history_pps (GstH264Parse * h264parse,
-    GstH264PPS * pps)
-{
-  gst_h264_parse_update_vcl_nal_history_sps (h264parse, pps->sequence);
-  h264parse->history_pps[HIST_IDX_PREV] = h264parse->history_pps[HIST_IDX_CURR];
-  h264parse->history_pps[HIST_IDX_CURR].id = pps->id;
-}
-
-static void
-gst_h264_parse_update_vcl_nal_history_nalu (GstH264Parse * h264parse,
-    GstH264NalUnit * nalu)
-{
-  h264parse->history_nalu[HIST_IDX_PREV]
-      = h264parse->history_nalu[HIST_IDX_CURR];
-  h264parse->history_nalu[HIST_IDX_CURR].ref_idc = nalu->ref_idc;
-  h264parse->history_nalu[HIST_IDX_CURR].idr_pic_flag = nalu->idr_pic_flag;
-
-  if (GST_H264_IS_MVC_NALU (nalu))
-    h264parse->history_nalu[HIST_IDX_CURR].view_id =
-        nalu->extension.mvc.view_id;
-}
-
-static void
-gst_h264_parse_update_vcl_nal_history (GstH264Parse * h264parse,
-    GstH264NalUnit * nalu, GstH264SliceHdr * slice)
-{
-  gst_h264_parse_update_vcl_nal_history_nalu (h264parse, nalu);
-  gst_h264_parse_update_vcl_nal_history_pps (h264parse, slice->pps);
-  h264parse->history_slice[HIST_IDX_PREV]
-      = h264parse->history_slice[HIST_IDX_CURR];
-  h264parse->history_slice[HIST_IDX_CURR].valid = TRUE;
-  h264parse->history_slice[HIST_IDX_CURR].frame_num = slice->frame_num;
-  h264parse->history_slice[HIST_IDX_CURR].field_pic_flag
-      = slice->field_pic_flag;
-  h264parse->history_slice[HIST_IDX_CURR].bottom_field_flag
-      = slice->bottom_field_flag;
-  h264parse->history_slice[HIST_IDX_CURR].idr_pic_id = slice->idr_pic_id;
-  h264parse->history_slice[HIST_IDX_CURR].delta_pic_order_cnt[0]
-      = slice->delta_pic_order_cnt[0];
-  h264parse->history_slice[HIST_IDX_CURR].delta_pic_order_cnt[1]
-      = slice->delta_pic_order_cnt[1];
-  h264parse->history_slice[HIST_IDX_CURR].pic_order_cnt_lsb
-      = slice->pic_order_cnt_lsb;
-  h264parse->history_slice[HIST_IDX_CURR].delta_pic_order_cnt_bottom
-      = slice->delta_pic_order_cnt_bottom;
-  h264parse->history_slice[HIST_IDX_CURR].first_mb_in_slice
-      = slice->first_mb_in_slice;
-}
-
 /* caller guarantees 2 bytes of nal payload */
 static gboolean
 gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
@@ -1265,6 +1183,44 @@ gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
   return TRUE;
 }
 
+/* caller guarantees at least 2 bytes of nal payload for each nal
+ * returns TRUE if next_nal indicates that nal terminates an AU */
+static inline gboolean
+gst_h264_parse_collect_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
+{
+  GstH264NalUnitType nal_type = nalu->type;
+  gboolean complete;
+
+  /* determine if AU complete */
+  GST_LOG_OBJECT (h264parse, "next nal type: %d %s (picture started %i)",
+      nal_type, _nal_name (nal_type), h264parse->picture_start);
+
+  /* consider a coded slices (IDR or not) to start a picture,
+   * (so ending the previous one) if first_mb_in_slice == 0
+   * (non-0 is part of previous one) */
+  /* NOTE this is not entirely according to Access Unit specs in 7.4.1.2.4,
+   * but in practice it works in sane cases, needs not much parsing,
+   * and also works with broken frame_num in NAL
+   * (where spec-wise would fail) */
+  complete = h264parse->picture_start && ((nal_type >= GST_H264_NAL_SEI &&
+          nal_type <= GST_H264_NAL_AU_DELIMITER) ||
+      (nal_type >= 14 && nal_type <= 18));
+
+  /* first_mb_in_slice == 0 considered start of frame */
+  if (nalu->size > nalu->header_bytes)
+    complete |= h264parse->picture_start && (nal_type == GST_H264_NAL_SLICE
+        || nal_type == GST_H264_NAL_SLICE_DPA
+        || nal_type == GST_H264_NAL_SLICE_IDR) &&
+        (nalu->data[nalu->offset + nalu->header_bytes] & 0x80);
+
+  GST_LOG_OBJECT (h264parse, "au complete: %d", complete);
+
+  if (complete)
+    h264parse->picture_start = FALSE;
+
+  return complete;
+}
+
 static guint8 au_delim[6] = {
   0x00, 0x00, 0x00, 0x01,       /* nal prefix */
   0x09,                         /* nal unit type = access unit delimiter */
@@ -1381,426 +1337,6 @@ gst_h264_parse_handle_frame_packetized (GstBaseParse * parse,
   return ret;
 }
 
-static gboolean
-gst_h264_parse_received_first_vcl_nal_base (GstH264Parse * h264parse,
-    GstH264ParseHistorySlice * slice_hdr_prev)
-{
-  /* Ref. ITU-T H.264, 7.4.1.2.4 */
-  GstH264ParseHistorySlice *slice_hdr_curr
-      = &h264parse->history_slice[HIST_IDX_CURR];
-  GstH264ParseHistoryPPS *pps_hist_curr
-      = &h264parse->history_pps[HIST_IDX_CURR];
-  GstH264ParseHistoryPPS *pps_hist_prev
-      = &h264parse->history_pps[HIST_IDX_PREV];
-  GstH264ParseHistorySPS *sps_hist_curr
-      = &h264parse->history_sps[HIST_IDX_CURR];
-  GstH264ParseHistorySPS *sps_hist_prev
-      = &h264parse->history_sps[HIST_IDX_PREV];
-  GstH264ParseHistoryNalUnit *nalu_hist_curr
-      = &h264parse->history_nalu[HIST_IDX_CURR];
-  GstH264ParseHistoryNalUnit *nalu_hist_prev
-      = &h264parse->history_nalu[HIST_IDX_PREV];
-
-  if (slice_hdr_curr->frame_num != slice_hdr_prev->frame_num) {
-    return TRUE;
-  } else if (pps_hist_curr->id != pps_hist_prev->id) {
-    return TRUE;
-  } else if (slice_hdr_curr->field_pic_flag != slice_hdr_prev->field_pic_flag) {
-    return TRUE;
-  } else if (slice_hdr_curr->field_pic_flag
-      && (slice_hdr_curr->bottom_field_flag
-          != slice_hdr_prev->bottom_field_flag)) {
-    return TRUE;
-  } else if ((nalu_hist_curr->ref_idc == 0 || nalu_hist_prev->ref_idc == 0)
-      && nalu_hist_curr->ref_idc != nalu_hist_prev->ref_idc) {
-    return TRUE;
-  } else if (sps_hist_curr->pic_order_cnt_type == 0
-      && sps_hist_prev->pic_order_cnt_type == 0
-      && (slice_hdr_curr->pic_order_cnt_lsb
-          != slice_hdr_prev->pic_order_cnt_lsb
-          || slice_hdr_curr->delta_pic_order_cnt_bottom
-          != slice_hdr_prev->delta_pic_order_cnt_bottom)) {
-    return TRUE;
-  } else if (sps_hist_curr->pic_order_cnt_type == 1
-      && sps_hist_prev->pic_order_cnt_type == 1
-      && (slice_hdr_curr->delta_pic_order_cnt[0]
-          != slice_hdr_prev->delta_pic_order_cnt[0]
-          || slice_hdr_curr->delta_pic_order_cnt[1]
-          != slice_hdr_prev->delta_pic_order_cnt[1])) {
-    return TRUE;
-  } else if (nalu_hist_curr->idr_pic_flag != nalu_hist_prev->idr_pic_flag) {
-    return TRUE;
-  } else if (nalu_hist_curr->idr_pic_flag == 1
-      && nalu_hist_prev->idr_pic_flag == 1
-      && (slice_hdr_curr->idr_pic_id != slice_hdr_prev->idr_pic_id)) {
-    return TRUE;
-  } else if (slice_hdr_curr->first_mb_in_slice
-      <= slice_hdr_prev->first_mb_in_slice) {
-    return TRUE;
-  }
-  return FALSE;
-}
-
-static gboolean
-gst_h264_parse_received_first_vcl_nal_mvc (GstH264Parse * h264parse,
-    GstH264ParseHistorySlice * slice_hdr_prev)
-{
-  /* Ref. ITU-T H.264, H.7.4.1.2.4 */
-  GstH264ParseHistoryNalUnit *nalu_hist_curr
-      = &h264parse->history_nalu[HIST_IDX_CURR];
-  GstH264ParseHistoryNalUnit *nalu_hist_prev
-      = &h264parse->history_nalu[HIST_IDX_PREV];
-
-  if (nalu_hist_curr->view_id != nalu_hist_prev->view_id)
-    return TRUE;
-
-  return gst_h264_parse_received_first_vcl_nal_base (h264parse, slice_hdr_prev);
-}
-
-static gboolean
-gst_h264_parse_received_first_vcl_nal (GstH264Parse * h264parse)
-{
-  GstH264ParseHistorySlice *slice_hdr_prev
-      = &h264parse->history_slice[HIST_IDX_PREV];
-  GstH264ParseHistorySPS *sps_hist_prev
-      = &h264parse->history_sps[HIST_IDX_PREV];
-
-  if (slice_hdr_prev->valid) {
-    switch (sps_hist_prev->profile_idc) {
-      case GST_H264_PROFILE_BASELINE:
-      case GST_H264_PROFILE_MAIN:
-      case GST_H264_PROFILE_EXTENDED:
-      case GST_H264_PROFILE_HIGH:
-      case GST_H264_PROFILE_HIGH10:
-      case GST_H264_PROFILE_HIGH_422:
-      case GST_H264_PROFILE_HIGH_444:
-        return gst_h264_parse_received_first_vcl_nal_base (h264parse,
-            slice_hdr_prev);
-      case GST_H264_PROFILE_MULTIVIEW_HIGH:
-      case GST_H264_PROFILE_STEREO_HIGH:
-        return gst_h264_parse_received_first_vcl_nal_mvc (h264parse,
-            slice_hdr_prev);
-      case GST_H264_PROFILE_SCALABLE_BASELINE:
-      case GST_H264_PROFILE_SCALABLE_HIGH:
-        /* SVC not supported, should not be reached */
-        g_return_val_if_reached (FALSE);
-      default:
-        return FALSE;
-    }
-    return gst_h264_parse_received_first_vcl_nal_base (h264parse,
-        slice_hdr_prev);
-  }
-  return FALSE;
-}
-
-static gboolean
-is_potential_nonvcl_au_limit (GstH264NalUnit * nalu)
-{
-  gboolean ret = FALSE;
-  switch (nalu->type) {
-    case GST_H264_NAL_AU_DELIMITER:
-    case GST_H264_NAL_SPS:
-    case GST_H264_NAL_PPS:
-    case GST_H264_NAL_SEI:
-    case GST_H264_NAL_PREFIX_UNIT:
-    case GST_H264_NAL_SUBSET_SPS:
-    case GST_H264_NAL_DEPTH_SPS:
-    case GST_H264_NAL_RSV_1:
-    case GST_H264_NAL_RSV_2:
-      ret = TRUE;
-      break;
-    default:
-      break;
-  }
-  return ret;
-}
-
-static GstH264ParseBacklogStatus
-gst_h264_parse_update_backlog (GstH264Parse * h264parse, GstH264NalUnit * nalu)
-{
-  gboolean is_first_vcl_nal = FALSE;
-  GstH264ParserResult pres;
-  GstH264SliceHdr slice_hdr;
-  gboolean nvcl_before_cau_vcl = FALSE;
-
-  g_array_append_val (h264parse->nal_backlog, *nalu);
-
-#define LAST_IDX  (h264parse->nal_backlog->len -1)
-
-  /* Update nal_backlog_potential_au_first_idx following
-   * ref. ITU-T H.264 7.4.1.2.3 */
-  switch (nalu->type) {
-    case GST_H264_NAL_SLICE:
-    case GST_H264_NAL_SLICE_DPA:
-    case GST_H264_NAL_SLICE_DPB:
-    case GST_H264_NAL_SLICE_DPC:
-    case GST_H264_NAL_SLICE_IDR:
-    case GST_H264_NAL_SLICE_EXT:
-      GST_DEBUG_OBJECT (h264parse, "vcl nal (%u) added to backlog", nalu->type);
-      pres = gst_h264_parser_parse_slice_hdr (h264parse->nalparser, nalu,
-          &slice_hdr, FALSE, FALSE);
-
-      if (pres == GST_H264_PARSER_OK) {
-        gst_h264_parse_update_vcl_nal_history (h264parse, nalu, &slice_hdr);
-        is_first_vcl_nal = gst_h264_parse_received_first_vcl_nal (h264parse);
-      } else {
-        /* Reset vcl nal history */
-        h264parse->history_slice[HIST_IDX_PREV].valid = FALSE;
-
-        /* Reset backlog */
-        h264parse->bl_curr_au_last_vcl = -1;
-        h264parse->bl_next_au_first_vcl = 1;
-        h264parse->bl_next_au_first_nal = 1;
-        h264parse->bl_next_nal = 0;
-        g_array_set_size (h264parse->nal_backlog, 0);
-
-        GST_DEBUG_OBJECT (h264parse, "Failed to parse slice header");
-        return GST_H264_PARSE_BACKLOG_STATUS_UPD_FAILED;
-      }
-
-      if (h264parse->bl_curr_au_last_vcl == -1) {
-        /* initialization, first vcl nal reception */
-        h264parse->bl_curr_au_last_vcl = LAST_IDX;
-
-        /* set index above backlog, meaning not received */
-        h264parse->bl_next_au_first_vcl = h264parse->nal_backlog->len;
-        h264parse->bl_next_au_first_nal = h264parse->bl_next_au_first_vcl;
-      } else {
-
-        if (is_first_vcl_nal) {
-          h264parse->bl_next_au_first_vcl = LAST_IDX;
-
-          /* First AUD, SPS, PPS, SEI, PREFIX_UNIT, SUBSET_SPS, DEPTH_SPS,
-           * RSV1, RSV2 between last vcl nal of current AU and first vcl nal
-           * of next AU define the first nal of the next AU, otherwise
-           * first vcl nal of next AU is the first nal on next AU.*/
-          if (h264parse->bl_next_au_first_nal <= h264parse->bl_curr_au_last_vcl)
-            h264parse->bl_next_au_first_nal = h264parse->bl_next_au_first_vcl;
-          else
-            g_assert (h264parse->bl_next_au_first_nal <=
-                h264parse->bl_next_au_first_vcl);
-
-        } else {
-          h264parse->bl_next_au_first_vcl = h264parse->nal_backlog->len;
-
-          /*if previous vcl nal was not the last, non vcl nal can't be last,
-           * therefore we move index of last nal to the last received vcl
-           * nal.*/
-          h264parse->bl_next_au_first_nal = h264parse->bl_next_au_first_vcl;
-        }
-      }
-
-      break;
-    default:
-      if (h264parse->bl_curr_au_last_vcl == -1) {
-        /* if we didn't receive any vcl, any nal from next au hasn't been
-         * received yet. In this state all nal from backlog belong to
-         * current AU. */
-        h264parse->bl_next_au_first_nal = h264parse->nal_backlog->len;
-        h264parse->bl_next_au_first_vcl = h264parse->nal_backlog->len;
-      }
-
-      nvcl_before_cau_vcl =
-          h264parse->bl_next_au_first_nal <= h264parse->bl_curr_au_last_vcl;
-
-      if (is_potential_nonvcl_au_limit (nalu)) {
-        /* these nal define the the first nal of a new AU if they are between
-         * the last vcl nal (of current AU) and first vcl (of next AU).*/
-        if (nvcl_before_cau_vcl)
-          h264parse->bl_next_au_first_nal = LAST_IDX;
-
-        /* Not the most efficient way has this will done again in _process_nal
-         * but sps and pps must be parsed before parsing s slice hdr. */
-        if (nalu->type == GST_H264_NAL_SPS) {
-          GstH264SPS sps;
-          gst_h264_parser_parse_sps (h264parse->nalparser, nalu, &sps);
-
-          g_return_val_if_fail (sps.profile_idc !=
-              GST_H264_PROFILE_SCALABLE_BASELINE
-              && sps.profile_idc != GST_H264_PROFILE_SCALABLE_HIGH,
-              GST_H264_PARSE_BACKLOG_STATUS_NOT_SUPPORTED);
-        } else if (nalu->type == GST_H264_NAL_PPS) {
-          GstH264PPS pps;
-          gst_h264_parser_parse_pps (h264parse->nalparser, nalu, &pps);
-        } else if (nalu->type == GST_H264_NAL_SUBSET_SPS) {
-          GstH264SPS sps;
-          gst_h264_parser_parse_subset_sps (h264parse->nalparser, nalu, &sps);
-          g_return_val_if_fail (sps.profile_idc !=
-              GST_H264_PROFILE_SCALABLE_BASELINE
-              && sps.profile_idc != GST_H264_PROFILE_SCALABLE_HIGH,
-              GST_H264_PARSE_BACKLOG_STATUS_NOT_SUPPORTED);
-        }
-      }
-      GST_DEBUG_OBJECT (h264parse, "Non-vcl nal (%u) added to backlog",
-          nalu->type);
-      break;
-  }
-
-  return is_first_vcl_nal ? GST_H264_PARSE_BACKLOG_STATUS_AU_COMPLETE :
-      GST_H264_PARSE_BACKLOG_STATUS_AU_INCOMPLETE;
-}
-
-static void
-gst_h264_parse_trim_backlog (GstH264Parse * h264parse)
-{
-  g_array_remove_range (h264parse->nal_backlog, 0,
-      h264parse->bl_next_au_first_nal);
-  h264parse->bl_next_nal = 0;
-  h264parse->bl_curr_au_last_vcl =
-      h264parse->bl_next_au_first_vcl - h264parse->bl_next_au_first_nal;
-  h264parse->bl_next_au_first_nal = h264parse->bl_curr_au_last_vcl + 1;
-  h264parse->bl_next_au_first_vcl = h264parse->bl_next_au_first_nal;
-}
-
-static void
-gst_h264_parse_clear_backlog (GstH264Parse * h264parse)
-{
-  h264parse->bl_next_nal = 0;
-  g_array_remove_range (h264parse->nal_backlog, 0, h264parse->nal_backlog->len);
-  h264parse->bl_curr_au_last_vcl = -1;
-  h264parse->bl_next_au_first_nal = 1;
-  h264parse->bl_next_au_first_vcl = 1;
-}
-
-static gboolean
-gst_h264_parse_process_backlog_loop (GstH264Parse * h264parse,
-    gint curr_next_thresh, gboolean * aud_insert, guint8 * data,
-    gint * framesize)
-{
-  GstH264NalUnit *bnalu;
-  gint i, size = 0;
-
-  for (i = h264parse->bl_next_nal; i < h264parse->nal_backlog->len; i++) {
-    bnalu = &g_array_index (h264parse->nal_backlog, GstH264NalUnit, i);
-    if (i < curr_next_thresh) {
-      if (aud_insert != NULL && i == 0 &&
-          bnalu->type != GST_H264_NAL_AU_DELIMITER)
-        *aud_insert = TRUE;
-
-      bnalu->data = (guint8 *) data;
-      if (gst_h264_parse_process_nal (h264parse, bnalu) == FALSE)
-        return FALSE;
-
-      size = bnalu->offset + bnalu->size;
-      h264parse->bl_next_nal = i + 1;
-    } else {
-      /* section of backlog that belong to next AU */
-      bnalu->offset -= size;
-      bnalu->sc_offset -= size;
-    }
-  }
-
-  *framesize += size;
-  return TRUE;
-}
-
-static gboolean
-gst_h264_parse_process_backlog_nal (GstH264Parse * h264parse, gint * proc_size,
-    gboolean * aud_insert, guint8 * data, gboolean clear_bl,
-    gboolean au_completed)
-{
-  GstH264NalUnit *bnalu;
-  gint framesize = 0;
-
-  g_assert (h264parse->nal_backlog != NULL);
-  g_assert (h264parse->nal_backlog->len > 0);
-
-  bnalu = &g_array_index (h264parse->nal_backlog, GstH264NalUnit,
-      h264parse->nal_backlog->len - 1);
-  h264parse->current_off = bnalu->offset + bnalu->size;
-
-  /* If the index of the first NAL from next AU is after the current AU vcl
-   * and the AU is not completed, we can't send the this nal downstream since
-   * we might need to insert a AUD before and we will only know this when we've
-   * received a new vcl nal. In this scenario even if we are in NAL alignment
-   * mode we have to keep non vcl NAL, that can start a AU, and only send them
-   * down stream when we know if the belong to current AU, in which case we
-   * just send them or belong if it belong to next AU where we might need to
-   * insert a AUD. If the first nal from next AU is a AUD we don't need to wait
-   * the completion the first vcl from next AU, AUD is the start of next AU.
-   */
-  if (gst_h264_parse_process_backlog_loop (h264parse,
-          h264parse->bl_next_au_first_nal, aud_insert, data,
-          &framesize) == FALSE)
-    goto fail;
-
-  /* We've processed a complete AU */
-  if (au_completed) {
-    gst_h264_parse_trim_backlog (h264parse);
-  }
-
-  /* Process all backlog. Used when draining or output in NAL mode. */
-  if (gst_h264_parse_process_backlog_loop (h264parse,
-          h264parse->nal_backlog->len, aud_insert, data, &framesize) == FALSE)
-    goto fail;
-
-  if (clear_bl) {
-    gst_h264_parse_clear_backlog (h264parse);
-  }
-
-  /* Backlog content doesn't need to parsed again, adjust offset accordingly. */
-  h264parse->current_off -= framesize;
-
-  if (proc_size)
-    *proc_size = framesize;
-
-  return TRUE;
-
-fail:
-  gst_h264_parse_clear_backlog (h264parse);
-  return FALSE;
-}
-
-static gboolean
-gst_h264_parse_process_backlog (GstH264Parse * h264parse, gint * proc_size,
-    gboolean * aud_insert, guint8 * data, gboolean proc_nau, gboolean clear_bl)
-{
-  GstH264NalUnit *bnalu;
-  gint framesize = 0;
-
-  g_assert (h264parse->nal_backlog != NULL);
-  g_assert (h264parse->nal_backlog->len > 0);
-
-  bnalu = &g_array_index (h264parse->nal_backlog, GstH264NalUnit,
-      h264parse->nal_backlog->len - 1);
-  h264parse->current_off = bnalu->offset + bnalu->size;
-
-  if (gst_h264_parse_process_backlog_loop (h264parse,
-          h264parse->bl_next_au_first_nal, aud_insert, data,
-          &framesize) == FALSE)
-    goto fail;
-
-  /* We've processed a complete AU */
-  if (h264parse->bl_next_au_first_nal < h264parse->nal_backlog->len) {
-    gst_h264_parse_trim_backlog (h264parse);
-  }
-
-  /* Process all backlog. Used when draining or output in NAL mode. */
-  if (proc_nau) {
-    if (gst_h264_parse_process_backlog_loop (h264parse,
-            h264parse->nal_backlog->len, aud_insert, data, &framesize) == FALSE)
-      goto fail;
-  }
-
-  if (clear_bl) {
-    gst_h264_parse_clear_backlog (h264parse);
-  }
-
-  /* What is in the backlog doesn't need to parsed again, adjust offset
-   * accordingly.*/
-  h264parse->current_off -= framesize;
-
-  if (proc_size)
-    *proc_size = framesize;
-
-  return TRUE;
-
-fail:
-  gst_h264_parse_clear_backlog (h264parse);
-
-  return FALSE;
-}
-
 static GstFlowReturn
 gst_h264_parse_handle_frame (GstBaseParse * parse,
     GstBaseParseFrame * frame, gint * skipsize)
@@ -1816,12 +1352,9 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
   GstH264NalUnit nalu;
   GstH264ParserResult pres;
   gint framesize;
-  GstH264ParseBacklogStatus blstatus = FALSE;
 
   if (G_UNLIKELY (GST_BUFFER_FLAG_IS_SET (frame->buffer,
               GST_BUFFER_FLAG_DISCONT))) {
-    // If any input buffer is marked discont we propagate discont
-    // to parsed output buffer.
     h264parse->discont = TRUE;
   }
 
@@ -1865,28 +1398,14 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
 
   /* The parser is being drain, but no new data was added, just prentend this
    * AU is complete */
-  if (current_off == size) {
-    if (drain) {
-      GST_DEBUG_OBJECT (h264parse, "draining with no new data");
-      framesize = current_off;
-      if (!gst_h264_parse_process_backlog (h264parse, &framesize,
-              &h264parse->aud_insert, data, TRUE, FALSE)) {
-        *skipsize = current_off;
-        goto skip;
-      }
-
-      goto end;
-    } else {
-      /* All data already parsed, we need more data. */
-      goto more;
-    }
+  if (drain && current_off == size) {
+    GST_DEBUG_OBJECT (h264parse, "draining with no new data");
+    nalu.size = 0;
+    nalu.offset = current_off;
+    goto end;
   }
 
-  /* In some case the base class can reduce the amount of data it gave us on
-   * previous call. When this happen we just ask for more data. */
-  if (current_off > size) {
-    goto more;
-  }
+  g_assert (current_off < size);
   GST_DEBUG_OBJECT (h264parse, "last parse position %d", current_off);
 
   /* check for initial skip */
@@ -1926,9 +1445,6 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
       case GST_H264_PARSER_OK:
         GST_DEBUG_OBJECT (h264parse, "complete nal (offset, size): (%u, %u) ",
             nalu.offset, nalu.size);
-
-        if ((nalu.offset + nalu.size) == size)
-          nonext = TRUE;
         break;
       case GST_H264_PARSER_NO_NAL:
         /* In NAL alignment, assume the NAL is broken */
@@ -1982,16 +1498,11 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
         if (current_off == 0) {
           GST_DEBUG_OBJECT (h264parse, "skipping broken nal");
           *skipsize = nalu.offset;
-          h264parse->current_off = -1;
           goto skip;
         } else {
           GST_DEBUG_OBJECT (h264parse, "terminating au");
-          framesize = nalu.sc_offset;
-          if (!gst_h264_parse_process_backlog (h264parse, &framesize,
-                  &h264parse->aud_insert, data, FALSE, TRUE)) {
-            *skipsize = current_off;
-            goto skip;
-          }
+          nalu.size = 0;
+          nalu.offset = nalu.sc_offset;
           goto end;
         }
         break;
@@ -2000,73 +1511,79 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
         break;
     }
 
-    blstatus = gst_h264_parse_update_backlog (h264parse, &nalu);
-    if (blstatus == GST_H264_PARSE_BACKLOG_STATUS_UPD_FAILED) {
-      *skipsize = current_off;
-      GST_ERROR_OBJECT (h264parse, "Failed to update backlog");
+    GST_DEBUG_OBJECT (h264parse, "%p complete nal found. Off: %u, Size: %u",
+        data, nalu.offset, nalu.size);
+
+    if (gst_h264_parse_collect_nal (h264parse, &nalu)) {
+      h264parse->aud_needed = TRUE;
+      /* complete current frame, if it exist */
+      if (current_off > 0) {
+        nalu.size = 0;
+        nalu.offset = nalu.sc_offset;
+        h264parse->marker = TRUE;
+        break;
+      }
+    }
+
+    if (!gst_h264_parse_process_nal (h264parse, &nalu)) {
+      GST_WARNING_OBJECT (h264parse,
+          "broken/invalid nal Type: %d %s, Size: %u will be dropped",
+          nalu.type, _nal_name (nalu.type), nalu.size);
+      *skipsize = nalu.size;
       goto skip;
-    } else if (blstatus == GST_H264_PARSE_BACKLOG_STATUS_NOT_SUPPORTED) {
-      /* SVC is not supported */
-      GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
-          ("Error parsing H.264 stream"), ("Not supported H.264 stream"));
-      goto invalid_stream;
     }
 
-    if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) {
-      if (!gst_h264_parse_process_backlog_nal (h264parse, &framesize,
-              &h264parse->aud_insert, data, FALSE,
-              blstatus == GST_H264_PARSE_BACKLOG_STATUS_AU_COMPLETE)) {
-        *skipsize = current_off;
-      }
-
-      if (framesize > 0)
-        goto end;
-
-    } else if (h264parse->align == GST_H264_PARSE_ALIGN_AU) {
-      if (h264parse->in_align != GST_H264_PARSE_ALIGN_AU) {
-        if (blstatus == GST_H264_PARSE_BACKLOG_STATUS_AU_COMPLETE) {
-          if (!gst_h264_parse_process_backlog (h264parse, &framesize,
-                  &h264parse->aud_insert, data, FALSE, FALSE)) {
-            *skipsize = current_off;
-            goto skip;
-          }
-
-          if (framesize > 0)
-            goto end;
-
-        } else if (drain && nonext) {
-          if (!gst_h264_parse_process_backlog (h264parse, &framesize,
-                  &h264parse->aud_insert, data, TRUE, FALSE)) {
-            *skipsize = current_off;
-            goto skip;
-          }
-          goto end;
-        }
-      } else {
-
-        /* Accumulate all NALs from current AU in backlog */
-        if (nonext) {
-          /* input and output alignment are AU, there's nothing to do more than
-           * inserting a AUD if it's missing. */
-          if (!gst_h264_parse_process_backlog (h264parse, &framesize,
-                  &h264parse->aud_insert, data, TRUE, TRUE)) {
-            *skipsize = current_off;
-            goto skip;
-          }
-          goto end;
-        }
-      }
+    /* Make sure the next buffer will contain an AUD */
+    if (h264parse->aud_needed) {
+      h264parse->aud_insert = TRUE;
+      h264parse->aud_needed = FALSE;
     }
 
+    /* Do not push immediately if we don't have all headers. This ensure that
+     * our caps are complete, avoiding a renegotiation */
+    if (h264parse->align == GST_H264_PARSE_ALIGN_NAL &&
+        !GST_H264_PARSE_STATE_VALID (h264parse,
+            GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS))
+      frame->flags |= GST_BASE_PARSE_FRAME_FLAG_QUEUE;
+
+    /* if no next nal, we reached the end of this buffer */
     if (nonext) {
+      /* If there is a marker flag, or input is AU, we know this is complete */
+      if (GST_BUFFER_FLAG_IS_SET (frame->buffer, GST_BUFFER_FLAG_MARKER) ||
+          h264parse->in_align == GST_H264_PARSE_ALIGN_AU) {
+        h264parse->marker = TRUE;
+        break;
+      }
+
+      /* or if we are draining */
+      if (drain || h264parse->align == GST_H264_PARSE_ALIGN_NAL)
+        break;
+
       current_off = nalu.offset + nalu.size;
       goto more;
     }
 
+    /* If the output is NAL, we are done */
+    if (h264parse->align == GST_H264_PARSE_ALIGN_NAL)
+      break;
+
+    GST_DEBUG_OBJECT (h264parse, "Looking for more");
     current_off = nalu.offset + nalu.size;
-  }                             /* while end */
+
+    /* expect at least 3 bytes start_code, and 1 bytes NALU header.
+     * the length of the NALU payload can be zero.
+     * (e.g. EOS/EOB placed at the end of an AU.) */
+    if (size - current_off < 4) {
+      /* Finish the frame if there is no more data in the stream */
+      if (drain)
+        break;
+
+      goto more;
+    }
+  }
 
 end:
+  framesize = nalu.offset + nalu.size;
 
   gst_buffer_unmap (buffer, &map);
 
diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.h b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.h
index 0ec18a87cf..1b6939de31 100644
--- a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.h
+++ b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.h
@@ -48,40 +48,6 @@ typedef struct _H264Params H264Params;
 
 GType gst_h264_parse_get_type (void);
 
-typedef struct
-{
-  gboolean valid;
-  guint16 frame_num;
-  guint8 field_pic_flag;
-  guint8 bottom_field_flag;
-  guint16 idr_pic_id;
-  gint32 delta_pic_order_cnt[2];
-  guint16 pic_order_cnt_lsb;
-  guint32 delta_pic_order_cnt_bottom;
-  guint32 first_mb_in_slice;
-} GstH264ParseHistorySlice;
-
-typedef struct
-{
-  guint16 ref_idc;
-  guint8 idr_pic_flag;
-
-  /* For MVC Extension */
-  guint16 view_id;
-} GstH264ParseHistoryNalUnit;
-
-typedef struct
-{
-  guint8 pic_order_cnt_type;
-  guint8 profile_idc;
-} GstH264ParseHistorySPS;
-
-typedef struct
-{
-  gint id;
-} GstH264ParseHistoryPPS;
-
-
 typedef struct _GstH264Parse GstH264Parse;
 typedef struct _GstH264ParseClass GstH264ParseClass;
 
@@ -191,21 +157,6 @@ struct _GstH264Parse
   GstVideoMultiviewFlags multiview_flags;
   gboolean first_in_bundle;
 
-  /* For insertion of AU Delimiter */
-  GArray *nal_backlog;
-
-  /* Index of last vcl nal of current AU in backlog */
-  gint bl_curr_au_last_vcl;
-
-  /* Index of first vcl nal of next AU in backlog */
-  gint bl_next_au_first_vcl;
-
-  /* Index of first nal of next AU in backlog */
-  gint bl_next_au_first_nal;
-
-  /* Index of next nal to be processed in backlog */
-  gint bl_next_nal;
-
   GstVideoParseUserData user_data;
   GstVideoParseUserDataUnregistered user_data_unregistered;
 
@@ -217,12 +168,6 @@ struct _GstH264Parse
 
   /* For forward predicted trickmode */
   gboolean discard_bidirectional;
-
-  /* First VCL NAL unit of primary code picuture detection context */
-  GstH264ParseHistorySlice history_slice[2];
-  GstH264ParseHistoryNalUnit history_nalu[2];
-  GstH264ParseHistorySPS history_sps[2];
-  GstH264ParseHistoryPPS history_pps[2];
 };
 
 struct _GstH264ParseClass