diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c
index 16850870ed..d813127ef8 100644
--- a/gst/videoparsers/gsth264parse.c
+++ b/gst/videoparsers/gsth264parse.c
@@ -653,7 +653,7 @@ gst_h264_parse_check_valid_frame (GstBaseParse * parse,
           *skipsize = nalu.offset;
 
           GST_DEBUG_OBJECT (h264parse, "skipping broken nal");
-          return FALSE;
+          goto invalid;
         } else {
           nalu.size = 0;
           goto end;
@@ -675,7 +675,7 @@ gst_h264_parse_check_valid_frame (GstBaseParse * parse,
           /*  Can't parse the nalu */
           if (size - h264parse->nalu.offset < 2) {
             *skipsize = nalu.offset;
-            return FALSE;
+            goto invalid;
           }
 
           /* We parse it anyway */
@@ -715,6 +715,10 @@ more:
   if (!h264parse->nalu.size) {
     /* skip up to initial startcode */
     *skipsize = h264parse->nalu.sc_offset;
+    /* but mind some stuff will have been skipped */
+    g_assert (current_off >= *skipsize);
+    current_off -= *skipsize;
+    h264parse->nalu.sc_offset = 0;
   } else {
     *skipsize = 0;
   }
@@ -723,6 +727,10 @@ more:
   h264parse->current_off = current_off;
 
   return FALSE;
+
+invalid:
+  gst_h264_parse_reset_frame (h264parse);
+  return FALSE;
 }
 
 /* byte together avc codec data based on collected pps and sps so far */