diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c
index 8f12151967..83c985393a 100644
--- a/gst/videoparsers/gsth264parse.c
+++ b/gst/videoparsers/gsth264parse.c
@@ -1384,7 +1384,8 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
         }
         break;
       case GST_H264_PARSER_NO_NAL:
-        *skipsize = size - 3;
+        /* Start code may have up to 4 bytes */
+        *skipsize = size - 4;
         goto skip;
         break;
       default:
diff --git a/gst/videoparsers/gsth265parse.c b/gst/videoparsers/gsth265parse.c
index 0c4ca177bd..849e743633 100644
--- a/gst/videoparsers/gsth265parse.c
+++ b/gst/videoparsers/gsth265parse.c
@@ -1208,7 +1208,10 @@ gst_h265_parse_handle_frame (GstBaseParse * parse,
         }
         break;
       case GST_H265_PARSER_NO_NAL:
-        *skipsize = size - 3;
+        /* start code may have up to 4 bytes, and we may also get that return
+         * value if only one of the two header bytes are present, make sure
+         * not to skip too much */
+        *skipsize = size - 5;
         goto skip;
       default:
         /* should not really occur either */