diff --git a/ext/jpeg/gstjpegdec.c b/ext/jpeg/gstjpegdec.c
index f2fb9904cf..b8f4ed6fb2 100644
--- a/ext/jpeg/gstjpegdec.c
+++ b/ext/jpeg/gstjpegdec.c
@@ -443,6 +443,10 @@ gst_jpeg_dec_parse (GstVideoDecoder * bdec, GstVideoCodecFrame * frame,
   if (at_eos) {
     GST_DEBUG ("Flushing all data out");
     toadd = size;
+
+    /* If we have leftover data, throw it away */
+    if (!dec->saw_header)
+      goto drop_frame;
     goto have_full_frame;
   }
 
@@ -460,6 +464,7 @@ gst_jpeg_dec_parse (GstVideoDecoder * bdec, GstVideoCodecFrame * frame,
   if (ret) {
     gst_adapter_flush (adapter, ret);
     size -= ret;
+    dec->saw_header = TRUE;
   }
 
   while (1) {
@@ -497,6 +502,7 @@ gst_jpeg_dec_parse (GstVideoDecoder * bdec, GstVideoCodecFrame * frame,
     if (value == 0xd9) {
       GST_DEBUG ("0x%08x: EOI marker", offset + 2);
       /* clear parse state */
+      dec->saw_header = FALSE;
       dec->parse_resync = FALSE;
       dec->parse_offset = 0;
       toadd = offset + 4;
@@ -589,6 +595,9 @@ have_full_frame:
   if (toadd)
     gst_video_decoder_add_to_frame (bdec, toadd);
   return gst_video_decoder_have_frame (bdec);
+
+drop_frame:
+  return GST_VIDEO_DECODER_FLOW_DROPPED;
 }
 
 
@@ -1407,6 +1416,7 @@ gst_jpeg_dec_reset (GstVideoDecoder * bdec, gboolean hard)
   dec->parse_offset = 0;
   dec->parse_entropy_len = 0;
   dec->parse_resync = FALSE;
+  dec->saw_header = FALSE;
 
   return TRUE;
 }