diff --git a/ext/vorbis/gstvorbisdec.c b/ext/vorbis/gstvorbisdec.c
index 6e6601a29c..9b0b50bd16 100644
--- a/ext/vorbis/gstvorbisdec.c
+++ b/ext/vorbis/gstvorbisdec.c
@@ -458,9 +458,6 @@ vorbis_handle_header_packet (GstVorbisDec * vd, ogg_packet * packet)
       break;
   }
 
-  /* consumer header packet/frame */
-  gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (vd), NULL, 1);
-
   return res;
 
   /* ERRORS */
@@ -693,6 +690,8 @@ vorbis_dec_handle_frame (GstAudioDecoder * dec, GstBuffer * buffer)
       goto done;
     }
     result = vorbis_handle_header_packet (vd, packet);
+    /* consumer header packet/frame */
+    gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (vd), NULL, 1);
   } else {
     GstClockTime timestamp, duration;