diff --git a/gst/matroska/matroska-mux.c b/gst/matroska/matroska-mux.c
index b5a2291224..babf2c948a 100644
--- a/gst/matroska/matroska-mux.c
+++ b/gst/matroska/matroska-mux.c
@@ -1621,7 +1621,8 @@ opus_streamheader_to_codecdata (const GValue * streamheader,
   if (bufarr->len != 1 && bufarr->len != 2)
     goto wrong_count;
 
-  context->xiph_headers_to_skip = bufarr->len;
+  /* Opus headers are not in-band */
+  context->xiph_headers_to_skip = 0;
 
   bufval = &g_array_index (bufarr, GValue, 0);
   if (G_VALUE_TYPE (bufval) != GST_TYPE_BUFFER) {