diff --git a/ext/ogg/gstoggdemux.c b/ext/ogg/gstoggdemux.c
index f31ecfa4d9..4cbe0cc403 100644
--- a/ext/ogg/gstoggdemux.c
+++ b/ext/ogg/gstoggdemux.c
@@ -784,7 +784,7 @@ gst_ogg_pad_submit_packet (GstOggPad * pad, ogg_packet * packet)
 
   if (!pad->have_type) {
     pad->have_type = gst_ogg_stream_setup_map (&pad->map, packet);
-    if (!pad->have_type) {
+    if (!pad->have_type && !pad->map.caps) {
       pad->map.caps = gst_caps_new_empty_simple ("application/x-unknown");
     }
     if (pad->map.is_skeleton) {