diff --git a/gst/isomp4/gstqtmux.c b/gst/isomp4/gstqtmux.c
index d9e843b5fe..5cabb8f380 100644
--- a/gst/isomp4/gstqtmux.c
+++ b/gst/isomp4/gstqtmux.c
@@ -3509,9 +3509,10 @@ gst_qt_mux_audio_sink_set_caps (GstQTPad * qtpad, GstCaps * caps)
               "assuming 'raw'");
         }
 
-        if (!codec_data || gst_buffer_get_size ((GstBuffer *) codec_data) < 2)
+        if (!codec_data || gst_buffer_get_size ((GstBuffer *) codec_data) < 2) {
           GST_WARNING_OBJECT (qtmux, "no (valid) codec_data for AAC audio");
-        else {
+          goto refuse_caps;
+        } else {
           guint8 profile;
 
           gst_buffer_extract ((GstBuffer *) codec_data, 0, &profile, 1);