diff --git a/subprojects/gst-plugins-good/gst/audioparsers/gstaacparse.c b/subprojects/gst-plugins-good/gst/audioparsers/gstaacparse.c
index b282873667..28e953ef58 100644
--- a/subprojects/gst-plugins-good/gst/audioparsers/gstaacparse.c
+++ b/subprojects/gst-plugins-good/gst/audioparsers/gstaacparse.c
@@ -210,7 +210,9 @@ gst_aac_parse_set_src_caps (GstAacParse * aacparse, GstCaps * sink_caps)
       stream_format = NULL;
   }
 
-  /* Generate codec data to be able to set profile/level on the caps */
+  /* Generate codec data to be able to set profile/level on the caps.
+   * The codec_data data is according to AudioSpecificConfig,
+   * ISO/IEC 14496-3, 1.6.2.1 */
   sample_rate_idx =
       gst_codec_utils_aac_get_index_from_sample_rate (aacparse->sample_rate);
   if (sample_rate_idx < 0)
@@ -231,39 +233,38 @@ gst_aac_parse_set_src_caps (GstAacParse * aacparse, GstCaps * sink_caps)
 
   peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (aacparse), NULL);
   if (peercaps && !gst_caps_can_intersect (src_caps, peercaps)) {
-    GST_DEBUG_OBJECT (GST_BASE_PARSE (aacparse)->srcpad,
-        "Caps can not intersect");
+    GstCaps *convcaps = gst_caps_copy (src_caps);
+    GstStructure *cs = gst_caps_get_structure (convcaps, 0);
+
+    GST_DEBUG_OBJECT (aacparse, "Caps do not intersect: parsed %" GST_PTR_FORMAT
+        " and peer %" GST_PTR_FORMAT, src_caps, peercaps);
+
     if (aacparse->header_type == DSPAAC_HEADER_ADTS) {
-      GST_DEBUG_OBJECT (GST_BASE_PARSE (aacparse)->srcpad,
-          "Input is ADTS, trying raw");
-      gst_caps_set_simple (src_caps, "stream-format", G_TYPE_STRING, "raw",
-          NULL);
-      if (gst_caps_can_intersect (src_caps, peercaps)) {
-        GstBuffer *codec_data_buffer;
+      GstBuffer *codec_data_buffer = gst_buffer_new_and_alloc (2);
 
-        GST_DEBUG_OBJECT (GST_BASE_PARSE (aacparse)->srcpad,
-            "Caps can intersect, we will drop the ADTS layer");
+      gst_buffer_fill (codec_data_buffer, 0, codec_data, 2);
+      gst_structure_set (cs, "stream-format", G_TYPE_STRING, "raw",
+          "codec_data", GST_TYPE_BUFFER, codec_data_buffer, NULL);
+
+      if (gst_caps_can_intersect (convcaps, peercaps)) {
+        GST_DEBUG_OBJECT (aacparse, "Converting from ADTS to raw");
         aacparse->output_header_type = DSPAAC_HEADER_NONE;
-
-        /* The codec_data data is according to AudioSpecificConfig,
-           ISO/IEC 14496-3, 1.6.2.1 */
-        codec_data_buffer = gst_buffer_new_and_alloc (2);
-        gst_buffer_fill (codec_data_buffer, 0, codec_data, 2);
-        gst_caps_set_simple (src_caps, "codec_data", GST_TYPE_BUFFER,
-            codec_data_buffer, NULL);
-        gst_buffer_unref (codec_data_buffer);
+        gst_caps_replace (&src_caps, convcaps);
       }
+
+      gst_buffer_unref (codec_data_buffer);
     } else if (aacparse->header_type == DSPAAC_HEADER_NONE) {
-      GST_DEBUG_OBJECT (GST_BASE_PARSE (aacparse)->srcpad,
-          "Input is raw, trying ADTS");
-      gst_caps_set_simple (src_caps, "stream-format", G_TYPE_STRING, "adts",
-          NULL);
-      if (gst_caps_can_intersect (src_caps, peercaps)) {
-        GST_DEBUG_OBJECT (GST_BASE_PARSE (aacparse)->srcpad,
-            "Caps can intersect, we will prepend ADTS headers");
+      gst_structure_set (cs, "stream-format", G_TYPE_STRING, "adts", NULL);
+      gst_structure_remove_field (cs, "codec_data");
+
+      if (gst_caps_can_intersect (convcaps, peercaps)) {
+        GST_DEBUG_OBJECT (aacparse, "Converting from raw to ADTS");
         aacparse->output_header_type = DSPAAC_HEADER_ADTS;
+        gst_caps_replace (&src_caps, convcaps);
       }
     }
+
+    gst_caps_unref (convcaps);
   }
   if (peercaps)
     gst_caps_unref (peercaps);