diff --git a/gst/matroska/matroska-demux.c b/gst/matroska/matroska-demux.c
index b67112cc68..4318a2f346 100644
--- a/gst/matroska/matroska-demux.c
+++ b/gst/matroska/matroska-demux.c
@@ -3278,6 +3278,7 @@ gst_matroska_demux_update_tracks (GstMatroskaDemux * demux, GstEbmlRead * ebml)
         if (!gst_caps_is_equal (old_track->caps, new_track->caps)) {
           gst_pad_set_caps (new_track->pad, new_track->caps);
         }
+        gst_caps_replace (&old_track->caps, NULL);
 
         if (!gst_tag_list_is_equal (old_track->tags, new_track->tags)) {
           GST_DEBUG_OBJECT (old_track->pad, "Sending tags %p: %"
diff --git a/gst/matroska/matroska-ids.c b/gst/matroska/matroska-ids.c
index 517956a3fe..5716b45874 100644
--- a/gst/matroska/matroska-ids.c
+++ b/gst/matroska/matroska-ids.c
@@ -335,6 +335,7 @@ gst_matroska_track_free (GstMatroskaTrackContext * track)
   g_free (track->language);
   g_free (track->codec_priv);
   g_free (track->codec_state);
+  gst_caps_replace (&track->caps, NULL);
 
   if (track->encodings != NULL) {
     int i;
diff --git a/gst/matroska/matroska-read-common.c b/gst/matroska/matroska-read-common.c
index 12085503ae..a952e740da 100644
--- a/gst/matroska/matroska-read-common.c
+++ b/gst/matroska/matroska-read-common.c
@@ -3289,7 +3289,6 @@ gst_matroska_read_common_reset (GstElement * element,
       if (context->pad != NULL)
         gst_element_remove_pad (element, context->pad);
 
-      gst_caps_replace (&context->caps, NULL);
       gst_matroska_track_free (context);
     }
     g_ptr_array_free (ctx->src, TRUE);