diff --git a/gst-libs/gst/adaptivedemux/gstadaptivedemux.c b/gst-libs/gst/adaptivedemux/gstadaptivedemux.c
index 5e8a9ff4c5..5867ed0568 100644
--- a/gst-libs/gst/adaptivedemux/gstadaptivedemux.c
+++ b/gst-libs/gst/adaptivedemux/gstadaptivedemux.c
@@ -1266,6 +1266,7 @@ gst_adaptive_demux_expose_streams (GstAdaptiveDemux * demux)
       gst_task_stop (stream->download_task);
       g_mutex_lock (&stream->fragment_download_lock);
       stream->cancelled = TRUE;
+      stream->replaced = TRUE;
       g_cond_signal (&stream->fragment_download_cond);
       g_mutex_unlock (&stream->fragment_download_lock);
     }
@@ -2010,6 +2011,7 @@ gst_adaptive_demux_start_tasks (GstAdaptiveDemux * demux,
     if (!start_preroll_streams) {
       g_mutex_lock (&stream->fragment_download_lock);
       stream->cancelled = FALSE;
+      stream->replaced = FALSE;
       g_mutex_unlock (&stream->fragment_download_lock);
     }
 
@@ -3778,6 +3780,9 @@ gst_adaptive_demux_stream_download_loop (GstAdaptiveDemuxStream * stream)
           goto end;
         }
         gst_task_stop (stream->download_task);
+        if (stream->replaced) {
+          goto end;
+        }
       } else {
         gst_task_stop (stream->download_task);
         if (gst_adaptive_demux_combine_flows (demux) == GST_FLOW_EOS) {
diff --git a/gst-libs/gst/adaptivedemux/gstadaptivedemux.h b/gst-libs/gst/adaptivedemux/gstadaptivedemux.h
index d59314c9b1..52e029e6d5 100644
--- a/gst-libs/gst/adaptivedemux/gstadaptivedemux.h
+++ b/gst-libs/gst/adaptivedemux/gstadaptivedemux.h
@@ -160,7 +160,8 @@ struct _GstAdaptiveDemuxStream
   GMutex fragment_download_lock;
   GCond fragment_download_cond;
   gboolean download_finished;   /* protected by fragment_download_lock */
-  gboolean cancelled;           /* protected by fragment_download_lock */
+  gboolean cancelled; /* protected by fragment_download_lock */
+  gboolean replaced; /* replaced in a bitrate switch (used with cancelled) */
   gboolean src_at_ready;     /* protected by fragment_download_lock */
   gboolean starting_fragment;
   gboolean first_fragment_buffer;