diff --git a/ext/hls/gsthlsdemux.c b/ext/hls/gsthlsdemux.c
index 5d39ed03c9..466f665bb6 100644
--- a/ext/hls/gsthlsdemux.c
+++ b/ext/hls/gsthlsdemux.c
@@ -805,7 +805,7 @@ _src_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer)
     gsize available;
 
     /* restart the decrypting lib for a new fragment */
-    if (demux->starting_fragment) {
+    if (demux->reset_crypto) {
       GstFragment *key_fragment;
       GstBuffer *key_buffer;
       GstMapInfo key_info;
@@ -842,6 +842,8 @@ _src_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer)
       gst_buffer_unmap (key_buffer, &key_info);
       gst_buffer_unref (key_buffer);
       g_object_unref (key_fragment);
+
+      demux->reset_crypto = FALSE;
     }
 
     gst_adapter_push (demux->adapter, buffer);
@@ -1989,6 +1991,7 @@ gst_hls_demux_get_next_fragment (GstHLSDemux * demux,
   demux->current_timestamp = timestamp;
   demux->current_duration = duration;
   demux->starting_fragment = TRUE;
+  demux->reset_crypto = TRUE;
   demux->current_key = key;
   demux->current_iv = iv;
 
diff --git a/ext/hls/gsthlsdemux.h b/ext/hls/gsthlsdemux.h
index 5b8e69b512..76f6f84d10 100644
--- a/ext/hls/gsthlsdemux.h
+++ b/ext/hls/gsthlsdemux.h
@@ -121,6 +121,7 @@ struct _GstHLSDemux
   GstClockTime current_timestamp;
   GstClockTime current_duration;
   gboolean starting_fragment;
+  gboolean reset_crypto;
   gint64 download_start_time;
   gint64 download_total_time;
   gint64 download_total_bytes;