diff --git a/subprojects/gst-plugins-good/gst/matroska/matroska-demux.c b/subprojects/gst-plugins-good/gst/matroska/matroska-demux.c
index ae3588487b..52a76b9d9c 100644
--- a/subprojects/gst-plugins-good/gst/matroska/matroska-demux.c
+++ b/subprojects/gst-plugins-good/gst/matroska/matroska-demux.c
@@ -3512,6 +3512,12 @@ gst_matroska_demux_update_tracks (GstMatroskaDemux * demux, GstEbmlRead * ebml)
         new_track->pad = old_track->pad;
         new_track->index = old_track->index;
         new_track->pos = old_track->pos;
+
+        /* If index table is empty, do not ref it, we will try to fallback
+         * to the generic one from read-common in such case */
+        if (old_track->index_table && old_track->index_table->len > 0)
+          new_track->index_table = g_array_ref (old_track->index_table);
+
         g_ptr_array_index (demux->common.src, old_track_index) = new_track;
         gst_pad_set_element_private (new_track->pad, new_track);
 
diff --git a/subprojects/gst-plugins-good/gst/matroska/matroska-read-common.c b/subprojects/gst-plugins-good/gst/matroska/matroska-read-common.c
index 2b63c03d56..eb317644cc 100644
--- a/subprojects/gst-plugins-good/gst/matroska/matroska-read-common.c
+++ b/subprojects/gst-plugins-good/gst/matroska/matroska-read-common.c
@@ -591,10 +591,12 @@ gst_matroska_read_common_do_index_seek (GstMatroskaReadCommon * common,
   GArray *index;
 
   /* find entry just before or at the requested position */
-  if (track && track->index_table)
+  if (track && track->index_table) {
     index = track->index_table;
-  else
+  } else {
+    GST_DEBUG_OBJECT (common->sinkpad, "Missing track index table");
     index = common->index;
+  }
 
   if (!index || !index->len)
     return NULL;