diff --git a/subprojects/gst-plugins-good/gst/matroska/matroska-demux.c b/subprojects/gst-plugins-good/gst/matroska/matroska-demux.c
index 01d754c3eb..f380419b89 100644
--- a/subprojects/gst-plugins-good/gst/matroska/matroska-demux.c
+++ b/subprojects/gst-plugins-good/gst/matroska/matroska-demux.c
@@ -351,6 +351,7 @@ gst_matroska_demux_reset (GstElement * element)
   demux->index_offset = 0;
   demux->seekable = FALSE;
   demux->need_segment = FALSE;
+  demux->upstream_format_is_time = FALSE;
   demux->segment_seqnum = 0;
   demux->requested_seek_time = GST_CLOCK_TIME_NONE;
   demux->seek_offset = -1;
@@ -4683,10 +4684,14 @@ gst_matroska_demux_parse_blockgroup_or_simpleblock (GstMatroskaDemux * demux,
             GST_TIME_ARGS (demux->common.segment.position));
         clace_time = demux->common.segment.position;
       }
-      segment->start = clace_time;
-      segment->stop = demux->common.segment.stop;
-      segment->time = segment->start - demux->stream_start_time;
-      segment->position = segment->start - demux->stream_start_time;
+
+      /* We shouldn't modify upstream driven TIME FORMAT segment */
+      if (!demux->upstream_format_is_time) {
+        segment->start = clace_time;
+        segment->stop = demux->common.segment.stop;
+        segment->time = segment->start - demux->stream_start_time;
+        segment->position = segment->start - demux->stream_start_time;
+      }
       GST_DEBUG_OBJECT (demux,
           "generated segment starting at %" GST_TIME_FORMAT ": %"
           GST_SEGMENT_FORMAT, GST_TIME_ARGS (lace_time), segment);
@@ -6219,6 +6224,17 @@ gst_matroska_demux_handle_sink_event (GstPad * pad, GstObject * parent,
         goto exit;
       }
 
+      if (segment->format == GST_FORMAT_TIME) {
+        demux->upstream_format_is_time = TRUE;
+        demux->segment_seqnum = gst_event_get_seqnum (event);
+        gst_segment_copy_into (segment, &demux->common.segment);
+        GST_DEBUG_OBJECT (demux, "Got segment in TIME format: %" GST_PTR_FORMAT,
+            event);
+        goto exit;
+      }
+
+      demux->upstream_format_is_time = FALSE;
+
       /* we only expect a BYTE segment, e.g. following a seek */
       if (segment->format != GST_FORMAT_BYTES) {
         GST_DEBUG_OBJECT (demux, "unsupported segment format, ignoring");
diff --git a/subprojects/gst-plugins-good/gst/matroska/matroska-demux.h b/subprojects/gst-plugins-good/gst/matroska/matroska-demux.h
index dda76692fa..d9e5b556fa 100644
--- a/subprojects/gst-plugins-good/gst/matroska/matroska-demux.h
+++ b/subprojects/gst-plugins-good/gst/matroska/matroska-demux.h
@@ -107,6 +107,10 @@ typedef struct _GstMatroskaDemux {
   GstPad                  *deferred_seek_pad;
   gboolean                 need_segment;
   guint32                  segment_seqnum;
+  /* If TRUE, matroskamux received upstream newsegment in TIME format
+   * which likely means that upstream is driving the pipeline (such as
+   * adaptive demuxers) */
+  gboolean                 upstream_format_is_time;
 
   /* reverse playback */
   GArray                  *seek_index;