diff --git a/gst/asfdemux/asfpacket.c b/gst/asfdemux/asfpacket.c
index fc9552d282..5942730dc9 100644
--- a/gst/asfdemux/asfpacket.c
+++ b/gst/asfdemux/asfpacket.c
@@ -148,6 +148,7 @@ gst_asf_payload_queue_for_stream (GstASFDemux * demux, AsfPayload * payload,
 
   /* make timestamps start from 0 */
   if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (demux->first_ts) &&
+          GST_CLOCK_TIME_IS_VALID (payload->ts) &&
           demux->first_ts < payload->ts))
     payload->ts -= demux->first_ts;
   else