diff --git a/gst/asfdemux/asfpacket.c b/gst/asfdemux/asfpacket.c
index e973a7e885..5266f075b8 100644
--- a/gst/asfdemux/asfpacket.c
+++ b/gst/asfdemux/asfpacket.c
@@ -121,6 +121,8 @@ static void
 gst_asf_payload_queue_for_stream (GstASFDemux * demux, AsfPayload * payload,
     AsfStream * stream)
 {
+  GST_DEBUG_OBJECT (demux, "Got payload for stream %d ts:%" GST_TIME_FORMAT,
+      stream->id, GST_TIME_ARGS (payload->ts));
   /* remember the first timestamp in the stream */
   if (!GST_CLOCK_TIME_IS_VALID (demux->first_ts) &&
       GST_CLOCK_TIME_IS_VALID (payload->ts)) {
@@ -134,19 +136,13 @@ gst_asf_payload_queue_for_stream (GstASFDemux * demux, AsfPayload * payload,
     }
   }
 
-  /* better drop a few frames at the beginning than send bogus timestamps */
-  if (G_UNLIKELY (payload->ts < demux->first_ts)) {
-    GST_LOG_OBJECT (stream->pad, "Dropping payload with timestamp %"
-        GST_TIME_FORMAT " which is before the first timestamp %"
-        GST_TIME_FORMAT, GST_TIME_ARGS (payload->ts),
-        GST_TIME_ARGS (demux->first_ts));
-    gst_buffer_replace (&payload->buf, NULL);
-    return;
-  }
-
   /* make timestamps start from 0 */
-  if (!demux->streaming)
-    payload->ts -= demux->first_ts;
+  if (!demux->streaming) {
+    if (demux->first_ts < payload->ts)
+      payload->ts -= demux->first_ts;
+    else
+      payload->ts = 0;
+  }
 
   /* remove any incomplete payloads that will never be completed */
   while (stream->payloads->len > 0) {