diff --git a/gst/flv/gstflvmux.c b/gst/flv/gstflvmux.c
index b241bcdd0b..516d45c1d8 100644
--- a/gst/flv/gstflvmux.c
+++ b/gst/flv/gstflvmux.c
@@ -1819,12 +1819,18 @@ static GstClockTime
 gst_flv_mux_get_next_time (GstAggregator * aggregator)
 {
   GstFlvMux *mux = GST_FLV_MUX (aggregator);
+  GstAggregatorPad *agg_audio_pad = GST_AGGREGATOR_PAD_CAST (mux->audio_pad);
+  GstAggregatorPad *agg_video_pad = GST_AGGREGATOR_PAD_CAST (mux->video_pad);
 
   GST_OBJECT_LOCK (aggregator);
   if (mux->state == GST_FLV_MUX_STATE_HEADER &&
       ((mux->audio_pad && mux->audio_pad->codec == G_MAXUINT) ||
           (mux->video_pad && mux->video_pad->codec == G_MAXUINT)))
     goto wait_for_data;
+
+  if (!((agg_audio_pad && gst_aggregator_pad_has_buffer (agg_audio_pad)) ||
+          (agg_video_pad && gst_aggregator_pad_has_buffer (agg_video_pad))))
+    goto wait_for_data;
   GST_OBJECT_UNLOCK (aggregator);
 
   return gst_flv_mux_get_next_time_for_segment (aggregator,