diff --git a/gst-libs/gst/audio/gstaudiodecoder.c b/gst-libs/gst/audio/gstaudiodecoder.c
index 82997f445d..2ba58aa725 100644
--- a/gst-libs/gst/audio/gstaudiodecoder.c
+++ b/gst-libs/gst/audio/gstaudiodecoder.c
@@ -3012,6 +3012,10 @@ gst_audio_decoder_set_latency (GstAudioDecoder * dec,
   dec->priv->ctx.min_latency = min;
   dec->priv->ctx.max_latency = max;
   GST_OBJECT_UNLOCK (dec);
+
+  /* post latency message on the bus */
+  gst_element_post_message (GST_ELEMENT (dec),
+      gst_message_new_latency (GST_OBJECT (dec)));
 }
 
 /**
diff --git a/gst-libs/gst/audio/gstaudioencoder.c b/gst-libs/gst/audio/gstaudioencoder.c
index 04d239ec24..3f2027264f 100644
--- a/gst-libs/gst/audio/gstaudioencoder.c
+++ b/gst-libs/gst/audio/gstaudioencoder.c
@@ -1250,8 +1250,6 @@ gst_audio_encoder_sink_setcaps (GstAudioEncoder * enc, GstCaps * caps)
   GstAudioInfo state;
   gboolean res = TRUE;
   guint old_rate;
-  GstClockTime old_min_latency;
-  GstClockTime old_max_latency;
 
   klass = GST_AUDIO_ENCODER_GET_CLASS (enc);
 
@@ -1294,12 +1292,6 @@ gst_audio_encoder_sink_setcaps (GstAudioEncoder * enc, GstCaps * caps)
   enc->priv->ctx.frame_max = 0;
   enc->priv->ctx.lookahead = 0;
 
-  /* element might report latency */
-  GST_OBJECT_LOCK (enc);
-  old_min_latency = ctx->min_latency;
-  old_max_latency = ctx->max_latency;
-  GST_OBJECT_UNLOCK (enc);
-
   if (klass->set_format)
     res = klass->set_format (enc, &state);
 
@@ -1313,18 +1305,6 @@ gst_audio_encoder_sink_setcaps (GstAudioEncoder * enc, GstCaps * caps)
     goto exit;
   }
 
-  /* notify if new latency */
-  GST_OBJECT_LOCK (enc);
-  if ((ctx->min_latency > 0 && ctx->min_latency != old_min_latency) ||
-      (ctx->max_latency > 0 && ctx->max_latency != old_max_latency)) {
-    GST_OBJECT_UNLOCK (enc);
-    /* post latency message on the bus */
-    gst_element_post_message (GST_ELEMENT (enc),
-        gst_message_new_latency (GST_OBJECT (enc)));
-    GST_OBJECT_LOCK (enc);
-  }
-  GST_OBJECT_UNLOCK (enc);
-
 exit:
 
   GST_AUDIO_ENCODER_STREAM_UNLOCK (enc);
@@ -2152,6 +2132,10 @@ gst_audio_encoder_set_latency (GstAudioEncoder * enc,
 
   GST_LOG_OBJECT (enc, "set to %" GST_TIME_FORMAT "-%" GST_TIME_FORMAT,
       GST_TIME_ARGS (min), GST_TIME_ARGS (max));
+
+  /* post latency message on the bus */
+  gst_element_post_message (GST_ELEMENT (enc),
+      gst_message_new_latency (GST_OBJECT (enc)));
 }
 
 /**