diff --git a/libs/gst/base/gstaggregator.c b/libs/gst/base/gstaggregator.c
index 9f29bec3d8..5fe19136eb 100644
--- a/libs/gst/base/gstaggregator.c
+++ b/libs/gst/base/gstaggregator.c
@@ -1163,8 +1163,8 @@ gst_aggregator_get_latency_unlocked (GstAggregator * self, gboolean * live,
     max += self->priv->sub_latency_max;
 
   our_latency = self->priv->latency;
-  if (GST_CLOCK_TIME_IS_VALID (our_latency))
-    min += our_latency;
+  else if (GST_CLOCK_TIME_IS_VALID (self->priv->sub_latency_max))
+    max = self->priv->sub_latency_max;
 
   if (live)
     *live = self->priv->latency_live;
@@ -1221,6 +1221,8 @@ gst_aggregator_query_latency (GstAggregator * self, GstQuery * query)
   if (GST_CLOCK_TIME_IS_VALID (self->priv->sub_latency_max)
       && GST_CLOCK_TIME_IS_VALID (data.max))
     data.max += self->priv->sub_latency_max;
+  else if (GST_CLOCK_TIME_IS_VALID (self->priv->sub_latency_max))
+    data.max = self->priv->sub_latency_max;
 
   if (data.live && GST_CLOCK_TIME_IS_VALID (our_latency) && data.min > data.max) {
     GST_ELEMENT_WARNING (self, CORE, NEGOTIATION,