diff --git a/gst-libs/gst/audio/gstbaseaudioencoder.c b/gst-libs/gst/audio/gstbaseaudioencoder.c
index a907198a94..3b4ea77b8f 100644
--- a/gst-libs/gst/audio/gstbaseaudioencoder.c
+++ b/gst-libs/gst/audio/gstbaseaudioencoder.c
@@ -500,11 +500,16 @@ gst_base_audio_encoder_finish_frame (GstBaseAudioEncoder * enc, GstBuffer * buf,
           GST_DEBUG_OBJECT (enc, "marked discont");
           priv->discont = TRUE;
         }
-        GST_LOG_OBJECT (enc, "new upstream ts %" GST_TIME_FORMAT
-            " at distance %" G_GUINT64_FORMAT, GST_TIME_ARGS (ts), distance);
-        /* re-sync to upstream ts */
-        priv->base_ts = ts;
-        priv->samples = distance;
+        if (TRUE || diff > GST_SECOND / ctx->state.rate / 2 ||
+            diff < -GST_SECOND / ctx->state.rate / 2) {
+          GST_LOG_OBJECT (enc, "new upstream ts %" GST_TIME_FORMAT
+              " at distance %" G_GUINT64_FORMAT, GST_TIME_ARGS (ts), distance);
+          /* re-sync to upstream ts */
+          priv->base_ts = ts;
+          priv->samples = distance;
+        } else {
+          GST_LOG_OBJECT (enc, "new upstream ts only introduces jitter");
+        }
       }
     }
     /* advance sample view */