diff --git a/gst/rtpmanager/gstrtpjitterbuffer.c b/gst/rtpmanager/gstrtpjitterbuffer.c
index fd8814f5b4..6eb0adc8e1 100644
--- a/gst/rtpmanager/gstrtpjitterbuffer.c
+++ b/gst/rtpmanager/gstrtpjitterbuffer.c
@@ -1788,6 +1788,11 @@ again:
 
     ret = gst_clock_id_wait (id, &clock_jitter);
 
+    JBUF_LOCK (priv);
+    /* and free the entry */
+    gst_clock_id_unref (id);
+    priv->clock_id = NULL;
+
     if (ret == GST_CLOCK_EARLY && gap > 0
         && clock_jitter > (priv->latency_ns + priv->peer_latency)) {
       GstClockTimeDiff total_duration;
@@ -1812,11 +1817,6 @@ again:
       lost_packets_late = TRUE;
     }
 
-    JBUF_LOCK (priv);
-    /* and free the entry */
-    gst_clock_id_unref (id);
-    priv->clock_id = NULL;
-
     /* at this point, the clock could have been unlocked by a timeout, a new
      * tail element was added to the queue or because we are shutting down. Check
      * for shutdown first. */