diff --git a/ChangeLog b/ChangeLog
index 02b5133d63..eba4c35563 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+2008-12-23  Wim Taymans  <wim.taymans@collabora.co.uk>
+
+	Patch by: Arnout Vandecappelle <arnout at mind dot be>
+
+	* gst/rtpmanager/rtpjitterbuffer.c: (rtp_jitter_buffer_reset_skew),
+	(calculate_skew):
+	* gst/rtpmanager/rtpjitterbuffer.h:
+	Keep track of the last outgoing timestamp and of the last sender-side
+	time.  Timestamps can only go forward if they do at the sender
+	side, can only go back if they do at the sender side, and remain the
+	same if they remain the same at the sender side. Fixes #565319.
+
 2008-12-22  Sebastian Dröge  <sebastian.droege@collabora.co.uk>
 
 	* gst/mxf/mxfmpeg.c: (mxf_mpeg_es_create_caps),
diff --git a/gst/rtpmanager/rtpjitterbuffer.c b/gst/rtpmanager/rtpjitterbuffer.c
index d027dc2e96..52b774b6b8 100644
--- a/gst/rtpmanager/rtpjitterbuffer.c
+++ b/gst/rtpmanager/rtpjitterbuffer.c
@@ -113,6 +113,7 @@ rtp_jitter_buffer_reset_skew (RTPJitterBuffer * jbuf)
   jbuf->window_min = 0;
   jbuf->skew = 0;
   jbuf->prev_send_diff = -1;
+  jbuf->prev_out_time = -1;
   GST_DEBUG ("reset skew correction");
 }
 
@@ -198,16 +199,20 @@ calculate_skew (RTPJitterBuffer * jbuf, guint32 rtptime, GstClockTime time,
     jbuf->base_time = -1;
     jbuf->base_rtptime = -1;
     jbuf->clock_rate = clock_rate;
+    jbuf->prev_out_time = -1;
+    jbuf->prev_send_diff = -1;
   }
 
   /* first time, lock on to time and gstrtptime */
   if (G_UNLIKELY (jbuf->base_time == -1)) {
     jbuf->base_time = time;
+    jbuf->prev_out_time = -1;
     GST_DEBUG ("Taking new base time %" GST_TIME_FORMAT, GST_TIME_ARGS (time));
   }
   if (G_UNLIKELY (jbuf->base_rtptime == -1)) {
     jbuf->base_rtptime = gstrtptime;
     jbuf->base_extrtp = ext_rtptime;
+    jbuf->prev_send_diff = -1;
     GST_DEBUG ("Taking new base rtptime %" GST_TIME_FORMAT,
         GST_TIME_ARGS (gstrtptime));
   }
@@ -221,6 +226,8 @@ calculate_skew (RTPJitterBuffer * jbuf, guint32 rtptime, GstClockTime time,
     jbuf->base_time = time;
     jbuf->base_rtptime = gstrtptime;
     jbuf->base_extrtp = ext_rtptime;
+    jbuf->prev_out_time = -1;
+    jbuf->prev_send_diff = -1;
     send_diff = 0;
   }
 
@@ -253,6 +260,8 @@ calculate_skew (RTPJitterBuffer * jbuf, guint32 rtptime, GstClockTime time,
     jbuf->base_time = time;
     jbuf->base_rtptime = gstrtptime;
     jbuf->base_extrtp = ext_rtptime;
+    jbuf->prev_out_time = -1;
+    jbuf->prev_send_diff = -1;
     send_diff = 0;
     delta = 0;
   }
@@ -333,11 +342,31 @@ calculate_skew (RTPJitterBuffer * jbuf, guint32 rtptime, GstClockTime time,
 no_skew:
   /* the output time is defined as the base timestamp plus the RTP time
    * adjusted for the clock skew .*/
-  if (jbuf->base_time != -1)
+  if (jbuf->base_time != -1) {
     out_time = jbuf->base_time + send_diff + jbuf->skew;
-  else
+    /* check if timestamps are not going backwards, we can only check this if we
+     * have a previous out time and a previous send_diff */
+    if (G_LIKELY (jbuf->prev_out_time != -1 && jbuf->prev_send_diff != -1)) {
+      /* now check for backwards timestamps */
+      if (G_UNLIKELY (
+              /* if the server timestamps went up and the out_time backwards */
+              (send_diff > jbuf->prev_send_diff
+                  && out_time < jbuf->prev_out_time) ||
+              /* if the server timestamps went backwards and the out_time forwards */
+              (send_diff < jbuf->prev_send_diff
+                  && out_time > jbuf->prev_out_time) ||
+              /* if the server timestamps did not change */
+              send_diff == jbuf->prev_send_diff)) {
+        GST_DEBUG ("backwards timestamps, using previous time");
+        out_time = jbuf->prev_out_time;
+      }
+    }
+  } else
     out_time = -1;
 
+  jbuf->prev_out_time = out_time;
+  jbuf->prev_send_diff = send_diff;
+
   GST_DEBUG ("skew %" G_GINT64_FORMAT ", out %" GST_TIME_FORMAT,
       jbuf->skew, GST_TIME_ARGS (out_time));
 
diff --git a/gst/rtpmanager/rtpjitterbuffer.h b/gst/rtpmanager/rtpjitterbuffer.h
index aa00919045..ff1a16b076 100644
--- a/gst/rtpmanager/rtpjitterbuffer.h
+++ b/gst/rtpmanager/rtpjitterbuffer.h
@@ -58,6 +58,7 @@ struct _RTPJitterBuffer {
   GstClockTime   base_rtptime;
   guint32        clock_rate;
   GstClockTime   base_extrtp;
+  GstClockTime   prev_out_time;
   guint64        ext_rtptime;
   guint64        last_rtptime;
   gint64         window[RTP_JITTER_BUFFER_MAX_WINDOW];