diff --git a/gst/rtpmanager/gstrtprtxreceive.c b/gst/rtpmanager/gstrtprtxreceive.c
index 4ae4e6508c..32933a1f75 100644
--- a/gst/rtpmanager/gstrtprtxreceive.c
+++ b/gst/rtpmanager/gstrtprtxreceive.c
@@ -427,12 +427,14 @@ _gst_rtp_buffer_new_from_rtx (GstRTPBuffer * rtp, guint32 ssrc1,
   guint payload_len = 0;
 
   /* copy fixed header */
-  mem = gst_memory_copy (rtp->map[0].memory, 0, rtp->size[0]);
+  mem = gst_memory_copy (rtp->map[0].memory,
+      (guint8 *) rtp->data[0] - rtp->map[0].data, rtp->size[0]);
   gst_buffer_append_memory (new_buffer, mem);
 
   /* copy extension if any */
   if (rtp->size[1]) {
-    mem = gst_memory_copy (rtp->map[1].memory, 0, rtp->size[1]);
+    mem = gst_memory_copy (rtp->map[1].memory,
+        (guint8 *) rtp->data[1] - rtp->map[1].data, rtp->size[1]);
     gst_buffer_append_memory (new_buffer, mem);
   }