diff --git a/gst/rtp/gstbuffermemory.c b/gst/rtp/gstbuffermemory.c
index 3b28417ba4..d6fd6477f3 100644
--- a/gst/rtp/gstbuffermemory.c
+++ b/gst/rtp/gstbuffermemory.c
@@ -45,6 +45,8 @@ gst_buffer_memory_map (GstBuffer * buffer, GstBufferMemoryMap * map)
   map->data = map->map.data;
   map->size = map->map.size;
   map->index = 0;
+  map->total_size = gst_buffer_get_size (buffer);
+  map->offset = 0;
 
   return TRUE;
 }
@@ -90,6 +92,8 @@ gst_buffer_memory_advance_bytes (GstBufferMemoryMap * map, gsize size)
 
   g_return_val_if_fail (map != NULL, FALSE);
 
+  map->offset += size;
+
   while (offset >= map->size) {
     offset -= map->size;
     GST_DEBUG ("switching memory");
diff --git a/gst/rtp/gstbuffermemory.h b/gst/rtp/gstbuffermemory.h
index cc00d4c1b1..3b78429898 100644
--- a/gst/rtp/gstbuffermemory.h
+++ b/gst/rtp/gstbuffermemory.h
@@ -32,11 +32,13 @@ struct _GstBufferMemoryMap
   GstMemory *mem;
   GstMapInfo map;
   guint index;
+  gsize total_size;
 
   /* public datas */
 
   /* data of the currently mapped memory */
   const guint8 *data;
+  guint offset;
 
   /* size of the currently mapped memory */
   gsize size;