diff --git a/gst-libs/gst/gl/gstglmemory.c b/gst-libs/gst/gl/gstglmemory.c
index e1818fbc83..5d3f73a05c 100644
--- a/gst-libs/gst/gl/gstglmemory.c
+++ b/gst-libs/gst/gl/gstglmemory.c
@@ -1508,22 +1508,26 @@ gst_gl_memory_setup_buffer (GstGLContext * context,
  * @valign: a #GstVideoInfo
  * @data: a list of per plane data pointers
  * @textures: (transfer out): a list of #GstGLMemory
+ * @user_data: user data for the destroy function
+ * @notify: A function called each time a memory is freed
  *
  * Wraps per plane data pointer in @data into the corresponding entry in
- * @textures based on @info and padding from @valign.
+ * @textures based on @info and padding from @valign. Note that the @notify
+ * will be called as many time as there is planes.
  *
  * Returns: whether the memory's were sucessfully created.
  */
 gboolean
 gst_gl_memory_setup_wrapped (GstGLContext * context, GstVideoInfo * info,
     GstVideoAlignment * valign, gpointer data[GST_VIDEO_MAX_PLANES],
-    GstGLMemory * textures[GST_VIDEO_MAX_PLANES])
+    GstGLMemory * textures[GST_VIDEO_MAX_PLANES], gpointer user_data,
+    GDestroyNotify notify)
 {
   gint i;
 
   for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) {
     textures[i] = (GstGLMemory *) gst_gl_memory_wrapped (context, info, i,
-        valign, data[i], NULL, NULL);
+        valign, data[i], user_data, notify);
   }
 
   return TRUE;
diff --git a/gst-libs/gst/gl/gstglmemory.h b/gst-libs/gst/gl/gstglmemory.h
index d124c7cbdf..7bc46070bd 100644
--- a/gst-libs/gst/gl/gstglmemory.h
+++ b/gst-libs/gst/gl/gstglmemory.h
@@ -142,7 +142,9 @@ gboolean      gst_gl_memory_setup_wrapped (GstGLContext * context,
                                            GstVideoInfo * info,
                                            GstVideoAlignment *valign,
                                            gpointer data[GST_VIDEO_MAX_PLANES],
-                                           GstGLMemory *textures[GST_VIDEO_MAX_PLANES]);
+                                           GstGLMemory *textures[GST_VIDEO_MAX_PLANES],
+                                           gpointer user_data,
+                                           GDestroyNotify notify);
 
 gint          gst_gl_memory_get_texture_width  (GstGLMemory * gl_mem);
 gint          gst_gl_memory_get_texture_height (GstGLMemory * gl_mem);
diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 2055c9a26a..3e2d28edee 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -696,12 +696,55 @@ static const UploadMethod _upload_meta_upload = {
   &_upload_meta_upload_free
 };
 
+struct RawUploadFrame
+{
+  gint ref_count;
+  GstVideoFrame frame;
+};
+
 struct RawUpload
 {
   GstGLUpload *upload;
-  GstVideoFrame in_frame;
+  struct RawUploadFrame *in_frame;
 };
 
+static struct RawUploadFrame *
+_raw_upload_frame_new (struct RawUpload *raw, GstBuffer * buffer)
+{
+  struct RawUploadFrame *frame;
+
+  if (!buffer)
+    return NULL;
+
+  frame = g_slice_new (struct RawUploadFrame);
+  frame->ref_count = 1;
+
+  if (!gst_video_frame_map (&frame->frame, &raw->upload->priv->in_info,
+          buffer, GST_MAP_READ)) {
+    g_slice_free (struct RawUploadFrame, frame);
+    return NULL;
+  }
+
+  raw->upload->priv->in_info = frame->frame.info;
+
+  return frame;
+}
+
+static void
+_raw_upload_frame_ref (struct RawUploadFrame *frame)
+{
+  g_atomic_int_inc (&frame->ref_count);
+}
+
+static void
+_raw_upload_frame_unref (struct RawUploadFrame *frame)
+{
+  if (g_atomic_int_dec_and_test (&frame->ref_count)) {
+    gst_video_frame_unmap (&frame->frame);
+    g_slice_free (struct RawUploadFrame, frame);
+  }
+}
+
 static gpointer
 _raw_data_upload_new (GstGLUpload * upload)
 {
@@ -738,15 +781,9 @@ _raw_data_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
   if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
     return FALSE;
 
-  if (buffer) {
-    if (!gst_video_frame_map (&raw->in_frame, &raw->upload->priv->in_info,
-            buffer, GST_MAP_READ))
-      return FALSE;
+  raw->in_frame = _raw_upload_frame_new (raw, buffer);
 
-    raw->upload->priv->in_info = raw->in_frame.info;
-  }
-
-  return TRUE;
+  return (raw->in_frame != NULL);
 }
 
 static void
@@ -772,10 +809,12 @@ _raw_data_upload_perform (gpointer impl, GstBuffer * buffer,
     max_planes *= GST_VIDEO_INFO_VIEWS (in_info);
 
   gst_gl_memory_setup_wrapped (raw->upload->context,
-      &raw->upload->priv->in_info, NULL, raw->in_frame.data, in_tex);
+      &raw->upload->priv->in_info, NULL, raw->in_frame->frame.data, in_tex,
+      raw->in_frame, (GDestroyNotify) _raw_upload_frame_unref);
 
   *outbuf = gst_buffer_new ();
   for (i = 0; i < max_planes; i++) {
+    _raw_upload_frame_ref (raw->in_frame);
     gst_buffer_append_memory (*outbuf, (GstMemory *) in_tex[i]);
   }
 
@@ -786,8 +825,8 @@ static void
 _raw_data_upload_release (gpointer impl, GstBuffer * buffer)
 {
   struct RawUpload *raw = impl;
-
-  gst_video_frame_unmap (&raw->in_frame);
+  _raw_upload_frame_unref (raw->in_frame);
+  raw->in_frame = NULL;
 }
 
 static void