diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c
index 246bb6e39b..c242940a92 100644
--- a/gst-libs/gst/gl/gstglcolorconvert.c
+++ b/gst-libs/gst/gl/gstglcolorconvert.c
@@ -50,8 +50,8 @@
 static void _do_convert (GstGLContext * context, GstGLColorConvert * convert);
 static gboolean _init_convert (GstGLColorConvert * convert);
 static gboolean _init_convert_fbo (GstGLColorConvert * convert);
-static gboolean _gst_gl_color_convert_perform_unlocked (GstGLColorConvert *
-    convert, GstBuffer * inbuf, GstBuffer * outbuf);
+static GstBuffer *_gst_gl_color_convert_perform_unlocked (GstGLColorConvert *
+    convert, GstBuffer * inbuf);
 
 static gboolean _do_convert_draw (GstGLContext * context,
     GstGLColorConvert * convert);
@@ -523,44 +523,46 @@ gst_gl_color_convert_set_format (GstGLColorConvert * convert,
  * gst_gl_color_convert_perform:
  * @convert: a #GstGLColorConvert
  * @inbuf: the texture ids for input formatted according to in_info
- * @outbuf: the texture ids for output formatted according to out_info
  *
- * Converts the data contained in in_tex into out_tex using the formats
- * specified by the #GstVideoInfo<!--  -->s passed to
- * gst_gl_color_convert_set_format() 
+ * Converts the data contained by @inbuf using the formats specified by the
+ * #GstVideoInfo<!--  -->s passed to gst_gl_color_convert_set_format() 
  *
- * Returns: whether the conversion was successful
+ * Returns: a converted #GstBuffer or %NULL%
  */
-gboolean
-gst_gl_color_convert_perform (GstGLColorConvert * convert, GstBuffer * inbuf,
-    GstBuffer * outbuf)
+GstBuffer *
+gst_gl_color_convert_perform (GstGLColorConvert * convert, GstBuffer * inbuf)
 {
-  gboolean ret;
+  GstBuffer *ret;
 
   g_return_val_if_fail (convert != NULL, FALSE);
 
   g_mutex_lock (&convert->lock);
-  ret = _gst_gl_color_convert_perform_unlocked (convert, inbuf, outbuf);
+  ret = _gst_gl_color_convert_perform_unlocked (convert, inbuf);
   g_mutex_unlock (&convert->lock);
 
   return ret;
 }
 
-static gboolean
+static GstBuffer *
 _gst_gl_color_convert_perform_unlocked (GstGLColorConvert * convert,
-    GstBuffer * inbuf, GstBuffer * outbuf)
+    GstBuffer * inbuf)
 {
   g_return_val_if_fail (convert != NULL, FALSE);
   g_return_val_if_fail (inbuf, FALSE);
-  g_return_val_if_fail (outbuf, FALSE);
 
   convert->inbuf = inbuf;
-  convert->outbuf = outbuf;
 
   gst_gl_context_thread_add (convert->context,
       (GstGLContextThreadFunc) _do_convert, convert);
 
-  return convert->priv->result;
+  if (!convert->priv->result) {
+    if (convert->outbuf)
+      gst_object_unref (convert->outbuf);
+    convert->outbuf = NULL;
+    return NULL;
+  }
+
+  return convert->outbuf;
 }
 
 static inline gboolean
@@ -1137,11 +1139,27 @@ _do_convert (GstGLContext * context, GstGLColorConvert * convert)
   in_width = GST_VIDEO_INFO_WIDTH (&convert->in_info);
   in_height = GST_VIDEO_INFO_HEIGHT (&convert->in_info);
 
+  convert->outbuf = NULL;
+
   if (!_init_convert (convert)) {
     convert->priv->result = FALSE;
     return;
   }
 
+  convert->outbuf = gst_buffer_new ();
+  if (!gst_gl_memory_setup_buffer (convert->context, &convert->out_info,
+          convert->outbuf)) {
+    convert->priv->result = FALSE;
+    return;
+  }
+
+  gst_buffer_add_video_meta_full (convert->outbuf, 0,
+      GST_VIDEO_INFO_FORMAT (&convert->out_info),
+      GST_VIDEO_INFO_WIDTH (&convert->out_info),
+      GST_VIDEO_INFO_HEIGHT (&convert->out_info),
+      GST_VIDEO_INFO_N_PLANES (&convert->out_info),
+      convert->out_info.offset, convert->out_info.stride);
+
   for (i = 0; i < c_info->in_n_textures; i++) {
     convert->priv->in_tex[i] =
         (GstGLMemory *) gst_buffer_peek_memory (convert->inbuf, i);
@@ -1206,8 +1224,6 @@ out:
         || out_width != out_tex->width || out_height != out_tex->height) {
       GstMapInfo to_info, from_info;
 
-      gst_memory_unmap ((GstMemory *) convert->priv->out_tex[j], &out_info[j]);
-
       if (!gst_memory_map ((GstMemory *) convert->priv->out_tex[j], &from_info,
               GST_MAP_READ | GST_MAP_GL)) {
         gst_gl_context_set_error (convert->context, "Failed to map "
@@ -1232,10 +1248,24 @@ out:
     }
   }
 
+  /* YV12 the same as I420 except planes 1+2 swapped */
+  if (GST_VIDEO_INFO_FORMAT (&convert->out_info) == GST_VIDEO_FORMAT_YV12) {
+    GstMemory *mem1 = gst_buffer_get_memory (convert->outbuf, 1);
+    GstMemory *mem2 = gst_buffer_get_memory (convert->outbuf, 2);
+
+    gst_buffer_replace_memory (convert->outbuf, 1, mem2);
+    gst_buffer_replace_memory (convert->outbuf, 2, mem1);
+  }
+
   for (i--; i >= 0; i--) {
     gst_memory_unmap ((GstMemory *) convert->priv->in_tex[i], &in_info[i]);
   }
 
+  if (!res) {
+    gst_buffer_unref (convert->outbuf);
+    convert->outbuf = NULL;
+  }
+
   convert->priv->result = res;
   return;
 }
diff --git a/gst-libs/gst/gl/gstglcolorconvert.h b/gst-libs/gst/gl/gstglcolorconvert.h
index b06f9e0a05..4b843fcbf5 100644
--- a/gst-libs/gst/gl/gstglcolorconvert.h
+++ b/gst-libs/gst/gl/gstglcolorconvert.h
@@ -105,8 +105,7 @@ void     gst_gl_color_convert_set_format    (GstGLColorConvert * convert,
                                              GstVideoInfo * in_info,
                                              GstVideoInfo * out_info);
 
-gboolean gst_gl_color_convert_perform        (GstGLColorConvert * convert,
-                                              GstBuffer * inbuf, GstBuffer * outbuf);
+GstBuffer * gst_gl_color_convert_perform    (GstGLColorConvert * convert, GstBuffer * inbuf);
 
 G_END_DECLS
 
diff --git a/gst-libs/gst/gl/gstgldownload.c b/gst-libs/gst/gl/gstgldownload.c
index d845bb7608..b785c64455 100644
--- a/gst-libs/gst/gl/gstgldownload.c
+++ b/gst-libs/gst/gl/gstgldownload.c
@@ -43,7 +43,8 @@
 #define USING_GLES2(context) (gst_gl_context_get_gl_api (context) & GST_GL_API_GLES2)
 #define USING_GLES3(context) (gst_gl_context_get_gl_api (context) & GST_GL_API_GLES3)
 
-static void _do_download (GstGLContext * context, GstGLDownload * download);
+static gboolean _do_download (GstGLDownload * download, guint texture_id,
+    gpointer data[GST_VIDEO_MAX_PLANES]);
 static gboolean _init_download (GstGLDownload * download);
 static gboolean _gst_gl_download_perform_with_data_unlocked (GstGLDownload *
     download, GLuint texture_id, gpointer data[GST_VIDEO_MAX_PLANES]);
@@ -59,10 +60,7 @@ struct _GstGLDownloadPrivate
   const gchar *ARGB;
   const gchar *vert_shader;
 
-  gboolean result;
-
   GstGLMemory *in_tex[GST_VIDEO_MAX_PLANES];
-  GstGLMemory *out_tex[GST_VIDEO_MAX_PLANES];
 };
 
 GST_DEBUG_CATEGORY_STATIC (gst_gl_download_debug);
@@ -145,13 +143,6 @@ gst_gl_download_reset (GstGLDownload * download)
 {
   guint i;
 
-  for (i = 0; i < GST_VIDEO_MAX_PLANES; i++) {
-    if (download->priv->out_tex[i]) {
-      gst_memory_unref ((GstMemory *) download->priv->out_tex[i]);
-      download->priv->out_tex[i] = NULL;
-    }
-  }
-
   for (i = 0; i < GST_VIDEO_MAX_PLANES; i++) {
     if (download->priv->in_tex[i]) {
       gst_memory_unref ((GstMemory *) download->priv->in_tex[i]);
@@ -225,7 +216,6 @@ static gboolean
 _gst_gl_download_perform_with_data_unlocked (GstGLDownload * download,
     GLuint texture_id, gpointer data[GST_VIDEO_MAX_PLANES])
 {
-  gpointer temp_data;
   guint i;
 
   g_return_val_if_fail (download != NULL, FALSE);
@@ -247,36 +237,7 @@ _gst_gl_download_perform_with_data_unlocked (GstGLDownload * download,
 
   download->priv->in_tex[0]->tex_id = texture_id;
 
-  if (!download->priv->out_tex[0]) {
-    if (GST_VIDEO_INFO_FORMAT (&download->info) == GST_VIDEO_FORMAT_YUY2
-        || GST_VIDEO_INFO_FORMAT (&download->info) == GST_VIDEO_FORMAT_UYVY) {
-      download->priv->out_tex[0] = gst_gl_memory_wrapped (download->context,
-          GST_VIDEO_GL_TEXTURE_TYPE_RGBA,
-          GST_VIDEO_INFO_COMP_WIDTH (&download->info, 1),
-          GST_VIDEO_INFO_HEIGHT (&download->info),
-          GST_VIDEO_INFO_PLANE_STRIDE (&download->info, 0), data[0], NULL,
-          NULL);
-    } else {
-      gst_gl_memory_setup_wrapped (download->context, &download->info,
-          data, download->priv->out_tex);
-    }
-  }
-
-  for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&download->info); i++) {
-    download->priv->out_tex[i]->data = data[i];
-  }
-
-  if (GST_VIDEO_INFO_FORMAT (&download->info) == GST_VIDEO_FORMAT_YV12) {
-    /* YV12 same as I420 except planes 1+2 swapped */
-    temp_data = download->priv->out_tex[1]->data;
-    download->priv->out_tex[1]->data = download->priv->out_tex[2]->data;
-    download->priv->out_tex[2]->data = temp_data;
-  }
-
-  gst_gl_context_thread_add (download->context,
-      (GstGLContextThreadFunc) _do_download, download);
-
-  return download->priv->result;
+  return _do_download (download, texture_id, data);
 }
 
 static gboolean
@@ -314,20 +275,22 @@ _init_download (GstGLDownload * download)
   return TRUE;
 }
 
-static void
-_do_download (GstGLContext * context, GstGLDownload * download)
+static gboolean
+_do_download (GstGLDownload * download, guint texture_id,
+    gpointer data[GST_VIDEO_MAX_PLANES])
 {
   guint out_width, out_height;
   GstBuffer *inbuf, *outbuf;
   GstMapInfo map_info;
+  gboolean ret = TRUE;
   gint i;
 
   out_width = GST_VIDEO_INFO_WIDTH (&download->info);
   out_height = GST_VIDEO_INFO_HEIGHT (&download->info);
 
   if (!download->initted) {
-    if (!(download->priv->result = _init_download (download)))
-      return;
+    if (!_init_download (download))
+      return FALSE;
   }
 
   GST_TRACE ("doing download of texture:%u (%ux%u)",
@@ -337,23 +300,24 @@ _do_download (GstGLContext * context, GstGLDownload * download)
   gst_buffer_append_memory (inbuf,
       gst_memory_ref ((GstMemory *) download->priv->in_tex[0]));
 
-  outbuf = gst_buffer_new ();
-  for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&download->info); i++) {
-    gst_buffer_append_memory (outbuf,
-        gst_memory_ref ((GstMemory *) download->priv->out_tex[0]));
-  }
-
-  download->priv->result =
-      gst_gl_color_convert_perform (download->convert, inbuf, outbuf);
-  if (!download->priv->result)
-    return;
+  outbuf = gst_gl_color_convert_perform (download->convert, inbuf);
+  if (!outbuf)
+    return FALSE;
 
   for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&download->info); i++) {
-    gst_memory_map ((GstMemory *) download->priv->out_tex[i], &map_info,
-        GST_MAP_READ);
-    gst_memory_unmap ((GstMemory *) download->priv->out_tex[i], &map_info);
+    GstMemory *out_mem = gst_buffer_peek_memory (outbuf, i);
+    gpointer temp_data = ((GstGLMemory *) out_mem)->data;
+    ((GstGLMemory *) out_mem)->data = data[i];
+
+    if (!gst_memory_map (out_mem, &map_info, GST_MAP_READ)) {
+      GST_ERROR_OBJECT (download, "Failed to map memory");
+      ret = FALSE;
+    }
+    gst_memory_unmap (out_mem, &map_info);
+    ((GstGLMemory *) out_mem)->data = temp_data;
   }
 
   gst_buffer_unref (inbuf);
-  gst_buffer_unref (outbuf);
+
+  return ret;
 }
diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 53959348f2..443e54689a 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -53,7 +53,7 @@ static gboolean _upload_memory (GstGLUpload * upload);
 static gboolean _init_upload (GstGLUpload * upload);
 //static gboolean _init_upload_fbo (GstGLContext * context, GstGLUpload * upload);
 static gboolean _gst_gl_upload_perform_with_data_unlocked (GstGLUpload * upload,
-    GLuint texture_id, gpointer data[GST_VIDEO_MAX_PLANES]);
+    GLuint * texture_id, gpointer data[GST_VIDEO_MAX_PLANES]);
 static void _do_upload_with_meta (GstGLContext * context, GstGLUpload * upload);
 static void gst_gl_upload_reset (GstGLUpload * upload);
 
@@ -84,6 +84,8 @@ struct _GstGLUploadPrivate
   GstVideoGLTextureUploadMeta *meta;
   guint tex_id;
   gboolean mapped;
+
+  GstBuffer *outbuf;
 };
 
 GST_DEBUG_CATEGORY_STATIC (gst_gl_upload_debug);
@@ -283,12 +285,6 @@ gst_gl_upload_perform_with_buffer (GstGLUpload * upload, GstBuffer * buffer,
       return TRUE;
     }
 
-    if (!upload->out_tex)
-      upload->out_tex = (GstGLMemory *) gst_gl_memory_alloc (upload->context,
-          GST_VIDEO_GL_TEXTURE_TYPE_RGBA, GST_VIDEO_INFO_WIDTH (&upload->in_info),
-          GST_VIDEO_INFO_HEIGHT (&upload->in_info),
-          4 * GST_VIDEO_INFO_WIDTH (&upload->in_info));
-
     GST_LOG_OBJECT (upload, "Attempting upload with GstGLMemory");
     for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&upload->in_info); i++) {
       upload->in_tex[i] = (GstGLMemory *) gst_buffer_peek_memory (buffer, i);
@@ -339,22 +335,16 @@ gst_gl_upload_perform_with_buffer (GstGLUpload * upload, GstBuffer * buffer,
     GST_ERROR_OBJECT (upload, "Failed to map memory");
     return FALSE;
   }
+  upload->priv->mapped = TRUE;
 
   /* update the video info from the one updated by frame_map using video meta */
   gst_gl_upload_set_format (upload, &upload->priv->frame.info);
 
-  if (!upload->priv->tex_id)
-    gst_gl_context_gen_texture (upload->context, &upload->priv->tex_id,
-        GST_VIDEO_FORMAT_RGBA, GST_VIDEO_INFO_WIDTH (&upload->in_info),
-        GST_VIDEO_INFO_HEIGHT (&upload->in_info));
-
-  if (!gst_gl_upload_perform_with_data (upload, upload->priv->tex_id,
+  if (!gst_gl_upload_perform_with_data (upload, tex_id,
           upload->priv->frame.data)) {
     return FALSE;
   }
 
-  upload->priv->mapped = TRUE;
-  *tex_id = upload->priv->tex_id;
   return TRUE;
 }
 
@@ -365,6 +355,12 @@ gst_gl_upload_release_buffer (GstGLUpload * upload)
 
   if (upload->priv->mapped)
     gst_video_frame_unmap (&upload->priv->frame);
+  upload->priv->mapped = FALSE;
+
+  if (upload->priv->outbuf) {
+    gst_buffer_unref (upload->priv->outbuf);
+    upload->priv->outbuf = NULL;
+  }
 }
 
 /*
@@ -439,7 +435,7 @@ gst_gl_upload_perform_with_gl_texture_upload_meta (GstGLUpload * upload,
 /**
  * gst_gl_upload_perform_with_data:
  * @upload: a #GstGLUpload
- * @texture_id: the texture id to download
+ * @texture_id: (out): the texture id to upload into
  * @data: where the downloaded data should go
  *
  * Uploads @data into @texture_id. data size and format is specified by
@@ -448,7 +444,7 @@ gst_gl_upload_perform_with_gl_texture_upload_meta (GstGLUpload * upload,
  * Returns: whether the upload was successful
  */
 gboolean
-gst_gl_upload_perform_with_data (GstGLUpload * upload, GLuint texture_id,
+gst_gl_upload_perform_with_data (GstGLUpload * upload, GLuint * texture_id,
     gpointer data[GST_VIDEO_MAX_PLANES])
 {
   gboolean ret;
@@ -457,17 +453,6 @@ gst_gl_upload_perform_with_data (GstGLUpload * upload, GLuint texture_id,
 
   g_mutex_lock (&upload->lock);
 
-  if (!upload->out_tex)
-    upload->out_tex = gst_gl_memory_wrapped_texture (upload->context, texture_id,
-        GST_VIDEO_GL_TEXTURE_TYPE_RGBA, GST_VIDEO_INFO_WIDTH (&upload->in_info),
-        GST_VIDEO_INFO_HEIGHT (&upload->in_info), NULL, NULL);
-
-  /* FIXME: kinda breaks the abstraction */
-  if (upload->out_tex->tex_id != texture_id) {
-    upload->out_tex->tex_id = texture_id;
-    GST_GL_MEMORY_FLAG_SET (upload->out_tex, GST_GL_MEMORY_FLAG_NEED_DOWNLOAD);
-  }
-
   ret = _gst_gl_upload_perform_with_data_unlocked (upload, texture_id, data);
 
   g_mutex_unlock (&upload->lock);
@@ -477,8 +462,9 @@ gst_gl_upload_perform_with_data (GstGLUpload * upload, GLuint texture_id,
 
 static gboolean
 _gst_gl_upload_perform_with_data_unlocked (GstGLUpload * upload,
-    GLuint texture_id, gpointer data[GST_VIDEO_MAX_PLANES])
+    GLuint * texture_id, gpointer data[GST_VIDEO_MAX_PLANES])
 {
+  gboolean ret;
   guint i;
 
   g_return_val_if_fail (upload != NULL, FALSE);
@@ -495,9 +481,10 @@ _gst_gl_upload_perform_with_data_unlocked (GstGLUpload * upload,
     }
   }
 
-  GST_LOG ("Uploading data into texture %u", texture_id);
+  ret = _upload_memory (upload);
+  *texture_id = upload->out_tex->tex_id;
 
-  return _upload_memory (upload);
+  return ret;
 }
 
 /* Called in the gl thread */
@@ -527,6 +514,10 @@ _init_upload (GstGLUpload * upload)
 
   gst_gl_color_convert_set_format (upload->convert, &upload->in_info, &out_info);
 
+  upload->out_tex = gst_gl_memory_wrapped_texture (upload->context, 0,
+      GST_VIDEO_GL_TEXTURE_TYPE_RGBA, GST_VIDEO_INFO_WIDTH (&upload->in_info),
+      GST_VIDEO_INFO_HEIGHT (&upload->in_info), NULL, NULL);
+
   upload->initted = TRUE;
 
   return TRUE;
@@ -540,9 +531,9 @@ _upload_memory (GstGLUpload * upload)
 {
   guint in_width, in_height;
   guint in_texture[GST_VIDEO_MAX_PLANES];
-  GstGLMemory *out_texture[GST_VIDEO_MAX_PLANES] = {upload->out_tex, 0, 0, 0};
-  GstBuffer *inbuf, *outbuf;
-  gboolean ret;
+  GstBuffer *inbuf;
+  GstVideoFrame out_frame;
+  GstVideoInfo out_info;
   gint i;
 
   in_width = GST_VIDEO_INFO_WIDTH (&upload->in_info);
@@ -559,17 +550,24 @@ _upload_memory (GstGLUpload * upload)
     in_texture[i] = upload->in_tex[i]->tex_id;
     gst_buffer_append_memory (inbuf, gst_memory_ref ((GstMemory *) upload->in_tex[i]));
   }
-  outbuf = gst_buffer_new ();
-  gst_buffer_append_memory (outbuf, gst_memory_ref ((GstMemory *) upload->out_tex));
 
-  GST_TRACE ("uploading to texture:%u with textures:%u,%u,%u dimensions:%ux%u", 
-      out_texture[0]->tex_id, in_texture[0], in_texture[1], in_texture[2],
-      in_width, in_height);
-
-  ret = gst_gl_color_convert_perform (upload->convert, inbuf, outbuf);
+  GST_TRACE ("uploading with textures:%u,%u,%u dimensions:%ux%u", 
+      in_texture[0], in_texture[1], in_texture[2], in_width, in_height);
 
+  upload->priv->outbuf = gst_gl_color_convert_perform (upload->convert, inbuf);
   gst_buffer_unref (inbuf);
-  gst_buffer_unref (outbuf);
 
-  return ret;
+  gst_video_info_set_format (&out_info, GST_VIDEO_FORMAT_RGBA, in_width, in_height);
+  if (!gst_video_frame_map (&out_frame, &out_info, upload->priv->outbuf,
+        GST_MAP_READ | GST_MAP_GL)) {
+    gst_buffer_unref (upload->priv->outbuf);
+    upload->priv->outbuf = NULL;
+    return FALSE;
+  }
+
+  upload->out_tex->tex_id = *(guint *) out_frame.data[0];
+
+  gst_video_frame_unmap (&out_frame);
+
+  return TRUE;
 }
diff --git a/gst-libs/gst/gl/gstglupload.h b/gst-libs/gst/gl/gstglupload.h
index 133e258f28..b4b5808662 100644
--- a/gst-libs/gst/gl/gstglupload.h
+++ b/gst-libs/gst/gl/gstglupload.h
@@ -82,7 +82,7 @@ GstVideoInfo * gst_gl_upload_get_format    (GstGLUpload * upload);
 
 gboolean gst_gl_upload_perform_with_buffer (GstGLUpload * upload, GstBuffer * buffer, guint * tex_id);
 void gst_gl_upload_release_buffer (GstGLUpload * upload);
-gboolean gst_gl_upload_perform_with_data          (GstGLUpload * upload, GLuint texture_id,
+gboolean gst_gl_upload_perform_with_data          (GstGLUpload * upload, GLuint * texture_id,
                                                    gpointer data[GST_VIDEO_MAX_PLANES]);
 
 gboolean gst_gl_upload_perform_with_gl_texture_upload_meta (GstGLUpload *upload, GstVideoGLTextureUploadMeta *meta, guint texture_id[4]);