diff --git a/gst/vaapi/gstvaapivideomemory.c b/gst/vaapi/gstvaapivideomemory.c
index 73cd0003a0..d287a75483 100644
--- a/gst/vaapi/gstvaapivideomemory.c
+++ b/gst/vaapi/gstvaapivideomemory.c
@@ -91,6 +91,23 @@ ensure_image (GstVaapiVideoMemory * mem)
   return TRUE;
 }
 
+static gboolean
+ensure_image_is_current (GstVaapiVideoMemory * mem)
+{
+  if (mem->use_direct_rendering)
+    return TRUE;
+
+  if (!GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET (mem,
+          GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT)) {
+    if (!gst_vaapi_surface_get_image (mem->surface, mem->image))
+      return FALSE;
+
+    GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem,
+        GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT);
+  }
+  return TRUE;
+}
+
 static GstVaapiSurface *
 new_surface (GstVaapiDisplay * display, const GstVideoInfo * vip)
 {
@@ -145,6 +162,23 @@ ensure_surface (GstVaapiVideoMemory * mem)
   return mem->surface != NULL;
 }
 
+static gboolean
+ensure_surface_is_current (GstVaapiVideoMemory * mem)
+{
+  if (mem->use_direct_rendering)
+    return TRUE;
+
+  if (!GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET (mem,
+          GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT)) {
+    if (!gst_vaapi_surface_put_image (mem->surface, mem->image))
+      return FALSE;
+
+    GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem,
+        GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT);
+  }
+  return TRUE;
+}
+
 gboolean
 gst_video_meta_map_vaapi_memory (GstVideoMeta * meta, guint plane,
     GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags)
@@ -168,12 +202,17 @@ gst_video_meta_map_vaapi_memory (GstVideoMeta * meta, guint plane,
       goto error_ensure_image;
 
     // Load VA image from surface
-    if ((flags & GST_MAP_READ) && !mem->use_direct_rendering)
-      gst_vaapi_surface_get_image (mem->surface, mem->image);
+    if ((flags & GST_MAP_READ) && !ensure_image_is_current (mem))
+      goto error_no_current_image;
 
     if (!gst_vaapi_image_map (mem->image))
       goto error_map_image;
     mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR;
+
+    // Mark surface as dirty and expect updates from image
+    if (flags & GST_MAP_WRITE)
+      GST_VAAPI_VIDEO_MEMORY_FLAG_UNSET (mem,
+          GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT);
   }
 
   *data = gst_vaapi_image_get_plane (mem->image, plane);
@@ -209,6 +248,11 @@ error_map_image:
         GST_VAAPI_ID_ARGS (gst_vaapi_image_get_id (mem->image)));
     return FALSE;
   }
+error_no_current_image:
+  {
+    GST_ERROR ("failed to make image current");
+    return FALSE;
+  }
 }
 
 gboolean
@@ -229,23 +273,16 @@ gst_video_meta_unmap_vaapi_memory (GstVideoMeta * meta, guint plane,
     mem->map_type = 0;
 
     /* Unmap VA image used for read/writes */
-    if (info->flags & GST_MAP_READWRITE)
+    if (info->flags & GST_MAP_READWRITE) {
       gst_vaapi_image_unmap (mem->image);
 
-    /* Commit VA image to surface */
-    if ((info->flags & GST_MAP_WRITE) && !mem->use_direct_rendering) {
-      if (!gst_vaapi_surface_put_image (mem->surface, mem->image))
-        goto error_upload_image;
+      if (info->flags & GST_MAP_WRITE) {
+        GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem,
+            GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT);
+      }
     }
   }
   return TRUE;
-
-  /* ERRORS */
-error_upload_image:
-  {
-    GST_ERROR ("failed to upload image");
-    return FALSE;
-  }
 }
 
 GstMemory *
@@ -275,6 +312,9 @@ gst_vaapi_video_memory_new (GstAllocator * base_allocator,
   mem->map_type = 0;
   mem->map_count = 0;
   mem->use_direct_rendering = allocator->has_direct_rendering;
+
+  GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem,
+      GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT);
   return GST_MEMORY_CAST (mem);
 }
 
@@ -313,6 +353,14 @@ gst_vaapi_video_memory_reset_surface (GstVaapiVideoMemory * mem)
     gst_vaapi_video_meta_set_surface_proxy (mem->meta, NULL);
 }
 
+gboolean
+gst_vaapi_video_memory_sync (GstVaapiVideoMemory * mem)
+{
+  g_return_val_if_fail (mem, NULL);
+
+  return ensure_surface_is_current (mem);
+}
+
 static gpointer
 gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize,
     guint flags)
@@ -330,6 +378,8 @@ gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize,
             gst_vaapi_video_meta_get_surface_proxy (mem->meta));
         if (!mem->proxy)
           goto error_no_surface_proxy;
+        if (!ensure_surface_is_current (mem))
+          goto error_no_current_surface;
         mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE;
         break;
       case GST_MAP_READ:
@@ -338,8 +388,8 @@ gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize,
           goto error_no_surface;
         if (!ensure_image (mem))
           goto error_no_image;
-        if (!mem->use_direct_rendering)
-          gst_vaapi_surface_get_image (mem->surface, mem->image);
+        if (!ensure_image_is_current (mem))
+          goto error_no_current_image;
         if (!gst_vaapi_image_map (mem->image))
           goto error_map_image;
         mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR;
@@ -379,9 +429,15 @@ error_no_surface_proxy:
 error_no_surface:
   GST_ERROR ("failed to extract VA surface from video buffer");
   return NULL;
+error_no_current_surface:
+  GST_ERROR ("failed to make surface current");
+  return NULL;
 error_no_image:
   GST_ERROR ("failed to extract VA image from video buffer");
   return NULL;
+error_no_current_image:
+  GST_ERROR ("failed to make image current");
+  return NULL;
 error_map_image:
   GST_ERROR ("failed to map VA image");
   return NULL;
@@ -429,6 +485,9 @@ gst_vaapi_video_memory_copy (GstVaapiVideoMemory * mem,
   if (offset != 0 || (size != -1 && (gsize) size != maxsize))
     goto error_unsupported;
 
+  if (!ensure_surface_is_current (mem))
+    goto error_no_current_surface;
+
   meta = gst_vaapi_video_meta_copy (mem->meta);
   if (!meta)
     goto error_allocate_memory;
@@ -440,6 +499,9 @@ gst_vaapi_video_memory_copy (GstVaapiVideoMemory * mem,
   return GST_VAAPI_VIDEO_MEMORY_CAST (out_mem);
 
   /* ERRORS */
+error_no_current_surface:
+  GST_ERROR ("failed to make surface current");
+  return NULL;
 error_unsupported:
   GST_ERROR ("failed to copy partial memory (unsupported operation)");
   return NULL;
diff --git a/gst/vaapi/gstvaapivideomemory.h b/gst/vaapi/gstvaapivideomemory.h
index 74af3681e4..5681a1c065 100644
--- a/gst/vaapi/gstvaapivideomemory.h
+++ b/gst/vaapi/gstvaapivideomemory.h
@@ -42,12 +42,22 @@ typedef struct _GstVaapiVideoAllocatorClass GstVaapiVideoAllocatorClass;
 #define GST_VAAPI_VIDEO_MEMORY_CAST(mem) \
   ((GstVaapiVideoMemory *) (mem))
 
+#define GST_VAAPI_IS_VIDEO_MEMORY(mem) \
+  ((mem) && (mem)->allocator && GST_VAAPI_IS_VIDEO_ALLOCATOR((mem)->allocator))
+
 #define GST_VAAPI_VIDEO_MEMORY_NAME             "GstVaapiVideoMemory"
 
 #if GST_CHECK_VERSION(1,1,0)
 #define GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE   "memory:VASurface"
 #endif
 
+#define GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET(mem, flag) \
+  GST_MEMORY_FLAG_IS_SET (mem, flag)
+#define GST_VAAPI_VIDEO_MEMORY_FLAG_SET(mem, flag) \
+  GST_MINI_OBJECT_FLAG_SET (mem, flag)
+#define GST_VAAPI_VIDEO_MEMORY_FLAG_UNSET(mem, flag) \
+  GST_MEMORY_FLAG_UNSET (mem, flag)
+
 /**
  * GstVaapiVideoMemoryMapType:
  * @GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE: map with gst_buffer_map()
@@ -66,6 +76,21 @@ typedef enum
   GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR
 } GstVaapiVideoMemoryMapType;
 
+/**
+ * GstVaapiVideoMemoryFlags:
+ * @GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT: The embedded
+ *   #GstVaapiSurface has the up-to-date video frame contents.
+ * @GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT: The embedded
+ *   #GstVaapiImage has the up-to-date video frame contents.
+ *
+ * The set of extended #GstMemory flags.
+ */
+typedef enum
+{
+  GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT = GST_MEMORY_FLAG_LAST << 0,
+  GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT = GST_MEMORY_FLAG_LAST << 1,
+} GstVaapiVideoMemoryFlags;
+
 /**
  * GstVaapiVideoMemory:
  *
@@ -106,6 +131,10 @@ G_GNUC_INTERNAL
 void
 gst_vaapi_video_memory_reset_surface (GstVaapiVideoMemory * mem);
 
+G_GNUC_INTERNAL
+gboolean
+gst_vaapi_video_memory_sync (GstVaapiVideoMemory * mem);
+
 /* ------------------------------------------------------------------------ */
 /* --- GstVaapiVideoAllocator                                           --- */
 /* ------------------------------------------------------------------------ */
diff --git a/gst/vaapi/gstvaapivideometa.c b/gst/vaapi/gstvaapivideometa.c
index d67f458c26..ab97be5f65 100644
--- a/gst/vaapi/gstvaapivideometa.c
+++ b/gst/vaapi/gstvaapivideometa.c
@@ -32,6 +32,10 @@
 #include <gst/vaapi/gstvaapisurfacepool.h>
 #include "gstvaapivideometa.h"
 
+#if GST_CHECK_VERSION(1,0,0)
+# include "gstvaapivideomemory.h"
+#endif
+
 #define GST_VAAPI_VIDEO_META(obj) \
   ((GstVaapiVideoMeta *) (obj))
 #define GST_VAAPI_IS_VIDEO_META(obj) \
@@ -39,6 +43,7 @@
 
 struct _GstVaapiVideoMeta
 {
+  GstBuffer *buffer;
   gint ref_count;
   GstVaapiDisplay *display;
   GstVaapiVideoPool *image_pool;
@@ -50,6 +55,23 @@ struct _GstVaapiVideoMeta
   guint has_render_rect:1;
 };
 
+static gboolean
+ensure_surface_proxy (GstVaapiVideoMeta * meta)
+{
+  if (!meta->proxy)
+    return FALSE;
+
+#if GST_CHECK_VERSION(1,0,0)
+  if (meta->buffer) {
+    GstMemory *const mem = gst_buffer_peek_memory (meta->buffer, 0);
+
+    if (GST_VAAPI_IS_VIDEO_MEMORY (mem))
+      return gst_vaapi_video_memory_sync (GST_VAAPI_VIDEO_MEMORY_CAST (mem));
+  }
+#endif
+  return TRUE;
+}
+
 static inline void
 set_display (GstVaapiVideoMeta * meta, GstVaapiDisplay * display)
 {
@@ -153,6 +175,7 @@ gst_vaapi_video_meta_finalize (GstVaapiVideoMeta * meta)
 static void
 gst_vaapi_video_meta_init (GstVaapiVideoMeta * meta)
 {
+  meta->buffer = NULL;
   meta->ref_count = 1;
   meta->display = NULL;
   meta->image_pool = NULL;
@@ -222,6 +245,7 @@ gst_vaapi_video_meta_copy (GstVaapiVideoMeta * meta)
   if (!copy)
     return NULL;
 
+  copy->buffer = NULL;
   copy->ref_count = 1;
   copy->display = gst_vaapi_display_ref (meta->display);
   copy->image_pool = NULL;
@@ -530,7 +554,8 @@ gst_vaapi_video_meta_get_surface (GstVaapiVideoMeta * meta)
 {
   g_return_val_if_fail (GST_VAAPI_IS_VIDEO_META (meta), NULL);
 
-  return meta->proxy ? GST_VAAPI_SURFACE_PROXY_SURFACE (meta->proxy) : NULL;
+  return ensure_surface_proxy (meta) ? GST_VAAPI_SURFACE_PROXY_SURFACE (meta->
+      proxy) : NULL;
 }
 
 /**
@@ -549,7 +574,7 @@ gst_vaapi_video_meta_get_surface_proxy (GstVaapiVideoMeta * meta)
 {
   g_return_val_if_fail (GST_VAAPI_IS_VIDEO_META (meta), NULL);
 
-  return meta->proxy;
+  return ensure_surface_proxy (meta) ? meta->proxy : NULL;
 }
 
 /**
@@ -760,6 +785,7 @@ gst_vaapi_video_meta_info_get (void)
 GstVaapiVideoMeta *
 gst_buffer_get_vaapi_video_meta (GstBuffer * buffer)
 {
+  GstVaapiVideoMeta *meta;
   GstMeta *m;
 
   g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL);
@@ -767,7 +793,11 @@ gst_buffer_get_vaapi_video_meta (GstBuffer * buffer)
   m = gst_buffer_get_meta (buffer, GST_VAAPI_VIDEO_META_API_TYPE);
   if (!m)
     return NULL;
-  return GST_VAAPI_VIDEO_META_HOLDER (m)->meta;
+
+  meta = GST_VAAPI_VIDEO_META_HOLDER (m)->meta;
+  if (meta)
+    meta->buffer = buffer;
+  return meta;
 }
 
 void
@@ -813,6 +843,7 @@ meta_quark_get (void)
 GstVaapiVideoMeta *
 gst_buffer_get_vaapi_video_meta (GstBuffer * buffer)
 {
+  GstVaapiVideoMeta *meta;
   const GstStructure *structure;
   const GValue *value;
 
@@ -826,7 +857,9 @@ gst_buffer_get_vaapi_video_meta (GstBuffer * buffer)
   if (!value)
     return NULL;
 
-  return GST_VAAPI_VIDEO_META (g_value_get_boxed (value));
+  meta = GST_VAAPI_VIDEO_META (g_value_get_boxed (value));
+  meta->buffer = buffer;
+  return meta;
 }
 
 void