plugins: ensure VA surface is current prior to using it.

When interacting with SW elements, the buffers and underlying video
memory could be mapped as read/write. However, we need to use those
buffers again as plain VA surfaces, we have to make sure the VA image
is thus committed back to VA surface memory.

This fixes pipelines involving avdec_* and vaapi{postproc,sink}.

gst/vaapi/gstvaapivideomemory.c

@@ -91,6 +91,23 @@ ensure_image (GstVaapiVideoMemory * mem)
   return TRUE;
 }
 
+static gboolean
+ensure_image_is_current (GstVaapiVideoMemory * mem)
+{
+  if (mem->use_direct_rendering)
+    return TRUE;
+
+  if (!GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET (mem,
+          GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT)) {
+    if (!gst_vaapi_surface_get_image (mem->surface, mem->image))
+      return FALSE;
+
+    GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem,
+        GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT);
+  }
+  return TRUE;
+}
+
 static GstVaapiSurface *
 new_surface (GstVaapiDisplay * display, const GstVideoInfo * vip)
 {
@@ -145,6 +162,23 @@ ensure_surface (GstVaapiVideoMemory * mem)
   return mem->surface != NULL;
 }
 
+static gboolean
+ensure_surface_is_current (GstVaapiVideoMemory * mem)
+{
+  if (mem->use_direct_rendering)
+    return TRUE;
+
+  if (!GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET (mem,
+          GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT)) {
+    if (!gst_vaapi_surface_put_image (mem->surface, mem->image))
+      return FALSE;
+
+    GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem,
+        GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT);
+  }
+  return TRUE;
+}
+
 gboolean
 gst_video_meta_map_vaapi_memory (GstVideoMeta * meta, guint plane,
     GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags)
@@ -168,12 +202,17 @@ gst_video_meta_map_vaapi_memory (GstVideoMeta * meta, guint plane,
       goto error_ensure_image;
 
     // Load VA image from surface
-    if ((flags & GST_MAP_READ) && !mem->use_direct_rendering)
+    if ((flags & GST_MAP_READ) && !ensure_image_is_current (mem))
-      gst_vaapi_surface_get_image (mem->surface, mem->image);
+      goto error_no_current_image;
 
     if (!gst_vaapi_image_map (mem->image))
       goto error_map_image;
     mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR;
+
+    // Mark surface as dirty and expect updates from image
+    if (flags & GST_MAP_WRITE)
+      GST_VAAPI_VIDEO_MEMORY_FLAG_UNSET (mem,
+          GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT);
   }
 
   *data = gst_vaapi_image_get_plane (mem->image, plane);
@@ -209,6 +248,11 @@ error_map_image:
         GST_VAAPI_ID_ARGS (gst_vaapi_image_get_id (mem->image)));
     return FALSE;
   }
+error_no_current_image:
+  {
+    GST_ERROR ("failed to make image current");
+    return FALSE;
+  }
 }
 
 gboolean
@@ -229,23 +273,16 @@ gst_video_meta_unmap_vaapi_memory (GstVideoMeta * meta, guint plane,
     mem->map_type = 0;
 
     /* Unmap VA image used for read/writes */
-    if (info->flags & GST_MAP_READWRITE)
+    if (info->flags & GST_MAP_READWRITE) {
       gst_vaapi_image_unmap (mem->image);
 
-    /* Commit VA image to surface */
+      if (info->flags & GST_MAP_WRITE) {
-    if ((info->flags & GST_MAP_WRITE) && !mem->use_direct_rendering) {
+        GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem,
-      if (!gst_vaapi_surface_put_image (mem->surface, mem->image))
+            GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT);
-        goto error_upload_image;
+      }
     }
   }
   return TRUE;
-
-  /* ERRORS */
-error_upload_image:
-  {
-    GST_ERROR ("failed to upload image");
-    return FALSE;
-  }
 }
 
 GstMemory *
@@ -275,6 +312,9 @@ gst_vaapi_video_memory_new (GstAllocator * base_allocator,
   mem->map_type = 0;
   mem->map_count = 0;
   mem->use_direct_rendering = allocator->has_direct_rendering;
+
+  GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem,
+      GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT);
   return GST_MEMORY_CAST (mem);
 }
 
@@ -313,6 +353,14 @@ gst_vaapi_video_memory_reset_surface (GstVaapiVideoMemory * mem)
     gst_vaapi_video_meta_set_surface_proxy (mem->meta, NULL);
 }
 
+gboolean
+gst_vaapi_video_memory_sync (GstVaapiVideoMemory * mem)
+{
+  g_return_val_if_fail (mem, NULL);
+
+  return ensure_surface_is_current (mem);
+}
+
 static gpointer
 gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize,
     guint flags)
@@ -330,6 +378,8 @@ gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize,
             gst_vaapi_video_meta_get_surface_proxy (mem->meta));
         if (!mem->proxy)
           goto error_no_surface_proxy;
+        if (!ensure_surface_is_current (mem))
+          goto error_no_current_surface;
         mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE;
         break;
       case GST_MAP_READ:
@@ -338,8 +388,8 @@ gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize,
           goto error_no_surface;
         if (!ensure_image (mem))
           goto error_no_image;
-        if (!mem->use_direct_rendering)
+        if (!ensure_image_is_current (mem))
-          gst_vaapi_surface_get_image (mem->surface, mem->image);
+          goto error_no_current_image;
         if (!gst_vaapi_image_map (mem->image))
           goto error_map_image;
         mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR;
@@ -379,9 +429,15 @@ error_no_surface_proxy:
 error_no_surface:
   GST_ERROR ("failed to extract VA surface from video buffer");
   return NULL;
+error_no_current_surface:
+  GST_ERROR ("failed to make surface current");
+  return NULL;
 error_no_image:
   GST_ERROR ("failed to extract VA image from video buffer");
   return NULL;
+error_no_current_image:
+  GST_ERROR ("failed to make image current");
+  return NULL;
 error_map_image:
   GST_ERROR ("failed to map VA image");
   return NULL;
@@ -429,6 +485,9 @@ gst_vaapi_video_memory_copy (GstVaapiVideoMemory * mem,
   if (offset != 0 || (size != -1 && (gsize) size != maxsize))
     goto error_unsupported;
 
+  if (!ensure_surface_is_current (mem))
+    goto error_no_current_surface;
+
   meta = gst_vaapi_video_meta_copy (mem->meta);
   if (!meta)
     goto error_allocate_memory;
@@ -440,6 +499,9 @@ gst_vaapi_video_memory_copy (GstVaapiVideoMemory * mem,
   return GST_VAAPI_VIDEO_MEMORY_CAST (out_mem);
 
   /* ERRORS */
+error_no_current_surface:
+  GST_ERROR ("failed to make surface current");
+  return NULL;
 error_unsupported:
   GST_ERROR ("failed to copy partial memory (unsupported operation)");
   return NULL;

gst/vaapi/gstvaapivideomemory.h

@@ -42,12 +42,22 @@ typedef struct _GstVaapiVideoAllocatorClass GstVaapiVideoAllocatorClass;
 #define GST_VAAPI_VIDEO_MEMORY_CAST(mem) \
   ((GstVaapiVideoMemory *) (mem))
 
+#define GST_VAAPI_IS_VIDEO_MEMORY(mem) \
+  ((mem) && (mem)->allocator && GST_VAAPI_IS_VIDEO_ALLOCATOR((mem)->allocator))
+
 #define GST_VAAPI_VIDEO_MEMORY_NAME             "GstVaapiVideoMemory"
 
 #if GST_CHECK_VERSION(1,1,0)
 #define GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE   "memory:VASurface"
 #endif
 
+#define GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET(mem, flag) \
+  GST_MEMORY_FLAG_IS_SET (mem, flag)
+#define GST_VAAPI_VIDEO_MEMORY_FLAG_SET(mem, flag) \
+  GST_MINI_OBJECT_FLAG_SET (mem, flag)
+#define GST_VAAPI_VIDEO_MEMORY_FLAG_UNSET(mem, flag) \
+  GST_MEMORY_FLAG_UNSET (mem, flag)
+
 /**
  * GstVaapiVideoMemoryMapType:
  * @GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE: map with gst_buffer_map()
@@ -66,6 +76,21 @@ typedef enum
   GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR
 } GstVaapiVideoMemoryMapType;
 
+/**
+ * GstVaapiVideoMemoryFlags:
+ * @GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT: The embedded
+ *   #GstVaapiSurface has the up-to-date video frame contents.
+ * @GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT: The embedded
+ *   #GstVaapiImage has the up-to-date video frame contents.
+ *
+ * The set of extended #GstMemory flags.
+ */
+typedef enum
+{
+  GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT = GST_MEMORY_FLAG_LAST << 0,
+  GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT = GST_MEMORY_FLAG_LAST << 1,
+} GstVaapiVideoMemoryFlags;
+
 /**
  * GstVaapiVideoMemory:
  *
@@ -106,6 +131,10 @@ G_GNUC_INTERNAL
 void
 gst_vaapi_video_memory_reset_surface (GstVaapiVideoMemory * mem);
 
+G_GNUC_INTERNAL
+gboolean
+gst_vaapi_video_memory_sync (GstVaapiVideoMemory * mem);
+
 /* ------------------------------------------------------------------------ */
 /* --- GstVaapiVideoAllocator                                           --- */
 /* ------------------------------------------------------------------------ */

gst/vaapi/gstvaapivideometa.c

@@ -32,6 +32,10 @@
 #include <gst/vaapi/gstvaapisurfacepool.h>
 #include "gstvaapivideometa.h"
 
+#if GST_CHECK_VERSION(1,0,0)
+# include "gstvaapivideomemory.h"
+#endif
+
 #define GST_VAAPI_VIDEO_META(obj) \
   ((GstVaapiVideoMeta *) (obj))
 #define GST_VAAPI_IS_VIDEO_META(obj) \
@@ -39,6 +43,7 @@
 
 struct _GstVaapiVideoMeta
 {
+  GstBuffer *buffer;
   gint ref_count;
   GstVaapiDisplay *display;
   GstVaapiVideoPool *image_pool;
@@ -50,6 +55,23 @@ struct _GstVaapiVideoMeta
   guint has_render_rect:1;
 };
 
+static gboolean
+ensure_surface_proxy (GstVaapiVideoMeta * meta)
+{
+  if (!meta->proxy)
+    return FALSE;
+
+#if GST_CHECK_VERSION(1,0,0)
+  if (meta->buffer) {
+    GstMemory *const mem = gst_buffer_peek_memory (meta->buffer, 0);
+
+    if (GST_VAAPI_IS_VIDEO_MEMORY (mem))
+      return gst_vaapi_video_memory_sync (GST_VAAPI_VIDEO_MEMORY_CAST (mem));
+  }
+#endif
+  return TRUE;
+}
+
 static inline void
 set_display (GstVaapiVideoMeta * meta, GstVaapiDisplay * display)
 {
@@ -153,6 +175,7 @@ gst_vaapi_video_meta_finalize (GstVaapiVideoMeta * meta)
 static void
 gst_vaapi_video_meta_init (GstVaapiVideoMeta * meta)
 {
+  meta->buffer = NULL;
   meta->ref_count = 1;
   meta->display = NULL;
   meta->image_pool = NULL;
@@ -222,6 +245,7 @@ gst_vaapi_video_meta_copy (GstVaapiVideoMeta * meta)
   if (!copy)
     return NULL;
 
+  copy->buffer = NULL;
   copy->ref_count = 1;
   copy->display = gst_vaapi_display_ref (meta->display);
   copy->image_pool = NULL;
@@ -530,7 +554,8 @@ gst_vaapi_video_meta_get_surface (GstVaapiVideoMeta * meta)
 {
   g_return_val_if_fail (GST_VAAPI_IS_VIDEO_META (meta), NULL);
 
-  return meta->proxy ? GST_VAAPI_SURFACE_PROXY_SURFACE (meta->proxy) : NULL;
+  return ensure_surface_proxy (meta) ? GST_VAAPI_SURFACE_PROXY_SURFACE (meta->
+      proxy) : NULL;
 }
 
 /**
@@ -549,7 +574,7 @@ gst_vaapi_video_meta_get_surface_proxy (GstVaapiVideoMeta * meta)
 {
   g_return_val_if_fail (GST_VAAPI_IS_VIDEO_META (meta), NULL);
 
-  return meta->proxy;
+  return ensure_surface_proxy (meta) ? meta->proxy : NULL;
 }
 
 /**
@@ -760,6 +785,7 @@ gst_vaapi_video_meta_info_get (void)
 GstVaapiVideoMeta *
 gst_buffer_get_vaapi_video_meta (GstBuffer * buffer)
 {
+  GstVaapiVideoMeta *meta;
   GstMeta *m;
 
   g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL);
@@ -767,7 +793,11 @@ gst_buffer_get_vaapi_video_meta (GstBuffer * buffer)
   m = gst_buffer_get_meta (buffer, GST_VAAPI_VIDEO_META_API_TYPE);
   if (!m)
     return NULL;
-  return GST_VAAPI_VIDEO_META_HOLDER (m)->meta;
+
+  meta = GST_VAAPI_VIDEO_META_HOLDER (m)->meta;
+  if (meta)
+    meta->buffer = buffer;
+  return meta;
 }
 
 void
@@ -813,6 +843,7 @@ meta_quark_get (void)
 GstVaapiVideoMeta *
 gst_buffer_get_vaapi_video_meta (GstBuffer * buffer)
 {
+  GstVaapiVideoMeta *meta;
   const GstStructure *structure;
   const GValue *value;
 
@@ -826,7 +857,9 @@ gst_buffer_get_vaapi_video_meta (GstBuffer * buffer)
   if (!value)
     return NULL;
 
-  return GST_VAAPI_VIDEO_META (g_value_get_boxed (value));
+  meta = GST_VAAPI_VIDEO_META (g_value_get_boxed (value));
+  meta->buffer = buffer;
+  return meta;
 }
 
 void