From ceb60285ea35699111ba56ec1b1bc8a8d1273545 Mon Sep 17 00:00:00 2001 From: Gwenole Beauchesne Date: Fri, 21 Nov 2014 15:23:13 +0100 Subject: [PATCH] plugins: ensure VA surface is current prior to using it. When interacting with SW elements, the buffers and underlying video memory could be mapped as read/write. However, we need to use those buffers again as plain VA surfaces, we have to make sure the VA image is thus committed back to VA surface memory. This fixes pipelines involving avdec_* and vaapi{postproc,sink}. --- gst/vaapi/gstvaapivideomemory.c | 94 +++++++++++++++++++++++++++------ gst/vaapi/gstvaapivideomemory.h | 29 ++++++++++ gst/vaapi/gstvaapivideometa.c | 41 ++++++++++++-- 3 files changed, 144 insertions(+), 20 deletions(-) diff --git a/gst/vaapi/gstvaapivideomemory.c b/gst/vaapi/gstvaapivideomemory.c index 73cd0003a0..d287a75483 100644 --- a/gst/vaapi/gstvaapivideomemory.c +++ b/gst/vaapi/gstvaapivideomemory.c @@ -91,6 +91,23 @@ ensure_image (GstVaapiVideoMemory * mem) return TRUE; } +static gboolean +ensure_image_is_current (GstVaapiVideoMemory * mem) +{ + if (mem->use_direct_rendering) + return TRUE; + + if (!GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET (mem, + GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT)) { + if (!gst_vaapi_surface_get_image (mem->surface, mem->image)) + return FALSE; + + GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem, + GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT); + } + return TRUE; +} + static GstVaapiSurface * new_surface (GstVaapiDisplay * display, const GstVideoInfo * vip) { @@ -145,6 +162,23 @@ ensure_surface (GstVaapiVideoMemory * mem) return mem->surface != NULL; } +static gboolean +ensure_surface_is_current (GstVaapiVideoMemory * mem) +{ + if (mem->use_direct_rendering) + return TRUE; + + if (!GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET (mem, + GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT)) { + if (!gst_vaapi_surface_put_image (mem->surface, mem->image)) + return FALSE; + + GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem, + GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT); + } + return TRUE; +} + gboolean gst_video_meta_map_vaapi_memory (GstVideoMeta * meta, guint plane, GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags) @@ -168,12 +202,17 @@ gst_video_meta_map_vaapi_memory (GstVideoMeta * meta, guint plane, goto error_ensure_image; // Load VA image from surface - if ((flags & GST_MAP_READ) && !mem->use_direct_rendering) - gst_vaapi_surface_get_image (mem->surface, mem->image); + if ((flags & GST_MAP_READ) && !ensure_image_is_current (mem)) + goto error_no_current_image; if (!gst_vaapi_image_map (mem->image)) goto error_map_image; mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR; + + // Mark surface as dirty and expect updates from image + if (flags & GST_MAP_WRITE) + GST_VAAPI_VIDEO_MEMORY_FLAG_UNSET (mem, + GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT); } *data = gst_vaapi_image_get_plane (mem->image, plane); @@ -209,6 +248,11 @@ error_map_image: GST_VAAPI_ID_ARGS (gst_vaapi_image_get_id (mem->image))); return FALSE; } +error_no_current_image: + { + GST_ERROR ("failed to make image current"); + return FALSE; + } } gboolean @@ -229,23 +273,16 @@ gst_video_meta_unmap_vaapi_memory (GstVideoMeta * meta, guint plane, mem->map_type = 0; /* Unmap VA image used for read/writes */ - if (info->flags & GST_MAP_READWRITE) + if (info->flags & GST_MAP_READWRITE) { gst_vaapi_image_unmap (mem->image); - /* Commit VA image to surface */ - if ((info->flags & GST_MAP_WRITE) && !mem->use_direct_rendering) { - if (!gst_vaapi_surface_put_image (mem->surface, mem->image)) - goto error_upload_image; + if (info->flags & GST_MAP_WRITE) { + GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem, + GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT); + } } } return TRUE; - - /* ERRORS */ -error_upload_image: - { - GST_ERROR ("failed to upload image"); - return FALSE; - } } GstMemory * @@ -275,6 +312,9 @@ gst_vaapi_video_memory_new (GstAllocator * base_allocator, mem->map_type = 0; mem->map_count = 0; mem->use_direct_rendering = allocator->has_direct_rendering; + + GST_VAAPI_VIDEO_MEMORY_FLAG_SET (mem, + GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT); return GST_MEMORY_CAST (mem); } @@ -313,6 +353,14 @@ gst_vaapi_video_memory_reset_surface (GstVaapiVideoMemory * mem) gst_vaapi_video_meta_set_surface_proxy (mem->meta, NULL); } +gboolean +gst_vaapi_video_memory_sync (GstVaapiVideoMemory * mem) +{ + g_return_val_if_fail (mem, NULL); + + return ensure_surface_is_current (mem); +} + static gpointer gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize, guint flags) @@ -330,6 +378,8 @@ gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize, gst_vaapi_video_meta_get_surface_proxy (mem->meta)); if (!mem->proxy) goto error_no_surface_proxy; + if (!ensure_surface_is_current (mem)) + goto error_no_current_surface; mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE; break; case GST_MAP_READ: @@ -338,8 +388,8 @@ gst_vaapi_video_memory_map (GstVaapiVideoMemory * mem, gsize maxsize, goto error_no_surface; if (!ensure_image (mem)) goto error_no_image; - if (!mem->use_direct_rendering) - gst_vaapi_surface_get_image (mem->surface, mem->image); + if (!ensure_image_is_current (mem)) + goto error_no_current_image; if (!gst_vaapi_image_map (mem->image)) goto error_map_image; mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR; @@ -379,9 +429,15 @@ error_no_surface_proxy: error_no_surface: GST_ERROR ("failed to extract VA surface from video buffer"); return NULL; +error_no_current_surface: + GST_ERROR ("failed to make surface current"); + return NULL; error_no_image: GST_ERROR ("failed to extract VA image from video buffer"); return NULL; +error_no_current_image: + GST_ERROR ("failed to make image current"); + return NULL; error_map_image: GST_ERROR ("failed to map VA image"); return NULL; @@ -429,6 +485,9 @@ gst_vaapi_video_memory_copy (GstVaapiVideoMemory * mem, if (offset != 0 || (size != -1 && (gsize) size != maxsize)) goto error_unsupported; + if (!ensure_surface_is_current (mem)) + goto error_no_current_surface; + meta = gst_vaapi_video_meta_copy (mem->meta); if (!meta) goto error_allocate_memory; @@ -440,6 +499,9 @@ gst_vaapi_video_memory_copy (GstVaapiVideoMemory * mem, return GST_VAAPI_VIDEO_MEMORY_CAST (out_mem); /* ERRORS */ +error_no_current_surface: + GST_ERROR ("failed to make surface current"); + return NULL; error_unsupported: GST_ERROR ("failed to copy partial memory (unsupported operation)"); return NULL; diff --git a/gst/vaapi/gstvaapivideomemory.h b/gst/vaapi/gstvaapivideomemory.h index 74af3681e4..5681a1c065 100644 --- a/gst/vaapi/gstvaapivideomemory.h +++ b/gst/vaapi/gstvaapivideomemory.h @@ -42,12 +42,22 @@ typedef struct _GstVaapiVideoAllocatorClass GstVaapiVideoAllocatorClass; #define GST_VAAPI_VIDEO_MEMORY_CAST(mem) \ ((GstVaapiVideoMemory *) (mem)) +#define GST_VAAPI_IS_VIDEO_MEMORY(mem) \ + ((mem) && (mem)->allocator && GST_VAAPI_IS_VIDEO_ALLOCATOR((mem)->allocator)) + #define GST_VAAPI_VIDEO_MEMORY_NAME "GstVaapiVideoMemory" #if GST_CHECK_VERSION(1,1,0) #define GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE "memory:VASurface" #endif +#define GST_VAAPI_VIDEO_MEMORY_FLAG_IS_SET(mem, flag) \ + GST_MEMORY_FLAG_IS_SET (mem, flag) +#define GST_VAAPI_VIDEO_MEMORY_FLAG_SET(mem, flag) \ + GST_MINI_OBJECT_FLAG_SET (mem, flag) +#define GST_VAAPI_VIDEO_MEMORY_FLAG_UNSET(mem, flag) \ + GST_MEMORY_FLAG_UNSET (mem, flag) + /** * GstVaapiVideoMemoryMapType: * @GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE: map with gst_buffer_map() @@ -66,6 +76,21 @@ typedef enum GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR } GstVaapiVideoMemoryMapType; +/** + * GstVaapiVideoMemoryFlags: + * @GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT: The embedded + * #GstVaapiSurface has the up-to-date video frame contents. + * @GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT: The embedded + * #GstVaapiImage has the up-to-date video frame contents. + * + * The set of extended #GstMemory flags. + */ +typedef enum +{ + GST_VAAPI_VIDEO_MEMORY_FLAG_SURFACE_IS_CURRENT = GST_MEMORY_FLAG_LAST << 0, + GST_VAAPI_VIDEO_MEMORY_FLAG_IMAGE_IS_CURRENT = GST_MEMORY_FLAG_LAST << 1, +} GstVaapiVideoMemoryFlags; + /** * GstVaapiVideoMemory: * @@ -106,6 +131,10 @@ G_GNUC_INTERNAL void gst_vaapi_video_memory_reset_surface (GstVaapiVideoMemory * mem); +G_GNUC_INTERNAL +gboolean +gst_vaapi_video_memory_sync (GstVaapiVideoMemory * mem); + /* ------------------------------------------------------------------------ */ /* --- GstVaapiVideoAllocator --- */ /* ------------------------------------------------------------------------ */ diff --git a/gst/vaapi/gstvaapivideometa.c b/gst/vaapi/gstvaapivideometa.c index d67f458c26..ab97be5f65 100644 --- a/gst/vaapi/gstvaapivideometa.c +++ b/gst/vaapi/gstvaapivideometa.c @@ -32,6 +32,10 @@ #include #include "gstvaapivideometa.h" +#if GST_CHECK_VERSION(1,0,0) +# include "gstvaapivideomemory.h" +#endif + #define GST_VAAPI_VIDEO_META(obj) \ ((GstVaapiVideoMeta *) (obj)) #define GST_VAAPI_IS_VIDEO_META(obj) \ @@ -39,6 +43,7 @@ struct _GstVaapiVideoMeta { + GstBuffer *buffer; gint ref_count; GstVaapiDisplay *display; GstVaapiVideoPool *image_pool; @@ -50,6 +55,23 @@ struct _GstVaapiVideoMeta guint has_render_rect:1; }; +static gboolean +ensure_surface_proxy (GstVaapiVideoMeta * meta) +{ + if (!meta->proxy) + return FALSE; + +#if GST_CHECK_VERSION(1,0,0) + if (meta->buffer) { + GstMemory *const mem = gst_buffer_peek_memory (meta->buffer, 0); + + if (GST_VAAPI_IS_VIDEO_MEMORY (mem)) + return gst_vaapi_video_memory_sync (GST_VAAPI_VIDEO_MEMORY_CAST (mem)); + } +#endif + return TRUE; +} + static inline void set_display (GstVaapiVideoMeta * meta, GstVaapiDisplay * display) { @@ -153,6 +175,7 @@ gst_vaapi_video_meta_finalize (GstVaapiVideoMeta * meta) static void gst_vaapi_video_meta_init (GstVaapiVideoMeta * meta) { + meta->buffer = NULL; meta->ref_count = 1; meta->display = NULL; meta->image_pool = NULL; @@ -222,6 +245,7 @@ gst_vaapi_video_meta_copy (GstVaapiVideoMeta * meta) if (!copy) return NULL; + copy->buffer = NULL; copy->ref_count = 1; copy->display = gst_vaapi_display_ref (meta->display); copy->image_pool = NULL; @@ -530,7 +554,8 @@ gst_vaapi_video_meta_get_surface (GstVaapiVideoMeta * meta) { g_return_val_if_fail (GST_VAAPI_IS_VIDEO_META (meta), NULL); - return meta->proxy ? GST_VAAPI_SURFACE_PROXY_SURFACE (meta->proxy) : NULL; + return ensure_surface_proxy (meta) ? GST_VAAPI_SURFACE_PROXY_SURFACE (meta-> + proxy) : NULL; } /** @@ -549,7 +574,7 @@ gst_vaapi_video_meta_get_surface_proxy (GstVaapiVideoMeta * meta) { g_return_val_if_fail (GST_VAAPI_IS_VIDEO_META (meta), NULL); - return meta->proxy; + return ensure_surface_proxy (meta) ? meta->proxy : NULL; } /** @@ -760,6 +785,7 @@ gst_vaapi_video_meta_info_get (void) GstVaapiVideoMeta * gst_buffer_get_vaapi_video_meta (GstBuffer * buffer) { + GstVaapiVideoMeta *meta; GstMeta *m; g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL); @@ -767,7 +793,11 @@ gst_buffer_get_vaapi_video_meta (GstBuffer * buffer) m = gst_buffer_get_meta (buffer, GST_VAAPI_VIDEO_META_API_TYPE); if (!m) return NULL; - return GST_VAAPI_VIDEO_META_HOLDER (m)->meta; + + meta = GST_VAAPI_VIDEO_META_HOLDER (m)->meta; + if (meta) + meta->buffer = buffer; + return meta; } void @@ -813,6 +843,7 @@ meta_quark_get (void) GstVaapiVideoMeta * gst_buffer_get_vaapi_video_meta (GstBuffer * buffer) { + GstVaapiVideoMeta *meta; const GstStructure *structure; const GValue *value; @@ -826,7 +857,9 @@ gst_buffer_get_vaapi_video_meta (GstBuffer * buffer) if (!value) return NULL; - return GST_VAAPI_VIDEO_META (g_value_get_boxed (value)); + meta = GST_VAAPI_VIDEO_META (g_value_get_boxed (value)); + meta->buffer = buffer; + return meta; } void