From f35eeaa02b71fbd017ffc4b1032cee41a0d54e2f Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Sat, 3 Oct 2020 18:53:46 +0900 Subject: [PATCH] d3d11: Don't hold staging texture Staging texture is used for memory transfer between system and gpu memory. Apart from d3d11{upload,download} elements, however, it should happen very rarely. Before this commit, d3d11bufferpool was allocating at least one staging texture in order to calculate cpu accessible memory size, and it wasn't freed for later use of the texture unconditionally. But it will increase system memory usage. Although GstD3D11memory object is implemented so that support CPU access, most memory transfer will happen in d3d11{upload,download} elements. By this commit, the initial staging texture will be freed immediately once cpu accessible memory size is calculated. Part-of: --- sys/d3d11/gstd3d11bufferpool.c | 70 ++++++++------- sys/d3d11/gstd3d11memory.c | 158 ++++++++++++++++++++++----------- sys/d3d11/gstd3d11memory.h | 21 +++-- sys/d3d11/gstd3d11utils.c | 74 +++++++++++++++ sys/d3d11/gstd3d11utils.h | 6 ++ 5 files changed, 235 insertions(+), 94 deletions(-) diff --git a/sys/d3d11/gstd3d11bufferpool.c b/sys/d3d11/gstd3d11bufferpool.c index 671f796a69..80cf647bb9 100644 --- a/sys/d3d11/gstd3d11bufferpool.c +++ b/sys/d3d11/gstd3d11bufferpool.c @@ -24,6 +24,9 @@ #include "gstd3d11bufferpool.h" #include "gstd3d11memory.h" #include "gstd3d11device.h" +#include "gstd3d11utils.h" + +#include GST_DEBUG_CATEGORY_STATIC (gst_d3d11_buffer_pool_debug); #define GST_CAT_DEFAULT gst_d3d11_buffer_pool_debug @@ -33,11 +36,12 @@ struct _GstD3D11BufferPoolPrivate GstD3D11Device *device; GstD3D11Allocator *allocator; - /* initial buffer used for calculating buffer size */ - GstBuffer *initial_buffer; - gboolean add_videometa; GstD3D11AllocationParams *d3d11_params; + + gint stride[GST_VIDEO_MAX_PLANES]; + gsize size[GST_VIDEO_MAX_PLANES]; + gsize offset[GST_VIDEO_MAX_PLANES]; }; #define gst_d3d11_buffer_pool_parent_class parent_class @@ -87,7 +91,6 @@ gst_d3d11_buffer_pool_dispose (GObject * object) gst_d3d11_allocation_params_free (priv->d3d11_params); priv->d3d11_params = NULL; - gst_clear_buffer (&priv->initial_buffer); gst_clear_object (&priv->device); gst_clear_object (&priv->allocator); @@ -114,6 +117,7 @@ gst_d3d11_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config) GstAllocator *allocator = NULL; gboolean ret = TRUE; D3D11_TEXTURE2D_DESC *desc; + GstBuffer *staging_buffer; gint i; if (!gst_buffer_pool_config_get_params (config, &caps, NULL, &min_buffers, @@ -133,7 +137,6 @@ gst_d3d11_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config) if (!gst_buffer_pool_config_get_allocator (config, &allocator, NULL)) goto wrong_config; - gst_clear_buffer (&priv->initial_buffer); gst_clear_object (&priv->allocator); if (allocator) { @@ -227,14 +230,33 @@ gst_d3d11_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config) } } - gst_d3d11_buffer_pool_alloc (pool, &priv->initial_buffer, NULL); + staging_buffer = gst_d3d11_allocate_staging_buffer (priv->allocator, + &info, priv->d3d11_params->d3d11_format, priv->d3d11_params->desc, TRUE); - if (!priv->initial_buffer) { - GST_ERROR_OBJECT (pool, "Could not create initial buffer"); + if (!staging_buffer) { + GST_ERROR_OBJECT (pool, "Couldn't allocated staging buffer"); return FALSE; + } else { + GstVideoMeta *meta = gst_buffer_get_video_meta (staging_buffer); + + if (!meta) { + GST_ERROR_OBJECT (pool, "Buffer doesn't have video meta"); + gst_buffer_unref (staging_buffer); + return FALSE; + } + + for (i = 0; i < gst_buffer_n_memory (staging_buffer); i++) { + GstMemory *mem = gst_buffer_peek_memory (staging_buffer, i); + + priv->size[i] = gst_memory_get_sizes (mem, NULL, NULL); + } + + memcpy (priv->offset, meta->offset, sizeof (priv->offset)); + memcpy (priv->stride, meta->stride, sizeof (priv->stride)); } - self->buffer_size = gst_buffer_get_size (priv->initial_buffer); + self->buffer_size = gst_buffer_get_size (staging_buffer); + gst_buffer_unref (staging_buffer); gst_buffer_pool_config_set_params (config, caps, self->buffer_size, min_buffers, max_buffers); @@ -275,34 +297,22 @@ gst_d3d11_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer, GstBuffer *buf; GstD3D11AllocationParams *d3d11_params = priv->d3d11_params; GstVideoInfo *info = &d3d11_params->info; - GstVideoInfo *aligned_info = &d3d11_params->aligned_info; - gint n_texture = 0; gint i; - gsize offset[GST_VIDEO_MAX_PLANES] = { 0, }; - - /* consume pre-allocated buffer if any */ - if (G_UNLIKELY (priv->initial_buffer)) { - *buffer = priv->initial_buffer; - priv->initial_buffer = NULL; - - return GST_FLOW_OK; - } buf = gst_buffer_new (); if (d3d11_params->d3d11_format->dxgi_format == DXGI_FORMAT_UNKNOWN) { - for (n_texture = 0; n_texture < GST_VIDEO_INFO_N_PLANES (info); n_texture++) { - d3d11_params->plane = n_texture; - mem = gst_d3d11_allocator_alloc (priv->allocator, d3d11_params); + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) { + mem = gst_d3d11_allocator_alloc (priv->allocator, &d3d11_params->desc[i], + d3d11_params->flags, priv->size[i]); if (!mem) goto error; gst_buffer_append_memory (buf, mem); } } else { - d3d11_params->plane = 0; - mem = gst_d3d11_allocator_alloc (priv->allocator, priv->d3d11_params); - n_texture++; + mem = gst_d3d11_allocator_alloc (priv->allocator, &d3d11_params->desc[0], + d3d11_params->flags, priv->size[0]); if (!mem) goto error; @@ -310,18 +320,12 @@ gst_d3d11_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer, gst_buffer_append_memory (buf, mem); } - /* calculate offset */ - for (i = 0; i < n_texture && i < GST_VIDEO_MAX_PLANES - 1; i++) { - offset[i + 1] = offset[i] + - d3d11_params->stride[i] * GST_VIDEO_INFO_COMP_HEIGHT (aligned_info, i); - } - if (priv->add_videometa) { GST_DEBUG_OBJECT (self, "adding GstVideoMeta"); gst_buffer_add_video_meta_full (buf, GST_VIDEO_FRAME_FLAG_NONE, GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info), GST_VIDEO_INFO_N_PLANES (info), - offset, d3d11_params->stride); + priv->offset, priv->stride); } *buffer = buf; diff --git a/sys/d3d11/gstd3d11memory.c b/sys/d3d11/gstd3d11memory.c index d1a43e5a4d..15257ccec7 100644 --- a/sys/d3d11/gstd3d11memory.c +++ b/sys/d3d11/gstd3d11memory.c @@ -261,11 +261,55 @@ map_cpu_access_data (GstD3D11Memory * dmem, D3D11_MAP map_type) return ret; } +static gpointer +gst_d3d11_memory_map_staging (GstMemory * mem, GstMapFlags flags) +{ + GstD3D11Memory *dmem = (GstD3D11Memory *) mem; + + g_mutex_lock (&dmem->lock); + if (dmem->cpu_map_count == 0) { + ID3D11DeviceContext *device_context = + gst_d3d11_device_get_device_context_handle (dmem->device); + D3D11_MAP map_type; + HRESULT hr; + gboolean ret = TRUE; + + map_type = gst_map_flags_to_d3d11 (flags); + + gst_d3d11_device_lock (dmem->device); + hr = ID3D11DeviceContext_Map (device_context, + (ID3D11Resource *) dmem->texture, 0, map_type, 0, &dmem->map); + if (!gst_d3d11_result (hr, dmem->device)) { + GST_ERROR_OBJECT (GST_MEMORY_CAST (dmem)->allocator, + "Failed to map staging texture (0x%x)", (guint) hr); + ret = FALSE; + } + gst_d3d11_device_unlock (dmem->device); + + if (!ret) { + g_mutex_unlock (&dmem->lock); + return NULL; + } + } + + dmem->cpu_map_count++; + g_mutex_unlock (&dmem->lock); + + return dmem->map.pData; +} + static gpointer gst_d3d11_memory_map (GstMemory * mem, gsize maxsize, GstMapFlags flags) { GstD3D11Memory *dmem = (GstD3D11Memory *) mem; + if (dmem->type == GST_D3D11_MEMORY_TYPE_STAGING) { + if ((flags & GST_MAP_D3D11) == GST_MAP_D3D11) + return dmem->texture; + + return gst_d3d11_memory_map_staging (mem, flags); + } + g_mutex_lock (&dmem->lock); if ((flags & GST_MAP_D3D11) == GST_MAP_D3D11) { if (dmem->staging && @@ -336,6 +380,9 @@ unmap_cpu_access_data (GstD3D11Memory * dmem) ID3D11DeviceContext *device_context = gst_d3d11_device_get_device_context_handle (dmem->device); + if (dmem->type == GST_D3D11_MEMORY_TYPE_STAGING) + staging = (ID3D11Resource *) dmem->texture; + gst_d3d11_device_lock (dmem->device); ID3D11DeviceContext_Unmap (device_context, staging, 0); gst_d3d11_device_unlock (dmem->device); @@ -348,14 +395,16 @@ gst_d3d11_memory_unmap_full (GstMemory * mem, GstMapInfo * info) g_mutex_lock (&dmem->lock); if ((info->flags & GST_MAP_D3D11) == GST_MAP_D3D11) { - if ((info->flags & GST_MAP_WRITE) == GST_MAP_WRITE) + if (dmem->type != GST_D3D11_MEMORY_TYPE_STAGING && + (info->flags & GST_MAP_WRITE) == GST_MAP_WRITE) GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD); g_mutex_unlock (&dmem->lock); return; } - if ((info->flags & GST_MAP_WRITE)) + if (dmem->type != GST_D3D11_MEMORY_TYPE_STAGING && + (info->flags & GST_MAP_WRITE)) GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_UPLOAD); dmem->cpu_map_count--; @@ -498,7 +547,7 @@ gst_d3d11_allocator_new (GstD3D11Device * device) static gboolean calculate_mem_size (GstD3D11Device * device, ID3D11Texture2D * texture, - D3D11_TEXTURE2D_DESC * desc, D3D11_MAP map_type, + const D3D11_TEXTURE2D_DESC * desc, D3D11_MAP map_type, gint stride[GST_VIDEO_MAX_PLANES], gsize * size) { HRESULT hr; @@ -684,31 +733,24 @@ error: GstMemory * gst_d3d11_allocator_alloc (GstD3D11Allocator * allocator, - GstD3D11AllocationParams * params) + const D3D11_TEXTURE2D_DESC * desc, GstD3D11AllocationFlags flags, + gsize size) { GstD3D11Memory *mem; GstD3D11Device *device; ID3D11Texture2D *texture = NULL; - ID3D11Texture2D *staging = NULL; - D3D11_TEXTURE2D_DESC *desc; - gsize *size; - gboolean is_first = FALSE; guint index_to_use = 0; GstD3D11AllocatorPrivate *priv; GstD3D11MemoryType type = GST_D3D11_MEMORY_TYPE_TEXTURE; g_return_val_if_fail (GST_IS_D3D11_ALLOCATOR (allocator), NULL); - g_return_val_if_fail (params != NULL, NULL); + g_return_val_if_fail (desc != NULL, NULL); + g_return_val_if_fail (size > 0, NULL); priv = allocator->priv; device = allocator->device; - desc = ¶ms->desc[params->plane]; - size = ¶ms->size[params->plane]; - if (*size == 0) - is_first = TRUE; - - if ((params->flags & GST_D3D11_ALLOCATION_FLAG_TEXTURE_ARRAY)) { + if ((flags & GST_D3D11_ALLOCATION_FLAG_TEXTURE_ARRAY)) { gint i; do_again: @@ -758,58 +800,74 @@ gst_d3d11_allocator_alloc (GstD3D11Allocator * allocator, } } - /* per plane, allocated staging texture to calculate actual size, - * stride, and offset */ - if (is_first) { - gint num_plane; - gint stride[GST_VIDEO_MAX_PLANES]; - gsize mem_size; - gint i; - - staging = create_staging_texture (device, desc); - if (!staging) { - GST_ERROR_OBJECT (allocator, "Couldn't create staging texture"); - goto error; - } - - if (!calculate_mem_size (device, - staging, desc, D3D11_MAP_READ, stride, &mem_size)) - goto error; - - num_plane = gst_d3d11_dxgi_format_n_planes (desc->Format); - - for (i = 0; i < num_plane; i++) { - params->stride[params->plane + i] = stride[i]; - } - - *size = mem_size; - } - mem = g_new0 (GstD3D11Memory, 1); gst_memory_init (GST_MEMORY_CAST (mem), - 0, GST_ALLOCATOR_CAST (allocator), NULL, *size, 0, 0, *size); + 0, GST_ALLOCATOR_CAST (allocator), NULL, size, 0, 0, size); g_mutex_init (&mem->lock); - mem->info = params->info; - mem->plane = params->plane; mem->desc = *desc; mem->texture = texture; - mem->staging = staging; mem->device = gst_object_ref (device); mem->type = type; mem->subresource_index = index_to_use; - if (staging) - GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD); - return GST_MEMORY_CAST (mem); error: if (texture) gst_d3d11_device_release_texture (device, texture); - if (staging) + return NULL; +} + +GstMemory * +gst_d3d11_allocator_alloc_staging (GstD3D11Allocator * allocator, + const D3D11_TEXTURE2D_DESC * desc, GstD3D11AllocationFlags flags, + gint * stride) +{ + GstD3D11Memory *mem; + GstD3D11Device *device; + ID3D11Texture2D *texture = NULL; + gsize mem_size = 0; + gint mem_stride[GST_VIDEO_MAX_PLANES]; + + g_return_val_if_fail (GST_IS_D3D11_ALLOCATOR (allocator), NULL); + g_return_val_if_fail (desc != NULL, NULL); + + device = allocator->device; + + texture = create_staging_texture (device, desc); + if (!texture) { + GST_ERROR_OBJECT (allocator, "Couldn't create staging texture"); + goto error; + } + + if (!calculate_mem_size (device, + texture, desc, D3D11_MAP_READ, mem_stride, &mem_size)) { + GST_ERROR_OBJECT (allocator, "Couldn't calculate staging texture size"); + goto error; + } + + mem = g_new0 (GstD3D11Memory, 1); + + gst_memory_init (GST_MEMORY_CAST (mem), + 0, GST_ALLOCATOR_CAST (allocator), NULL, mem_size, 0, 0, mem_size); + + g_mutex_init (&mem->lock); + mem->desc = *desc; + mem->texture = texture; + mem->device = gst_object_ref (device); + mem->type = GST_D3D11_MEMORY_TYPE_STAGING; + + /* every plan will have identical size */ + if (stride) + *stride = mem_stride[0]; + + return GST_MEMORY_CAST (mem); + +error: + if (texture) gst_d3d11_device_release_texture (device, texture); return NULL; diff --git a/sys/d3d11/gstd3d11memory.h b/sys/d3d11/gstd3d11memory.h index 178e0fc434..42da320aed 100644 --- a/sys/d3d11/gstd3d11memory.h +++ b/sys/d3d11/gstd3d11memory.h @@ -84,13 +84,6 @@ struct _GstD3D11AllocationParams GstVideoInfo aligned_info; const GstD3D11Format *d3d11_format; - /* size and stride of staging texture, set by allocator */ - gint stride[GST_VIDEO_MAX_PLANES]; - gsize size[GST_VIDEO_MAX_PLANES]; - - /* Current target plane for allocation */ - guint plane; - GstD3D11AllocationFlags flags; /*< private >*/ @@ -101,6 +94,7 @@ typedef enum { GST_D3D11_MEMORY_TYPE_TEXTURE = 0, GST_D3D11_MEMORY_TYPE_ARRAY = 1, + GST_D3D11_MEMORY_TYPE_STAGING = 2, } GstD3D11MemoryType; struct _GstD3D11Memory @@ -119,9 +113,6 @@ struct _GstD3D11Memory ID3D11RenderTargetView *render_target_view[GST_VIDEO_MAX_PLANES]; guint num_render_target_views; - GstVideoInfo info; - - guint plane; GstD3D11MemoryType type; /* > 0 if this is Array typed memory */ @@ -173,7 +164,15 @@ GType gst_d3d11_allocator_get_type (void); GstD3D11Allocator * gst_d3d11_allocator_new (GstD3D11Device *device); GstMemory * gst_d3d11_allocator_alloc (GstD3D11Allocator * allocator, - GstD3D11AllocationParams * params); + const D3D11_TEXTURE2D_DESC * desc, + GstD3D11AllocationFlags flags, + gsize size); + +GstMemory * gst_d3d11_allocator_alloc_staging (GstD3D11Allocator * allocator, + const D3D11_TEXTURE2D_DESC * desc, + GstD3D11AllocationFlags flags, + gint * stride); + void gst_d3d11_allocator_set_flushing (GstD3D11Allocator * allocator, gboolean flushing); diff --git a/sys/d3d11/gstd3d11utils.c b/sys/d3d11/gstd3d11utils.c index 1d4d30d539..e97bc7e6c7 100644 --- a/sys/d3d11/gstd3d11utils.c +++ b/sys/d3d11/gstd3d11utils.c @@ -23,6 +23,7 @@ #include "gstd3d11utils.h" #include "gstd3d11device.h" +#include "gstd3d11memory.h" #include #include @@ -397,6 +398,79 @@ gst_d3d11_get_device_vendor (GstD3D11Device * device) return vendor; } +GstBuffer * +gst_d3d11_allocate_staging_buffer (GstD3D11Allocator * allocator, + const GstVideoInfo * info, const GstD3D11Format * format, + const D3D11_TEXTURE2D_DESC desc[GST_VIDEO_MAX_PLANES], + gboolean add_videometa) +{ + GstBuffer *buffer; + gint i; + gint stride[GST_VIDEO_MAX_PLANES] = { 0, }; + gsize offset[GST_VIDEO_MAX_PLANES] = { 0, }; + GstMemory *mem; + + g_return_val_if_fail (GST_IS_D3D11_ALLOCATOR (allocator), NULL); + g_return_val_if_fail (info != NULL, NULL); + g_return_val_if_fail (format != NULL, NULL); + g_return_val_if_fail (desc != NULL, NULL); + + buffer = gst_buffer_new (); + + if (format->dxgi_format == DXGI_FORMAT_UNKNOWN) { + gsize size[GST_VIDEO_MAX_PLANES] = { 0, }; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) { + mem = gst_d3d11_allocator_alloc_staging (allocator, &desc[i], 0, + &stride[i]); + + if (!mem) { + GST_ERROR_OBJECT (allocator, "Couldn't allocate memory for plane %d", + i); + goto error; + } + + size[i] = gst_memory_get_sizes (mem, NULL, NULL); + if (i > 0) + offset[i] = offset[i - 1] + size[i - 1]; + gst_buffer_append_memory (buffer, mem); + } + } else { + /* must be YUV semi-planar or single plane */ + g_assert (GST_VIDEO_INFO_N_PLANES (info) <= 2); + + mem = gst_d3d11_allocator_alloc_staging (allocator, &desc[0], 0, + &stride[0]); + + if (!mem) { + GST_ERROR_OBJECT (allocator, "Couldn't allocate memory"); + goto error; + } + + gst_memory_get_sizes (mem, NULL, NULL); + gst_buffer_append_memory (buffer, mem); + + if (GST_VIDEO_INFO_N_PLANES (info) == 2) { + stride[1] = stride[0]; + offset[1] = stride[0] * desc[0].Height; + } + } + + if (add_videometa) { + gst_buffer_add_video_meta_full (buffer, GST_VIDEO_FRAME_FLAG_NONE, + GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info), + GST_VIDEO_INFO_HEIGHT (info), GST_VIDEO_INFO_N_PLANES (info), + offset, stride); + } + + return buffer; + +error: + gst_buffer_unref (buffer); + + return NULL; +} + gboolean _gst_d3d11_result (HRESULT hr, GstD3D11Device * device, GstDebugCategory * cat, const gchar * file, const gchar * function, gint line) diff --git a/sys/d3d11/gstd3d11utils.h b/sys/d3d11/gstd3d11utils.h index 5aa37fbe8e..109b208d9f 100644 --- a/sys/d3d11/gstd3d11utils.h +++ b/sys/d3d11/gstd3d11utils.h @@ -54,6 +54,12 @@ gboolean gst_d3d11_is_windows_8_or_greater (void); GstD3D11DeviceVendor gst_d3d11_get_device_vendor (GstD3D11Device * device); +GstBuffer * gst_d3d11_allocate_staging_buffer (GstD3D11Allocator * allocator, + const GstVideoInfo * info, + const GstD3D11Format * format, + const D3D11_TEXTURE2D_DESC desc[GST_VIDEO_MAX_PLANES], + gboolean add_videometa); + gboolean _gst_d3d11_result (HRESULT hr, GstD3D11Device * device, GstDebugCategory * cat,