diff --git a/ext/vulkan/meson.build b/ext/vulkan/meson.build index 933688b3a2..682da6d52d 100644 --- a/ext/vulkan/meson.build +++ b/ext/vulkan/meson.build @@ -23,7 +23,7 @@ vulkan_sources = [ 'vkdownload.c', 'vkdeviceprovider.c', 'vkelementutils.c', - 'vkfullscreenrender.c', + 'vkfullscreenquad.c', 'vkimageidentity.c', 'vksink.c', 'vkshader.c', diff --git a/ext/vulkan/shaders/ayuv_to_rgb.frag b/ext/vulkan/shaders/ayuv_to_rgb.frag index c876bd3f90..41640ee5a0 100644 --- a/ext/vulkan/shaders/ayuv_to_rgb.frag +++ b/ext/vulkan/shaders/ayuv_to_rgb.frag @@ -6,13 +6,13 @@ layout(location = 0) in vec2 inTexCoord; -layout(set = 0, binding = 0) uniform sampler2D inTexture0; -layout(set = 0, binding = 1) uniform reorder { +layout(set = 0, binding = 0) uniform reorder { ivec4 in_reorder_idx; ivec4 out_reorder_idx; ivec2 texSize; ColorMatrices matrices; }; +layout(set = 0, binding = 1) uniform sampler2D inTexture0; layout(location = 0) out vec4 outColor0; diff --git a/ext/vulkan/shaders/identity.frag b/ext/vulkan/shaders/identity.frag index a998ee44d7..97a7b3332d 100644 --- a/ext/vulkan/shaders/identity.frag +++ b/ext/vulkan/shaders/identity.frag @@ -2,7 +2,7 @@ layout(location = 0) in vec2 inTexCoord; -layout(set = 0, binding = 0) uniform sampler2D inTexture; +layout(set = 0, binding = 1) uniform sampler2D inTexture; layout(location = 0) out vec4 outColor; diff --git a/ext/vulkan/shaders/nv12_to_rgb.frag b/ext/vulkan/shaders/nv12_to_rgb.frag index 09e389cd3e..5498e9f256 100644 --- a/ext/vulkan/shaders/nv12_to_rgb.frag +++ b/ext/vulkan/shaders/nv12_to_rgb.frag @@ -6,14 +6,14 @@ layout(location = 0) in vec2 inTexCoord; -layout(set = 0, binding = 0) uniform sampler2D inTexture0; -layout(set = 0, binding = 1) uniform sampler2D inTexture1; -layout(set = 0, binding = 2) uniform reorder { +layout(set = 0, binding = 0) uniform reorder { ivec4 in_reorder_idx; ivec4 out_reorder_idx; ivec2 texSize; ColorMatrices matrices; }; +layout(set = 0, binding = 1) uniform sampler2D inTexture0; +layout(set = 0, binding = 2) uniform sampler2D inTexture1; layout(location = 0) out vec4 outColor0; diff --git a/ext/vulkan/shaders/rgb_to_ayuv.frag b/ext/vulkan/shaders/rgb_to_ayuv.frag index da26040edf..9134c169b1 100644 --- a/ext/vulkan/shaders/rgb_to_ayuv.frag +++ b/ext/vulkan/shaders/rgb_to_ayuv.frag @@ -5,13 +5,13 @@ layout(location = 0) in vec2 inTexCoord; -layout(set = 0, binding = 0) uniform sampler2D inTexture0; -layout(set = 0, binding = 1) uniform reorder { +layout(set = 0, binding = 0) uniform reorder { ivec4 in_reorder_idx; ivec4 out_reorder_idx; ivec2 texSize; ColorMatrices matrices; }; +layout(set = 0, binding = 1) uniform sampler2D inTexture0; layout(location = 0) out vec4 outColor0; diff --git a/ext/vulkan/shaders/rgb_to_nv12.frag b/ext/vulkan/shaders/rgb_to_nv12.frag index 4c574b39ed..d4ae731fcd 100644 --- a/ext/vulkan/shaders/rgb_to_nv12.frag +++ b/ext/vulkan/shaders/rgb_to_nv12.frag @@ -5,13 +5,13 @@ layout(location = 0) in vec2 inTexCoord; -layout(set = 0, binding = 0) uniform sampler2D inTexture0; -layout(set = 0, binding = 1) uniform reorder { +layout(set = 0, binding = 0) uniform reorder { ivec4 in_reorder_idx; ivec4 out_reorder_idx; ivec2 texSize; ColorMatrices matrices; }; +layout(set = 0, binding = 1) uniform sampler2D inTexture0; layout(location = 0) out vec4 outColor0; layout(location = 1) out vec4 outColor1; diff --git a/ext/vulkan/shaders/rgb_to_yuy2.frag b/ext/vulkan/shaders/rgb_to_yuy2.frag index 06e9a4e78a..945926fd01 100644 --- a/ext/vulkan/shaders/rgb_to_yuy2.frag +++ b/ext/vulkan/shaders/rgb_to_yuy2.frag @@ -5,13 +5,13 @@ layout(location = 0) in vec2 inTexCoord; -layout(set = 0, binding = 0) uniform sampler2D inTexture0; -layout(set = 0, binding = 1) uniform reorder { +layout(set = 0, binding = 0) uniform reorder { ivec4 in_reorder_idx; ivec4 out_reorder_idx; ivec2 texSize; ColorMatrices matrices; }; +layout(set = 0, binding = 1) uniform sampler2D inTexture0; layout(location = 0) out vec4 outColor0; diff --git a/ext/vulkan/shaders/swizzle.frag b/ext/vulkan/shaders/swizzle.frag index 06845a0eee..a2ab72a0e6 100644 --- a/ext/vulkan/shaders/swizzle.frag +++ b/ext/vulkan/shaders/swizzle.frag @@ -4,11 +4,11 @@ layout(location = 0) in vec2 inTexCoord; -layout(push_constant) uniform reorder { +layout(set = 0, binding = 0) uniform reorder { ivec4 in_reorder_idx; ivec4 out_reorder_idx; }; -layout(set = 0, binding = 0) uniform sampler2D inTexture0; +layout(set = 0, binding = 1) uniform sampler2D inTexture0; layout(location = 0) out vec4 outColor0; diff --git a/ext/vulkan/shaders/swizzle_and_clobber_alpha.frag b/ext/vulkan/shaders/swizzle_and_clobber_alpha.frag index 77419bd23a..65ffe86dfd 100644 --- a/ext/vulkan/shaders/swizzle_and_clobber_alpha.frag +++ b/ext/vulkan/shaders/swizzle_and_clobber_alpha.frag @@ -4,11 +4,11 @@ layout(location = 0) in vec2 inTexCoord; -layout(push_constant) uniform reorder { +layout(set = 0, binding = 0) uniform reorder { ivec4 in_reorder_idx; ivec4 out_reorder_idx; }; -layout(set = 0, binding = 0) uniform sampler2D inTexture0; +layout(set = 0, binding = 1) uniform sampler2D inTexture0; layout(location = 0) out vec4 outColor0; diff --git a/ext/vulkan/shaders/uyvy_to_rgb.frag b/ext/vulkan/shaders/uyvy_to_rgb.frag deleted file mode 100644 index 0f245dcec9..0000000000 --- a/ext/vulkan/shaders/uyvy_to_rgb.frag +++ /dev/null @@ -1,30 +0,0 @@ -#version 450 core - -#include "yuy2_uyvy_to_rgb.glsl" - -layout(location = 0) in vec2 inTexCoord; - -layout(set = 0, binding = 0) uniform sampler2D inTexture0; -layout(set = 0, binding = 1) uniform YUVCoefficients_ubo { YUVCoefficients coeff; }; - -layout(set = 0, binding = 2) uniform TexelOrdering -{ - vec2 tex_size; - vec2 poffset; - ivec4 in_reorder_idx; - ivec4 out_reorder_idx; -} ordering; - -layout(location = 0) out vec4 outColor0; - -void main() -{ - float dx; - if (mod (inTexCoord.x * ordering.tex_size.x, 2.0) < 1.0) { - dx = -ordering.poffset.x; - } else { - dx = 0.0; - } - - outColor0 = YUY2_UYVY_to_RGB (inTexture0, inTexCoord, dx, coeff, ordering.in_reorder_idx, ordering.out_reorder_idx); -} diff --git a/ext/vulkan/shaders/view_convert.frag b/ext/vulkan/shaders/view_convert.frag index be19968610..640260aaa7 100644 --- a/ext/vulkan/shaders/view_convert.frag +++ b/ext/vulkan/shaders/view_convert.frag @@ -5,9 +5,7 @@ layout(location = 0) in vec2 inTexCoord; -layout(set = 0, binding = 0) uniform sampler2D l_tex; -layout(set = 0, binding = 1) uniform sampler2D r_tex; -layout(set = 0, binding = 2) uniform ViewConvert { +layout(set = 0, binding = 0) uniform ViewConvert { ivec4 in_reorder_idx; ivec4 out_reorder_idx; vec4 tex_offset; @@ -16,6 +14,8 @@ layout(set = 0, binding = 2) uniform ViewConvert { int output_type; mat3 downmix[2]; }; +layout(set = 0, binding = 1) uniform sampler2D l_tex; +layout(set = 0, binding = 2) uniform sampler2D r_tex; layout(location = 0) out vec4 outColor0; diff --git a/ext/vulkan/shaders/yuy2_to_rgb.frag b/ext/vulkan/shaders/yuy2_to_rgb.frag index 67bb2b371c..2e31e387c7 100644 --- a/ext/vulkan/shaders/yuy2_to_rgb.frag +++ b/ext/vulkan/shaders/yuy2_to_rgb.frag @@ -6,13 +6,13 @@ layout(location = 0) in vec2 inTexCoord; -layout(set = 0, binding = 0) uniform sampler2D inTexture0; -layout(set = 0, binding = 1) uniform reorder { +layout(set = 0, binding = 0) uniform reorder { ivec4 in_reorder_idx; ivec4 out_reorder_idx; ivec2 texSize; ColorMatrices matrices; }; +layout(set = 0, binding = 1) uniform sampler2D inTexture0; layout(location = 0) out vec4 outColor0; diff --git a/ext/vulkan/vkcolorconvert.c b/ext/vulkan/vkcolorconvert.c index d503306c9b..ca2b169426 100644 --- a/ext/vulkan/vkcolorconvert.c +++ b/ext/vulkan/vkcolorconvert.c @@ -19,10 +19,10 @@ */ /** - * SECTION:element-vulkanimageidentity - * @title: vulkanimgeidentity + * SECTION:element-vulkancolorconvert + * @title: vulkancolorconvert * - * vulkanimageidentity produces a vulkan image that is a copy of the input image. + * vulkancolorconvert converts between different video colorspaces. */ #ifdef HAVE_CONFIG_H @@ -51,12 +51,6 @@ GST_DEBUG_CATEGORY (gst_debug_vulkan_color_convert); #define N_SHADER_INFO (8*4*4) static shader_info shader_infos[N_SHADER_INFO]; -#define PUSH_CONSTANT_RANGE_NULL_INIT (VkPushConstantRange) { \ - .stageFlags = 0, \ - .offset = 0, \ - .size = 0, \ -} - static void get_rgb_format_swizzle_order (GstVideoFormat format, gint swizzle[GST_VIDEO_MAX_COMPONENTS]) @@ -494,82 +488,6 @@ convert_info_new (GstVideoInfo * in_info, GstVideoInfo * out_info) return conv; } -static GstVulkanDescriptorSet * -_create_descriptor_set (GstVulkanColorConvert * conv) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - GstVulkanDescriptorSet *ret; - GError *error = NULL; - - ret = gst_vulkan_descriptor_cache_acquire (conv->descriptor_pool, &error); - if (!ret) { - GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s", - error->message); - g_clear_error (&error); - return NULL; - } - - return ret; -} - -static void -update_descriptor_set (GstVulkanColorConvert * conv, - VkDescriptorSet descriptor_set, VkImageView * views, guint n_views) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - VkDescriptorBufferInfo buffer_info; - VkDescriptorImageInfo image_info[GST_VIDEO_MAX_PLANES]; - VkWriteDescriptorSet writes[5]; - guint i = 0; - - for (; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { - /* *INDENT-OFF* */ - image_info[i] = (VkDescriptorImageInfo) { - .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - .imageView = views[i], - .sampler = conv->sampler - }; - - g_assert (i < n_views); - g_assert (i < GST_VIDEO_MAX_PLANES); - - writes[i] = (VkWriteDescriptorSet) { - .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, - .pNext = NULL, - .dstSet = descriptor_set, - .dstBinding = i, - .dstArrayElement = 0, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = 1, - .pImageInfo = &image_info[i] - }; - /* *INDENT-ON* */ - } - if (conv->uniform && conv->current_shader->uniform_size) { - /* *INDENT-OFF* */ - buffer_info = (VkDescriptorBufferInfo) { - .buffer = ((GstVulkanBufferMemory *) conv->uniform)->buffer, - .offset = 0, - .range = conv->current_shader->uniform_size - }; - writes[i] = (VkWriteDescriptorSet) { - .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, - .pNext = NULL, - .dstSet = descriptor_set, - .dstBinding = i, - .dstArrayElement = 0, - .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, - .descriptorCount = 1, - .pBufferInfo = &buffer_info - }; - /* *INDENT-ON* */ - i++; - }; - g_assert (i <= G_N_ELEMENTS (writes)); - - vkUpdateDescriptorSets (render->device->device, i, writes, 0, NULL); -} - static void video_format_to_reorder (GstVideoFormat v_format, gint * reorder, gboolean input) @@ -718,33 +636,47 @@ calculate_reorder_indexes (GstVideoFormat in_format, ret_out[3]); } -static gboolean -swizzle_rgb_update_command_state (GstVulkanColorConvert * conv, - VkCommandBuffer cmd, shader_info * sinfo, - GstVulkanImageView ** in_views, GstVulkanImageView ** out_views, - GstVulkanFence * fence) +struct RGBUpdateData { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - GstVulkanDescriptorSet *descriptor_set; - gint reorder[8]; + int in_reorder[4]; + int out_reorder[4]; +}; - calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), - in_views, GST_VIDEO_INFO_FORMAT (&render->out_info), - out_views, reorder, &reorder[4]); +static GstMemory * +swizzle_rgb_create_uniform_memory (GstVulkanColorConvert * conv, + shader_info * sinfo, GstVulkanImageView ** in_views, + GstVulkanImageView ** out_views) +{ + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv); - vkCmdPushConstants (cmd, render->pipeline_layout, - VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof (reorder), - (const void *) reorder); - descriptor_set = _create_descriptor_set (conv); - update_descriptor_set (conv, descriptor_set->set, &in_views[0]->view, 1); - vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, - render->pipeline_layout, 0, 1, &descriptor_set->set, 0, NULL); + if (sinfo->user_data) { + return gst_memory_ref (sinfo->user_data); + } else { + struct RGBUpdateData data = { 0, }; + GstMapInfo map_info; + GstMemory *uniforms; - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - (GstMiniObject *) descriptor_set)); + uniforms = + gst_vulkan_buffer_memory_alloc (vfilter->device, + sizeof (struct RGBUpdateData), + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - return TRUE; + calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&conv->quad->in_info), + in_views, GST_VIDEO_INFO_FORMAT (&conv->quad->out_info), + out_views, data.in_reorder, data.out_reorder); + + if (!gst_memory_map (uniforms, &map_info, GST_MAP_WRITE)) { + gst_memory_unref (uniforms); + return NULL; + } + memcpy (map_info.data, &data, sizeof (data)); + gst_memory_unmap (uniforms, &map_info); + + sinfo->user_data = gst_memory_ref (uniforms); + return uniforms; + } } struct ColorMatrices @@ -764,62 +696,60 @@ struct YUVUpdateData struct ColorMatrices matrices; }; -static gboolean -yuv_to_rgb_update_command_state (GstVulkanColorConvert * conv, - VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageView ** in_views, - GstVulkanImageView ** out_views, GstVulkanFence * fence) +static GstMemory * +yuv_to_rgb_create_uniform_memory (GstVulkanColorConvert * conv, + shader_info * sinfo, GstVulkanImageView ** in_views, + GstVulkanImageView ** out_views) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - VkImageView views[GST_VIDEO_MAX_PLANES]; - GstVulkanDescriptorSet *descriptor_set; - int i; + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv); - if (!GPOINTER_TO_INT (sinfo->user_data)) { + if (sinfo->user_data) { + return gst_memory_ref (sinfo->user_data); + } else { struct YUVUpdateData data; ConvertInfo *conv_info; GstMapInfo map_info; + GstMemory *uniforms; - calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), - in_views, GST_VIDEO_INFO_FORMAT (&render->out_info), + uniforms = + gst_vulkan_buffer_memory_alloc (vfilter->device, + sizeof (struct YUVUpdateData), + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + + calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&conv->quad->in_info), + in_views, GST_VIDEO_INFO_FORMAT (&conv->quad->out_info), out_views, data.in_reorder, data.out_reorder); - conv_info = convert_info_new (&render->in_info, &render->out_info); + conv_info = convert_info_new (&conv->quad->in_info, &conv->quad->out_info); matrix_to_float (&conv_info->to_RGB_matrix, data.matrices.to_RGB); matrix_to_float (&conv_info->convert_matrix, data.matrices.primaries); matrix_to_float (&conv_info->to_YUV_matrix, data.matrices.to_YUV); /* FIXME: keep this around */ g_free (conv_info); - data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&render->in_info); - data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&render->in_info); + data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&conv->quad->in_info); + data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&conv->quad->in_info); - if (!gst_memory_map (conv->uniform, &map_info, GST_MAP_WRITE)) { - return FALSE; + if (!gst_memory_map (uniforms, &map_info, GST_MAP_WRITE)) { + gst_memory_unref (uniforms); + return NULL; } memcpy (map_info.data, &data, sizeof (data)); - gst_memory_unmap (conv->uniform, &map_info); + gst_memory_unmap (uniforms, &map_info); - sinfo->user_data = GINT_TO_POINTER (1); + sinfo->user_data = gst_memory_ref (uniforms); + return uniforms; } - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) - views[i] = in_views[i]->view; - descriptor_set = _create_descriptor_set (conv); - update_descriptor_set (conv, descriptor_set->set, views, - GST_VIDEO_INFO_N_PLANES (&render->in_info)); - vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, - render->pipeline_layout, 0, 1, &descriptor_set->set, 0, NULL); - - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - (GstMiniObject *) descriptor_set)); - - return TRUE; } static void -clear_user_data_flag (shader_info * sinfo) +unref_memory_if_set (shader_info * sinfo) { + if (sinfo->user_data) + gst_memory_unref (sinfo->user_data); sinfo->user_data = NULL; } @@ -833,22 +763,6 @@ static GstFlowReturn gst_vulkan_color_convert_transform (GstBaseTransform * bt, static gboolean gst_vulkan_color_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, GstCaps * out_caps); -static VkAttachmentReference - * gst_vulkan_color_convert_render_pass_attachment_references - (GstVulkanFullScreenRender * render, guint * n_attachments); -static VkAttachmentDescription - * gst_vulkan_color_convert_render_pass_attachment_descriptions - (GstVulkanFullScreenRender * render, guint * n_descriptions); -static VkDescriptorSetLayoutBinding - * gst_vulkan_color_convert_descriptor_set_layout_bindings - (GstVulkanFullScreenRender * render, guint * n_bindings); -static void -gst_vulkan_color_convert_shader_create_info (GstVulkanFullScreenRender * - render); -static VkPushConstantRange - * gst_vulkan_color_convert_push_constant_ranges (GstVulkanFullScreenRender * - render, guint * n_constants); - static GstStaticPadTemplate gst_vulkan_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, @@ -880,7 +794,7 @@ enum #define gst_vulkan_color_convert_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstVulkanColorConvert, gst_vulkan_color_convert, - GST_TYPE_VULKAN_FULL_SCREEN_RENDER, + GST_TYPE_VULKAN_VIDEO_FILTER, GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_color_convert, "vulkancolorconvert", 0, "Vulkan Color Convert")); @@ -933,17 +847,11 @@ fill_shader_info (void) shader_infos[info_i++] = (shader_info) { .from = rgbs[i], .to = rgbs[j], - .cmd_state_update = swizzle_rgb_update_command_state, + .cmd_create_uniform = swizzle_rgb_create_uniform_memory, .frag_code = clobber_alpha ? swizzle_and_clobber_alpha_frag : swizzle_frag, .frag_size = clobber_alpha ? swizzle_and_clobber_alpha_frag_size : swizzle_frag_size, - .push_constant_ranges = { - { - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT, - .offset = 0, - .size = 8 * sizeof(gint32), - }, PUSH_CONSTANT_RANGE_NULL_INIT, - }, - .notify = (GDestroyNotify) clear_user_data_flag, + .uniform_size = sizeof (struct RGBUpdateData), + .notify = (GDestroyNotify) unref_memory_if_set, .user_data = NULL, }; } @@ -954,24 +862,22 @@ fill_shader_info (void) shader_infos[info_i++] = (shader_info) { .from = rgbs[i], .to = yuvs[j].format, - .cmd_state_update = yuv_to_rgb_update_command_state, + .cmd_create_uniform = yuv_to_rgb_create_uniform_memory, .frag_code = yuvs[j].to_frag, .frag_size = yuvs[j].to_frag_size, - .push_constant_ranges = { PUSH_CONSTANT_RANGE_NULL_INIT }, .uniform_size = sizeof(struct YUVUpdateData), - .notify = (GDestroyNotify) clear_user_data_flag, + .notify = (GDestroyNotify) unref_memory_if_set, .user_data = NULL, }; GST_TRACE ("Initializing info for %s -> %s", to_finfo->name, from_finfo->name); shader_infos[info_i++] = (shader_info) { .from = yuvs[j].format, .to = rgbs[i], - .cmd_state_update = yuv_to_rgb_update_command_state, + .cmd_create_uniform = yuv_to_rgb_create_uniform_memory, .frag_code = yuvs[j].from_frag, .frag_size = yuvs[j].from_frag_size, - .push_constant_ranges = { PUSH_CONSTANT_RANGE_NULL_INIT }, .uniform_size = sizeof(struct YUVUpdateData), - .notify = (GDestroyNotify) clear_user_data_flag, + .notify = (GDestroyNotify) unref_memory_if_set, .user_data = NULL, }; } @@ -987,11 +893,9 @@ gst_vulkan_color_convert_class_init (GstVulkanColorConvertClass * klass) { GstElementClass *gstelement_class; GstBaseTransformClass *gstbasetransform_class; - GstVulkanFullScreenRenderClass *fullscreenrender_class; gstelement_class = (GstElementClass *) klass; gstbasetransform_class = (GstBaseTransformClass *) klass; - fullscreenrender_class = (GstVulkanFullScreenRenderClass *) klass; gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader", "Filter/Video/Convert", "A Vulkan Color Convert", @@ -1011,17 +915,6 @@ gst_vulkan_color_convert_class_init (GstVulkanColorConvertClass * klass) gstbasetransform_class->set_caps = gst_vulkan_color_convert_set_caps; gstbasetransform_class->transform = gst_vulkan_color_convert_transform; - fullscreenrender_class->render_pass_attachment_references = - gst_vulkan_color_convert_render_pass_attachment_references; - fullscreenrender_class->render_pass_attachment_descriptions = - gst_vulkan_color_convert_render_pass_attachment_descriptions; - fullscreenrender_class->descriptor_set_layout_bindings = - gst_vulkan_color_convert_descriptor_set_layout_bindings; - fullscreenrender_class->shader_create_info = - gst_vulkan_color_convert_shader_create_info; - fullscreenrender_class->push_constant_ranges = - gst_vulkan_color_convert_push_constant_ranges; - fill_shader_info (); } @@ -1188,9 +1081,9 @@ static GstCaps * gst_vulkan_color_convert_transform_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt); - caps = gst_vulkan_color_convert_transform_format_info (render->device, + caps = gst_vulkan_color_convert_transform_format_info (vfilter->device, direction == GST_PAD_SRC, caps); if (filter) { @@ -1204,300 +1097,16 @@ gst_vulkan_color_convert_transform_caps (GstBaseTransform * bt, return caps; } -static void -destroy_shader_create_info (GstVulkanFullScreenRender * render, gpointer data) -{ - VkPipelineShaderStageCreateInfo *info = data; - int i; - - for (i = 0; i < render->n_shader_stages; i++) { - vkDestroyShaderModule (render->device->device, info[i].module, NULL); - } - - g_free (info); -} - -static void -gst_vulkan_color_convert_shader_create_info (GstVulkanFullScreenRender * render) -{ - GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (render); - VkShaderModule vert_module, frag_module; - - vert_module = - _vk_create_shader (render->device, identity_vert, identity_vert_size, - NULL); - frag_module = - _vk_create_shader (render->device, conv->current_shader->frag_code, - conv->current_shader->frag_size, NULL); - - render->n_shader_stages = 2; - render->shader_create_info = - g_new0 (VkPipelineShaderStageCreateInfo, render->n_shader_stages); - render->destroy_shader_create_info = destroy_shader_create_info; - - /* *INDENT-OFF* */ - render->shader_create_info[0] = (VkPipelineShaderStageCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, - .pNext = NULL, - .stage = VK_SHADER_STAGE_VERTEX_BIT, - .module = vert_module, - .pName = "main" - }; - - render->shader_create_info[1] = (VkPipelineShaderStageCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, - .pNext = NULL, - .stage = VK_SHADER_STAGE_FRAGMENT_BIT, - .module = frag_module, - .pName = "main" - }; - /* *INDENT-ON* */ -} - -static VkPushConstantRange * -gst_vulkan_color_convert_push_constant_ranges (GstVulkanFullScreenRender * - render, guint * n_constants) -{ - GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (render); - VkPushConstantRange *ranges; - int i; - - for (i = 0; i < G_N_ELEMENTS (conv->current_shader->push_constant_ranges); - i++) { - if (conv->current_shader->push_constant_ranges[i].stageFlags == 0) { - break; - } - } - - *n_constants = i; - GST_DEBUG_OBJECT (conv, "%s->%s has %u push constants", - gst_video_format_to_string (conv->current_shader->from), - gst_video_format_to_string (conv->current_shader->to), *n_constants); - if (*n_constants <= 0) - ranges = NULL; - else - ranges = - g_memdup (conv->current_shader->push_constant_ranges, - sizeof (VkPushConstantRange) * *n_constants); - - return ranges; -} - -static VkDescriptorSetLayoutBinding - * gst_vulkan_color_convert_descriptor_set_layout_bindings - (GstVulkanFullScreenRender * render, guint * n_bindings) -{ - GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (render); - VkDescriptorSetLayoutBinding *bindings; - guint i; - - *n_bindings = 0; - *n_bindings += GST_VIDEO_INFO_N_PLANES (&render->in_info); - if (conv->current_shader->uniform_size) - *n_bindings += 1; - bindings = g_new0 (VkDescriptorSetLayoutBinding, *n_bindings); - - GST_DEBUG_OBJECT (conv, "%s->%s has %u descriptor set layout bindings", - gst_video_format_to_string (conv->current_shader->from), - gst_video_format_to_string (conv->current_shader->to), *n_bindings); - - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { - /* *INDENT-OFF* */ - bindings[i] = (VkDescriptorSetLayoutBinding) { - .binding = i, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .pImmutableSamplers = NULL, - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT - }; - /* *INDENT-ON* */ - } - if (conv->current_shader->uniform_size) { - /* *INDENT-OFF* */ - bindings[i] = (VkDescriptorSetLayoutBinding) { - .binding = i, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, - .pImmutableSamplers = NULL, - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT - }; - /* *INDENT-ON* */ - i++; - } - - g_assert (i == *n_bindings); - - return bindings; -} - -static VkAttachmentReference - * gst_vulkan_color_convert_render_pass_attachment_references - (GstVulkanFullScreenRender * render, guint * n_attachments) -{ - VkAttachmentReference *attachments; - int i; - - *n_attachments = GST_VIDEO_INFO_N_PLANES (&render->out_info); - attachments = g_new0 (VkAttachmentReference, *n_attachments); - - for (i = 0; i < *n_attachments; i++) { - /* *INDENT-OFF* */ - attachments[i] = (VkAttachmentReference) { - .attachment = i, - .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL - }; - /* *INDENT-ON* */ - } - - return attachments; -} - -static VkAttachmentDescription - * gst_vulkan_color_convert_render_pass_attachment_descriptions - (GstVulkanFullScreenRender * render, guint * n_descriptions) -{ - VkAttachmentDescription *color_attachments; - int i; - - *n_descriptions = GST_VIDEO_INFO_N_PLANES (&render->out_info); - color_attachments = g_new0 (VkAttachmentDescription, *n_descriptions); - for (i = 0; i < *n_descriptions; i++) { - /* *INDENT-OFF* */ - color_attachments[i] = (VkAttachmentDescription) { - .format = gst_vulkan_format_from_video_info (&render->out_info, i), - .samples = VK_SAMPLE_COUNT_1_BIT, - .loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR, - .storeOp = VK_ATTACHMENT_STORE_OP_STORE, - .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE, - .stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE, - /* FIXME: share this between elements to avoid pipeline barriers */ - .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, - .finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL - }; - /* *INDENT-ON* */ - } - - return color_attachments; -} - -static VkSampler -_create_sampler (GstVulkanColorConvert * conv) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - - /* *INDENT-OFF* */ - VkSamplerCreateInfo samplerInfo = { - .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, - .pNext = NULL, - .flags = 0, - .magFilter = VK_FILTER_LINEAR, - .minFilter = VK_FILTER_LINEAR, - .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .anisotropyEnable = VK_FALSE, - .maxAnisotropy = 1, - .borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK, - .unnormalizedCoordinates = VK_FALSE, - .compareEnable = VK_FALSE, - .compareOp = VK_COMPARE_OP_ALWAYS, - .mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR, - .mipLodBias = 0.0f, - .minLod = 0.0f, - .maxLod = 0.0f - }; - /* *INDENT-ON* */ - GError *error = NULL; - VkSampler sampler; - VkResult err; - - err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) { - GST_ERROR_OBJECT (conv, "Failed to create sampler: %s", error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - return sampler; -} - static gboolean gst_vulkan_color_convert_start (GstBaseTransform * bt) { GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (bt); + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv); if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt)) return FALSE; - if (!(conv->sampler = _create_sampler (conv))) - return FALSE; - - return TRUE; -} - -static GstVulkanDescriptorCache * -_create_descriptor_pool (GstVulkanColorConvert * conv) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - gsize max_sets = 32; /* FIXME: don't hardcode this! */ - - /* *INDENT-OFF* */ - VkDescriptorPoolSize pool_sizes[] = { - { - .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = max_sets * GST_VIDEO_INFO_N_PLANES (&render->in_info), - }, - { - .type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, - .descriptorCount = max_sets - }, - }; - - VkDescriptorPoolCreateInfo pool_info = { - .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, - .pNext = NULL, - .flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, - .poolSizeCount = G_N_ELEMENTS (pool_sizes), - .pPoolSizes = pool_sizes, - .maxSets = max_sets - }; - /* *INDENT-ON* */ - VkDescriptorPool pool; - GstVulkanDescriptorPool *ret; - GstVulkanDescriptorCache *cache; - GError *error = NULL; - VkResult err; - - err = - vkCreateDescriptorPool (render->device->device, &pool_info, NULL, &pool); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateDescriptorPool") < 0) { - GST_ERROR_OBJECT (render, "Failed to create descriptor pool: %s", - error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - ret = gst_vulkan_descriptor_pool_new_wrapped (render->device, pool, max_sets); - cache = - gst_vulkan_descriptor_cache_new (ret, 1, &render->descriptor_set_layout); - gst_object_unref (ret); - - return cache; -} - -static gboolean -_create_uniform_buffer (GstVulkanColorConvert * conv) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - - if (conv->current_shader->uniform_size) { - conv->uniform = - gst_vulkan_buffer_memory_alloc (render->device, - conv->current_shader->uniform_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - } + conv->quad = gst_vulkan_full_screen_quad_new (vfilter->queue); return TRUE; } @@ -1506,15 +1115,17 @@ static gboolean gst_vulkan_color_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, GstCaps * out_caps) { + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt); GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (bt); - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - GstVideoInfo in_info, out_info; - GstVulkanFence *last_fence; + GstVulkanHandle *vert, *frag; int i; - if (!gst_video_info_from_caps (&in_info, in_caps)) + if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps, + out_caps)) return FALSE; - if (!gst_video_info_from_caps (&out_info, out_caps)) + + if (!gst_vulkan_full_screen_quad_set_info (conv->quad, &vfilter->in_info, + &vfilter->out_info)) return FALSE; if (conv->current_shader) { @@ -1523,15 +1134,16 @@ gst_vulkan_color_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, } for (i = 0; i < G_N_ELEMENTS (shader_infos); i++) { - if (shader_infos[i].from != GST_VIDEO_INFO_FORMAT (&in_info)) + if (shader_infos[i].from != GST_VIDEO_INFO_FORMAT (&vfilter->in_info)) continue; - if (shader_infos[i].to != GST_VIDEO_INFO_FORMAT (&out_info)) + if (shader_infos[i].to != GST_VIDEO_INFO_FORMAT (&vfilter->out_info)) continue; GST_INFO_OBJECT (conv, "Found compatible conversion information from %s to %s", - gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&in_info)), - gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&out_info))); + gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&vfilter->in_info)), + gst_video_format_to_string (GST_VIDEO_INFO_FORMAT + (&vfilter->out_info))); conv->current_shader = &shader_infos[i]; } @@ -1541,33 +1153,26 @@ gst_vulkan_color_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, return FALSE; } - if (render->last_fence) - last_fence = gst_vulkan_fence_ref (render->last_fence); - else - last_fence = gst_vulkan_fence_new_always_signalled (render->device); - - if (conv->descriptor_pool) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_object_unref (last_fence, - (GstObject *) conv->descriptor_pool)); - conv->descriptor_pool = NULL; - if (conv->uniform) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (last_fence, - (GstMiniObject *) conv->uniform)); - conv->uniform = NULL; - - gst_vulkan_fence_unref (last_fence); - - if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps, - out_caps)) + if (!(vert = + _vk_create_shader (vfilter->device, identity_vert, identity_vert_size, + NULL))) { return FALSE; - - if (!(conv->descriptor_pool = _create_descriptor_pool (conv))) + } + if (!(frag = + _vk_create_shader (vfilter->device, conv->current_shader->frag_code, + conv->current_shader->frag_size, NULL))) { + gst_vulkan_handle_unref (vert); return FALSE; + } - if (!_create_uniform_buffer (conv)) + if (!gst_vulkan_full_screen_quad_set_shaders (conv->quad, vert, frag)) { + gst_vulkan_handle_unref (vert); + gst_vulkan_handle_unref (frag); return FALSE; + } + + gst_vulkan_handle_unref (vert); + gst_vulkan_handle_unref (frag); return TRUE; } @@ -1576,131 +1181,146 @@ static gboolean gst_vulkan_color_convert_stop (GstBaseTransform * bt) { GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (bt); - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - if (render->device) { - GstVulkanFence *last_fence; - - if (render->last_fence) - last_fence = gst_vulkan_fence_ref (render->last_fence); - else - last_fence = gst_vulkan_fence_new_always_signalled (render->device); - - if (conv->descriptor_pool) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_object_unref (last_fence, - (GstObject *) conv->descriptor_pool)); - conv->descriptor_pool = NULL; - if (conv->sampler) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_sampler (last_fence, conv->sampler)); - conv->sampler = VK_NULL_HANDLE; - if (conv->uniform) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (last_fence, - (GstMiniObject *) conv->uniform)); - conv->uniform = NULL; - - gst_vulkan_fence_unref (last_fence); + if (conv->current_shader) { + conv->current_shader->notify (conv->current_shader); + conv->current_shader = NULL; } - if (conv->cmd_pool) - gst_object_unref (conv->cmd_pool); - conv->cmd_pool = NULL; - - conv->current_shader = NULL; + gst_clear_object (&conv->quad); return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt); } -static VkFramebuffer -_create_framebuffer (GstVulkanColorConvert * conv, guint n_views, - VkImageView * views) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - - /* *INDENT-OFF* */ - VkFramebufferCreateInfo framebuffer_info = { - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = NULL, - .renderPass = render->render_pass, - .attachmentCount = n_views, - .pAttachments = views, - .width = GST_VIDEO_INFO_WIDTH (&render->out_info), - .height = GST_VIDEO_INFO_HEIGHT (&render->out_info), - .layers = 1 - }; - /* *INDENT-ON* */ - VkFramebuffer framebuffer; - GError *error = NULL; - VkResult err; - - err = - vkCreateFramebuffer (render->device->device, &framebuffer_info, NULL, - &framebuffer); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateFramebuffer") < 0) { - GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s", - error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - return framebuffer; -} - static GstFlowReturn gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer * outbuf) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (bt); - GstVulkanImageMemory *in_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt); GstVulkanImageView *in_img_views[GST_VIDEO_MAX_PLANES] = { NULL, }; GstVulkanImageMemory *render_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; GstVulkanImageView *render_img_views[GST_VIDEO_MAX_PLANES] = { NULL, }; GstVulkanImageMemory *out_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstBuffer *render_buf = NULL; GstVulkanFence *fence = NULL; - VkFramebuffer framebuffer; GstVulkanCommandBuffer *cmd_buf; GError *error = NULL; VkResult err; int i; - fence = gst_vulkan_fence_new (render->device, 0, &error); + fence = gst_vulkan_fence_new (vfilter->device, 0, &error); if (!fence) goto error; - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { - GstMemory *mem = gst_buffer_peek_memory (inbuf, i); - if (!gst_is_vulkan_image_memory (mem)) { + if (!gst_vulkan_full_screen_quad_set_input_buffer (conv->quad, inbuf, &error)) + goto error; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->in_info); i++) { + GstMemory *img_mem = gst_buffer_peek_memory (inbuf, i); + if (!gst_is_vulkan_image_memory (img_mem)) { g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, "Input memory must be a GstVulkanImageMemory"); goto error; } - in_img_mems[i] = (GstVulkanImageMemory *) mem; - in_img_views[i] = get_or_create_image_view (in_img_mems[i]); - gst_vulkan_trash_list_add (render->trash_list, + in_img_views[i] = + get_or_create_image_view ((GstVulkanImageMemory *) img_mem); + gst_vulkan_trash_list_add (conv->quad->trash_list, gst_vulkan_trash_new_mini_object_unref (fence, (GstMiniObject *) in_img_views[i])); } - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { - GstMemory *mem = gst_buffer_peek_memory (outbuf, i); - if (!gst_is_vulkan_image_memory (mem)) { - g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, - "Output memory must be a GstVulkanImageMemory"); + { + gboolean need_render_buf = FALSE; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->out_info); i++) { + GstMemory *mem = gst_buffer_peek_memory (outbuf, i); + if (!gst_is_vulkan_image_memory (mem)) { + g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Output memory must be a GstVulkanImageMemory"); + goto error; + } + out_img_mems[i] = (GstVulkanImageMemory *) mem; + + if (GST_VIDEO_INFO_WIDTH (&conv->quad->out_info) == + GST_VIDEO_INFO_COMP_WIDTH (&conv->quad->out_info, i) + && GST_VIDEO_INFO_HEIGHT (&conv->quad->out_info) == + GST_VIDEO_INFO_COMP_HEIGHT (&conv->quad->out_info, i)) { + render_img_mems[i] = out_img_mems[i]; + GST_LOG_OBJECT (conv, "using original output memory %p for plane %u", + out_img_mems[i], i); + } else { + /* we need a scratch buffer because framebuffers can only output to + * attachments of at least the same size which means no sub-sampled + * rendering */ + VkImageTiling tiling = VK_IMAGE_TILING_OPTIMAL; + VkFormat vk_format; + GstMemory *mem; + + vk_format = + gst_vulkan_format_from_video_info (&conv->quad->out_info, i); + + mem = gst_vulkan_image_memory_alloc (vfilter->device, + vk_format, GST_VIDEO_INFO_WIDTH (&conv->quad->out_info), + GST_VIDEO_INFO_HEIGHT (&conv->quad->out_info), tiling, + VK_IMAGE_USAGE_TRANSFER_SRC_BIT | + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + render_img_mems[i] = (GstVulkanImageMemory *) mem; + need_render_buf = TRUE; + GST_LOG_OBJECT (conv, "using replacement output memory %p for plane %u", + mem, i); + } + } + + if (need_render_buf) { + render_buf = gst_buffer_new (); + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->out_info); i++) { + gst_buffer_append_memory (render_buf, + gst_memory_ref ((GstMemory *) render_img_mems[i])); + } + gst_vulkan_trash_list_add (conv->quad->trash_list, + gst_vulkan_trash_new_mini_object_unref (fence, + (GstMiniObject *) render_buf)); + } else { + render_buf = outbuf; + } + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->out_info); i++) { + GstMemory *img_mem = gst_buffer_peek_memory (render_buf, i); + if (!gst_is_vulkan_image_memory (img_mem)) { + g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Input memory must be a GstVulkanImageMemory"); + goto error; + } + render_img_views[i] = + get_or_create_image_view ((GstVulkanImageMemory *) img_mem); + gst_vulkan_trash_list_add (conv->quad->trash_list, + gst_vulkan_trash_new_mini_object_unref (fence, + (GstMiniObject *) render_img_views[i])); + } + } + + if (!gst_vulkan_full_screen_quad_set_output_buffer (conv->quad, render_buf, + &error)) + goto error; + + { + GstMemory *uniforms = conv->current_shader->cmd_create_uniform (conv, + conv->current_shader, in_img_views, render_img_views); + if (!gst_vulkan_full_screen_quad_set_uniform_buffer (conv->quad, uniforms, + &error)) { + gst_memory_unref (uniforms); goto error; } - out_img_mems[i] = (GstVulkanImageMemory *) mem; + gst_memory_unref (uniforms); } - if (!conv->cmd_pool) { - if (!(conv->cmd_pool = - gst_vulkan_queue_create_command_pool (render->queue, &error))) - goto error; - } + if (!gst_vulkan_full_screen_quad_prepare_draw (conv->quad, fence, &error)) + goto error; - if (!(cmd_buf = gst_vulkan_command_pool_create (conv->cmd_pool, &error))) + if (!(cmd_buf = + gst_vulkan_command_pool_create (conv->quad->cmd_pool, &error))) goto error; { @@ -1721,116 +1341,11 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, goto error; } - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { - /* *INDENT-OFF* */ - VkImageMemoryBarrier in_image_memory_barrier = { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = NULL, - .srcAccessMask = in_img_mems[i]->barrier.parent.access_flags, - .dstAccessMask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT, - .oldLayout = in_img_mems[i]->barrier.image_layout, - .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - /* FIXME: implement exclusive transfers */ - .srcQueueFamilyIndex = 0, - .dstQueueFamilyIndex = 0, - .image = in_img_mems[i]->image, - .subresourceRange = in_img_mems[i]->barrier.subresource_range - }; - /* *INDENT-ON* */ - - vkCmdPipelineBarrier (cmd_buf->cmd, - in_img_mems[i]->barrier.parent.pipeline_stages, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, NULL, 0, NULL, 1, - &in_image_memory_barrier); - - in_img_mems[i]->barrier.parent.pipeline_stages = - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; - in_img_mems[i]->barrier.parent.access_flags = - in_image_memory_barrier.dstAccessMask; - in_img_mems[i]->barrier.image_layout = in_image_memory_barrier.newLayout; - } - - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { - VkImageMemoryBarrier render_image_memory_barrier; - - if (GST_VIDEO_INFO_WIDTH (&render->out_info) == - GST_VIDEO_INFO_COMP_WIDTH (&render->out_info, i)) { - render_img_mems[i] = out_img_mems[i]; - } else { - /* we need a scratch buffer because framebuffers can only output to - * attachments of at least the same size which means no sub-sampled - * rendering */ - VkImageTiling tiling = VK_IMAGE_TILING_OPTIMAL; - VkFormat vk_format; - GstMemory *mem; - - vk_format = gst_vulkan_format_from_video_info (&render->out_info, i); - - mem = gst_vulkan_image_memory_alloc (render->device, - vk_format, GST_VIDEO_INFO_WIDTH (&render->out_info), - GST_VIDEO_INFO_HEIGHT (&render->out_info), tiling, - VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, - VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - render_img_mems[i] = (GstVulkanImageMemory *) mem; - } - - /* *INDENT-OFF* */ - render_image_memory_barrier = (VkImageMemoryBarrier) { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = NULL, - .srcAccessMask = render_img_mems[i]->barrier.parent.access_flags, - .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, - .oldLayout = render_img_mems[i]->barrier.image_layout, - .newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, - /* FIXME: implement exclusive transfers */ - .srcQueueFamilyIndex = 0, - .dstQueueFamilyIndex = 0, - .image = render_img_mems[i]->image, - .subresourceRange = render_img_mems[i]->barrier.subresource_range - }; - /* *INDENT-ON* */ - - vkCmdPipelineBarrier (cmd_buf->cmd, - render_img_mems[i]->barrier.parent.pipeline_stages, - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, NULL, 0, NULL, 1, - &render_image_memory_barrier); - - render_img_mems[i]->barrier.parent.pipeline_stages = - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; - render_img_mems[i]->barrier.parent.access_flags = - render_image_memory_barrier.dstAccessMask; - render_img_mems[i]->barrier.image_layout = - render_image_memory_barrier.newLayout; - } - - { - VkImageView attachments[4] = { 0, }; - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { - render_img_views[i] = get_or_create_image_view (render_img_mems[i]); - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - (GstMiniObject *) render_img_views[i])); - attachments[i] = render_img_views[i]->view; - } - - if (!(framebuffer = _create_framebuffer (conv, - GST_VIDEO_INFO_N_PLANES (&render->out_info), attachments))) { - g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, - "Failed to create framebuffer"); - goto error; - } - } - - conv->current_shader->cmd_state_update (conv, cmd_buf->cmd, - conv->current_shader, in_img_views, render_img_views, fence); - if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd_buf->cmd, - framebuffer)) { - g_set_error (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, - "Failed to fill framebuffer"); + if (!gst_vulkan_full_screen_quad_fill_command_buffer (conv->quad, cmd_buf, + fence, &error)) goto unlock_error; - } - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->out_info); i++) { if (render_img_mems[i] != out_img_mems[i]) { VkImageMemoryBarrier out_image_memory_barrier; VkImageMemoryBarrier render_image_memory_barrier; @@ -1871,8 +1386,12 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, .layerCount = 1, }, .srcOffsets = { - {0, 0, 0}, - {GST_VIDEO_INFO_COMP_WIDTH (&render->out_info, i), GST_VIDEO_INFO_COMP_HEIGHT (&render->out_info, i), 1}, + { 0, 0, 0 }, + { + GST_VIDEO_INFO_COMP_WIDTH (&conv->quad->out_info, i), + GST_VIDEO_INFO_COMP_HEIGHT (&conv->quad->out_info, i), + 1 + }, }, .dstSubresource = { .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, @@ -1881,12 +1400,19 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, .layerCount = 1, }, .dstOffsets = { - {0, 0, 0}, - {GST_VIDEO_INFO_COMP_WIDTH (&render->out_info, i), GST_VIDEO_INFO_COMP_HEIGHT (&render->out_info, i), 1}, + { 0, 0, 0 }, + { + GST_VIDEO_INFO_COMP_WIDTH (&conv->quad->out_info, i), + GST_VIDEO_INFO_COMP_HEIGHT (&conv->quad->out_info, i), + 1 + }, }, }; /* *INDENT-ON* */ + GST_LOG_OBJECT (conv, "blitting plane %u from %p to %p", i, + render_img_mems[i], out_img_mems[i]); + vkCmdPipelineBarrier (cmd_buf->cmd, render_img_mems[i]->barrier.parent.pipeline_stages, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 0, NULL, 1, @@ -1918,7 +1444,7 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, out_img_mems[i]->barrier.image_layout, 1, &blit, VK_FILTER_LINEAR); /* XXX: try to reuse this image later */ - gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_list_add (conv->quad->trash_list, gst_vulkan_trash_new_mini_object_unref (fence, (GstMiniObject *) render_img_mems[i])); } @@ -1929,14 +1455,10 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0) goto error; - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_framebuffer (fence, framebuffer)); - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - GST_MINI_OBJECT_CAST (cmd_buf))); + if (!gst_vulkan_full_screen_quad_submit (conv->quad, cmd_buf, fence, &error)) + goto error; - if (!gst_vulkan_full_screen_render_submit (render, cmd_buf->cmd, fence)) - return GST_FLOW_ERROR; + gst_vulkan_fence_unref (fence); return GST_FLOW_OK; @@ -1946,6 +1468,8 @@ unlock_error: gst_vulkan_command_buffer_unref (cmd_buf); } error: + gst_clear_mini_object ((GstMiniObject **) & fence); + GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL)); g_clear_error (&error); return GST_FLOW_ERROR; diff --git a/ext/vulkan/vkcolorconvert.h b/ext/vulkan/vkcolorconvert.h index 227dc39740..f27734b5f8 100644 --- a/ext/vulkan/vkcolorconvert.h +++ b/ext/vulkan/vkcolorconvert.h @@ -24,7 +24,7 @@ #include #include #include -#include "vkfullscreenrender.h" +#include "vkfullscreenquad.h" G_BEGIN_DECLS @@ -41,16 +41,15 @@ typedef struct _GstVulkanColorConvertClass GstVulkanColorConvertClass; typedef struct _shader_info shader_info; -typedef gboolean (*CommandStateUpdate) (GstVulkanColorConvert * conv, VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageView ** src_views, GstVulkanImageView ** dest_views, GstVulkanFence * fence); +typedef GstMemory * (*CommandCreateUniformMemory) (GstVulkanColorConvert * conv, shader_info * sinfo, GstVulkanImageView ** src_views, GstVulkanImageView ** dest_views); struct _shader_info { GstVideoFormat from; GstVideoFormat to; - CommandStateUpdate cmd_state_update; + CommandCreateUniformMemory cmd_create_uniform; gchar *frag_code; gsize frag_size; - VkPushConstantRange push_constant_ranges[MAX_PUSH_CONSTANTS]; gsize uniform_size; GDestroyNotify notify; gpointer user_data; @@ -58,26 +57,16 @@ struct _shader_info struct _GstVulkanColorConvert { - GstVulkanFullScreenRender parent; + GstVulkanVideoFilter parent; - GstVulkanCommandPool *cmd_pool; - - VkSampler sampler; - GstVulkanDescriptorCache *descriptor_pool; - - VkShaderModule vert_module; - VkShaderModule frag_module; - - VkDescriptorSetLayoutBinding sampler_layout_binding; - VkDescriptorSetLayoutCreateInfo layout_info; + GstVulkanFullScreenQuad *quad; shader_info *current_shader; - GstMemory *uniform; }; struct _GstVulkanColorConvertClass { - GstVulkanFullScreenRenderClass parent_class; + GstVulkanVideoFilterClass parent_class; }; GType gst_vulkan_color_convert_get_type(void); diff --git a/ext/vulkan/vkfullscreenquad.c b/ext/vulkan/vkfullscreenquad.c new file mode 100644 index 0000000000..00f11c14ec --- /dev/null +++ b/ext/vulkan/vkfullscreenquad.c @@ -0,0 +1,1325 @@ +/* + * GStreamer Plugins Vulkan + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "vkfullscreenquad.h" +#include "vkelementutils.h" + +#define GST_CAT_DEFAULT gst_vulkan_full_screen_quad_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +typedef struct _GstVulkanFullScreenQuad GstVulkanFullScreenQuad; + +struct _GstVulkanFullScreenQuadPrivate +{ + GstBuffer *inbuf; + GstBuffer *outbuf; + + GstMemory *vertices; + GstMemory *indices; + gsize n_indices; + GstMemory *uniforms; + gsize uniform_size; + + GstMemory *push_constants; + gsize push_constants_size; + + GstVulkanHandle *vert; + GstVulkanHandle *frag; +}; + +G_DEFINE_TYPE_WITH_CODE (GstVulkanFullScreenQuad, gst_vulkan_full_screen_quad, + GST_TYPE_OBJECT, GST_DEBUG_CATEGORY_INIT (gst_vulkan_full_screen_quad_debug, + "vulkanfullscreenquad", 0, "vulkan fullscreen quad render"); + G_ADD_PRIVATE (GstVulkanFullScreenQuad)); + +#define GET_PRIV(self) gst_vulkan_full_screen_quad_get_instance_private (self) + +struct Vertex vertices[] = { + {-1.0f, -1.0f, 0.0f, 0.0f, 0.0f}, + {1.0f, -1.0f, 0.0f, 1.0f, 0.0f}, + {1.0f, 1.0f, 0.0f, 1.0f, 1.0f}, + {-1.0f, 1.0f, 0.0f, 0.0f, 1.0f}, +}; + +gushort indices[] = { + 0, 1, 2, 0, 2, 3, +}; + +static gboolean +create_sampler (GstVulkanFullScreenQuad * self, GError ** error) +{ + /* *INDENT-OFF* */ + VkSamplerCreateInfo samplerInfo = { + .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, + .magFilter = VK_FILTER_LINEAR, + .minFilter = VK_FILTER_LINEAR, + .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, + .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, + .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, + .anisotropyEnable = VK_FALSE, + .maxAnisotropy = 1, + .borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK, + .unnormalizedCoordinates = VK_FALSE, + .compareEnable = VK_FALSE, + .compareOp = VK_COMPARE_OP_ALWAYS, + .mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR, + .mipLodBias = 0.0f, + .minLod = 0.0f, + .maxLod = 0.0f + }; + /* *INDENT-ON* */ + VkSampler sampler; + VkResult err; + + err = + vkCreateSampler (self->queue->device->device, &samplerInfo, NULL, + &sampler); + if (gst_vulkan_error_to_g_error (err, error, "vkCreateSampler") < 0) { + return FALSE; + } + + self->sampler = gst_vulkan_handle_new_wrapped (self->queue->device, + GST_VULKAN_HANDLE_TYPE_SAMPLER, (GstVulkanHandleTypedef) sampler, + gst_vulkan_handle_free_sampler, NULL); + + return TRUE; +} + +static GstVulkanDescriptorSet * +get_and_update_descriptor_set (GstVulkanFullScreenQuad * self, + GstVulkanImageView ** views, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv = GET_PRIV (self); + GstVulkanDescriptorSet *set; + + if (!self->sampler) + if (!create_sampler (self, error)) + return NULL; + + if (!(set = + gst_vulkan_descriptor_cache_acquire (self->descriptor_cache, error))) + return NULL; + + { + VkWriteDescriptorSet writes[GST_VIDEO_MAX_PLANES + 1]; + VkDescriptorImageInfo image_info[GST_VIDEO_MAX_PLANES]; + VkDescriptorBufferInfo buffer_info; + int write_n = 0; + int i; + + /* *INDENT-OFF* */ + if (priv->uniforms) { + buffer_info = (VkDescriptorBufferInfo) { + .buffer = ((GstVulkanBufferMemory *) priv->uniforms)->buffer, + .offset = 0, + .range = priv->uniform_size + }; + + writes[write_n++] = (VkWriteDescriptorSet) { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = NULL, + .dstSet = set->set, + .dstBinding = 0, + .dstArrayElement = 0, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .descriptorCount = 1, + .pBufferInfo = &buffer_info + }; + } + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->in_info); i++) { + image_info[i] = (VkDescriptorImageInfo) { + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + .imageView = views[i]->view, + .sampler = (VkSampler) self->sampler->handle + }; + + writes[write_n++] = (VkWriteDescriptorSet) { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = NULL, + .dstSet = set->set, + .dstBinding = i + 1, + .dstArrayElement = 0, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .descriptorCount = 1, + .pImageInfo = &image_info[i] + }; + } + /* *INDENT-ON* */ + vkUpdateDescriptorSets (self->queue->device->device, write_n, writes, 0, + NULL); + } + + return set; +} + +static gboolean +create_descriptor_set_layout (GstVulkanFullScreenQuad * self, GError ** error) +{ + VkDescriptorSetLayoutBinding bindings[GST_VIDEO_MAX_PLANES + 1] = { {0,} }; + VkDescriptorSetLayoutCreateInfo layout_info; + VkDescriptorSetLayout descriptor_set_layout; + int descriptor_n = 0; + VkResult err; + int i; + + /* *INDENT-OFF* */ + bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) { + .binding = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT + }; + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->in_info); i++) { + bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) { + .binding = i+1, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT + }; + }; + + layout_info = (VkDescriptorSetLayoutCreateInfo) { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .pNext = NULL, + .bindingCount = descriptor_n, + .pBindings = bindings + }; + /* *INDENT-ON* */ + + err = + vkCreateDescriptorSetLayout (self->queue->device->device, &layout_info, + NULL, &descriptor_set_layout); + if (gst_vulkan_error_to_g_error (err, error, + "vkCreateDescriptorSetLayout") < 0) { + return FALSE; + } + + self->descriptor_set_layout = + gst_vulkan_handle_new_wrapped (self->queue->device, + GST_VULKAN_HANDLE_TYPE_DESCRIPTOR_SET_LAYOUT, + (GstVulkanHandleTypedef) descriptor_set_layout, + gst_vulkan_handle_free_descriptor_set_layout, NULL); + + return TRUE; +} + +static gboolean +create_pipeline_layout (GstVulkanFullScreenQuad * self, GError ** error) +{ + VkPipelineLayoutCreateInfo pipeline_layout_info; + VkPipelineLayout pipeline_layout; + VkResult err; + + if (!self->descriptor_set_layout) + if (!create_descriptor_set_layout (self, error)) + return FALSE; + + /* *INDENT-OFF* */ + pipeline_layout_info = (VkPipelineLayoutCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, + .pNext = NULL, + .setLayoutCount = 1, + .pSetLayouts = (VkDescriptorSetLayout *) &self->descriptor_set_layout->handle, + .pushConstantRangeCount = 0, + .pPushConstantRanges = NULL, + }; + /* *INDENT-ON* */ + + err = + vkCreatePipelineLayout (self->queue->device->device, + &pipeline_layout_info, NULL, &pipeline_layout); + if (gst_vulkan_error_to_g_error (err, error, "vkCreatePipelineLayout") < 0) { + return FALSE; + } + + self->pipeline_layout = gst_vulkan_handle_new_wrapped (self->queue->device, + GST_VULKAN_HANDLE_TYPE_PIPELINE_LAYOUT, + (GstVulkanHandleTypedef) pipeline_layout, + gst_vulkan_handle_free_pipeline_layout, NULL); + + return TRUE; +} + +static gboolean +create_render_pass (GstVulkanFullScreenQuad * self, GError ** error) +{ + VkAttachmentDescription color_attachments[GST_VIDEO_MAX_PLANES]; + VkAttachmentReference color_attachment_refs[GST_VIDEO_MAX_PLANES]; + VkRenderPassCreateInfo render_pass_info; + VkSubpassDescription subpass; + VkRenderPass render_pass; + VkResult err; + int i; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->out_info); i++) { + /* *INDENT-OFF* */ + color_attachments[i] = (VkAttachmentDescription) { + .format = gst_vulkan_format_from_video_info (&self->out_info, i), + .samples = VK_SAMPLE_COUNT_1_BIT, + .loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE, + .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE, + .stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE, + /* FIXME: share this between elements to avoid pipeline barriers */ + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL + }; + + color_attachment_refs[i] = (VkAttachmentReference) { + .attachment = i, + .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL + }; + /* *INDENT-ON* */ + } + + /* *INDENT-OFF* */ + subpass = (VkSubpassDescription) { + .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, + .colorAttachmentCount = GST_VIDEO_INFO_N_PLANES (&self->out_info), + .pColorAttachments = color_attachment_refs + }; + + render_pass_info = (VkRenderPassCreateInfo) { + .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, + .pNext = NULL, + .attachmentCount = GST_VIDEO_INFO_N_PLANES (&self->out_info), + .pAttachments = color_attachments, + .subpassCount = 1, + .pSubpasses = &subpass + }; + /* *INDENT-ON* */ + + err = + vkCreateRenderPass (self->queue->device->device, &render_pass_info, NULL, + &render_pass); + if (gst_vulkan_error_to_g_error (err, error, "vkCreateRenderPass") < 0) { + return FALSE; + } + + self->render_pass = gst_vulkan_handle_new_wrapped (self->queue->device, + GST_VULKAN_HANDLE_TYPE_RENDER_PASS, + (GstVulkanHandleTypedef) render_pass, + gst_vulkan_handle_free_render_pass, NULL); + + return TRUE; +} + +static gboolean +create_pipeline (GstVulkanFullScreenQuad * self, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv = GET_PRIV (self); + VkVertexInputBindingDescription vertex_binding; + VkVertexInputAttributeDescription attribute_descriptions[2]; + VkPipelineShaderStageCreateInfo shader_create_info[2]; + VkPipelineVertexInputStateCreateInfo vertex_input_info; + VkPipelineInputAssemblyStateCreateInfo input_assembly; + VkPipelineViewportStateCreateInfo viewport_state; + VkPipelineRasterizationStateCreateInfo rasterizer; + VkPipelineMultisampleStateCreateInfo multisampling; + VkPipelineColorBlendAttachmentState + color_blend_attachments[GST_VIDEO_MAX_PLANES]; + VkPipelineColorBlendStateCreateInfo color_blending; + VkGraphicsPipelineCreateInfo pipeline_create_info; + VkPipeline pipeline; + VkResult err; + + if (!priv->vert || !priv->frag) { + g_set_error_literal (error, GST_VULKAN_ERROR, + VK_ERROR_INITIALIZATION_FAILED, "Missing shader information"); + return FALSE; + } + + if (!self->pipeline_layout) + if (!create_pipeline_layout (self, error)) + return FALSE; + + if (!self->render_pass) + if (!create_render_pass (self, error)) + return FALSE; + + /* *INDENT-OFF* */ + shader_create_info[0] = (VkPipelineShaderStageCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = NULL, + .stage = VK_SHADER_STAGE_VERTEX_BIT, + .module = (VkShaderModule) priv->vert->handle, + .pName = "main" + }; + + shader_create_info[1] = (VkPipelineShaderStageCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = NULL, + .stage = VK_SHADER_STAGE_FRAGMENT_BIT, + .module = (VkShaderModule) priv->frag->handle, + .pName = "main" + }; + + /* *INDENT-OFF* */ + vertex_binding = (VkVertexInputBindingDescription) { + .binding = 0, + .stride = sizeof (struct Vertex), + .inputRate = VK_VERTEX_INPUT_RATE_VERTEX + }; + + attribute_descriptions[0] = (VkVertexInputAttributeDescription) { + .binding = 0, + .location = 0, + .format = VK_FORMAT_R32G32B32_SFLOAT, + .offset = G_STRUCT_OFFSET (struct Vertex, x) + }; + attribute_descriptions[1] = (VkVertexInputAttributeDescription) { + .binding = 0, + .location = 1, + .format = VK_FORMAT_R32G32_SFLOAT, + .offset = G_STRUCT_OFFSET (struct Vertex, s) + }; + + vertex_input_info = (VkPipelineVertexInputStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, + .pNext = NULL, + .vertexBindingDescriptionCount = 1, + .pVertexBindingDescriptions = &vertex_binding, + .vertexAttributeDescriptionCount = 2, + .pVertexAttributeDescriptions = attribute_descriptions, + }; + + input_assembly = (VkPipelineInputAssemblyStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, + .pNext = NULL, + .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, + .primitiveRestartEnable = VK_FALSE + }; + + viewport_state = (VkPipelineViewportStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, + .pNext = NULL, + .viewportCount = 1, + .pViewports = &(VkViewport) { + .x = 0.0f, + .y = 0.0f, + .width = (float) GST_VIDEO_INFO_WIDTH (&self->out_info), + .height = (float) GST_VIDEO_INFO_HEIGHT (&self->out_info), + .minDepth = 0.0f, + .maxDepth = 1.0f + }, + .scissorCount = 1, + .pScissors = &(VkRect2D) { + .offset = { 0, 0 }, + .extent = { + GST_VIDEO_INFO_WIDTH (&self->out_info), + GST_VIDEO_INFO_HEIGHT (&self->out_info) + } + } + }; + + rasterizer = (VkPipelineRasterizationStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, + .pNext = NULL, + .depthClampEnable = VK_FALSE, + .rasterizerDiscardEnable = VK_FALSE, + .polygonMode = VK_POLYGON_MODE_FILL, + .lineWidth = 1.0f, + .cullMode = VK_CULL_MODE_NONE, + .frontFace = VK_FRONT_FACE_CLOCKWISE, + .depthBiasEnable = VK_FALSE + }; + + multisampling = (VkPipelineMultisampleStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, + .pNext = NULL, + .sampleShadingEnable = VK_FALSE, + .rasterizationSamples = VK_SAMPLE_COUNT_1_BIT + }; + + color_blend_attachments[0] = (VkPipelineColorBlendAttachmentState) { + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + .blendEnable = VK_FALSE + }; + color_blend_attachments[1] = (VkPipelineColorBlendAttachmentState) { + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + .blendEnable = VK_FALSE + }; + color_blend_attachments[2] = (VkPipelineColorBlendAttachmentState) { + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + .blendEnable = VK_FALSE + }; + color_blend_attachments[3] = (VkPipelineColorBlendAttachmentState) { + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + .blendEnable = VK_FALSE + }; + + color_blending = (VkPipelineColorBlendStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = NULL, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_COPY, + .attachmentCount = GST_VIDEO_INFO_N_PLANES (&self->out_info), + .pAttachments = color_blend_attachments, + .blendConstants = { 0.0f, 0.0f, 0.0f, 0.0f } + }; + + pipeline_create_info = (VkGraphicsPipelineCreateInfo) { + .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, + .pNext = NULL, + .stageCount = 2, + .pStages = shader_create_info, + .pVertexInputState = &vertex_input_info, + .pInputAssemblyState = &input_assembly, + .pViewportState = &viewport_state, + .pRasterizationState = &rasterizer, + .pMultisampleState = &multisampling, + .pColorBlendState = &color_blending, + .layout = (VkPipelineLayout) self->pipeline_layout->handle, + .renderPass = (VkRenderPass) self->render_pass->handle, + .subpass = 0, + .basePipelineHandle = VK_NULL_HANDLE + }; + /* *INDENT-ON* */ + + err = + vkCreateGraphicsPipelines (self->queue->device->device, VK_NULL_HANDLE, 1, + &pipeline_create_info, NULL, &pipeline); + if (gst_vulkan_error_to_g_error (err, error, "vkCreateGraphicsPipelines") < 0) { + return FALSE; + } + + self->graphics_pipeline = gst_vulkan_handle_new_wrapped (self->queue->device, + GST_VULKAN_HANDLE_TYPE_PIPELINE, (GstVulkanHandleTypedef) pipeline, + gst_vulkan_handle_free_pipeline, NULL); + + return TRUE; +} + +static gboolean +create_descriptor_pool (GstVulkanFullScreenQuad * self, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv = GET_PRIV (self); + VkDescriptorPoolCreateInfo pool_info; + gsize max_sets = 32; /* FIXME: don't hardcode this! */ + guint n_pools = 1; + VkDescriptorPoolSize pool_sizes[2]; + VkDescriptorPool pool; + GstVulkanDescriptorPool *ret; + VkResult err; + + /* *INDENT-OFF* */ + pool_sizes[0] = (VkDescriptorPoolSize) { + .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .descriptorCount = max_sets * GST_VIDEO_INFO_N_PLANES (&self->in_info), + }; + + if (priv->uniforms) { + pool_sizes[1] = (VkDescriptorPoolSize) { + .type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .descriptorCount = max_sets + }; + n_pools++; + } + + pool_info = (VkDescriptorPoolCreateInfo) { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, + .pNext = NULL, + .flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, + .poolSizeCount = n_pools, + .pPoolSizes = pool_sizes, + .maxSets = max_sets + }; + /* *INDENT-ON* */ + + err = + vkCreateDescriptorPool (self->queue->device->device, &pool_info, NULL, + &pool); + if (gst_vulkan_error_to_g_error (err, error, "vkCreateDescriptorPool") < 0) { + return FALSE; + } + + ret = + gst_vulkan_descriptor_pool_new_wrapped (self->queue->device, pool, + max_sets); + self->descriptor_cache = + gst_vulkan_descriptor_cache_new (ret, 1, &self->descriptor_set_layout); + gst_object_unref (ret); + + return TRUE; +} + +static gboolean +create_framebuffer (GstVulkanFullScreenQuad * self, GstVulkanImageView ** views, + GError ** error) +{ + VkImageView attachments[GST_VIDEO_MAX_PLANES] = { 0, }; + VkFramebufferCreateInfo framebuffer_info; + VkFramebuffer framebuffer; + VkResult err; + int i; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->out_info); i++) { + attachments[i] = views[i]->view; + } + + /* *INDENT-OFF* */ + framebuffer_info = (VkFramebufferCreateInfo) { + .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, + .pNext = NULL, + .renderPass = (VkRenderPass) self->render_pass->handle, + .attachmentCount = GST_VIDEO_INFO_N_PLANES (&self->out_info), + .pAttachments = attachments, + .width = GST_VIDEO_INFO_WIDTH (&self->out_info), + .height = GST_VIDEO_INFO_HEIGHT (&self->out_info), + .layers = 1 + }; + /* *INDENT-ON* */ + + err = + vkCreateFramebuffer (self->queue->device->device, &framebuffer_info, NULL, + &framebuffer); + if (gst_vulkan_error_to_g_error (err, error, "vkCreateFramebuffer") < 0) { + return FALSE; + } + + self->framebuffer = gst_vulkan_handle_new_wrapped (self->queue->device, + GST_VULKAN_HANDLE_TYPE_FRAMEBUFFER, (GstVulkanHandleTypedef) framebuffer, + gst_vulkan_handle_free_framebuffer, NULL); + + return TRUE; +} + +#define LAST_FENCE_OR_ALWAYS_SIGNALLED(self,device) \ + self->last_fence ? gst_vulkan_fence_ref (self->last_fence) : gst_vulkan_fence_new_always_signalled (device) + +GstVulkanFence * +gst_vulkan_full_screen_quad_get_last_fence (GstVulkanFullScreenQuad * self) +{ + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), NULL); + + return LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); +} + +static void +clear_descriptor_set (GstVulkanFullScreenQuad * self) +{ + GstVulkanFence *last_fence = + LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); + + if (self->descriptor_set) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) self->descriptor_set)); + self->descriptor_set = NULL; + + gst_vulkan_fence_unref (last_fence); +} + +static void +clear_framebuffer (GstVulkanFullScreenQuad * self) +{ + GstVulkanFence *last_fence = + LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); + + if (self->framebuffer) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) self->framebuffer)); + self->framebuffer = NULL; + + gst_vulkan_fence_unref (last_fence); +} + +static void +clear_command_pool (GstVulkanFullScreenQuad * self) +{ + GstVulkanFence *last_fence = + LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); + + if (self->cmd_pool) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_object_unref (last_fence, + (GstObject *) self->cmd_pool)); + self->cmd_pool = NULL; + + gst_vulkan_fence_unref (last_fence); +} + +static void +clear_sampler (GstVulkanFullScreenQuad * self) +{ + GstVulkanFence *last_fence = + LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); + + if (self->sampler) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) self->sampler)); + self->sampler = NULL; + + gst_vulkan_fence_unref (last_fence); +} + +static void +clear_descriptor_cache (GstVulkanFullScreenQuad * self) +{ + GstVulkanFence *last_fence = + LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); + + if (self->descriptor_cache) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_object_unref (last_fence, + (GstObject *) self->descriptor_cache)); + self->descriptor_cache = NULL; + + gst_vulkan_fence_unref (last_fence); +} + +static void +clear_shaders (GstVulkanFullScreenQuad * self) +{ + GstVulkanFullScreenQuadPrivate *priv = GET_PRIV (self); + GstVulkanFence *last_fence = + LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); + + if (priv->vert) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) priv->vert)); + priv->vert = NULL; + + if (priv->frag) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) priv->frag)); + priv->frag = NULL; + + gst_vulkan_fence_unref (last_fence); +} + +static void +clear_uniform_data (GstVulkanFullScreenQuad * self) +{ + GstVulkanFullScreenQuadPrivate *priv = GET_PRIV (self); + GstVulkanFence *last_fence = + LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); + + if (priv->uniforms) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) priv->uniforms)); + priv->uniforms = NULL; + priv->uniform_size = 0; + + gst_vulkan_fence_unref (last_fence); +} + +static void +destroy_pipeline (GstVulkanFullScreenQuad * self) +{ + GstVulkanFence *last_fence = + LAST_FENCE_OR_ALWAYS_SIGNALLED (self, self->queue->device); + + if (self->render_pass) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) self->render_pass)); + self->render_pass = NULL; + if (self->pipeline_layout) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) self->pipeline_layout)); + self->pipeline_layout = NULL; + if (self->graphics_pipeline) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) self->graphics_pipeline)); + self->graphics_pipeline = NULL; + if (self->descriptor_set_layout) + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (last_fence, + (GstMiniObject *) self->descriptor_set_layout)); + self->descriptor_set_layout = NULL; + + gst_vulkan_fence_unref (last_fence); + + gst_vulkan_trash_list_gc (self->trash_list); +} + +void +gst_vulkan_full_screen_quad_init (GstVulkanFullScreenQuad * self) +{ + self->trash_list = gst_vulkan_trash_fence_list_new (); +} + +GstVulkanFullScreenQuad * +gst_vulkan_full_screen_quad_new (GstVulkanQueue * queue) +{ + GstVulkanFullScreenQuad *self; + + g_return_val_if_fail (GST_IS_VULKAN_QUEUE (queue), NULL); + + self = g_object_new (GST_TYPE_VULKAN_FULL_SCREEN_QUAD, NULL); + self->queue = gst_object_ref (queue); + + return self; +} + +static void +gst_vulkan_full_screen_quad_finalize (GObject * object) +{ + GstVulkanFullScreenQuad *self = GST_VULKAN_FULL_SCREEN_QUAD (object); + GstVulkanFullScreenQuadPrivate *priv = GET_PRIV (self); + + destroy_pipeline (self); + clear_command_pool (self); + clear_sampler (self); + clear_framebuffer (self); + clear_descriptor_set (self); + clear_descriptor_cache (self); + clear_shaders (self); + clear_uniform_data (self); + + gst_vulkan_trash_list_wait (self->trash_list, -1); + gst_vulkan_trash_list_gc (self->trash_list); + gst_clear_object (&self->trash_list); + + gst_clear_mini_object (((GstMiniObject **) & priv->vertices)); + gst_clear_mini_object (((GstMiniObject **) & priv->indices)); + + gst_clear_mini_object (((GstMiniObject **) & self->last_fence)); + + gst_clear_object (&self->queue); + + gst_clear_buffer (&priv->inbuf); + gst_clear_buffer (&priv->outbuf); + + G_OBJECT_CLASS (gst_vulkan_full_screen_quad_parent_class)->finalize (object); +} + +static void +gst_vulkan_full_screen_quad_class_init (GstVulkanFullScreenQuadClass * klass) +{ + GObjectClass *obj_class = G_OBJECT_CLASS (klass); + + obj_class->finalize = gst_vulkan_full_screen_quad_finalize; +} + +gboolean +gst_vulkan_full_screen_quad_set_info (GstVulkanFullScreenQuad * self, + GstVideoInfo * in_info, GstVideoInfo * out_info) +{ + self->out_info = *out_info; + self->in_info = *in_info; + + destroy_pipeline (self); + clear_framebuffer (self); + clear_descriptor_set (self); + clear_descriptor_cache (self); + clear_uniform_data (self); + + return TRUE; +} + +gboolean +gst_vulkan_full_screen_quad_set_input_buffer (GstVulkanFullScreenQuad * self, + GstBuffer * buffer, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv; + + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), FALSE); + + priv = GET_PRIV (self); + + gst_buffer_replace (&priv->inbuf, buffer); + clear_descriptor_set (self); + return TRUE; +} + +gboolean +gst_vulkan_full_screen_quad_set_output_buffer (GstVulkanFullScreenQuad * self, + GstBuffer * buffer, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv; + + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), FALSE); + + priv = GET_PRIV (self); + + gst_buffer_replace (&priv->outbuf, buffer); + clear_framebuffer (self); + return TRUE; +} + +gboolean +gst_vulkan_full_screen_quad_set_shaders (GstVulkanFullScreenQuad * self, + GstVulkanHandle * vert, GstVulkanHandle * frag) +{ + GstVulkanFullScreenQuadPrivate *priv; + + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), FALSE); + g_return_val_if_fail (vert != NULL, FALSE); + g_return_val_if_fail (vert->type == GST_VULKAN_HANDLE_TYPE_SHADER, FALSE); + g_return_val_if_fail (frag != NULL, FALSE); + g_return_val_if_fail (frag->type == GST_VULKAN_HANDLE_TYPE_SHADER, FALSE); + + priv = GET_PRIV (self); + + clear_shaders (self); + destroy_pipeline (self); + + priv->vert = gst_vulkan_handle_ref (vert); + priv->frag = gst_vulkan_handle_ref (frag); + + return TRUE; +} + +gboolean +gst_vulkan_full_screen_quad_set_uniform_buffer (GstVulkanFullScreenQuad * self, + GstMemory * uniforms, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv; + + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), FALSE); + g_return_val_if_fail (uniforms == NULL + || gst_is_vulkan_buffer_memory (uniforms), FALSE); + + priv = GET_PRIV (self); + + clear_uniform_data (self); + if (uniforms) { + priv->uniforms = gst_memory_ref (uniforms); + priv->uniform_size = gst_memory_get_sizes (uniforms, NULL, NULL); + } + + return TRUE; +} + +static GstVulkanImageMemory * +peek_image_from_buffer (GstBuffer * buffer, guint i) +{ + GstMemory *mem = gst_buffer_peek_memory (buffer, i); + g_return_val_if_fail (gst_is_vulkan_image_memory (mem), NULL); + return (GstVulkanImageMemory *) mem; +} + +static gboolean +ensure_vertex_data (GstVulkanFullScreenQuad * self, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv = GET_PRIV (self); + GstMapInfo map_info; + + if (!priv->vertices) { + priv->vertices = gst_vulkan_buffer_memory_alloc (self->queue->device, + sizeof (vertices), VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + } + + if (!gst_memory_map (priv->vertices, &map_info, GST_MAP_WRITE)) { + g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_MEMORY_MAP_FAILED, + "Failed to map memory"); + goto failure; + } + + memcpy (map_info.data, vertices, map_info.size); + gst_memory_unmap (priv->vertices, &map_info); + + if (!priv->indices) { + priv->indices = gst_vulkan_buffer_memory_alloc (self->queue->device, + sizeof (indices), VK_BUFFER_USAGE_INDEX_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + } + + if (!gst_memory_map (priv->indices, &map_info, GST_MAP_WRITE)) { + g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_MEMORY_MAP_FAILED, + "Failed to map memory"); + goto failure; + } + + memcpy (map_info.data, indices, map_info.size); + gst_memory_unmap (priv->indices, &map_info); + + priv->n_indices = G_N_ELEMENTS (indices); + + return TRUE; + +failure: + if (priv->vertices) + gst_memory_unref (priv->vertices); + priv->vertices = NULL; + if (priv->indices) + gst_memory_unref (priv->indices); + priv->indices = NULL; + priv->n_indices = 0; + return FALSE; +} + +gboolean +gst_vulkan_full_screen_quad_draw (GstVulkanFullScreenQuad * self, + GError ** error) +{ + GstVulkanCommandBuffer *cmd = NULL; + GstVulkanFence *fence = NULL; + VkResult err; + + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), FALSE); + + fence = gst_vulkan_fence_new (self->queue->device, 0, error); + if (!fence) + goto error; + + if (!gst_vulkan_full_screen_quad_prepare_draw (self, fence, error)) + goto error; + + if (!(cmd = gst_vulkan_command_pool_create (self->cmd_pool, error))) + goto error; + + { + VkCommandBufferBeginInfo cmd_buf_info = { 0, }; + + /* *INDENT-OFF* */ + cmd_buf_info = (VkCommandBufferBeginInfo) { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .pNext = NULL, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, + .pInheritanceInfo = NULL + }; + /* *INDENT-ON* */ + + gst_vulkan_command_buffer_lock (cmd); + err = vkBeginCommandBuffer (cmd->cmd, &cmd_buf_info); + if (gst_vulkan_error_to_g_error (err, error, "vkBeginCommandBuffer") < 0) + goto unlock_error; + } + + if (!gst_vulkan_full_screen_quad_fill_command_buffer (self, cmd, fence, + error)) + goto unlock_error; + + err = vkEndCommandBuffer (cmd->cmd); + gst_vulkan_command_buffer_unlock (cmd); + if (gst_vulkan_error_to_g_error (err, error, "vkEndCommandBuffer") < 0) + goto error; + + if (!gst_vulkan_full_screen_quad_submit (self, cmd, fence, error)) + goto error; + + gst_vulkan_fence_unref (fence); + + return TRUE; + +unlock_error: + gst_vulkan_command_buffer_unlock (cmd); + +error: + gst_clear_mini_object ((GstMiniObject **) & cmd); + gst_clear_mini_object ((GstMiniObject **) & fence); + return FALSE; +} + +gboolean +gst_vulkan_full_screen_quad_prepare_draw (GstVulkanFullScreenQuad * self, + GstVulkanFence * fence, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv; + GstVulkanImageView *in_views[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanImageView *out_views[GST_VIDEO_MAX_PLANES] = { NULL, }; + int i; + + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), FALSE); + g_return_val_if_fail (fence != NULL, FALSE); + + priv = GET_PRIV (self); + + if (!self->graphics_pipeline) + if (!create_pipeline (self, error)) + return FALSE; + + if (!ensure_vertex_data (self, error)) + goto error; + + if (!self->descriptor_cache) + if (!create_descriptor_pool (self, error)) + goto error; + + if (!self->descriptor_set) { + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->in_info); i++) { + GstVulkanImageMemory *img_mem = peek_image_from_buffer (priv->inbuf, i); + if (!gst_is_vulkan_image_memory ((GstMemory *) img_mem)) { + g_set_error_literal (error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Input memory must be a GstVulkanImageMemory"); + goto error; + } + in_views[i] = get_or_create_image_view (img_mem); + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (fence, + (GstMiniObject *) in_views[i])); + } + if (!(self->descriptor_set = + get_and_update_descriptor_set (self, in_views, error))) + goto error; + } + + if (!self->framebuffer) { + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->out_info); i++) { + GstVulkanImageMemory *img_mem = peek_image_from_buffer (priv->outbuf, i); + if (!gst_is_vulkan_image_memory ((GstMemory *) img_mem)) { + g_set_error_literal (error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Output memory must be a GstVulkanImageMemory"); + goto error; + } + out_views[i] = get_or_create_image_view (img_mem); + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (fence, + (GstMiniObject *) out_views[i])); + } + if (!create_framebuffer (self, out_views, error)) + goto error; + } + + if (!self->cmd_pool) + if (!(self->cmd_pool = + gst_vulkan_queue_create_command_pool (self->queue, error))) + goto error; + + return TRUE; + +error: + for (i = 0; i < GST_VIDEO_MAX_PLANES; i++) + gst_clear_mini_object ((GstMiniObject **) & in_views[i]); + for (i = 0; i < GST_VIDEO_MAX_PLANES; i++) + gst_clear_mini_object ((GstMiniObject **) & out_views[i]); + return FALSE; +} + +/** + * gst_vulkan_full_screen_quad_fill_command_buffer: + * @self: a #GstVulkanFullScreenQuad + * @cmd: the #GstVulkanCommandBuffer to fill with commands + * @error: a #GError to fill on error + * + * Returns: whether @cmd could be filled with the necessary commands + */ +gboolean +gst_vulkan_full_screen_quad_fill_command_buffer (GstVulkanFullScreenQuad * self, + GstVulkanCommandBuffer * cmd, GstVulkanFence * fence, GError ** error) +{ + GstVulkanFullScreenQuadPrivate *priv; + GstVulkanImageView *in_views[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanImageView *out_views[GST_VIDEO_MAX_PLANES] = { NULL, }; + int i; + + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), FALSE); + g_return_val_if_fail (cmd != NULL, FALSE); + g_return_val_if_fail (fence != NULL, FALSE); + + priv = GET_PRIV (self); + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->in_info); i++) { + GstVulkanImageMemory *img_mem = peek_image_from_buffer (priv->inbuf, i); + if (!gst_is_vulkan_image_memory ((GstMemory *) img_mem)) { + g_set_error_literal (error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Input memory must be a GstVulkanImageMemory"); + goto error; + } + in_views[i] = get_or_create_image_view (img_mem); + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (fence, + (GstMiniObject *) in_views[i])); + } + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->out_info); i++) { + GstVulkanImageMemory *img_mem = peek_image_from_buffer (priv->outbuf, i); + if (!gst_is_vulkan_image_memory ((GstMemory *) img_mem)) { + g_set_error_literal (error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Output memory must be a GstVulkanImageMemory"); + goto error; + } + out_views[i] = get_or_create_image_view (img_mem); + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (fence, + (GstMiniObject *) out_views[i])); + } + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->in_info); i++) { + /* *INDENT-OFF* */ + VkImageMemoryBarrier in_image_memory_barrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = NULL, + .srcAccessMask = in_views[i]->image->barrier.parent.access_flags, + .dstAccessMask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT, + .oldLayout = in_views[i]->image->barrier.image_layout, + .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + /* FIXME: implement exclusive transfers */ + .srcQueueFamilyIndex = 0, + .dstQueueFamilyIndex = 0, + .image = in_views[i]->image->image, + .subresourceRange = in_views[i]->image->barrier.subresource_range + }; + /* *INDENT-ON* */ + + vkCmdPipelineBarrier (cmd->cmd, + in_views[i]->image->barrier.parent.pipeline_stages, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, NULL, 0, NULL, 1, + &in_image_memory_barrier); + + in_views[i]->image->barrier.parent.pipeline_stages = + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + in_views[i]->image->barrier.parent.access_flags = + in_image_memory_barrier.dstAccessMask; + in_views[i]->image->barrier.image_layout = + in_image_memory_barrier.newLayout; + } + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->out_info); i++) { + /* *INDENT-OFF* */ + VkImageMemoryBarrier out_image_memory_barrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = NULL, + .srcAccessMask = out_views[i]->image->barrier.parent.access_flags, + .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .oldLayout = out_views[i]->image->barrier.image_layout, + .newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + /* FIXME: implement exclusive transfers */ + .srcQueueFamilyIndex = 0, + .dstQueueFamilyIndex = 0, + .image = out_views[i]->image->image, + .subresourceRange = out_views[i]->image->barrier.subresource_range + }; + /* *INDENT-ON* */ + + vkCmdPipelineBarrier (cmd->cmd, + out_views[i]->image->barrier.parent.pipeline_stages, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, NULL, 0, NULL, 1, + &out_image_memory_barrier); + + out_views[i]->image->barrier.parent.pipeline_stages = + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + out_views[i]->image->barrier.parent.access_flags = + out_image_memory_barrier.dstAccessMask; + out_views[i]->image->barrier.image_layout = + out_image_memory_barrier.newLayout; + } + + { + /* *INDENT-OFF* */ + VkClearValue clearColor = {{{ 0.0f, 0.0f, 0.0f, 1.0f }}}; + VkClearValue clearColors[GST_VIDEO_MAX_PLANES] = { + clearColor, clearColor, clearColor, clearColor, + }; + VkRenderPassBeginInfo render_pass_info = { + .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, + .renderPass = (VkRenderPass) self->render_pass->handle, + .framebuffer = (VkFramebuffer) self->framebuffer->handle, + .renderArea.offset = { 0, 0 }, + .renderArea.extent = { + GST_VIDEO_INFO_WIDTH (&self->out_info), + GST_VIDEO_INFO_HEIGHT (&self->out_info) + }, + .clearValueCount = GST_VIDEO_INFO_N_PLANES (&self->out_info), + .pClearValues = clearColors, + }; + /* *INDENT-ON* */ + VkDeviceSize offsets[] = { 0 }; + + vkCmdBindDescriptorSets (cmd->cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, + (VkPipelineLayout) self->pipeline_layout->handle, 0, 1, + &self->descriptor_set->set, 0, NULL); + + vkCmdBeginRenderPass (cmd->cmd, &render_pass_info, + VK_SUBPASS_CONTENTS_INLINE); + vkCmdBindPipeline (cmd->cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, + (VkPipeline) self->graphics_pipeline->handle); + vkCmdBindVertexBuffers (cmd->cmd, 0, 1, + &((GstVulkanBufferMemory *) priv->vertices)->buffer, offsets); + vkCmdBindIndexBuffer (cmd->cmd, + ((GstVulkanBufferMemory *) priv->indices)->buffer, 0, + VK_INDEX_TYPE_UINT16); + vkCmdDrawIndexed (cmd->cmd, priv->n_indices, 1, 0, 0, 0); + vkCmdEndRenderPass (cmd->cmd); + } + + return TRUE; + +error: + return FALSE; +} + +/** + * gst_vulkan_full_screen_quad_submit: + * @self: a #GstVulkanFullScreenQuad + * @cmd: (transfer full): a #GstVulkanCommandBuffer to submit + * @fence: a #GstVulkanFence to signal on completion + * @error: a #GError to fill on error + * + * Returns: whether @cmd could be submitted to the queue + */ +gboolean +gst_vulkan_full_screen_quad_submit (GstVulkanFullScreenQuad * self, + GstVulkanCommandBuffer * cmd, GstVulkanFence * fence, GError ** error) +{ + VkResult err; + + g_return_val_if_fail (GST_IS_VULKAN_FULL_SCREEN_QUAD (self), FALSE); + g_return_val_if_fail (cmd != NULL, FALSE); + g_return_val_if_fail (fence != NULL, FALSE); + + { + /* *INDENT-OFF* */ + VkSubmitInfo submit_info = { + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, + .pNext = NULL, + .waitSemaphoreCount = 0, + .pWaitSemaphores = NULL, + .pWaitDstStageMask = NULL, + .commandBufferCount = 1, + .pCommandBuffers = &cmd->cmd, + .signalSemaphoreCount = 0, + .pSignalSemaphores = NULL, + }; + /* *INDENT-ON* */ + + gst_vulkan_queue_submit_lock (self->queue); + err = + vkQueueSubmit (self->queue->queue, 1, &submit_info, + GST_VULKAN_FENCE_FENCE (fence)); + gst_vulkan_queue_submit_unlock (self->queue); + if (gst_vulkan_error_to_g_error (err, error, "vkQueueSubmit") < 0) + goto error; + } + + gst_vulkan_trash_list_add (self->trash_list, + gst_vulkan_trash_new_mini_object_unref (fence, + GST_MINI_OBJECT_CAST (cmd))); + + gst_vulkan_trash_list_gc (self->trash_list); + + if (self->last_fence) + gst_vulkan_fence_unref (self->last_fence); + self->last_fence = gst_vulkan_fence_ref (fence); + + return TRUE; + +error: + return FALSE; +} diff --git a/ext/vulkan/vkfullscreenquad.h b/ext/vulkan/vkfullscreenquad.h new file mode 100644 index 0000000000..f73748ccf3 --- /dev/null +++ b/ext/vulkan/vkfullscreenquad.h @@ -0,0 +1,96 @@ +/* + * GStreamer Plugins + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_VULKAN_FULL_SCREEN_QUAD_H__ +#define __GST_VULKAN_FULL_SCREEN_QUAD_H__ + + +#include + +#include + +G_BEGIN_DECLS + +GType gst_vulkan_full_screen_quad_get_type (void); +#define GST_TYPE_VULKAN_FULL_SCREEN_QUAD (gst_vulkan_full_screen_quad_get_type ()) +#define GST_VULKAN_FULL_SCREEN_QUAD(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_VULKAN_FULL_SCREEN_QUAD, GstVulkanFullScreenQuad)) +#define GST_VULKAN_FULL_SCREEN_QUAD_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_VULKAN_FULL_SCREEN_QUAD, GstVulkanFullScreenQuadClass)) +#define GST_IS_VULKAN_FULL_SCREEN_QUAD(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_VULKAN_FULL_SCREEN_QUAD)) +#define GST_IS_VULKAN_FULL_SCREEN_QUAD_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_VULKAN_FULL_SCREEN_QUAD)) +#define GST_VULKAN_FULL_SCREEN_QUAD_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_VULKAN_FULL_SCREEN_QUAD, GstVulkanFullScreenQuadClass)) + +/* XXX: privatise this on moving to lib */ +struct Vertex +{ + float x, y, z; + float s, t; +}; + +typedef struct _GstVulkanFullScreenQuad GstVulkanFullScreenQuad; +typedef struct _GstVulkanFullScreenQuadClass GstVulkanFullScreenQuadClass; +typedef struct _GstVulkanFullScreenQuadPrivate GstVulkanFullScreenQuadPrivate; + +struct _GstVulkanFullScreenQuad +{ + GstObject parent; + + GstVideoInfo out_info; + GstVideoInfo in_info; + + GstVulkanQueue *queue; + + GstVulkanHandle *render_pass; + GstVulkanHandle *pipeline_layout; + GstVulkanHandle *graphics_pipeline; + GstVulkanHandle *descriptor_set_layout; + GstVulkanDescriptorCache *descriptor_cache; + GstVulkanDescriptorSet *descriptor_set; + GstVulkanHandle *framebuffer; + GstVulkanHandle *sampler; + + GstVulkanCommandPool *cmd_pool; + + GstVulkanTrashList *trash_list; + GstVulkanFence *last_fence; +}; + +struct _GstVulkanFullScreenQuadClass +{ + GstObjectClass parent_class; +}; + +GstVulkanFullScreenQuad * gst_vulkan_full_screen_quad_new (GstVulkanQueue * queue); + +gboolean gst_vulkan_full_screen_quad_set_info (GstVulkanFullScreenQuad * self, GstVideoInfo *in_info, GstVideoInfo * out_info); +gboolean gst_vulkan_full_screen_quad_set_shaders (GstVulkanFullScreenQuad * self, GstVulkanHandle * vert, GstVulkanHandle * frag); +gboolean gst_vulkan_full_screen_quad_set_uniform_buffer (GstVulkanFullScreenQuad * self, GstMemory * uniforms, GError ** error); + +gboolean gst_vulkan_full_screen_quad_set_input_buffer (GstVulkanFullScreenQuad * self, GstBuffer * buffer, GError ** error); +gboolean gst_vulkan_full_screen_quad_set_output_buffer (GstVulkanFullScreenQuad * self, GstBuffer * buffer, GError ** error); + +gboolean gst_vulkan_full_screen_quad_prepare_draw (GstVulkanFullScreenQuad * self, GstVulkanFence * fence, GError ** error); +gboolean gst_vulkan_full_screen_quad_fill_command_buffer (GstVulkanFullScreenQuad * self, GstVulkanCommandBuffer * cmd, GstVulkanFence * fence, GError ** error); +gboolean gst_vulkan_full_screen_quad_submit (GstVulkanFullScreenQuad * self, GstVulkanCommandBuffer * cmd, GstVulkanFence * fence, GError ** error); +gboolean gst_vulkan_full_screen_quad_draw (GstVulkanFullScreenQuad * self, GError ** error); + +GstVulkanFence * gst_vulkan_full_screen_quad_get_last_fence (GstVulkanFullScreenQuad * self); + +G_END_DECLS +#endif /* __GST_VULKAN_FULL_SCREEN_QUAD_H__ */ diff --git a/ext/vulkan/vkfullscreenrender.c b/ext/vulkan/vkfullscreenrender.c deleted file mode 100644 index d323f25d40..0000000000 --- a/ext/vulkan/vkfullscreenrender.c +++ /dev/null @@ -1,921 +0,0 @@ -/* - * GStreamer - * Copyright (C) 2019 Matthew Waters - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -/** - * SECTION:element-vulkanimageidentity - * @title: vulkanimgeidentity - * - * vulkanimageidentity produces a vulkan image that is a copy of the input image. - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include - -#include "vkimageidentity.h" -#include "vkshader.h" - -GST_DEBUG_CATEGORY (gst_debug_vulkan_full_screen_render); -#define GST_CAT_DEFAULT gst_debug_vulkan_full_screen_render - -struct Vertex vertices[] = { - {-1.0f, -1.0f, 0.0f, 0.0f, 0.0f}, - {1.0f, -1.0f, 0.0f, 1.0f, 0.0f}, - {1.0f, 1.0f, 0.0f, 1.0f, 1.0f}, - {-1.0f, 1.0f, 0.0f, 0.0f, 1.0f}, -}; - -gushort indices[] = { - 0, 1, 2, 0, 2, 3, -}; - -static void gst_vulkan_full_screen_render_finalize (GObject * object); - -static gboolean gst_vulkan_full_screen_render_query (GstBaseTransform * bt, - GstPadDirection direction, GstQuery * query); -static void gst_vulkan_full_screen_render_set_context (GstElement * element, - GstContext * context); - -static gboolean gst_vulkan_full_screen_render_start (GstBaseTransform * bt); -static gboolean gst_vulkan_full_screen_render_stop (GstBaseTransform * bt); - -static gboolean gst_vulkan_full_screen_render_set_caps (GstBaseTransform * bt, - GstCaps * in_caps, GstCaps * out_caps); -static GstCaps *gst_vulkan_full_screen_render_transform_caps (GstBaseTransform * - bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter); -static gboolean -gst_vulkan_full_screen_render_propose_allocation (GstBaseTransform * bt, - GstQuery * decide_query, GstQuery * query); -static gboolean -gst_vulkan_full_screen_render_decide_allocation (GstBaseTransform * bt, - GstQuery * query); - -#define IMAGE_FORMATS " { BGRA }" - -static GstStaticPadTemplate gst_vulkan_sink_template = -GST_STATIC_PAD_TEMPLATE ("sink", - GST_PAD_SINK, - GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, - IMAGE_FORMATS))); - -static GstStaticPadTemplate gst_vulkan_src_template = -GST_STATIC_PAD_TEMPLATE ("src", - GST_PAD_SRC, - GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, - IMAGE_FORMATS))); - -enum -{ - PROP_0, -}; - -enum -{ - SIGNAL_0, - LAST_SIGNAL -}; - -/* static guint gst_vulkan_full_screen_render_signals[LAST_SIGNAL] = { 0 }; */ - -#define gst_vulkan_full_screen_render_parent_class parent_class -G_DEFINE_TYPE_WITH_CODE (GstVulkanFullScreenRender, - gst_vulkan_full_screen_render, GST_TYPE_BASE_TRANSFORM, - GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_full_screen_render, - "vulkanimageidentity", 0, "Vulkan Image identity")); - -static void -gst_vulkan_full_screen_render_class_init (GstVulkanFullScreenRenderClass * - klass) -{ - GObjectClass *gobject_class; - GstElementClass *gstelement_class; - GstBaseTransformClass *gstbasetransform_class; - - gobject_class = (GObjectClass *) klass; - gstelement_class = (GstElementClass *) klass; - gstbasetransform_class = (GstBaseTransformClass *) klass; - - gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader", - "Filter/Video", "A Vulkan image copier", - "Matthew Waters "); - - gst_element_class_add_static_pad_template (gstelement_class, - &gst_vulkan_sink_template); - gst_element_class_add_static_pad_template (gstelement_class, - &gst_vulkan_src_template); - - gobject_class->finalize = gst_vulkan_full_screen_render_finalize; - - gstelement_class->set_context = gst_vulkan_full_screen_render_set_context; - gstbasetransform_class->start = - GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_start); - gstbasetransform_class->stop = - GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_stop); - gstbasetransform_class->query = - GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_query); - gstbasetransform_class->set_caps = gst_vulkan_full_screen_render_set_caps; - gstbasetransform_class->transform_caps = - gst_vulkan_full_screen_render_transform_caps; - gstbasetransform_class->propose_allocation = - gst_vulkan_full_screen_render_propose_allocation; - gstbasetransform_class->decide_allocation = - gst_vulkan_full_screen_render_decide_allocation; -} - -static void -gst_vulkan_full_screen_render_init (GstVulkanFullScreenRender * render) -{ -} - -static void -gst_vulkan_full_screen_render_finalize (GObject * object) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (object); - - gst_caps_replace (&render->in_caps, NULL); - gst_caps_replace (&render->out_caps, NULL); - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - -static gboolean -gst_vulkan_full_screen_render_query (GstBaseTransform * bt, - GstPadDirection direction, GstQuery * query) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_CONTEXT:{ - if (gst_vulkan_handle_context_query (GST_ELEMENT (render), query, - NULL, render->instance, render->device)) - return TRUE; - - if (gst_vulkan_queue_handle_context_query (GST_ELEMENT (render), - query, render->queue)) - return TRUE; - - break; - } - default: - break; - } - - return GST_BASE_TRANSFORM_CLASS (parent_class)->query (bt, direction, query); -} - -static void -gst_vulkan_full_screen_render_set_context (GstElement * element, - GstContext * context) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (element); - - gst_vulkan_handle_set_context (element, context, NULL, &render->instance); - - GST_ELEMENT_CLASS (parent_class)->set_context (element, context); -} - -struct choose_data -{ - GstVulkanFullScreenRender *upload; - GstVulkanQueue *queue; -}; - -static gboolean -_choose_queue (GstVulkanDevice * device, GstVulkanQueue * queue, - struct choose_data *data) -{ - guint flags = - device->physical_device->queue_family_props[queue->family].queueFlags; - - GST_ERROR ("flags 0x%x", flags); - - if ((flags & VK_QUEUE_GRAPHICS_BIT) != 0) { - if (data->queue) - gst_object_unref (data->queue); - data->queue = gst_object_ref (queue); - return FALSE; - } - - return TRUE; -} - -static GstVulkanQueue * -_find_graphics_queue (GstVulkanFullScreenRender * upload) -{ - struct choose_data data; - - data.upload = upload; - data.queue = NULL; - - gst_vulkan_device_foreach_queue (upload->device, - (GstVulkanDeviceForEachQueueFunc) _choose_queue, &data); - - return data.queue; -} - -static GstCaps * -gst_vulkan_full_screen_render_transform_caps (GstBaseTransform * bt, - GstPadDirection direction, GstCaps * caps, GstCaps * filter) -{ - GstCaps *result, *tmp; - - tmp = gst_caps_copy (caps); - - if (filter) { - result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); - gst_caps_unref (tmp); - } else { - result = tmp; - } - - return result; -} - -static void -clear_shader_create_info (GstVulkanFullScreenRender * render) -{ - if (render->shader_create_info) { - if (render->destroy_shader_create_info) - render->destroy_shader_create_info (render, render->shader_create_info); - } - render->n_shader_stages = 0; - render->shader_create_info = NULL; - render->destroy_shader_create_info = NULL; -} - -static VkPipeline -_create_pipeline (GstVulkanFullScreenRender * render) -{ - GstVulkanFullScreenRenderClass *render_class = - GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render); - VkVertexInputBindingDescription vertex_binding_description; - VkVertexInputAttributeDescription vertex_attribute_description[2]; - VkPipelineVertexInputStateCreateInfo vertex_input_info; - VkPipelineInputAssemblyStateCreateInfo input_assembly; - VkPipelineViewportStateCreateInfo viewport_state; - VkPipelineRasterizationStateCreateInfo rasterizer; - VkPipelineMultisampleStateCreateInfo multisampling; - VkPipelineColorBlendAttachmentState - color_blend_attachments[GST_VIDEO_MAX_PLANES]; - VkPipelineColorBlendStateCreateInfo color_blending; - VkGraphicsPipelineCreateInfo pipeline_info; - VkPipeline pipeline; - GError *error = NULL; - VkResult err; - - render_class->shader_create_info (render); - - /* *INDENT-OFF* */ - vertex_binding_description = (VkVertexInputBindingDescription) { - .binding = 0, - .stride = sizeof (struct Vertex), - .inputRate = VK_VERTEX_INPUT_RATE_VERTEX - }; - - vertex_attribute_description[0] = (VkVertexInputAttributeDescription) { - .binding = 0, - .location = 0, - .format = VK_FORMAT_R32G32B32_SFLOAT, - .offset = G_STRUCT_OFFSET (struct Vertex, x) - }; - vertex_attribute_description[1] = (VkVertexInputAttributeDescription) { - .binding = 0, - .location = 1, - .format = VK_FORMAT_R32G32_SFLOAT, - .offset = G_STRUCT_OFFSET (struct Vertex, s) - }; - - vertex_input_info = (VkPipelineVertexInputStateCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, - .pNext = NULL, - .vertexBindingDescriptionCount = 1, - .pVertexBindingDescriptions = &vertex_binding_description, - .vertexAttributeDescriptionCount = G_N_ELEMENTS (vertex_attribute_description), - .pVertexAttributeDescriptions = vertex_attribute_description - }; - - input_assembly = (VkPipelineInputAssemblyStateCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, - .pNext = NULL, - .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, - .primitiveRestartEnable = VK_FALSE - }; - - viewport_state = (VkPipelineViewportStateCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, - .pNext = NULL, - .viewportCount = 1, - .pViewports = &(VkViewport) { - .x = 0.0f, - .y = 0.0f, - .width = (float) GST_VIDEO_INFO_WIDTH (&render->out_info), - .height = (float) GST_VIDEO_INFO_HEIGHT (&render->out_info), - .minDepth = 0.0f, - .maxDepth = 1.0f - }, - .scissorCount = 1, - .pScissors = &(VkRect2D) { - .offset = { 0, 0 }, - .extent = { - GST_VIDEO_INFO_WIDTH (&render->out_info), - GST_VIDEO_INFO_HEIGHT (&render->out_info) - } - } - }; - - rasterizer = (VkPipelineRasterizationStateCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, - .pNext = NULL, - .depthClampEnable = VK_FALSE, - .rasterizerDiscardEnable = VK_FALSE, - .polygonMode = VK_POLYGON_MODE_FILL, - .lineWidth = 1.0f, - .cullMode = VK_CULL_MODE_BACK_BIT, - .frontFace = VK_FRONT_FACE_CLOCKWISE, - .depthBiasEnable = VK_FALSE - }; - - multisampling = (VkPipelineMultisampleStateCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, - .pNext = NULL, - .sampleShadingEnable = VK_FALSE, - .rasterizationSamples = VK_SAMPLE_COUNT_1_BIT - }; - - color_blend_attachments[0] = (VkPipelineColorBlendAttachmentState) { - .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, - .blendEnable = VK_FALSE - }; - color_blend_attachments[1] = (VkPipelineColorBlendAttachmentState) { - .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, - .blendEnable = VK_FALSE - }; - color_blend_attachments[2] = (VkPipelineColorBlendAttachmentState) { - .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, - .blendEnable = VK_FALSE - }; - color_blend_attachments[3] = (VkPipelineColorBlendAttachmentState) { - .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, - .blendEnable = VK_FALSE - }; - - color_blending = (VkPipelineColorBlendStateCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, - .pNext = NULL, - .logicOpEnable = VK_FALSE, - .logicOp = VK_LOGIC_OP_COPY, - .attachmentCount = GST_VIDEO_INFO_N_PLANES (&render->out_info), - .pAttachments = color_blend_attachments, - .blendConstants = { 0.0f, 0.0f, 0.0f, 0.0f } - }; - - pipeline_info = (VkGraphicsPipelineCreateInfo) { - .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = NULL, - .stageCount = render->n_shader_stages, - .pStages = render->shader_create_info, - .pVertexInputState = &vertex_input_info, - .pInputAssemblyState = &input_assembly, - .pViewportState = &viewport_state, - .pRasterizationState = &rasterizer, - .pMultisampleState = &multisampling, - .pColorBlendState = &color_blending, - .layout = render->pipeline_layout, - .renderPass = render->render_pass, - .subpass = 0, - .basePipelineHandle = VK_NULL_HANDLE - }; - /* *INDENT-ON* */ - - err = - vkCreateGraphicsPipelines (render->device->device, VK_NULL_HANDLE, 1, - &pipeline_info, NULL, &pipeline); - clear_shader_create_info (render); - if (gst_vulkan_error_to_g_error (err, &error, - "vkCreateGraphicsPipelines") < 0) { - GST_ERROR_OBJECT (render, "Failed to create pipeline layout: %s", - error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - return pipeline; -} - -static VkPipelineLayout -_create_pipeline_layout (GstVulkanFullScreenRender * render) -{ - GstVulkanFullScreenRenderClass *render_class = - GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render); - VkPipelineLayoutCreateInfo pipeline_layout_info; - VkPipelineLayout pipeline_layout; - VkPushConstantRange *constants = NULL; - guint n_constants = 0; - GError *error = NULL; - VkResult err; - - if (render_class->push_constant_ranges) - constants = render_class->push_constant_ranges (render, &n_constants); - - /* *INDENT-OFF* */ - pipeline_layout_info = (VkPipelineLayoutCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, - .pNext = NULL, - .setLayoutCount = 1, - .pSetLayouts = (VkDescriptorSetLayout *) &render->descriptor_set_layout->handle, - .pushConstantRangeCount = n_constants, - .pPushConstantRanges = constants, - }; - /* *INDENT-ON* */ - - err = - vkCreatePipelineLayout (render->device->device, - &pipeline_layout_info, NULL, &pipeline_layout); - g_free (constants); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreatePipelineLayout") < 0) { - GST_ERROR_OBJECT (render, "Failed to create pipeline layout: %s", - error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - return pipeline_layout; -} - -static VkRenderPass -_create_render_pass (GstVulkanFullScreenRender * render) -{ - GstVulkanFullScreenRenderClass *render_class = - GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render); - - guint n_descriptions; - VkAttachmentDescription *descriptions = - render_class->render_pass_attachment_descriptions (render, - &n_descriptions); - - guint n_refs; - VkAttachmentReference *color_attachment_refs = - render_class->render_pass_attachment_references (render, &n_refs); - - /* *INDENT-OFF* */ - VkSubpassDescription subpass = { - .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, - .colorAttachmentCount = n_refs, - .pColorAttachments = color_attachment_refs - }; - - VkRenderPassCreateInfo render_pass_info = { - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, - .pNext = NULL, - .attachmentCount = n_descriptions, - .pAttachments = descriptions, - .subpassCount = 1, - .pSubpasses = &subpass - }; - /* *INDENT-ON* */ - VkRenderPass render_pass = VK_NULL_HANDLE; - VkResult err; - GError *error = NULL; - - err = - vkCreateRenderPass (render->device->device, &render_pass_info, NULL, - &render_pass); - g_free (color_attachment_refs); - g_free (descriptions); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateRenderPass") < 0) { - GST_ERROR_OBJECT (render, "Failed to create renderpass: %s", - error->message); - return VK_NULL_HANDLE; - } - - return render_pass; -} - -static GstVulkanHandle * -_create_descriptor_set_layout (GstVulkanFullScreenRender * render) -{ - GstVulkanFullScreenRenderClass *render_class = - GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render); - guint n_bindings; - VkDescriptorSetLayoutBinding *bindings = - render_class->descriptor_set_layout_bindings (render, &n_bindings); - - /* *INDENT-OFF* */ - VkDescriptorSetLayoutCreateInfo layout_info = { - .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, - .pNext = NULL, - .bindingCount = n_bindings, - .pBindings = bindings - }; - /* *INDENT-ON* */ - VkDescriptorSetLayout descriptor_set_layout; - GstVulkanHandle *ret; - VkResult err; - GError *error = NULL; - - err = - vkCreateDescriptorSetLayout (render->device->device, &layout_info, - NULL, &descriptor_set_layout); - g_free (bindings); - if (gst_vulkan_error_to_g_error (err, &error, - "vkCreateDescriptorSetLayout") < 0) { - GST_ERROR_OBJECT (render, "Failed to create renderpass: %s", - error->message); - return VK_NULL_HANDLE; - } - - ret = gst_vulkan_handle_new_wrapped (render->device, - GST_VULKAN_HANDLE_TYPE_DESCRIPTOR_SET_LAYOUT, - (GstVulkanHandleTypedef) descriptor_set_layout, - gst_vulkan_handle_free_descriptor_set_layout, NULL); - - return ret; -} - -static gboolean -gst_vulkan_full_screen_render_set_caps (GstBaseTransform * bt, - GstCaps * in_caps, GstCaps * out_caps) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - GstVulkanFence *last_fence; - - if (!gst_video_info_from_caps (&render->in_info, in_caps)) - return FALSE; - if (!gst_video_info_from_caps (&render->out_info, out_caps)) - return FALSE; - - gst_caps_replace (&render->in_caps, in_caps); - gst_caps_replace (&render->out_caps, out_caps); - - if (render->last_fence) - last_fence = gst_vulkan_fence_ref (render->last_fence); - else - last_fence = gst_vulkan_fence_new_always_signalled (render->device); - - if (render->descriptor_set_layout) { - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (last_fence, - (GstMiniObject *) render->descriptor_set_layout)); - render->descriptor_set_layout = NULL; - } - if (render->pipeline_layout) { - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_pipeline_layout (last_fence, - render->pipeline_layout)); - render->pipeline_layout = VK_NULL_HANDLE; - } - if (render->render_pass) { - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_render_pass (last_fence, - render->render_pass)); - render->render_pass = VK_NULL_HANDLE; - } - if (render->graphics_pipeline) { - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_pipeline (last_fence, - render->graphics_pipeline)); - render->graphics_pipeline = VK_NULL_HANDLE; - } - - gst_vulkan_fence_unref (last_fence); - - if (!(render->descriptor_set_layout = _create_descriptor_set_layout (render))) - return FALSE; - if (!(render->pipeline_layout = _create_pipeline_layout (render))) - return FALSE; - if (!(render->render_pass = _create_render_pass (render))) - return FALSE; - if (!(render->graphics_pipeline = _create_pipeline (render))) - return FALSE; - - GST_DEBUG_OBJECT (bt, "set caps: %" GST_PTR_FORMAT, in_caps); - - return TRUE; -} - -static gboolean -gst_vulkan_full_screen_render_propose_allocation (GstBaseTransform * bt, - GstQuery * decide_query, GstQuery * query) -{ - /* FIXME: */ - return FALSE; -} - -static gboolean -gst_vulkan_full_screen_render_decide_allocation (GstBaseTransform * bt, - GstQuery * query) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - GstBufferPool *pool = NULL; - GstStructure *config; - GstCaps *caps; - guint min, max, size; - gboolean update_pool; - - gst_query_parse_allocation (query, &caps, NULL); - if (!caps) - return FALSE; - - if (gst_query_get_n_allocation_pools (query) > 0) { - gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); - - update_pool = TRUE; - } else { - GstVideoInfo vinfo; - - gst_video_info_init (&vinfo); - gst_video_info_from_caps (&vinfo, caps); - size = vinfo.size; - min = max = 0; - update_pool = FALSE; - } - - if (!pool || !GST_IS_VULKAN_IMAGE_BUFFER_POOL (pool)) { - if (pool) - gst_object_unref (pool); - pool = gst_vulkan_image_buffer_pool_new (render->device); - } - - config = gst_buffer_pool_get_config (pool); - - gst_buffer_pool_config_set_params (config, caps, size, min, max); - gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); - - gst_buffer_pool_set_config (pool, config); - - if (update_pool) - gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); - else - gst_query_add_allocation_pool (query, pool, size, min, max); - - gst_object_unref (pool); - - return TRUE; -} - -static gboolean -_create_vertex_buffers (GstVulkanFullScreenRender * render) -{ - GstMapInfo map_info; - - render->vertices = - gst_vulkan_buffer_memory_alloc (render->device, sizeof (vertices), - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - - if (!gst_memory_map (render->vertices, &map_info, GST_MAP_WRITE)) { - gst_memory_unref (render->vertices); - render->vertices = NULL; - return FALSE; - } - memcpy (map_info.data, vertices, sizeof (vertices)); - gst_memory_unmap (render->vertices, &map_info); - - render->indices = - gst_vulkan_buffer_memory_alloc (render->device, sizeof (indices), - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - - if (!gst_memory_map (render->indices, &map_info, GST_MAP_WRITE)) { - gst_memory_unref (render->vertices); - render->vertices = NULL; - gst_memory_unref (render->indices); - render->indices = NULL; - return FALSE; - } - memcpy (map_info.data, indices, sizeof (indices)); - gst_memory_unmap (render->indices, &map_info); - - return TRUE; -} - -static gboolean -gst_vulkan_full_screen_render_start (GstBaseTransform * bt) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - - if (!gst_vulkan_ensure_element_data (GST_ELEMENT (bt), NULL, - &render->instance)) { - GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND, - ("Failed to retrieve vulkan instance"), (NULL)); - return FALSE; - } - if (!gst_vulkan_device_run_context_query (GST_ELEMENT (render), - &render->device)) { - GError *error = NULL; - GST_DEBUG_OBJECT (render, "No device retrieved from peer elements"); - if (!(render->device = - gst_vulkan_instance_create_device (render->instance, &error))) { - GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND, - ("Failed to create vulkan device"), ("%s", error->message)); - g_clear_error (&error); - return FALSE; - } - } - - if (!gst_vulkan_queue_run_context_query (GST_ELEMENT (render), - &render->queue)) { - GST_DEBUG_OBJECT (render, "No queue retrieved from peer elements"); - render->queue = _find_graphics_queue (render); - } - if (!render->queue) - return FALSE; - - if (!_create_vertex_buffers (render)) - return FALSE; - - render->trash_list = gst_vulkan_trash_fence_list_new (); - - return TRUE; -} - -static gboolean -gst_vulkan_full_screen_render_stop (GstBaseTransform * bt) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - - if (render->device) { - GstVulkanFence *last_fence; - - if (render->last_fence) - last_fence = gst_vulkan_fence_ref (render->last_fence); - else - last_fence = gst_vulkan_fence_new_always_signalled (render->device); - - if (render->graphics_pipeline) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_pipeline (last_fence, - render->graphics_pipeline)); - render->graphics_pipeline = VK_NULL_HANDLE; - if (render->pipeline_layout) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_pipeline_layout (last_fence, - render->pipeline_layout)); - render->pipeline_layout = VK_NULL_HANDLE; - if (render->render_pass) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_render_pass (last_fence, - render->render_pass)); - render->render_pass = VK_NULL_HANDLE; - if (render->descriptor_set_layout) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (last_fence, - (GstMiniObject *) render->descriptor_set_layout)); - render->descriptor_set_layout = NULL; - - gst_vulkan_fence_unref (last_fence); - - if (render->last_fence) - gst_vulkan_fence_unref (render->last_fence); - render->last_fence = NULL; - - if (!gst_vulkan_trash_list_wait (render->trash_list, -1)) - GST_WARNING_OBJECT (render, - "Failed to wait for all resources to be freed"); - gst_object_unref (render->trash_list); - render->trash_list = NULL; - - if (render->vertices) - gst_memory_unref (render->vertices); - render->vertices = NULL; - - if (render->indices) - gst_memory_unref (render->indices); - render->indices = NULL; - - gst_object_unref (render->device); - } - render->device = NULL; - - if (render->queue) - gst_object_unref (render->queue); - render->queue = NULL; - - if (render->instance) - gst_object_unref (render->instance); - render->instance = NULL; - - return TRUE; -} - -/** - * gst_vulkan_full_screen_render_fill_command_buffer: - * @render: a #GstVulkanFullScreenRender - * @cmd: a `VkCommandBuffer` - * @framebuffer: a `VkFramebuffer` - * - * Returns: whether @cmd could be filled with the commands necessary to render - */ -gboolean -gst_vulkan_full_screen_render_fill_command_buffer (GstVulkanFullScreenRender * - render, VkCommandBuffer cmd, VkFramebuffer framebuffer) -{ - /* *INDENT-OFF* */ - VkClearValue clearColor = {{{ 0.0f, 0.0f, 0.0f, 1.0f }}}; - VkClearValue clearColors[GST_VIDEO_MAX_PLANES] = { - clearColor, clearColor, clearColor, clearColor, - }; - VkRenderPassBeginInfo render_pass_info = { - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, - .renderPass = render->render_pass, - .framebuffer = framebuffer, - .renderArea.offset = { 0, 0 }, - .renderArea.extent = { - GST_VIDEO_INFO_WIDTH (&render->out_info), - GST_VIDEO_INFO_HEIGHT (&render->out_info) - }, - .clearValueCount = GST_VIDEO_INFO_N_PLANES (&render->out_info), - .pClearValues = clearColors, - }; - /* *INDENT-ON* */ - VkDeviceSize offsets[] = { 0 }; - - vkCmdBeginRenderPass (cmd, &render_pass_info, VK_SUBPASS_CONTENTS_INLINE); - vkCmdBindPipeline (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, - render->graphics_pipeline); - vkCmdBindVertexBuffers (cmd, 0, 1, - &((GstVulkanBufferMemory *) render->vertices)->buffer, offsets); - vkCmdBindIndexBuffer (cmd, - ((GstVulkanBufferMemory *) render->indices)->buffer, 0, - VK_INDEX_TYPE_UINT16); - vkCmdDrawIndexed (cmd, G_N_ELEMENTS (indices), 1, 0, 0, 0); - vkCmdEndRenderPass (cmd); - - return TRUE; -} - -gboolean -gst_vulkan_full_screen_render_submit (GstVulkanFullScreenRender * render, - VkCommandBuffer cmd, GstVulkanFence * fence) -{ - VkSubmitInfo submit_info; - GError *error = NULL; - VkResult err; - - if (!fence) - fence = gst_vulkan_fence_new (render->device, 0, &error); - if (!fence) - goto error; - - /* *INDENT-OFF* */ - submit_info = (VkSubmitInfo) { - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, - .pNext = NULL, - .waitSemaphoreCount = 0, - .pWaitSemaphores = NULL, - .pWaitDstStageMask = NULL, - .commandBufferCount = 1, - .pCommandBuffers = &cmd, - .signalSemaphoreCount = 0, - .pSignalSemaphores = NULL, - }; - /* *INDENT-ON* */ - - if (render->last_fence) - gst_vulkan_fence_unref (render->last_fence); - render->last_fence = gst_vulkan_fence_ref (fence); - - gst_vulkan_queue_submit_lock (render->queue); - err = - vkQueueSubmit (render->queue->queue, 1, &submit_info, - GST_VULKAN_FENCE_FENCE (fence)); - gst_vulkan_queue_submit_unlock (render->queue); - if (gst_vulkan_error_to_g_error (err, &error, "vkQueueSubmit") < 0) - goto error; - - gst_vulkan_trash_list_gc (render->trash_list); - - gst_vulkan_fence_unref (fence); - - return TRUE; - -error: - GST_ELEMENT_ERROR (render, LIBRARY, FAILED, ("%s", error->message), (NULL)); - g_clear_error (&error); - return FALSE; -} diff --git a/ext/vulkan/vkfullscreenrender.h b/ext/vulkan/vkfullscreenrender.h deleted file mode 100644 index 1e01812626..0000000000 --- a/ext/vulkan/vkfullscreenrender.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * GStreamer - * Copyright (C) 2019 Matthew Waters - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#ifndef _VK_FULL_SCREEN_RENDER_H_ -#define _VK_FULL_SCREEN_RENDER_H_ - -#include -#include -#include - -G_BEGIN_DECLS - -#define GST_TYPE_VULKAN_FULL_SCREEN_RENDER (gst_vulkan_full_screen_render_get_type()) -#define GST_VULKAN_FULL_SCREEN_RENDER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRender)) -#define GST_VULKAN_FULL_SCREEN_RENDER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRenderClass)) -#define GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRenderClass)) -#define GST_IS_VULKAN_FULL_SCREEN_RENDER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER)) -#define GST_IS_VULKAN_FULL_SCREEN_RENDER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_FULL_SCREEN_RENDER)) - -typedef struct _GstVulkanFullScreenRender GstVulkanFullScreenRender; -typedef struct _GstVulkanFullScreenRenderClass GstVulkanFullScreenRenderClass; - -typedef void (*GstVulkanFullScreenRenderDestroyInfoNotify) - (GstVulkanFullScreenRender * render, gpointer info); - -struct Vertex -{ - gfloat x, y, z; - gfloat s, t; -}; - -struct _GstVulkanFullScreenRender -{ - GstBaseTransform parent; - - GstVulkanInstance *instance; - GstVulkanDevice *device; - GstVulkanQueue *queue; - - GstCaps *in_caps; - GstVideoInfo in_info; - GstCaps *out_caps; - GstVideoInfo out_info; - - guint n_shader_stages; - VkPipelineShaderStageCreateInfo *shader_create_info; - GstVulkanFullScreenRenderDestroyInfoNotify destroy_shader_create_info; - - VkRenderPass render_pass; - VkPipelineLayout pipeline_layout; - VkPipeline graphics_pipeline; - GstVulkanHandle *descriptor_set_layout; - - GstMemory *vertices; - GstMemory *indices; - - GstVulkanTrashList *trash_list; - GstVulkanFence *last_fence; -}; - -struct _GstVulkanFullScreenRenderClass -{ - GstBaseTransformClass video_sink_class; - - void (*shader_create_info) (GstVulkanFullScreenRender * render); - VkDescriptorSetLayoutBinding * (*descriptor_set_layout_bindings) (GstVulkanFullScreenRender * render, guint * n_bindings); - VkAttachmentReference * (*render_pass_attachment_references) (GstVulkanFullScreenRender * render, guint * n_refs); - VkAttachmentDescription * (*render_pass_attachment_descriptions) (GstVulkanFullScreenRender * render, guint * n_descriptions); - VkPushConstantRange * (*push_constant_ranges) (GstVulkanFullScreenRender * render, guint * n_constants); -}; - -gboolean gst_vulkan_full_screen_render_fill_command_buffer (GstVulkanFullScreenRender * render, VkCommandBuffer cmd, VkFramebuffer framebuffer); -gboolean gst_vulkan_full_screen_render_submit (GstVulkanFullScreenRender * render, VkCommandBuffer cmd, GstVulkanFence *fence); - -GType gst_vulkan_full_screen_render_get_type(void); - -G_END_DECLS - -#endif diff --git a/ext/vulkan/vkimageidentity.c b/ext/vulkan/vkimageidentity.c index 0844640096..e1c471f90c 100644 --- a/ext/vulkan/vkimageidentity.c +++ b/ext/vulkan/vkimageidentity.c @@ -44,26 +44,11 @@ GST_DEBUG_CATEGORY (gst_debug_vulkan_image_identity); static gboolean gst_vulkan_image_identity_start (GstBaseTransform * bt); static gboolean gst_vulkan_image_identity_stop (GstBaseTransform * bt); -static GstCaps *gst_vulkan_image_identity_transform_caps (GstBaseTransform * bt, - GstPadDirection direction, GstCaps * caps, GstCaps * filter); static GstFlowReturn gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer * outbuf); static gboolean gst_vulkan_image_identity_set_caps (GstBaseTransform * bt, GstCaps * in_caps, GstCaps * out_caps); -static VkAttachmentReference - * gst_vulkan_image_identity_render_pass_attachment_references - (GstVulkanFullScreenRender * render, guint * n_attachments); -static VkAttachmentDescription - * gst_vulkan_image_identity_render_pass_attachment_descriptions - (GstVulkanFullScreenRender * render, guint * n_descriptions); -static VkDescriptorSetLayoutBinding - * gst_vulkan_image_identity_descriptor_set_layout_bindings - (GstVulkanFullScreenRender * render, guint * n_bindings); -static void -gst_vulkan_image_identity_shader_create_info (GstVulkanFullScreenRender * - render); - #define IMAGE_FORMATS " { BGRA }" static GstStaticPadTemplate gst_vulkan_sink_template = @@ -97,7 +82,7 @@ enum #define gst_vulkan_image_identity_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstVulkanImageIdentity, gst_vulkan_image_identity, - GST_TYPE_VULKAN_FULL_SCREEN_RENDER, + GST_TYPE_VULKAN_VIDEO_FILTER, GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_image_identity, "vulkanimageidentity", 0, "Vulkan Image identity")); @@ -106,13 +91,11 @@ gst_vulkan_image_identity_class_init (GstVulkanImageIdentityClass * klass) { GstElementClass *gstelement_class; GstBaseTransformClass *gstbasetransform_class; - GstVulkanFullScreenRenderClass *fullscreenrender_class; gstelement_class = (GstElementClass *) klass; gstbasetransform_class = (GstBaseTransformClass *) klass; - fullscreenrender_class = (GstVulkanFullScreenRenderClass *) klass; - gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader", + gst_element_class_set_metadata (gstelement_class, "Vulkan Image Identity", "Filter/Video", "A Vulkan image copier", "Matthew Waters "); @@ -125,19 +108,8 @@ gst_vulkan_image_identity_class_init (GstVulkanImageIdentityClass * klass) GST_DEBUG_FUNCPTR (gst_vulkan_image_identity_start); gstbasetransform_class->stop = GST_DEBUG_FUNCPTR (gst_vulkan_image_identity_stop); - gstbasetransform_class->transform_caps = - gst_vulkan_image_identity_transform_caps; gstbasetransform_class->set_caps = gst_vulkan_image_identity_set_caps; gstbasetransform_class->transform = gst_vulkan_image_identity_transform; - - fullscreenrender_class->render_pass_attachment_references = - gst_vulkan_image_identity_render_pass_attachment_references; - fullscreenrender_class->render_pass_attachment_descriptions = - gst_vulkan_image_identity_render_pass_attachment_descriptions; - fullscreenrender_class->descriptor_set_layout_bindings = - gst_vulkan_image_identity_descriptor_set_layout_bindings; - fullscreenrender_class->shader_create_info = - gst_vulkan_image_identity_shader_create_info; } static void @@ -145,535 +117,86 @@ gst_vulkan_image_identity_init (GstVulkanImageIdentity * vk_identity) { } -static GstCaps * -gst_vulkan_image_identity_transform_caps (GstBaseTransform * bt, - GstPadDirection direction, GstCaps * caps, GstCaps * filter) -{ - GstCaps *result, *tmp; - - tmp = gst_caps_copy (caps); - - if (filter) { - result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); - gst_caps_unref (tmp); - } else { - result = tmp; - } - - return result; -} - -static void -destroy_shader_create_info (GstVulkanFullScreenRender * render, gpointer data) -{ - VkPipelineShaderStageCreateInfo *info = data; - int i; - - for (i = 0; i < render->n_shader_stages; i++) { - vkDestroyShaderModule (render->device->device, info[i].module, NULL); - } - - g_free (info); -} - -static void -gst_vulkan_image_identity_shader_create_info (GstVulkanFullScreenRender * - render) -{ - VkShaderModule vert_module, frag_module; - - vert_module = - _vk_create_shader (render->device, identity_vert, identity_vert_size, - NULL); - frag_module = - _vk_create_shader (render->device, identity_frag, identity_frag_size, - NULL); - - render->n_shader_stages = 2; - render->shader_create_info = - g_new0 (VkPipelineShaderStageCreateInfo, render->n_shader_stages); - render->destroy_shader_create_info = destroy_shader_create_info; - - /* *INDENT-OFF* */ - render->shader_create_info[0] = (VkPipelineShaderStageCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, - .pNext = NULL, - .stage = VK_SHADER_STAGE_VERTEX_BIT, - .module = vert_module, - .pName = "main" - }; - - render->shader_create_info[1] = (VkPipelineShaderStageCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, - .pNext = NULL, - .stage = VK_SHADER_STAGE_FRAGMENT_BIT, - .module = frag_module, - .pName = "main" - }; -} - -static VkDescriptorSetLayoutBinding * -gst_vulkan_image_identity_descriptor_set_layout_bindings (GstVulkanFullScreenRender * render, guint * n_bindings) -{ - VkDescriptorSetLayoutBinding *bindings; - - *n_bindings = 1; - bindings = g_new0 (VkDescriptorSetLayoutBinding, *n_bindings); - - /* *INDENT-OFF* */ - bindings[0] = (VkDescriptorSetLayoutBinding) { - .binding = 0, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .pImmutableSamplers = NULL, - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT - }; - /* *INDENT-ON* */ - - return bindings; -} - -static VkAttachmentReference - * gst_vulkan_image_identity_render_pass_attachment_references - (GstVulkanFullScreenRender * render, guint * n_attachments) -{ - VkAttachmentReference *attachments; - - *n_attachments = 1; - attachments = g_new0 (VkAttachmentReference, *n_attachments); - /* *INDENT-OFF* */ - attachments[0] = (VkAttachmentReference) { - .attachment = 0, - .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL - }; - /* *INDENT-ON* */ - - return attachments; -} - -static VkAttachmentDescription - * gst_vulkan_image_identity_render_pass_attachment_descriptions - (GstVulkanFullScreenRender * render, guint * n_descriptions) -{ - VkAttachmentDescription *color_attachments; - - *n_descriptions = 1; - color_attachments = g_new0 (VkAttachmentDescription, *n_descriptions); - /* *INDENT-OFF* */ - color_attachments[0] = (VkAttachmentDescription) { - .format = gst_vulkan_format_from_video_info (&render->in_info, 0), - .samples = VK_SAMPLE_COUNT_1_BIT, - .loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR, - .storeOp = VK_ATTACHMENT_STORE_OP_STORE, - .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE, - .stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE, - /* FIXME: share this between elements to avoid pipeline barriers */ - .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, - .finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL - }; - /* *INDENT-ON* */ - - return color_attachments; -} - -static GstVulkanDescriptorCache * -_create_descriptor_pool (GstVulkanImageIdentity * vk_identity) -{ - GstVulkanFullScreenRender *render = - GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); - guint max_sets = 32; /* FIXME: Don't hardcode this! */ - - /* *INDENT-OFF* */ - VkDescriptorPoolSize pool_sizes = { - .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = max_sets - }; - - VkDescriptorPoolCreateInfo pool_info = { - .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, - .pNext = NULL, - .poolSizeCount = 1, - .pPoolSizes = &pool_sizes, - .maxSets = max_sets - }; - /* *INDENT-ON* */ - VkDescriptorPool pool; - GstVulkanDescriptorPool *ret; - GstVulkanDescriptorCache *cache; - GError *error = NULL; - VkResult err; - - err = - vkCreateDescriptorPool (render->device->device, &pool_info, NULL, &pool); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateDescriptorPool") < 0) { - GST_ERROR_OBJECT (render, "Failed to create descriptor pool: %s", - error->message); - g_clear_error (&error); - return NULL; - } - - ret = gst_vulkan_descriptor_pool_new_wrapped (render->device, pool, max_sets); - cache = - gst_vulkan_descriptor_cache_new (ret, 1, &render->descriptor_set_layout); - gst_object_unref (ret); - - return cache; -} - static gboolean gst_vulkan_image_identity_set_caps (GstBaseTransform * bt, GstCaps * in_caps, GstCaps * out_caps) { + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt); GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - GstVulkanFence *last_fence; if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps, out_caps)) return FALSE; - if (render->last_fence) - last_fence = gst_vulkan_fence_ref (render->last_fence); - else - last_fence = gst_vulkan_fence_new_always_signalled (render->device); - - if (vk_identity->descriptor_pool) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_object_unref (last_fence, - (GstObject *) vk_identity->descriptor_pool)); - vk_identity->descriptor_pool = NULL; - - gst_vulkan_fence_unref (last_fence); - - if (!(vk_identity->descriptor_pool = _create_descriptor_pool (vk_identity))) + if (!gst_vulkan_full_screen_quad_set_info (vk_identity->quad, + &vfilter->in_info, &vfilter->out_info)) return FALSE; return TRUE; } -static VkSampler -_create_sampler (GstVulkanImageIdentity * vk_identity) -{ - GstVulkanFullScreenRender *render = - GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); - - /* *INDENT-OFF* */ - VkSamplerCreateInfo samplerInfo = { - .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, - .magFilter = VK_FILTER_LINEAR, - .minFilter = VK_FILTER_LINEAR, - .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .anisotropyEnable = VK_FALSE, - .maxAnisotropy = 1, - .borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK, - .unnormalizedCoordinates = VK_FALSE, - .compareEnable = VK_FALSE, - .compareOp = VK_COMPARE_OP_ALWAYS, - .mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR, - .mipLodBias = 0.0f, - .minLod = 0.0f, - .maxLod = 0.0f - }; - /* *INDENT-ON* */ - GError *error = NULL; - VkSampler sampler; - VkResult err; - - err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) { - GST_ERROR_OBJECT (vk_identity, "Failed to create sampler: %s", - error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - return sampler; -} - static gboolean gst_vulkan_image_identity_start (GstBaseTransform * bt) { GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (vk_identity); + GstVulkanHandle *vert, *frag; + GError *error = NULL; if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt)) return FALSE; - if (!(vk_identity->sampler = _create_sampler (vk_identity))) - return FALSE; + vk_identity->quad = gst_vulkan_full_screen_quad_new (vfilter->queue); + + if (!(vert = _vk_create_shader (vfilter->device, identity_vert, + identity_vert_size, &error))) + goto error; + if (!(frag = _vk_create_shader (vfilter->device, identity_frag, + identity_frag_size, &error))) { + gst_vulkan_handle_unref (vert); + goto error; + } + gst_vulkan_full_screen_quad_set_shaders (vk_identity->quad, vert, frag); + + gst_vulkan_handle_unref (vert); + gst_vulkan_handle_unref (frag); return TRUE; + +error: + GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, ("%s", error->message), (NULL)); + return FALSE; } static gboolean gst_vulkan_image_identity_stop (GstBaseTransform * bt) { GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - if (render->device) { - GstVulkanFence *last_fence; - - if (render->last_fence) - last_fence = gst_vulkan_fence_ref (render->last_fence); - else - last_fence = gst_vulkan_fence_new_always_signalled (render->device); - - if (vk_identity->descriptor_pool) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_object_unref (last_fence, - (GstObject *) vk_identity->descriptor_pool)); - vk_identity->descriptor_pool = NULL; - if (vk_identity->sampler) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_sampler (last_fence, vk_identity->sampler)); - vk_identity->sampler = VK_NULL_HANDLE; - - gst_vulkan_fence_unref (last_fence); - } - - if (vk_identity->cmd_pool) - gst_object_unref (vk_identity->cmd_pool); - vk_identity->cmd_pool = VK_NULL_HANDLE; + gst_clear_object (&vk_identity->quad); return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt); } -static void -update_descriptor_set (GstVulkanImageIdentity * vk_identity, - VkDescriptorSet set, VkImageView view) -{ - GstVulkanFullScreenRender *render = - GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); - - /* *INDENT-OFF* */ - VkDescriptorImageInfo image_info = { - .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - .imageView = view, - .sampler = vk_identity->sampler - }; - - VkWriteDescriptorSet writes = { - .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, - .pNext = NULL, - .dstSet = set, - .dstBinding = 0, - .dstArrayElement = 0, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = 1, - .pImageInfo = &image_info - }; - /* *INDENT-ON* */ - vkUpdateDescriptorSets (render->device->device, 1, &writes, 0, NULL); -} - -static VkFramebuffer -_create_framebuffer (GstVulkanImageIdentity * vk_identity, VkImageView view) -{ - GstVulkanFullScreenRender *render = - GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); - - /* *INDENT-OFF* */ - VkImageView attachments[] = { - view, - }; - VkFramebufferCreateInfo framebuffer_info = { - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = NULL, - .renderPass = render->render_pass, - .attachmentCount = 1, - .pAttachments = attachments, - .width = GST_VIDEO_INFO_WIDTH (&render->in_info), - .height = GST_VIDEO_INFO_HEIGHT (&render->in_info), - .layers = 1 - }; - /* *INDENT-ON* */ - VkFramebuffer framebuffer; - GError *error = NULL; - VkResult err; - - err = - vkCreateFramebuffer (render->device->device, &framebuffer_info, NULL, - &framebuffer); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateFramebuffer") < 0) { - GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s", - error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - return framebuffer; -} - static GstFlowReturn gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer * outbuf) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); - GstVulkanImageMemory *in_img_mem, *out_img_mem; - GstVulkanImageView *in_img_view, *out_img_view; - GstVulkanFence *fence = NULL; - GstVulkanDescriptorSet *set; - GstMemory *in_mem, *out_mem; - VkFramebuffer framebuffer; - GstVulkanCommandBuffer *cmd_buf; GError *error = NULL; - VkResult err; - fence = gst_vulkan_fence_new (render->device, 0, &error); - if (!fence) + if (!gst_vulkan_full_screen_quad_set_input_buffer (vk_identity->quad, inbuf, + &error)) + goto error; + if (!gst_vulkan_full_screen_quad_set_output_buffer (vk_identity->quad, outbuf, + &error)) goto error; - in_mem = gst_buffer_peek_memory (inbuf, 0); - if (!gst_is_vulkan_image_memory (in_mem)) { - g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, - "Input memory must be a GstVulkanImageMemory"); + if (!gst_vulkan_full_screen_quad_draw (vk_identity->quad, &error)) goto error; - } - in_img_mem = (GstVulkanImageMemory *) in_mem; - in_img_view = get_or_create_image_view (in_img_mem); - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - GST_MINI_OBJECT_CAST (in_img_view))); - - out_mem = gst_buffer_peek_memory (outbuf, 0); - if (!gst_is_vulkan_image_memory (out_mem)) { - g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, - "Input memory must be a GstVulkanImageMemory"); - goto error; - } - out_img_mem = (GstVulkanImageMemory *) out_mem; - out_img_view = get_or_create_image_view (out_img_mem); - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - GST_MINI_OBJECT_CAST (out_img_view))); - - if (!vk_identity->cmd_pool) { - if (!(vk_identity->cmd_pool = - gst_vulkan_queue_create_command_pool (render->queue, &error))) - goto error; - } - if (!(set = - gst_vulkan_descriptor_cache_acquire (vk_identity->descriptor_pool, - &error))) - goto error; - update_descriptor_set (vk_identity, set->set, in_img_view->view); - - if (!(cmd_buf = - gst_vulkan_command_pool_create (vk_identity->cmd_pool, &error))) - goto error; - - if (!(framebuffer = _create_framebuffer (vk_identity, out_img_view->view))) { - g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, - "Failed to create framebuffer"); - goto error; - } - - - { - VkCommandBufferBeginInfo cmd_buf_info = { 0, }; - - /* *INDENT-OFF* */ - cmd_buf_info = (VkCommandBufferBeginInfo) { - .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, - .pNext = NULL, - .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, - .pInheritanceInfo = NULL - }; - /* *INDENT-ON* */ - - gst_vulkan_command_buffer_lock (cmd_buf); - err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info); - if (gst_vulkan_error_to_g_error (err, &error, "vkBeginCommandBuffer") < 0) - goto unlock_error; - } - - { - /* *INDENT-OFF* */ - VkImageMemoryBarrier in_image_memory_barrier = { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = NULL, - .srcAccessMask = in_img_mem->barrier.parent.access_flags, - .dstAccessMask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT, - .oldLayout = in_img_mem->barrier.image_layout, - .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - /* FIXME: implement exclusive transfers */ - .srcQueueFamilyIndex = 0, - .dstQueueFamilyIndex = 0, - .image = in_img_mem->image, - .subresourceRange = in_img_mem->barrier.subresource_range - }; - - VkImageMemoryBarrier out_image_memory_barrier = { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = NULL, - .srcAccessMask = out_img_mem->barrier.parent.access_flags, - .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, - .oldLayout = out_img_mem->barrier.image_layout, - .newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, - /* FIXME: implement exclusive transfers */ - .srcQueueFamilyIndex = 0, - .dstQueueFamilyIndex = 0, - .image = out_img_mem->image, - .subresourceRange = out_img_mem->barrier.subresource_range - }; - /* *INDENT-ON* */ - - vkCmdPipelineBarrier (cmd_buf->cmd, - in_img_mem->barrier.parent.pipeline_stages, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, NULL, 0, NULL, 1, - &in_image_memory_barrier); - - in_img_mem->barrier.parent.pipeline_stages = - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; - in_img_mem->barrier.parent.access_flags = - in_image_memory_barrier.dstAccessMask; - in_img_mem->barrier.image_layout = in_image_memory_barrier.newLayout; - - vkCmdPipelineBarrier (cmd_buf->cmd, - out_img_mem->barrier.parent.pipeline_stages, - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, NULL, 0, NULL, 1, - &out_image_memory_barrier); - - out_img_mem->barrier.parent.pipeline_stages = - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; - out_img_mem->barrier.parent.access_flags = - out_image_memory_barrier.dstAccessMask; - out_img_mem->barrier.image_layout = out_image_memory_barrier.newLayout; - } - - vkCmdBindDescriptorSets (cmd_buf->cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, - render->pipeline_layout, 0, 1, &set->set, 0, NULL); - if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd_buf->cmd, - framebuffer)) - goto unlock_error; - - err = vkEndCommandBuffer (cmd_buf->cmd); - gst_vulkan_command_buffer_unlock (cmd_buf); - if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0) - goto error; - - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - GST_MINI_OBJECT_CAST (set))); - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_framebuffer (fence, framebuffer)); - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - GST_MINI_OBJECT_CAST (cmd_buf))); - - if (!gst_vulkan_full_screen_render_submit (render, cmd_buf->cmd, fence)) - return GST_FLOW_ERROR; return GST_FLOW_OK; -unlock_error: - if (cmd_buf) { - gst_vulkan_command_buffer_unlock (cmd_buf); - gst_vulkan_command_buffer_unref (cmd_buf); - } error: GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL)); g_clear_error (&error); diff --git a/ext/vulkan/vkimageidentity.h b/ext/vulkan/vkimageidentity.h index 5ab1de4f05..c24e5d8167 100644 --- a/ext/vulkan/vkimageidentity.h +++ b/ext/vulkan/vkimageidentity.h @@ -24,7 +24,7 @@ #include #include #include -#include "vkfullscreenrender.h" +#include "vkfullscreenquad.h" G_BEGIN_DECLS @@ -39,20 +39,15 @@ typedef struct _GstVulkanImageIdentityClass GstVulkanImageIdentityClass; struct _GstVulkanImageIdentity { - GstVulkanFullScreenRender parent; + GstVulkanVideoFilter parent; - GstVulkanCommandPool *cmd_pool; - - VkSampler sampler; - GstVulkanDescriptorCache *descriptor_pool; - - VkDescriptorSetLayoutBinding sampler_layout_binding; - VkDescriptorSetLayoutCreateInfo layout_info; + GstVulkanFullScreenQuad *quad; + GstMemory *uniforms; }; struct _GstVulkanImageIdentityClass { - GstVulkanFullScreenRenderClass parent_class; + GstVulkanVideoFilterClass parent_class; }; GType gst_vulkan_image_identity_get_type(void); diff --git a/ext/vulkan/vkshader.c b/ext/vulkan/vkshader.c index 2f9bbf5470..54a7862822 100644 --- a/ext/vulkan/vkshader.c +++ b/ext/vulkan/vkshader.c @@ -27,11 +27,11 @@ #define SPIRV_MAGIC_NUMBER_NE 0x07230203 #define SPIRV_MAGIC_NUMBER_OE 0x03022307 -VkShaderModule +GstVulkanHandle * _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size, GError ** error) { - VkShaderModule ret; + VkShaderModule shader; VkResult res; /* *INDENT-OFF* */ @@ -49,7 +49,7 @@ _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size, g_return_val_if_fail (size >= 4, VK_NULL_HANDLE); g_return_val_if_fail (size % 4 == 0, VK_NULL_HANDLE); - first_word = ((guint32 *) code)[0]; + first_word = code[0] | code[1] << 8 | code[2] << 16 | code[3] << 24; g_return_val_if_fail (first_word == SPIRV_MAGIC_NUMBER_NE || first_word == SPIRV_MAGIC_NUMBER_OE, VK_NULL_HANDLE); if (first_word == SPIRV_MAGIC_NUMBER_OE) { @@ -78,12 +78,11 @@ _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size, info.pCode = new_code; } - res = vkCreateShaderModule (device->device, &info, NULL, &ret); + res = vkCreateShaderModule (device->device, &info, NULL, &shader); g_free (new_code); if (gst_vulkan_error_to_g_error (res, error, "VkCreateShaderModule") < 0) - return VK_NULL_HANDLE; + return NULL; - g_free (new_code); - - return ret; + return gst_vulkan_handle_new_wrapped (device, GST_VULKAN_HANDLE_TYPE_SHADER, + (GstVulkanHandleTypedef) shader, gst_vulkan_handle_free_shader, NULL); } diff --git a/ext/vulkan/vkshader.h b/ext/vulkan/vkshader.h index 5fa74f250b..b819c48488 100644 --- a/ext/vulkan/vkshader.h +++ b/ext/vulkan/vkshader.h @@ -26,6 +26,6 @@ G_BEGIN_DECLS -VkShaderModule _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size, GError ** error); +GstVulkanHandle * _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size, GError ** error); #endif diff --git a/ext/vulkan/vkviewconvert.c b/ext/vulkan/vkviewconvert.c index a8dc4d9aee..8f58325fe2 100644 --- a/ext/vulkan/vkviewconvert.c +++ b/ext/vulkan/vkviewconvert.c @@ -293,30 +293,48 @@ calculate_reorder_indexes (GstVideoFormat in_format, static void update_descriptor_set (GstVulkanViewConvert * conv, - VkDescriptorSet descriptor_set, VkImageView * views, guint n_views) + GstVulkanImageView ** in_views) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - VkDescriptorBufferInfo buffer_info; - VkDescriptorImageInfo image_info[2 * GST_VIDEO_MAX_PLANES]; - VkWriteDescriptorSet writes[10]; + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv); + VkDescriptorImageInfo image_info[GST_VIDEO_MAX_PLANES]; + VkWriteDescriptorSet writes[GST_VIDEO_MAX_PLANES]; + GstVideoMultiviewMode in_mode; + GstVideoMultiviewFlags in_flags, out_flags; + VkImageView views[GST_VIDEO_MAX_PLANES]; guint i = 0; - for (; i < GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2; i++) { + in_mode = conv->input_mode_override; + in_flags = conv->input_flags_override; + if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) + in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info); + out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info); + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info); i++) { + if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) == + (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) { + views[2 * i] = in_views[i]->view; + views[2 * i + 1] = in_views[i]->view; + } else { + views[2 * i] = in_views[i]->view; + views[2 * i + 1] = in_views[i]->view; + } + } + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info) * 2; i++) { /* *INDENT-OFF* */ image_info[i] = (VkDescriptorImageInfo) { .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .imageView = views[i], - .sampler = conv->sampler + .sampler = (VkSampler) conv->quad->sampler->handle }; - g_assert (i < n_views); g_assert (i < GST_VIDEO_MAX_PLANES); writes[i] = (VkWriteDescriptorSet) { .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .pNext = NULL, - .dstSet = descriptor_set, - .dstBinding = i, + .dstSet = conv->quad->descriptor_set->set, + .dstBinding = i+1, .dstArrayElement = 0, .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .descriptorCount = 1, @@ -324,34 +342,31 @@ update_descriptor_set (GstVulkanViewConvert * conv, }; /* *INDENT-ON* */ } - /* *INDENT-OFF* */ - buffer_info = (VkDescriptorBufferInfo) { - .buffer = ((GstVulkanBufferMemory *) conv->uniform)->buffer, - .offset = 0, - .range = sizeof (struct ViewUpdate), - }; - writes[i] = (VkWriteDescriptorSet) { - .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, - .pNext = NULL, - .dstSet = descriptor_set, - .dstBinding = i, - .dstArrayElement = 0, - .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, - .descriptorCount = 1, - .pBufferInfo = &buffer_info - }; - /* *INDENT-ON* */ - i++; g_assert (i <= G_N_ELEMENTS (writes)); - vkUpdateDescriptorSets (render->device->device, i, writes, 0, NULL); + vkUpdateDescriptorSets (vfilter->device->device, i, writes, 0, NULL); } static gboolean -_update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views, - GstVulkanImageView ** out_views, VkImageView views[GST_VIDEO_MAX_PLANES]) +create_uniform_buffer (GstVulkanViewConvert * conv) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv); + + conv->uniform = + gst_vulkan_buffer_memory_alloc (vfilter->device, + sizeof (struct ViewUpdate), + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + + return TRUE; +} + +static gboolean +update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views, + GstVulkanImageView ** out_views) +{ + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv); GstVideoMultiviewMode in_mode, out_mode; GstVideoMultiviewFlags in_flags, out_flags; struct ViewUpdate data; @@ -359,8 +374,8 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views, guint l_index, r_index; gboolean mono_input = FALSE; - calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), - in_views, GST_VIDEO_INFO_FORMAT (&render->out_info), + calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&vfilter->in_info), + in_views, GST_VIDEO_INFO_FORMAT (&vfilter->out_info), out_views, data.in_reorder_idx, data.out_reorder_idx); data.tex_scale[0][0] = data.tex_scale[0][1] = 1.; @@ -371,29 +386,24 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views, in_mode = conv->input_mode_override; in_flags = conv->input_flags_override; if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) { - in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&render->in_info); - in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&render->in_info); + in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->in_info); + in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info); } /* Configured output mode already takes any override * into account */ - out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&render->out_info); - out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&render->out_info); + out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->out_info); + out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info); if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) == (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) { l_index = 0; r_index = 1; } else { - VkImageView scratch_view; - GST_LOG_OBJECT (conv, "Switching left/right views"); l_index = 1; r_index = 0; - scratch_view = views[0]; - views[0] = views[1]; - views[1] = scratch_view; } if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) { /* unknown/mono/left/right single image */ @@ -470,8 +480,8 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views, data.output_type = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE; } - data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&render->out_info); - data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&render->out_info); + data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&vfilter->out_info); + data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&vfilter->out_info); memcpy (&data.downmix[0], &downmix_matrices[conv->downmix_mode][0], sizeof (data.downmix[0])); memcpy (&data.downmix[1], &downmix_matrices[conv->downmix_mode][1], @@ -482,59 +492,25 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views, } memcpy (map_info.data, &data, sizeof (data)); gst_memory_unmap (conv->uniform, &map_info); - conv->descriptor_up_to_date = TRUE; return TRUE; } -static GstVulkanDescriptorSet * -_create_descriptor_set (GstVulkanViewConvert * conv) +static GstMemory * +get_uniforms (GstVulkanViewConvert * conv, + GstVulkanImageView ** in_views, GstVulkanImageView ** out_views) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - GstVulkanDescriptorSet *ret; - GError *error = NULL; - - ret = gst_vulkan_descriptor_cache_acquire (conv->descriptor_pool, &error); - if (!ret) { - GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s", - error->message); - g_clear_error (&error); - return NULL; - } - - return ret; -} - -static gboolean -view_convert_update_command_state (GstVulkanViewConvert * conv, - VkCommandBuffer cmd, GstVulkanImageView ** in_views, - GstVulkanImageView ** out_views, GstVulkanFence * fence) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - GstVulkanDescriptorSet *set; - VkImageView views[GST_VIDEO_MAX_PLANES]; - int i; - - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { - views[2 * i] = in_views[i]->view; - views[2 * i + 1] = in_views[i]->view; - } - - if (!conv->descriptor_up_to_date) { - if (!_update_uniform (conv, in_views, out_views, views)) + if (!conv->uniform) { + if (!create_uniform_buffer (conv)) + return NULL; + if (!update_uniform (conv, in_views, out_views)) { + gst_memory_unref (conv->uniform); + conv->uniform = NULL; return FALSE; + } } - set = _create_descriptor_set (conv); - update_descriptor_set (conv, set->set, views, - GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2); - vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, - render->pipeline_layout, 0, 1, &set->set, 0, NULL); - - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, (GstMiniObject *) set)); - - return TRUE; + return gst_memory_ref (conv->uniform); } static void gst_vulkan_view_convert_set_property (GObject * object, @@ -554,21 +530,6 @@ static GstFlowReturn gst_vulkan_view_convert_transform (GstBaseTransform * bt, static gboolean gst_vulkan_view_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, GstCaps * out_caps); -static VkAttachmentReference - * gst_vulkan_view_convert_render_pass_attachment_references - (GstVulkanFullScreenRender * render, guint * n_attachments); -static VkAttachmentDescription - * gst_vulkan_view_convert_render_pass_attachment_descriptions - (GstVulkanFullScreenRender * render, guint * n_descriptions); -static VkDescriptorSetLayoutBinding - * gst_vulkan_view_convert_descriptor_set_layout_bindings - (GstVulkanFullScreenRender * render, guint * n_bindings); -static void -gst_vulkan_view_convert_shader_create_info (GstVulkanFullScreenRender * render); -static VkPushConstantRange - * gst_vulkan_view_convert_push_constant_ranges (GstVulkanFullScreenRender * - render, guint * n_constants); - static GstStaticPadTemplate gst_vulkan_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, @@ -599,7 +560,7 @@ enum #define gst_vulkan_view_convert_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstVulkanViewConvert, gst_vulkan_view_convert, - GST_TYPE_VULKAN_FULL_SCREEN_RENDER, + GST_TYPE_VULKAN_VIDEO_FILTER, GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_view_convert, "vulkanviewconvert", 0, "Vulkan View Convert")); @@ -609,12 +570,10 @@ gst_vulkan_view_convert_class_init (GstVulkanViewConvertClass * klass) GObjectClass *gobject_class; GstElementClass *gstelement_class; GstBaseTransformClass *gstbasetransform_class; - GstVulkanFullScreenRenderClass *fullscreenrender_class; gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; gstbasetransform_class = (GstBaseTransformClass *) klass; - fullscreenrender_class = (GstVulkanFullScreenRenderClass *) klass; gobject_class->set_property = gst_vulkan_view_convert_set_property; gobject_class->get_property = gst_vulkan_view_convert_get_property; @@ -668,17 +627,6 @@ gst_vulkan_view_convert_class_init (GstVulkanViewConvertClass * klass) gstbasetransform_class->fixate_caps = gst_vulkan_view_convert_fixate_caps; gstbasetransform_class->set_caps = gst_vulkan_view_convert_set_caps; gstbasetransform_class->transform = gst_vulkan_view_convert_transform; - - fullscreenrender_class->render_pass_attachment_references = - gst_vulkan_view_convert_render_pass_attachment_references; - fullscreenrender_class->render_pass_attachment_descriptions = - gst_vulkan_view_convert_render_pass_attachment_descriptions; - fullscreenrender_class->descriptor_set_layout_bindings = - gst_vulkan_view_convert_descriptor_set_layout_bindings; - fullscreenrender_class->shader_create_info = - gst_vulkan_view_convert_shader_create_info; - fullscreenrender_class->push_constant_ranges = - gst_vulkan_view_convert_push_constant_ranges; } static void @@ -722,9 +670,6 @@ gst_vulkan_view_convert_set_property (GObject * object, guint prop_id, G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } - GST_OBJECT_LOCK (conv); - conv->descriptor_up_to_date = FALSE; - GST_OBJECT_UNLOCK (conv); } static void @@ -1829,272 +1774,92 @@ done: return othercaps; } -static void -destroy_shader_create_info (GstVulkanFullScreenRender * render, gpointer data) +static gboolean +create_descriptor_set_layout (GstVulkanViewConvert * conv, GError ** error) { - VkPipelineShaderStageCreateInfo *info = data; + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv); + VkDescriptorSetLayoutBinding bindings[GST_VIDEO_MAX_PLANES * 2 + 1] = + { {0,}, }; + VkDescriptorSetLayoutCreateInfo layout_info; + VkDescriptorSetLayout descriptor_set_layout; + int descriptor_n = 0; + VkResult err; int i; - for (i = 0; i < render->n_shader_stages; i++) { - vkDestroyShaderModule (render->device->device, info[i].module, NULL); - } - - g_free (info); -} - -static void -gst_vulkan_view_convert_shader_create_info (GstVulkanFullScreenRender * render) -{ - VkShaderModule vert_module, frag_module; - - vert_module = - _vk_create_shader (render->device, identity_vert, identity_vert_size, - NULL); - frag_module = - _vk_create_shader (render->device, view_convert_frag, - view_convert_frag_size, NULL); - - render->n_shader_stages = 2; - render->shader_create_info = - g_new0 (VkPipelineShaderStageCreateInfo, render->n_shader_stages); - render->destroy_shader_create_info = destroy_shader_create_info; - /* *INDENT-OFF* */ - render->shader_create_info[0] = (VkPipelineShaderStageCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, - .pNext = NULL, - .stage = VK_SHADER_STAGE_VERTEX_BIT, - .module = vert_module, - .pName = "main" - }; - - render->shader_create_info[1] = (VkPipelineShaderStageCreateInfo) { - .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, - .pNext = NULL, - .stage = VK_SHADER_STAGE_FRAGMENT_BIT, - .module = frag_module, - .pName = "main" - }; - /* *INDENT-ON* */ -} - -static VkPushConstantRange * -gst_vulkan_view_convert_push_constant_ranges (GstVulkanFullScreenRender * - render, guint * n_constants) -{ - *n_constants = 0; - return NULL; -} - -static VkDescriptorSetLayoutBinding - * gst_vulkan_view_convert_descriptor_set_layout_bindings - (GstVulkanFullScreenRender * render, guint * n_bindings) -{ - VkDescriptorSetLayoutBinding *bindings; - guint i; - - *n_bindings = 0; - *n_bindings += GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2; - *n_bindings += 1; /* uniform binding */ - bindings = g_new0 (VkDescriptorSetLayoutBinding, *n_bindings); - - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2; i++) { - /* *INDENT-OFF* */ - bindings[i] = (VkDescriptorSetLayoutBinding) { - .binding = i, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .pImmutableSamplers = NULL, - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT - }; - i++; - bindings[i] = (VkDescriptorSetLayoutBinding) { - .binding = i, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .pImmutableSamplers = NULL, - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT - }; - /* *INDENT-ON* */ - } - /* *INDENT-OFF* */ - bindings[i] = (VkDescriptorSetLayoutBinding) { - .binding = i, + bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) { + .binding = 0, .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT + }; + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info) * 2; i++) { + bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) { + .binding = i+1, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImmutableSamplers = NULL, .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT + }; + }; + + layout_info = (VkDescriptorSetLayoutCreateInfo) { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .pNext = NULL, + .bindingCount = descriptor_n, + .pBindings = bindings }; /* *INDENT-ON* */ - i++; - g_assert (i == *n_bindings); - - return bindings; -} - -static VkAttachmentReference - * gst_vulkan_view_convert_render_pass_attachment_references - (GstVulkanFullScreenRender * render, guint * n_attachments) -{ - VkAttachmentReference *attachments; - int i; - - *n_attachments = GST_VIDEO_INFO_N_PLANES (&render->out_info); - attachments = g_new0 (VkAttachmentReference, *n_attachments); - - for (i = 0; i < *n_attachments; i++) { - /* *INDENT-OFF* */ - attachments[i] = (VkAttachmentReference) { - .attachment = i, - .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL - }; - /* *INDENT-ON* */ + err = + vkCreateDescriptorSetLayout (vfilter->device->device, &layout_info, + NULL, &descriptor_set_layout); + if (gst_vulkan_error_to_g_error (err, error, + "vkCreateDescriptorSetLayout") < 0) { + return FALSE; } - return attachments; -} + conv->quad->descriptor_set_layout = + gst_vulkan_handle_new_wrapped (vfilter->device, + GST_VULKAN_HANDLE_TYPE_DESCRIPTOR_SET_LAYOUT, + (GstVulkanHandleTypedef) descriptor_set_layout, + gst_vulkan_handle_free_descriptor_set_layout, NULL); -static VkAttachmentDescription - * gst_vulkan_view_convert_render_pass_attachment_descriptions - (GstVulkanFullScreenRender * render, guint * n_descriptions) -{ - VkAttachmentDescription *color_attachments; - int i; - - *n_descriptions = GST_VIDEO_INFO_N_PLANES (&render->out_info); - color_attachments = g_new0 (VkAttachmentDescription, *n_descriptions); - for (i = 0; i < *n_descriptions; i++) { - /* *INDENT-OFF* */ - color_attachments[i] = (VkAttachmentDescription) { - .format = gst_vulkan_format_from_video_info (&render->out_info, i), - .samples = VK_SAMPLE_COUNT_1_BIT, - .loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR, - .storeOp = VK_ATTACHMENT_STORE_OP_STORE, - .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE, - .stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE, - /* FIXME: share this between elements to avoid pipeline barriers */ - .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, - .finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL - }; - /* *INDENT-ON* */ - } - - return color_attachments; -} - -static VkSampler -_create_sampler (GstVulkanViewConvert * conv) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - - /* *INDENT-OFF* */ - VkSamplerCreateInfo samplerInfo = { - .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, - .magFilter = VK_FILTER_NEAREST, - .minFilter = VK_FILTER_NEAREST, - .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, - .anisotropyEnable = VK_FALSE, - .maxAnisotropy = 1, - .borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK, - .unnormalizedCoordinates = VK_FALSE, - .compareEnable = VK_FALSE, - .compareOp = VK_COMPARE_OP_ALWAYS, - .mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR, - .mipLodBias = 0.0f, - .minLod = 0.0f, - .maxLod = 0.0f - }; - /* *INDENT-ON* */ - GError *error = NULL; - VkSampler sampler; - VkResult err; - - err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) { - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - return sampler; + return TRUE; } static gboolean gst_vulkan_view_convert_start (GstBaseTransform * bt) { + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt); GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); + GstVulkanHandle *vert, *frag; if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt)) return FALSE; - if (!(conv->sampler = _create_sampler (conv))) + conv->quad = gst_vulkan_full_screen_quad_new (vfilter->queue); + + if (!(vert = + _vk_create_shader (vfilter->device, identity_vert, identity_vert_size, + NULL))) { + return FALSE; + } + if (!(frag = + _vk_create_shader (vfilter->device, view_convert_frag, + view_convert_frag_size, NULL))) { + gst_vulkan_handle_unref (vert); return FALSE; - - return TRUE; -} - -static GstVulkanDescriptorCache * -_create_descriptor_pool (GstVulkanViewConvert * conv) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - guint max_sets = 32; /* FIXME: don't hardcode this! */ - - /* *INDENT-OFF* */ - VkDescriptorPoolSize pool_sizes[] = { - { - .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = max_sets * GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2, - }, - { - .type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, - .descriptorCount = max_sets - }, - }; - - VkDescriptorPoolCreateInfo pool_info = { - .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, - .pNext = NULL, - .poolSizeCount = G_N_ELEMENTS (pool_sizes), - .pPoolSizes = pool_sizes, - .maxSets = max_sets - }; - /* *INDENT-ON* */ - VkDescriptorPool pool; - GstVulkanDescriptorPool *ret; - GstVulkanDescriptorCache *cache; - GError *error = NULL; - VkResult err; - - err = - vkCreateDescriptorPool (render->device->device, &pool_info, NULL, &pool); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateDescriptorPool") < 0) { - GST_ERROR_OBJECT (render, "Failed to create descriptor pool: %s", - error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; } - ret = gst_vulkan_descriptor_pool_new_wrapped (render->device, pool, 32); - cache = - gst_vulkan_descriptor_cache_new (ret, 1, &render->descriptor_set_layout); - gst_object_unref (ret); - - return cache; -} - -static gboolean -_create_uniform_buffer (GstVulkanViewConvert * conv) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - - conv->uniform = - gst_vulkan_buffer_memory_alloc (render->device, - sizeof (struct ViewUpdate), - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + if (!gst_vulkan_full_screen_quad_set_shaders (conv->quad, vert, frag)) { + gst_vulkan_handle_unref (vert); + gst_vulkan_handle_unref (frag); + return FALSE; + } + gst_vulkan_handle_unref (vert); + gst_vulkan_handle_unref (frag); return TRUE; } @@ -2103,36 +1868,15 @@ static gboolean gst_vulkan_view_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, GstCaps * out_caps) { + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt); GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - GstVulkanFence *last_fence; - - if (render->last_fence) - last_fence = gst_vulkan_fence_ref (render->last_fence); - else - last_fence = gst_vulkan_fence_new_always_signalled (render->device); - - if (conv->descriptor_pool) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_object_unref (last_fence, - (GstObject *) conv->descriptor_pool)); - conv->descriptor_pool = NULL; - if (conv->uniform) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (last_fence, - (GstMiniObject *) conv->uniform)); - conv->uniform = NULL; - - gst_vulkan_fence_unref (last_fence); if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps, out_caps)) return FALSE; - if (!(conv->descriptor_pool = _create_descriptor_pool (conv))) - return FALSE; - - if (!_create_uniform_buffer (conv)) + if (!gst_vulkan_full_screen_quad_set_info (conv->quad, &vfilter->in_info, + &vfilter->out_info)) return FALSE; return TRUE; @@ -2142,134 +1886,82 @@ static gboolean gst_vulkan_view_convert_stop (GstBaseTransform * bt) { GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - conv->descriptor_up_to_date = FALSE; - - if (render->device) { - GstVulkanFence *last_fence; - - if (render->last_fence) - last_fence = gst_vulkan_fence_ref (render->last_fence); - else - last_fence = gst_vulkan_fence_new_always_signalled (render->device); - - if (conv->descriptor_pool) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_object_unref (last_fence, - (GstObject *) conv->descriptor_pool)); - conv->descriptor_pool = NULL; - if (conv->sampler) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_sampler (last_fence, conv->sampler)); - conv->sampler = VK_NULL_HANDLE; - if (conv->uniform) - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (last_fence, - (GstMiniObject *) conv->uniform)); - conv->uniform = NULL; - - gst_vulkan_fence_unref (last_fence); - } - - if (conv->cmd_pool) - gst_object_unref (conv->cmd_pool); - conv->cmd_pool = NULL; + gst_clear_object (&conv->quad); + if (conv->uniform) + gst_memory_unref (conv->uniform); + conv->uniform = NULL; return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt); } -static VkFramebuffer -_create_framebuffer (GstVulkanViewConvert * conv, guint n_views, - VkImageView * views) -{ - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - - /* *INDENT-OFF* */ - VkFramebufferCreateInfo framebuffer_info = { - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = NULL, - .renderPass = render->render_pass, - .attachmentCount = n_views, - .pAttachments = views, - .width = GST_VIDEO_INFO_WIDTH (&render->out_info), - .height = GST_VIDEO_INFO_HEIGHT (&render->out_info), - .layers = 1 - }; - /* *INDENT-ON* */ - VkFramebuffer framebuffer; - GError *error = NULL; - VkResult err; - - err = - vkCreateFramebuffer (render->device->device, &framebuffer_info, NULL, - &framebuffer); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateFramebuffer") < 0) { - GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s", - error->message); - g_clear_error (&error); - return VK_NULL_HANDLE; - } - - return framebuffer; -} - static GstFlowReturn gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer * outbuf) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt); GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); - GstVulkanImageMemory *in_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; GstVulkanImageView *in_img_views[GST_VIDEO_MAX_PLANES] = { NULL, }; - GstVulkanImageMemory *out_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; GstVulkanImageView *out_img_views[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanCommandBuffer *cmd_buf = NULL; GstVulkanFence *fence = NULL; - GstVulkanCommandBuffer *cmd_buf; - VkFramebuffer framebuffer; GError *error = NULL; VkResult err; int i; - fence = gst_vulkan_fence_new (render->device, 0, &error); + if (!gst_vulkan_full_screen_quad_set_input_buffer (conv->quad, inbuf, &error)) + goto error; + if (!gst_vulkan_full_screen_quad_set_output_buffer (conv->quad, outbuf, + &error)) + goto error; + + fence = gst_vulkan_fence_new (vfilter->device, 0, &error); if (!fence) goto error; - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { - GstMemory *mem = gst_buffer_peek_memory (inbuf, i); - if (!gst_is_vulkan_image_memory (mem)) { + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->in_info); i++) { + GstMemory *img_mem = gst_buffer_peek_memory (inbuf, i); + if (!gst_is_vulkan_image_memory (img_mem)) { g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, "Input memory must be a GstVulkanImageMemory"); goto error; } - in_img_mems[i] = (GstVulkanImageMemory *) mem; - in_img_views[i] = get_or_create_image_view (in_img_mems[i]); - gst_vulkan_trash_list_add (render->trash_list, + in_img_views[i] = + get_or_create_image_view ((GstVulkanImageMemory *) img_mem); + gst_vulkan_trash_list_add (conv->quad->trash_list, gst_vulkan_trash_new_mini_object_unref (fence, (GstMiniObject *) in_img_views[i])); } - - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->out_info); i++) { GstMemory *mem = gst_buffer_peek_memory (outbuf, i); if (!gst_is_vulkan_image_memory (mem)) { g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, "Output memory must be a GstVulkanImageMemory"); goto error; } - out_img_mems[i] = (GstVulkanImageMemory *) mem; - out_img_views[i] = get_or_create_image_view (out_img_mems[i]); - gst_vulkan_trash_list_add (render->trash_list, + out_img_views[i] = get_or_create_image_view ((GstVulkanImageMemory *) mem); + gst_vulkan_trash_list_add (conv->quad->trash_list, gst_vulkan_trash_new_mini_object_unref (fence, (GstMiniObject *) out_img_views[i])); } - if (!conv->cmd_pool) { - if (!(conv->cmd_pool = - gst_vulkan_queue_create_command_pool (render->queue, &error))) + { + GstMemory *uniforms = get_uniforms (conv, in_img_views, out_img_views); + if (!gst_vulkan_full_screen_quad_set_uniform_buffer (conv->quad, uniforms, + &error)) goto error; + gst_memory_unref (uniforms); } - if (!(cmd_buf = gst_vulkan_command_pool_create (conv->cmd_pool, &error))) + if (!conv->quad->descriptor_set_layout) + if (!create_descriptor_set_layout (conv, &error)) + goto error; + + if (!gst_vulkan_full_screen_quad_prepare_draw (conv->quad, fence, &error)) + goto error; + + if (!(cmd_buf = + gst_vulkan_command_pool_create (conv->quad->cmd_pool, &error))) goto error; { @@ -2287,106 +1979,23 @@ gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, gst_vulkan_command_buffer_lock (cmd_buf); err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info); if (gst_vulkan_error_to_g_error (err, &error, "vkBeginCommandBuffer") < 0) - goto unlock_error; + goto error; } - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { - /* *INDENT-OFF* */ - VkImageMemoryBarrier in_image_memory_barrier = { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = NULL, - .srcAccessMask = in_img_mems[i]->barrier.parent.access_flags, - .dstAccessMask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT, - .oldLayout = in_img_mems[i]->barrier.image_layout, - .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - /* FIXME: implement exclusive transfers */ - .srcQueueFamilyIndex = 0, - .dstQueueFamilyIndex = 0, - .image = in_img_mems[i]->image, - .subresourceRange = in_img_mems[i]->barrier.subresource_range - }; - /* *INDENT-ON* */ - - vkCmdPipelineBarrier (cmd_buf->cmd, - in_img_mems[i]->barrier.parent.pipeline_stages, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, NULL, 0, NULL, 1, - &in_image_memory_barrier); - - in_img_mems[i]->barrier.parent.pipeline_stages = - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; - in_img_mems[i]->barrier.parent.access_flags = - in_image_memory_barrier.dstAccessMask; - in_img_mems[i]->barrier.image_layout = in_image_memory_barrier.newLayout; - } - - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { - VkImageMemoryBarrier out_image_memory_barrier; - - /* *INDENT-OFF* */ - out_image_memory_barrier = (VkImageMemoryBarrier) { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = NULL, - .srcAccessMask = out_img_mems[i]->barrier.parent.access_flags, - .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, - .oldLayout = out_img_mems[i]->barrier.image_layout, - .newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, - /* FIXME: implement exclusive transfers */ - .srcQueueFamilyIndex = 0, - .dstQueueFamilyIndex = 0, - .image = out_img_mems[i]->image, - .subresourceRange = out_img_mems[i]->barrier.subresource_range - }; - /* *INDENT-ON* */ - - vkCmdPipelineBarrier (cmd_buf->cmd, - out_img_mems[i]->barrier.parent.pipeline_stages, - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, NULL, 0, NULL, 1, - &out_image_memory_barrier); - - out_img_mems[i]->barrier.parent.pipeline_stages = - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; - out_img_mems[i]->barrier.parent.access_flags = - out_image_memory_barrier.dstAccessMask; - out_img_mems[i]->barrier.image_layout = out_image_memory_barrier.newLayout; - } - - { - VkImageView attachments[4] = { 0, }; - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { - attachments[i] = out_img_views[i]->view; - } - - if (!(framebuffer = _create_framebuffer (conv, - GST_VIDEO_INFO_N_PLANES (&render->out_info), attachments))) { - g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, - "Failed to create framebuffer"); - goto unlock_error; - } - } - - view_convert_update_command_state (conv, cmd_buf->cmd, in_img_views, - out_img_views, fence); - - if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd_buf->cmd, - framebuffer)) { - g_set_error (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, - "Failed to fill framebuffer"); + update_descriptor_set (conv, in_img_views); + if (!gst_vulkan_full_screen_quad_fill_command_buffer (conv->quad, cmd_buf, + fence, &error)) goto unlock_error; - } err = vkEndCommandBuffer (cmd_buf->cmd); gst_vulkan_command_buffer_unlock (cmd_buf); if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0) goto error; - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_free_framebuffer (fence, framebuffer)); - gst_vulkan_trash_list_add (render->trash_list, - gst_vulkan_trash_new_mini_object_unref (fence, - GST_MINI_OBJECT_CAST (cmd_buf))); + if (!gst_vulkan_full_screen_quad_submit (conv->quad, cmd_buf, fence, &error)) + goto error; - if (!gst_vulkan_full_screen_render_submit (render, cmd_buf->cmd, fence)) - return GST_FLOW_ERROR; + gst_vulkan_fence_unref (fence); return GST_FLOW_OK; @@ -2395,7 +2004,10 @@ unlock_error: gst_vulkan_command_buffer_unlock (cmd_buf); gst_vulkan_command_buffer_unref (cmd_buf); } + error: + gst_clear_mini_object ((GstMiniObject **) & fence); + GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL)); g_clear_error (&error); return GST_FLOW_ERROR; diff --git a/ext/vulkan/vkviewconvert.h b/ext/vulkan/vkviewconvert.h index d95808cc40..f05f5ac176 100644 --- a/ext/vulkan/vkviewconvert.h +++ b/ext/vulkan/vkviewconvert.h @@ -24,7 +24,7 @@ #include #include #include -#include "vkfullscreenrender.h" +#include "vkfullscreenquad.h" G_BEGIN_DECLS @@ -46,34 +46,24 @@ typedef enum struct _GstVulkanViewConvert { - GstVulkanFullScreenRender parent; + GstVulkanVideoFilter parent; - GstVulkanCommandPool *cmd_pool; - - VkSampler sampler; - GstVulkanDescriptorCache *descriptor_pool; - - VkShaderModule vert_module; - VkShaderModule frag_module; - - VkDescriptorSetLayoutBinding sampler_layout_binding; - VkDescriptorSetLayoutCreateInfo layout_info; - - GstMemory *uniform; - gboolean descriptor_up_to_date; + GstVulkanFullScreenQuad *quad; /* properties */ - GstVideoMultiviewMode input_mode_override; - GstVideoMultiviewFlags input_flags_override; - GstVideoMultiviewMode output_mode_override; - GstVideoMultiviewFlags output_flags_override; + GstVideoMultiviewMode input_mode_override; + GstVideoMultiviewFlags input_flags_override; + GstVideoMultiviewMode output_mode_override; + GstVideoMultiviewFlags output_flags_override; - GstVulkanStereoDownmix downmix_mode; + GstVulkanStereoDownmix downmix_mode; + + GstMemory *uniform; }; struct _GstVulkanViewConvertClass { - GstVulkanFullScreenRenderClass parent_class; + GstVulkanVideoFilterClass parent_class; }; GType gst_vulkan_view_convert_get_type(void); diff --git a/gst-libs/gst/vulkan/gstvkvideofilter.c b/gst-libs/gst/vulkan/gstvkvideofilter.c new file mode 100644 index 0000000000..2fa8381999 --- /dev/null +++ b/gst-libs/gst/vulkan/gstvkvideofilter.c @@ -0,0 +1,338 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:vulkanvideofilter + * @title: vulkanvideofilter + * + * vulkanvideofilter is a helper base class for retrieving and holding the + * #GstVulkanInstance, #GstVulkanDevice and #GstVulkanQueue used by an element. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include + +GST_DEBUG_CATEGORY (gst_debug_vulkan_video_filter); +#define GST_CAT_DEFAULT gst_debug_vulkan_video_filter + +static void gst_vulkan_video_filter_finalize (GObject * object); + +static gboolean gst_vulkan_video_filter_query (GstBaseTransform * bt, + GstPadDirection direction, GstQuery * query); +static void gst_vulkan_video_filter_set_context (GstElement * element, + GstContext * context); + +static gboolean gst_vulkan_video_filter_start (GstBaseTransform * bt); +static gboolean gst_vulkan_video_filter_stop (GstBaseTransform * bt); + +static gboolean gst_vulkan_video_filter_set_caps (GstBaseTransform * bt, + GstCaps * in_caps, GstCaps * out_caps); +static GstCaps *gst_vulkan_video_filter_transform_caps (GstBaseTransform * + bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter); +static gboolean +gst_vulkan_video_filter_propose_allocation (GstBaseTransform * bt, + GstQuery * decide_query, GstQuery * query); +static gboolean +gst_vulkan_video_filter_decide_allocation (GstBaseTransform * bt, + GstQuery * query); + +enum +{ + PROP_0, +}; + +enum +{ + SIGNAL_0, + LAST_SIGNAL +}; + +/* static guint gst_vulkan_video_filter_signals[LAST_SIGNAL] = { 0 }; */ + +#define gst_vulkan_video_filter_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstVulkanVideoFilter, + gst_vulkan_video_filter, GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_video_filter, + "vulkanvideofilter", 0, "Vulkan Video Filter")); + +static void +gst_vulkan_video_filter_class_init (GstVulkanVideoFilterClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstBaseTransformClass *gstbasetransform_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstbasetransform_class = (GstBaseTransformClass *) klass; + + gobject_class->finalize = gst_vulkan_video_filter_finalize; + + gstelement_class->set_context = gst_vulkan_video_filter_set_context; + gstbasetransform_class->start = + GST_DEBUG_FUNCPTR (gst_vulkan_video_filter_start); + gstbasetransform_class->stop = + GST_DEBUG_FUNCPTR (gst_vulkan_video_filter_stop); + gstbasetransform_class->query = + GST_DEBUG_FUNCPTR (gst_vulkan_video_filter_query); + gstbasetransform_class->set_caps = gst_vulkan_video_filter_set_caps; + gstbasetransform_class->transform_caps = + gst_vulkan_video_filter_transform_caps; + gstbasetransform_class->propose_allocation = + gst_vulkan_video_filter_propose_allocation; + gstbasetransform_class->decide_allocation = + gst_vulkan_video_filter_decide_allocation; +} + +static void +gst_vulkan_video_filter_init (GstVulkanVideoFilter * render) +{ +} + +static void +gst_vulkan_video_filter_finalize (GObject * object) +{ + GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (object); + + gst_caps_replace (&render->in_caps, NULL); + gst_caps_replace (&render->out_caps, NULL); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static gboolean +gst_vulkan_video_filter_query (GstBaseTransform * bt, + GstPadDirection direction, GstQuery * query) +{ + GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT:{ + if (gst_vulkan_handle_context_query (GST_ELEMENT (render), query, + NULL, render->instance, render->device)) + return TRUE; + + if (gst_vulkan_queue_handle_context_query (GST_ELEMENT (render), + query, render->queue)) + return TRUE; + + break; + } + default: + break; + } + + return GST_BASE_TRANSFORM_CLASS (parent_class)->query (bt, direction, query); +} + +static void +gst_vulkan_video_filter_set_context (GstElement * element, GstContext * context) +{ + GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (element); + + gst_vulkan_handle_set_context (element, context, NULL, &render->instance); + + GST_ELEMENT_CLASS (parent_class)->set_context (element, context); +} + +struct choose_data +{ + GstVulkanVideoFilter *upload; + GstVulkanQueue *queue; +}; + +static gboolean +_choose_queue (GstVulkanDevice * device, GstVulkanQueue * queue, + struct choose_data *data) +{ + guint flags = + device->physical_device->queue_family_props[queue->family].queueFlags; + + if ((flags & VK_QUEUE_GRAPHICS_BIT) != 0) { + if (data->queue) + gst_object_unref (data->queue); + data->queue = gst_object_ref (queue); + return FALSE; + } + + return TRUE; +} + +static GstVulkanQueue * +_find_graphics_queue (GstVulkanVideoFilter * upload) +{ + struct choose_data data; + + data.upload = upload; + data.queue = NULL; + + gst_vulkan_device_foreach_queue (upload->device, + (GstVulkanDeviceForEachQueueFunc) _choose_queue, &data); + + return data.queue; +} + +static GstCaps * +gst_vulkan_video_filter_transform_caps (GstBaseTransform * bt, + GstPadDirection direction, GstCaps * caps, GstCaps * filter) +{ + GstCaps *result, *tmp; + + tmp = gst_caps_copy (caps); + + if (filter) { + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } else { + result = tmp; + } + + return result; +} + +static gboolean +gst_vulkan_video_filter_set_caps (GstBaseTransform * bt, + GstCaps * in_caps, GstCaps * out_caps) +{ + GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt); + + if (!gst_video_info_from_caps (&render->in_info, in_caps)) + return FALSE; + if (!gst_video_info_from_caps (&render->out_info, out_caps)) + return FALSE; + + gst_caps_replace (&render->in_caps, in_caps); + gst_caps_replace (&render->out_caps, out_caps); + GST_DEBUG_OBJECT (bt, "set caps: %" GST_PTR_FORMAT, in_caps); + + return TRUE; +} + +static gboolean +gst_vulkan_video_filter_propose_allocation (GstBaseTransform * bt, + GstQuery * decide_query, GstQuery * query) +{ + /* FIXME: */ + return FALSE; +} + +static gboolean +gst_vulkan_video_filter_decide_allocation (GstBaseTransform * bt, + GstQuery * query) +{ + GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt); + GstBufferPool *pool = NULL; + GstStructure *config; + GstCaps *caps; + guint min, max, size; + gboolean update_pool; + + gst_query_parse_allocation (query, &caps, NULL); + if (!caps) + return FALSE; + + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + update_pool = TRUE; + } else { + GstVideoInfo vinfo; + + gst_video_info_init (&vinfo); + gst_video_info_from_caps (&vinfo, caps); + size = vinfo.size; + min = max = 0; + update_pool = FALSE; + } + + if (!pool || !GST_IS_VULKAN_IMAGE_BUFFER_POOL (pool)) { + if (pool) + gst_object_unref (pool); + pool = gst_vulkan_image_buffer_pool_new (render->device); + } + + config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + gst_buffer_pool_set_config (pool, config); + + if (update_pool) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + + gst_object_unref (pool); + + return TRUE; +} + +static gboolean +gst_vulkan_video_filter_start (GstBaseTransform * bt) +{ + GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt); + + if (!gst_vulkan_ensure_element_data (GST_ELEMENT (bt), NULL, + &render->instance)) { + GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND, + ("Failed to retrieve vulkan instance"), (NULL)); + return FALSE; + } + if (!gst_vulkan_device_run_context_query (GST_ELEMENT (render), + &render->device)) { + GError *error = NULL; + GST_DEBUG_OBJECT (render, "No device retrieved from peer elements"); + if (!(render->device = + gst_vulkan_instance_create_device (render->instance, &error))) { + GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND, + ("Failed to create vulkan device"), ("%s", error->message)); + g_clear_error (&error); + return FALSE; + } + } + + if (!gst_vulkan_queue_run_context_query (GST_ELEMENT (render), + &render->queue)) { + GST_DEBUG_OBJECT (render, "No queue retrieved from peer elements"); + render->queue = _find_graphics_queue (render); + } + if (!render->queue) + return FALSE; + + return TRUE; +} + +static gboolean +gst_vulkan_video_filter_stop (GstBaseTransform * bt) +{ + GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt); + + gst_clear_object (&render->device); + gst_clear_object (&render->queue); + gst_clear_object (&render->instance); + + return TRUE; +} diff --git a/gst-libs/gst/vulkan/gstvkvideofilter.h b/gst-libs/gst/vulkan/gstvkvideofilter.h new file mode 100644 index 0000000000..986615b0e0 --- /dev/null +++ b/gst-libs/gst/vulkan/gstvkvideofilter.h @@ -0,0 +1,63 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _VK_FULL_SCREEN_RENDER_H_ +#define _VK_FULL_SCREEN_RENDER_H_ + +#include +#include +#include + +G_BEGIN_DECLS + +GST_VULKAN_API +GType gst_vulkan_video_filter_get_type(void); +#define GST_TYPE_VULKAN_VIDEO_FILTER (gst_vulkan_video_filter_get_type()) +#define GST_VULKAN_VIDEO_FILTER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_VIDEO_FILTER,GstVulkanVideoFilter)) +#define GST_VULKAN_VIDEO_FILTER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_VIDEO_FILTER,GstVulkanVideoFilterClass)) +#define GST_VULKAN_VIDEO_FILTER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VULKAN_VIDEO_FILTER,GstVulkanVideoFilterClass)) +#define GST_IS_VULKAN_VIDEO_FILTER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_VIDEO_FILTER)) +#define GST_IS_VULKAN_VIDEO_FILTER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_VIDEO_FILTER)) + +typedef struct _GstVulkanVideoFilter GstVulkanVideoFilter; +typedef struct _GstVulkanVideoFilterClass GstVulkanVideoFilterClass; + +struct _GstVulkanVideoFilter +{ + GstBaseTransform parent; + + GstVulkanInstance *instance; + GstVulkanDevice *device; + GstVulkanQueue *queue; + + GstCaps *in_caps; + GstVideoInfo in_info; + GstCaps *out_caps; + GstVideoInfo out_info; +}; + +struct _GstVulkanVideoFilterClass +{ + GstBaseTransformClass video_sink_class; +}; + +G_END_DECLS + +#endif diff --git a/gst-libs/gst/vulkan/meson.build b/gst-libs/gst/vulkan/meson.build index ccd2052b47..2878566b18 100644 --- a/gst-libs/gst/vulkan/meson.build +++ b/gst-libs/gst/vulkan/meson.build @@ -27,6 +27,7 @@ vulkan_sources = [ 'gstvkqueue.c', 'gstvkswapper.c', 'gstvktrash.c', + 'gstvkvideofilter.c', 'gstvkutils.c', 'gstvkwindow.c', ] @@ -58,6 +59,7 @@ vulkan_headers = [ 'gstvkswapper.h', 'gstvktrash.h', 'gstvkutils.h', + 'gstvkvideofilter.h', 'gstvkwindow.h', 'vulkan-prelude.h', 'vulkan_fwd.h', diff --git a/gst-libs/gst/vulkan/vulkan.h b/gst-libs/gst/vulkan/vulkan.h index d12371453c..013379304e 100644 --- a/gst-libs/gst/vulkan/vulkan.h +++ b/gst-libs/gst/vulkan/vulkan.h @@ -25,6 +25,7 @@ #include +/* helper vulkan objects */ #include #include #include @@ -52,4 +53,7 @@ #include #include +/* helper elements */ +#include + #endif /* __GST_VULKAN_H__ */