mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2025-01-03 14:08:56 +00:00
d3d11videosink: Fixing unexpected overlay composition meta drops
A buffer can hold multiple GstVideoOverlayCompositionMeta objects. Should scan all metas on a buffer and render all overlay rectangles if any. Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4862>
This commit is contained in:
Seungha Yang
GStreamer Marge Bot
parent
f88cf8b8b0
commit
94eadc9810
2 changed files with 141 additions and 148 deletions
|
|
@ -24,6 +24,9 @@
|
|||
#include "gstd3d11overlaycompositor.h"
|
||||
#include "gstd3d11pluginutils.h"
|
||||
#include <wrl.h>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
|
||||
GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_overlay_compositor_debug);
|
||||
#define GST_CAT_DEFAULT gst_d3d11_overlay_compositor_debug
|
||||
|
|
@ -95,7 +98,23 @@ static const gchar templ_vertex_shader[] =
|
|||
"{\n"
|
||||
" return input;\n"
|
||||
"}\n";
|
||||
/* *INDENT-ON* */
|
||||
|
||||
struct GstD3D11CompositionOverlay
|
||||
{
|
||||
~GstD3D11CompositionOverlay ()
|
||||
{
|
||||
if (overlay_rect)
|
||||
gst_video_overlay_rectangle_unref (overlay_rect);
|
||||
}
|
||||
|
||||
GstVideoOverlayRectangle *overlay_rect = nullptr;
|
||||
ComPtr<ID3D11Texture2D> texture;
|
||||
ComPtr<ID3D11ShaderResourceView> srv;
|
||||
ComPtr<ID3D11Buffer> vertex_buffer;
|
||||
gboolean premul_alpha = FALSE;
|
||||
};
|
||||
|
||||
typedef std::shared_ptr<GstD3D11CompositionOverlay> GstD3D11CompositionOverlayPtr;
|
||||
|
||||
struct _GstD3D11OverlayCompositorPrivate
|
||||
{
|
||||
|
|
@ -103,33 +122,22 @@ struct _GstD3D11OverlayCompositorPrivate
|
|||
|
||||
D3D11_VIEWPORT viewport;
|
||||
|
||||
ID3D11PixelShader *ps;
|
||||
ID3D11PixelShader *premul_ps;
|
||||
ID3D11VertexShader *vs;
|
||||
ID3D11InputLayout *layout;
|
||||
ID3D11SamplerState *sampler;
|
||||
ID3D11BlendState *blend;
|
||||
ID3D11Buffer *index_buffer;
|
||||
ComPtr<ID3D11PixelShader> ps;
|
||||
ComPtr<ID3D11PixelShader> premul_ps;
|
||||
ComPtr<ID3D11VertexShader> vs;
|
||||
ComPtr<ID3D11InputLayout> layout;
|
||||
ComPtr<ID3D11SamplerState> sampler;
|
||||
ComPtr<ID3D11BlendState> blend;
|
||||
ComPtr<ID3D11Buffer> index_buffer;
|
||||
|
||||
/* GstD3D11CompositionOverlay */
|
||||
GList *overlays;
|
||||
std::vector<GstD3D11CompositionOverlayPtr> overlays;
|
||||
};
|
||||
/* *INDENT-ON* */
|
||||
|
||||
typedef struct
|
||||
{
|
||||
GstVideoOverlayRectangle *overlay_rect;
|
||||
ID3D11Texture2D *texture;
|
||||
ID3D11ShaderResourceView *srv;
|
||||
ID3D11Buffer *vertex_buffer;
|
||||
gboolean premul_alpha;
|
||||
} GstD3D11CompositionOverlay;
|
||||
|
||||
static void gst_d3d11_overlay_compositor_dispose (GObject * object);
|
||||
static void
|
||||
gst_d3d11_overlay_compositor_free_overlays (GstD3D11OverlayCompositor * self);
|
||||
static void gst_d3d11_overlay_compositor_finalize (GObject * object);
|
||||
|
||||
#define gst_d3d11_overlay_compositor_parent_class parent_class
|
||||
G_DEFINE_TYPE_WITH_PRIVATE (GstD3D11OverlayCompositor,
|
||||
G_DEFINE_TYPE (GstD3D11OverlayCompositor,
|
||||
gst_d3d11_overlay_compositor, GST_TYPE_OBJECT);
|
||||
|
||||
static void
|
||||
|
|
@ -137,43 +145,32 @@ gst_d3d11_overlay_compositor_class_init (GstD3D11OverlayCompositorClass * klass)
|
|||
{
|
||||
GObjectClass *object_class = G_OBJECT_CLASS (klass);
|
||||
|
||||
object_class->dispose = gst_d3d11_overlay_compositor_dispose;
|
||||
object_class->finalize = gst_d3d11_overlay_compositor_finalize;
|
||||
}
|
||||
|
||||
static void
|
||||
gst_d3d11_overlay_compositor_init (GstD3D11OverlayCompositor * self)
|
||||
{
|
||||
self->priv = (GstD3D11OverlayCompositorPrivate *)
|
||||
gst_d3d11_overlay_compositor_get_instance_private (self);
|
||||
self->priv = new GstD3D11OverlayCompositorPrivate ();
|
||||
}
|
||||
|
||||
static void
|
||||
gst_d3d11_overlay_compositor_dispose (GObject * object)
|
||||
gst_d3d11_overlay_compositor_finalize (GObject * object)
|
||||
{
|
||||
GstD3D11OverlayCompositor *self = GST_D3D11_OVERLAY_COMPOSITOR (object);
|
||||
GstD3D11OverlayCompositorPrivate *priv = self->priv;
|
||||
|
||||
gst_d3d11_overlay_compositor_free_overlays (self);
|
||||
|
||||
GST_D3D11_CLEAR_COM (priv->ps);
|
||||
GST_D3D11_CLEAR_COM (priv->premul_ps);
|
||||
GST_D3D11_CLEAR_COM (priv->vs);
|
||||
GST_D3D11_CLEAR_COM (priv->layout);
|
||||
GST_D3D11_CLEAR_COM (priv->sampler);
|
||||
GST_D3D11_CLEAR_COM (priv->blend);
|
||||
GST_D3D11_CLEAR_COM (priv->index_buffer);
|
||||
delete self->priv;
|
||||
|
||||
gst_clear_object (&self->device);
|
||||
|
||||
G_OBJECT_CLASS (parent_class)->dispose (object);
|
||||
G_OBJECT_CLASS (parent_class)->finalize (object);
|
||||
}
|
||||
|
||||
static GstD3D11CompositionOverlay *
|
||||
static GstD3D11CompositionOverlayPtr
|
||||
gst_d3d11_composition_overlay_new (GstD3D11OverlayCompositor * self,
|
||||
GstVideoOverlayRectangle * overlay_rect)
|
||||
{
|
||||
GstD3D11OverlayCompositorPrivate *priv = self->priv;
|
||||
GstD3D11CompositionOverlay *overlay = nullptr;
|
||||
gint x, y;
|
||||
guint width, height;
|
||||
D3D11_SUBRESOURCE_DATA subresource_data;
|
||||
|
|
@ -343,32 +340,16 @@ gst_d3d11_composition_overlay_new (GstD3D11OverlayCompositor * self,
|
|||
|
||||
context_handle->Unmap (vertex_buffer.Get (), 0);
|
||||
|
||||
overlay = g_new0 (GstD3D11CompositionOverlay, 1);
|
||||
auto overlay = std::make_shared < GstD3D11CompositionOverlay > ();
|
||||
overlay->overlay_rect = gst_video_overlay_rectangle_ref (overlay_rect);
|
||||
overlay->texture = texture.Detach ();
|
||||
overlay->srv = srv.Detach ();
|
||||
overlay->vertex_buffer = vertex_buffer.Detach ();
|
||||
overlay->texture = texture;
|
||||
overlay->srv = srv;
|
||||
overlay->vertex_buffer = vertex_buffer;
|
||||
overlay->premul_alpha = premul_alpha;
|
||||
|
||||
return overlay;
|
||||
}
|
||||
|
||||
static void
|
||||
gst_d3d11_composition_overlay_free (GstD3D11CompositionOverlay * overlay)
|
||||
{
|
||||
if (!overlay)
|
||||
return;
|
||||
|
||||
if (overlay->overlay_rect)
|
||||
gst_video_overlay_rectangle_unref (overlay->overlay_rect);
|
||||
|
||||
GST_D3D11_CLEAR_COM (overlay->srv);
|
||||
GST_D3D11_CLEAR_COM (overlay->texture);
|
||||
GST_D3D11_CLEAR_COM (overlay->vertex_buffer);
|
||||
|
||||
g_free (overlay);
|
||||
}
|
||||
|
||||
static gboolean
|
||||
gst_d3d11_overlay_compositor_setup_shader (GstD3D11OverlayCompositor * self)
|
||||
{
|
||||
|
|
@ -550,39 +531,27 @@ gst_d3d11_overlay_compositor_new (GstD3D11Device * device,
|
|||
return self;
|
||||
}
|
||||
|
||||
static void
|
||||
gst_d3d11_overlay_compositor_free_overlays (GstD3D11OverlayCompositor * self)
|
||||
{
|
||||
GstD3D11OverlayCompositorPrivate *priv = self->priv;
|
||||
|
||||
if (priv->overlays) {
|
||||
g_list_free_full (priv->overlays,
|
||||
(GDestroyNotify) gst_d3d11_composition_overlay_free);
|
||||
|
||||
priv->overlays = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
static gint
|
||||
find_in_compositor (const GstD3D11CompositionOverlay * overlay,
|
||||
const GstVideoOverlayRectangle * rect)
|
||||
{
|
||||
return !(overlay->overlay_rect == rect);
|
||||
}
|
||||
|
||||
static gboolean
|
||||
is_in_video_overlay_composition (GstVideoOverlayComposition * voc,
|
||||
GstD3D11CompositionOverlay * overlay)
|
||||
gst_d3d11_overlay_compositor_foreach_meta (GstBuffer * buffer, GstMeta ** meta,
|
||||
std::vector < GstVideoOverlayRectangle * >*overlay_rect)
|
||||
{
|
||||
guint i;
|
||||
GstVideoOverlayCompositionMeta *cmeta;
|
||||
guint num_rect;
|
||||
|
||||
for (i = 0; i < gst_video_overlay_composition_n_rectangles (voc); i++) {
|
||||
GstVideoOverlayRectangle *rectangle =
|
||||
gst_video_overlay_composition_get_rectangle (voc, i);
|
||||
if (overlay->overlay_rect == rectangle)
|
||||
return TRUE;
|
||||
if ((*meta)->info->api != GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE)
|
||||
return TRUE;
|
||||
|
||||
cmeta = (GstVideoOverlayCompositionMeta *) (*meta);
|
||||
if (!cmeta->overlay)
|
||||
return TRUE;
|
||||
|
||||
num_rect = gst_video_overlay_composition_n_rectangles (cmeta->overlay);
|
||||
for (guint i = 0; i < num_rect; i++) {
|
||||
auto rect = gst_video_overlay_composition_get_rectangle (cmeta->overlay, i);
|
||||
overlay_rect->push_back (rect);
|
||||
}
|
||||
return FALSE;
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
gboolean
|
||||
|
|
@ -590,61 +559,64 @@ gst_d3d11_overlay_compositor_upload (GstD3D11OverlayCompositor * compositor,
|
|||
GstBuffer * buf)
|
||||
{
|
||||
GstD3D11OverlayCompositorPrivate *priv;
|
||||
GstVideoOverlayCompositionMeta *meta;
|
||||
gint i, num_overlays;
|
||||
GList *iter;
|
||||
std::vector < GstVideoOverlayRectangle * >new_overlay_rect;
|
||||
|
||||
g_return_val_if_fail (compositor != nullptr, FALSE);
|
||||
g_return_val_if_fail (GST_IS_BUFFER (buf), FALSE);
|
||||
|
||||
priv = compositor->priv;
|
||||
|
||||
meta = gst_buffer_get_video_overlay_composition_meta (buf);
|
||||
if (!meta) {
|
||||
gst_d3d11_overlay_compositor_free_overlays (compositor);
|
||||
gst_buffer_foreach_meta (buf,
|
||||
(GstBufferForeachMetaFunc) gst_d3d11_overlay_compositor_foreach_meta,
|
||||
&new_overlay_rect);
|
||||
|
||||
if (new_overlay_rect.empty ()) {
|
||||
priv->overlays.clear ();
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
num_overlays = gst_video_overlay_composition_n_rectangles (meta->overlay);
|
||||
if (!num_overlays) {
|
||||
gst_d3d11_overlay_compositor_free_overlays (compositor);
|
||||
return TRUE;
|
||||
}
|
||||
GST_LOG_OBJECT (compositor, "Found %" G_GSIZE_FORMAT
|
||||
" overlay rectangles, %" G_GSIZE_FORMAT " in current queue",
|
||||
new_overlay_rect.size (), priv->overlays.size ());
|
||||
|
||||
GST_LOG_OBJECT (compositor, "Upload %d overlay rectangles", num_overlays);
|
||||
|
||||
/* Upload new overlay */
|
||||
for (i = 0; i < num_overlays; i++) {
|
||||
GstVideoOverlayRectangle *rectangle =
|
||||
gst_video_overlay_composition_get_rectangle (meta->overlay, i);
|
||||
|
||||
if (!g_list_find_custom (priv->overlays,
|
||||
rectangle, (GCompareFunc) find_in_compositor)) {
|
||||
GstD3D11CompositionOverlay *overlay = nullptr;
|
||||
|
||||
overlay = gst_d3d11_composition_overlay_new (compositor, rectangle);
|
||||
|
||||
if (!overlay)
|
||||
/* *INDENT-OFF* */
|
||||
for (auto it : new_overlay_rect) {
|
||||
if (std::find_if (priv->overlays.begin (), priv->overlays.end (),
|
||||
[&] (const auto & overlay) -> bool {
|
||||
return overlay->overlay_rect == it;
|
||||
}) == priv->overlays.end ()) {
|
||||
auto new_overlay = gst_d3d11_composition_overlay_new (compositor, it);
|
||||
if (!new_overlay)
|
||||
return FALSE;
|
||||
|
||||
priv->overlays = g_list_append (priv->overlays, overlay);
|
||||
priv->overlays.push_back (new_overlay);
|
||||
}
|
||||
}
|
||||
/* *INDENT-ON* */
|
||||
|
||||
GST_LOG_OBJECT (compositor, "Overlay rectangles in queue after uploaded %"
|
||||
G_GSIZE_FORMAT, priv->overlays.size ());
|
||||
|
||||
/* Remove old overlay */
|
||||
iter = priv->overlays;
|
||||
while (iter) {
|
||||
GstD3D11CompositionOverlay *overlay =
|
||||
(GstD3D11CompositionOverlay *) iter->data;
|
||||
GList *next = iter->next;
|
||||
|
||||
if (!is_in_video_overlay_composition (meta->overlay, overlay)) {
|
||||
priv->overlays = g_list_delete_link (priv->overlays, iter);
|
||||
gst_d3d11_composition_overlay_free (overlay);
|
||||
/* *INDENT-OFF* */
|
||||
auto it = priv->overlays.begin ();
|
||||
while (it != priv->overlays.end ()) {
|
||||
auto old_overlay = *it;
|
||||
if (std::find_if (new_overlay_rect.begin (), new_overlay_rect.end (),
|
||||
[&] (const auto & overlay) -> bool {
|
||||
return overlay == old_overlay->overlay_rect;
|
||||
}) == new_overlay_rect.end ()) {
|
||||
GST_LOG_OBJECT (compositor, "Removing %p from queue",
|
||||
old_overlay->overlay_rect);
|
||||
it = priv->overlays.erase (it);
|
||||
} else {
|
||||
it++;
|
||||
}
|
||||
|
||||
iter = next;
|
||||
}
|
||||
/* *INDENT-ON* */
|
||||
|
||||
GST_LOG_OBJECT (compositor, "Final queue size %" G_GSIZE_FORMAT,
|
||||
priv->overlays.size ());
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
|
@ -677,45 +649,49 @@ gst_d3d11_overlay_compositor_draw_unlocked (GstD3D11OverlayCompositor *
|
|||
compositor, ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES])
|
||||
{
|
||||
GstD3D11OverlayCompositorPrivate *priv;
|
||||
GList *iter;
|
||||
ID3D11DeviceContext *context;
|
||||
ID3D11ShaderResourceView *clear_view[GST_VIDEO_MAX_PLANES] = { nullptr, };
|
||||
UINT strides = sizeof (VertexData);
|
||||
UINT offsets = 0;
|
||||
ID3D11SamplerState *samplers[1];
|
||||
|
||||
g_return_val_if_fail (compositor != nullptr, FALSE);
|
||||
g_return_val_if_fail (rtv != nullptr, FALSE);
|
||||
|
||||
priv = compositor->priv;
|
||||
|
||||
if (!priv->overlays)
|
||||
if (priv->overlays.empty ())
|
||||
return TRUE;
|
||||
|
||||
samplers[0] = priv->sampler.Get ();
|
||||
|
||||
context = gst_d3d11_device_get_device_context_handle (compositor->device);
|
||||
context->IASetPrimitiveTopology (D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
|
||||
context->IASetInputLayout (priv->layout);
|
||||
context->IASetIndexBuffer (priv->index_buffer, DXGI_FORMAT_R16_UINT, 0);
|
||||
context->PSSetSamplers (0, 1, &priv->sampler);
|
||||
context->VSSetShader (priv->vs, nullptr, 0);
|
||||
context->IASetInputLayout (priv->layout.Get ());
|
||||
context->IASetIndexBuffer (priv->index_buffer.Get (),
|
||||
DXGI_FORMAT_R16_UINT, 0);
|
||||
context->PSSetSamplers (0, 1, samplers);
|
||||
context->VSSetShader (priv->vs.Get (), nullptr, 0);
|
||||
context->RSSetViewports (1, &priv->viewport);
|
||||
context->OMSetRenderTargets (1, rtv, nullptr);
|
||||
context->OMSetBlendState (priv->blend, nullptr, 0xffffffff);
|
||||
context->OMSetBlendState (priv->blend.Get (), nullptr, 0xffffffff);
|
||||
|
||||
for (iter = priv->overlays; iter; iter = g_list_next (iter)) {
|
||||
GstD3D11CompositionOverlay *overlay =
|
||||
(GstD3D11CompositionOverlay *) iter->data;
|
||||
/* *INDENT-OFF* */
|
||||
for (auto overlay : priv->overlays) {
|
||||
ID3D11ShaderResourceView *srv[] = { overlay->srv.Get () };
|
||||
ID3D11Buffer *vertex_buf[] = { overlay->vertex_buffer.Get () };
|
||||
|
||||
if (overlay->premul_alpha)
|
||||
context->PSSetShader (priv->premul_ps, nullptr, 0);
|
||||
context->PSSetShader (priv->premul_ps.Get (), nullptr, 0);
|
||||
else
|
||||
context->PSSetShader (priv->ps, nullptr, 0);
|
||||
context->PSSetShader (priv->ps.Get (), nullptr, 0);
|
||||
|
||||
context->PSSetShaderResources (0, 1, &overlay->srv);
|
||||
context->IASetVertexBuffers (0,
|
||||
1, &overlay->vertex_buffer, &strides, &offsets);
|
||||
context->PSSetShaderResources (0, 1, srv);
|
||||
context->IASetVertexBuffers (0, 1, vertex_buf, &strides, &offsets);
|
||||
|
||||
context->DrawIndexed (6, 0, 0);
|
||||
}
|
||||
/* *INDENT-ON* */
|
||||
|
||||
context->PSSetShaderResources (0, 1, clear_view);
|
||||
context->OMSetRenderTargets (0, nullptr, nullptr);
|
||||
|
|
|
|||
|
|
@ -1358,6 +1358,27 @@ gst_d3d11_video_sink_check_device_update (GstD3D11VideoSink * self,
|
|||
self->device = (GstD3D11Device *) gst_object_ref (dmem->device);
|
||||
}
|
||||
|
||||
static gboolean
|
||||
gst_d3d11_video_sink_foreach_meta (GstBuffer * buffer, GstMeta ** meta,
|
||||
GstBuffer * uploaded)
|
||||
{
|
||||
GstVideoOverlayCompositionMeta *cmeta;
|
||||
|
||||
if ((*meta)->info->api != GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE)
|
||||
return TRUE;
|
||||
|
||||
cmeta = (GstVideoOverlayCompositionMeta *) (*meta);
|
||||
if (!cmeta->overlay)
|
||||
return TRUE;
|
||||
|
||||
if (gst_video_overlay_composition_n_rectangles (cmeta->overlay) == 0)
|
||||
return TRUE;
|
||||
|
||||
gst_buffer_add_video_overlay_composition_meta (uploaded, cmeta->overlay);
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static GstFlowReturn
|
||||
gst_d3d11_video_sink_prepare (GstBaseSink * sink, GstBuffer * buffer)
|
||||
{
|
||||
|
|
@ -1382,8 +1403,6 @@ gst_d3d11_video_sink_prepare (GstBaseSink * sink, GstBuffer * buffer)
|
|||
}
|
||||
|
||||
if (!gst_is_d3d11_buffer (buffer)) {
|
||||
GstVideoOverlayCompositionMeta *overlay_meta;
|
||||
|
||||
ret = gst_buffer_pool_acquire_buffer (self->pool, &self->prepared_buffer,
|
||||
nullptr);
|
||||
if (ret != GST_FLOW_OK)
|
||||
|
|
@ -1404,11 +1423,9 @@ gst_d3d11_video_sink_prepare (GstBaseSink * sink, GstBuffer * buffer)
|
|||
gst_memory_unmap (mem, &info);
|
||||
}
|
||||
|
||||
overlay_meta = gst_buffer_get_video_overlay_composition_meta (buffer);
|
||||
if (overlay_meta) {
|
||||
gst_buffer_add_video_overlay_composition_meta (self->prepared_buffer,
|
||||
overlay_meta->overlay);
|
||||
}
|
||||
gst_buffer_foreach_meta (buffer,
|
||||
(GstBufferForeachMetaFunc) gst_d3d11_video_sink_foreach_meta,
|
||||
self->prepared_buffer);
|
||||
} else {
|
||||
self->prepared_buffer = gst_buffer_ref (buffer);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue