2019-12-30 09:58:59 +00:00
|
|
|
/*
|
|
|
|
|
* GStreamer
|
|
|
|
|
* Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
|
|
|
|
|
*
|
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
|
* modify it under the terms of the GNU Library General Public
|
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
|
* version 2 of the License, or (at your option) any later version.
|
|
|
|
|
*
|
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
|
* Library General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU Library General Public
|
|
|
|
|
* License along with this library; if not, write to the
|
|
|
|
|
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
|
|
|
|
* Boston, MA 02110-1301, USA.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
|
|
|
#include "config.h"
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#include "gstd3d11window.h"
|
2020-12-20 17:47:45 +00:00
|
|
|
#include "gstd3d11pluginutils.h"
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2021-01-18 10:17:14 +00:00
|
|
|
#if GST_D3D11_WINAPI_APP
|
2019-12-30 09:58:59 +00:00
|
|
|
/* workaround for GetCurrentTime collision */
|
|
|
|
|
#ifdef GetCurrentTime
|
|
|
|
|
#undef GetCurrentTime
|
|
|
|
|
#endif
|
|
|
|
|
#include <windows.ui.xaml.h>
|
|
|
|
|
#include <windows.applicationmodel.core.h>
|
2020-11-21 19:39:57 +00:00
|
|
|
#endif
|
|
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
#include <wrl.h>
|
2021-09-10 15:43:26 +00:00
|
|
|
|
2021-03-13 08:40:57 +00:00
|
|
|
/* *INDENT-OFF* */
|
2019-12-30 09:58:59 +00:00
|
|
|
using namespace Microsoft::WRL;
|
2021-09-10 15:43:26 +00:00
|
|
|
/* *INDENT-ON* */
|
2021-03-13 08:40:57 +00:00
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_window_debug);
|
|
|
|
|
#define GST_CAT_DEFAULT gst_d3d11_window_debug
|
2021-03-13 08:40:57 +00:00
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
enum
|
|
|
|
|
{
|
|
|
|
|
PROP_0,
|
|
|
|
|
PROP_D3D11_DEVICE,
|
|
|
|
|
PROP_FORCE_ASPECT_RATIO,
|
|
|
|
|
PROP_ENABLE_NAVIGATION_EVENTS,
|
|
|
|
|
PROP_FULLSCREEN_TOGGLE_MODE,
|
|
|
|
|
PROP_FULLSCREEN,
|
|
|
|
|
PROP_WINDOW_HANDLE,
|
2020-11-21 19:39:57 +00:00
|
|
|
PROP_RENDER_STATS,
|
2019-12-30 09:58:59 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
#define DEFAULT_ENABLE_NAVIGATION_EVENTS TRUE
|
|
|
|
|
#define DEFAULT_FORCE_ASPECT_RATIO TRUE
|
|
|
|
|
#define DEFAULT_FULLSCREEN_TOGGLE_MODE GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_NONE
|
|
|
|
|
#define DEFAULT_FULLSCREEN FALSE
|
2020-11-21 19:39:57 +00:00
|
|
|
#define DEFAULT_RENDER_STATS FALSE
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
enum
|
|
|
|
|
{
|
|
|
|
|
SIGNAL_KEY_EVENT,
|
|
|
|
|
SIGNAL_MOUSE_EVENT,
|
|
|
|
|
SIGNAL_LAST
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static guint d3d11_window_signals[SIGNAL_LAST] = { 0, };
|
|
|
|
|
|
|
|
|
|
GType
|
|
|
|
|
gst_d3d11_window_fullscreen_toggle_mode_type (void)
|
|
|
|
|
{
|
2021-03-22 03:34:36 +00:00
|
|
|
static gsize mode_type = 0;
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
if (g_once_init_enter (&mode_type)) {
|
|
|
|
|
static const GFlagsValue mode_types[] = {
|
|
|
|
|
{GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_NONE,
|
|
|
|
|
"GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_NONE", "none"},
|
|
|
|
|
{GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_ALT_ENTER,
|
|
|
|
|
"GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_ALT_ENTER", "alt-enter"},
|
|
|
|
|
{GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_PROPERTY,
|
|
|
|
|
"GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_PROPERTY", "property"},
|
|
|
|
|
{0, NULL, NULL},
|
|
|
|
|
};
|
|
|
|
|
GType tmp = g_flags_register_static ("GstD3D11WindowFullscreenToggleMode",
|
|
|
|
|
mode_types);
|
|
|
|
|
g_once_init_leave (&mode_type, tmp);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return (GType) mode_type;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define gst_d3d11_window_parent_class parent_class
|
2021-03-25 12:17:07 +00:00
|
|
|
G_DEFINE_ABSTRACT_TYPE (GstD3D11Window, gst_d3d11_window, GST_TYPE_OBJECT);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
static void gst_d3d11_window_set_property (GObject * object, guint prop_id,
|
|
|
|
|
const GValue * value, GParamSpec * pspec);
|
|
|
|
|
static void gst_d3d11_window_get_property (GObject * object, guint prop_id,
|
|
|
|
|
GValue * value, GParamSpec * pspec);
|
|
|
|
|
static void gst_d3d11_window_dispose (GObject * object);
|
|
|
|
|
static GstFlowReturn gst_d3d111_window_present (GstD3D11Window * self,
|
2021-03-25 12:17:07 +00:00
|
|
|
GstBuffer * buffer, ID3D11VideoProcessorOutputView * pov,
|
|
|
|
|
ID3D11RenderTargetView * rtv);
|
2020-01-10 12:45:43 +00:00
|
|
|
static void gst_d3d11_window_on_resize_default (GstD3D11Window * window,
|
|
|
|
|
guint width, guint height);
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
static gboolean gst_d3d11_window_prepare_default (GstD3D11Window * window,
|
|
|
|
|
guint display_width, guint display_height, GstCaps * caps,
|
|
|
|
|
gboolean * video_processor_available, GError ** error);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
gst_d3d11_window_class_init (GstD3D11WindowClass * klass)
|
|
|
|
|
{
|
|
|
|
|
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
|
|
|
|
|
|
|
|
|
|
gobject_class->set_property = gst_d3d11_window_set_property;
|
|
|
|
|
gobject_class->get_property = gst_d3d11_window_get_property;
|
|
|
|
|
gobject_class->dispose = gst_d3d11_window_dispose;
|
|
|
|
|
|
2020-01-10 12:45:43 +00:00
|
|
|
klass->on_resize = GST_DEBUG_FUNCPTR (gst_d3d11_window_on_resize_default);
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
klass->prepare = GST_DEBUG_FUNCPTR (gst_d3d11_window_prepare_default);
|
2020-01-10 12:45:43 +00:00
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
g_object_class_install_property (gobject_class, PROP_D3D11_DEVICE,
|
|
|
|
|
g_param_spec_object ("d3d11device", "D3D11 Device",
|
|
|
|
|
"GstD3D11Device object for creating swapchain",
|
|
|
|
|
GST_TYPE_D3D11_DEVICE,
|
|
|
|
|
(GParamFlags) (G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY |
|
|
|
|
|
G_PARAM_STATIC_STRINGS)));
|
|
|
|
|
|
|
|
|
|
g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO,
|
|
|
|
|
g_param_spec_boolean ("force-aspect-ratio",
|
|
|
|
|
"Force aspect ratio",
|
|
|
|
|
"When enabled, scaling will respect original aspect ratio",
|
|
|
|
|
DEFAULT_FORCE_ASPECT_RATIO,
|
|
|
|
|
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
|
|
|
|
|
|
|
|
|
|
g_object_class_install_property (gobject_class, PROP_ENABLE_NAVIGATION_EVENTS,
|
|
|
|
|
g_param_spec_boolean ("enable-navigation-events",
|
|
|
|
|
"Enable navigation events",
|
|
|
|
|
"When enabled, signals for navigation events are emitted",
|
|
|
|
|
DEFAULT_ENABLE_NAVIGATION_EVENTS,
|
|
|
|
|
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
|
|
|
|
|
|
|
|
|
|
g_object_class_install_property (gobject_class, PROP_FULLSCREEN_TOGGLE_MODE,
|
|
|
|
|
g_param_spec_flags ("fullscreen-toggle-mode",
|
|
|
|
|
"Full screen toggle mode",
|
|
|
|
|
"Full screen toggle mode used to trigger fullscreen mode change",
|
|
|
|
|
GST_D3D11_WINDOW_TOGGLE_MODE_GET_TYPE, DEFAULT_FULLSCREEN_TOGGLE_MODE,
|
|
|
|
|
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
|
|
|
|
|
|
|
|
|
|
g_object_class_install_property (gobject_class, PROP_FULLSCREEN,
|
|
|
|
|
g_param_spec_boolean ("fullscreen",
|
|
|
|
|
"fullscreen",
|
|
|
|
|
"Ignored when \"fullscreen-toggle-mode\" does not include \"property\"",
|
|
|
|
|
DEFAULT_FULLSCREEN,
|
|
|
|
|
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
|
|
|
|
|
|
|
|
|
|
g_object_class_install_property (gobject_class, PROP_WINDOW_HANDLE,
|
|
|
|
|
g_param_spec_pointer ("window-handle",
|
|
|
|
|
"Window Handle", "External Window Handle",
|
|
|
|
|
(GParamFlags) (G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY |
|
|
|
|
|
G_PARAM_STATIC_STRINGS)));
|
|
|
|
|
|
|
|
|
|
d3d11_window_signals[SIGNAL_KEY_EVENT] =
|
|
|
|
|
g_signal_new ("key-event", G_TYPE_FROM_CLASS (klass),
|
|
|
|
|
G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL,
|
|
|
|
|
G_TYPE_NONE, 2, G_TYPE_STRING, G_TYPE_STRING);
|
|
|
|
|
|
|
|
|
|
d3d11_window_signals[SIGNAL_MOUSE_EVENT] =
|
|
|
|
|
g_signal_new ("mouse-event", G_TYPE_FROM_CLASS (klass),
|
|
|
|
|
G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL,
|
|
|
|
|
G_TYPE_NONE, 4, G_TYPE_STRING, G_TYPE_INT, G_TYPE_DOUBLE, G_TYPE_DOUBLE);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
gst_d3d11_window_init (GstD3D11Window * self)
|
|
|
|
|
{
|
|
|
|
|
self->force_aspect_ratio = DEFAULT_FORCE_ASPECT_RATIO;
|
|
|
|
|
self->enable_navigation_events = DEFAULT_ENABLE_NAVIGATION_EVENTS;
|
|
|
|
|
self->fullscreen_toggle_mode = GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_NONE;
|
|
|
|
|
self->fullscreen = DEFAULT_FULLSCREEN;
|
2020-11-21 19:39:57 +00:00
|
|
|
self->render_stats = DEFAULT_RENDER_STATS;
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
gst_d3d11_window_set_property (GObject * object, guint prop_id,
|
|
|
|
|
const GValue * value, GParamSpec * pspec)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11Window *self = GST_D3D11_WINDOW (object);
|
|
|
|
|
GstD3D11WindowClass *klass = GST_D3D11_WINDOW_GET_CLASS (object);
|
|
|
|
|
|
|
|
|
|
switch (prop_id) {
|
|
|
|
|
case PROP_D3D11_DEVICE:
|
|
|
|
|
self->device = (GstD3D11Device *) g_value_dup_object (value);
|
|
|
|
|
break;
|
|
|
|
|
case PROP_FORCE_ASPECT_RATIO:
|
|
|
|
|
{
|
|
|
|
|
self->force_aspect_ratio = g_value_get_boolean (value);
|
|
|
|
|
if (self->swap_chain)
|
|
|
|
|
klass->update_swap_chain (self);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case PROP_ENABLE_NAVIGATION_EVENTS:
|
|
|
|
|
self->enable_navigation_events = g_value_get_boolean (value);
|
|
|
|
|
break;
|
|
|
|
|
case PROP_FULLSCREEN_TOGGLE_MODE:
|
|
|
|
|
self->fullscreen_toggle_mode =
|
|
|
|
|
(GstD3D11WindowFullscreenToggleMode) g_value_get_flags (value);
|
|
|
|
|
break;
|
|
|
|
|
case PROP_FULLSCREEN:
|
|
|
|
|
{
|
|
|
|
|
self->requested_fullscreen = g_value_get_boolean (value);
|
|
|
|
|
if (self->swap_chain)
|
|
|
|
|
klass->change_fullscreen_mode (self);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case PROP_WINDOW_HANDLE:
|
|
|
|
|
self->external_handle = (guintptr) g_value_get_pointer (value);
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
gst_d3d11_window_get_property (GObject * object, guint prop_id,
|
|
|
|
|
GValue * value, GParamSpec * pspec)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11Window *self = GST_D3D11_WINDOW (object);
|
|
|
|
|
|
|
|
|
|
switch (prop_id) {
|
|
|
|
|
case PROP_ENABLE_NAVIGATION_EVENTS:
|
|
|
|
|
g_value_set_boolean (value, self->enable_navigation_events);
|
|
|
|
|
break;
|
|
|
|
|
case PROP_FORCE_ASPECT_RATIO:
|
|
|
|
|
g_value_set_boolean (value, self->force_aspect_ratio);
|
|
|
|
|
break;
|
|
|
|
|
case PROP_FULLSCREEN_TOGGLE_MODE:
|
|
|
|
|
g_value_set_flags (value, self->fullscreen_toggle_mode);
|
|
|
|
|
break;
|
|
|
|
|
case PROP_FULLSCREEN:
|
|
|
|
|
g_value_set_boolean (value, self->fullscreen);
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
gst_d3d11_window_release_resources (GstD3D11Device * device,
|
|
|
|
|
GstD3D11Window * window)
|
|
|
|
|
{
|
2021-03-13 08:40:57 +00:00
|
|
|
GST_D3D11_CLEAR_COM (window->rtv);
|
|
|
|
|
GST_D3D11_CLEAR_COM (window->pov);
|
|
|
|
|
GST_D3D11_CLEAR_COM (window->swap_chain);
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
gst_d3d11_window_dispose (GObject * object)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11Window *self = GST_D3D11_WINDOW (object);
|
|
|
|
|
|
2022-06-15 17:18:43 +00:00
|
|
|
if (self->device)
|
2019-12-30 09:58:59 +00:00
|
|
|
gst_d3d11_window_release_resources (self->device, self);
|
|
|
|
|
|
2020-04-21 06:30:03 +00:00
|
|
|
g_clear_pointer (&self->processor, gst_d3d11_video_processor_free);
|
|
|
|
|
g_clear_pointer (&self->compositor, gst_d3d11_overlay_compositor_free);
|
2022-06-15 17:18:43 +00:00
|
|
|
gst_clear_object (&self->converter);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
gst_clear_buffer (&self->cached_buffer);
|
|
|
|
|
gst_clear_object (&self->device);
|
|
|
|
|
|
|
|
|
|
G_OBJECT_CLASS (parent_class)->dispose (object);
|
|
|
|
|
}
|
|
|
|
|
|
2020-01-10 12:45:43 +00:00
|
|
|
static void
|
|
|
|
|
gst_d3d11_window_on_resize_default (GstD3D11Window * window, guint width,
|
|
|
|
|
guint height)
|
2019-12-30 09:58:59 +00:00
|
|
|
{
|
|
|
|
|
HRESULT hr;
|
|
|
|
|
ID3D11Device *device_handle;
|
|
|
|
|
D3D11_TEXTURE2D_DESC desc;
|
|
|
|
|
DXGI_SWAP_CHAIN_DESC swap_desc;
|
|
|
|
|
ID3D11Texture2D *backbuffer = NULL;
|
|
|
|
|
GstVideoRectangle src_rect, dst_rect, rst_rect;
|
|
|
|
|
IDXGISwapChain *swap_chain;
|
|
|
|
|
|
|
|
|
|
gst_d3d11_device_lock (window->device);
|
|
|
|
|
if (!window->swap_chain)
|
|
|
|
|
goto done;
|
|
|
|
|
|
|
|
|
|
device_handle = gst_d3d11_device_get_device_handle (window->device);
|
|
|
|
|
swap_chain = window->swap_chain;
|
|
|
|
|
|
2021-03-13 08:40:57 +00:00
|
|
|
GST_D3D11_CLEAR_COM (window->rtv);
|
|
|
|
|
GST_D3D11_CLEAR_COM (window->pov);
|
2020-01-29 12:10:00 +00:00
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
swap_chain->GetDesc (&swap_desc);
|
2020-03-26 12:20:20 +00:00
|
|
|
hr = swap_chain->ResizeBuffers (0, width, height, window->dxgi_format,
|
2019-12-30 09:58:59 +00:00
|
|
|
swap_desc.Flags);
|
|
|
|
|
if (!gst_d3d11_result (hr, window->device)) {
|
|
|
|
|
GST_ERROR_OBJECT (window, "Couldn't resize buffers, hr: 0x%x", (guint) hr);
|
|
|
|
|
goto done;
|
|
|
|
|
}
|
|
|
|
|
|
2021-08-09 17:48:01 +00:00
|
|
|
hr = swap_chain->GetBuffer (0, IID_PPV_ARGS (&backbuffer));
|
2019-12-30 09:58:59 +00:00
|
|
|
if (!gst_d3d11_result (hr, window->device)) {
|
|
|
|
|
GST_ERROR_OBJECT (window,
|
|
|
|
|
"Cannot get backbuffer from swapchain, hr: 0x%x", (guint) hr);
|
|
|
|
|
goto done;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
backbuffer->GetDesc (&desc);
|
|
|
|
|
window->surface_width = desc.Width;
|
|
|
|
|
window->surface_height = desc.Height;
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
dst_rect.x = 0;
|
|
|
|
|
dst_rect.y = 0;
|
|
|
|
|
dst_rect.w = window->surface_width;
|
|
|
|
|
dst_rect.h = window->surface_height;
|
|
|
|
|
|
|
|
|
|
if (window->force_aspect_ratio) {
|
2020-03-02 11:55:29 +00:00
|
|
|
src_rect.x = 0;
|
|
|
|
|
src_rect.y = 0;
|
|
|
|
|
src_rect.w = GST_VIDEO_INFO_WIDTH (&window->render_info);
|
|
|
|
|
src_rect.h = GST_VIDEO_INFO_HEIGHT (&window->render_info);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
gst_video_sink_center_rect (src_rect, dst_rect, &rst_rect, TRUE);
|
|
|
|
|
} else {
|
|
|
|
|
rst_rect = dst_rect;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
window->render_rect.left = rst_rect.x;
|
|
|
|
|
window->render_rect.top = rst_rect.y;
|
|
|
|
|
window->render_rect.right = rst_rect.x + rst_rect.w;
|
|
|
|
|
window->render_rect.bottom = rst_rect.y + rst_rect.h;
|
|
|
|
|
|
|
|
|
|
GST_LOG_OBJECT (window,
|
|
|
|
|
"New client area %dx%d, render rect x: %d, y: %d, %dx%d",
|
|
|
|
|
desc.Width, desc.Height, rst_rect.x, rst_rect.y, rst_rect.w, rst_rect.h);
|
|
|
|
|
|
2021-08-09 17:48:01 +00:00
|
|
|
hr = device_handle->CreateRenderTargetView (backbuffer, NULL, &window->rtv);
|
2019-12-30 09:58:59 +00:00
|
|
|
if (!gst_d3d11_result (hr, window->device)) {
|
|
|
|
|
GST_ERROR_OBJECT (window, "Cannot create render target view, hr: 0x%x",
|
|
|
|
|
(guint) hr);
|
|
|
|
|
|
|
|
|
|
goto done;
|
|
|
|
|
}
|
|
|
|
|
|
2020-01-29 12:10:00 +00:00
|
|
|
if (window->processor) {
|
|
|
|
|
D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC pov_desc;
|
|
|
|
|
|
|
|
|
|
pov_desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D;
|
|
|
|
|
pov_desc.Texture2D.MipSlice = 0;
|
|
|
|
|
|
|
|
|
|
if (!gst_d3d11_video_processor_create_output_view (window->processor,
|
2021-08-09 17:48:01 +00:00
|
|
|
&pov_desc, backbuffer, &window->pov))
|
2020-01-29 12:10:00 +00:00
|
|
|
goto done;
|
|
|
|
|
}
|
2020-11-21 19:39:57 +00:00
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
window->first_present = TRUE;
|
|
|
|
|
|
|
|
|
|
/* redraw the last scene if cached buffer exits */
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
if (window->cached_buffer) {
|
2021-03-25 12:17:07 +00:00
|
|
|
gst_d3d111_window_present (window, window->cached_buffer,
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
window->pov, window->rtv);
|
|
|
|
|
}
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
done:
|
2021-03-13 08:40:57 +00:00
|
|
|
GST_D3D11_CLEAR_COM (backbuffer);
|
2021-03-20 07:15:35 +00:00
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
gst_d3d11_device_unlock (window->device);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
gst_d3d11_window_on_key_event (GstD3D11Window * window, const gchar * event,
|
|
|
|
|
const gchar * key)
|
|
|
|
|
{
|
|
|
|
|
g_return_if_fail (GST_IS_D3D11_WINDOW (window));
|
|
|
|
|
|
|
|
|
|
if (!window->enable_navigation_events)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
g_signal_emit (window, d3d11_window_signals[SIGNAL_KEY_EVENT], 0, event, key);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
gst_d3d11_window_on_mouse_event (GstD3D11Window * window, const gchar * event,
|
|
|
|
|
gint button, gdouble x, gdouble y)
|
|
|
|
|
{
|
|
|
|
|
g_return_if_fail (GST_IS_D3D11_WINDOW (window));
|
|
|
|
|
|
|
|
|
|
if (!window->enable_navigation_events)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
g_signal_emit (window, d3d11_window_signals[SIGNAL_MOUSE_EVENT], 0,
|
|
|
|
|
event, button, x, y);
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
typedef struct
|
|
|
|
|
{
|
|
|
|
|
DXGI_FORMAT dxgi_format;
|
|
|
|
|
GstVideoFormat gst_format;
|
|
|
|
|
gboolean supported;
|
|
|
|
|
} GstD3D11WindowDisplayFormat;
|
|
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
gboolean
|
2020-03-02 11:55:29 +00:00
|
|
|
gst_d3d11_window_prepare (GstD3D11Window * window, guint display_width,
|
|
|
|
|
guint display_height, GstCaps * caps, gboolean * video_processor_available,
|
|
|
|
|
GError ** error)
|
2019-12-30 09:58:59 +00:00
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass;
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
|
|
|
|
|
g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE);
|
|
|
|
|
|
|
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
g_assert (klass->prepare != NULL);
|
|
|
|
|
|
|
|
|
|
GST_DEBUG_OBJECT (window, "Prepare window, display resolution %dx%d, caps %"
|
|
|
|
|
GST_PTR_FORMAT, display_width, display_height, caps);
|
|
|
|
|
|
|
|
|
|
return klass->prepare (window, display_width, display_height, caps,
|
|
|
|
|
video_processor_available, error);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static gboolean
|
|
|
|
|
gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
|
|
|
|
|
guint display_height, GstCaps * caps, gboolean * video_processor_available,
|
|
|
|
|
GError ** error)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass;
|
2019-12-30 09:58:59 +00:00
|
|
|
guint swapchain_flags = 0;
|
2020-03-09 11:29:17 +00:00
|
|
|
ID3D11Device *device_handle;
|
|
|
|
|
guint i;
|
|
|
|
|
guint num_supported_format = 0;
|
|
|
|
|
HRESULT hr;
|
|
|
|
|
UINT display_flags =
|
|
|
|
|
D3D11_FORMAT_SUPPORT_TEXTURE2D | D3D11_FORMAT_SUPPORT_DISPLAY;
|
|
|
|
|
UINT supported_flags = 0;
|
|
|
|
|
GstD3D11WindowDisplayFormat formats[] = {
|
2021-03-14 04:01:37 +00:00
|
|
|
{DXGI_FORMAT_R8G8B8A8_UNORM, GST_VIDEO_FORMAT_RGBA, FALSE},
|
|
|
|
|
{DXGI_FORMAT_B8G8R8A8_UNORM, GST_VIDEO_FORMAT_BGRA, FALSE},
|
|
|
|
|
{DXGI_FORMAT_R10G10B10A2_UNORM, GST_VIDEO_FORMAT_RGB10A2_LE, FALSE},
|
2020-03-09 11:29:17 +00:00
|
|
|
};
|
|
|
|
|
const GstD3D11WindowDisplayFormat *chosen_format = NULL;
|
2021-03-14 04:01:37 +00:00
|
|
|
const GstDxgiColorSpace *chosen_colorspace = NULL;
|
2020-03-09 11:29:17 +00:00
|
|
|
gboolean have_hdr10 = FALSE;
|
|
|
|
|
DXGI_COLOR_SPACE_TYPE native_colorspace_type =
|
|
|
|
|
DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709;
|
|
|
|
|
DXGI_HDR_METADATA_HDR10 hdr10_metadata = { 0, };
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
/* Step 1: Clear old resources and objects */
|
2020-02-05 13:12:15 +00:00
|
|
|
gst_clear_buffer (&window->cached_buffer);
|
2020-03-09 11:29:17 +00:00
|
|
|
g_clear_pointer (&window->processor, gst_d3d11_video_processor_free);
|
|
|
|
|
g_clear_pointer (&window->compositor, gst_d3d11_overlay_compositor_free);
|
2022-06-15 17:18:43 +00:00
|
|
|
gst_clear_object (&window->converter);
|
2020-03-09 11:29:17 +00:00
|
|
|
|
2021-03-15 10:48:56 +00:00
|
|
|
window->processor_in_use = FALSE;
|
|
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
/* Step 2: Decide display color format
|
|
|
|
|
* If upstream format is 10bits, try DXGI_FORMAT_R10G10B10A2_UNORM first
|
|
|
|
|
* Otherwise, use DXGI_FORMAT_B8G8R8A8_UNORM or DXGI_FORMAT_B8G8R8A8_UNORM
|
|
|
|
|
*/
|
|
|
|
|
gst_video_info_from_caps (&window->info, caps);
|
|
|
|
|
device_handle = gst_d3d11_device_get_device_handle (window->device);
|
|
|
|
|
for (i = 0; i < G_N_ELEMENTS (formats); i++) {
|
|
|
|
|
hr = device_handle->CheckFormatSupport (formats[i].dxgi_format,
|
|
|
|
|
&supported_flags);
|
|
|
|
|
if (SUCCEEDED (hr) && (supported_flags & display_flags) == display_flags) {
|
|
|
|
|
GST_DEBUG_OBJECT (window, "Device supports format %s (DXGI_FORMAT %d)",
|
|
|
|
|
gst_video_format_to_string (formats[i].gst_format),
|
|
|
|
|
formats[i].dxgi_format);
|
|
|
|
|
formats[i].supported = TRUE;
|
|
|
|
|
num_supported_format++;
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-02-05 13:12:15 +00:00
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
if (num_supported_format == 0) {
|
|
|
|
|
GST_ERROR_OBJECT (window, "Cannot determine render format");
|
|
|
|
|
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
|
|
|
|
|
"Cannot determine render format");
|
|
|
|
|
return FALSE;
|
|
|
|
|
}
|
2021-03-25 12:17:07 +00:00
|
|
|
|
|
|
|
|
for (i = 0; i < GST_VIDEO_INFO_N_COMPONENTS (&window->info); i++) {
|
|
|
|
|
if (GST_VIDEO_INFO_COMP_DEPTH (&window->info, i) > 8) {
|
|
|
|
|
if (formats[2].supported) {
|
|
|
|
|
chosen_format = &formats[2];
|
2020-03-09 11:29:17 +00:00
|
|
|
}
|
2021-03-25 12:17:07 +00:00
|
|
|
break;
|
2020-03-09 11:29:17 +00:00
|
|
|
}
|
|
|
|
|
}
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
if (!chosen_format) {
|
|
|
|
|
/* prefer native format over conversion */
|
|
|
|
|
for (i = 0; i < 2; i++) {
|
|
|
|
|
if (formats[i].supported &&
|
|
|
|
|
formats[i].gst_format == GST_VIDEO_INFO_FORMAT (&window->info)) {
|
|
|
|
|
chosen_format = &formats[i];
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
/* choose any color space then */
|
|
|
|
|
if (!chosen_format) {
|
|
|
|
|
for (i = 0; i < G_N_ELEMENTS (formats); i++) {
|
|
|
|
|
if (formats[i].supported) {
|
|
|
|
|
chosen_format = &formats[i];
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
g_assert (chosen_format != NULL);
|
|
|
|
|
|
2021-12-09 10:51:04 +00:00
|
|
|
GST_DEBUG_OBJECT (window, "chosen render format %s (DXGI_FORMAT %d)",
|
2020-03-09 11:29:17 +00:00
|
|
|
gst_video_format_to_string (chosen_format->gst_format),
|
|
|
|
|
chosen_format->dxgi_format);
|
|
|
|
|
|
|
|
|
|
/* Step 3: Create swapchain
|
|
|
|
|
* (or reuse old swapchain if the format is not changed) */
|
|
|
|
|
window->allow_tearing = FALSE;
|
2021-10-08 08:16:02 +00:00
|
|
|
|
|
|
|
|
{
|
|
|
|
|
ComPtr < IDXGIFactory5 > factory5;
|
|
|
|
|
IDXGIFactory1 *factory_handle;
|
|
|
|
|
BOOL allow_tearing = FALSE;
|
|
|
|
|
|
|
|
|
|
factory_handle = gst_d3d11_device_get_dxgi_factory_handle (window->device);
|
|
|
|
|
hr = factory_handle->QueryInterface (IID_PPV_ARGS (&factory5));
|
|
|
|
|
if (SUCCEEDED (hr)) {
|
|
|
|
|
hr = factory5->CheckFeatureSupport (DXGI_FEATURE_PRESENT_ALLOW_TEARING,
|
|
|
|
|
(void *) &allow_tearing, sizeof (allow_tearing));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (SUCCEEDED (hr) && allow_tearing)
|
|
|
|
|
window->allow_tearing = allow_tearing;
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
if (window->allow_tearing) {
|
2022-06-21 11:04:15 +00:00
|
|
|
GST_DEBUG_OBJECT (window, "device supports tearing");
|
2020-03-09 11:29:17 +00:00
|
|
|
swapchain_flags |= DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING;
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_lock (window->device);
|
2020-03-09 11:29:17 +00:00
|
|
|
window->dxgi_format = chosen_format->dxgi_format;
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
if (!window->swap_chain &&
|
|
|
|
|
!klass->create_swap_chain (window, window->dxgi_format,
|
2021-03-14 04:01:37 +00:00
|
|
|
display_width, display_height, swapchain_flags,
|
|
|
|
|
&window->swap_chain)) {
|
2020-03-09 11:29:17 +00:00
|
|
|
GST_ERROR_OBJECT (window, "Cannot create swapchain");
|
2019-12-30 09:58:59 +00:00
|
|
|
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
|
2020-03-09 11:29:17 +00:00
|
|
|
"Cannot create swapchain");
|
2020-12-03 18:40:17 +00:00
|
|
|
goto error;
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
/* this rect struct will be used to calculate render area */
|
|
|
|
|
window->render_rect.left = 0;
|
|
|
|
|
window->render_rect.top = 0;
|
|
|
|
|
window->render_rect.right = display_width;
|
|
|
|
|
window->render_rect.bottom = display_height;
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
window->input_rect.left = 0;
|
|
|
|
|
window->input_rect.top = 0;
|
|
|
|
|
window->input_rect.right = GST_VIDEO_INFO_WIDTH (&window->info);
|
|
|
|
|
window->input_rect.bottom = GST_VIDEO_INFO_HEIGHT (&window->info);
|
2020-01-29 12:10:00 +00:00
|
|
|
|
2022-06-15 17:18:43 +00:00
|
|
|
window->prev_input_rect = window->input_rect;
|
|
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
/* Step 4: Decide render color space and set it on converter/processor */
|
|
|
|
|
{
|
|
|
|
|
GstVideoMasteringDisplayInfo minfo;
|
|
|
|
|
GstVideoContentLightLevel cll;
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2020-03-09 11:29:17 +00:00
|
|
|
if (gst_video_mastering_display_info_from_caps (&minfo, caps) &&
|
|
|
|
|
gst_video_content_light_level_from_caps (&cll, caps)) {
|
2021-08-09 17:48:01 +00:00
|
|
|
ComPtr < IDXGISwapChain4 > swapchain4;
|
2020-03-09 11:29:17 +00:00
|
|
|
HRESULT hr;
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2021-08-09 17:48:01 +00:00
|
|
|
hr = window->swap_chain->QueryInterface (IID_PPV_ARGS (&swapchain4));
|
2020-03-09 11:29:17 +00:00
|
|
|
if (gst_d3d11_result (hr, window->device)) {
|
|
|
|
|
GST_DEBUG_OBJECT (window, "Have HDR metadata, set to DXGI swapchain");
|
|
|
|
|
|
|
|
|
|
gst_d3d11_hdr_meta_data_to_dxgi (&minfo, &cll, &hdr10_metadata);
|
|
|
|
|
|
|
|
|
|
hr = swapchain4->SetHDRMetaData (DXGI_HDR_METADATA_TYPE_HDR10,
|
|
|
|
|
sizeof (DXGI_HDR_METADATA_HDR10), &hdr10_metadata);
|
|
|
|
|
if (!gst_d3d11_result (hr, window->device)) {
|
|
|
|
|
GST_WARNING_OBJECT (window, "Couldn't set HDR metadata, hr 0x%x",
|
|
|
|
|
(guint) hr);
|
|
|
|
|
} else {
|
|
|
|
|
have_hdr10 = TRUE;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Step 5: Choose display color space */
|
|
|
|
|
gst_video_info_set_format (&window->render_info,
|
|
|
|
|
chosen_format->gst_format, display_width, display_height);
|
|
|
|
|
|
|
|
|
|
/* preserve upstream colorimetry */
|
2019-12-30 09:58:59 +00:00
|
|
|
window->render_info.colorimetry.primaries =
|
|
|
|
|
window->info.colorimetry.primaries;
|
|
|
|
|
window->render_info.colorimetry.transfer = window->info.colorimetry.transfer;
|
2020-11-16 11:44:06 +00:00
|
|
|
/* prefer FULL range RGB. STUDIO range doesn't seem to be well supported
|
|
|
|
|
* color space by GPUs and we don't need to preserve color range for
|
|
|
|
|
* target display color space type */
|
|
|
|
|
window->render_info.colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255;
|
2020-03-09 11:29:17 +00:00
|
|
|
|
|
|
|
|
{
|
2021-08-09 17:48:01 +00:00
|
|
|
ComPtr < IDXGISwapChain3 > swapchain3;
|
2020-03-09 11:29:17 +00:00
|
|
|
HRESULT hr;
|
|
|
|
|
|
2021-08-09 17:48:01 +00:00
|
|
|
hr = window->swap_chain->QueryInterface (IID_PPV_ARGS (&swapchain3));
|
2020-03-09 11:29:17 +00:00
|
|
|
|
|
|
|
|
if (gst_d3d11_result (hr, window->device)) {
|
2021-03-14 04:01:37 +00:00
|
|
|
chosen_colorspace =
|
|
|
|
|
gst_d3d11_find_swap_chain_color_space (&window->render_info,
|
2021-12-09 10:55:04 +00:00
|
|
|
swapchain3.Get ());
|
2020-03-09 11:29:17 +00:00
|
|
|
if (chosen_colorspace) {
|
|
|
|
|
native_colorspace_type =
|
|
|
|
|
(DXGI_COLOR_SPACE_TYPE) chosen_colorspace->dxgi_color_space_type;
|
|
|
|
|
hr = swapchain3->SetColorSpace1 (native_colorspace_type);
|
|
|
|
|
if (!gst_d3d11_result (hr, window->device)) {
|
|
|
|
|
GST_WARNING_OBJECT (window, "Failed to set colorspace %d, hr: 0x%x",
|
2021-03-14 04:01:37 +00:00
|
|
|
native_colorspace_type, (guint) hr);
|
2020-03-09 11:29:17 +00:00
|
|
|
chosen_colorspace = NULL;
|
|
|
|
|
native_colorspace_type = DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709;
|
|
|
|
|
} else {
|
|
|
|
|
GST_DEBUG_OBJECT (window,
|
|
|
|
|
"Set colorspace %d", native_colorspace_type);
|
|
|
|
|
|
|
|
|
|
/* update with selected display color space */
|
|
|
|
|
window->render_info.colorimetry.primaries =
|
|
|
|
|
chosen_colorspace->primaries;
|
|
|
|
|
window->render_info.colorimetry.transfer =
|
|
|
|
|
chosen_colorspace->transfer;
|
2021-03-14 04:01:37 +00:00
|
|
|
window->render_info.colorimetry.range = chosen_colorspace->range;
|
|
|
|
|
window->render_info.colorimetry.matrix = chosen_colorspace->matrix;
|
2020-03-09 11:29:17 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* otherwise, use most common DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709
|
|
|
|
|
* color space */
|
|
|
|
|
if (!chosen_colorspace) {
|
|
|
|
|
GST_DEBUG_OBJECT (window, "No selected render color space, use BT709");
|
|
|
|
|
window->render_info.colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_BT709;
|
|
|
|
|
window->render_info.colorimetry.transfer = GST_VIDEO_TRANSFER_BT709;
|
|
|
|
|
window->render_info.colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255;
|
|
|
|
|
}
|
2022-02-10 13:31:06 +00:00
|
|
|
|
2020-10-30 18:28:55 +00:00
|
|
|
if (chosen_colorspace) {
|
|
|
|
|
const GstDxgiColorSpace *in_color_space =
|
|
|
|
|
gst_d3d11_video_info_to_dxgi_color_space (&window->info);
|
2022-02-12 19:36:39 +00:00
|
|
|
GstD3D11Format in_format;
|
2020-10-30 18:28:55 +00:00
|
|
|
gboolean hardware = FALSE;
|
|
|
|
|
GstD3D11VideoProcessor *processor = NULL;
|
2022-02-12 19:36:39 +00:00
|
|
|
DXGI_FORMAT in_dxgi_format;
|
2020-07-21 14:44:03 +00:00
|
|
|
|
2022-02-12 19:36:39 +00:00
|
|
|
gst_d3d11_device_get_format (window->device,
|
|
|
|
|
GST_VIDEO_INFO_FORMAT (&window->info), &in_format);
|
|
|
|
|
in_dxgi_format = in_format.dxgi_format;
|
|
|
|
|
|
|
|
|
|
if (in_color_space && in_format.dxgi_format != DXGI_FORMAT_UNKNOWN) {
|
2020-10-30 18:28:55 +00:00
|
|
|
g_object_get (window->device, "hardware", &hardware, NULL);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (hardware) {
|
|
|
|
|
processor =
|
2020-06-12 11:18:53 +00:00
|
|
|
gst_d3d11_video_processor_new (window->device,
|
|
|
|
|
GST_VIDEO_INFO_WIDTH (&window->info),
|
2021-03-14 04:01:37 +00:00
|
|
|
GST_VIDEO_INFO_HEIGHT (&window->info), display_width, display_height);
|
2020-01-29 12:10:00 +00:00
|
|
|
}
|
|
|
|
|
|
2020-10-30 18:28:55 +00:00
|
|
|
if (processor) {
|
|
|
|
|
DXGI_FORMAT out_dxgi_format = chosen_format->dxgi_format;
|
|
|
|
|
DXGI_COLOR_SPACE_TYPE in_dxgi_color_space =
|
|
|
|
|
(DXGI_COLOR_SPACE_TYPE) in_color_space->dxgi_color_space_type;
|
|
|
|
|
DXGI_COLOR_SPACE_TYPE out_dxgi_color_space = native_colorspace_type;
|
|
|
|
|
|
|
|
|
|
if (!gst_d3d11_video_processor_check_format_conversion (processor,
|
|
|
|
|
in_dxgi_format, in_dxgi_color_space, out_dxgi_format,
|
|
|
|
|
out_dxgi_color_space)) {
|
|
|
|
|
GST_DEBUG_OBJECT (window, "Conversion is not supported by device");
|
|
|
|
|
gst_d3d11_video_processor_free (processor);
|
|
|
|
|
processor = NULL;
|
|
|
|
|
} else {
|
|
|
|
|
GST_DEBUG_OBJECT (window, "video processor supports conversion");
|
|
|
|
|
gst_d3d11_video_processor_set_input_dxgi_color_space (processor,
|
|
|
|
|
in_dxgi_color_space);
|
|
|
|
|
gst_d3d11_video_processor_set_output_dxgi_color_space (processor,
|
|
|
|
|
out_dxgi_color_space);
|
2020-03-09 11:29:17 +00:00
|
|
|
|
2020-10-30 18:28:55 +00:00
|
|
|
if (have_hdr10) {
|
|
|
|
|
GST_DEBUG_OBJECT (window, "Set HDR metadata on video processor");
|
|
|
|
|
gst_d3d11_video_processor_set_input_hdr10_metadata (processor,
|
|
|
|
|
&hdr10_metadata);
|
|
|
|
|
gst_d3d11_video_processor_set_output_hdr10_metadata (processor,
|
|
|
|
|
&hdr10_metadata);
|
2020-03-09 11:29:17 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-30 18:28:55 +00:00
|
|
|
window->processor = processor;
|
2020-01-29 12:10:00 +00:00
|
|
|
}
|
|
|
|
|
}
|
2022-02-10 13:31:06 +00:00
|
|
|
|
2020-10-30 18:28:55 +00:00
|
|
|
*video_processor_available = !!window->processor;
|
2020-01-29 12:10:00 +00:00
|
|
|
|
|
|
|
|
/* configure shader even if video processor is available for fallback */
|
2019-12-30 09:58:59 +00:00
|
|
|
window->converter =
|
2021-03-13 08:40:57 +00:00
|
|
|
gst_d3d11_converter_new (window->device, &window->info,
|
2022-06-15 17:18:43 +00:00
|
|
|
&window->render_info);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
if (!window->converter) {
|
|
|
|
|
GST_ERROR_OBJECT (window, "Cannot create converter");
|
|
|
|
|
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
|
|
|
|
|
"Cannot create converter");
|
2020-12-03 18:40:17 +00:00
|
|
|
goto error;
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
window->compositor =
|
|
|
|
|
gst_d3d11_overlay_compositor_new (window->device, &window->render_info);
|
|
|
|
|
if (!window->compositor) {
|
|
|
|
|
GST_ERROR_OBJECT (window, "Cannot create overlay compositor");
|
|
|
|
|
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
|
|
|
|
|
"Cannot create overlay compositor");
|
2020-12-03 18:40:17 +00:00
|
|
|
goto error;
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_unlock (window->device);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2020-01-10 12:45:43 +00:00
|
|
|
/* call resize to allocated resources */
|
2020-03-02 11:55:29 +00:00
|
|
|
klass->on_resize (window, display_width, display_height);
|
2020-01-10 12:45:43 +00:00
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
if (window->requested_fullscreen != window->fullscreen) {
|
|
|
|
|
klass->change_fullscreen_mode (window);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
GST_DEBUG_OBJECT (window, "New swap chain 0x%p created", window->swap_chain);
|
|
|
|
|
|
|
|
|
|
return TRUE;
|
2020-12-03 18:40:17 +00:00
|
|
|
|
|
|
|
|
error:
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_unlock (window->device);
|
2020-12-03 18:40:17 +00:00
|
|
|
|
|
|
|
|
return FALSE;
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
gst_d3d11_window_show (GstD3D11Window * window)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass;
|
|
|
|
|
|
|
|
|
|
g_return_if_fail (GST_IS_D3D11_WINDOW (window));
|
|
|
|
|
|
|
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
|
|
|
|
|
if (klass->show)
|
|
|
|
|
klass->show (window);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
2021-07-30 15:59:14 +00:00
|
|
|
gst_d3d11_window_set_render_rectangle (GstD3D11Window * window,
|
|
|
|
|
const GstVideoRectangle * rect)
|
2019-12-30 09:58:59 +00:00
|
|
|
{
|
2021-07-30 15:59:14 +00:00
|
|
|
GstD3D11WindowClass *klass;
|
|
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
g_return_if_fail (GST_IS_D3D11_WINDOW (window));
|
|
|
|
|
|
2021-07-30 15:59:14 +00:00
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
|
|
|
|
|
if (klass->set_render_rectangle)
|
|
|
|
|
klass->set_render_rectangle (window, rect);
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
2021-09-17 17:27:51 +00:00
|
|
|
void
|
|
|
|
|
gst_d3d11_window_set_title (GstD3D11Window * window, const gchar * title)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass;
|
|
|
|
|
|
|
|
|
|
g_return_if_fail (GST_IS_D3D11_WINDOW (window));
|
|
|
|
|
|
|
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
|
|
|
|
|
if (klass->set_title)
|
|
|
|
|
klass->set_title (window, title);
|
|
|
|
|
}
|
|
|
|
|
|
2020-01-29 12:10:00 +00:00
|
|
|
static gboolean
|
|
|
|
|
gst_d3d11_window_buffer_ensure_processor_input (GstD3D11Window * self,
|
|
|
|
|
GstBuffer * buffer, ID3D11VideoProcessorInputView ** in_view)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11Memory *mem;
|
2020-12-19 17:39:40 +00:00
|
|
|
ID3D11VideoProcessorInputView *piv;
|
2020-01-29 12:10:00 +00:00
|
|
|
|
|
|
|
|
if (!self->processor)
|
|
|
|
|
return FALSE;
|
|
|
|
|
|
|
|
|
|
if (gst_buffer_n_memory (buffer) != 1)
|
|
|
|
|
return FALSE;
|
|
|
|
|
|
|
|
|
|
mem = (GstD3D11Memory *) gst_buffer_peek_memory (buffer, 0);
|
2020-12-19 17:39:40 +00:00
|
|
|
piv = gst_d3d11_video_processor_get_input_view (self->processor, mem);
|
|
|
|
|
if (!piv) {
|
|
|
|
|
GST_LOG_OBJECT (self, "Failed to get processor input view");
|
2020-10-23 07:59:00 +00:00
|
|
|
return FALSE;
|
2020-01-29 12:10:00 +00:00
|
|
|
}
|
|
|
|
|
|
2020-12-19 17:39:40 +00:00
|
|
|
*in_view = piv;
|
2020-01-29 12:10:00 +00:00
|
|
|
|
|
|
|
|
return TRUE;
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-15 10:48:56 +00:00
|
|
|
static gboolean
|
|
|
|
|
gst_d3d11_window_do_processor (GstD3D11Window * self,
|
2021-09-18 14:37:20 +00:00
|
|
|
ID3D11VideoProcessorInputView * piv, ID3D11VideoProcessorOutputView * pov,
|
|
|
|
|
RECT * input_rect)
|
2021-03-15 10:48:56 +00:00
|
|
|
{
|
|
|
|
|
gboolean ret;
|
|
|
|
|
|
|
|
|
|
ret = gst_d3d11_video_processor_render_unlocked (self->processor,
|
2021-09-18 14:37:20 +00:00
|
|
|
input_rect, piv, &self->render_rect, pov);
|
2021-03-15 10:48:56 +00:00
|
|
|
if (!ret) {
|
|
|
|
|
GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using processor");
|
|
|
|
|
} else {
|
|
|
|
|
GST_TRACE_OBJECT (self, "Rendered using processor");
|
|
|
|
|
self->processor_in_use = TRUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static gboolean
|
|
|
|
|
gst_d3d11_window_do_convert (GstD3D11Window * self,
|
|
|
|
|
ID3D11ShaderResourceView * srv[GST_VIDEO_MAX_PLANES],
|
2021-09-18 14:37:20 +00:00
|
|
|
ID3D11RenderTargetView * rtv, RECT * input_rect)
|
2021-03-15 10:48:56 +00:00
|
|
|
{
|
2022-06-15 17:18:43 +00:00
|
|
|
RECT *prev = &self->prev_input_rect;
|
|
|
|
|
|
|
|
|
|
if (input_rect->left != prev->left || input_rect->top != prev->top ||
|
|
|
|
|
input_rect->right != prev->right || input_rect->bottom != prev->bottom) {
|
|
|
|
|
g_object_set (self->converter, "src-x", (gint) input_rect->left,
|
|
|
|
|
"src-y", (gint) input_rect->top,
|
|
|
|
|
"src-width", (gint) (input_rect->right - input_rect->left),
|
|
|
|
|
"src-height", (gint) (input_rect->bottom - input_rect->top), nullptr);
|
|
|
|
|
|
|
|
|
|
*prev = *input_rect;
|
2021-09-18 14:37:20 +00:00
|
|
|
}
|
|
|
|
|
|
2022-06-15 17:18:43 +00:00
|
|
|
if (!gst_d3d11_converter_convert_unlocked (self->converter, srv, &rtv)) {
|
2021-03-15 10:48:56 +00:00
|
|
|
GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using converter");
|
|
|
|
|
return FALSE;
|
|
|
|
|
} else {
|
|
|
|
|
GST_TRACE_OBJECT (self, "Rendered using converter");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return TRUE;
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
static GstFlowReturn
|
2020-11-21 19:39:57 +00:00
|
|
|
gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer,
|
2021-03-25 12:17:07 +00:00
|
|
|
ID3D11VideoProcessorOutputView * pov, ID3D11RenderTargetView * rtv)
|
2019-12-30 09:58:59 +00:00
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass = GST_D3D11_WINDOW_GET_CLASS (self);
|
|
|
|
|
GstFlowReturn ret = GST_FLOW_OK;
|
|
|
|
|
guint present_flags = 0;
|
|
|
|
|
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
if (!buffer)
|
|
|
|
|
return GST_FLOW_OK;
|
2019-12-30 09:58:59 +00:00
|
|
|
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
{
|
2020-12-19 17:39:40 +00:00
|
|
|
GstMapInfo infos[GST_VIDEO_MAX_PLANES];
|
2019-12-30 09:58:59 +00:00
|
|
|
ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES];
|
2020-01-29 12:10:00 +00:00
|
|
|
ID3D11VideoProcessorInputView *piv = NULL;
|
2020-12-19 17:39:40 +00:00
|
|
|
ID3D11Device *device_handle =
|
|
|
|
|
gst_d3d11_device_get_device_handle (self->device);
|
2021-03-15 10:48:56 +00:00
|
|
|
gboolean can_convert = FALSE;
|
|
|
|
|
gboolean can_process = FALSE;
|
|
|
|
|
gboolean convert_ret = FALSE;
|
2021-09-18 14:37:20 +00:00
|
|
|
RECT input_rect = self->input_rect;
|
|
|
|
|
GstVideoCropMeta *crop_meta;
|
2020-12-19 17:39:40 +00:00
|
|
|
|
|
|
|
|
/* Map memory in any case so that we can upload pending stage texture */
|
2021-03-14 04:01:37 +00:00
|
|
|
if (!gst_d3d11_buffer_map (buffer, device_handle, infos, GST_MAP_READ)) {
|
2020-12-19 17:39:40 +00:00
|
|
|
GST_ERROR_OBJECT (self, "Couldn't map buffer");
|
|
|
|
|
return GST_FLOW_ERROR;
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-15 10:48:56 +00:00
|
|
|
can_convert = gst_d3d11_buffer_get_shader_resource_view (buffer, srv);
|
|
|
|
|
if (pov) {
|
|
|
|
|
can_process = gst_d3d11_window_buffer_ensure_processor_input (self,
|
|
|
|
|
buffer, &piv);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!can_convert && !can_process) {
|
|
|
|
|
GST_ERROR_OBJECT (self, "Input texture cannot be used for converter");
|
|
|
|
|
return GST_FLOW_ERROR;
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
2021-09-18 14:37:20 +00:00
|
|
|
crop_meta = gst_buffer_get_video_crop_meta (buffer);
|
|
|
|
|
/* Do minimal validate */
|
|
|
|
|
if (crop_meta) {
|
|
|
|
|
ID3D11Texture2D *texture = (ID3D11Texture2D *) infos[0].data;
|
|
|
|
|
D3D11_TEXTURE2D_DESC desc = { 0, };
|
|
|
|
|
|
|
|
|
|
texture->GetDesc (&desc);
|
|
|
|
|
|
|
|
|
|
if (desc.Width < crop_meta->x + crop_meta->width ||
|
|
|
|
|
desc.Height < crop_meta->y + crop_meta->height) {
|
|
|
|
|
GST_WARNING_OBJECT (self, "Invalid crop meta, ignore");
|
|
|
|
|
|
|
|
|
|
crop_meta = nullptr;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (crop_meta) {
|
|
|
|
|
input_rect.left = crop_meta->x;
|
|
|
|
|
input_rect.right = crop_meta->x + crop_meta->width;
|
|
|
|
|
input_rect.top = crop_meta->y;
|
|
|
|
|
input_rect.bottom = crop_meta->y + crop_meta->height;
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
if (self->first_present) {
|
2020-06-02 15:46:13 +00:00
|
|
|
D3D11_VIEWPORT viewport;
|
|
|
|
|
|
|
|
|
|
viewport.TopLeftX = self->render_rect.left;
|
|
|
|
|
viewport.TopLeftY = self->render_rect.top;
|
|
|
|
|
viewport.Width = self->render_rect.right - self->render_rect.left;
|
|
|
|
|
viewport.Height = self->render_rect.bottom - self->render_rect.top;
|
|
|
|
|
viewport.MinDepth = 0.0f;
|
|
|
|
|
viewport.MaxDepth = 1.0f;
|
2022-06-15 17:18:43 +00:00
|
|
|
|
|
|
|
|
g_object_set (self->converter, "dest-x", (gint) self->render_rect.left,
|
|
|
|
|
"dest-y", (gint) self->render_rect.top,
|
|
|
|
|
"dest-width",
|
|
|
|
|
(gint) (self->render_rect.right - self->render_rect.left),
|
|
|
|
|
"dest-height",
|
|
|
|
|
(gint) (self->render_rect.bottom - self->render_rect.top), nullptr);
|
2020-06-02 15:46:13 +00:00
|
|
|
gst_d3d11_overlay_compositor_update_viewport (self->compositor,
|
|
|
|
|
&viewport);
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
2021-03-15 10:48:56 +00:00
|
|
|
/* Converter preference order
|
|
|
|
|
* 1) If this texture can be converted via processor, and we used processor
|
|
|
|
|
* previously, use processor
|
|
|
|
|
* 2) If SRV is available, use converter
|
|
|
|
|
* 3) otherwise, use processor
|
|
|
|
|
*/
|
|
|
|
|
if (can_process && self->processor_in_use) {
|
2021-09-18 14:37:20 +00:00
|
|
|
convert_ret = gst_d3d11_window_do_processor (self, piv, pov, &input_rect);
|
2021-03-15 10:48:56 +00:00
|
|
|
} else if (can_convert) {
|
2021-09-18 14:37:20 +00:00
|
|
|
convert_ret = gst_d3d11_window_do_convert (self, srv, rtv, &input_rect);
|
2021-03-15 10:48:56 +00:00
|
|
|
} else if (can_process) {
|
2021-09-18 14:37:20 +00:00
|
|
|
convert_ret = gst_d3d11_window_do_processor (self, piv, pov, &input_rect);
|
2020-01-29 12:10:00 +00:00
|
|
|
} else {
|
2021-03-15 10:48:56 +00:00
|
|
|
g_assert_not_reached ();
|
|
|
|
|
ret = GST_FLOW_ERROR;
|
|
|
|
|
goto unmap_and_out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!convert_ret) {
|
|
|
|
|
ret = GST_FLOW_ERROR;
|
|
|
|
|
goto unmap_and_out;
|
2020-01-29 12:10:00 +00:00
|
|
|
}
|
2019-12-30 09:58:59 +00:00
|
|
|
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
gst_d3d11_overlay_compositor_upload (self->compositor, buffer);
|
|
|
|
|
gst_d3d11_overlay_compositor_draw_unlocked (self->compositor, &rtv);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2020-02-26 04:11:30 +00:00
|
|
|
if (self->allow_tearing && self->fullscreen) {
|
2019-12-30 09:58:59 +00:00
|
|
|
present_flags |= DXGI_PRESENT_ALLOW_TEARING;
|
|
|
|
|
}
|
|
|
|
|
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
if (klass->present)
|
|
|
|
|
ret = klass->present (self, present_flags);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
self->first_present = FALSE;
|
2020-12-19 17:39:40 +00:00
|
|
|
|
2021-03-14 04:01:37 +00:00
|
|
|
unmap_and_out:
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
gst_d3d11_buffer_unmap (buffer, infos);
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
GstFlowReturn
|
2021-08-02 07:22:06 +00:00
|
|
|
gst_d3d11_window_render (GstD3D11Window * window, GstBuffer * buffer)
|
2019-12-30 09:58:59 +00:00
|
|
|
{
|
|
|
|
|
GstMemory *mem;
|
|
|
|
|
GstFlowReturn ret;
|
|
|
|
|
|
|
|
|
|
g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), GST_FLOW_ERROR);
|
|
|
|
|
|
2021-08-02 07:22:06 +00:00
|
|
|
if (buffer) {
|
|
|
|
|
mem = gst_buffer_peek_memory (buffer, 0);
|
|
|
|
|
if (!gst_is_d3d11_memory (mem)) {
|
|
|
|
|
GST_ERROR_OBJECT (window, "Invalid buffer");
|
2019-12-30 09:58:59 +00:00
|
|
|
|
2021-08-02 07:22:06 +00:00
|
|
|
return GST_FLOW_ERROR;
|
|
|
|
|
}
|
2019-12-30 09:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_lock (window->device);
|
2021-08-02 07:22:06 +00:00
|
|
|
if (buffer)
|
|
|
|
|
gst_buffer_replace (&window->cached_buffer, buffer);
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
|
2021-03-25 12:17:07 +00:00
|
|
|
ret = gst_d3d111_window_present (window, window->cached_buffer,
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
window->pov, window->rtv);
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_unlock (window->device);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
GstFlowReturn
|
|
|
|
|
gst_d3d11_window_render_on_shared_handle (GstD3D11Window * window,
|
|
|
|
|
GstBuffer * buffer, HANDLE shared_handle, guint texture_misc_flags,
|
|
|
|
|
guint64 acquire_key, guint64 release_key)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass;
|
|
|
|
|
GstMemory *mem;
|
|
|
|
|
GstFlowReturn ret = GST_FLOW_OK;
|
|
|
|
|
GstD3D11WindowSharedHandleData data = { NULL, };
|
|
|
|
|
ID3D11VideoProcessorOutputView *pov = NULL;
|
|
|
|
|
ID3D11RenderTargetView *rtv = NULL;
|
|
|
|
|
|
|
|
|
|
g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), GST_FLOW_ERROR);
|
|
|
|
|
|
|
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
|
|
|
|
|
g_assert (klass->open_shared_handle != NULL);
|
|
|
|
|
g_assert (klass->release_shared_handle != NULL);
|
|
|
|
|
|
|
|
|
|
mem = gst_buffer_peek_memory (buffer, 0);
|
|
|
|
|
if (!gst_is_d3d11_memory (mem)) {
|
|
|
|
|
GST_ERROR_OBJECT (window, "Invalid buffer");
|
|
|
|
|
|
|
|
|
|
return GST_FLOW_ERROR;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
data.shared_handle = shared_handle;
|
|
|
|
|
data.texture_misc_flags = texture_misc_flags;
|
|
|
|
|
data.acquire_key = acquire_key;
|
|
|
|
|
data.release_key = release_key;
|
|
|
|
|
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_lock (window->device);
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
if (!klass->open_shared_handle (window, &data)) {
|
|
|
|
|
GST_ERROR_OBJECT (window, "Couldn't open shared handle");
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_unlock (window->device);
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
return GST_FLOW_OK;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (data.fallback_rtv) {
|
|
|
|
|
rtv = data.fallback_rtv;
|
|
|
|
|
pov = data.fallback_pov;
|
|
|
|
|
} else {
|
|
|
|
|
rtv = data.rtv;
|
|
|
|
|
pov = data.pov;
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-25 12:17:07 +00:00
|
|
|
ret = gst_d3d111_window_present (window, buffer, pov, rtv);
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
|
|
|
|
|
klass->release_shared_handle (window, &data);
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_unlock (window->device);
|
d3d11videosink: Add support for drawing on application's own texture
Add a way to support drawing on application's texture instead of
usual window handle.
To make use of this new feature, application should follow below step.
1) Enable this feature by using "draw-on-shared-texture" property
2) Watch "begin-draw" signal
3) On "begin-draw" signal handler, application can request drawing
by using "draw" signal action. Note that "draw" signal action
should be happen before "begin-draw" signal handler is returned
NOTE 1) For texture sharing, creating a texture with
D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag is strongly recommend
if possible because we cannot ensure sync a texture
which was created with D3D11_RESOURCE_MISC_SHARED
and it would cause glitch with ID3D11VideoProcessor use case.
NOTE 2) Direct9Ex doesn't support texture sharing which was
created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX. In other words,
D3D11_RESOURCE_MISC_SHARED is the only option for Direct3D11/Direct9Ex interop.
NOTE 3) Because of missing synchronization around ID3D11VideoProcessor,
If shared texture was created with D3D11_RESOURCE_MISC_SHARED,
d3d11videosink might use fallback texture to convert DXVA texture
to normal Direct3D texture. Then converted texture will be
copied to user-provided shared texture.
* Why not use generic appsink approach?
In order for application to be able to store video data
which was produced by GStreamer in application's own texture,
there would be two possible approaches,
one is copying our texture into application's own texture,
and the other is drawing on application's own texture directly.
The former (appsink way) cannot be a zero-copy by nature.
In order to support zero-copy processing, we need to draw on
application's own texture directly.
For example, assume that application wants RGBA texture.
Then we can imagine following case.
"d3d11h264dec ! d3d11convert ! video/x-raw(memory:D3D11Memory),format=RGBA ! appsink"
^
|_ allocate new Direct3D texture for RGBA format
In above case, d3d11convert will allocate new texture(s) for RGBA format
and then application will copy again the our RGBA texutre into
application's own texture. One texture allocation plus per frame GPU copy will hanppen
in that case therefore.
Moreover, in order for application to be able to access
our texture, we need to allocate texture with additional flags for
application's Direct3D11 device to be able to read texture data.
That would be another implementation burden on our side
But with this MR, we can configure pipeline in this way
"d3d11h264dec ! d3d11videosink".
In that way, we can save at least one texture allocation and
per frame texutre copy since d3d11videosink will convert incoming texture
into application's texture format directly without copy.
* What if we expose texture without conversion and application does
conversion by itself?
As mentioned above, for application to be able to access our texture
from application's Direct3D11 device, we need to allocate texture
in a special form. But in some case, that might not be possible.
Also, if a texture belongs to decoder DPB, exposing such texture
to application is unsafe and usual Direct3D11 shader cannot handle
such texture. To convert format, ID3D11VideoProcessor API needs to
be used but that would be a implementation burden for application.
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1873>
2020-12-23 14:49:12 +00:00
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
gboolean
|
|
|
|
|
gst_d3d11_window_unlock (GstD3D11Window * window)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass;
|
|
|
|
|
gboolean ret = TRUE;
|
|
|
|
|
|
|
|
|
|
g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE);
|
|
|
|
|
|
|
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
|
|
|
|
|
if (klass->unlock)
|
|
|
|
|
ret = klass->unlock (window);
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
gboolean
|
|
|
|
|
gst_d3d11_window_unlock_stop (GstD3D11Window * window)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass;
|
|
|
|
|
gboolean ret = TRUE;
|
|
|
|
|
|
|
|
|
|
g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE);
|
|
|
|
|
|
|
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
|
|
|
|
|
if (klass->unlock_stop)
|
|
|
|
|
ret = klass->unlock_stop (window);
|
|
|
|
|
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_lock (window->device);
|
2019-12-30 09:58:59 +00:00
|
|
|
gst_clear_buffer (&window->cached_buffer);
|
2021-03-20 07:15:35 +00:00
|
|
|
gst_d3d11_device_unlock (window->device);
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-26 16:52:59 +00:00
|
|
|
void
|
|
|
|
|
gst_d3d11_window_unprepare (GstD3D11Window * window)
|
|
|
|
|
{
|
|
|
|
|
GstD3D11WindowClass *klass;
|
|
|
|
|
|
|
|
|
|
g_return_if_fail (GST_IS_D3D11_WINDOW (window));
|
|
|
|
|
|
|
|
|
|
klass = GST_D3D11_WINDOW_GET_CLASS (window);
|
|
|
|
|
|
|
|
|
|
if (klass->unprepare)
|
|
|
|
|
klass->unprepare (window);
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-30 09:58:59 +00:00
|
|
|
GstD3D11WindowNativeType
|
|
|
|
|
gst_d3d11_window_get_native_type_from_handle (guintptr handle)
|
|
|
|
|
{
|
|
|
|
|
if (!handle)
|
|
|
|
|
return GST_D3D11_WINDOW_NATIVE_TYPE_NONE;
|
|
|
|
|
|
|
|
|
|
#if (!GST_D3D11_WINAPI_ONLY_APP)
|
|
|
|
|
if (IsWindow ((HWND) handle))
|
|
|
|
|
return GST_D3D11_WINDOW_NATIVE_TYPE_HWND;
|
2021-01-18 10:17:14 +00:00
|
|
|
#endif
|
|
|
|
|
#if GST_D3D11_WINAPI_ONLY_APP
|
2019-12-30 09:58:59 +00:00
|
|
|
{
|
2021-03-14 04:01:37 +00:00
|
|
|
/* *INDENT-OFF* */
|
2019-12-30 09:58:59 +00:00
|
|
|
ComPtr<IInspectable> window = reinterpret_cast<IInspectable*> (handle);
|
|
|
|
|
ComPtr<ABI::Windows::UI::Core::ICoreWindow> core_window;
|
|
|
|
|
ComPtr<ABI::Windows::UI::Xaml::Controls::ISwapChainPanel> panel;
|
2021-03-14 04:01:37 +00:00
|
|
|
/* *INDENT-ON* */
|
2019-12-30 09:58:59 +00:00
|
|
|
|
|
|
|
|
if (SUCCEEDED (window.As (&core_window)))
|
|
|
|
|
return GST_D3D11_WINDOW_NATIVE_TYPE_CORE_WINDOW;
|
|
|
|
|
|
|
|
|
|
if (SUCCEEDED (window.As (&panel)))
|
|
|
|
|
return GST_D3D11_WINDOW_NATIVE_TYPE_SWAP_CHAIN_PANEL;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
return GST_D3D11_WINDOW_NATIVE_TYPE_NONE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const gchar *
|
|
|
|
|
gst_d3d11_window_get_native_type_to_string (GstD3D11WindowNativeType type)
|
|
|
|
|
{
|
|
|
|
|
switch (type) {
|
|
|
|
|
case GST_D3D11_WINDOW_NATIVE_TYPE_NONE:
|
|
|
|
|
return "none";
|
|
|
|
|
case GST_D3D11_WINDOW_NATIVE_TYPE_HWND:
|
|
|
|
|
return "hwnd";
|
|
|
|
|
case GST_D3D11_WINDOW_NATIVE_TYPE_CORE_WINDOW:
|
|
|
|
|
return "core-window";
|
|
|
|
|
case GST_D3D11_WINDOW_NATIVE_TYPE_SWAP_CHAIN_PANEL:
|
|
|
|
|
return "swap-chain-panel";
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return "none";
|
2021-03-14 04:01:37 +00:00
|
|
|
}
|
