d3d11videosink: Add display-format property

Make swapchian's display format configurable, since some DXGI formats
we can use for swapchain are not API interop compatible.
For instance, BGRA format should be used for Direct2D interop.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2923>

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp

@@ -63,6 +63,7 @@ enum
   PROP_ROTATE_METHOD,
   PROP_GAMMA_MODE,
   PROP_PRIMARIES_MODE,
+  PROP_DISPLAY_FORMAT,
 };
 
 #define DEFAULT_ADAPTER                   -1
@@ -73,6 +74,32 @@ enum
 #define DEFAULT_DRAW_ON_SHARED_TEXTURE    FALSE
 #define DEFAULT_GAMMA_MODE                GST_VIDEO_GAMMA_MODE_NONE
 #define DEFAULT_PRIMARIES_MODE            GST_VIDEO_PRIMARIES_MODE_NONE
+#define DEFAULT_DISPLAY_FORMAT            DXGI_FORMAT_UNKNOWN
+
+#define GST_TYPE_D3D11_VIDEO_SINK_DISPLAY_FORMAT (gst_d3d11_video_sink_display_format_type())
+static GType
+gst_d3d11_video_sink_display_format_type (void)
+{
+  static GType format_type = 0;
+
+  GST_D3D11_CALL_ONCE_BEGIN {
+    static const GEnumValue format_types[] = {
+      {DXGI_FORMAT_UNKNOWN, "DXGI_FORMAT_UNKNOWN", "unknown"},
+      {DXGI_FORMAT_R10G10B10A2_UNORM,
+          "DXGI_FORMAT_R10G10B10A2_UNORM", "r10g10b10a2-unorm"},
+      {DXGI_FORMAT_R8G8B8A8_UNORM,
+          "DXGI_FORMAT_R8G8B8A8_UNORM", "r8g8b8a8-unorm"},
+      {DXGI_FORMAT_B8G8R8A8_UNORM,
+          "DXGI_FORMAT_B8G8R8A8_UNORM", "b8g8r8a8-unorm"},
+      {0, nullptr, nullptr},
+    };
+
+    format_type = g_enum_register_static ("GstD3D11VideoSinkDisplayFormat",
+        format_types);
+  } GST_D3D11_CALL_ONCE_END;
+
+  return format_type;
+}
 
 enum
 {
@@ -127,6 +154,7 @@ struct _GstD3D11VideoSink
   gboolean draw_on_shared_texture;
   GstVideoGammaMode gamma_mode;
   GstVideoPrimariesMode primaries_mode;
+  DXGI_FORMAT display_format;
 
   /* saved render rectangle until we have a window */
   GstVideoRectangle render_rect;
@@ -311,6 +339,19 @@ gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass)
           DEFAULT_PRIMARIES_MODE, (GParamFlags) (GST_PARAM_MUTABLE_READY |
               G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
 
+  /**
+   * GstD3D11VideoSink:display-format:
+   *
+   * Swapchain display format
+   *
+   * Since: 1.22
+   */
+  g_object_class_install_property (gobject_class, PROP_DISPLAY_FORMAT,
+      g_param_spec_enum ("display-format", "Display Format",
+          "Swapchain display format", GST_TYPE_D3D11_VIDEO_SINK_DISPLAY_FORMAT,
+          DEFAULT_DISPLAY_FORMAT, (GParamFlags) (G_PARAM_READWRITE |
+              GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS)));
+
   /**
    * GstD3D11VideoSink::begin-draw:
    * @videosink: the #d3d11videosink
@@ -409,6 +450,8 @@ gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass)
 
   gst_type_mark_as_plugin_api (GST_D3D11_WINDOW_TOGGLE_MODE_GET_TYPE,
       (GstPluginAPIFlags) 0);
+  gst_type_mark_as_plugin_api (GST_TYPE_D3D11_VIDEO_SINK_DISPLAY_FORMAT,
+      (GstPluginAPIFlags) 0);
 }
 
 static void
@@ -422,6 +465,7 @@ gst_d3d11_video_sink_init (GstD3D11VideoSink * self)
   self->draw_on_shared_texture = DEFAULT_DRAW_ON_SHARED_TEXTURE;
   self->gamma_mode = DEFAULT_GAMMA_MODE;
   self->primaries_mode = DEFAULT_PRIMARIES_MODE;
+  self->display_format = DEFAULT_DISPLAY_FORMAT;
 
   InitializeCriticalSection (&self->lock);
 }
@@ -477,6 +521,9 @@ gst_d3d11_videosink_set_property (GObject * object, guint prop_id,
     case PROP_PRIMARIES_MODE:
       self->primaries_mode = (GstVideoPrimariesMode) g_value_get_enum (value);
       break;
+    case PROP_DISPLAY_FORMAT:
+      self->display_format = (DXGI_FORMAT) g_value_get_enum (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -522,6 +569,9 @@ gst_d3d11_videosink_get_property (GObject * object, guint prop_id,
     case PROP_PRIMARIES_MODE:
       g_value_set_enum (value, self->primaries_mode);
       break;
+    case PROP_DISPLAY_FORMAT:
+      g_value_set_enum (value, self->display_format);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -715,7 +765,8 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
       GST_TYPE_VIDEO_PRIMARIES_MODE, self->primaries_mode, nullptr);
 
   if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self),
-          GST_VIDEO_SINK_HEIGHT (self), caps, config, &error)) {
+          GST_VIDEO_SINK_HEIGHT (self), caps, config, self->display_format,
+          &error)) {
     GstMessage *error_msg;
 
     LeaveCriticalSection (&self->lock);

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.cpp

@@ -110,7 +110,7 @@ static void gst_d3d11_window_on_resize_default (GstD3D11Window * window,
     guint width, guint height);
 static gboolean gst_d3d11_window_prepare_default (GstD3D11Window * window,
     guint display_width, guint display_height, GstCaps * caps,
-    GstStructure * config, GError ** error);
+    GstStructure * config, DXGI_FORMAT display_format, GError ** error);
 
 static void
 gst_d3d11_window_class_init (GstD3D11WindowClass * klass)
@@ -515,7 +515,7 @@ typedef struct
 gboolean
 gst_d3d11_window_prepare (GstD3D11Window * window, guint display_width,
     guint display_height, GstCaps * caps, GstStructure * config,
-    GError ** error)
+    DXGI_FORMAT display_format, GError ** error)
 {
   GstD3D11WindowClass *klass;
 
@@ -528,13 +528,13 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint display_width,
       GST_PTR_FORMAT, display_width, display_height, caps);
 
   return klass->prepare (window, display_width, display_height, caps, config,
-      error);
+      display_format, error);
 }
 
 static gboolean
 gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
     guint display_height, GstCaps * caps, GstStructure * config,
-    GError ** error)
+    DXGI_FORMAT display_format, GError ** error)
 {
   GstD3D11Device *device = window->device;
   GstD3D11WindowClass *klass;
@@ -546,11 +546,11 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
       D3D11_FORMAT_SUPPORT_TEXTURE2D | D3D11_FORMAT_SUPPORT_DISPLAY;
   UINT supported_flags = 0;
   GstD3D11WindowDisplayFormat formats[] = {
-    {DXGI_FORMAT_R8G8B8A8_UNORM, GST_VIDEO_FORMAT_RGBA, FALSE},
     {DXGI_FORMAT_B8G8R8A8_UNORM, GST_VIDEO_FORMAT_BGRA, FALSE},
+    {DXGI_FORMAT_R8G8B8A8_UNORM, GST_VIDEO_FORMAT_RGBA, FALSE},
     {DXGI_FORMAT_R10G10B10A2_UNORM, GST_VIDEO_FORMAT_RGB10A2_LE, FALSE},
   };
-  const GstD3D11WindowDisplayFormat *chosen_format = NULL;
+  const GstD3D11WindowDisplayFormat *chosen_format = nullptr;
   DXGI_COLOR_SPACE_TYPE swapchain_colorspace =
       DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709;
   gboolean hdr10_aware = FALSE;
@@ -598,6 +598,27 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
     return FALSE;
   }
 
+  if (display_format != DXGI_FORMAT_UNKNOWN) {
+    for (guint i = 0; i < G_N_ELEMENTS (formats); i++) {
+      if (display_format == formats[i].dxgi_format && formats[i].supported) {
+        GST_DEBUG_OBJECT (window, "Requested format %s is supported",
+            gst_d3d11_dxgi_format_to_string (display_format));
+        chosen_format = &formats[i];
+        break;
+      }
+    }
+
+    if (!chosen_format) {
+      GST_ERROR_OBJECT (window, "Requested DXGI FORMAT %d is not supported",
+          display_format);
+      g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
+          "Cannot determine render format");
+      if (config)
+        gst_structure_free (config);
+
+      return FALSE;
+    }
+  } else {
     for (guint i = 0; i < GST_VIDEO_INFO_N_COMPONENTS (&window->info); i++) {
       if (GST_VIDEO_INFO_COMP_DEPTH (&window->info, i) > 8) {
         if (formats[2].supported) {
@@ -606,6 +627,7 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
         break;
       }
     }
+  }
 
   if (!chosen_format) {
     /* prefer native format over conversion */

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.h

@@ -150,6 +150,7 @@ struct _GstD3D11WindowClass
                                            guint display_height,
                                            GstCaps * caps,
                                            GstStructure * config,
+                                           DXGI_FORMAT display_format,
                                            GError ** error);
 
   void          (*unprepare)              (GstD3D11Window * window);
@@ -185,6 +186,7 @@ gboolean      gst_d3d11_window_prepare              (GstD3D11Window * window,
                                                      guint display_height,
                                                      GstCaps * caps,
                                                      GstStructure * config,
+                                                     DXGI_FORMAT display_format,
                                                      GError ** error);
 
 GstFlowReturn gst_d3d11_window_render               (GstD3D11Window * window,

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window_dummy.cpp

@@ -52,7 +52,7 @@ static void gst_d3d11_window_dummy_on_resize (GstD3D11Window * window,
     guint width, guint height);
 static gboolean gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
     guint display_width, guint display_height, GstCaps * caps,
-    GstStructure * config, GError ** error);
+    GstStructure * config, DXGI_FORMAT display_format, GError ** error);
 static void gst_d3d11_window_dummy_unprepare (GstD3D11Window * window);
 static gboolean
 gst_d3d11_window_dummy_open_shared_handle (GstD3D11Window * window,
@@ -85,7 +85,7 @@ gst_d3d11_window_dummy_init (GstD3D11WindowDummy * self)
 static gboolean
 gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
     guint display_width, guint display_height, GstCaps * caps,
-    GstStructure * config, GError ** error)
+    GstStructure * config, DXGI_FORMAT display_format, GError ** error)
 {
   gst_clear_object (&window->compositor);
   gst_clear_object (&window->converter);