d3d11device: Enhance format enumeration and debugging

* Enhance debug log to print human readable D3D11_FORMAT_SUPPORT flags
  value, instead of packed numeric flagset value.
* Only device supported format will be added to format table.
  Depending on device feature level (i.e., D3D9 feature devices),
  16bits formats will not be supported. Although there might be formats
  we deinfed but not supported, it will not be a major issue in practice
  since our D3D11 implementation does not support legacy devices already
  (known limitation) and also old d3dvideosink will be promoted in that case.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2441>

subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11_private.h

@@ -29,6 +29,9 @@
 
 G_BEGIN_DECLS
 
+#define GST_TYPE_D3D11_FORMAT_SUPPORT (gst_d3d11_format_support_get_type())
+GType gst_d3d11_format_support_get_type (void);
+
 void  gst_d3d11_device_d3d11_debug (GstD3D11Device * device,
                                     const gchar * file,
                                     const gchar * function,

subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11device.cpp

@@ -111,7 +111,7 @@ struct _GstD3D11DevicePrivate
   ID3D11VideoContext *video_context;
 
   IDXGIFactory1 *factory;
-  GstD3D11Format format_table[GST_D3D11_N_FORMATS];
+  GArray *format_table;
 
   GRecMutex extern_lock;
   GMutex resource_lock;
@@ -413,6 +413,8 @@ gst_d3d11_device_init (GstD3D11Device * self)
   priv = (GstD3D11DevicePrivate *)
       gst_d3d11_device_get_instance_private (self);
   priv->adapter = DEFAULT_ADAPTER;
+  priv->format_table = g_array_sized_new (FALSE, FALSE,
+      sizeof (GstD3D11Format), GST_D3D11_N_FORMATS);
 
   g_rec_mutex_init (&priv->extern_lock);
   g_mutex_init (&priv->resource_lock);
@@ -440,87 +442,29 @@ is_windows_8_or_greater (void)
   return ret;
 }
 
-inline D3D11_FORMAT_SUPPORT
+static guint
-operator | (D3D11_FORMAT_SUPPORT lhs, D3D11_FORMAT_SUPPORT rhs)
+check_format_support (GstD3D11Device * self, DXGI_FORMAT format)
-{
-  return static_cast < D3D11_FORMAT_SUPPORT > (static_cast < UINT >
-      (lhs) | static_cast < UINT > (rhs));
-}
-
-inline D3D11_FORMAT_SUPPORT
-operator |= (D3D11_FORMAT_SUPPORT lhs, D3D11_FORMAT_SUPPORT rhs)
-{
-  return lhs | rhs;
-}
-
-static gboolean
-can_support_format (GstD3D11Device * self, DXGI_FORMAT format,
-    D3D11_FORMAT_SUPPORT extra_flags)
 {
   GstD3D11DevicePrivate *priv = self->priv;
   ID3D11Device *handle = priv->device;
   HRESULT hr;
-  UINT supported;
+  UINT format_support;
-  D3D11_FORMAT_SUPPORT flags = D3D11_FORMAT_SUPPORT_TEXTURE2D;
-  const gchar *format_name = gst_d3d11_dxgi_format_to_string (format);
 
-  flags |= extra_flags;
+  hr = handle->CheckFormatSupport (format, &format_support);
+  if (FAILED (hr) || format_support == 0)
+    return 0;
 
-  if (!is_windows_8_or_greater ()) {
+  return format_support;
-    GST_INFO_OBJECT (self, "DXGI_FORMAT_%s (%d) needs Windows 8 or greater",
-        format_name, (guint) format);
-    return FALSE;
-  }
-
-  hr = handle->CheckFormatSupport (format, &supported);
-  if (FAILED (hr)) {
-    GST_DEBUG_OBJECT (self, "DXGI_FORMAT_%s (%d) is not supported by device",
-        format_name, (guint) format);
-    return FALSE;
-  }
-
-  if ((supported & flags) != flags) {
-    GST_DEBUG_OBJECT (self,
-        "DXGI_FORMAT_%s (%d) doesn't support flag 0x%x (supported flag 0x%x)",
-        format_name, (guint) format, (guint) supported, (guint) flags);
-    return FALSE;
-  }
-
-  GST_INFO_OBJECT (self, "Device supports DXGI_FORMAT_%s (%d)",
-      format_name, (guint) format);
-
-  return TRUE;
-}
-
-static void
-update_format_support_flag (GstD3D11Device * self, GstD3D11Format * format)
-{
-  GstD3D11DevicePrivate *priv = self->priv;
-  ID3D11Device *handle = priv->device;
-  HRESULT hr;
-  UINT supported;
-  DXGI_FORMAT dxgi_format;
-
-  if (format->dxgi_format != DXGI_FORMAT_UNKNOWN)
-    dxgi_format = format->dxgi_format;
-  else
-    dxgi_format = format->resource_format[0];
-
-  hr = handle->CheckFormatSupport (dxgi_format, &supported);
-  if (FAILED (hr)) {
-    GST_DEBUG_OBJECT (self, "DXGI_FORMAT_%s (%d) is not supported by device",
-        gst_d3d11_dxgi_format_to_string (dxgi_format), (guint) dxgi_format);
-    return;
-  }
-
-  format->support_flags = (D3D11_FORMAT_SUPPORT) supported;
 }
 
 static void
 dump_format (GstD3D11Device * self, GstD3D11Format * format)
 {
-  GST_LOG_OBJECT (self, "%s -> DXGI_FORMAT_%s (%d), "
+  gchar *format_support_str = g_flags_to_string (GST_TYPE_D3D11_FORMAT_SUPPORT,
-      "resource format: %s (%d), %s (%d), %s (%d), %s (%d), flags 0x%x",
+      format->format_support[0]);
+
+  GST_LOG_OBJECT (self, "%s -> %s (%d), "
+      "resource format: %s (%d), %s (%d), %s (%d), %s (%d), flags (0x%x) %s",
       gst_video_format_to_string (format->format),
       gst_d3d11_dxgi_format_to_string (format->dxgi_format),
       format->dxgi_format,
@@ -531,7 +475,10 @@ dump_format (GstD3D11Device * self, GstD3D11Format * format)
       gst_d3d11_dxgi_format_to_string (format->resource_format[2]),
       format->resource_format[2],
       gst_d3d11_dxgi_format_to_string (format->resource_format[3]),
-      format->resource_format[3], (guint) format->support_flags);
+      format->resource_format[3], format->format_support[0],
+      format_support_str);
+
+  g_free (format_support_str);
 }
 
 static void
@@ -539,21 +486,31 @@ gst_d3d11_device_setup_format_table (GstD3D11Device * self)
 {
   GstD3D11DevicePrivate *priv = self->priv;
 
-  for (guint i = 0; i < G_N_ELEMENTS (priv->format_table); i++) {
+  for (guint i = 0; i < G_N_ELEMENTS (_gst_d3d11_default_format_map); i++) {
-    GstD3D11Format *format = &priv->format_table[i];
+    const GstD3D11Format *iter = &_gst_d3d11_default_format_map[i];
+    GstD3D11Format format;
+    guint support[GST_VIDEO_MAX_PLANES] = { 0, };
+    gboolean native = TRUE;
 
-    *format = _gst_d3d11_default_format_map[i];
+    switch (iter->format) {
-
+        /* RGB/GRAY */
-    switch (format->format) {
-        /* RGB */
       case GST_VIDEO_FORMAT_BGRA:
       case GST_VIDEO_FORMAT_BGRx:
       case GST_VIDEO_FORMAT_RGBA:
       case GST_VIDEO_FORMAT_RGBx:
       case GST_VIDEO_FORMAT_RGB10A2_LE:
       case GST_VIDEO_FORMAT_RGBA64_LE:
-        g_assert (format->dxgi_format != DXGI_FORMAT_UNKNOWN);
+      case GST_VIDEO_FORMAT_GRAY8:
-        update_format_support_flag (self, format);
+      case GST_VIDEO_FORMAT_GRAY16_LE:
+        support[0] = check_format_support (self, iter->dxgi_format);
+        if (!support[0]) {
+          const gchar *format_name =
+              gst_d3d11_dxgi_format_to_string (iter->dxgi_format);
+          GST_INFO_OBJECT (self, "DXGI_FORMAT_%s (%d) for %s is not supported",
+              format_name, (guint) iter->dxgi_format,
+              gst_video_format_to_string (iter->format));
+          continue;
+        }
         break;
         /* YUV DXGI native formats */
       case GST_VIDEO_FORMAT_VUYA:
@@ -561,13 +518,40 @@ gst_d3d11_device_setup_format_table (GstD3D11Device * self)
       case GST_VIDEO_FORMAT_NV12:
       case GST_VIDEO_FORMAT_P010_10LE:
       case GST_VIDEO_FORMAT_P012_LE:
-      case GST_VIDEO_FORMAT_P016_LE:
+      case GST_VIDEO_FORMAT_P016_LE:{
-        if (!can_support_format (self,
+        gboolean supported = TRUE;
-                format->dxgi_format, format->support_flags)) {
+
-          format->dxgi_format = DXGI_FORMAT_UNKNOWN;
+        if (is_windows_8_or_greater ())
+          support[0] = check_format_support (self, iter->dxgi_format);
+
+        if (!support[0]) {
+          GST_DEBUG_OBJECT (self,
+              "DXGI_FORMAT_%s (%d) for %s is not supported, "
+              "checking resource format",
+              gst_d3d11_dxgi_format_to_string (iter->dxgi_format),
+              (guint) iter->dxgi_format,
+              gst_video_format_to_string (iter->format));
+
+          native = FALSE;
+          for (guint j = 0; j < GST_VIDEO_MAX_PLANES; j++) {
+            if (iter->resource_format[j] == DXGI_FORMAT_UNKNOWN)
+              break;
+
+            support[j] = check_format_support (self, iter->resource_format[j]);
+            if (support[j] == 0) {
+              supported = FALSE;
+              break;
+            }
+          }
+
+          if (!supported) {
+            GST_INFO_OBJECT (self, "%s is not supported",
+                gst_video_format_to_string (iter->format));
+            continue;
+          }
         }
-        update_format_support_flag (self, format);
         break;
+      }
         /* YUV non-DXGI native formats */
       case GST_VIDEO_FORMAT_NV21:
       case GST_VIDEO_FORMAT_I420:
@@ -582,23 +566,45 @@ gst_d3d11_device_setup_format_table (GstD3D11Device * self)
       case GST_VIDEO_FORMAT_Y444_12LE:
       case GST_VIDEO_FORMAT_Y444_16LE:
       case GST_VIDEO_FORMAT_AYUV:
-      case GST_VIDEO_FORMAT_AYUV64:
+      case GST_VIDEO_FORMAT_AYUV64:{
-      {
+        gboolean supported = TRUE;
-        g_assert (format->dxgi_format == DXGI_FORMAT_UNKNOWN);
+
-        update_format_support_flag (self, format);
+        native = FALSE;
+        for (guint j = 0; j < GST_VIDEO_MAX_PLANES; j++) {
+          if (iter->resource_format[j] == DXGI_FORMAT_UNKNOWN)
+            break;
+
+          support[j] = check_format_support (self, iter->resource_format[j]);
+          if (support[j] == 0) {
+            supported = FALSE;
+            break;
+          }
+        }
+
+        if (!supported) {
+          GST_INFO_OBJECT (self, "%s is not supported",
+              gst_video_format_to_string (iter->format));
+          continue;
+        }
         break;
       }
-      case GST_VIDEO_FORMAT_GRAY8:
-      case GST_VIDEO_FORMAT_GRAY16_LE:
-        g_assert (format->dxgi_format != DXGI_FORMAT_UNKNOWN);
-        update_format_support_flag (self, format);
-        break;
       default:
         g_assert_not_reached ();
-        break;
+        return;
     }
 
-    dump_format (self, format);
+    format = *iter;
+
+    if (!native)
+      format.dxgi_format = DXGI_FORMAT_UNKNOWN;
+
+    for (guint j = 0; j < GST_VIDEO_MAX_PLANES; j++)
+      format.format_support[j] = support[j];
+
+    if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_LOG)
+      dump_format (self, &format);
+
+    g_array_append_val (priv->format_table, format);
   }
 
   /* FIXME: d3d11 sampler doesn't support packed-and-subsampled formats
@@ -739,6 +745,7 @@ gst_d3d11_device_finalize (GObject * object)
 
   GST_LOG_OBJECT (self, "finalize");
 
+  g_array_unref (priv->format_table);
   g_rec_mutex_clear (&priv->extern_lock);
   g_mutex_clear (&priv->resource_lock);
   g_free (priv->description);
@@ -1357,12 +1364,15 @@ gst_d3d11_device_get_format (GstD3D11Device * device, GstVideoFormat format,
 
   priv = device->priv;
 
-  for (guint i = 0; i < G_N_ELEMENTS (priv->format_table); i++) {
+  for (guint i = 0; i < priv->format_table->len; i++) {
-    if (priv->format_table[i].format != format)
+    const GstD3D11Format *d3d11_fmt =
+        &g_array_index (priv->format_table, GstD3D11Format, i);
+
+    if (d3d11_fmt->format != format)
       continue;
 
     if (device_format)
-      *device_format = priv->format_table[i];
+      *device_format = *d3d11_fmt;
 
     return TRUE;
   }

subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11format.cpp

@@ -51,6 +51,67 @@ ensure_debug_category (void)
 #define ensure_debug_category() /* NOOP */
 #endif /* GST_DISABLE_GST_DEBUG */
 
+GType
+gst_d3d11_format_support_get_type (void)
+{
+  static gsize support_type = 0;
+  static const GFlagsValue support_values[] = {
+    {D3D11_FORMAT_SUPPORT_BUFFER, "BUFFER", "buffer"},
+    {D3D11_FORMAT_SUPPORT_IA_VERTEX_BUFFER, "IA_VERTEX_BUFFER",
+        "ia-vertex-buffer"},
+    {D3D11_FORMAT_SUPPORT_IA_INDEX_BUFFER, "IA_INDEX_BUFFER",
+        "ia-index-buffer"},
+    {D3D11_FORMAT_SUPPORT_SO_BUFFER, "SO_BUFFER", "so-buffer"},
+    {D3D11_FORMAT_SUPPORT_TEXTURE1D, "TEXTURE1D", "texture1d"},
+    {D3D11_FORMAT_SUPPORT_TEXTURE2D, "TEXTURE2D", "texture2d"},
+    {D3D11_FORMAT_SUPPORT_TEXTURE3D, "TEXTURE3D", "texture3d"},
+    {D3D11_FORMAT_SUPPORT_TEXTURECUBE, "TEXTURECUBE", "texturecube"},
+    {D3D11_FORMAT_SUPPORT_SHADER_LOAD, "SHADER_LOAD", "shader-load"},
+    {D3D11_FORMAT_SUPPORT_SHADER_SAMPLE, "SHADER_SAMPLE", "shader-sample"},
+    {D3D11_FORMAT_SUPPORT_SHADER_SAMPLE_COMPARISON, "SHADER_COMPARISION",
+        "shader-comparision"},
+    {D3D11_FORMAT_SUPPORT_SHADER_SAMPLE_MONO_TEXT, "SHADER_SAMPLE_MONO_TEXT",
+        "shader-sample-mono-text"},
+    {D3D11_FORMAT_SUPPORT_MIP, "MIP", "mip"},
+    {D3D11_FORMAT_SUPPORT_MIP_AUTOGEN, "MIP_AUTOGEN", "mip-autogen"},
+    {D3D11_FORMAT_SUPPORT_RENDER_TARGET, "RENDER_TARGET", "render-target"},
+    {D3D11_FORMAT_SUPPORT_BLENDABLE, "BLANDABLE", "blandable"},
+    {D3D11_FORMAT_SUPPORT_DEPTH_STENCIL, "DEPTH_STENCIL", "depth-stencil"},
+    {D3D11_FORMAT_SUPPORT_CPU_LOCKABLE, "CPU_LOCKABLE", "cpu-lockable"},
+    {D3D11_FORMAT_SUPPORT_MULTISAMPLE_RESOLVE, "MULTISAMPLE_RESOLVE",
+        "multisample-resolve"},
+    {D3D11_FORMAT_SUPPORT_DISPLAY, "DISPLAY", "display"},
+    {D3D11_FORMAT_SUPPORT_CAST_WITHIN_BIT_LAYOUT, "CAST_WITHIN_BIT_LAYOUT",
+        "cast-within-bit-layout"},
+    {D3D11_FORMAT_SUPPORT_MULTISAMPLE_RENDERTARGET, "MULTISAMPLE_RENDERTARGET",
+        "multisample-rendertarget"},
+    {D3D11_FORMAT_SUPPORT_MULTISAMPLE_LOAD, "MULTISAMPLE_LOAD",
+        "multisample-load"},
+    {D3D11_FORMAT_SUPPORT_SHADER_GATHER, "SHADER_GATHER", "shader-gether"},
+    {D3D11_FORMAT_SUPPORT_BACK_BUFFER_CAST, "BACK_BUFFER_CAST",
+        "back-buffer-cast"},
+    {D3D11_FORMAT_SUPPORT_TYPED_UNORDERED_ACCESS_VIEW, "UNORDERED_ACCESS_VIEW",
+        "unordered-access-view"},
+    {D3D11_FORMAT_SUPPORT_SHADER_GATHER_COMPARISON, "SHADER_GATHER_COMPARISON",
+        "shader-gether-comparision"},
+    {D3D11_FORMAT_SUPPORT_DECODER_OUTPUT, "DECODER_OUTPUT", "decoder-output"},
+    {D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT, "VIDEO_PROCESSOR_OUTPUT",
+        "video-processor-output"},
+    {D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_INPUT, "VIDEO_PROCESSOR_INPUT",
+        "video-processor-input"},
+    {D3D11_FORMAT_SUPPORT_VIDEO_ENCODER, "VIDEO_ENCODER", "video-encoder"},
+    {0, nullptr, nullptr}
+  };
+
+  if (g_once_init_enter (&support_type)) {
+    GType tmp = g_flags_register_static ("GstD3D11FormatSupport",
+        support_values);
+    g_once_init_leave (&support_type, tmp);
+  }
+
+  return (GType) support_type;
+}
+
 /**
  * gst_d3d11_dxgi_format_get_size:
  * @format: a DXGI_FORMAT

subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11format.h

@@ -54,11 +54,17 @@ struct _GstD3D11Format
   /* formats for texture processing */
   DXGI_FORMAT resource_format[GST_VIDEO_MAX_PLANES];
 
-  /* extra format used for unordered access view */
+  /* extra format used for unordered access view (unused) */
   DXGI_FORMAT uav_format[GST_VIDEO_MAX_PLANES];
 
-  /* cached flags */
+  /* D3D11_FORMAT_SUPPORT flags */
-  D3D11_FORMAT_SUPPORT support_flags;
+  guint format_support[GST_VIDEO_MAX_PLANES];
+
+  /* D3D11_FORMAT_SUPPORT2 flags (unused) */
+  guint format_support2[GST_VIDEO_MAX_PLANES];
+
+  /*< private >*/
+  guint padding[GST_PADDING_LARGE];
 };
 
 GST_D3D11_API