d3d12: Add AV12 format support

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7093>

subprojects/gst-plugins-bad/gst-libs/gst/d3d12/gstd3d12-private.h

@@ -50,7 +50,7 @@
 #define GST_D3D12_TIER_1_FORMATS \
     "AYUV64, GBRA_12LE, GBRA_10LE, AYUV, ABGR, ARGB, GBRA, Y444_16LE, " \
     "A444_16LE, A444_12LE, A444_10LE, A444, " \
-    "A422_16LE, A422_12LE, A422_10LE, A422, A420_16LE, A420_12LE, A420_10LE, A420, " \
+    "A422_16LE, A422_12LE, A422_10LE, A422, A420_16LE, A420_12LE, A420_10LE, A420, AV12, " \
     "GBR_16LE, Y444_12LE, GBR_12LE, I422_12LE, I420_12LE, Y444_10LE, GBR_10LE, " \
     "I422_10LE, I420_10LE, Y444, BGRP, GBR, RGBP, xBGR, xRGB, Y42B, NV24, NV16, NV61, NV21, " \
     "I420, YV12, Y41B, YUV9, YVU9, GRAY16_LE, GRAY8"

subprojects/gst-plugins-bad/gst-libs/gst/d3d12/gstd3d12bufferpool.cpp

@@ -212,12 +212,18 @@ gst_d3d12_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
 
     priv->alloc[0] = alloc;
   } else {
+    auto finfo = params->aligned_info.finfo;
+
     for (guint i = 0; i < GST_VIDEO_MAX_PLANES; i++) {
       if (params->d3d12_format.resource_format[i] == DXGI_FORMAT_UNKNOWN)
         break;
 
-      guint width = GST_VIDEO_INFO_COMP_WIDTH (&params->aligned_info, i);
+      gint comp[GST_VIDEO_MAX_COMPONENTS];
-      guint height = GST_VIDEO_INFO_COMP_HEIGHT (&params->aligned_info, i);
+      gst_video_format_info_component (finfo, i, comp);
+
+      guint width = GST_VIDEO_INFO_COMP_WIDTH (&params->aligned_info, comp[0]);
+      guint height =
+          GST_VIDEO_INFO_COMP_HEIGHT (&params->aligned_info, comp[0]);
       width = MAX (width, 1);
       height = MAX (height, 1);
 

subprojects/gst-plugins-bad/gst-libs/gst/d3d12/gstd3d12converter.cpp

@@ -602,6 +602,58 @@ gst_d3d12_converter_get_gamma_enc_table (GstVideoTransferFunction func)
   return table;
 }
 
+static guint
+reorder_rtv_index (GstVideoFormat output_format, guint index)
+{
+  switch (output_format) {
+    case GST_VIDEO_FORMAT_A420:
+    case GST_VIDEO_FORMAT_A420_10LE:
+    case GST_VIDEO_FORMAT_A420_12LE:
+    case GST_VIDEO_FORMAT_A420_16LE:
+    case GST_VIDEO_FORMAT_A422:
+    case GST_VIDEO_FORMAT_A422_10LE:
+    case GST_VIDEO_FORMAT_A422_12LE:
+    case GST_VIDEO_FORMAT_A422_16LE:
+    {
+      switch (index) {
+        case 0:
+          return 0;
+        case 1:
+          return 3;
+        case 2:
+          return 1;
+        case 3:
+          return 2;
+        default:
+          g_assert_not_reached ();
+          break;
+      }
+      return 0;
+    }
+    case GST_VIDEO_FORMAT_AV12:
+    {
+      switch (index) {
+        case 0:
+          return 0;
+        case 1:
+          return 2;
+        case 2:
+          return 1;
+        case 3:
+          return 3;
+        default:
+          g_assert_not_reached ();
+          break;
+      }
+      return 0;
+    }
+    default:
+      break;
+  }
+
+  return index;
+}
+
 static gboolean
 gst_d3d12_converter_setup_resource (GstD3D12Converter * self,
     const GstVideoInfo * in_info, const GstVideoInfo * out_info,
@@ -659,8 +711,10 @@ gst_d3d12_converter_setup_resource (GstD3D12Converter * self,
   }
 
   std::queue < DXGI_FORMAT > rtv_formats;
+  auto output_format = GST_VIDEO_INFO_FORMAT (out_info);
   for (guint i = 0; i < 4; i++) {
-    auto format = out_format->resource_format[i];
+    auto index = reorder_rtv_index (output_format, i);
+    auto format = out_format->resource_format[index];
     if (format == DXGI_FORMAT_UNKNOWN)
       break;
 
@@ -1243,6 +1297,7 @@ gst_d3d12_converter_update_dest_rect (GstD3D12Converter * self)
     case GST_VIDEO_FORMAT_A420_10LE:
     case GST_VIDEO_FORMAT_A420_12LE:
     case GST_VIDEO_FORMAT_A420_16LE:
+    case GST_VIDEO_FORMAT_AV12:
       priv->viewport[1].TopLeftX = priv->viewport[0].TopLeftX / 2;
       priv->viewport[1].TopLeftY = priv->viewport[0].TopLeftY / 2;
       priv->viewport[1].Width = priv->viewport[0].Width / 2;
@@ -1617,6 +1672,12 @@ gst_d3d12_converter_calculate_border_color (GstD3D12Converter * self)
         priv->clear_color[1][2] = 0;
         priv->clear_color[1][3] = 1.0;
         break;
+      case GST_VIDEO_FORMAT_AV12:
+        priv->clear_color[0][0] = converted[0];
+        priv->clear_color[1][0] = converted[1];
+        priv->clear_color[1][1] = converted[2];
+        priv->clear_color[2][0] = a;
+        break;
       case GST_VIDEO_FORMAT_YUV9:
       case GST_VIDEO_FORMAT_YVU9:
       case GST_VIDEO_FORMAT_Y41B:
@@ -1963,26 +2024,10 @@ static void
 reorder_rtv_handles (GstVideoFormat output_format,
     D3D12_CPU_DESCRIPTOR_HANDLE * src, D3D12_CPU_DESCRIPTOR_HANDLE * dst)
 {
-  switch (output_format) {
+  for (guint i = 0; i < GST_VIDEO_MAX_PLANES; i++) {
-    case GST_VIDEO_FORMAT_A420:
+    auto index = reorder_rtv_index (output_format, i);
-    case GST_VIDEO_FORMAT_A420_10LE:
+    dst[i] = src[index];
-    case GST_VIDEO_FORMAT_A420_12LE:
-    case GST_VIDEO_FORMAT_A420_16LE:
-    case GST_VIDEO_FORMAT_A422:
-    case GST_VIDEO_FORMAT_A422_10LE:
-    case GST_VIDEO_FORMAT_A422_12LE:
-    case GST_VIDEO_FORMAT_A422_16LE:
-      dst[0] = src[0];
-      dst[1] = src[3];
-      dst[2] = src[1];
-      dst[3] = src[2];
-      return;
-    default:
-      break;
   }
-
-  for (guint i = 0; i < GST_VIDEO_MAX_PLANES; i++)
-    dst[i] = src[i];
 }
 
 static gboolean

subprojects/gst-plugins-bad/gst-libs/gst/d3d12/gstd3d12format.cpp

@@ -155,6 +155,22 @@ struct FormatBuilder : public GstD3D12Format
         Support1, Support2);
   }
 
+  static inline FormatBuilder YuvSemiPlanarWithAlpha (
+      GstVideoFormat Format,
+      DXGI_FORMAT ResourceFormatY,
+      DXGI_FORMAT ResourceFormatUV,
+      DXGI_FORMAT ResourceFormatA,
+      D3D12_FORMAT_SUPPORT1 Support1 = kDefaultFormatSupport1,
+      D3D12_FORMAT_SUPPORT2 Support2 = D3D12_FORMAT_SUPPORT2_NONE
+    )
+  {
+    DXGI_FORMAT resource_format[] = { ResourceFormatY, ResourceFormatUV,
+        ResourceFormatA, DXGI_FORMAT_UNKNOWN };
+    return FormatBuilder (Format, GST_D3D12_FORMAT_FLAG_NONE,
+        D3D12_RESOURCE_DIMENSION_TEXTURE2D, DXGI_FORMAT_UNKNOWN,
+        resource_format, Support1, Support2);
+  }
+
   static inline FormatBuilder YuvPacked (
       GstVideoFormat Format,
       DXGI_FORMAT DxgiFormat,
@@ -358,7 +374,8 @@ static const GstD3D12Format g_format_map[] = {
   FormatBuilder::NotSupported(GST_VIDEO_FORMAT_NV12_32L32),
   FormatBuilder::Planar (GST_VIDEO_FORMAT_RGBP),
   FormatBuilder::Planar (GST_VIDEO_FORMAT_BGRP),
-  FormatBuilder::NotSupported(GST_VIDEO_FORMAT_AV12),
+  FormatBuilder::YuvSemiPlanarWithAlpha (GST_VIDEO_FORMAT_AV12,
+      DXGI_FORMAT_R8_UNORM, DXGI_FORMAT_R8G8_UNORM, DXGI_FORMAT_R8_UNORM),
   FormatBuilder::NotSupported(GST_VIDEO_FORMAT_ARGB64_LE),
   FormatBuilder::NotSupported(GST_VIDEO_FORMAT_ARGB64_BE),
   FormatBuilder::RgbPacked (GST_VIDEO_FORMAT_RGBA64_LE,

subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/converter-hlsl/PSMain_converter.hlsl

@@ -162,6 +162,18 @@ class SamplerNV21 : ISampler
   }
 };
 
+class SamplerAV12 : ISampler
+{
+  float4 Execute (float2 uv)
+  {
+    float4 sample;
+    sample.x = shaderTexture_0.Sample(samplerState, uv).x;
+    sample.yz = shaderTexture_1.Sample(samplerState, uv).xy;
+    sample.a = shaderTexture_2.Sample(samplerState, uv).x;
+    return sample;
+  }
+};
+
 class SamplerI420 : ISampler
 {
   float4 Execute (float2 uv)
@@ -1489,6 +1501,18 @@ static const char str_PSMain_converter[] =
 "  }\n"
 "};\n"
 "\n"
+"class SamplerAV12 : ISampler\n"
+"{\n"
+"  float4 Execute (float2 uv)\n"
+"  {\n"
+"    float4 sample;\n"
+"    sample.x = shaderTexture_0.Sample(samplerState, uv).x;\n"
+"    sample.yz = shaderTexture_1.Sample(samplerState, uv).xy;\n"
+"    sample.a = shaderTexture_2.Sample(samplerState, uv).x;\n"
+"    return sample;\n"
+"  }\n"
+"};\n"
+"\n"
 "class SamplerI420 : ISampler\n"
 "{\n"
 "  float4 Execute (float2 uv)\n"

subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/gstd3dshadercache.cpp

@@ -657,6 +657,8 @@ conv_ps_make_input (GstVideoFormat format, gboolean premul)
     case GST_VIDEO_FORMAT_NV21:
     case GST_VIDEO_FORMAT_NV61:
       return "NV21";
+    case GST_VIDEO_FORMAT_AV12:
+      return "AV12";
     case GST_VIDEO_FORMAT_YUV9:
     case GST_VIDEO_FORMAT_Y41B:
     case GST_VIDEO_FORMAT_I420:
@@ -810,6 +812,10 @@ conv_ps_make_output (GstVideoFormat format, gboolean premul)
       ret.push_back({PS_OUTPUT::LUMA, "Luma"});
       ret.push_back({PS_OUTPUT::CHROMA, "ChromaNV21"});
       break;
+    case GST_VIDEO_FORMAT_AV12:
+      ret.push_back({PS_OUTPUT::LUMA_ALPHA, "LumaAlphaA420"});
+      ret.push_back({PS_OUTPUT::CHROMA, "ChromaNV12"});
+      break;
     case GST_VIDEO_FORMAT_YUV9:
     case GST_VIDEO_FORMAT_Y41B:
     case GST_VIDEO_FORMAT_I420: