d3d11videosink: Add support for transform and MSAA

Adding properties for 3D rotation with arbitrary angle
and scaling. And adding Multi Sampling Anti-Aliasing rendering
support to smooth borders if arbitrary angle is applied

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5532>

subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json

@@ -10212,6 +10212,20 @@
                         "type": "gboolean",
                         "writable": true
                     },
+                    "fov": {
+                        "blurb": "Field of view angle in degrees",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "90",
+                        "max": "3.40282e+38",
+                        "min": "0",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gfloat",
+                        "writable": true
+                    },
                     "fullscreen": {
                         "blurb": "Ignored when \"fullscreen-toggle-mode\" does not include \"property\"",
                         "conditionally-available": false,
@@ -10248,6 +10262,30 @@
                         "type": "GstVideoGammaMode",
                         "writable": true
                     },
+                    "msaa": {
+                        "blurb": "MSAA (Multi-Sampling Anti-Aliasing) level",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "disabled (0)",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "GstD3D11MSAAMode",
+                        "writable": true
+                    },
+                    "ortho": {
+                        "blurb": "Use orthographic projection",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "false",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gboolean",
+                        "writable": true
+                    },
                     "primaries-mode": {
                         "blurb": "Primaries conversion mode",
                         "conditionally-available": false,
@@ -10282,6 +10320,76 @@
                         "readable": true,
                         "type": "GstVideoOrientationMethod",
                         "writable": true
+                    },
+                    "rotation-x": {
+                        "blurb": "x-axis rotation angle in degrees",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "0",
+                        "max": "3.40282e+38",
+                        "min": "-3.40282e+38",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gfloat",
+                        "writable": true
+                    },
+                    "rotation-y": {
+                        "blurb": "y-axis rotation angle in degrees",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "0",
+                        "max": "3.40282e+38",
+                        "min": "-3.40282e+38",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gfloat",
+                        "writable": true
+                    },
+                    "rotation-z": {
+                        "blurb": "z-axis rotation angle in degrees",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "0",
+                        "max": "3.40282e+38",
+                        "min": "-3.40282e+38",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gfloat",
+                        "writable": true
+                    },
+                    "scale-x": {
+                        "blurb": "Scale multiplier for x-axis",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "1",
+                        "max": "3.40282e+38",
+                        "min": "-3.40282e+38",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gfloat",
+                        "writable": true
+                    },
+                    "scale-y": {
+                        "blurb": "Scale multiplier for y-axis",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "1",
+                        "max": "3.40282e+38",
+                        "min": "-3.40282e+38",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gfloat",
+                        "writable": true
                     }
                 },
                 "rank": "primary",
@@ -10778,6 +10886,31 @@
                     }
                 ]
             },
+            "GstD3D11MSAAMode": {
+                "kind": "enum",
+                "values": [
+                    {
+                        "desc": "Disabled",
+                        "name": "disabled",
+                        "value": "0"
+                    },
+                    {
+                        "desc": "2x MSAA",
+                        "name": "2x",
+                        "value": "1"
+                    },
+                    {
+                        "desc": "4x MSAA",
+                        "name": "4x",
+                        "value": "2"
+                    },
+                    {
+                        "desc": "8x MSAA",
+                        "name": "8x",
+                        "value": "3"
+                    }
+                ]
+            },
             "GstD3D11SamplingMethod": {
                 "kind": "enum",
                 "values": [

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.cpp

@@ -77,6 +77,30 @@ gst_d3d11_alpha_mode_get_type (void)
   return type;
 }
 
+/**
+ * GstD3D11MSAAMode:
+ *
+ * Since: 1.24
+ */
+GType
+gst_d3d11_msaa_mode_get_type (void)
+{
+  static GType type = 0;
+  static const GEnumValue msaa_mode[] = {
+    {GST_D3D11_MSAA_DISABLED, "Disabled", "disabled"},
+    {GST_D3D11_MSAA_2X, "2x MSAA", "2x"},
+    {GST_D3D11_MSAA_4X, "4x MSAA", "4x"},
+    {GST_D3D11_MSAA_8X, "8x MSAA", "8x"},
+    {0, nullptr, nullptr},
+  };
+
+  GST_D3D11_CALL_ONCE_BEGIN {
+    type = g_enum_register_static ("GstD3D11MSAAMode", msaa_mode);
+  } GST_D3D11_CALL_ONCE_END;
+
+  return type;
+}
+
 /* Max Texture Dimension for feature level 11_0 ~ 12_1 */
 static guint _gst_d3d11_texture_max_dimension = 16384;
 

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.h

@@ -50,6 +50,17 @@ typedef enum
 #define GST_TYPE_D3D11_ALPHA_MODE (gst_d3d11_alpha_mode_get_type())
 GType gst_d3d11_alpha_mode_get_type (void);
 
+typedef enum
+{
+  GST_D3D11_MSAA_DISABLED,
+  GST_D3D11_MSAA_2X,
+  GST_D3D11_MSAA_4X,
+  GST_D3D11_MSAA_8X,
+} GstD3D11MSAAMode;
+
+#define GST_TYPE_D3D11_MSAA_MODE (gst_d3d11_msaa_mode_get_type())
+GType gst_d3d11_msaa_mode_get_type (void);
+
 void            gst_d3d11_plugin_utils_init         (D3D_FEATURE_LEVEL feature_level);
 
 GstCaps *       gst_d3d11_get_updated_template_caps (GstStaticCaps * template_caps);

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp

@@ -65,6 +65,14 @@ enum
   PROP_PRIMARIES_MODE,
   PROP_DISPLAY_FORMAT,
   PROP_EMIT_PRESENT,
+  PROP_FOV,
+  PROP_ORTHO,
+  PROP_ROTATION_X,
+  PROP_ROTATION_Y,
+  PROP_ROTATION_Z,
+  PROP_SCALE_X,
+  PROP_SCALE_Y,
+  PROP_MSAA,
   PROP_RENDER_RECTANGE,
 };
 
@@ -78,6 +86,11 @@ enum
 #define DEFAULT_PRIMARIES_MODE            GST_VIDEO_PRIMARIES_MODE_NONE
 #define DEFAULT_DISPLAY_FORMAT            DXGI_FORMAT_UNKNOWN
 #define DEFAULT_EMIT_PRESENT              FALSE
+#define DEFAULT_ROTATION                  0.0f
+#define DEFAULT_SCALE                     1.0f
+#define DEFAULT_FOV                       90.0f
+#define DEFAULT_ORTHO                     FALSE
+#define DEFAULT_MSAA                      GST_D3D11_MSAA_DISABLED
 
 /**
  * GstD3D11VideoSinkDisplayFormat:
@@ -188,6 +201,14 @@ struct _GstD3D11VideoSink
   GstVideoPrimariesMode primaries_mode;
   DXGI_FORMAT display_format;
   gboolean emit_present;
+  gfloat fov;
+  gboolean ortho;
+  gfloat rotation_x;
+  gfloat rotation_y;
+  gfloat rotation_z;
+  gfloat scale_x;
+  gfloat scale_y;
+  GstD3D11MSAAMode msaa;
 
   /* saved render rectangle until we have a window */
   GstVideoRectangle render_rect;
@@ -399,6 +420,110 @@ gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass)
           (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
               G_PARAM_STATIC_STRINGS)));
 
+  /**
+   * GstD3D11VideoSink:fov:
+   *
+   * Field of view angle in degrees
+   *
+   * Since: 1.24
+   */
+  g_object_class_install_property (gobject_class, PROP_FOV,
+      g_param_spec_float ("fov", "Fov",
+          "Field of view angle in degrees",
+          0, G_MAXFLOAT, DEFAULT_FOV,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+  /**
+   * GstD3D11VideoSink:ortho:
+   *
+   * Use orthographic projection
+   *
+   * Since: 1.24
+   */
+  g_object_class_install_property (gobject_class, PROP_ORTHO,
+      g_param_spec_boolean ("ortho", "Orthographic",
+          "Use orthographic projection", DEFAULT_ORTHO,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+  /**
+   * GstD3D11VideoSink:rotation-x:
+   *
+   * x-axis rotation angle to be applied prior to "rotate-method"
+   *
+   * Since: 1.24
+   */
+  g_object_class_install_property (gobject_class, PROP_ROTATION_X,
+      g_param_spec_float ("rotation-x", "Rotation X",
+          "x-axis rotation angle in degrees",
+          -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_ROTATION,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+  /**
+   * GstD3D11VideoSink:rotation-y:
+   *
+   * y-axis rotation angle to be applied prior to "rotate-method"
+   *
+   * Since: 1.24
+   */
+  g_object_class_install_property (gobject_class, PROP_ROTATION_Y,
+      g_param_spec_float ("rotation-y", "Rotation Y",
+          "y-axis rotation angle in degrees",
+          -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_ROTATION,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+  /**
+   * GstD3D11VideoSink:rotation-z:
+   *
+   * z-axis rotation angle to be applied prior to "rotate-method"
+   *
+   * Since: 1.24
+   */
+  g_object_class_install_property (gobject_class, PROP_ROTATION_Z,
+      g_param_spec_float ("rotation-z", "Rotation Z",
+          "z-axis rotation angle in degrees",
+          -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_ROTATION,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+  /**
+   * GstD3D11VideoSink:scale-x:
+   *
+   * Scale multiplier for x-axis
+   *
+   * Since: 1.24
+   */
+  g_object_class_install_property (gobject_class, PROP_SCALE_X,
+      g_param_spec_float ("scale-x", "Scale X",
+          "Scale multiplier for x-axis",
+          -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_SCALE,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+  /**
+   * GstD3D11VideoSink:scale-y:
+   *
+   * Scale multiplier for y-axis
+   *
+   * Since: 1.24
+   */
+  g_object_class_install_property (gobject_class, PROP_SCALE_Y,
+      g_param_spec_float ("scale-y", "Scale Y",
+          "Scale multiplier for y-axis",
+          -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_SCALE,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+  /**
+   * GstD3D11VideoSink:msaa:
+   *
+   * MSAA (Multi-Sampling Anti-Aliasing) level
+   *
+   * Since: 1.24
+   */
+  g_object_class_install_property (gobject_class, PROP_MSAA,
+      g_param_spec_enum ("msaa", "MSAA",
+          "MSAA (Multi-Sampling Anti-Aliasing) level",
+          GST_TYPE_D3D11_MSAA_MODE, DEFAULT_MSAA,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+
   /**
    * GstD3D11VideoSink:render-rectangle:
    *
@@ -503,6 +628,7 @@ gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass)
       (GstPluginAPIFlags) 0);
   gst_type_mark_as_plugin_api (GST_TYPE_D3D11_VIDEO_SINK_DISPLAY_FORMAT,
       (GstPluginAPIFlags) 0);
+  gst_type_mark_as_plugin_api (GST_TYPE_D3D11_MSAA_MODE, (GstPluginAPIFlags) 0);
 }
 
 static void
@@ -518,6 +644,14 @@ gst_d3d11_video_sink_init (GstD3D11VideoSink * self)
   self->primaries_mode = DEFAULT_PRIMARIES_MODE;
   self->display_format = DEFAULT_DISPLAY_FORMAT;
   self->emit_present = DEFAULT_EMIT_PRESENT;
+  self->fov = DEFAULT_FOV;
+  self->ortho = DEFAULT_ORTHO;
+  self->rotation_x = DEFAULT_ROTATION;
+  self->rotation_y = DEFAULT_ROTATION;
+  self->rotation_z = DEFAULT_ROTATION;
+  self->scale_x = DEFAULT_SCALE;
+  self->scale_y = DEFAULT_SCALE;
+  self->msaa = DEFAULT_MSAA;
 
   InitializeCriticalSection (&self->lock);
 }
@@ -579,6 +713,39 @@ gst_d3d11_videosink_set_property (GObject * object, guint prop_id,
     case PROP_EMIT_PRESENT:
       self->emit_present = g_value_get_boolean (value);
       break;
+    case PROP_FOV:
+      self->fov = g_value_get_float (value);
+      gst_d3d11_video_sink_set_orientation (self, self->method, FALSE);
+      break;
+    case PROP_ORTHO:
+      self->ortho = g_value_get_boolean (value);
+      gst_d3d11_video_sink_set_orientation (self, self->method, FALSE);
+      break;
+    case PROP_ROTATION_X:
+      self->rotation_x = g_value_get_float (value);
+      gst_d3d11_video_sink_set_orientation (self, self->method, FALSE);
+      break;
+    case PROP_ROTATION_Y:
+      self->rotation_y = g_value_get_float (value);
+      gst_d3d11_video_sink_set_orientation (self, self->method, FALSE);
+      break;
+    case PROP_ROTATION_Z:
+      self->rotation_z = g_value_get_float (value);
+      gst_d3d11_video_sink_set_orientation (self, self->method, FALSE);
+      break;
+    case PROP_SCALE_X:
+      self->scale_x = g_value_get_float (value);
+      gst_d3d11_video_sink_set_orientation (self, self->method, FALSE);
+      break;
+    case PROP_SCALE_Y:
+      self->scale_y = g_value_get_float (value);
+      gst_d3d11_video_sink_set_orientation (self, self->method, FALSE);
+      break;
+    case PROP_MSAA:
+      self->msaa = (GstD3D11MSAAMode) g_value_get_enum (value);
+      if (self->window)
+        gst_d3d11_window_set_msaa_mode (self->window, self->msaa);
+      break;
     case PROP_RENDER_RECTANGE:
       gst_video_overlay_set_property (object, PROP_RENDER_RECTANGE,
           PROP_RENDER_RECTANGE, value);
@@ -634,6 +801,30 @@ gst_d3d11_videosink_get_property (GObject * object, guint prop_id,
     case PROP_EMIT_PRESENT:
       g_value_set_boolean (value, self->emit_present);
       break;
+    case PROP_FOV:
+      g_value_set_float (value, self->fov);
+      break;
+    case PROP_ORTHO:
+      g_value_set_boolean (value, self->ortho);
+      break;
+    case PROP_ROTATION_X:
+      g_value_set_float (value, self->rotation_x);
+      break;
+    case PROP_ROTATION_Y:
+      g_value_set_float (value, self->rotation_x);
+      break;
+    case PROP_ROTATION_Z:
+      g_value_set_float (value, self->rotation_z);
+      break;
+    case PROP_SCALE_X:
+      g_value_set_float (value, self->scale_x);
+      break;
+    case PROP_SCALE_Y:
+      g_value_set_float (value, self->scale_y);
+      break;
+    case PROP_MSAA:
+      g_value_set_enum (value, self->msaa);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -1056,7 +1247,10 @@ done:
       "enable-navigation-events", self->enable_navigation_events,
       "emit-present", self->emit_present, nullptr);
 
-  gst_d3d11_window_set_orientation (self->window, self->selected_method);
+  gst_d3d11_window_set_orientation (self->window, self->selected_method,
+      self->fov, self->ortho, self->rotation_x, self->rotation_y,
+      self->rotation_z, self->scale_x, self->scale_y);
+  gst_d3d11_window_set_msaa_mode (self->window, self->msaa);
 
   g_signal_connect (self->window, "key-event",
       G_CALLBACK (gst_d3d11_video_sink_key_event), self);
@@ -1320,8 +1514,11 @@ gst_d3d11_video_sink_set_orientation (GstD3D11VideoSink * self,
     self->selected_method = self->method;
   }
 
-  if (self->window)
+  if (self->window) {
-    gst_d3d11_window_set_orientation (self->window, self->selected_method);
+    gst_d3d11_window_set_orientation (self->window, self->selected_method,
+        self->fov, self->ortho, self->rotation_x, self->rotation_y,
+        self->rotation_z, self->scale_x, self->scale_y);
+  }
 }
 
 static void

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.cpp

@@ -36,8 +36,13 @@
 
 #include <wrl.h>
 
+/* Disable platform-specific intrinsics */
+#define _XM_NO_INTRINSICS_
+#include <DirectXMath.h>
+
 /* *INDENT-OFF* */
 using namespace Microsoft::WRL;
+using namespace DirectX;
 /* *INDENT-ON* */
 
 GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_window_debug);
@@ -105,7 +110,7 @@ static void gst_d3d11_window_get_property (GObject * object, guint prop_id,
     GValue * value, GParamSpec * pspec);
 static void gst_d3d11_window_dispose (GObject * object);
 static GstFlowReturn gst_d3d11_window_present (GstD3D11Window * self,
-    GstBuffer * buffer, GstBuffer * render_target);
+    GstBuffer * buffer, GstBuffer * render_target, GstBuffer * multisample);
 static void gst_d3d11_window_on_resize_default (GstD3D11Window * window,
     guint width, guint height);
 static GstFlowReturn gst_d3d11_window_prepare_default (GstD3D11Window * window,
@@ -194,6 +199,14 @@ gst_d3d11_window_init (GstD3D11Window * self)
   self->fullscreen_toggle_mode = GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_NONE;
   self->fullscreen = DEFAULT_FULLSCREEN;
   self->emit_present = DEFAULT_EMIT_PRESENT;
+  self->fov = 90.0f;
+  self->ortho = FALSE;
+  self->rotation_x = 0.0f;
+  self->rotation_y = 0.0f;
+  self->rotation_z = 0.0f;
+  self->scale_x = 1.0f;
+  self->scale_y = 1.0f;
+  self->msaa = GST_D3D11_MSAA_DISABLED;
 }
 
 static void
@@ -279,6 +292,7 @@ gst_d3d11_window_dispose (GObject * object)
   gst_clear_object (&self->compositor);
   gst_clear_object (&self->converter);
 
+  gst_clear_buffer (&self->msaa_buffer);
   gst_clear_buffer (&self->cached_buffer);
   gst_clear_object (&self->device);
 
@@ -297,14 +311,19 @@ gst_d3d11_window_on_resize_default (GstD3D11Window * self, guint width,
   GstVideoRectangle src_rect, dst_rect, rst_rect;
   IDXGISwapChain *swap_chain;
   GstMemory *mem;
+  GstMemory *msaa_mem = nullptr;
   GstD3D11Memory *dmem;
   ID3D11RenderTargetView *rtv;
   ID3D11DeviceContext *context;
+  ID3D11Device *device_handle;
   gsize size;
   GstD3D11DeviceLockGuard lk (device);
   const FLOAT clear_color[] = { 0.0, 0.0, 0.0, 1.0 };
+  UINT quality_levels = 0;
+  UINT sample_count = 1;
 
   gst_clear_buffer (&self->backbuffer);
+  gst_clear_buffer (&self->msaa_buffer);
   if (!self->swap_chain)
     return;
 
@@ -350,12 +369,56 @@ gst_d3d11_window_on_resize_default (GstD3D11Window * self, guint width,
     return;
   }
 
+  switch (self->msaa) {
+    case GST_D3D11_MSAA_2X:
+      sample_count = 2;
+      break;
+    case GST_D3D11_MSAA_4X:
+      sample_count = 4;
+      break;
+    case GST_D3D11_MSAA_8X:
+      sample_count = 8;
+      break;
+    default:
+      break;
+  }
+
+  device_handle = gst_d3d11_device_get_device_handle (self->device);
+  while (sample_count > 1) {
+    hr = device_handle->CheckMultisampleQualityLevels (desc.Format,
+        sample_count, &quality_levels);
+    if (gst_d3d11_result (hr, device) && quality_levels > 0)
+      break;
+
+    sample_count = sample_count / 2;
+  };
+
+  if (sample_count > 1 && quality_levels > 0) {
+    ComPtr < ID3D11Texture2D > multisample_texture;
+    desc.SampleDesc.Count = sample_count;
+    desc.SampleDesc.Quality = quality_levels - 1;
+    device_handle->CreateTexture2D (&desc, nullptr, &multisample_texture);
+
+    if (multisample_texture) {
+      msaa_mem = gst_d3d11_allocator_alloc_wrapped (nullptr,
+          self->device, multisample_texture.Get (), size, nullptr, nullptr);
+
+      dmem = GST_D3D11_MEMORY_CAST (msaa_mem);
+      rtv = gst_d3d11_memory_get_render_target_view (dmem, 0);
+    }
+  }
+
   context = gst_d3d11_device_get_device_context_handle (self->device);
   context->ClearRenderTargetView (rtv, clear_color);
 
   self->backbuffer = gst_buffer_new ();
   gst_buffer_append_memory (self->backbuffer, mem);
 
+  if (msaa_mem) {
+    self->msaa_buffer = gst_buffer_new ();
+    gst_buffer_append_memory (self->msaa_buffer, msaa_mem);
+  }
+
   self->surface_width = desc.Width;
   self->surface_height = desc.Height;
 
@@ -399,8 +462,10 @@ gst_d3d11_window_on_resize_default (GstD3D11Window * self, guint width,
   self->first_present = TRUE;
 
   /* redraw the last scene if cached buffer exits */
-  if (self->cached_buffer)
+  if (self->cached_buffer) {
-    gst_d3d11_window_present (self, self->cached_buffer, self->backbuffer);
+    gst_d3d11_window_present (self, self->cached_buffer, self->backbuffer,
+        self->msaa_buffer);
+  }
 }
 
 void
@@ -847,9 +912,123 @@ gst_d3d11_window_set_title (GstD3D11Window * window, const gchar * title)
     klass->set_title (window, title);
 }
 
+static const XMFLOAT4X4 g_matrix_90r = XMFLOAT4X4 (0.0f, -1.0f, 0.0f, 0.0f,
+    1.0f, 0.0f, 0.0f, 0.0f,
+    0.0f, 0.0f, 1.0f, 0.0f,
+    0.0f, 0.0f, 0.0f, 1.0f);
+
+static const XMFLOAT4X4 g_matrix_180 = XMFLOAT4X4 (-1.0f, 0.0f, 0.0f, 0.0f,
+    0.0f, -1.0f, 0.0f, 0.0f,
+    0.0f, 0.0f, 1.0f, 0.0f,
+    0.0f, 0.0f, 0.0f, 1.0f);
+
+static const XMFLOAT4X4 g_matrix_90l = XMFLOAT4X4 (0.0f, 1.0f, 0.0f, 0.0f,
+    -1.0f, 0.0f, 0.0f, 0.0f,
+    0.0f, 0.0f, 1.0f, 0.0f,
+    0.0f, 0.0f, 0.0f, 1.0f);
+
+static const XMFLOAT4X4 g_matrix_horiz = XMFLOAT4X4 (-1.0f, 0.0f, 0.0f, 0.0f,
+    0.0f, 1.0f, 0.0f, 0.0f,
+    0.0f, 0.0f, 1.0f, 0.0f,
+    0.0f, 0.0f, 0.0f, 1.0f);
+
+static const XMFLOAT4X4 g_matrix_vert = XMFLOAT4X4 (1.0f, 0.0f, 0.0f, 0.0f,
+    0.0f, -1.0f, 0.0f, 0.0f,
+    0.0f, 0.0f, 1.0f, 0.0f,
+    0.0f, 0.0f, 0.0f, 1.0f);
+
+static const XMFLOAT4X4 g_matrix_ul_lr = XMFLOAT4X4 (0.0f, -1.0f, 0.0f, 0.0f,
+    -1.0f, 0.0f, 0.0f, 0.0f,
+    0.0f, 0.0f, 1.0f, 0.0f,
+    0.0f, 0.0f, 0.0f, 1.0f);
+
+static const XMFLOAT4X4 g_matrix_ur_ll = XMFLOAT4X4 (0.0f, 1.0f, 0.0f, 0.0f,
+    1.0f, 0.0f, 0.0f, 0.0f,
+    0.0f, 0.0f, 1.0f, 0.0f,
+    0.0f, 0.0f, 0.0f, 1.0f);
+
+static void
+gst_d3d11_window_calculate_matrix (GstD3D11Window * self,
+    gfloat viewport_width, gfloat viewport_height, gfloat transform_matrix[16])
+{
+  gfloat aspect_ratio;
+  gboolean rotated = FALSE;
+  XMMATRIX rotate_matrix = XMMatrixIdentity ();
+
+  switch (self->method) {
+    case GST_VIDEO_ORIENTATION_IDENTITY:
+    case GST_VIDEO_ORIENTATION_AUTO:
+    case GST_VIDEO_ORIENTATION_CUSTOM:
+    default:
+      break;
+    case GST_VIDEO_ORIENTATION_90R:
+      rotate_matrix = XMLoadFloat4x4 (&g_matrix_90r);
+      rotated = TRUE;
+      break;
+    case GST_VIDEO_ORIENTATION_180:
+      rotate_matrix = XMLoadFloat4x4 (&g_matrix_180);
+      break;
+    case GST_VIDEO_ORIENTATION_90L:
+      rotate_matrix = XMLoadFloat4x4 (&g_matrix_90l);
+      rotated = TRUE;
+      break;
+    case GST_VIDEO_ORIENTATION_HORIZ:
+      rotate_matrix = XMLoadFloat4x4 (&g_matrix_horiz);
+      break;
+    case GST_VIDEO_ORIENTATION_VERT:
+      rotate_matrix = XMLoadFloat4x4 (&g_matrix_vert);
+      break;
+    case GST_VIDEO_ORIENTATION_UL_LR:
+      rotate_matrix = XMLoadFloat4x4 (&g_matrix_ul_lr);
+      rotated = TRUE;
+      break;
+    case GST_VIDEO_ORIENTATION_UR_LL:
+      rotate_matrix = XMLoadFloat4x4 (&g_matrix_ur_ll);
+      rotated = TRUE;
+      break;
+  }
+
+  if (rotated)
+    aspect_ratio = viewport_height / viewport_width;
+  else
+    aspect_ratio = viewport_width / viewport_height;
+
+  /* Apply user specified transform matrix first, then rotate-method */
+  XMMATRIX scale =
+      XMMatrixScaling (self->scale_x * aspect_ratio, self->scale_y, 1.0);
+
+  XMMATRIX rotate =
+      XMMatrixRotationX (XMConvertToRadians (self->rotation_x)) *
+      XMMatrixRotationY (XMConvertToRadians (-self->rotation_y)) *
+      XMMatrixRotationZ (XMConvertToRadians (-self->rotation_z));
+
+  XMMATRIX view = XMMatrixLookAtLH (XMVectorSet (0.0, 0.0, -1.0, 0.0),
+      XMVectorSet (0.0, 0.0, 0.0, 0.0), XMVectorSet (0.0, 1.0, 0.0, 0.0));
+
+  XMMATRIX proj;
+  if (self->ortho) {
+    proj = XMMatrixOrthographicOffCenterLH (-aspect_ratio,
+        aspect_ratio, -1.0, 1.0, 0.1, 100.0);
+  } else {
+    proj = XMMatrixPerspectiveFovLH (XMConvertToRadians (self->fov),
+        aspect_ratio, 0.1, 100.0);
+  }
+
+  XMMATRIX mvp = scale * rotate * view * proj * rotate_matrix;
+
+  XMFLOAT4X4 matrix;
+  XMStoreFloat4x4 (&matrix, mvp);
+
+  for (guint i = 0; i < 4; i++) {
+    for (guint j = 0; j < 4; j++) {
+      transform_matrix[i * 4 + j] = matrix.m[i][j];
+    }
+  }
+}
+
 static GstFlowReturn
 gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer,
-    GstBuffer * backbuffer)
+    GstBuffer * backbuffer, GstBuffer * multisample)
 {
   GstD3D11WindowClass *klass = GST_D3D11_WINDOW_GET_CLASS (self);
   GstFlowReturn ret = GST_FLOW_OK;
@@ -859,6 +1038,8 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer,
   ID3D11RenderTargetView *rtv;
   GstMemory *mem;
   GstD3D11Memory *dmem;
+  GstBuffer *target_buf;
+  ID3D11DeviceContext *context;
 
   if (!buffer)
     return GST_FLOW_OK;
@@ -868,7 +1049,12 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer,
     return GST_FLOW_ERROR;
   }
 
-  mem = gst_buffer_peek_memory (backbuffer, 0);
+  if (multisample)
+    target_buf = multisample;
+  else
+    target_buf = backbuffer;
+
+  mem = gst_buffer_peek_memory (target_buf, 0);
   if (!gst_is_d3d11_memory (mem)) {
     GST_ERROR_OBJECT (self, "Invalid back buffer");
     return GST_FLOW_ERROR;
@@ -881,14 +1067,13 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer,
     return GST_FLOW_ERROR;
   }
 
+  context = gst_d3d11_device_get_device_context_handle (self->device);
+
   /* We use flip mode swapchain and will not redraw borders.
    * So backbuffer should be cleared manually in order to remove artifact of
    * previous client's rendering on present signal */
   if (self->emit_present) {
     const FLOAT clear_color[] = { 0.0f, 0.0f, 0.0f, 1.0f };
-    ID3D11DeviceContext *context =
-        gst_d3d11_device_get_device_context_handle (self->device);
-
     context->ClearRenderTargetView (rtv, clear_color);
   }
 
@@ -913,6 +1098,7 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer,
 
   if (self->first_present) {
     D3D11_VIEWPORT viewport;
+    const gfloat min_diff = 0.00001f;
 
     viewport.TopLeftX = self->render_rect.left;
     viewport.TopLeftY = self->render_rect.top;
@@ -926,17 +1112,51 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer,
         "dest-width",
         (gint) (self->render_rect.right - self->render_rect.left),
         "dest-height",
-        (gint) (self->render_rect.bottom - self->render_rect.top),
+        (gint) (self->render_rect.bottom - self->render_rect.top), nullptr);
-        "video-direction", self->method, nullptr);
+
+    if (!XMScalarNearEqual (self->rotation_x, 0.0f, min_diff) &&
+        !XMScalarNearEqual (self->rotation_y, 0.0f, min_diff) &&
+        !XMScalarNearEqual (self->rotation_z, 0.0f, min_diff) &&
+        !XMScalarNearEqual (self->scale_x, 1.0f, min_diff) &&
+        !XMScalarNearEqual (self->scale_y, 1.0f, min_diff)) {
+      g_object_set (self->converter, "video-direction", self->method, nullptr);
+    } else {
+      gfloat transform_matrix[16];
+
+      GST_DEBUG_OBJECT (self, "Applying custom transform");
+
+      gst_d3d11_window_calculate_matrix (self,
+          viewport.Width, viewport.Height, transform_matrix);
+      g_object_set (self->converter,
+          "video-direction", GST_VIDEO_ORIENTATION_CUSTOM, nullptr);
+      gst_d3d11_converter_set_transform_matrix (self->converter,
+          transform_matrix);
+    }
+
     gst_d3d11_overlay_compositor_update_viewport (self->compositor, &viewport);
   }
 
   if (!gst_d3d11_converter_convert_buffer_unlocked (self->converter,
-          buffer, backbuffer)) {
+          buffer, target_buf)) {
     GST_ERROR_OBJECT (self, "Couldn't render buffer");
     return GST_FLOW_ERROR;
   }
 
+  if (multisample) {
+    GstD3D11Memory *src_mem;
+    GstD3D11Memory *dst_mem;
+
+    src_mem = (GstD3D11Memory *) gst_buffer_peek_memory (multisample, 0);
+    dst_mem = (GstD3D11Memory *) gst_buffer_peek_memory (backbuffer, 0);
+
+    auto src_tex = gst_d3d11_memory_get_resource_handle (src_mem);
+    auto dst_tex = gst_d3d11_memory_get_resource_handle (dst_mem);
+
+    context->ResolveSubresource (dst_tex, 0, src_tex, 0, self->dxgi_format);
+
+    rtv = gst_d3d11_memory_get_render_target_view (dst_mem, 0);
+  }
+
   gst_d3d11_overlay_compositor_upload (self->compositor, buffer);
   gst_d3d11_overlay_compositor_draw_unlocked (self->compositor, &rtv);
 
@@ -963,7 +1183,7 @@ gst_d3d11_window_render (GstD3D11Window * window, GstBuffer * buffer)
     gst_buffer_replace (&window->cached_buffer, buffer);
 
   return gst_d3d11_window_present (window, window->cached_buffer,
-      window->backbuffer);
+      window->backbuffer, window->msaa_buffer);
 }
 
 GstFlowReturn
@@ -993,7 +1213,7 @@ gst_d3d11_window_render_on_shared_handle (GstD3D11Window * window,
     return GST_FLOW_OK;
   }
 
-  ret = gst_d3d11_window_present (window, buffer, data.render_target);
+  ret = gst_d3d11_window_present (window, buffer, data.render_target, nullptr);
 
   klass->release_shared_handle (window, &data);
 
@@ -1098,16 +1318,36 @@ gst_d3d11_window_get_native_type_to_string (GstD3D11WindowNativeType type)
 
 void
 gst_d3d11_window_set_orientation (GstD3D11Window * window,
-    GstVideoOrientationMethod method)
+    GstVideoOrientationMethod method, gfloat fov, gboolean ortho,
+    gfloat rotation_x, gfloat rotation_y, gfloat rotation_z,
+    gfloat scale_x, gfloat scale_y)
 {
-  if (method == GST_VIDEO_ORIENTATION_AUTO ||
-      method == GST_VIDEO_ORIENTATION_CUSTOM) {
-    return;
-  }
-
   GstD3D11DeviceLockGuard lk (window->device);
-  if (window->method != method) {
+  if (window->method != method || window->fov != fov || window->ortho != ortho
+      || window->rotation_x != rotation_x || window->rotation_y != rotation_y
+      || window->rotation_z != rotation_z || window->scale_x != scale_x
+      || window->scale_y != scale_y) {
     window->method = method;
+    window->fov = fov;
+    window->ortho = ortho;
+    window->rotation_x = rotation_x;
+    window->rotation_y = rotation_y;
+    window->rotation_z = rotation_z;
+    window->scale_x = scale_y;
+    if (window->swap_chain) {
+      GstD3D11WindowClass *klass = GST_D3D11_WINDOW_GET_CLASS (window);
+
+      klass->on_resize (window, window->surface_width, window->surface_height);
+    }
+  }
+}
+
+void
+gst_d3d11_window_set_msaa_mode (GstD3D11Window * window, GstD3D11MSAAMode mode)
+{
+  GstD3D11DeviceLockGuard lk (window->device);
+  if (window->msaa != mode) {
+    window->msaa = mode;
     if (window->swap_chain) {
       GstD3D11WindowClass *klass = GST_D3D11_WINDOW_GET_CLASS (window);
 

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.h

@@ -109,11 +109,20 @@ struct _GstD3D11Window
   IDXGISwapChain *swap_chain;
   GstBuffer *backbuffer;
   DXGI_FORMAT dxgi_format;
+  GstBuffer *msaa_buffer;
 
   GstBuffer *cached_buffer;
   gboolean first_present;
 
   GstVideoOrientationMethod method;
+  gfloat fov;
+  gboolean ortho;
+  gfloat rotation_x;
+  gfloat rotation_y;
+  gfloat rotation_z;
+  gfloat scale_x;
+  gfloat scale_y;
+  GstD3D11MSAAMode msaa;
 };
 
 struct _GstD3D11WindowClass
@@ -178,7 +187,17 @@ void          gst_d3d11_window_set_title            (GstD3D11Window * window,
                                                      const gchar *title);
 
 void          gst_d3d11_window_set_orientation      (GstD3D11Window * window,
-                                                     GstVideoOrientationMethod method);
+                                                     GstVideoOrientationMethod method,
+                                                     gfloat fov,
+                                                     gboolean ortho,
+                                                     gfloat rotation_x,
+                                                     gfloat rotation_y,
+                                                     gfloat rotation_z,
+                                                     gfloat scale_x,
+                                                     gfloat scale_y);
+
+void          gst_d3d11_window_set_msaa_mode        (GstD3D11Window * window,
+                                                     GstD3D11MSAAMode mode);
 
 GstFlowReturn gst_d3d11_window_prepare              (GstD3D11Window * window,
                                                      guint display_width,

subprojects/gst-plugins-bad/sys/d3d11/meson.build

@@ -130,6 +130,31 @@ if cc.get_id() == 'msvc' and fxc.found()
   extra_args += ['-DHLSL_PRECOMPILED']
 endif
 
+have_dx_math = cxx.compiles('''
+    #include <windows.h>
+    #include <DirectXMath.h>
+    using namespace DirectX;
+    int main(int argc, char ** argv) {
+      XMMATRIX matrix;
+      XMFLOAT4X4 dump;
+      matrix = XMMatrixIdentity ();
+      XMStoreFloat4x4 (&dump, matrix);
+      return 0;
+    }
+    ''',
+    name: 'DirectXMath suupport in Windows SDK')
+
+if not have_dx_math
+  directxmath_dep = dependency('directxmath',
+      fallback: ['directxmath', 'directxmath_dep'],
+      required: d3d11_option)
+  if not directxmath_dep.found()
+    subdir_done ()
+  endif
+
+  extra_dep += [directxmath_dep]
+endif
+
 gstd3d11 = library('gstd3d11',
   d3d11_sources + hlsl_precompiled,
   c_args : gst_plugins_bad_args + extra_c_args + extra_args,