d3d11screencapturesrc: Add "window-capture-mode" property

... to support capturing only window's client area

Fixes: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/2425
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4324>

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapturesrc.cpp

@@ -66,6 +66,7 @@ enum
   PROP_SHOW_BORDER,
   PROP_CAPTURE_API,
   PROP_ADAPTER,
+  PROP_WINDOW_CAPTURE_MODE,
 };
 
 typedef enum
@@ -74,6 +75,12 @@ typedef enum
   GST_D3D11_SCREEN_CAPTURE_API_WGC,
 } GstD3D11ScreenCaptureAPI;
 
+typedef enum
+{
+  GST_D3D11_WINDOW_CAPTURE_DEFAULT,
+  GST_D3D11_WINDOW_CAPTURE_CLIENT,
+} GstD3D11WindowCaptureMode;
+
 #ifdef HAVE_WINRT_CAPTURE
 /**
  * GstD3D11ScreenCaptureAPI:
@@ -109,6 +116,42 @@ gst_d3d11_screen_capture_api_get_type (void)
 
   return api_type;
 }
+
+/**
+ * GstD3D11WindowCaptureMode:
+ *
+ * Since: 1.24
+ */
+#define GST_TYPE_D3D11_WINDOW_CAPTURE_MODE (gst_d3d11_window_capture_mode_get_type())
+static GType
+gst_d3d11_window_capture_mode_get_type (void)
+{
+  static GType type = 0;
+
+  GST_D3D11_CALL_ONCE_BEGIN {
+    static const GEnumValue hwnd_modes[] = {
+      /**
+       * GstD3D11WindowCaptureMode::default:
+       *
+       * Since: 1.24
+       */
+      {GST_D3D11_WINDOW_CAPTURE_DEFAULT,
+          "Capture entire window area", "default"},
+
+      /**
+       * GstD3D11WindowCaptureMode::client:
+       *
+       * Since: 1.24
+       */
+      {GST_D3D11_WINDOW_CAPTURE_CLIENT, "Capture client area", "client"},
+      {0, nullptr, nullptr},
+    };
+
+    type = g_enum_register_static ("GstD3D11WindowCaptureMode", hwnd_modes);
+  } GST_D3D11_CALL_ONCE_END;
+
+  return type;
+}
 #endif
 
 #define DEFAULT_MONITOR_INDEX -1
@@ -116,6 +159,7 @@ gst_d3d11_screen_capture_api_get_type (void)
 #define DEFAULT_SHOW_BORDER FALSE
 #define DEFAULT_CAPTURE_API GST_D3D11_SCREEN_CAPTURE_API_DXGI
 #define DEFAULT_ADAPTER -1
+#define DEFAULT_WINDOW_CAPTURE_MODE GST_D3D11_WINDOW_CAPTURE_DEFAULT
 
 static GstStaticCaps template_caps =
     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
@@ -142,6 +186,7 @@ struct _GstD3D11ScreenCaptureSrc
   gboolean show_cursor;
   gboolean show_border;
   GstD3D11ScreenCaptureAPI capture_api;
+  GstD3D11WindowCaptureMode hwnd_capture_mode;
   gint adapter;
 
   guint crop_x;
@@ -343,6 +388,20 @@ gst_d3d11_screen_capture_src_class_init (GstD3D11ScreenCaptureSrcClass * klass)
             -1, G_MAXINT32, DEFAULT_ADAPTER,
             (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
                 GST_PARAM_CONDITIONALLY_AVAILABLE | G_PARAM_STATIC_STRINGS)));
+
+    /**
+     * GstD3D11ScreenCaptureSrc:hwnd-capture-mode:
+     *
+     * Since: 1.24
+     */
+    g_object_class_install_property (gobject_class, PROP_WINDOW_CAPTURE_MODE,
+        g_param_spec_enum ("window-capture-mode", "Window Capture Mode",
+            "Window capture mode to use if \"window-handle\" is set",
+            GST_TYPE_D3D11_WINDOW_CAPTURE_MODE, DEFAULT_WINDOW_CAPTURE_MODE,
+            (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
+                GST_PARAM_CONDITIONALLY_AVAILABLE | G_PARAM_STATIC_STRINGS)));
+    gst_type_mark_as_plugin_api (GST_TYPE_D3D11_WINDOW_CAPTURE_MODE,
+        (GstPluginAPIFlags) 0);
   }
 #endif
 
@@ -388,6 +447,7 @@ gst_d3d11_screen_capture_src_init (GstD3D11ScreenCaptureSrc * self)
   self->show_cursor = DEFAULT_SHOW_CURSOR;
   self->show_border = DEFAULT_SHOW_BORDER;
   self->capture_api = DEFAULT_CAPTURE_API;
+  self->hwnd_capture_mode = DEFAULT_WINDOW_CAPTURE_MODE;
   self->adapter = DEFAULT_ADAPTER;
   self->min_latency = GST_CLOCK_TIME_NONE;
   self->max_latency = GST_CLOCK_TIME_NONE;
@@ -459,6 +519,10 @@ gst_d3d11_screen_capture_src_set_property (GObject * object, guint prop_id,
     case PROP_ADAPTER:
       self->adapter = g_value_get_int (value);
       break;
+    case PROP_WINDOW_CAPTURE_MODE:
+      self->hwnd_capture_mode =
+          (GstD3D11WindowCaptureMode) g_value_get_enum (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -505,6 +569,9 @@ gst_d3d11_screen_capture_src_get_property (GObject * object, guint prop_id,
     case PROP_ADAPTER:
       g_value_set_int (value, self->adapter);
       break;
+    case PROP_WINDOW_CAPTURE_MODE:
+      g_value_set_enum (value, self->hwnd_capture_mode);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -965,9 +1032,11 @@ gst_d3d11_screen_capture_src_start (GstBaseSrc * bsrc)
 #ifdef HAVE_WINRT_CAPTURE
   if (self->window_handle) {
     capture = gst_d3d11_winrt_capture_new (self->device, nullptr,
-        self->window_handle);
+        self->window_handle,
+        self->hwnd_capture_mode == GST_D3D11_WINDOW_CAPTURE_CLIENT);
   } else if (self->capture_api == GST_D3D11_SCREEN_CAPTURE_API_WGC) {
-    capture = gst_d3d11_winrt_capture_new (self->device, monitor, nullptr);
+    capture = gst_d3d11_winrt_capture_new (self->device,
+        monitor, nullptr, FALSE);
   }
 #endif
 
@@ -990,7 +1059,8 @@ gst_d3d11_screen_capture_src_start (GstBaseSrc * bsrc)
         self->capture_api = GST_D3D11_SCREEN_CAPTURE_API_WGC;
         gst_clear_object (&capture);
         GST_WARNING_OBJECT (self, "DXGI capture is not available");
-        capture = gst_d3d11_winrt_capture_new (self->device, monitor, nullptr);
+        capture = gst_d3d11_winrt_capture_new (self->device,
+            monitor, nullptr, FALSE);
         if (capture
             && gst_d3d11_screen_capture_prepare (capture) == GST_FLOW_OK) {
           GST_INFO_OBJECT (self, "Fallback to Windows Graphics Capture");

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11winrtcapture.cpp

@@ -38,6 +38,7 @@
 #include <windows.graphics.directx.direct3d11.h>
 #include <windows.graphics.directx.direct3d11.interop.h>
 #include <string.h>
+#include <dwmapi.h>
 
 #include <wrl.h>
 
@@ -79,6 +80,7 @@ typedef struct
   HRESULT (WINAPI * WindowsDeleteString) (HSTRING string);
   HRESULT (WINAPI * RoGetActivationFactory) (HSTRING activatable_class_id,
       REFIID iid, void ** factory);
+  DPI_AWARENESS_CONTEXT (WINAPI * SetThreadDpiAwarenessContext) (DPI_AWARENESS_CONTEXT context);
 } GstD3D11WinRTVTable;
 
 static GstD3D11WinRTVTable winrt_vtable = { FALSE, };
@@ -153,6 +155,8 @@ struct GstD3D11WinRTCaptureInner
       g_module_close (d3d11_module); \
     if (combase_module) \
       g_module_close (combase_module); \
+    if (user32_module) \
+      g_module_close (user32_module); \
     return; \
   } \
 } G_STMT_END
@@ -162,6 +166,7 @@ gst_d3d11_winrt_capture_load_library (void)
 {
   static GModule *d3d11_module = nullptr;
   static GModule *combase_module = nullptr;
+  static GModule *user32_module = nullptr;
 
   GST_D3D11_CALL_ONCE_BEGIN {
     d3d11_module = g_module_open ("d3d11.dll", G_MODULE_BIND_LAZY);
@@ -175,6 +180,13 @@ gst_d3d11_winrt_capture_load_library (void)
       return;
     }
 
+    user32_module = g_module_open ("user32.dll", G_MODULE_BIND_LAZY);
+    if (!user32_module) {
+      g_module_close (combase_module);
+      g_module_close (d3d11_module);
+      return;
+    }
+
     LOAD_SYMBOL (d3d11_module, CreateDirect3D11DeviceFromDXGIDevice,
         CreateDirect3D11DeviceFromDXGIDevice);
     LOAD_SYMBOL (combase_module, RoInitialize, RoInitialize);
@@ -183,6 +195,8 @@ gst_d3d11_winrt_capture_load_library (void)
     LOAD_SYMBOL (combase_module, WindowsDeleteString, WindowsDeleteString);
     LOAD_SYMBOL (combase_module, RoGetActivationFactory,
         RoGetActivationFactory);
+    LOAD_SYMBOL (user32_module, SetThreadDpiAwarenessContext,
+        SetThreadDpiAwarenessContext);
 
     winrt_vtable.loaded = TRUE;
   }
@@ -197,6 +211,7 @@ enum
   PROP_D3D11_DEVICE,
   PROP_MONITOR_HANDLE,
   PROP_WINDOW_HANDLE,
+  PROP_CLIENT_ONLY,
 };
 
 struct _GstD3D11WinRTCapture
@@ -210,6 +225,8 @@ struct _GstD3D11WinRTCapture
   /* Actual texture resolution */
   UINT width;
   UINT height;
+  UINT capture_width;
+  UINT capture_height;
 
   gboolean flushing;
   boolean show_mouse;
@@ -225,6 +242,7 @@ struct _GstD3D11WinRTCapture
 
   HMONITOR monitor_handle;
   HWND window_handle;
+  gboolean client_only;
 
   HWND hidden_window;
 };
@@ -292,6 +310,11 @@ gst_d3d11_winrt_capture_class_init (GstD3D11WinRTCaptureClass * klass)
           "A HWND handle of window to capture", (GParamFlags)
           (G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY |
               G_PARAM_STATIC_STRINGS)));
+  g_object_class_install_property (gobject_class, PROP_CLIENT_ONLY,
+      g_param_spec_boolean ("client-only",
+          "Client Only", "Captures only client area", FALSE,
+          (GParamFlags) (G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY |
+              G_PARAM_STATIC_STRINGS)));
 
   capture_class->prepare = GST_DEBUG_FUNCPTR (gst_d3d11_winrt_capture_prepare);
   capture_class->get_size =
@@ -372,6 +395,9 @@ gst_d3d11_winrt_capture_set_property (GObject * object, guint prop_id,
     case PROP_WINDOW_HANDLE:
       self->window_handle = (HWND) g_value_get_pointer (value);
       break;
+    case PROP_CLIENT_ONLY:
+      self->client_only = g_value_get_boolean (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -478,8 +504,26 @@ gst_d3d11_winrt_configure (GstD3D11WinRTCapture * self)
     goto error;
   }
 
-  self->width = (UINT) self->pool_size.Width;
-  self->height = (UINT) self->pool_size.Height;
+  self->width = self->capture_width = (UINT) self->pool_size.Width;
+  self->height = self->capture_height = (UINT) self->pool_size.Height;
+
+  if (self->window_handle && self->client_only) {
+    RECT rect;
+    if (!GetClientRect (self->window_handle, &rect)) {
+      GST_ERROR_OBJECT (self, "Could not get client rect");
+      goto error;
+    }
+
+    self->capture_width = rect.right - rect.left;
+    self->capture_height = rect.bottom - rect.top;
+
+    self->capture_width = MAX (self->capture_width, 1);
+    self->capture_height = MAX (self->capture_height, 1);
+
+    GST_DEBUG_OBJECT (self, "Client rect %d:%d:%d:%d, pool size %dx%d",
+        rect.left, rect.top, rect.right, rect.bottom,
+        self->width, self->height);
+  }
 
   hr = pool_statics2->CreateFreeThreaded (inner->d3d_device.Get (),
       DirectXPixelFormat::DirectXPixelFormat_B8G8R8A8UIntNormalized,
@@ -652,6 +696,9 @@ gst_d3d11_winrt_capture_thread_func (GstD3D11WinRTCapture * self)
   }
 #endif
 
+  winrt_vtable.SetThreadDpiAwarenessContext
+      (DPI_AWARENESS_CONTEXT_SYSTEM_AWARE);
+
   QueryPerformanceFrequency (&self->frequency);
 
   winrt_vtable.RoInitialize (RO_INIT_MULTITHREADED);
@@ -744,8 +791,8 @@ gst_d3d11_winrt_capture_get_size (GstD3D11ScreenCapture * capture,
 {
   GstD3D11WinRTCapture *self = GST_D3D11_WINRT_CAPTURE (capture);
 
-  *width = self->width;
-  *height = self->height;
+  *width = self->capture_width;
+  *height = self->capture_height;
 
   return TRUE;
 }
@@ -819,6 +866,7 @@ gst_d3d11_winrt_capture_do_capture (GstD3D11ScreenCapture * capture,
   LONGLONG timeout;
   D3D11_TEXTURE2D_DESC desc;
   gboolean size_changed = FALSE;
+  D3D11_BOX box = *crop_box;
 
   GstD3D11CSLockGuard lk (&self->lock);
 again:
@@ -933,23 +981,93 @@ again:
         self->width, self->height, desc.Width, desc.Height);
     self->width = desc.Width;
     self->height = desc.Height;
+    if (!self->window_handle || !self->client_only) {
+      self->capture_width = self->width;
+      self->capture_height = self->capture_height;
+    }
+
     size_changed = TRUE;
   }
 
+  if (self->window_handle && self->client_only) {
+    RECT client_rect, bound_rect;
+    POINT client_pos = { 0, };
+    UINT width, height;
+    DPI_AWARENESS_CONTEXT prev;
+    BOOL ret;
+
+    prev = winrt_vtable.SetThreadDpiAwarenessContext
+        (DPI_AWARENESS_CONTEXT_SYSTEM_AWARE);
+    ret = GetClientRect (self->window_handle, &client_rect) &&
+        DwmGetWindowAttribute (self->window_handle,
+        DWMWA_EXTENDED_FRAME_BOUNDS, &bound_rect, sizeof (RECT)) == S_OK &&
+        ClientToScreen (self->window_handle, &client_pos);
+
+    if (prev)
+      winrt_vtable.SetThreadDpiAwarenessContext (prev);
+
+    if (!ret) {
+      GST_ERROR_OBJECT (self, "Could not get client rect");
+      return GST_FLOW_ERROR;
+    }
+
+    width = client_rect.right - client_rect.left;
+    height = client_rect.bottom - client_rect.top;
+
+    width = MAX (width, 1);
+    height = MAX (height, 1);
+
+    if (self->capture_width != width || self->capture_height != height) {
+      GST_DEBUG_OBJECT (self, "Client rect size changed %dx%d -> %dx%d",
+          self->capture_width, self->capture_height, width, height);
+      self->capture_width = width;
+      self->capture_height = height;
+      size_changed = TRUE;
+    } else {
+      UINT x_offset = 0;
+      UINT y_offset = 0;
+
+      GST_LOG_OBJECT (self, "bound-rect: %d:%d:%d:%d, "
+          "client-rect: %d:%d:%d:%d, client-upper-left: %d:%d",
+          bound_rect.left, bound_rect.top, bound_rect.right, bound_rect.bottom,
+          client_rect.left, client_rect.top, client_rect.right,
+          client_rect.bottom, client_pos.x, client_pos.y);
+
+      if (client_pos.x > bound_rect.left)
+        x_offset = client_pos.x - bound_rect.left;
+
+      if (client_pos.y > bound_rect.top)
+        y_offset = client_pos.y - bound_rect.top;
+
+      box.left += x_offset;
+      box.top += y_offset;
+
+      box.right += x_offset;
+      box.bottom += y_offset;
+
+      /* left and top is inclusive */
+      box.left = MIN (desc.Width - 1, box.left);
+      box.top = MIN (desc.Height - 1, box.top);
+
+      box.right = MIN (desc.Width, box.right);
+      box.bottom = MIN (desc.Height, box.bottom);
+    }
+  }
+
   if (size_changed)
     return GST_D3D11_SCREEN_CAPTURE_FLOW_SIZE_CHANGED;
 
   context_handle = gst_d3d11_device_get_device_context_handle (self->device);
   GstD3D11DeviceLockGuard device_lk (self->device);
   context_handle->CopySubresourceRegion (texture, 0, 0, 0, 0,
-      captured_texture.Get (), 0, crop_box);
+      captured_texture.Get (), 0, &box);
 
   return GST_FLOW_OK;
 }
 
 GstD3D11ScreenCapture *
 gst_d3d11_winrt_capture_new (GstD3D11Device * device, HMONITOR monitor_handle,
-    HWND window_handle)
+    HWND window_handle, gboolean client_only)
 {
   GstD3D11WinRTCapture *self;
 
@@ -963,7 +1081,9 @@ gst_d3d11_winrt_capture_new (GstD3D11Device * device, HMONITOR monitor_handle,
 
   self = (GstD3D11WinRTCapture *) g_object_new (GST_TYPE_D3D11_WINRT_CAPTURE,
       "d3d11device", device, "monitor-handle", (gpointer) monitor_handle,
-      "window-handle", (gpointer) window_handle, nullptr);
+      "window-handle", (gpointer) window_handle, "client-only", client_only,
+      nullptr);
+
   if (!self->inner) {
     gst_clear_object (&self);
     return nullptr;

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11winrtcapture.h

@@ -35,7 +35,8 @@ gboolean                gst_d3d11_winrt_capture_load_library (void);
 
 GstD3D11ScreenCapture * gst_d3d11_winrt_capture_new (GstD3D11Device * device,
                                                      HMONITOR monitor_handle,
-                                                     HWND window_handle);
+                                                     HWND window_handle,
+                                                     gboolean client_only);
 
 G_END_DECLS
 

subprojects/gst-plugins-bad/sys/d3d11/meson.build

@@ -91,9 +91,11 @@ if d3d11_winapi_desktop
     extra_dep += [winmm_lib]
   endif
 
-  if have_wgc
+  dwmapi_lib = cc.find_library('dwmapi', required: false)
+  if have_wgc and dwmapi_lib.found()
     d3d11_sources += ['gstd3d11winrtcapture.cpp']
     extra_args += ['-DHAVE_WINRT_CAPTURE']
+    extra_dep += [dwmapi_lib]
   endif
 endif