d3d11videosink: Move potentially time-consuming operations to ::prepare()

Move following tasks to ::prepare() from ::show_frame()
* CPU -> GPU upload
* GstD3D11Window object setup, including input caps change handling

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3574>

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp

@@ -170,12 +170,11 @@ struct _GstD3D11VideoSink
   GstD3D11Window *window;
   gint video_width;
   gint video_height;
-
   GstVideoInfo info;
-
   guintptr window_id;
-
   gboolean caps_updated;
+  GstBuffer *prepared_buffer;
+  GstBufferPool *pool;
 
   /* properties */
   gint adapter;
@@ -195,7 +194,6 @@ struct _GstD3D11VideoSink
 
   /* For drawing on user texture */
   gboolean drawing;
-  GstBuffer *current_buffer;
   CRITICAL_SECTION lock;
 
   gchar *title;
@@ -240,6 +238,8 @@ static gboolean gst_d3d11_video_sink_unlock (GstBaseSink * sink);
 static gboolean gst_d3d11_video_sink_unlock_stop (GstBaseSink * sink);
 static gboolean gst_d3d11_video_sink_event (GstBaseSink * sink,
     GstEvent * event);
+static GstFlowReturn gst_d3d11_video_sink_prepare (GstBaseSink * sink,
+    GstBuffer * buffer);
 static GstFlowReturn
 gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf);
 static gboolean gst_d3d11_video_sink_prepare_window (GstD3D11VideoSink * self);
@@ -484,6 +484,7 @@ gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass)
   basesink_class->unlock_stop =
       GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_unlock_stop);
   basesink_class->event = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_event);
+  basesink_class->prepare = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_prepare);
 
   videosink_class->show_frame =
       GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_show_frame);
@@ -718,10 +719,17 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
   GstStructure *config;
   GstD3D11Window *window;
   GstFlowReturn ret = GST_FLOW_OK;
+  GstD3D11AllocationParams *params;
+  guint bind_flags = D3D11_BIND_SHADER_RESOURCE;
+  GstD3D11Format device_format;
 
   GST_DEBUG_OBJECT (self, "Updating window with caps %" GST_PTR_FORMAT, caps);
 
   self->caps_updated = FALSE;
+  if (self->pool) {
+    gst_buffer_pool_set_active (self->pool, FALSE);
+    gst_clear_object (&self->pool);
+  }
 
   EnterCriticalSection (&self->lock);
   if (!gst_d3d11_video_sink_prepare_window (self)) {
@@ -849,6 +857,32 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
 
   gst_object_unref (window);
 
+  self->pool = gst_d3d11_buffer_pool_new (self->device);
+  config = gst_buffer_pool_get_config (self->pool);
+
+  if (gst_d3d11_device_get_format (self->device,
+          GST_VIDEO_INFO_FORMAT (&self->info), &device_format) &&
+      (device_format.format_support[0] &
+          (guint) D3D11_FORMAT_SUPPORT_RENDER_TARGET) != 0) {
+    bind_flags |= D3D11_BIND_RENDER_TARGET;
+  }
+
+  params = gst_d3d11_allocation_params_new (self->device, &self->info,
+      GST_D3D11_ALLOCATION_FLAG_DEFAULT, bind_flags, 0);
+  gst_buffer_pool_config_set_d3d11_allocation_params (config, params);
+  gst_d3d11_allocation_params_free (params);
+
+  gst_buffer_pool_config_set_params (config, caps, self->info.size, 0, 0);
+  if (!gst_buffer_pool_set_config (self->pool, config) ||
+      !gst_buffer_pool_set_active (self->pool, TRUE)) {
+    GST_ERROR_OBJECT (self, "Couldn't setup buffer pool");
+    gst_clear_object (&self->pool);
+
+    GST_ELEMENT_ERROR (self, RESOURCE, FAILED, (nullptr),
+        ("Couldn't setup buffer pool"));
+    return GST_FLOW_ERROR;
+  }
+
   return GST_FLOW_OK;
 }
 
@@ -1024,6 +1058,12 @@ gst_d3d11_video_sink_stop (GstBaseSink * sink)
 
   GST_DEBUG_OBJECT (self, "Stop");
 
+  gst_clear_buffer (&self->prepared_buffer);
+  if (self->pool) {
+    gst_buffer_pool_set_active (self->pool, FALSE);
+    gst_clear_object (&self->pool);
+  }
+
   if (self->window)
     gst_d3d11_window_unprepare (self->window);
 
@@ -1187,6 +1227,8 @@ gst_d3d11_video_sink_unlock_stop (GstBaseSink * sink)
   if (self->window)
     gst_d3d11_window_unlock_stop (self->window);
 
+  gst_clear_buffer (&self->prepared_buffer);
+
   return TRUE;
 }
 
@@ -1306,13 +1348,14 @@ gst_d3d11_video_sink_check_device_update (GstD3D11VideoSink * self,
 }
 
 static GstFlowReturn
-gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
+gst_d3d11_video_sink_prepare (GstBaseSink * sink, GstBuffer * buffer)
 {
   GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
-  GstFlowReturn ret = GST_FLOW_OK;
+  GstFlowReturn ret;
 
-  gst_d3d11_video_sink_check_device_update (self, buf);
+  gst_clear_buffer (&self->prepared_buffer);
 
+  gst_d3d11_video_sink_check_device_update (self, buffer);
   if (self->caps_updated || !self->window) {
     GstCaps *caps = gst_pad_get_current_caps (GST_BASE_SINK_PAD (sink));
 
@@ -1327,12 +1370,55 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
       return ret;
   }
 
-  gst_d3d11_window_show (self->window);
+  if (!gst_is_d3d11_buffer (buffer)) {
+    GstVideoOverlayCompositionMeta *overlay_meta;
+
+    ret = gst_buffer_pool_acquire_buffer (self->pool, &self->prepared_buffer,
+        nullptr);
+    if (ret != GST_FLOW_OK)
+      return ret;
+
+    gst_d3d11_buffer_copy_into (self->prepared_buffer, buffer, &self->info);
+    /* Upload to default texture */
+    for (guint i = 0; i < gst_buffer_n_memory (self->prepared_buffer); i++) {
+      GstMemory *mem = gst_buffer_peek_memory (self->prepared_buffer, i);
+      GstMapInfo info;
+      if (!gst_memory_map (mem,
+              &info, (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) {
+        GST_ERROR_OBJECT (self, "Couldn't map fallback buffer");
+        gst_clear_buffer (&self->prepared_buffer);
+        return GST_FLOW_ERROR;
+      }
+
+      gst_memory_unmap (mem, &info);
+    }
+
+    overlay_meta = gst_buffer_get_video_overlay_composition_meta (buffer);
+    if (overlay_meta) {
+      gst_buffer_add_video_overlay_composition_meta (self->prepared_buffer,
+          overlay_meta->overlay);
+    }
+  } else {
+    self->prepared_buffer = gst_buffer_ref (buffer);
+  }
+
+  return GST_FLOW_OK;
+}
+
+static GstFlowReturn
+gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
+{
+  GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
+  GstFlowReturn ret = GST_FLOW_OK;
+
+  if (!self->prepared_buffer) {
+    GST_ERROR_OBJECT (self, "No prepared buffer");
+    return GST_FLOW_ERROR;
+  }
 
   if (self->draw_on_shared_texture) {
     GstD3D11CSLockGuard lk (&self->lock);
 
-    self->current_buffer = buf;
     self->drawing = TRUE;
 
     GST_LOG_OBJECT (self, "Begin drawing");
@@ -1343,9 +1429,8 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
 
     GST_LOG_OBJECT (self, "End drawing");
     self->drawing = FALSE;
-    self->current_buffer = nullptr;
   } else {
-    ret = gst_d3d11_window_render (self->window, buf);
+    ret = gst_d3d11_window_render (self->window, self->prepared_buffer);
   }
 
   if (ret == GST_D3D11_WINDOW_FLOW_CLOSED) {
@@ -1466,7 +1551,7 @@ gst_d3d11_video_sink_draw_action (GstD3D11VideoSink * self,
   }
 
   GstD3D11CSLockGuard lk (&self->lock);
-  if (!self->drawing || !self->current_buffer) {
+  if (!self->drawing || !self->prepared_buffer) {
     GST_WARNING_OBJECT (self, "Nothing to draw");
     return FALSE;
   }
@@ -1477,7 +1562,7 @@ gst_d3d11_video_sink_draw_action (GstD3D11VideoSink * self,
       release_key);
 
   ret = gst_d3d11_window_render_on_shared_handle (self->window,
-      self->current_buffer, shared_handle, texture_misc_flags, acquire_key,
+      self->prepared_buffer, shared_handle, texture_misc_flags, acquire_key,
       release_key);
 
   return ret == GST_FLOW_OK;

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.cpp

@@ -814,19 +814,6 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
   return GST_FLOW_OK;
 }
 
-void
-gst_d3d11_window_show (GstD3D11Window * window)
-{
-  GstD3D11WindowClass *klass;
-
-  g_return_if_fail (GST_IS_D3D11_WINDOW (window));
-
-  klass = GST_D3D11_WINDOW_GET_CLASS (window);
-
-  if (klass->show)
-    klass->show (window);
-}
-
 void
 gst_d3d11_window_set_render_rectangle (GstD3D11Window * window,
     const GstVideoRectangle * rect)

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.h

@@ -121,8 +121,6 @@ struct _GstD3D11WindowClass
 {
   GstObjectClass object_class;
 
-  void          (*show)                   (GstD3D11Window * window);
-
   void          (*update_swap_chain)      (GstD3D11Window * window);
 
   void          (*change_fullscreen_mode) (GstD3D11Window * window);
@@ -170,8 +168,6 @@ struct _GstD3D11WindowClass
 
 GType         gst_d3d11_window_get_type             (void);
 
-void          gst_d3d11_window_show                 (GstD3D11Window * window);
-
 void          gst_d3d11_window_set_render_rectangle (GstD3D11Window * window,
                                                      const GstVideoRectangle * rect);
 

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window_win32.cpp

@@ -43,7 +43,6 @@ G_LOCK_DEFINE_STATIC (create_lock);
 #define WM_GST_D3D11_CONSTRUCT_INTERNAL_WINDOW (WM_USER + 2)
 #define WM_GST_D3D11_DESTROY_INTERNAL_WINDOW (WM_USER + 3)
 #define WM_GST_D3D11_MOVE_WINDOW (WM_USER + 4)
-#define WM_GST_D3D11_SHOW_WINDOW (WM_USER + 5)
 
 static LRESULT CALLBACK window_proc (HWND hWnd, UINT uMsg, WPARAM wParam,
     LPARAM lParam);
@@ -67,8 +66,6 @@ struct _GstD3D11WindowWin32
   GMainContext *main_context;
   GMainLoop *loop;
 
-  gboolean visible;
-
   GSource *msg_source;
   GIOChannel *msg_io_channel;
 
@@ -105,7 +102,6 @@ G_DEFINE_TYPE (GstD3D11WindowWin32, gst_d3d11_window_win32,
 static void gst_d3d11_window_win32_constructed (GObject * object);
 static void gst_d3d11_window_win32_dispose (GObject * object);
 
-static void gst_d3d11_window_win32_show (GstD3D11Window * window);
 static void gst_d3d11_window_win32_update_swap_chain (GstD3D11Window * window);
 static void
 gst_d3d11_window_win32_change_fullscreen_mode (GstD3D11Window * window);
@@ -147,7 +143,6 @@ gst_d3d11_window_win32_class_init (GstD3D11WindowWin32Class * klass)
   gobject_class->constructed = gst_d3d11_window_win32_constructed;
   gobject_class->dispose = gst_d3d11_window_win32_dispose;
 
-  window_class->show = GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_show);
   window_class->update_swap_chain =
       GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_update_swap_chain);
   window_class->change_fullscreen_mode =
@@ -216,9 +211,32 @@ gst_d3d11_window_win32_prepare (GstD3D11Window * window, guint display_width,
   GstD3D11WindowWin32 *self = GST_D3D11_WINDOW_WIN32 (window);
   HWND hwnd;
   GstFlowReturn ret;
+  gint width, height;
 
-  if (!self->setup_external_hwnd)
+  switch (window->method) {
+    case GST_VIDEO_ORIENTATION_90R:
+    case GST_VIDEO_ORIENTATION_90L:
+    case GST_VIDEO_ORIENTATION_UL_LR:
+    case GST_VIDEO_ORIENTATION_UR_LL:
+      width = display_height;
+      height = display_width;
+      break;
+    default:
+      width = display_width;
+      height = display_height;
+      break;
+  }
+
+  if (!self->setup_external_hwnd) {
+    RECT rect;
+    GetClientRect (self->internal_hwnd, &rect);
+    width += 2 * GetSystemMetrics (SM_CXSIZEFRAME);
+    height +=
+        2 * GetSystemMetrics (SM_CYSIZEFRAME) + GetSystemMetrics (SM_CYCAPTION);
+    MoveWindow (self->internal_hwnd, rect.left, rect.top, width, height, FALSE);
+    ShowWindow (self->internal_hwnd, SW_SHOW);
     goto done;
+  }
 
   hwnd = (HWND) window->external_handle;
   if (!IsWindow (hwnd)) {
@@ -574,9 +592,6 @@ gst_d3d11_window_win32_create_internal_window (GstD3D11WindowWin32 * self)
   }
 
   self->device_handle = 0;
-  self->internal_hwnd = 0;
-  self->visible = FALSE;
-
   self->internal_hwnd = CreateWindowExA (0,
       "GSTD3D11",
       "Direct3D11 renderer",
@@ -823,9 +838,6 @@ gst_d3d11_window_win32_handle_window_proc (GstD3D11WindowWin32 * self,
         }
       }
       break;
-    case WM_GST_D3D11_SHOW_WINDOW:
-      ShowWindow (self->internal_hwnd, SW_SHOW);
-      break;
     default:
       break;
   }
@@ -917,6 +929,7 @@ sub_class_proc (HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
         SWP_FRAMECHANGED | SWP_NOACTIVATE);
     MoveWindow (self->internal_hwnd, rect.left, rect.top, rect.right,
         rect.bottom, FALSE);
+    ShowWindow (self->internal_hwnd, SW_SHOW);
 
     GstD3D11SRWLockGuard lk (&self->lock);
     self->overlay_state = GST_D3D11_WINDOW_WIN32_OVERLAY_STATE_OPENED;
@@ -1112,51 +1125,6 @@ gst_d3d11_window_win32_create_swap_chain (GstD3D11Window * window,
   return TRUE;
 }
 
-static void
-gst_d3d11_window_win32_show (GstD3D11Window * window)
-{
-  GstD3D11WindowWin32 *self = GST_D3D11_WINDOW_WIN32 (window);
-  gint width, height;
-
-  switch (window->method) {
-    case GST_VIDEO_ORIENTATION_90R:
-    case GST_VIDEO_ORIENTATION_90L:
-    case GST_VIDEO_ORIENTATION_UL_LR:
-    case GST_VIDEO_ORIENTATION_UR_LL:
-      width = GST_VIDEO_INFO_HEIGHT (&window->render_info);
-      height = GST_VIDEO_INFO_WIDTH (&window->render_info);
-      break;
-    default:
-      width = GST_VIDEO_INFO_WIDTH (&window->render_info);
-      height = GST_VIDEO_INFO_HEIGHT (&window->render_info);
-      break;
-  }
-
-  if (!self->visible) {
-    /* if no parent the real size has to be set now because this has not been done
-     * when at window creation */
-    if (!self->external_hwnd) {
-      RECT rect;
-      GetClientRect (self->internal_hwnd, &rect);
-      width += 2 * GetSystemMetrics (SM_CXSIZEFRAME);
-      height +=
-          2 * GetSystemMetrics (SM_CYSIZEFRAME) +
-          GetSystemMetrics (SM_CYCAPTION);
-      MoveWindow (self->internal_hwnd, rect.left, rect.top, width,
-          height, FALSE);
-      ShowWindow (self->internal_hwnd, SW_SHOW);
-    } else if (self->internal_hwnd) {
-      /* ShowWindow will throw message to message pumping thread (app thread)
-       * synchroniously, which can be blocked at the moment.
-       * Post message to internal hwnd and do that from message pumping thread
-       */
-      PostMessageA (self->internal_hwnd, WM_GST_D3D11_SHOW_WINDOW, 0, 0);
-    }
-
-    self->visible = TRUE;
-  }
-}
-
 static GstFlowReturn
 gst_d3d11_window_win32_present (GstD3D11Window * window, guint present_flags)
 {