d3d11: Handle device change

If incoming buffer holds other d3d11 device, and user wants any device
(i.e., adapter index wasn't specified explicitly) update our device
with that of buffer.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2191>

sys/d3d11/gstd3d11basefilter.cpp

@@ -55,6 +55,8 @@ static gboolean gst_d3d11_base_filter_get_unit_size (GstBaseTransform * trans,
 static gboolean
 gst_d3d11_base_filter_query (GstBaseTransform * trans,
     GstPadDirection direction, GstQuery * query);
+static void gst_d3d11_base_filter_before_transform (GstBaseTransform * trans,
+    GstBuffer * buffer);
 
 static void
 gst_d3d11_base_filter_class_init (GstD3D11BaseFilterClass * klass)
@@ -92,6 +94,8 @@ gst_d3d11_base_filter_class_init (GstD3D11BaseFilterClass * klass)
   trans_class->get_unit_size =
       GST_DEBUG_FUNCPTR (gst_d3d11_base_filter_get_unit_size);
   trans_class->query = GST_DEBUG_FUNCPTR (gst_d3d11_base_filter_query);
+  trans_class->before_transform =
+      GST_DEBUG_FUNCPTR (gst_d3d11_base_filter_before_transform);
 
   gst_type_mark_as_plugin_api (GST_TYPE_D3D11_BASE_FILTER,
       (GstPluginAPIFlags) 0);
@@ -260,3 +264,72 @@ gst_d3d11_base_filter_query (GstBaseTransform * trans,
   return GST_BASE_TRANSFORM_CLASS (parent_class)->query (trans, direction,
       query);
 }
+
+static void
+gst_d3d11_base_filter_before_transform (GstBaseTransform * trans,
+    GstBuffer * buffer)
+{
+  GstD3D11BaseFilter *self = GST_D3D11_BASE_FILTER (trans);
+  GstD3D11Memory *dmem;
+  GstMemory *mem;
+  gboolean update_device = FALSE;
+  GstCaps *in_caps = NULL;
+  GstCaps *out_caps = NULL;
+
+  mem = gst_buffer_peek_memory (buffer, 0);
+  /* Can happens (e.g., d3d11upload) */
+  if (!gst_is_d3d11_memory (mem))
+    return;
+
+  dmem = GST_D3D11_MEMORY_CAST (mem);
+  /* Same device, nothing to do */
+  if (dmem->device == self->device)
+    return;
+
+  /* Can accept any device, update */
+  if (self->adapter < 0) {
+    update_device = TRUE;
+  } else {
+    guint adapter = 0;
+
+    g_object_get (dmem->device, "adapter", &adapter, NULL);
+    /* The same GPU as what user wanted, update */
+    if (adapter == (guint) self->adapter)
+      update_device = TRUE;
+  }
+
+  if (!update_device)
+    return;
+
+  GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %"
+      GST_PTR_FORMAT, self->device, dmem->device);
+
+  gst_object_unref (self->device);
+  self->device = (GstD3D11Device *) gst_object_ref (dmem->device);
+
+  in_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM_SINK_PAD (trans));
+  if (!in_caps) {
+    GST_WARNING_OBJECT (self, "sinkpad has null caps");
+    goto out;
+  }
+
+  out_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM_SRC_PAD (trans));
+  if (!out_caps) {
+    GST_WARNING_OBJECT (self, "Has no configured output caps");
+    goto out;
+  }
+
+  /* subclass will update internal object.
+   * Note that gst_base_transform_reconfigure() might not trigger this
+   * unless caps was changed meanwhile */
+  gst_d3d11_base_filter_set_caps (trans, in_caps, out_caps);
+
+  /* Mark reconfigure so that we can update pool */
+  gst_base_transform_reconfigure_src (trans);
+
+out:
+  gst_clear_caps (&in_caps);
+  gst_clear_caps (&out_caps);
+
+  return;
+}

sys/d3d11/gstd3d11compositor.cpp

@@ -1312,10 +1312,12 @@ static gboolean gst_d3d11_compositor_decide_allocation (GstAggregator *
     aggregator, GstQuery * query);
 static gboolean gst_d3d11_compositor_sink_event (GstAggregator * agg,
     GstAggregatorPad * pad, GstEvent * event);
-
 static GstFlowReturn
 gst_d3d11_compositor_aggregate_frames (GstVideoAggregator * vagg,
     GstBuffer * outbuf);
+static GstFlowReturn
+gst_d3d11_compositor_create_output_buffer (GstVideoAggregator * vagg,
+    GstBuffer ** outbuffer);
 
 #define gst_d3d11_compositor_parent_class parent_class
 G_DEFINE_TYPE_WITH_CODE (GstD3D11Compositor, gst_d3d11_compositor,
@@ -1372,6 +1374,8 @@ gst_d3d11_compositor_class_init (GstD3D11CompositorClass * klass)
 
   vagg_class->aggregate_frames =
       GST_DEBUG_FUNCPTR (gst_d3d11_compositor_aggregate_frames);
+  vagg_class->create_output_buffer =
+      GST_DEBUG_FUNCPTR (gst_d3d11_compositor_create_output_buffer);
 
   caps = gst_d3d11_get_updated_template_caps (&pad_template_caps);
   gst_element_class_add_pad_template (element_class,
@@ -1519,6 +1523,21 @@ could_not_create:
   }
 }
 
+static gboolean
+gst_d3d11_compositor_pad_clear_resource (GstD3D11Compositor * self,
+    GstD3D11CompositorPad * cpad, gpointer user_data)
+{
+  gst_clear_buffer (&cpad->fallback_buf);
+  if (cpad->fallback_pool) {
+    gst_buffer_pool_set_active (cpad->fallback_pool, FALSE);
+    gst_clear_object (&cpad->fallback_pool);
+  }
+  g_clear_pointer (&cpad->convert, gst_d3d11_converter_free);
+  GST_D3D11_CLEAR_COM (cpad->blend);
+
+  return TRUE;
+}
+
 static void
 gst_d3d11_compositor_release_pad (GstElement * element, GstPad * pad)
 {
@@ -1530,13 +1549,7 @@ gst_d3d11_compositor_release_pad (GstElement * element, GstPad * pad)
   gst_child_proxy_child_removed (GST_CHILD_PROXY (self), G_OBJECT (pad),
       GST_OBJECT_NAME (pad));
 
-  gst_clear_buffer (&cpad->fallback_buf);
-  if (cpad->fallback_pool) {
-    gst_buffer_pool_set_active (cpad->fallback_pool, FALSE);
-    gst_clear_object (&cpad->fallback_pool);
-  }
-  g_clear_pointer (&cpad->convert, gst_d3d11_converter_free);
-  GST_D3D11_CLEAR_COM (cpad->blend);
+  gst_d3d11_compositor_pad_clear_resource (self, cpad, NULL);
 
   GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad);
 }
@@ -2255,3 +2268,109 @@ done:
 
   return ret;
 }
+
+typedef struct
+{
+  /* without holding ref */
+  GstD3D11Device *other_device;
+  gboolean have_same_device;
+} DeviceCheckData;
+
+static gboolean
+gst_d3d11_compositor_check_device_update (GstElement * agg,
+    GstVideoAggregatorPad * vpad, DeviceCheckData * data)
+{
+  GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (agg);
+  GstBuffer *buf;
+  GstMemory *mem;
+  GstD3D11Memory *dmem;
+  gboolean update_device = FALSE;
+
+  buf = gst_video_aggregator_pad_get_current_buffer (vpad);
+  if (!buf)
+    return TRUE;
+
+  mem = gst_buffer_peek_memory (buf, 0);
+  /* FIXME: we should be able to accept non-d3d11 memory later once
+   * we remove intermediate elements (d3d11upload and d3d11colorconvert)
+   */
+  if (!gst_is_d3d11_memory (mem)) {
+    GST_ELEMENT_ERROR (agg, CORE, FAILED, (NULL), ("Invalid memory"));
+    return FALSE;
+  }
+
+  dmem = GST_D3D11_MEMORY_CAST (mem);
+
+  /* We can use existing device */
+  if (dmem->device == self->device) {
+    data->have_same_device = TRUE;
+    return FALSE;
+  }
+
+  if (self->adapter < 0) {
+    update_device = TRUE;
+  } else {
+    guint adapter = 0;
+
+    g_object_get (dmem->device, "adapter", &adapter, NULL);
+    /* The same GPU as what user wanted, update */
+    if (adapter == (guint) self->adapter)
+      update_device = TRUE;
+  }
+
+  if (!update_device)
+    return TRUE;
+
+  data->other_device = dmem->device;
+
+  /* Keep iterate since there might be one buffer which holds the same device
+   * as ours */
+  return TRUE;
+}
+
+static GstFlowReturn
+gst_d3d11_compositor_create_output_buffer (GstVideoAggregator * vagg,
+    GstBuffer ** outbuffer)
+{
+  GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg);
+  DeviceCheckData data;
+
+  /* Check whether there is at least one sinkpad which holds d3d11 buffer
+   * with compatible device, and if not, update our device */
+  data.other_device = NULL;
+  data.have_same_device = FALSE;
+
+  gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg),
+      (GstElementForeachPadFunc) gst_d3d11_compositor_check_device_update,
+      &data);
+
+  if (data.have_same_device || !data.other_device)
+    goto done;
+
+  /* Clear all device dependent resources */
+  gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg),
+      (GstElementForeachPadFunc) gst_d3d11_compositor_pad_clear_resource, NULL);
+
+  gst_clear_buffer (&self->fallback_buf);
+  if (self->fallback_pool) {
+    gst_buffer_pool_set_active (self->fallback_pool, FALSE);
+    gst_clear_object (&self->fallback_pool);
+  }
+  g_clear_pointer (&self->checker_background, gst_d3d11_quad_free);
+
+  GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %"
+      GST_PTR_FORMAT, self->device, data.other_device);
+  gst_object_unref (self->device);
+  self->device = (GstD3D11Device *) gst_object_ref (data.other_device);
+
+  /* We cannot call gst_aggregator_negotiate() here, since GstVideoAggregator
+   * is holding GST_VIDEO_AGGREGATOR_LOCK() already.
+   * Mark reconfigure and do reconfigure later */
+  gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg));
+
+  return GST_AGGREGATOR_FLOW_NEED_DATA;
+
+done:
+  return GST_VIDEO_AGGREGATOR_CLASS (parent_class)->create_output_buffer (vagg,
+      outbuffer);
+}

sys/d3d11/gstd3d11deinterlace.cpp

@@ -328,6 +328,8 @@ gst_d3d11_deinterlace_transform (GstBaseTransform * trans, GstBuffer * inbuf,
     GstBuffer * outbuf);
 static gboolean gst_d3d11_deinterlace_sink_event (GstBaseTransform * trans,
     GstEvent * event);
+static void gst_d3d11_deinterlace_before_transform (GstBaseTransform * trans,
+    GstBuffer * buffer);
 
 static void
 gst_d3d11_deinterlace_class_init (GstD3D11DeinterlaceClass * klass,
@@ -413,6 +415,8 @@ gst_d3d11_deinterlace_class_init (GstD3D11DeinterlaceClass * klass,
   trans_class->transform = GST_DEBUG_FUNCPTR (gst_d3d11_deinterlace_transform);
   trans_class->sink_event =
       GST_DEBUG_FUNCPTR (gst_d3d11_deinterlace_sink_event);
+  trans_class->before_transform =
+      GST_DEBUG_FUNCPTR (gst_d3d11_deinterlace_before_transform);
 
   klass->adapter = cdata->adapter;
   klass->device_id = cdata->device_id;
@@ -1829,6 +1833,68 @@ gst_d3d11_deinterlace_sink_event (GstBaseTransform * trans, GstEvent * event)
   return GST_BASE_TRANSFORM_CLASS (parent_class)->sink_event (trans, event);
 }
 
+static void
+gst_d3d11_deinterlace_before_transform (GstBaseTransform * trans,
+    GstBuffer * buffer)
+{
+  GstD3D11Deinterlace *self = GST_D3D11_DEINTERLACE (trans);
+  GstD3D11DeinterlaceClass *klass = GST_D3D11_DEINTERLACE_GET_CLASS (self);
+  GstD3D11Memory *dmem;
+  GstMemory *mem;
+  GstCaps *in_caps = NULL;
+  GstCaps *out_caps = NULL;
+  guint adapter = 0;
+
+  mem = gst_buffer_peek_memory (buffer, 0);
+  if (!gst_is_d3d11_memory (mem)) {
+    GST_ELEMENT_ERROR (self, CORE, FAILED, (NULL), ("Invalid memory"));
+    return;
+  }
+
+  dmem = GST_D3D11_MEMORY_CAST (mem);
+  /* Same device, nothing to do */
+  if (dmem->device == self->device)
+    return;
+
+  g_object_get (dmem->device, "adapter", &adapter, NULL);
+  /* We have per-GPU deinterlace elements because of different capability
+   * per GPU. so, cannot accept other GPU at the moment */
+  if (adapter != klass->adapter)
+    return;
+
+  GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %"
+      GST_PTR_FORMAT, self->device, dmem->device);
+
+  /* Drain buffers before updating device */
+  gst_d3d11_deinterlace_drain (self);
+
+  gst_object_unref (self->device);
+  self->device = (GstD3D11Device *) gst_object_ref (dmem->device);
+
+  in_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM_SINK_PAD (trans));
+  if (!in_caps) {
+    GST_WARNING_OBJECT (self, "sinkpad has null caps");
+    goto out;
+  }
+
+  out_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM_SRC_PAD (trans));
+  if (!out_caps) {
+    GST_WARNING_OBJECT (self, "Has no configured output caps");
+    goto out;
+  }
+
+  gst_d3d11_deinterlace_set_caps (trans, in_caps, out_caps);
+
+  /* Mark reconfigure so that we can update pool */
+  gst_base_transform_reconfigure_src (trans);
+
+out:
+  gst_clear_caps (&in_caps);
+  gst_clear_caps (&out_caps);
+
+  return;
+}
+
 /* FIXME: might be job of basetransform */
 static GstFlowReturn
 gst_d3d11_deinterlace_drain (GstD3D11Deinterlace * self)

sys/d3d11/gstd3d11videosink.cpp

@@ -1049,6 +1049,48 @@ error:
   return FALSE;
 }
 
+static void
+gst_d3d11_video_sink_check_device_update (GstD3D11VideoSink * self,
+    GstBuffer * buf)
+{
+  GstMemory *mem;
+  GstD3D11Memory *dmem;
+  gboolean update_device = FALSE;
+
+  /* We have configured window already, cannot update device */
+  if (self->window)
+    return;
+
+  mem = gst_buffer_peek_memory (buf, 0);
+  if (!gst_is_d3d11_memory (mem))
+    return;
+
+  dmem = GST_D3D11_MEMORY_CAST (mem);
+  /* Same device, nothing to do */
+  if (dmem->device == self->device)
+    return;
+
+  if (self->adapter < 0) {
+    update_device = TRUE;
+  } else {
+    guint adapter = 0;
+
+    g_object_get (dmem->device, "adapter", &adapter, NULL);
+    /* The same GPU as what user wanted, update */
+    if (adapter == (guint) self->adapter)
+      update_device = TRUE;
+  }
+
+  if (!update_device)
+    return;
+
+  GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %"
+      GST_PTR_FORMAT, self->device, dmem->device);
+
+  gst_object_unref (self->device);
+  self->device = (GstD3D11Device *) gst_object_ref (dmem->device);
+}
+
 static GstFlowReturn
 gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
 {
@@ -1060,6 +1102,8 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
       gst_d3d11_device_get_device_handle (self->device);
   ID3D11ShaderResourceView *view[GST_VIDEO_MAX_PLANES];
 
+  gst_d3d11_video_sink_check_device_update (self, buf);
+
   if (self->caps_updated || !self->window) {
     GstCaps *caps = gst_pad_get_current_caps (GST_BASE_SINK_PAD (sink));
     gboolean update_ret;