diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11bufferpool.cpp b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11bufferpool.cpp
index fbf1e8dc93..d5a86b64c2 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11bufferpool.cpp
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11bufferpool.cpp
@@ -243,9 +243,9 @@ gst_d3d11_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
 
     pool_alloc = GST_D3D11_POOL_ALLOCATOR (alloc);
     flow_ret = gst_d3d11_pool_allocator_acquire_memory (pool_alloc, &mem);
+    gst_d3d11_allocator_set_active (alloc, FALSE);
     if (flow_ret != GST_FLOW_OK) {
       GST_ERROR_OBJECT (self, "Failed to allocate initial memory");
-      gst_d3d11_allocator_set_active (alloc, FALSE);
       gst_object_unref (alloc);
       return FALSE;
     }
@@ -254,7 +254,6 @@ gst_d3d11_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
             &stride) || stride < desc[i].Width) {
       GST_ERROR_OBJECT (self, "Failed to calculate stride");
 
-      gst_d3d11_allocator_set_active (alloc, FALSE);
       gst_object_unref (alloc);
       gst_memory_unref (mem);
 
diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.cpp b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.cpp
index 7c39236c76..d443ff1eab 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.cpp
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.cpp
@@ -2208,6 +2208,8 @@ gst_d3d11_pool_allocator_release_memory (GstD3D11PoolAllocator * self,
   /* keep it around in our queue */
   priv->queue.push (mem);
   priv->outstanding--;
+  if (priv->outstanding == 0 && priv->flushing)
+    gst_d3d11_pool_allocator_stop (self);
   WakeAllConditionVariable (&priv->cond);
   ReleaseSRWLockExclusive (&priv->lock);
 
@@ -2232,13 +2234,6 @@ gst_d3d11_memory_release (GstMiniObject * object)
   priv = alloc->priv;
 
   AcquireSRWLockExclusive (&priv->lock);
-  /* if flushing, free this memory */
-  if (alloc->priv->flushing) {
-    ReleaseSRWLockExclusive (&priv->lock);
-    GST_LOG_OBJECT (alloc, "allocator is flushing, free %p", mem);
-    return TRUE;
-  }
-
   /* return the memory to the allocator */
   gst_memory_ref (mem);
   gst_d3d11_pool_allocator_release_memory (alloc, mem);
diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12memory.cpp b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12memory.cpp
index 6af65b476f..ad9c4efced 100644
--- a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12memory.cpp
+++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12memory.cpp
@@ -1100,6 +1100,8 @@ gst_d3d12_pool_allocator_release_memory (GstD3D12PoolAllocator * self,
   /* keep it around in our queue */
   priv->queue.push (mem);
   priv->outstanding--;
+  if (priv->outstanding == 0 && priv->flushing)
+    gst_d3d12_pool_allocator_stop (self);
   priv->cond.notify_all ();
   priv->lock.unlock ();
 
@@ -1124,13 +1126,6 @@ gst_d3d12_memory_release (GstMiniObject * mini_object)
   priv = alloc->priv;
 
   priv->lock.lock ();
-  /* if flushing, free this memory */
-  if (alloc->priv->flushing) {
-    priv->lock.unlock ();
-    GST_LOG_OBJECT (alloc, "allocator is flushing, free %p", mem);
-    return TRUE;
-  }
-
   /* return the memory to the allocator */
   gst_memory_ref (mem);
   gst_d3d12_pool_allocator_release_memory (alloc, mem);