diff --git a/gst/vaapi/gstvaapipluginbase.c b/gst/vaapi/gstvaapipluginbase.c
index 32cfd120e1..47e3fd7429 100644
--- a/gst/vaapi/gstvaapipluginbase.c
+++ b/gst/vaapi/gstvaapipluginbase.c
@@ -469,6 +469,28 @@ has_dmabuf_capable_peer (GstVaapiPluginBase * plugin, GstPad * pad)
   return is_dmabuf_capable;
 }
 
+static gboolean
+set_dmabuf_allocator (GstVaapiPluginBase * plugin, GstBufferPool * pool,
+    GstCaps * caps)
+{
+  GstStructure *config;
+  GstAllocator *allocator;
+  GstVideoInfo vi;
+  gboolean ret;
+
+  if (!gst_video_info_from_caps (&vi, caps))
+    return FALSE;
+  config = gst_buffer_pool_get_config (pool);
+  allocator = gst_vaapi_dmabuf_allocator_new (plugin->display, &vi,
+      GST_VAAPI_SURFACE_ALLOC_FLAG_LINEAR_STORAGE);
+  if (!allocator)
+    return FALSE;
+  gst_buffer_pool_config_set_allocator (config, allocator, NULL);
+  ret = gst_buffer_pool_set_config (pool, config);
+  gst_object_unref (allocator);
+  return ret;
+}
+
 static gboolean
 gst_vaapi_buffer_pool_caps_is_equal (GstBufferPool * pool, GstCaps * newcaps)
 {
@@ -502,6 +524,8 @@ ensure_sinkpad_buffer_pool (GstVaapiPluginBase * plugin, GstCaps * caps)
   GstBufferPool *pool;
   GstStructure *config;
   GstVideoInfo vi;
+  GstAllocator *allocator;
+  gboolean configured;
 
   /* video decoders don't use a buffer pool in the sink pad */
   if (GST_IS_VIDEO_DECODER (plugin))
@@ -527,13 +551,20 @@ ensure_sinkpad_buffer_pool (GstVaapiPluginBase * plugin, GstCaps * caps)
   gst_video_info_force_nv12_if_encoded (&vi);
   plugin->sinkpad_buffer_size = GST_VIDEO_INFO_SIZE (&vi);
 
+  allocator = gst_vaapi_video_allocator_new (plugin->display, &vi, 0);
+  if (!allocator)
+    goto error_create_allocator;
+
   config = gst_buffer_pool_get_config (pool);
   gst_buffer_pool_config_set_params (config, caps,
       plugin->sinkpad_buffer_size, 0, 0);
+  gst_buffer_pool_config_set_allocator (config, allocator, NULL);
   gst_buffer_pool_config_add_option (config,
       GST_BUFFER_POOL_OPTION_VAAPI_VIDEO_META);
   gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
-  if (!gst_buffer_pool_set_config (pool, config))
+  configured = gst_buffer_pool_set_config (pool, config);
+  gst_object_unref (allocator);
+  if (!configured)
     goto error_pool_config;
   plugin->sinkpad_buffer_pool = pool;
   return TRUE;
@@ -545,6 +576,12 @@ error_invalid_caps:
     gst_object_unref (pool);
     return FALSE;
   }
+error_create_allocator:
+  {
+    GST_ERROR_OBJECT (plugin, "failed to create allocator");
+    gst_object_unref (pool);
+    return FALSE;
+  }
 error_create_pool:
   {
     GST_ERROR_OBJECT (plugin, "failed to create buffer pool");
@@ -618,12 +655,7 @@ gst_vaapi_plugin_base_propose_allocation (GstVaapiPluginBase * plugin,
         plugin->sinkpad_buffer_size, 0, 0);
 
     if (has_dmabuf_capable_peer (plugin, plugin->sinkpad)) {
-      GstStructure *const config =
-          gst_buffer_pool_get_config (plugin->sinkpad_buffer_pool);
-
-      gst_buffer_pool_config_add_option (config,
-          GST_BUFFER_POOL_OPTION_DMABUF_MEMORY);
-      if (!gst_buffer_pool_set_config (plugin->sinkpad_buffer_pool, config))
+      if (!set_dmabuf_allocator (plugin, plugin->sinkpad_buffer_pool, caps))
         goto error_pool_config;
     }
   }
@@ -668,6 +700,7 @@ gst_vaapi_plugin_base_decide_allocation (GstVaapiPluginBase * plugin,
   gboolean update_pool = FALSE;
   gboolean has_video_meta = FALSE;
   gboolean has_video_alignment = FALSE;
+  GstAllocator *allocator;
 #if (USE_GLX || USE_EGL)
   gboolean has_texture_upload_meta = FALSE;
   guint idx;
@@ -749,10 +782,17 @@ gst_vaapi_plugin_base_decide_allocation (GstVaapiPluginBase * plugin,
     if (!pool)
       goto error_create_pool;
 
+    allocator = gst_vaapi_video_allocator_new (plugin->display, &vi, 0);
+    if (!allocator)
+      goto error_create_allocator;
+
     config = gst_buffer_pool_get_config (pool);
     gst_buffer_pool_config_set_params (config, caps, size, min, max);
     gst_buffer_pool_config_add_option (config,
         GST_BUFFER_POOL_OPTION_VAAPI_VIDEO_META);
+    gst_buffer_pool_config_set_allocator (config, allocator, NULL);
+
+    gst_object_unref (allocator);
   }
 
   if (!config)
@@ -809,6 +849,12 @@ error_create_pool:
     GST_ERROR_OBJECT (plugin, "failed to create buffer pool");
     return FALSE;
   }
+error_create_allocator:
+  {
+    GST_ERROR_OBJECT (plugin, "failed to create allocator");
+    gst_object_unref (pool);
+    return FALSE;
+  }
 error_config_failed:
   {
     if (pool)
diff --git a/gst/vaapi/gstvaapivideobufferpool.c b/gst/vaapi/gstvaapivideobufferpool.c
index 41eac91499..b424027557 100644
--- a/gst/vaapi/gstvaapivideobufferpool.c
+++ b/gst/vaapi/gstvaapivideobufferpool.c
@@ -136,13 +136,15 @@ gst_vaapi_video_buffer_pool_set_config (GstBufferPool * pool,
 {
   GstVaapiVideoBufferPoolPrivate *const priv =
       GST_VAAPI_VIDEO_BUFFER_POOL (pool)->priv;
-  GstCaps *caps = NULL;
-  GstVideoInfo *const cur_vip = &priv->video_info;
+  GstCaps *caps;
   GstVideoInfo new_vip;
+  const GstVideoInfo *alloc_vip;
   GstVideoAlignment align;
   GstAllocator *allocator;
-  gboolean changed_caps, use_dmabuf_memory;
 
+  GST_DEBUG_OBJECT (pool, "config %" GST_PTR_FORMAT, config);
+
+  caps = NULL;
   if (!gst_buffer_pool_config_get_params (config, &caps, NULL, NULL, NULL))
     goto error_invalid_config;
   if (!caps)
@@ -150,43 +152,36 @@ gst_vaapi_video_buffer_pool_set_config (GstBufferPool * pool,
   if (!gst_video_info_from_caps (&new_vip, caps))
     goto error_invalid_caps;
 
-  use_dmabuf_memory = gst_buffer_pool_config_has_option (config,
-      GST_BUFFER_POOL_OPTION_DMABUF_MEMORY);
-  if (priv->use_dmabuf_memory != use_dmabuf_memory) {
-    priv->use_dmabuf_memory = use_dmabuf_memory;
-    g_clear_object (&priv->allocator);
-  }
+  allocator = NULL;
+  if (!gst_buffer_pool_config_get_allocator (config, &allocator, NULL))
+    goto error_invalid_allocator;
 
-  changed_caps = !priv->allocator || gst_video_info_changed (cur_vip, &new_vip);
-  if (changed_caps) {
-    const GstVideoInfo *alloc_vip;
-    guint flags = 0;
-
-    if (use_dmabuf_memory) {
-      /* XXX: also needs fixed strides/offsets */
-      flags |= GST_VAAPI_SURFACE_ALLOC_FLAG_LINEAR_STORAGE;
-      allocator =
-          gst_vaapi_dmabuf_allocator_new (priv->display, &new_vip, flags);
-    } else {
-      allocator = gst_vaapi_video_allocator_new (priv->display, &new_vip, 0);
-    }
-    if (!allocator)
-      goto error_create_allocator;
-    gst_object_replace ((GstObject **) & priv->allocator,
-        GST_OBJECT_CAST (allocator));
-    gst_object_unref (allocator);
-    priv->video_info = new_vip;
-
-    alloc_vip = gst_allocator_get_vaapi_video_info (allocator, NULL);
-    if (!alloc_vip)
-      goto error_create_allocator_info;
-    priv->alloc_info = *alloc_vip;
-  }
+  if (gst_video_info_changed (&priv->video_info, &new_vip))
+    gst_object_replace ((GstObject **) & priv->allocator, NULL);
+  priv->video_info = new_vip;
 
   if (!gst_buffer_pool_config_has_option (config,
           GST_BUFFER_POOL_OPTION_VAAPI_VIDEO_META))
     goto error_no_vaapi_video_meta_option;
 
+  /* not our allocator, not our buffers */
+  if (allocator) {
+    priv->use_dmabuf_memory = gst_vaapi_is_dmabuf_allocator (allocator);
+    if (priv->use_dmabuf_memory ||
+        g_strcmp0 (allocator->mem_type, GST_VAAPI_VIDEO_MEMORY_NAME) == 0) {
+      if (priv->allocator)
+        gst_object_unref (priv->allocator);
+      if ((priv->allocator = allocator))
+        gst_object_ref (allocator);
+      alloc_vip = gst_allocator_get_vaapi_video_info (priv->allocator, NULL);
+      if (!alloc_vip)
+        goto error_create_allocator_info;
+      priv->alloc_info = *alloc_vip;
+    }
+  }
+  if (!priv->allocator)
+    goto error_no_allocator;
+
   priv->has_video_meta = gst_buffer_pool_config_has_option (config,
       GST_BUFFER_POOL_OPTION_VIDEO_META);
 
@@ -213,7 +208,7 @@ error_invalid_config:
   }
 error_no_caps:
   {
-    GST_ERROR_OBJECT (pool, "no valid caps in config");
+    GST_ERROR_OBJECT (pool, "no caps in config");
     return FALSE;
   }
 error_invalid_caps:
@@ -221,9 +216,14 @@ error_invalid_caps:
     GST_ERROR_OBJECT (pool, "invalid caps %" GST_PTR_FORMAT, caps);
     return FALSE;
   }
-error_create_allocator:
+error_invalid_allocator:
   {
-    GST_ERROR_OBJECT (pool, "failed to create GstVaapiVideoAllocator object");
+    GST_ERROR_OBJECT (pool, "no allocator in config");
+    return FALSE;
+  }
+error_no_vaapi_video_meta_option:
+  {
+    GST_ERROR_OBJECT (pool, "no GstVaapiVideoMeta option in config");
     return FALSE;
   }
 error_create_allocator_info:
@@ -232,9 +232,9 @@ error_create_allocator_info:
         "failed to create GstVaapiVideoAllocator `video-info'");
     return FALSE;
   }
-error_no_vaapi_video_meta_option:
+error_no_allocator:
   {
-    GST_ERROR_OBJECT (pool, "no GstVaapiVideoMeta option");
+    GST_ERROR_OBJECT (pool, "no allocator defined");
     return FALSE;
   }
 }
diff --git a/gst/vaapi/gstvaapivideobufferpool.h b/gst/vaapi/gstvaapivideobufferpool.h
index 07b28a7a45..8f691caad7 100644
--- a/gst/vaapi/gstvaapivideobufferpool.h
+++ b/gst/vaapi/gstvaapivideobufferpool.h
@@ -54,17 +54,6 @@ typedef struct _GstVaapiVideoBufferPoolPrivate GstVaapiVideoBufferPoolPrivate;
 #define GST_BUFFER_POOL_OPTION_VAAPI_VIDEO_META \
   "GstBufferPoolOptionVaapiVideoMeta"
 
-/**
- * GST_BUFFER_POOL_OPTION_DMABUF_MEMORY:
- *
- * An option that can be activated on bufferpool to request dmabuf
- * handles on buffers from the pool.
- */
-#ifndef GST_BUFFER_POOL_OPTION_DMABUF_MEMORY
-#define GST_BUFFER_POOL_OPTION_DMABUF_MEMORY \
-  "GstBufferPoolOptionDMABUFMemory"
-#endif
-
 /**
  * GstVaapiVideoBufferPoolAcquireFlags:
  * @GST_VAAPI_VIDEO_BUFFER_POOL_ACQUIRE_FLAG_NO_ALLOC: option to
diff --git a/gst/vaapi/gstvaapivideomemory.c b/gst/vaapi/gstvaapivideomemory.c
index 5eb0cb94bc..ea09177faf 100644
--- a/gst/vaapi/gstvaapivideomemory.c
+++ b/gst/vaapi/gstvaapivideomemory.c
@@ -1035,3 +1035,16 @@ gst_allocator_set_vaapi_video_info (GstAllocator * allocator,
 
   return TRUE;
 }
+
+gboolean
+gst_vaapi_is_dmabuf_allocator (GstAllocator * allocator)
+{
+  GstStructure *st;
+
+  g_return_val_if_fail (GST_IS_ALLOCATOR (allocator), FALSE);
+
+  if (g_strcmp0 (allocator->mem_type, GST_ALLOCATOR_DMABUF) != 0)
+    return FALSE;
+  st = g_object_get_qdata (G_OBJECT (allocator), GST_VAAPI_VIDEO_INFO_QUARK);
+  return (st != NULL);
+}
diff --git a/gst/vaapi/gstvaapivideomemory.h b/gst/vaapi/gstvaapivideomemory.h
index 43994c7a50..6254f324e3 100644
--- a/gst/vaapi/gstvaapivideomemory.h
+++ b/gst/vaapi/gstvaapivideomemory.h
@@ -214,6 +214,9 @@ gboolean
 gst_allocator_set_vaapi_video_info (GstAllocator * allocator,
     const GstVideoInfo * vip, guint flags);
 
+gboolean
+gst_vaapi_is_dmabuf_allocator (GstAllocator * allocator);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_VIDEO_MEMORY_H */