diff --git a/gst/vaapi/gstvaapivideobufferpool.c b/gst/vaapi/gstvaapivideobufferpool.c
index a331d4ff61..d99f306e84 100644
--- a/gst/vaapi/gstvaapivideobufferpool.c
+++ b/gst/vaapi/gstvaapivideobufferpool.c
@@ -166,14 +166,8 @@ gst_vaapi_video_buffer_pool_set_config (GstBufferPool * pool,
   if (allocator
       && (g_strcmp0 (allocator->mem_type, GST_VAAPI_VIDEO_MEMORY_NAME) != 0
           && g_strcmp0 (allocator->mem_type,
-              GST_VAAPI_DMABUF_ALLOCATOR_NAME) != 0)) {
-    /* if pool has already an allocator, try it and ignore the one in
-     * configuration */
-    if (priv->allocator)
-      allocator = priv->allocator;
-    else
-      allocator = NULL;
-  }
+              GST_VAAPI_DMABUF_ALLOCATOR_NAME) != 0))
+    goto error_invalid_allocator;
 
   /* get the allocator properties */
   if (allocator) {