diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c
index 79be36660c..b6d917aa7f 100644
--- a/gst/vaapi/gstvaapidecode.c
+++ b/gst/vaapi/gstvaapidecode.c
@@ -238,8 +238,9 @@ gst_vaapidecode_ensure_allowed_srcpad_caps (GstVaapiDecode * decode)
   out_caps = gst_caps_make_writable (out_caps);
   gst_caps_append (out_caps, gst_caps_from_string (GST_VAAPI_MAKE_DMABUF_CAPS));
 
-  raw_caps = gst_vaapi_plugin_base_get_allowed_raw_caps
-      (GST_VAAPI_PLUGIN_BASE (decode));
+  raw_caps = gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps
+      (GST_VAAPI_PLUGIN_BASE (decode),
+      GST_VIDEO_INFO_FORMAT (&decode->decoded_info));
   if (!raw_caps) {
     gst_caps_unref (out_caps);
     GST_WARNING_OBJECT (decode, "failed to create raw sink caps");
diff --git a/gst/vaapi/gstvaapipluginbase.c b/gst/vaapi/gstvaapipluginbase.c
index 6192fefdfa..c98fe05cd5 100644
--- a/gst/vaapi/gstvaapipluginbase.c
+++ b/gst/vaapi/gstvaapipluginbase.c
@@ -1254,43 +1254,70 @@ no_valid_gl_display:
 }
 
 static GArray *
-extract_allowed_surface_formats (GstVaapiDisplay * display, GArray * formats)
+extract_allowed_surface_formats (GstVaapiDisplay * display,
+    GArray * img_formats, GstVideoFormat specified_format,
+    GstPadDirection direction)
 {
   guint i;
   GArray *out_formats;
+  GstVaapiSurface *surface = NULL;
+
+  g_assert (direction == GST_PAD_SRC || direction == GST_PAD_SINK);
 
   out_formats =
-      g_array_sized_new (FALSE, FALSE, sizeof (GstVideoFormat), formats->len);
+      g_array_sized_new (FALSE, FALSE, sizeof (GstVideoFormat),
+      img_formats->len);
   if (!out_formats)
     return NULL;
 
-  for (i = 0; i < formats->len; i++) {
-    const GstVideoFormat format = g_array_index (formats, GstVideoFormat, i);
-    GstVaapiSurface *surface;
+  for (i = 0; i < img_formats->len; i++) {
+    const GstVideoFormat img_format =
+        g_array_index (img_formats, GstVideoFormat, i);
     GstVaapiImage *image;
     GstVideoInfo vi;
+    GstVideoFormat surface_format;
+    gboolean res;
 
-    if (format == GST_VIDEO_FORMAT_UNKNOWN)
+    if (img_format == GST_VIDEO_FORMAT_UNKNOWN)
       continue;
 
-    gst_video_info_set_format (&vi, format, 64, 64);
-    surface = gst_vaapi_surface_new_full (display, &vi, 0);
-    if (!surface)
-      continue;
+    surface_format =
+        (specified_format == GST_VIDEO_FORMAT_UNKNOWN) ?
+        img_format : specified_format;
+    if (!surface) {
+      gst_video_info_set_format (&vi, surface_format, 64, 64);
+      surface = gst_vaapi_surface_new_full (display, &vi, 0);
+      if (!surface)
+        continue;
+    }
 
-    image = gst_vaapi_image_new (display, format, 64, 64);
+    image = gst_vaapi_image_new (display, img_format, 64, 64);
     if (!image) {
-      gst_vaapi_object_unref (surface);
+      /* Just reuse the surface if the format is specified */
+      if (specified_format == GST_VIDEO_FORMAT_UNKNOWN)
+        gst_vaapi_object_replace (&surface, NULL);
+
       continue;
     }
 
-    if (gst_vaapi_surface_put_image (surface, image))
-      g_array_append_val (out_formats, format);
+    res = FALSE;
+    if (direction == GST_PAD_SRC) {
+      res = gst_vaapi_surface_get_image (surface, image);
+    } else {
+      res = gst_vaapi_surface_put_image (surface, image);
+    }
+    if (res)
+      g_array_append_val (out_formats, img_format);
 
     gst_vaapi_object_unref (image);
-    gst_vaapi_object_unref (surface);
+    /* Just reuse the surface if the format is specified */
+    if (specified_format == GST_VIDEO_FORMAT_UNKNOWN)
+      gst_vaapi_object_replace (&surface, NULL);
   }
 
+  if (surface)
+    gst_vaapi_object_unref (surface);
+
   if (out_formats->len == 0) {
     g_array_unref (out_formats);
     return NULL;
@@ -1299,7 +1326,8 @@ extract_allowed_surface_formats (GstVaapiDisplay * display, GArray * formats)
 }
 
 static gboolean
-ensure_allowed_raw_caps (GstVaapiPluginBase * plugin)
+ensure_allowed_raw_caps (GstVaapiPluginBase * plugin, GstVideoFormat format,
+    GstPadDirection direction)
 {
   GArray *formats, *out_formats;
   GstVaapiDisplay *display;
@@ -1314,7 +1342,8 @@ ensure_allowed_raw_caps (GstVaapiPluginBase * plugin)
   formats = gst_vaapi_display_get_image_formats (display);
   if (!formats)
     goto bail;
-  out_formats = extract_allowed_surface_formats (display, formats);
+  out_formats =
+      extract_allowed_surface_formats (display, formats, format, direction);
   if (!out_formats)
     goto bail;
   out_caps = gst_vaapi_video_format_new_template_caps_from_list (out_formats);
@@ -1336,7 +1365,7 @@ bail:
 }
 
 /**
- * gst_vaapi_plugin_base_get_allowed_raw_caps:
+ * gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps:
  * @plugin: a #GstVaapiPluginBase
  *
  * Returns the raw #GstCaps allowed by the element.
@@ -1344,9 +1373,27 @@ bail:
  * Returns: the allowed raw #GstCaps or %NULL
  **/
 GstCaps *
-gst_vaapi_plugin_base_get_allowed_raw_caps (GstVaapiPluginBase * plugin)
+gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps (GstVaapiPluginBase * plugin)
 {
-  if (!ensure_allowed_raw_caps (plugin))
+  if (!ensure_allowed_raw_caps (plugin, GST_VIDEO_FORMAT_UNKNOWN, GST_PAD_SINK))
+    return NULL;
+  return plugin->allowed_raw_caps;
+}
+
+/**
+ * gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps:
+ * @plugin: a #GstVaapiPluginBase
+ * @format: a #GstVideoFormat, the format we need to check
+ *
+ * Returns the raw #GstCaps allowed by the element.
+ *
+ * Returns: the allowed raw #GstCaps or %NULL
+ **/
+GstCaps *
+gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps (GstVaapiPluginBase *
+    plugin, GstVideoFormat format)
+{
+  if (!ensure_allowed_raw_caps (plugin, format, GST_PAD_SRC))
     return NULL;
   return plugin->allowed_raw_caps;
 }
diff --git a/gst/vaapi/gstvaapipluginbase.h b/gst/vaapi/gstvaapipluginbase.h
index c0d07496aa..9faa063952 100644
--- a/gst/vaapi/gstvaapipluginbase.h
+++ b/gst/vaapi/gstvaapipluginbase.h
@@ -251,7 +251,12 @@ gst_vaapi_plugin_base_create_gl_context (GstVaapiPluginBase * plugin);
 
 G_GNUC_INTERNAL
 GstCaps *
-gst_vaapi_plugin_base_get_allowed_raw_caps (GstVaapiPluginBase * plugin);
+gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps (GstVaapiPluginBase * plugin);
+
+G_GNUC_INTERNAL
+GstCaps *
+gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps (
+    GstVaapiPluginBase * plugin, GstVideoFormat format);
 
 G_GNUC_INTERNAL
 void
diff --git a/gst/vaapi/gstvaapipostproc.c b/gst/vaapi/gstvaapipostproc.c
index a2e75e8487..24d93f69e2 100644
--- a/gst/vaapi/gstvaapipostproc.c
+++ b/gst/vaapi/gstvaapipostproc.c
@@ -1094,7 +1094,7 @@ ensure_allowed_sinkpad_caps (GstVaapiPostproc * postproc)
     return FALSE;
   }
 
-  raw_caps = gst_vaapi_plugin_base_get_allowed_raw_caps
+  raw_caps = gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps
       (GST_VAAPI_PLUGIN_BASE (postproc));
   if (!raw_caps) {
     gst_caps_unref (out_caps);
diff --git a/gst/vaapi/gstvaapisink.c b/gst/vaapi/gstvaapisink.c
index 9333f43824..c19658332e 100644
--- a/gst/vaapi/gstvaapisink.c
+++ b/gst/vaapi/gstvaapisink.c
@@ -1215,7 +1215,7 @@ gst_vaapisink_start (GstBaseSink * base_sink)
 
   /* Ensures possible raw caps earlier to avoid race conditions at
    * get_caps() */
-  if (!gst_vaapi_plugin_base_get_allowed_raw_caps (plugin))
+  if (!gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps (plugin))
     return FALSE;
 
   return TRUE;
@@ -1251,7 +1251,8 @@ gst_vaapisink_get_caps_impl (GstBaseSink * base_sink)
 
   out_caps = gst_caps_from_string (surface_caps_str);
   raw_caps =
-      gst_vaapi_plugin_base_get_allowed_raw_caps (GST_VAAPI_PLUGIN_BASE (sink));
+      gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps (GST_VAAPI_PLUGIN_BASE
+      (sink));
   if (!raw_caps)
     return out_caps;