diff --git a/gst/vaapi/gstvaapipluginbase.c b/gst/vaapi/gstvaapipluginbase.c
index c5689e3637..ad254f2bd1 100644
--- a/gst/vaapi/gstvaapipluginbase.c
+++ b/gst/vaapi/gstvaapipluginbase.c
@@ -389,6 +389,7 @@ gst_vaapi_plugin_base_set_caps (GstVaapiPluginBase * plugin, GstCaps * incaps,
     if (!gst_video_info_from_caps (&plugin->sinkpad_info, incaps))
       return FALSE;
     plugin->sinkpad_caps_changed = TRUE;
+    plugin->sinkpad_caps_is_raw = !gst_caps_has_vaapi_surface (incaps);
   }
 
   if (outcaps && outcaps != plugin->srcpad_caps) {
@@ -398,7 +399,7 @@ gst_vaapi_plugin_base_set_caps (GstVaapiPluginBase * plugin, GstCaps * incaps,
     plugin->srcpad_caps_changed = TRUE;
   }
 
-  if (plugin->uploader && GST_VIDEO_INFO_IS_YUV (&plugin->sinkpad_info)) {
+  if (plugin->uploader && plugin->sinkpad_caps_is_raw) {
     if (!gst_vaapi_uploader_ensure_display (plugin->uploader, plugin->display))
       return FALSE;
     if (!gst_vaapi_uploader_ensure_caps (plugin->uploader,
@@ -477,7 +478,7 @@ gst_vaapi_plugin_base_allocate_input_buffer (GstVaapiPluginBase * plugin,
     plugin->sinkpad_caps_changed = TRUE;
   }
 
-  if (!GST_VIDEO_INFO_IS_YUV (&plugin->sinkpad_info))
+  if (!plugin->sinkpad_caps_is_raw)
     return GST_FLOW_OK;
 
   if (!gst_vaapi_uploader_ensure_display (plugin->uploader, plugin->display))
@@ -529,7 +530,7 @@ gst_vaapi_plugin_base_get_input_buffer (GstVaapiPluginBase * plugin,
     return GST_FLOW_OK;
   }
 
-  if (!GST_VIDEO_INFO_IS_YUV (&plugin->sinkpad_info))
+  if (!plugin->sinkpad_caps_is_raw)
     goto error_invalid_buffer;
 
   if (!plugin->sinkpad_buffer_pool)
@@ -586,7 +587,7 @@ error_map_src_buffer:
 #else
   if (meta)
     outbuf = gst_buffer_ref (inbuf);
-  else if (GST_VIDEO_INFO_IS_YUV (&plugin->sinkpad_info)) {
+  else if (plugin->sinkpad_caps_is_raw) {
     outbuf = gst_vaapi_uploader_get_buffer (plugin->uploader);
     if (!outbuf)
       goto error_create_buffer;
@@ -595,7 +596,7 @@ error_map_src_buffer:
   } else
     goto error_invalid_buffer;
 
-  if (GST_VIDEO_INFO_IS_YUV (&plugin->sinkpad_info) &&
+  if (plugin->sinkpad_caps_is_raw &&
       !gst_vaapi_uploader_process (plugin->uploader, inbuf, outbuf))
     goto error_copy_buffer;
 
diff --git a/gst/vaapi/gstvaapipluginbase.h b/gst/vaapi/gstvaapipluginbase.h
index f0ee4295c9..76413b3d18 100644
--- a/gst/vaapi/gstvaapipluginbase.h
+++ b/gst/vaapi/gstvaapipluginbase.h
@@ -123,6 +123,7 @@ struct _GstVaapiPluginBase
   GstPad *sinkpad;
   GstCaps *sinkpad_caps;
   gboolean sinkpad_caps_changed;
+  gboolean sinkpad_caps_is_raw;
   GstVideoInfo sinkpad_info;
   GstPadQueryFunction sinkpad_query;
 #if GST_CHECK_VERSION(1,0,0)