diff --git a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvert.c b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvert.c
index 8424bbfe2b..ff8b4f10de 100644
--- a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvert.c
+++ b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvert.c
@@ -49,7 +49,7 @@ GST_ELEMENT_REGISTER_DEFINE (videoconvert, "videoconvert",
 static void
 gst_video_convert_class_init (GstVideoConvertClass * klass)
 {
-
+  ((GstVideoConvertScaleClass *) klass)->any_memory = TRUE;
 }
 
 static void
diff --git a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.c b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.c
index df168caf82..a5d9b52de4 100644
--- a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.c
+++ b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.c
@@ -617,6 +617,7 @@ static GstCaps *
 gst_video_convert_scale_transform_caps (GstBaseTransform * trans,
     GstPadDirection direction, GstCaps * caps, GstCaps * filter)
 {
+  gint i;
   GstCaps *ret;
 
   GST_DEBUG_OBJECT (trans,
@@ -633,6 +634,28 @@ gst_video_convert_scale_transform_caps (GstBaseTransform * trans,
     ret = intersection;
   }
 
+  if (GST_VIDEO_CONVERT_SCALE_GET_CLASS (trans)->any_memory)
+    return ret;
+
+  for (i = 0; i < gst_caps_get_size (ret); i++) {
+    gint j;
+    GstCapsFeatures *f = gst_caps_get_features (ret, i);
+
+    if (!f || gst_caps_features_is_any (f) ||
+        gst_caps_features_is_equal (f, GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY))
+      continue;
+
+    for (j = 0; j < gst_caps_features_get_size (f); j++) {
+      const gchar *feature = gst_caps_features_get_nth (f, j);
+
+      if (g_str_has_prefix (feature, "memory:")) {
+        GST_DEBUG_OBJECT (trans, "Can not work with memory `%s`", feature);
+        gst_caps_remove_structure (ret, i);
+        break;
+      }
+    }
+  }
+
   GST_DEBUG_OBJECT (trans, "returning caps: %" GST_PTR_FORMAT, ret);
 
   return ret;
diff --git a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.h b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.h
index 9d25e039b6..64be8b1eaa 100644
--- a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.h
+++ b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.h
@@ -36,6 +36,8 @@ G_DECLARE_DERIVABLE_TYPE (GstVideoConvertScale, gst_video_convert_scale, GST, VI
 struct _GstVideoConvertScaleClass
 {
   GstVideoFilterClass parent;
+
+  gboolean any_memory;
 };
 
 /**
diff --git a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoscale.c b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoscale.c
index b246bf97b1..e8e41f152c 100644
--- a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoscale.c
+++ b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoscale.c
@@ -112,6 +112,8 @@ gst_video_scale_class_init (GstVideoScaleClass * klass)
   gobject_class->set_property = gst_video_scale_set_property;
   gobject_class->get_property = gst_video_scale_get_property;
 
+  ((GstVideoConvertScaleClass *) klass)->any_memory = TRUE;
+
   g_object_class_install_property (gobject_class, PROP_GAMMA_DECODE,
       g_param_spec_boolean ("gamma-decode", "Gamma Decode",
           "Decode gamma before scaling", DEFAULT_PROP_GAMMA_DECODE,