diff --git a/gst/vaapi/gstvaapidownload.c b/gst/vaapi/gstvaapidownload.c
index 904570bfc5..9a9a3ca5e2 100644
--- a/gst/vaapi/gstvaapidownload.c
+++ b/gst/vaapi/gstvaapidownload.c
@@ -389,10 +389,9 @@ gst_vaapidownload_transform_caps(
 {
     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
     GstPad *srcpad;
-    GstCaps *allowed_caps, *inter_caps, *out_caps = NULL;
+    GstCaps *allowed_caps, *tmp_caps, *out_caps = NULL;
     GstStructure *structure;
     GArray *formats;
-    guint i;
 
     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
 
@@ -411,35 +410,31 @@ gst_vaapidownload_transform_caps(
         if (download->allowed_caps)
             allowed_caps = gst_caps_ref(download->allowed_caps);
         else {
-            allowed_caps = gst_caps_new_empty();
-            if (!allowed_caps)
-                return NULL;
-
             formats = gst_vaapi_display_get_image_formats(
                 GST_VAAPI_PLUGIN_BASE_DISPLAY(download));
-            if (formats) {
-                for (i = 0; i < formats->len; i++) {
-                    const GstVideoFormat format =
-                        g_array_index(formats, GstVideoFormat, i);
-                    gst_caps_merge(allowed_caps,
-                        gst_vaapi_video_format_to_caps(format));
-                }
+            if (G_UNLIKELY(!formats))
+                allowed_caps = gst_caps_new_empty();
+            else {
+                allowed_caps =
+                    gst_vaapi_video_format_new_template_caps_from_list(formats);
                 g_array_unref(formats);
             }
+            if (!allowed_caps)
+                return NULL;
         }
-        inter_caps = gst_caps_intersect(out_caps, allowed_caps);
+        tmp_caps = gst_caps_intersect(out_caps, allowed_caps);
         gst_caps_unref(allowed_caps);
         gst_caps_unref(out_caps);
-        out_caps = inter_caps;
+        out_caps = tmp_caps;
 
         /* Intersect with allowed caps from the peer, if any */
         srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
         allowed_caps = gst_pad_peer_get_caps(srcpad);
         if (allowed_caps) {
-            inter_caps = gst_caps_intersect(out_caps, allowed_caps);
+            tmp_caps = gst_caps_intersect(out_caps, allowed_caps);
             gst_caps_unref(allowed_caps);
             gst_caps_unref(out_caps);
-            out_caps = inter_caps;
+            out_caps = tmp_caps;
         }
     }
     else {
diff --git a/gst/vaapi/gstvaapipluginutil.c b/gst/vaapi/gstvaapipluginutil.c
index c4652b839b..29711687c1 100644
--- a/gst/vaapi/gstvaapipluginutil.c
+++ b/gst/vaapi/gstvaapipluginutil.c
@@ -404,6 +404,77 @@ gst_vaapi_value_set_format_list (GValue * value, GArray * formats)
   return TRUE;
 }
 
+void
+set_video_template_caps (GstCaps * caps)
+{
+  GstStructure *const structure = gst_caps_get_structure (caps, 0);
+
+  gst_structure_set (structure,
+      "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
+      "height", GST_TYPE_INT_RANGE, 1, G_MAXINT,
+      "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1,
+      "pixel-aspect-ratio", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL);
+}
+
+GstCaps *
+gst_vaapi_video_format_new_template_caps (GstVideoFormat format)
+{
+#if GST_CHECK_VERSION(1,0,0)
+  GstCaps *caps;
+
+  g_return_val_if_fail (format != GST_VIDEO_FORMAT_UNKNOWN, NULL);
+
+  caps = gst_caps_new_empty_simple ("video/x-raw");
+  if (!caps)
+    return NULL;
+
+  gst_caps_set_simple (caps,
+      "format", G_TYPE_STRING, gst_video_format_to_string (format), NULL);
+  set_video_template_caps (caps);
+  return caps;
+#else
+  return gst_video_format_new_template_caps (format);
+#endif
+}
+
+GstCaps *
+gst_vaapi_video_format_new_template_caps_from_list (GArray * formats)
+{
+#if GST_CHECK_VERSION(1,0,0)
+  GValue v_formats = G_VALUE_INIT;
+  GstCaps *caps;
+
+  caps = gst_caps_new_empty_simple ("video/x-raw");
+  if (!caps)
+    return NULL;
+
+  if (!gst_vaapi_value_set_format_list (&v_formats, formats)) {
+    gst_caps_unref (caps);
+    return NULL;
+  }
+
+  gst_caps_set_value (caps, "format", &v_formats);
+  set_video_template_caps (caps);
+#else
+  GstCaps *caps, *tmp_caps;
+  guint i;
+
+  g_return_val_if_fail (formats != NULL, NULL);
+
+  caps = gst_caps_new_empty ();
+  if (!caps)
+    return NULL;
+
+  for (i = 0; i < formats->len; i++) {
+    const GstVideoFormat format = g_array_index (formats, GstVideoFormat, i);
+    tmp_caps = gst_vaapi_video_format_new_template_caps (format);
+    if (tmp_caps)
+      gst_caps_append (caps, tmp_caps);
+  }
+#endif
+  return caps;
+}
+
 gboolean
 gst_caps_set_interlaced (GstCaps * caps, GstVideoInfo * vip)
 {
diff --git a/gst/vaapi/gstvaapipluginutil.h b/gst/vaapi/gstvaapipluginutil.h
index 3d04588c5c..0c6e55703c 100644
--- a/gst/vaapi/gstvaapipluginutil.h
+++ b/gst/vaapi/gstvaapipluginutil.h
@@ -64,6 +64,15 @@ G_GNUC_INTERNAL
 gboolean
 gst_vaapi_value_set_format_list (GValue * value, GArray * formats);
 
+/* Helpers to build video caps */
+G_GNUC_INTERNAL
+GstCaps *
+gst_vaapi_video_format_new_template_caps (GstVideoFormat format);
+
+G_GNUC_INTERNAL
+GstCaps *
+gst_vaapi_video_format_new_template_caps_from_list (GArray * formats);
+
 /* Helpers to handle interlaced contents */
 #if GST_CHECK_VERSION(1,0,0)
 # define GST_CAPS_INTERLACED_MODES \
diff --git a/gst/vaapi/gstvaapiuploader.c b/gst/vaapi/gstvaapiuploader.c
index b28cfe8038..6d99c855fd 100644
--- a/gst/vaapi/gstvaapiuploader.c
+++ b/gst/vaapi/gstvaapiuploader.c
@@ -30,6 +30,7 @@
 #include <gst/vaapi/gstvaapisurfacepool.h>
 
 #include "gstvaapiuploader.h"
+#include "gstvaapipluginutil.h"
 #include "gstvaapivideobuffer.h"
 
 #define GST_HELPER_NAME "vaapiupload"
@@ -112,7 +113,7 @@ ensure_allowed_caps(GstVaapiUploader *uploader)
 {
     GstVaapiUploaderPrivate * const priv = uploader->priv;
     GstVaapiSurface *surface = NULL;
-    GArray *image_formats = NULL;
+    GArray *formats = NULL, *out_formats = NULL;
     GstCaps *out_caps;
     guint i;
     gboolean success = FALSE;
@@ -122,22 +123,23 @@ ensure_allowed_caps(GstVaapiUploader *uploader)
     if (priv->allowed_caps)
         return TRUE;
 
-    out_caps = gst_caps_new_empty();
-    if (!out_caps)
-        return FALSE;
+    formats = gst_vaapi_display_get_image_formats(priv->display);
+    if (!formats)
+        goto cleanup;
 
-    image_formats = gst_vaapi_display_get_image_formats(priv->display);
-    if (!image_formats)
-        goto end;
+    out_formats = g_array_sized_new(FALSE, FALSE, sizeof(GstVideoFormat),
+        formats->len);
+    if (!out_formats)
+        goto cleanup;
 
     surface = gst_vaapi_surface_new(priv->display,
         GST_VAAPI_CHROMA_TYPE_YUV420, WIDTH, HEIGHT);
     if (!surface)
-        goto end;
+        goto cleanup;
 
-    for (i = 0; i < image_formats->len; i++) {
+    for (i = 0; i < formats->len; i++) {
         const GstVideoFormat format =
-            g_array_index(image_formats, GstVideoFormat, i);
+            g_array_index(formats, GstVideoFormat, i);
         GstVaapiImage *image;
 
         if (format == GST_VIDEO_FORMAT_UNKNOWN)
@@ -146,17 +148,23 @@ ensure_allowed_caps(GstVaapiUploader *uploader)
         if (!image)
             continue;
         if (ensure_image(image) && gst_vaapi_surface_put_image(surface, image))
-            gst_caps_append(out_caps, gst_vaapi_video_format_to_caps(format));
+            g_array_append_val(out_formats, format);
         gst_vaapi_object_unref(image);
     }
 
+    out_caps = gst_vaapi_video_format_new_template_caps_from_list(out_formats);
+    if (!out_caps)
+        goto cleanup;
+
     gst_caps_replace(&priv->allowed_caps, out_caps);
+    gst_caps_unref(out_caps);
     success = TRUE;
 
-end:
-    gst_caps_unref(out_caps);
-    if (image_formats)
-        g_array_unref(image_formats);
+cleanup:
+    if (out_formats)
+        g_array_unref(out_formats);
+    if (formats)
+        g_array_unref(formats);
     if (surface)
         gst_vaapi_object_unref(surface);
     return success;