diff --git a/gst-libs/gst/vaapi/gstvaapidisplay.c b/gst-libs/gst/vaapi/gstvaapidisplay.c
index a733423dbe..32738a116e 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay.c
+++ b/gst-libs/gst/vaapi/gstvaapidisplay.c
@@ -32,6 +32,7 @@
 #include "gstvaapiutils.h"
 #include "gstvaapivalue.h"
 #include "gstvaapidisplay.h"
+#include "gstvaapitexturemap.h"
 #include "gstvaapidisplay_priv.h"
 #include "gstvaapiworkarounds.h"
 #include "gstvaapiversion.h"
@@ -2149,3 +2150,28 @@ gst_vaapi_display_has_opengl (GstVaapiDisplay * display)
   return (klass->display_type == GST_VAAPI_DISPLAY_TYPE_GLX ||
       klass->display_type == GST_VAAPI_DISPLAY_TYPE_EGL);
 }
+
+/**
+ * gst_vaapi_display_reset_texture_map:
+ * @display: a #GstVaapiDisplay
+ *
+ * Reset the internal #GstVaapiTextureMap if available.
+ *
+ * This function is thread safe.
+ */
+void
+gst_vaapi_display_reset_texture_map (GstVaapiDisplay * display)
+{
+  GstVaapiDisplayClass *klass;
+  GstVaapiTextureMap *map;
+
+  g_return_if_fail (display != NULL);
+
+  if (!gst_vaapi_display_has_opengl (display))
+    return;
+  klass = GST_VAAPI_DISPLAY_GET_CLASS (display);
+  if (!klass->get_texture_map)
+    return;
+  if ((map = klass->get_texture_map (display)))
+    gst_vaapi_texture_map_reset (map);
+}
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay.h b/gst-libs/gst/vaapi/gstvaapidisplay.h
index f4cc8653d9..600f276eca 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay.h
+++ b/gst-libs/gst/vaapi/gstvaapidisplay.h
@@ -257,6 +257,9 @@ gst_vaapi_display_get_vendor_string (GstVaapiDisplay * display);
 gboolean
 gst_vaapi_display_has_opengl (GstVaapiDisplay * display);
 
+void
+gst_vaapi_display_reset_texture_map (GstVaapiDisplay * display);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_DISPLAY_H */
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_egl.c b/gst-libs/gst/vaapi/gstvaapidisplay_egl.c
index feab6408fb..f049de98ca 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_egl.c
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_egl.c
@@ -232,14 +232,49 @@ gst_vaapi_display_egl_create_window (GstVaapiDisplay * display, GstVaapiID id,
   return gst_vaapi_window_egl_new (display, width, height);
 }
 
+static void
+ensure_texture_map (GstVaapiDisplayEGL * display)
+{
+  if (!display->texture_map)
+    display->texture_map = gst_vaapi_texture_map_new ();
+}
+
 static GstVaapiTexture *
 gst_vaapi_display_egl_create_texture (GstVaapiDisplay * display, GstVaapiID id,
     guint target, guint format, guint width, guint height)
 {
-  if (id != GST_VAAPI_ID_INVALID)
-    return gst_vaapi_texture_egl_new_wrapped (display, id, target, format,
-        width, height);
-  return gst_vaapi_texture_egl_new (display, target, format, width, height);
+  GstVaapiDisplayEGL *dpy = GST_VAAPI_DISPLAY_EGL (display);
+  GstVaapiTexture *texture;
+
+  if (id == GST_VAAPI_ID_INVALID)
+    return gst_vaapi_texture_egl_new (display, target, format, width, height);
+
+  ensure_texture_map (dpy);
+  if (!(texture = gst_vaapi_texture_map_lookup (dpy->texture_map, id))) {
+    if ((texture =
+            gst_vaapi_texture_egl_new_wrapped (display, id, target, format,
+                width, height))) {
+      gst_vaapi_texture_map_add (dpy->texture_map, texture, id);
+    }
+  }
+
+  return texture;
+}
+
+static GstVaapiTextureMap *
+gst_vaapi_display_egl_get_texture_map (GstVaapiDisplay * display)
+{
+  return GST_VAAPI_DISPLAY_EGL (display)->texture_map;
+}
+
+static void
+gst_vaapi_display_egl_finalize (GstVaapiDisplay * display)
+{
+  GstVaapiDisplayEGL *dpy = GST_VAAPI_DISPLAY_EGL (display);
+
+  if (dpy->texture_map)
+    gst_object_unref (dpy->texture_map);
+  GST_VAAPI_DISPLAY_EGL_GET_CLASS (display)->parent_finalize (display);
 }
 
 static void
@@ -254,6 +289,10 @@ gst_vaapi_display_egl_class_init (GstVaapiDisplayEGLClass * klass)
 
   gst_vaapi_display_class_init (dpy_class);
 
+  /* chain up destructor */
+  klass->parent_finalize = object_class->finalize;
+  object_class->finalize = (GDestroyNotify) gst_vaapi_display_egl_finalize;
+
   object_class->size = sizeof (GstVaapiDisplayEGL);
   dpy_class->display_type = GST_VAAPI_DISPLAY_TYPE_EGL;
   dpy_class->bind_display = (GstVaapiDisplayBindFunc)
@@ -280,6 +319,7 @@ gst_vaapi_display_egl_class_init (GstVaapiDisplayEGLClass * klass)
       gst_vaapi_display_egl_create_window;
   dpy_class->create_texture = (GstVaapiDisplayCreateTextureFunc)
       gst_vaapi_display_egl_create_texture;
+  dpy_class->get_texture_map = gst_vaapi_display_egl_get_texture_map;
 }
 
 static inline const GstVaapiDisplayClass *
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_egl_priv.h b/gst-libs/gst/vaapi/gstvaapidisplay_egl_priv.h
index 3db5ad8980..96e2b43b85 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_egl_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_egl_priv.h
@@ -24,6 +24,7 @@
 #define GST_VAAPI_DISPLAY_EGL_PRIV_H
 
 #include <gst/vaapi/gstvaapiwindow.h>
+#include <gst/vaapi/gstvaapitexturemap.h>
 #include "gstvaapidisplay_egl.h"
 #include "gstvaapidisplay_priv.h"
 #include "gstvaapiutils_egl.h"
@@ -79,6 +80,7 @@ struct _GstVaapiDisplayEGL
   EglDisplay *egl_display;
   EglContext *egl_context;
   guint gles_version;
+  GstVaapiTextureMap *texture_map;
 };
 
 /**
@@ -90,6 +92,7 @@ struct _GstVaapiDisplayEGLClass
 {
   /*< private >*/
   GstVaapiDisplayClass parent_class;
+  GDestroyNotify parent_finalize;
 };
 
 G_GNUC_INTERNAL
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_glx.c b/gst-libs/gst/vaapi/gstvaapidisplay_glx.c
index 32aa714113..9b4a007cbc 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_glx.c
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_glx.c
@@ -50,13 +50,48 @@ gst_vaapi_display_glx_create_window (GstVaapiDisplay * display, GstVaapiID id,
       gst_vaapi_window_glx_new (display, width, height);
 }
 
+static void
+ensure_texture_map (GstVaapiDisplayGLX * display)
+{
+  if (!display->texture_map)
+    display->texture_map = gst_vaapi_texture_map_new ();
+}
+
 static GstVaapiTexture *
 gst_vaapi_display_glx_create_texture (GstVaapiDisplay * display, GstVaapiID id,
     guint target, guint format, guint width, guint height)
 {
-  return id != GST_VAAPI_ID_INVALID ?
-      gst_vaapi_texture_glx_new_wrapped (display, id, target, format) :
-      gst_vaapi_texture_glx_new (display, target, format, width, height);
+  GstVaapiTexture *texture;
+  GstVaapiDisplayGLX *dpy = GST_VAAPI_DISPLAY_GLX (display);
+
+  if (id == GST_VAAPI_ID_INVALID)
+    return gst_vaapi_texture_glx_new (display, target, format, width, height);
+
+  ensure_texture_map (dpy);
+  if (!(texture = gst_vaapi_texture_map_lookup (dpy->texture_map, id))) {
+    if ((texture =
+            gst_vaapi_texture_glx_new_wrapped (display, id, target, format))) {
+      gst_vaapi_texture_map_add (dpy->texture_map, texture, id);
+    }
+  }
+
+  return texture;
+}
+
+static GstVaapiTextureMap *
+gst_vaapi_display_glx_get_texture_map (GstVaapiDisplay * display)
+{
+  return GST_VAAPI_DISPLAY_GLX (display)->texture_map;
+}
+
+static void
+gst_vaapi_display_glx_finalize (GstVaapiDisplay * display)
+{
+  GstVaapiDisplayGLX *dpy = GST_VAAPI_DISPLAY_GLX (display);
+
+  if (dpy->texture_map)
+    gst_object_unref (dpy->texture_map);
+  GST_VAAPI_DISPLAY_GLX_GET_CLASS (display)->parent_finalize (display);
 }
 
 static void
@@ -68,10 +103,15 @@ gst_vaapi_display_glx_class_init (GstVaapiDisplayGLXClass * klass)
 
   gst_vaapi_display_x11_class_init (&klass->parent_class);
 
+  /* chain up destructor */
+  klass->parent_finalize = object_class->finalize;
+  object_class->finalize = (GDestroyNotify) gst_vaapi_display_glx_finalize;
+
   object_class->size = sizeof (GstVaapiDisplayGLX);
   dpy_class->display_type = GST_VAAPI_DISPLAY_TYPE_GLX;
   dpy_class->create_window = gst_vaapi_display_glx_create_window;
   dpy_class->create_texture = gst_vaapi_display_glx_create_texture;
+  dpy_class->get_texture_map = gst_vaapi_display_glx_get_texture_map;
 }
 
 static inline const GstVaapiDisplayClass *
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_glx_priv.h b/gst-libs/gst/vaapi/gstvaapidisplay_glx_priv.h
index 63f7175ce7..4da88228a6 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_glx_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_glx_priv.h
@@ -25,6 +25,7 @@
 
 #include <gst/vaapi/gstvaapiutils_glx.h>
 #include <gst/vaapi/gstvaapidisplay_glx.h>
+#include <gst/vaapi/gstvaapitexturemap.h>
 #include "gstvaapidisplay_x11_priv.h"
 
 G_BEGIN_DECLS
@@ -53,6 +54,7 @@ struct _GstVaapiDisplayGLX
 {
   /*< private >*/
   GstVaapiDisplayX11 parent_instance;
+  GstVaapiTextureMap *texture_map;
 };
 
 /**
@@ -64,6 +66,7 @@ struct _GstVaapiDisplayGLXClass
 {
   /*< private >*/
   GstVaapiDisplayX11Class parent_class;
+  GDestroyNotify parent_finalize;
 };
 
 G_END_DECLS
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_priv.h b/gst-libs/gst/vaapi/gstvaapidisplay_priv.h
index fd50fe79e2..12b5184bfe 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_priv.h
@@ -29,6 +29,7 @@
 #include <gst/vaapi/gstvaapidisplaycache.h>
 #include <gst/vaapi/gstvaapiwindow.h>
 #include <gst/vaapi/gstvaapitexture.h>
+#include <gst/vaapi/gstvaapitexturemap.h>
 #include "gstvaapiminiobject.h"
 
 G_BEGIN_DECLS
@@ -73,6 +74,8 @@ typedef GstVaapiWindow *(*GstVaapiDisplayCreateWindowFunc) (
 typedef GstVaapiTexture *(*GstVaapiDisplayCreateTextureFunc) (
     GstVaapiDisplay * display, GstVaapiID id, guint target, guint format,
     guint width, guint height);
+typedef GstVaapiTextureMap *(*GstVaapiDisplayGetTextureMapFunc) (
+    GstVaapiDisplay * display);
 
 typedef guintptr (*GstVaapiDisplayGetVisualIdFunc) (GstVaapiDisplay * display,
     GstVaapiWindow * window);
@@ -226,6 +229,7 @@ struct _GstVaapiDisplayClass
   GstVaapiDisplayGetColormapFunc get_colormap;
   GstVaapiDisplayCreateWindowFunc create_window;
   GstVaapiDisplayCreateTextureFunc create_texture;
+  GstVaapiDisplayGetTextureMapFunc get_texture_map;
 };
 
 /* Initialization types */
diff --git a/gst/vaapi/gstvaapivideometa_texture.c b/gst/vaapi/gstvaapivideometa_texture.c
index 01e33852fd..fb1535a760 100644
--- a/gst/vaapi/gstvaapivideometa_texture.c
+++ b/gst/vaapi/gstvaapivideometa_texture.c
@@ -176,26 +176,34 @@ gst_vaapi_texture_upload (GstVideoGLTextureUploadMeta * meta,
       gst_vaapi_video_meta_get_surface_proxy (vmeta);
   GstVaapiSurface *const surface = gst_vaapi_surface_proxy_get_surface (proxy);
   GstVaapiDisplay *const dpy = GST_VAAPI_OBJECT_DISPLAY (surface);
+  GstVaapiTexture *texture = NULL;
 
   if (!gst_vaapi_display_has_opengl (dpy))
     return FALSE;
 
-  if (!meta_texture->texture ||
+  if (meta_texture->texture
       /* Check whether VA display changed */
-      GST_VAAPI_OBJECT_DISPLAY (meta_texture->texture) != dpy ||
+      && GST_VAAPI_OBJECT_DISPLAY (meta_texture->texture) == dpy
       /* Check whether texture id changed */
-      gst_vaapi_texture_get_id (meta_texture->texture) != texture_id[0]) {
+      && (gst_vaapi_texture_get_id (meta_texture->texture) == texture_id[0])) {
+    texture = meta_texture->texture;
+  }
+
+  if (!texture) {
     /* FIXME: should we assume target? */
-    GstVaapiTexture *const texture =
+    texture =
         gst_vaapi_texture_new_wrapped (dpy, texture_id[0],
         GL_TEXTURE_2D, meta_texture->gl_format, meta_texture->width,
         meta_texture->height);
-    gst_vaapi_texture_replace (&meta_texture->texture, texture);
-    if (!texture)
-      return FALSE;
-    gst_vaapi_texture_unref (texture);
   }
 
+  if (meta_texture->texture != texture) {
+    gst_vaapi_texture_replace (&meta_texture->texture, texture);
+  }
+
+  if (!texture)
+    return FALSE;
+
   gst_vaapi_texture_set_orientation_flags (meta_texture->texture,
       get_texture_orientation_flags (meta->texture_orientation));