diff --git a/gst-libs/gst/vaapi/gstvaapidisplay.c b/gst-libs/gst/vaapi/gstvaapidisplay.c
index 39d7540d14..fb65e55987 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay.c
+++ b/gst-libs/gst/vaapi/gstvaapidisplay.c
@@ -827,7 +827,8 @@ gst_vaapi_display_destroy (GstVaapiDisplay * display)
   }
 
   if (priv->display) {
-    vaTerminate (priv->display);
+    if (!priv->parent)
+      vaTerminate (priv->display);
     priv->display = NULL;
   }
 
@@ -842,6 +843,8 @@ gst_vaapi_display_destroy (GstVaapiDisplay * display)
 
   g_free (priv->vendor_string);
   priv->vendor_string = NULL;
+
+  gst_vaapi_display_replace (&priv->parent, NULL);
 }
 
 static gboolean
@@ -888,8 +891,10 @@ gst_vaapi_display_create_unlocked (GstVaapiDisplay * display,
   if (!priv->display)
     return FALSE;
 
-  if (!vaapi_initialize (priv->display))
-    return FALSE;
+  if (!priv->parent) {
+    if (!vaapi_initialize (priv->display))
+      return FALSE;
+  }
 
   GST_INFO_OBJECT (display, "new display addr=%p", display);
   g_free (priv->display_name);
@@ -909,6 +914,8 @@ gst_vaapi_display_lock_default (GstVaapiDisplay * display)
 {
   GstVaapiDisplayPrivate *priv = GST_VAAPI_DISPLAY_GET_PRIVATE (display);
 
+  if (priv->parent)
+    priv = GST_VAAPI_DISPLAY_GET_PRIVATE (priv->parent);
   g_rec_mutex_lock (&priv->mutex);
 }
 
@@ -917,6 +924,8 @@ gst_vaapi_display_unlock_default (GstVaapiDisplay * display)
 {
   GstVaapiDisplayPrivate *priv = GST_VAAPI_DISPLAY_GET_PRIVATE (display);
 
+  if (priv->parent)
+    priv = GST_VAAPI_DISPLAY_GET_PRIVATE (priv->parent);
   g_rec_mutex_unlock (&priv->mutex);
 }
 
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_egl.c b/gst-libs/gst/vaapi/gstvaapidisplay_egl.c
index 6e83bd3e37..27d70743cc 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_egl.c
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_egl.c
@@ -104,6 +104,7 @@ gst_vaapi_display_egl_bind_display (GstVaapiDisplay * base_display,
   EGLDisplay *native_egl_display;
   guint gl_platform = EGL_PLATFORM_UNKNOWN;
   const InitParams *params = (InitParams *) native_params;
+  GstVaapiDisplayPrivate *const priv = GST_VAAPI_DISPLAY_GET_PRIVATE (display);
 
   native_vaapi_display = params->display;
   native_egl_display = params->gl_display;
@@ -120,14 +121,14 @@ gst_vaapi_display_egl_bind_display (GstVaapiDisplay * base_display,
       native_vaapi_display = gst_vaapi_display_wayland_new (NULL);
 #endif
   } else {
-    /* thus it could be unrefed */
+    /* thus it could be assigned to parent */
     gst_object_ref (native_vaapi_display);
   }
   if (!native_vaapi_display)
     return FALSE;
 
   gst_vaapi_display_replace (&display->display, native_vaapi_display);
-  gst_object_unref (native_vaapi_display);
+  priv->parent = native_vaapi_display;
 
   switch (GST_VAAPI_DISPLAY_GET_CLASS_TYPE (display->display)) {
     case GST_VAAPI_DISPLAY_TYPE_X11:
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_priv.h b/gst-libs/gst/vaapi/gstvaapidisplay_priv.h
index 0394a351c9..95272a79b0 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_priv.h
@@ -109,6 +109,7 @@ typedef enum _GstVaapiDisplayInitType           GstVaapiDisplayInitType;
 
 struct _GstVaapiDisplayPrivate
 {
+  GstVaapiDisplay *parent;
   GRecMutex mutex;
   gchar *display_name;
   VADisplay display;