diff --git a/gst-libs/gst/gl/gstglutils.c b/gst-libs/gst/gl/gstglutils.c
index 2d90714b53..2e1b0dd4cf 100644
--- a/gst-libs/gst/gl/gstglutils.c
+++ b/gst-libs/gst/gl/gstglutils.c
@@ -767,8 +767,11 @@ gst_gl_handle_set_context (GstElement * element, GstContext * context,
   }
 #endif
 
-  if (replacement)
-    gst_object_replace ((GstObject **) display, (GstObject *) replacement);
+  if (replacement) {
+    GstGLDisplay *old = *display;
+    *display = replacement;
+    gst_object_unref (old);
+  }
 
   return TRUE;
 }