diff --git a/gst-libs/gst/vaapi/gstvaapisurface.c b/gst-libs/gst/vaapi/gstvaapisurface.c
index 18fd414a35..57eab42f8b 100644
--- a/gst-libs/gst/vaapi/gstvaapisurface.c
+++ b/gst-libs/gst/vaapi/gstvaapisurface.c
@@ -648,15 +648,19 @@ gst_vaapi_surface_associate_subpicture(
     g_return_val_if_fail(GST_VAAPI_IS_SURFACE(surface), FALSE);
     g_return_val_if_fail(GST_VAAPI_IS_SUBPICTURE(subpicture), FALSE);
 
-    if (!gst_vaapi_surface_deassociate_subpicture(surface, subpicture))
-        return FALSE;
-
     if (!surface->priv->subpictures) {
         surface->priv->subpictures = g_ptr_array_new();
         if (!surface->priv->subpictures)
             return FALSE;
     }
 
+    if (g_ptr_array_remove_fast(surface->priv->subpictures, subpicture)) {
+        success = _gst_vaapi_surface_deassociate_subpicture(surface, subpicture);
+        g_object_unref(subpicture);
+        if (!success)
+            return FALSE;
+    }
+
     success = _gst_vaapi_surface_associate_subpicture(
         surface,
         subpicture,