diff --git a/subprojects/gst-plugins-base/sys/xvimage/xvimageallocator.c b/subprojects/gst-plugins-base/sys/xvimage/xvimageallocator.c
index bf1f2bd9e4..a6d84325b8 100644
--- a/subprojects/gst-plugins-base/sys/xvimage/xvimageallocator.c
+++ b/subprojects/gst-plugins-base/sys/xvimage/xvimageallocator.c
@@ -383,6 +383,13 @@ gst_xvimage_allocator_alloc (GstXvImageAllocator * allocator, gint im_format,
         context->xv_port_id, im_format, NULL, padded_width, padded_height,
         &mem->SHMInfo);
     if (!mem->xvimage || error_caught) {
+
+      /* If the memory was allocated, drop it */
+      if (mem->xvimage) {
+        XFree (mem->xvimage);
+        mem->xvimage = NULL;
+      }
+
       g_mutex_unlock (&context->lock);
 
       /* Reset error flag */