diff --git a/gst/gstallocator.c b/gst/gstallocator.c
index df983d922e..367fc2a93f 100644
--- a/gst/gstallocator.c
+++ b/gst/gstallocator.c
@@ -210,6 +210,8 @@ gst_allocator_register (const gchar * name, GstAllocator * allocator)
       allocator, name);
 
   g_rw_lock_writer_lock (&lock);
+  /* The ref will never be released */
+  GST_OBJECT_FLAG_SET (allocator, GST_OBJECT_FLAG_MAY_BE_LEAKED);
   g_hash_table_insert (allocators, (gpointer) name, (gpointer) allocator);
   g_rw_lock_writer_unlock (&lock);
 }