diff --git a/gst/gstcontroller.c b/gst/gstcontroller.c
index c58a1ff956..c9f033cdb1 100644
--- a/gst/gstcontroller.c
+++ b/gst/gstcontroller.c
@@ -279,8 +279,12 @@ gst_controller_new_valist (GstObject * object, va_list var_args)
 
   g_return_val_if_fail (G_IS_OBJECT (object), NULL);
 
+  /* FIXME: storing the controller into the object is ugly
+   * we'd like to make the controller object completely internal
+   */
   self = g_object_newv (GST_TYPE_CONTROLLER, 0, NULL);
   self->object = g_object_ref (object);
+  object->ctrl = g_object_ref (self);
 
   /* create GstControlledProperty for each property */
   while ((name = va_arg (var_args, gchar *))) {
@@ -314,6 +318,7 @@ gst_controller_new_list (GstObject * object, GList * list)
 
   self = g_object_newv (GST_TYPE_CONTROLLER, 0, NULL);
   self->object = g_object_ref (object);
+  object->ctrl = g_object_ref (self);
 
   /* create GstControlledProperty for each property */
   for (node = list; node; node = g_list_next (node)) {
diff --git a/gst/gstobject.c b/gst/gstobject.c
index 5a00ee031f..47cd08b56f 100644
--- a/gst/gstobject.c
+++ b/gst/gstobject.c
@@ -982,7 +982,12 @@ gst_object_control_properties (GstObject * object, ...)
 
   va_start (var_args, object);
   if (object->ctrl) {
-    object->ctrl = gst_controller_new_valist (object, var_args);
+    GstController *ctrl = gst_controller_new_valist (object, var_args);
+
+    /* FIXME: see gst_controller_new_*() */
+    g_object_unref (object->ctrl);
+    object->ctrl = ctrl;
+
     res = (object->ctrl != NULL);
   } else {
     res = gst_controller_add_properties_valist ((GstController *) object->ctrl,