diff --git a/gst/schedulers/gstbasicscheduler.c b/gst/schedulers/gstbasicscheduler.c
index 58ca419964..e6359f6b6a 100644
--- a/gst/schedulers/gstbasicscheduler.c
+++ b/gst/schedulers/gstbasicscheduler.c
@@ -1068,10 +1068,9 @@ gst_basic_scheduler_remove_element (GstScheduler * sched, GstElement * element)
     /* remove it from the list of elements */
     bsched->elements = g_list_remove (bsched->elements, element);
     bsched->num_elements--;
-
-    /* unset the scheduler pointer in the element */
-    GST_ELEMENT_SCHED (element) = NULL;
   }
+  /* unset the scheduler pointer in the element */
+  GST_ELEMENT_SCHED (element) = NULL;
 }
 
 static GstElementStateReturn