diff --git a/plugins/elements/gstinputselector.c b/plugins/elements/gstinputselector.c
index 626fd3c15e..d19b6500b4 100644
--- a/plugins/elements/gstinputselector.c
+++ b/plugins/elements/gstinputselector.c
@@ -364,7 +364,8 @@ gst_selector_pad_new_cached_buffer (GstSelectorPad * selpad, GstBuffer * buffer)
 static void
 gst_selector_pad_free_cached_buffer (GstSelectorPadCachedBuffer * cached_buffer)
 {
-  gst_buffer_unref (cached_buffer->buffer);
+  if (cached_buffer->buffer)
+    gst_buffer_unref (cached_buffer->buffer);
   g_slice_free (GstSelectorPadCachedBuffer, cached_buffer);
 }
 
@@ -971,6 +972,10 @@ gst_selector_pad_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
           gst_selector_pad_chain (pad, parent, cached_buffer->buffer);
           GST_INPUT_SELECTOR_LOCK (sel);
 
+          /* We just passed the ownership of the buffer to the chain function */
+          cached_buffer->buffer = NULL;
+          gst_selector_pad_free_cached_buffer (cached_buffer);
+
           /* we may have cleaned up the queue in the meantime because of
            * old buffers */
           if (!selpad->cached_buffers) {