diff --git a/gst/rtp/gstrtptheorapay.c b/gst/rtp/gstrtptheorapay.c
index 379e04ddec..10055c0f4e 100644
--- a/gst/rtp/gstrtptheorapay.c
+++ b/gst/rtp/gstrtptheorapay.c
@@ -771,8 +771,14 @@ gst_rtp_theora_pay_handle_buffer (GstRTPBasePayload * basepayload,
     ret = GST_FLOW_OK;
     goto done;
   } else if (rtptheorapay->headers) {
-    if (!gst_rtp_theora_pay_finish_headers (basepayload))
-      goto header_error;
+    if (rtptheorapay->need_headers) {
+      if (!gst_rtp_theora_pay_finish_headers (basepayload))
+        goto header_error;
+    } else {
+      g_list_free_full (rtptheorapay->headers,
+          (GDestroyNotify) gst_buffer_unref);
+      rtptheorapay->headers = NULL;
+    }
   }
 
   /* there is a config request, see if we need to insert it */