diff --git a/ext/x264/gstx264enc.c b/ext/x264/gstx264enc.c
index 61e7698f5e..3d81d962ea 100644
--- a/ext/x264/gstx264enc.c
+++ b/ext/x264/gstx264enc.c
@@ -1804,6 +1804,7 @@ gst_x264_enc_set_format (GstVideoEncoder * video_enc,
 
     if (gst_caps_is_empty (allowed_caps)) {
       gst_caps_unref (allowed_caps);
+      gst_caps_unref (template_caps);
       return FALSE;
     }