diff --git a/gst-libs/gst/video/gstbasevideoencoder.c b/gst-libs/gst/video/gstbasevideoencoder.c
index 3919f9a909..afc060c353 100644
--- a/gst-libs/gst/video/gstbasevideoencoder.c
+++ b/gst-libs/gst/video/gstbasevideoencoder.c
@@ -809,16 +809,19 @@ gst_base_video_encoder_finish_frame (GstBaseVideoEncoder * base_video_encoder,
    * and subsequently set.  subclass is adult enough to set_caps itself ...
    * so simply check/ensure/assert that src pad caps are set by now */
   if (!base_video_encoder->set_output_caps) {
-    GstCaps *caps;
+    if (!GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder))) {
+      GstCaps *caps;
 
-    if (base_video_encoder_class->get_caps) {
-      caps = base_video_encoder_class->get_caps (base_video_encoder);
-    } else {
-      caps = gst_caps_new_simple ("video/unknown", NULL);
+      if (base_video_encoder_class->get_caps) {
+        caps = base_video_encoder_class->get_caps (base_video_encoder);
+      } else {
+        caps = gst_caps_new_simple ("video/unknown", NULL);
+      }
+      GST_DEBUG_OBJECT (base_video_encoder, "src caps %" GST_PTR_FORMAT, caps);
+      gst_pad_set_caps (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder),
+          caps);
+      gst_caps_unref (caps);
     }
-    GST_DEBUG_OBJECT (base_video_encoder, "src caps %" GST_PTR_FORMAT, caps);
-    gst_pad_set_caps (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), caps);
-    gst_caps_unref (caps);
     base_video_encoder->set_output_caps = TRUE;
   }