diff --git a/ext/dirac/gstdiracenc.cc b/ext/dirac/gstdiracenc.cc
index 6e3129aa04..3a112cdf13 100644
--- a/ext/dirac/gstdiracenc.cc
+++ b/ext/dirac/gstdiracenc.cc
@@ -1266,7 +1266,8 @@ gst_dirac_enc_shape_output_ogg (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_OFFSET_END (buf) = dirac_enc->last_granulepos;
   }
 
-  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC(base_video_encoder)->caps);
+  gst_buffer_set_caps (buf,
+      GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder)));
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
@@ -1297,7 +1298,8 @@ gst_dirac_enc_shape_output_quicktime (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC(base_video_encoder)->caps);
+  gst_buffer_set_caps (buf,
+      GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder)));
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
@@ -1333,7 +1335,8 @@ gst_dirac_enc_shape_output_mp4 (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC(base_video_encoder)->caps);
+  gst_buffer_set_caps (buf,
+      GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder)));
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
diff --git a/ext/schroedinger/gstschroenc.c b/ext/schroedinger/gstschroenc.c
index faf5a7ffee..e8fabd1d14 100644
--- a/ext/schroedinger/gstschroenc.c
+++ b/ext/schroedinger/gstschroenc.c
@@ -648,7 +648,8 @@ gst_schro_enc_shape_output_ogg (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_OFFSET_END (buf) = schro_enc->last_granulepos;
   }
 
-  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
+  gst_buffer_set_caps (buf,
+      GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder)));
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
@@ -680,7 +681,8 @@ gst_schro_enc_shape_output_quicktime (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
+  gst_buffer_set_caps (buf,
+      GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder)));
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
@@ -716,7 +718,8 @@ gst_schro_enc_shape_output_mp4 (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
+  gst_buffer_set_caps (buf,
+      GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder)));
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
diff --git a/ext/vp8/gstvp8enc.c b/ext/vp8/gstvp8enc.c
index 9c1d8915a6..8fcaea3be1 100644
--- a/ext/vp8/gstvp8enc.c
+++ b/ext/vp8/gstvp8enc.c
@@ -936,7 +936,8 @@ gst_vp8_enc_shape_output (GstBaseVideoEncoder * base_video_encoder,
         gst_util_uint64_scale (frame->presentation_frame_number + 1,
         GST_SECOND * state->fps_d, state->fps_n);
 
-    gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
+    gst_buffer_set_caps (buf,
+        GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder)));
     ret = gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 
     if (ret != GST_FLOW_OK) {
diff --git a/gst-libs/gst/video/gstbasevideocodec.c b/gst-libs/gst/video/gstbasevideocodec.c
index cb29593e45..8db2eeff34 100644
--- a/gst-libs/gst/video/gstbasevideocodec.c
+++ b/gst-libs/gst/video/gstbasevideocodec.c
@@ -112,11 +112,6 @@ gst_base_video_codec_reset (GstBaseVideoCodec * base_video_codec)
   g_list_free (base_video_codec->frames);
   base_video_codec->frames = NULL;
 
-  if (base_video_codec->caps) {
-    gst_caps_unref (base_video_codec->caps);
-    base_video_codec->caps = NULL;
-  }
-
 }
 
 static void
diff --git a/gst-libs/gst/video/gstbasevideocodec.h b/gst-libs/gst/video/gstbasevideocodec.h
index fc8fb0b720..526026a1c6 100644
--- a/gst-libs/gst/video/gstbasevideocodec.h
+++ b/gst-libs/gst/video/gstbasevideocodec.h
@@ -147,8 +147,6 @@ struct _GstBaseVideoCodec
   GstVideoState state;
   GstSegment segment;
 
-  GstCaps *caps;
-
   gdouble proportion;
   GstClockTime earliest_time;
 
diff --git a/gst-libs/gst/video/gstbasevideoencoder.c b/gst-libs/gst/video/gstbasevideoencoder.c
index ad33f60e0f..f84c09dae9 100644
--- a/gst-libs/gst/video/gstbasevideoencoder.c
+++ b/gst-libs/gst/video/gstbasevideoencoder.c
@@ -540,22 +540,21 @@ gst_base_video_encoder_finish_frame (GstBaseVideoEncoder * base_video_encoder,
       g_list_remove (GST_BASE_VIDEO_CODEC (base_video_encoder)->frames, frame);
 
   if (!base_video_encoder->set_output_caps) {
+    GstCaps *caps;
+
     if (base_video_encoder_class->get_caps) {
-      GST_BASE_VIDEO_CODEC (base_video_encoder)->caps =
-          base_video_encoder_class->get_caps (base_video_encoder);
+      caps = base_video_encoder_class->get_caps (base_video_encoder);
     } else {
-      GST_BASE_VIDEO_CODEC (base_video_encoder)->caps =
-          gst_caps_new_simple ("video/unknown", NULL);
+      caps = gst_caps_new_simple ("video/unknown", NULL);
     }
-    GST_DEBUG_OBJECT (base_video_encoder, "src caps %" GST_PTR_FORMAT,
-        GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
-    gst_pad_set_caps (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder),
-        GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
+    GST_DEBUG_OBJECT (base_video_encoder, "src caps %" GST_PTR_FORMAT, caps);
+    gst_pad_set_caps (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), caps);
+    gst_caps_unref (caps);
     base_video_encoder->set_output_caps = TRUE;
   }
 
   gst_buffer_set_caps (GST_BUFFER (frame->src_buffer),
-      GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
+      GST_PAD_CAPS (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder)));
 
   if (frame->force_keyframe) {
     GstClockTime stream_time;