diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index 1cd3491ef0..c4a29712a8 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -2159,6 +2159,10 @@ gst_video_decoder_drop_frame (GstVideoDecoder * dec, GstVideoCodecFrame * frame)
  * If no output data is provided, @frame is considered skipped.
  * In any case, the frame is considered finished and released.
  *
+ * After calling this function the output buffer of the frame is to be
+ * considered read-only. This function will also change the metadata
+ * of the buffer.
+ *
  * Returns: a #GstFlowReturn resulting from sending data downstream
  *
  * Since: 0.10.36
@@ -2189,12 +2193,7 @@ gst_video_decoder_finish_frame (GstVideoDecoder * decoder,
     goto done;
   }
 
-  /* A reference always needs to be owned by the frame on the buffer.
-   * For that reason, we use a complete sub-buffer (zero-cost) to push
-   * downstream.
-   * The original buffer will be free-ed only when downstream AND the
-   * current implementation are done with the frame. */
-  output_buffer = gst_buffer_copy (frame->output_buffer);
+  output_buffer = frame->output_buffer;
 
   GST_BUFFER_FLAG_UNSET (output_buffer, GST_BUFFER_FLAG_DELTA_UNIT);
 
@@ -2211,6 +2210,12 @@ gst_video_decoder_finish_frame (GstVideoDecoder * decoder,
     priv->discont = FALSE;
   }
 
+  /* Get an additional ref to the buffer, which is going to be pushed
+   * downstream, the original ref is owned by the frame
+   *
+   * FIXME: clip_and_push_buf() changes buffer metadata but the buffer
+   * might have a refcount > 1 */
+  output_buffer = gst_buffer_ref (output_buffer);
   if (decoder->output_segment.rate < 0.0) {
     GST_LOG_OBJECT (decoder, "queued frame");
     priv->output_queued = g_list_prepend (priv->output_queued, output_buffer);
@@ -2827,6 +2832,9 @@ gst_video_decoder_alloc_output_buffer (GstVideoDecoder * decoder)
  * current #GstVideoCodecState.  Subclass should already have configured video
  * state and set src pad caps.
  *
+ * The buffer allocated here is owned by the frame and you should only
+ * keep references to the frame, not the buffer.
+ *
  * Returns: %GST_FLOW_OK if an output buffer could be allocated
  *
  * Since: 0.10.36
diff --git a/gst-libs/gst/video/gstvideoencoder.c b/gst-libs/gst/video/gstvideoencoder.c
index 335220942b..ed28a19866 100644
--- a/gst-libs/gst/video/gstvideoencoder.c
+++ b/gst-libs/gst/video/gstvideoencoder.c
@@ -1328,6 +1328,10 @@ gst_video_encoder_set_src_caps (GstVideoEncoder * encoder)
  * It is subsequently pushed downstream or provided to @pre_push.
  * In any case, the frame is considered finished and released.
  *
+ * After calling this function the output buffer of the frame is to be
+ * considered read-only. This function will also change the metadata
+ * of the buffer.
+ *
  * Returns: a #GstFlowReturn resulting from sending data downstream
  *
  * Since: 0.10.36
diff --git a/gst-libs/gst/video/gstvideoutils.h b/gst-libs/gst/video/gstvideoutils.h
index 0628afdb7b..60e27eabfc 100644
--- a/gst-libs/gst/video/gstvideoutils.h
+++ b/gst-libs/gst/video/gstvideoutils.h
@@ -201,10 +201,13 @@ typedef enum
  *       Typical usage in decoders is to set this on the opaque value provided
  *       to the library and get back the frame using gst_video_decoder_get_frame()
  * @distance_from_sync: Distance in frames from the last synchronization point.
- * @input_buffer: the input #GstBuffer that created this frame.
+ * @input_buffer: the input #GstBuffer that created this frame. The buffer is owned
+ *           by the frame and references to the frame instead of the buffer should
  * @output_buffer: the output #GstBuffer. Implementations should set this either
  *           directly, or by using the @gst_video_decoder_alloc_output_frame() or
- *           @gst_video_decoder_alloc_output_buffer() methods.
+ *           @gst_video_decoder_alloc_output_buffer() methods. The buffer is owned
+ *           by the frame and references to the frame instead of the buffer should
+ *           be kept.
  * @deadline: Running time when the frame will be used.
  * @events: Events that will be pushed downstream before this frame is pushed.
  *