diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_jpeg.c b/gst-libs/gst/vaapi/gstvaapidecoder_jpeg.c
index 65e648059d..5fc7f16024 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_jpeg.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_jpeg.c
@@ -490,19 +490,15 @@ decode_scan(
 }
 
 static GstVaapiDecoderStatus
-decode_buffer(GstVaapiDecoderJpeg *decoder, GstBuffer *buffer)
+decode_buffer(GstVaapiDecoderJpeg *decoder, guchar *buf, guint buf_size)
 {
     GstVaapiDecoderJpegPrivate * const priv = decoder->priv;
     GstVaapiDecoderStatus status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
     GstJpegMarkerSegment seg;
     GstJpegScanSegment scan_seg;
-    guchar *buf;
-    guint buf_size, ofs;
+    guint ofs;
     gboolean append_ecs;
 
-    buf      = GST_BUFFER_DATA(buffer);
-    buf_size = GST_BUFFER_SIZE(buffer);
-
     memset(&scan_seg, 0, sizeof(scan_seg));
 
     ofs = 0;
@@ -683,16 +679,17 @@ gst_vaapi_decoder_jpeg_decode(GstVaapiDecoder *base_decoder,
     GstVaapiDecoderStatus status;
     GstBuffer * const buffer =
         GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer;
+    guchar *buf;
+    guint buf_size;
 
     status = ensure_decoder(decoder);
     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
         return status;
 
-    unit->buffer = gst_buffer_create_sub(buffer, unit->offset, unit->size);
-    if (!unit->buffer)
-        return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
+    buf      = GST_BUFFER_DATA(buffer) + unit->offset;
+    buf_size = unit->size;
 
-    status = decode_buffer(decoder, unit->buffer);
+    status = decode_buffer(decoder, buf, buf_size);
     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
         return status;
     return GST_VAAPI_DECODER_STATUS_SUCCESS;
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_mpeg4.c b/gst-libs/gst/vaapi/gstvaapidecoder_mpeg4.c
index 8d800b4104..c3dd265eb1 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_mpeg4.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_mpeg4.c
@@ -890,16 +890,12 @@ decode_packet(GstVaapiDecoderMpeg4 *decoder, GstMpeg4Packet packet)
 }
 
 static GstVaapiDecoderStatus
-decode_buffer(GstVaapiDecoderMpeg4 *decoder, GstBuffer *buffer)
+decode_buffer(GstVaapiDecoderMpeg4 *decoder, const guchar *buf, guint buf_size)
 {
     GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
     GstVaapiDecoderStatus status;
     GstMpeg4Packet packet;
-    const guchar *buf;
-    guint buf_size, ofs;
-
-    buf      = GST_BUFFER_DATA(buffer);
-    buf_size = GST_BUFFER_SIZE(buffer);
+    guint ofs;
 
     if (priv->is_svh) {
         status = decode_picture(decoder, buf, buf_size);
@@ -1088,16 +1084,17 @@ gst_vaapi_decoder_mpeg4_decode(GstVaapiDecoder *base_decoder,
     GstVaapiDecoderStatus status;
     GstBuffer * const buffer =
         GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer;
+    const guchar *buf;
+    guint buf_size;
 
     status = ensure_decoder(decoder);
     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
         return status;
 
-    unit->buffer = gst_buffer_create_sub(buffer, unit->offset, unit->size);
-    if (!unit->buffer)
-        return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
+    buf      = GST_BUFFER_DATA(buffer) + unit->offset;
+    buf_size = unit->size;
 
-    status = decode_buffer(decoder, unit->buffer);
+    status = decode_buffer(decoder, buf, buf_size);
     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
         return status;
     return GST_VAAPI_DECODER_STATUS_SUCCESS;
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_unit.c b/gst-libs/gst/vaapi/gstvaapidecoder_unit.c
index af37947900..57c4d2b3b5 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_unit.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_unit.c
@@ -42,7 +42,6 @@ gst_vaapi_decoder_unit_init(GstVaapiDecoderUnit *unit)
     unit->flags = 0;
     unit->size = 0;
     unit->offset = 0;
-    unit->buffer = NULL;
 
     unit->parsed_info = NULL;
     unit->parsed_info_destroy_notify = NULL;
@@ -61,26 +60,9 @@ gst_vaapi_decoder_unit_init(GstVaapiDecoderUnit *unit)
 void
 gst_vaapi_decoder_unit_clear(GstVaapiDecoderUnit *unit)
 {
-    gst_buffer_replace(&unit->buffer, NULL);
     gst_vaapi_decoder_unit_set_parsed_info(unit, NULL, NULL);
 }
 
-/**
- * gst_vaapi_decoder_unit_set_buffer:
- * @unit: a #GstVaapiDecoderUnit
- * @buffer: the new #GstBuffer to set
- *
- * Sets new buffer to the supplied decoder unit. The @unit holds an
- * extra reference to the @buffer if it is not NULL.
- */
-void
-gst_vaapi_decoder_unit_set_buffer(GstVaapiDecoderUnit *unit, GstBuffer *buffer)
-{
-    g_return_if_fail(GST_VAAPI_IS_DECODER_UNIT(unit));
-
-    gst_buffer_replace(&unit->buffer, buffer);
-}
-
 /**
  * gst_vaapi_decoder_unit_set_parsed_info:
  * @unit: a #GstVaapiDecoderUnit
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_unit.h b/gst-libs/gst/vaapi/gstvaapidecoder_unit.h
index 6ccc59bddf..717e805526 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_unit.h
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_unit.h
@@ -22,8 +22,6 @@
 #ifndef GST_VAAPI_DECODER_UNIT_H
 #define GST_VAAPI_DECODER_UNIT_H
 
-#include <gst/gstbuffer.h>
-
 G_BEGIN_DECLS
 
 typedef struct _GstVaapiDecoderUnit             GstVaapiDecoderUnit;
@@ -155,7 +153,6 @@ typedef enum {
  * @size: size in bytes of this bitstream unit
  * @offset: relative offset in bytes to bitstream unit within the
  *    associated #GstVideoCodecFrame input_buffer
- * @buffer: (optional) associated buffer or sub-buffer
  * @parsed_info: parser-specific data (this is codec specific)
  * @parsed_info_destroy_notify: function used to release @parsed_info data
  *
@@ -165,7 +162,6 @@ struct _GstVaapiDecoderUnit {
     guint               flags;
     guint               size;
     guint               offset;
-    GstBuffer          *buffer;
     gpointer            parsed_info;
     GDestroyNotify      parsed_info_destroy_notify;
 };
@@ -182,10 +178,6 @@ G_GNUC_INTERNAL
 GstVaapiDecoderUnit *
 gst_vaapi_decoder_unit_new(void);
 
-G_GNUC_INTERNAL
-void
-gst_vaapi_decoder_unit_set_buffer(GstVaapiDecoderUnit *unit, GstBuffer *buffer);
-
 G_GNUC_INTERNAL
 void
 gst_vaapi_decoder_unit_set_parsed_info(GstVaapiDecoderUnit *unit,