diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_objects.c b/gst-libs/gst/vaapi/gstvaapiencoder_objects.c
index e87538ce17..2a6173625a 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_objects.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_objects.c
@@ -25,6 +25,7 @@
 #include "gstvaapiencoder_objects.h"
 #include "gstvaapiencoder.h"
 #include "gstvaapiencoder_priv.h"
+#include "gstvaapisurfaceproxy_priv.h"
 #include "gstvaapiutils.h"
 
 #define DEBUG 1
@@ -387,7 +388,7 @@ gst_vaapi_enc_picture_destroy (GstVaapiEncPicture * picture)
   gst_vaapi_mini_object_replace (
       (GstVaapiMiniObject **) (&picture->sequence), NULL);
 
-  g_assert (picture->surface);
+  gst_vaapi_surface_proxy_replace (&picture->proxy, NULL);
   picture->surface_id = VA_INVALID_ID;
   picture->surface = NULL;
 
@@ -405,19 +406,22 @@ gboolean
 gst_vaapi_enc_picture_create (GstVaapiEncPicture * picture,
     const GstVaapiCodecObjectConstructorArgs * args)
 {
-  GstVideoCodecFrame *frame;
-  GstVaapiSurface *surface;
-  GstVaapiEncObjUserDataHead *user_data;
+  GstVideoCodecFrame *const frame = (GstVideoCodecFrame *)args->data;
   gboolean success;
 
-  g_assert (args->data);
-  g_return_val_if_fail (args->data, FALSE);
+  g_return_val_if_fail (frame != NULL, FALSE);
 
-  frame = (GstVideoCodecFrame *) args->data;
-  user_data = gst_video_codec_frame_get_user_data (frame);
-  g_assert (user_data);
-  surface = user_data->surface;
-  g_return_val_if_fail (surface, FALSE);
+  picture->proxy = gst_video_codec_frame_get_user_data (frame);
+  if (!gst_vaapi_surface_proxy_ref (picture->proxy))
+    return FALSE;
+
+  picture->surface = GST_VAAPI_SURFACE_PROXY_SURFACE (picture->proxy);
+  if (!picture->surface)
+    return FALSE;
+
+  picture->surface_id = GST_VAAPI_OBJECT_ID (picture->surface);
+  if (picture->surface_id == VA_INVALID_ID)
+    return FALSE;
 
   picture->sequence = NULL;
   picture->type = GST_VAAPI_PICTURE_TYPE_NONE;
@@ -446,10 +450,6 @@ gst_vaapi_enc_picture_create (GstVaapiEncPicture * picture,
     return FALSE;
 
   picture->frame = gst_video_codec_frame_ref (frame);
-  picture->surface = surface;
-  g_assert (picture->surface);
-  picture->surface_id = gst_vaapi_surface_get_id (picture->surface);
-  g_assert (picture->surface_id != VA_INVALID_SURFACE);
 
   return TRUE;
 }
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_objects.h b/gst-libs/gst/vaapi/gstvaapiencoder_objects.h
index 240d55b521..fdc960ee24 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_objects.h
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_objects.h
@@ -261,11 +261,6 @@ typedef enum
 #define GST_VAAPI_ENC_PICTURE_GET_FRAME(picture) \
     (picture)->frame
 
-typedef struct
-{
-  GstVaapiSurface *surface;
-} GstVaapiEncObjUserDataHead;
-
 /**
  * GstVaapiEncPicture:
  *
@@ -276,6 +271,7 @@ struct _GstVaapiEncPicture
   /*< private > */
   GstVaapiCodecObject parent_instance;
   GstVideoCodecFrame *frame;
+  GstVaapiSurfaceProxy *proxy;
   GstVaapiSurface *surface;
   GstVaapiEncSequence *sequence;
   /*< private >, picture packed header */
diff --git a/gst/vaapi/gstvaapiencode.c b/gst/vaapi/gstvaapiencode.c
index 86b630c9b5..a6bab1360d 100644
--- a/gst/vaapi/gstvaapiencode.c
+++ b/gst/vaapi/gstvaapiencode.c
@@ -41,12 +41,6 @@
 #define GST_VAAPI_ENCODE_FLOW_CONVERT_ERROR     GST_FLOW_CUSTOM_ERROR_1
 #define GST_VAAPI_ENCODE_FLOW_CODEC_DATA_ERROR  GST_FLOW_CUSTOM_ERROR_2
 
-typedef struct _GstVaapiEncodeFrameUserData
-{
-  GstVaapiEncObjUserDataHead head;
-  GstBuffer *vaapi_buf;
-} GstVaapiEncodeFrameUserData;
-
 GST_DEBUG_CATEGORY_STATIC (gst_vaapiencode_debug);
 #define GST_CAT_DEFAULT gst_vaapiencode_debug
 
@@ -630,7 +624,6 @@ get_source_buffer (GstVaapiEncode * encode, GstBuffer * src_buffer,
   gboolean success;
 #endif
 
-  *out_buffer_ptr = NULL;
   meta = gst_buffer_get_vaapi_video_meta (src_buffer);
   if (meta) {
     *out_buffer_ptr = gst_buffer_ref (src_buffer);
@@ -728,77 +721,73 @@ error_copy_buffer:
   }
 }
 
-static inline gpointer
-_create_user_data (GstBuffer * buf)
-{
-  GstVaapiVideoMeta *meta;
-  GstVaapiSurface *surface;
-  GstVaapiEncodeFrameUserData *user_data;
-
-  meta = gst_buffer_get_vaapi_video_meta (buf);
-  if (!meta) {
-    GST_DEBUG ("convert to vaapi buffer failed");
-    return NULL;
-  }
-  surface = gst_vaapi_video_meta_get_surface (meta);
-  if (!surface) {
-    GST_DEBUG ("vaapi_meta of codec frame doesn't have vaapisurfaceproxy");
-    return NULL;
-  }
-
-  user_data = g_slice_new0 (GstVaapiEncodeFrameUserData);
-  user_data->head.surface = surface;
-  user_data->vaapi_buf = gst_buffer_ref (buf);
-  return user_data;
-}
-
-static void
-_destroy_user_data (gpointer data)
-{
-  GstVaapiEncodeFrameUserData *user_data = (GstVaapiEncodeFrameUserData *) data;
-
-  g_assert (data);
-  if (!user_data)
-    return;
-  gst_buffer_replace (&user_data->vaapi_buf, NULL);
-  g_slice_free (GstVaapiEncodeFrameUserData, user_data);
-}
-
 static GstFlowReturn
 gst_vaapiencode_handle_frame (GstVideoEncoder * venc,
     GstVideoCodecFrame * frame)
 {
   GstVaapiEncode *const encode = GST_VAAPIENCODE_CAST (venc);
-  GstFlowReturn ret = GST_FLOW_OK;
-  GstVaapiEncoderStatus status = GST_VAAPI_ENCODER_STATUS_SUCCESS;
+  GstVaapiEncoderStatus status;
+  GstVaapiVideoMeta *meta;
+  GstVaapiSurfaceProxy *proxy;
+  GstFlowReturn ret;
   GstBuffer *buf;
-  gpointer user_data;
-
-  g_assert (encode && encode->encoder);
-  g_assert (frame && frame->input_buffer);
 
+  buf = NULL;
   ret = get_source_buffer (encode, frame->input_buffer, &buf);
   if (ret != GST_FLOW_OK)
-    return ret;
+    goto error_buffer_invalid;
 
-  user_data = _create_user_data (buf);
-  GST_VAAPI_ENCODER_CHECK_STATUS (user_data,
-      ret, "create frame user data failed");
+  gst_buffer_replace (&frame->input_buffer, buf);
+  gst_buffer_unref (buf);
 
-  gst_video_codec_frame_set_user_data (frame, user_data, _destroy_user_data);
+  meta = gst_buffer_get_vaapi_video_meta (buf);
+  if (!meta)
+    goto error_buffer_no_meta;
+
+  proxy = gst_vaapi_video_meta_get_surface_proxy (meta);
+  if (!proxy)
+    goto error_buffer_no_surface_proxy;
+
+  gst_video_codec_frame_set_user_data (frame,
+      gst_vaapi_surface_proxy_ref (proxy),
+      (GDestroyNotify)gst_vaapi_surface_proxy_unref);
 
   GST_VIDEO_ENCODER_STREAM_UNLOCK (encode);
-  /*encoding frames */
   status = gst_vaapi_encoder_put_frame (encode->encoder, frame);
   GST_VIDEO_ENCODER_STREAM_LOCK (encode);
+  if (status < GST_VAAPI_ENCODER_STATUS_SUCCESS)
+    goto error_encode_frame;
 
-  GST_VAAPI_ENCODER_CHECK_STATUS (GST_VAAPI_ENCODER_STATUS_SUCCESS <=
-      status, GST_FLOW_ERROR, "gst_vaapiencoder_encode failed.");
-
-end:
   gst_video_codec_frame_unref (frame);
-  gst_buffer_replace (&buf, NULL);
-  return ret;
+  return GST_FLOW_OK;
+
+  /* ERRORS */
+error_buffer_invalid:
+  {
+    if (buf)
+      gst_buffer_unref (buf);
+    gst_video_codec_frame_unref (frame);
+    return ret;
+  }
+error_buffer_no_meta:
+  {
+    GST_ERROR ("failed to get GstVaapiVideoMeta information");
+    gst_video_codec_frame_unref (frame);
+    return GST_FLOW_ERROR;
+  }
+error_buffer_no_surface_proxy:
+  {
+    GST_ERROR ("failed to get VA surface proxy");
+    gst_video_codec_frame_unref (frame);
+    return GST_FLOW_ERROR;
+  }
+error_encode_frame:
+  {
+    GST_ERROR ("failed to encode frame %d (status %d)",
+        frame->system_frame_number, status);
+    gst_video_codec_frame_unref (frame);
+    return GST_FLOW_ERROR;
+  }
 }
 
 static GstFlowReturn