diff --git a/sys/nvcodec/gstnvdecoder.c b/sys/nvcodec/gstnvdecoder.c
index 902db4d071..19a710073f 100644
--- a/sys/nvcodec/gstnvdecoder.c
+++ b/sys/nvcodec/gstnvdecoder.c
@@ -306,6 +306,7 @@ gst_nv_decoder_new_frame (GstNvDecoder * decoder)
   frame = g_new0 (GstNvDecoderFrame, 1);
   frame->index = index_to_use;
   frame->decoder = gst_object_ref (decoder);
+  frame->ref_count = 1;
 
   GST_LOG_OBJECT (decoder, "New frame %p (index %d)", frame, frame->index);
 
@@ -369,33 +370,45 @@ gst_nv_decoder_frame_unmap (GstNvDecoderFrame * frame)
   frame->mapped = FALSE;
 }
 
+GstNvDecoderFrame *
+gst_nv_decoder_frame_ref (GstNvDecoderFrame * frame)
+{
+  g_assert (frame != NULL);
+
+  g_atomic_int_add (&frame->ref_count, 1);
+
+  return frame;
+}
+
 void
-gst_nv_decoder_frame_free (GstNvDecoderFrame * frame)
+gst_nv_decoder_frame_unref (GstNvDecoderFrame * frame)
 {
   GstNvDecoder *self;
 
   g_assert (frame != NULL);
 
-  GST_LOG ("Free frame %p (index %d)", frame, frame->index);
+  if (g_atomic_int_dec_and_test (&frame->ref_count)) {
+    GST_LOG ("Free frame %p (index %d)", frame, frame->index);
 
-  if (frame->decoder) {
-    self = frame->decoder;
-    if (frame->mapped && gst_cuda_context_push (self->context)) {
-      gst_nv_decoder_frame_unmap (frame);
-      gst_cuda_context_pop (NULL);
+    if (frame->decoder) {
+      self = frame->decoder;
+      if (frame->mapped && gst_cuda_context_push (self->context)) {
+        gst_nv_decoder_frame_unmap (frame);
+        gst_cuda_context_pop (NULL);
+      }
+
+      if (frame->index < self->pool_size) {
+        self->frame_pool[frame->index].available = TRUE;
+      } else {
+        GST_WARNING_OBJECT (self,
+            "Frame %p has invalid index %d", frame, frame->index);
+      }
+
+      gst_object_unref (self);
     }
 
-    if (frame->index < self->pool_size) {
-      self->frame_pool[frame->index].available = TRUE;
-    } else {
-      GST_WARNING_OBJECT (self,
-          "Frame %p has invalid index %d", frame, frame->index);
-    }
-
-    gst_object_unref (self);
+    g_free (frame);
   }
-
-  g_free (frame);
 }
 
 gboolean
diff --git a/sys/nvcodec/gstnvdecoder.h b/sys/nvcodec/gstnvdecoder.h
index befbb8ec27..c3d4473bea 100644
--- a/sys/nvcodec/gstnvdecoder.h
+++ b/sys/nvcodec/gstnvdecoder.h
@@ -42,6 +42,8 @@ typedef struct _GstNvDecoderFrame
 
   /*< private >*/
   GstNvDecoder *decoder;
+
+  gint ref_count;
 } GstNvDecoderFrame;
 
 typedef enum
@@ -59,7 +61,9 @@ GstNvDecoder * gst_nv_decoder_new (GstCudaContext * context,
 
 GstNvDecoderFrame * gst_nv_decoder_new_frame (GstNvDecoder * decoder);
 
-void gst_nv_decoder_frame_free (GstNvDecoderFrame * frame);
+GstNvDecoderFrame * gst_nv_decoder_frame_ref (GstNvDecoderFrame * frame);
+
+void gst_nv_decoder_frame_unref (GstNvDecoderFrame * frame);
 
 gboolean gst_nv_decoder_decode_picture (GstNvDecoder * decoder,
                                         CUVIDPICPARAMS * params);
diff --git a/sys/nvcodec/gstnvh264dec.c b/sys/nvcodec/gstnvh264dec.c
index 8712b24996..f09c5d2588 100644
--- a/sys/nvcodec/gstnvh264dec.c
+++ b/sys/nvcodec/gstnvh264dec.c
@@ -432,7 +432,7 @@ gst_nv_h264_dec_new_picture (GstH264Decoder * decoder,
       "New decoder frame %p (index %d)", nv_frame, nv_frame->index);
 
   gst_h264_picture_set_user_data (picture,
-      nv_frame, (GDestroyNotify) gst_nv_decoder_frame_free);
+      nv_frame, (GDestroyNotify) gst_nv_decoder_frame_unref);
 
   return TRUE;
 }
diff --git a/sys/nvcodec/gstnvh265dec.c b/sys/nvcodec/gstnvh265dec.c
index 1440cd26f6..33bb94d0b4 100644
--- a/sys/nvcodec/gstnvh265dec.c
+++ b/sys/nvcodec/gstnvh265dec.c
@@ -430,7 +430,7 @@ gst_nv_h265_dec_new_picture (GstH265Decoder * decoder,
   GST_LOG_OBJECT (self, "New decoder frame %p (index %d)", frame, frame->index);
 
   gst_h265_picture_set_user_data (picture,
-      frame, (GDestroyNotify) gst_nv_decoder_frame_free);
+      frame, (GDestroyNotify) gst_nv_decoder_frame_unref);
 
   return TRUE;
 }
diff --git a/sys/nvcodec/gstnvvp8dec.c b/sys/nvcodec/gstnvvp8dec.c
index b47298087a..da200d930f 100644
--- a/sys/nvcodec/gstnvvp8dec.c
+++ b/sys/nvcodec/gstnvvp8dec.c
@@ -300,7 +300,7 @@ gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
       "New decoder frame %p (index %d)", nv_frame, nv_frame->index);
 
   gst_vp8_picture_set_user_data (picture,
-      nv_frame, (GDestroyNotify) gst_nv_decoder_frame_free);
+      nv_frame, (GDestroyNotify) gst_nv_decoder_frame_unref);
 
   return TRUE;
 }