diff --git a/sys/msdk/gstmsdkallocator_d3d.c b/sys/msdk/gstmsdkallocator_d3d.c
index 75b147e86b..27e457288b 100644
--- a/sys/msdk/gstmsdkallocator_d3d.c
+++ b/sys/msdk/gstmsdkallocator_d3d.c
@@ -75,6 +75,5 @@ gst_msdk_set_frame_allocator (GstMsdkContext * context)
     .Free = gst_msdk_frame_free,
   };
 
-  MFXVideoCORE_SetFrameAllocator (gst_msdk_context_get_session (context),
-      &gst_msdk_frame_allocator);
+  gst_msdk_context_set_frame_allocator (context, &gst_msdk_frame_allocator);
 }
diff --git a/sys/msdk/gstmsdkallocator_libva.c b/sys/msdk/gstmsdkallocator_libva.c
index 74be0dffc1..e5200a3fe5 100644
--- a/sys/msdk/gstmsdkallocator_libva.c
+++ b/sys/msdk/gstmsdkallocator_libva.c
@@ -366,8 +366,7 @@ gst_msdk_set_frame_allocator (GstMsdkContext * context)
     .Free = gst_msdk_frame_free,
   };
 
-  MFXVideoCORE_SetFrameAllocator (gst_msdk_context_get_session (context),
-      &gst_msdk_frame_allocator);
+  gst_msdk_context_set_frame_allocator (context, &gst_msdk_frame_allocator);
 }
 
 gboolean
diff --git a/sys/msdk/gstmsdkcontext.c b/sys/msdk/gstmsdkcontext.c
index ae7b305fda..f1615ad603 100644
--- a/sys/msdk/gstmsdkcontext.c
+++ b/sys/msdk/gstmsdkcontext.c
@@ -47,6 +47,7 @@ struct _GstMsdkContextPrivate
   GList *cached_alloc_responses;
   gboolean hardware;
   gboolean is_joined;
+  gboolean has_frame_allocator;
   GstMsdkContextJobType job_type;
   gint shared_async_depth;
   GMutex mutex;
@@ -597,3 +598,25 @@ gst_msdk_context_add_shared_async_depth (GstMsdkContext * context,
 {
   context->priv->shared_async_depth += async_depth;
 }
+
+void
+gst_msdk_context_set_frame_allocator (GstMsdkContext * context,
+    mfxFrameAllocator * allocator)
+{
+  GstMsdkContextPrivate *priv = context->priv;
+
+  g_mutex_lock (&priv->mutex);
+
+  if (!priv->has_frame_allocator) {
+    mfxStatus status;
+
+    status = MFXVideoCORE_SetFrameAllocator (priv->session, allocator);
+
+    if (status != MFX_ERR_NONE)
+      GST_ERROR ("Failed to set frame allocator");
+    else
+      priv->has_frame_allocator = 1;
+  }
+
+  g_mutex_unlock (&priv->mutex);
+}
diff --git a/sys/msdk/gstmsdkcontext.h b/sys/msdk/gstmsdkcontext.h
index cbaf689379..6abfa86ef9 100644
--- a/sys/msdk/gstmsdkcontext.h
+++ b/sys/msdk/gstmsdkcontext.h
@@ -142,6 +142,10 @@ gst_msdk_context_get_shared_async_depth (GstMsdkContext * context);
 void
 gst_msdk_context_add_shared_async_depth (GstMsdkContext * context, gint async_depth);
 
+void
+gst_msdk_context_set_frame_allocator (GstMsdkContext * context,
+    mfxFrameAllocator * allocator);
+
 G_END_DECLS
 
 #endif /* GST_MSDK_CONTEXT_H */