From 38c55de132b2a9baba1fbba3922c2e8b9a3af233 Mon Sep 17 00:00:00 2001
From: Haihao Xiang <haihao.xiang@intel.com>
Date: Wed, 21 Aug 2019 12:45:37 +0800
Subject: [PATCH] msdk: free mfx frames to avoid memory leaks in encoder and
 vpp

---
 sys/msdk/gstmsdkenc.c |  7 +++++--
 sys/msdk/gstmsdkvpp.c | 13 ++++++++++++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/sys/msdk/gstmsdkenc.c b/sys/msdk/gstmsdkenc.c
index cf72d74726..1b926b8c64 100644
--- a/sys/msdk/gstmsdkenc.c
+++ b/sys/msdk/gstmsdkenc.c
@@ -359,7 +359,7 @@ gst_msdkenc_init_encoder (GstMsdkEnc * thiz)
     if (status < MFX_ERR_NONE) {
       GST_ERROR_OBJECT (thiz, "Init failed (%s)",
           msdk_status_to_string (status));
-      goto no_vpp;
+      goto no_vpp_free_resource;
     } else if (status > MFX_ERR_NONE) {
       GST_WARNING_OBJECT (thiz, "Init returned: %s",
           msdk_status_to_string (status));
@@ -375,7 +375,7 @@ gst_msdkenc_init_encoder (GstMsdkEnc * thiz)
         GST_WARNING_OBJECT (thiz, "VPP close failed (%s)",
             msdk_status_to_string (status1));
 
-      goto no_vpp;
+      goto no_vpp_free_resource;
     } else if (status > MFX_ERR_NONE) {
       GST_WARNING_OBJECT (thiz, "Get VPP Parameters returned: %s",
           msdk_status_to_string (status));
@@ -529,6 +529,9 @@ gst_msdkenc_init_encoder (GstMsdkEnc * thiz)
 
   return TRUE;
 
+no_vpp_free_resource:
+  if (thiz->use_video_memory)
+    gst_msdk_frame_free (thiz->context, &thiz->vpp_alloc_resp);
 no_vpp:
 failed:
   GST_OBJECT_UNLOCK (thiz);
diff --git a/sys/msdk/gstmsdkvpp.c b/sys/msdk/gstmsdkvpp.c
index b50dfcdf99..88940f8bb0 100644
--- a/sys/msdk/gstmsdkvpp.c
+++ b/sys/msdk/gstmsdkvpp.c
@@ -804,6 +804,11 @@ gst_msdkvpp_close (GstMsdkVPP * thiz)
   if (!thiz->context)
     return;
 
+  if (thiz->use_video_memory) {
+    gst_msdk_frame_free (thiz->context, &thiz->in_alloc_resp);
+    gst_msdk_frame_free (thiz->context, &thiz->out_alloc_resp);
+  }
+
   GST_DEBUG_OBJECT (thiz, "Closing VPP 0x%p", thiz->context);
   status = MFXVideoVPP_Close (gst_msdk_context_get_session (thiz->context));
   if (status != MFX_ERR_NONE && status != MFX_ERR_NOT_INITIALIZED) {
@@ -956,8 +961,14 @@ gst_msdkvpp_initialize (GstMsdkVPP * thiz)
    * otherwise the subsequent function call of MFXVideoVPP_Init() will
    * fail
    */
-  if (thiz->initialized)
+  if (thiz->initialized) {
+    if (thiz->use_video_memory) {
+      gst_msdk_frame_free (thiz->context, &thiz->in_alloc_resp);
+      gst_msdk_frame_free (thiz->context, &thiz->out_alloc_resp);
+    }
+
     MFXVideoVPP_Close (session);
+  }
 
   if (thiz->use_video_memory) {
     gst_msdk_set_frame_allocator (thiz->context);