gstreamer/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c

/* GStreamer Intel MSDK plugin
 * Copyright (c) 2016, Intel Corporation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif

#include <stdlib.h>

#include "gstmsdkdec.h"
#include "gstmsdkcontextutil.h"
#include "gstmsdkallocator.h"
#include "gstmsdkcaps.h"

#ifndef _WIN32
#include <drm_fourcc.h>
#include <gst/va/gstvaallocator.h>
#include <gst/va/gstvavideoformat.h>
#else
#include <gst/d3d11/gstd3d11.h>
#endif

GST_DEBUG_CATEGORY_EXTERN (gst_msdkdec_debug);
#define GST_CAT_DEFAULT gst_msdkdec_debug

#define PROP_HARDWARE_DEFAULT            TRUE
#define PROP_ASYNC_DEPTH_DEFAULT         1

#define IS_ALIGNED(i, n) (((i) & ((n)-1)) == 0)

#define GST_TO_MFX_TIME(time) ((time) == GST_CLOCK_TIME_NONE ? \
    MFX_TIMESTAMP_UNKNOWN : gst_util_uint64_scale_round ((time), 9, 100000))

#define MFX_TO_GST_TIME(time) ((time) == MFX_TIMESTAMP_UNKNOWN ? \
    GST_CLOCK_TIME_NONE : gst_util_uint64_scale_round ((time), 100000, 9))

#define MFX_TIME_IS_VALID(time) ((time) != MFX_TIMESTAMP_UNKNOWN)

#define GST_MSDK_FRAME_SURFACE gst_msdk_frame_surface_quark_get ()

#define gst_msdkdec_parent_class parent_class
G_DEFINE_TYPE (GstMsdkDec, gst_msdkdec, GST_TYPE_VIDEO_DECODER);

struct _MsdkDecTask
{
  GstMsdkSurface *surface;
  mfxSyncPoint sync_point;

  gboolean decode_only;
};

static GstFlowReturn gst_msdkdec_drain (GstVideoDecoder * decoder);
static gboolean gst_msdkdec_flush (GstVideoDecoder * decoder);
static gboolean gst_msdkdec_negotiate (GstMsdkDec * thiz, gboolean hard_reset);

void
gst_msdkdec_add_bs_extra_param (GstMsdkDec * thiz, mfxExtBuffer * param)
{
  if (thiz->num_bs_extra_params < MAX_BS_EXTRA_PARAMS) {
    thiz->bs_extra_params[thiz->num_bs_extra_params] = param;
    thiz->num_bs_extra_params++;
  }
}

void
gst_msdkdec_add_video_extra_param (GstMsdkDec * thiz, mfxExtBuffer * param)
{
  if (thiz->num_video_extra_params < MAX_VIDEO_EXTRA_PARAMS) {
    thiz->video_extra_params[thiz->num_video_extra_params] = param;
    thiz->num_video_extra_params++;
  }
}

static GstVideoCodecFrame *
gst_msdkdec_get_oldest_frame (GstVideoDecoder * decoder)
{
  GstVideoCodecFrame *frame = NULL, *old_frame = NULL;
  GList *frames, *l;
  gint count = 0;

  frames = gst_video_decoder_get_frames (decoder);

  for (l = frames; l != NULL; l = l->next) {
    GstVideoCodecFrame *f = l->data;

    if (!GST_CLOCK_TIME_IS_VALID (f->pts)) {
      GST_INFO
          ("Frame doesn't have a valid pts yet, Use gst_video_decoder_get_oldest_frame()"
          "with out considering the PTS for selecting the frame to be finished");
      old_frame = gst_video_decoder_get_oldest_frame (decoder);
      break;
    }

    if (!frame || frame->pts > f->pts)
      frame = f;

    count++;
  }

  if (old_frame)
    frame = old_frame;

  if (frame) {
    GST_LOG_OBJECT (decoder,
        "Oldest frame is %d %" GST_TIME_FORMAT " and %d frames left",
        frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1);
    gst_video_codec_frame_ref (frame);
  }

  if (old_frame)
    gst_video_codec_frame_unref (old_frame);

  g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref);

  return frame;
}

static inline void
free_surface (GstMsdkSurface * s)
{
  gst_buffer_unref (s->buf);
  g_slice_free (GstMsdkSurface, s);
}

static gboolean
gst_msdkdec_free_unlocked_msdk_surfaces (GstMsdkDec * thiz,
    gboolean check_avail_surface)
{
  GList *l;
  GstMsdkSurface *surface;

  for (l = thiz->locked_msdk_surfaces; l;) {
    GList *next = l->next;
    surface = l->data;
    if (surface->surface->Data.Locked == 0 &&
        GST_MINI_OBJECT_REFCOUNT_VALUE (surface->buf) == 1) {
      free_surface (surface);
      thiz->locked_msdk_surfaces =
          g_list_delete_link (thiz->locked_msdk_surfaces, l);

      /* When check_avail_surface flag is enabled, it means we only
       * need to find one available surface instead of releasing all
       * the unlocked surfaces, so we can return TEUR here.
       */
      if (check_avail_surface)
        return TRUE;
    }
    l = next;
  }
  /* We need to check if all surfaces are in used */
  if (g_list_length (thiz->locked_msdk_surfaces) ==
      thiz->alloc_resp.NumFrameActual)
    return FALSE;
  else
    return TRUE;
}

static GstMsdkSurface *
allocate_output_surface (GstMsdkDec * thiz)
{
  GstMsdkSurface *msdk_surface = NULL;
  GstBuffer *out_buffer = NULL;
  GstMemory *mem = NULL;
  mfxFrameSurface1 *mfx_surface = NULL;
  gint n = 0;
  guint retry_times = 1000;
#ifdef _WIN32
  GstMapInfo map_info;
#endif

  /* Free un-unsed msdk surfaces firstly, hence the associated mfx
   * surfaces will be moved from used list to available list */
  if (!gst_msdkdec_free_unlocked_msdk_surfaces (thiz, FALSE)) {
    for (n = 0; n < retry_times; n++) {
      /* It is MediaSDK/oneVPL's requirement that only the pre-allocated
       * surfaces can be used during the whole decoding process.
       * In the case of decoder plus multi-encoders, it is possible
       * that all surfaces are used by downstreams and no more surfaces
       * available for decoder. So here we need to wait until there is at
       * least one surface is free for decoder.
       */
      g_usleep (1000);
      if (gst_msdkdec_free_unlocked_msdk_surfaces (thiz, TRUE))
        break;
    }
    if (n == retry_times) {
      GST_WARNING ("No available unlocked msdk surfaces");
      return NULL;
    }
  }

  if ((gst_buffer_pool_acquire_buffer (thiz->alloc_pool, &out_buffer, NULL))
      != GST_FLOW_OK) {
    GST_ERROR_OBJECT (thiz, "Failed to allocate output buffer");
    return NULL;
  }
#ifdef _WIN32
  /* For d3d11 we should call gst_buffer_map with GST_MAP_WRITE |
   * GST_MAP_D3D11 flags to make sure the staging texture has been uploaded
   */
  if (!gst_buffer_map (out_buffer, &map_info, GST_MAP_WRITE | GST_MAP_D3D11)) {
    GST_ERROR ("Failed to map buffer");
    return NULL;
  }
#endif
  mem = gst_buffer_peek_memory (out_buffer, 0);
  msdk_surface = g_slice_new0 (GstMsdkSurface);

  if ((mfx_surface = gst_mini_object_get_qdata (GST_MINI_OBJECT_CAST (mem),
              GST_MSDK_FRAME_SURFACE))) {
    msdk_surface->surface = mfx_surface;
    msdk_surface->from_qdata = TRUE;
#ifdef _WIN32
    gst_buffer_unmap (out_buffer, &map_info);
#endif
  } else {
    GST_ERROR ("No available surfaces");
    g_slice_free (GstMsdkSurface, msdk_surface);
    return NULL;
  }

  msdk_surface->buf = out_buffer;

  if (!thiz->sfc)
    gst_msdk_update_mfx_frame_info_from_mfx_video_param
        (&msdk_surface->surface->Info, &thiz->param);

  thiz->locked_msdk_surfaces =
      g_list_append (thiz->locked_msdk_surfaces, msdk_surface);

  return msdk_surface;
}

static void
gst_msdkdec_close_decoder (GstMsdkDec * thiz, gboolean reset_param)
{
  mfxStatus status;

  if (!thiz->context || !thiz->initialized)
    return;

  GST_DEBUG_OBJECT (thiz, "Closing decoder with context %" GST_PTR_FORMAT,
      thiz->context);

  gst_msdk_frame_free (thiz->context, &thiz->alloc_resp);

  status = MFXVideoDECODE_Close (gst_msdk_context_get_session (thiz->context));
  if (status != MFX_ERR_NONE && status != MFX_ERR_NOT_INITIALIZED) {
    GST_WARNING_OBJECT (thiz, "Decoder close failed (%s)",
        msdk_status_to_string (status));
  }

  g_array_set_size (thiz->tasks, 0);

  if (reset_param)
    memset (&thiz->param, 0, sizeof (thiz->param));

  thiz->num_bs_extra_params = 0;
  thiz->num_video_extra_params = 0;
  thiz->initialized = FALSE;
  gst_adapter_clear (thiz->adapter);
}

static void
gst_msdkdec_set_context (GstElement * element, GstContext * context)
{
  GstMsdkContext *msdk_context = NULL;
  GstMsdkDec *thiz = GST_MSDKDEC (element);

  if (gst_msdk_context_get_context (context, &msdk_context)) {
    gst_object_replace ((GstObject **) & thiz->context,
        (GstObject *) msdk_context);
    gst_object_unref (msdk_context);
  } else
#ifndef _WIN32
  if (gst_msdk_context_from_external_va_display (context,
          thiz->hardware, GST_MSDK_JOB_DECODER, &msdk_context)) {
    gst_object_replace ((GstObject **) & thiz->context,
        (GstObject *) msdk_context);
    gst_object_unref (msdk_context);
  }
#else
  if (gst_msdk_context_from_external_d3d11_device (context,
          thiz->hardware, GST_MSDK_JOB_DECODER, &msdk_context)) {
    gst_object_replace ((GstObject **) & thiz->context,
        (GstObject *) msdk_context);
    gst_object_unref (msdk_context);
  }
#endif

  GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}

static gboolean
gst_msdkdec_init_decoder (GstMsdkDec * thiz)
{
  GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
  GstVideoInfo *info, *output_info;
  mfxSession session;
  mfxStatus status;
  mfxFrameAllocRequest request;
  gint shared_async_depth;
#if (MFX_VERSION >= 1022)
  mfxExtDecVideoProcessing ext_dec_video_proc;
#endif

  if (thiz->initialized)
    return TRUE;

  if (!thiz->context) {
    GST_WARNING_OBJECT (thiz, "No MSDK Context");
    return FALSE;
  }

  if (!thiz->input_state) {
    GST_DEBUG_OBJECT (thiz, "Have no input state yet");
    return FALSE;
  }
  info = &thiz->input_state->info;
  output_info = &thiz->output_state->info;

  GST_OBJECT_LOCK (thiz);

  gst_msdk_set_frame_allocator (thiz->context);
  thiz->param.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY;

  thiz->param.AsyncDepth = thiz->async_depth;

  /* We expect msdk to fill the width and height values */
  if (!(thiz->param.mfx.FrameInfo.Width && thiz->param.mfx.FrameInfo.Height))
    goto failed;

  klass->preinit_decoder (thiz);

  /* Set frame rate only if provided.
   * If not, frame rate will be assumed inside the driver.
   * Also we respect the upstream provided fps values */
  if (info->fps_n > 0 && info->fps_d > 0
      && info->fps_n != thiz->param.mfx.FrameInfo.FrameRateExtN
      && info->fps_d != thiz->param.mfx.FrameInfo.FrameRateExtD) {
    thiz->param.mfx.FrameInfo.FrameRateExtN = info->fps_n;
    thiz->param.mfx.FrameInfo.FrameRateExtD = info->fps_d;
  }

  if (info->par_n && info->par_d && !thiz->param.mfx.FrameInfo.AspectRatioW
      && !thiz->param.mfx.FrameInfo.AspectRatioH) {
    thiz->param.mfx.FrameInfo.AspectRatioW = info->par_n;
    thiz->param.mfx.FrameInfo.AspectRatioH = info->par_d;
  }

  thiz->param.mfx.FrameInfo.FourCC =
      thiz->param.mfx.FrameInfo.FourCC ? thiz->param.mfx.
      FrameInfo.FourCC : MFX_FOURCC_NV12;
  thiz->param.mfx.FrameInfo.ChromaFormat =
      thiz->param.mfx.FrameInfo.ChromaFormat ? thiz->param.mfx.
      FrameInfo.ChromaFormat : MFX_CHROMAFORMAT_YUV420;

#if (MFX_VERSION >= 1022)
  if (output_info && thiz->sfc) {
    memset (&ext_dec_video_proc, 0, sizeof (ext_dec_video_proc));
    ext_dec_video_proc.Header.BufferId = MFX_EXTBUFF_DEC_VIDEO_PROCESSING;
    ext_dec_video_proc.Header.BufferSz = sizeof (ext_dec_video_proc);
    ext_dec_video_proc.In.CropW = thiz->param.mfx.FrameInfo.CropW;
    ext_dec_video_proc.In.CropH = thiz->param.mfx.FrameInfo.CropH;
    ext_dec_video_proc.In.CropX = 0;
    ext_dec_video_proc.In.CropY = 0;
    ext_dec_video_proc.Out.FourCC =
        gst_msdk_get_mfx_fourcc_from_format (output_info->finfo->format);
    ext_dec_video_proc.Out.ChromaFormat =
        gst_msdk_get_mfx_chroma_from_format (output_info->finfo->format);
    ext_dec_video_proc.Out.Width = GST_ROUND_UP_16 (output_info->width);
    ext_dec_video_proc.Out.Height = GST_ROUND_UP_32 (output_info->height);
    ext_dec_video_proc.Out.CropW = output_info->width;
    ext_dec_video_proc.Out.CropH = output_info->height;
    ext_dec_video_proc.Out.CropX = 0;
    ext_dec_video_proc.Out.CropY = 0;
    gst_msdkdec_add_video_extra_param (thiz,
        (mfxExtBuffer *) & ext_dec_video_proc);
  }
#endif

  if (thiz->num_video_extra_params) {
    thiz->param.NumExtParam = thiz->num_video_extra_params;
    thiz->param.ExtParam = thiz->video_extra_params;
  }

  session = gst_msdk_context_get_session (thiz->context);
  /* validate parameters and allow MFX to make adjustments */
  status = MFXVideoDECODE_Query (session, &thiz->param, &thiz->param);
  if (status < MFX_ERR_NONE) {
    GST_ERROR_OBJECT (thiz, "Video Decode Query failed (%s)",
        msdk_status_to_string (status));
    goto failed;
  } else if (status > MFX_ERR_NONE) {
    GST_WARNING_OBJECT (thiz, "Video Decode Query returned: %s",
        msdk_status_to_string (status));
  }

  klass->postinit_decoder (thiz);

  status = MFXVideoDECODE_QueryIOSurf (session, &thiz->param, &request);
  if (status < MFX_ERR_NONE) {
    GST_ERROR_OBJECT (thiz, "Query IO surfaces failed (%s)",
        msdk_status_to_string (status));
    goto failed;
  } else if (status > MFX_ERR_NONE) {
    GST_WARNING_OBJECT (thiz, "Query IO surfaces returned: %s",
        msdk_status_to_string (status));
  }

  if (request.NumFrameSuggested < thiz->param.AsyncDepth) {
    GST_ERROR_OBJECT (thiz, "Required %d surfaces (%d suggested), async %d",
        request.NumFrameMin, request.NumFrameSuggested, thiz->param.AsyncDepth);
    goto failed;
  }

  /* account for downstream requirement */
  if (G_LIKELY (thiz->min_prealloc_buffers))
    request.NumFrameSuggested += thiz->min_prealloc_buffers;
  else
    GST_WARNING_OBJECT (thiz,
        "Allocating resources without considering the downstream requirement"
        "or extra scratch surface count");

  shared_async_depth = gst_msdk_context_get_shared_async_depth (thiz->context);
  request.NumFrameSuggested += shared_async_depth;

  request.Type |= MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
  if (thiz->use_dmabuf)
    request.Type |= MFX_MEMTYPE_EXPORT_FRAME;
#if (MFX_VERSION >= 1022)
  if (thiz->sfc) {
    request.Info.Width = ext_dec_video_proc.Out.Width;
    request.Info.Height = ext_dec_video_proc.Out.Height;
  }
#endif

  gst_msdk_frame_alloc (thiz->context, &request, &thiz->alloc_resp);
  thiz->alloc_pool = gst_msdk_context_get_alloc_pool (thiz->context);

  /* update the prealloc_buffer count, which will be used later
   * as GstBufferPool min_buffers */
  thiz->min_prealloc_buffers = request.NumFrameSuggested;

  GST_DEBUG_OBJECT (thiz, "Required %d surfaces (%d suggested)",
      request.NumFrameMin, request.NumFrameSuggested);

  status = MFXVideoDECODE_Init (session, &thiz->param);
  if (status < MFX_ERR_NONE) {
    GST_ERROR_OBJECT (thiz, "Init failed (%s)", msdk_status_to_string (status));
    goto failed;
  } else if (status > MFX_ERR_NONE) {
    GST_WARNING_OBJECT (thiz, "Init returned: %s",
        msdk_status_to_string (status));
  }

  status = MFXVideoDECODE_GetVideoParam (session, &thiz->param);
  if (status < MFX_ERR_NONE) {
    GST_ERROR_OBJECT (thiz, "Get Video Parameters failed (%s)",
        msdk_status_to_string (status));
    goto failed;
  } else if (status > MFX_ERR_NONE) {
    GST_WARNING_OBJECT (thiz, "Get Video Parameters returned: %s",
        msdk_status_to_string (status));
  }

  g_array_set_size (thiz->tasks, 0);    /* resets array content */
  g_array_set_size (thiz->tasks, thiz->param.AsyncDepth);
  thiz->next_task = 0;

  GST_OBJECT_UNLOCK (thiz);

  thiz->initialized = TRUE;
  return TRUE;

failed:
  GST_OBJECT_UNLOCK (thiz);
  return FALSE;
}

static gboolean
pad_accept_memory (GstMsdkDec * thiz, const gchar * mem_type, GstCaps ** filter)
{
  gboolean ret = FALSE;
  GstCaps *caps, *out_caps;
  GstPad *pad;

  pad = GST_VIDEO_DECODER_SRC_PAD (thiz);

  caps = gst_caps_copy (*filter);
  gst_caps_set_features (caps, 0,
      gst_caps_features_new_single_static_str (mem_type));

#ifndef _WIN32
  /* In dma caps case, we need to set drm-format for raw caps */
  if (gst_msdkcaps_has_feature (caps, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
    GstStructure *s = gst_caps_get_structure (caps, 0);

    if (gst_structure_has_field (s, "format")) {
      const gchar *fmt_str = gst_structure_get_string (s, "format");
      GstVideoFormat fmt = gst_video_format_from_string (fmt_str);
      guint32 fourcc = gst_va_drm_fourcc_from_video_format (fmt);
      gchar *drm_str =
          gst_video_dma_drm_fourcc_to_string (fourcc, thiz->modifier);
      gst_msdkcaps_set_strings (caps, mem_type, "drm-format", drm_str);
      gst_caps_set_simple (caps, "format", G_TYPE_STRING, "DMA_DRM", NULL);
      g_free (drm_str);
    }
  }
#endif

  out_caps = gst_pad_peer_query_caps (pad, caps);

  if (!out_caps || gst_caps_is_empty (out_caps))
    goto done;

  if (gst_msdkcaps_has_feature (out_caps, mem_type)) {
    gst_caps_replace (filter, caps);
    ret = TRUE;
  }

done:
  if (caps)
    gst_caps_unref (caps);
  if (out_caps)
    gst_caps_unref (out_caps);
  return ret;
}

static GstCaps *
gst_msdkdec_getcaps (GstVideoDecoder * decoder, GstCaps * filter)
{
  GstCaps *caps, *tmp = NULL;

  caps = gst_pad_get_pad_template_caps (decoder->sinkpad);
  if (caps) {
    if (filter) {
      tmp = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
      gst_caps_unref (caps);
      caps = tmp;
    }
  } else {
    caps = gst_video_decoder_proxy_getcaps (decoder, NULL, filter);
  }

  return caps;
}

static gboolean
gst_msdkdec_fixate_format (GstMsdkDec * thiz, GstCaps * caps,
    GstVideoFormat fmt)
{
  GValue gfmt = G_VALUE_INIT;

  g_return_val_if_fail (caps != NULL, FALSE);
  g_return_val_if_fail (gst_caps_is_writable (caps), FALSE);
  g_return_val_if_fail (fmt != GST_VIDEO_FORMAT_UNKNOWN, FALSE);

  g_value_init (&gfmt, G_TYPE_STRING);

  for (guint i = 0; i < gst_caps_get_size (caps); i++) {
    GstStructure *s = gst_caps_get_structure (caps, i);

#ifndef _WIN32
    GstCapsFeatures *f = gst_caps_get_features (caps, i);
    if (gst_caps_features_contains (f, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
      const GValue *drm_fmts = gst_structure_get_value (s, "drm-format");
      guint32 fourcc = gst_va_drm_fourcc_from_video_format (fmt);
      gchar *drm_str =
          gst_video_dma_drm_fourcc_to_string (fourcc, thiz->modifier);
      g_value_set_string (&gfmt, drm_str);
      g_free (drm_str);
      if (!gst_value_can_intersect (&gfmt, drm_fmts))
        goto failed;
      gst_structure_set_value (s, "drm-format", &gfmt);
      continue;
    }
#endif

    const GValue *fmts = gst_structure_get_value (s, "format");
    g_value_set_string (&gfmt, gst_video_format_to_string (fmt));
    if (!gst_value_can_intersect (&gfmt, fmts))
      goto failed;
    gst_structure_set_value (s, "format", &gfmt);
  }

  g_value_unset (&gfmt);
  return TRUE;

failed:
  g_value_unset (&gfmt);
  return FALSE;
}

static gboolean
gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation)
{
  GstVideoInfo vinfo;
  GstVideoAlignment align;
  GstCaps *allocation_caps = NULL;
  GstCaps *allowed_caps = NULL, *temp_caps, *out_caps, *src_caps;
  GstVideoFormat format;
  guint width, height;
  guint alloc_w, alloc_h;
  int out_width = 0, out_height = 0;
  gint dar_n = -1, dar_d = -1;
  GstStructure *outs = NULL;
  const gchar *out_format;
  GValue v_width = G_VALUE_INIT;
  GValue v_height = G_VALUE_INIT;
  gboolean has_drm_format = FALSE;

  /* use display width and display height in output state, which
   * will be used for caps negotiation */
  width =
      thiz->param.mfx.FrameInfo.CropW ? thiz->param.mfx.
      FrameInfo.CropW : GST_VIDEO_INFO_WIDTH (&thiz->input_state->info);
  height =
      thiz->param.mfx.FrameInfo.CropH ? thiz->param.mfx.
      FrameInfo.CropH : GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info);

  format =
      gst_msdk_get_video_format_from_mfx_fourcc (thiz->param.mfx.
      FrameInfo.FourCC);

  if (format == GST_VIDEO_FORMAT_UNKNOWN) {
    GST_WARNING_OBJECT (thiz, "Failed to find a valid video format");
    return FALSE;
  }
#ifndef _WIN32
  /* Get possible modifier before negotiation really happens */
  GstVaDisplay *display =
      (GstVaDisplay *) gst_msdk_context_get_va_display (thiz->context);
  thiz->modifier = gst_va_dmabuf_get_modifier_for_format (display, format,
      VA_SURFACE_ATTRIB_USAGE_HINT_DECODER);
  gst_object_unref (display);
#endif

#if (MFX_VERSION >= 1022)
  /* SFC is triggered (for AVC and HEVC) when default output format is not
   * accepted by downstream or when downstream requests for a smaller
   * resolution (i.e. SFC supports down-scaling)
   * Here we need to do the query twice: the first time uses default color
   * format and bitstream's original size to query peer pad, empty caps
   * means default format and/or size are not accepted by downstream;
   * then we need the second query to decide src caps' color format and size,
   * and let SFC work. */
  if (thiz->param.mfx.CodecId == MFX_CODEC_AVC ||
      thiz->param.mfx.CodecId == MFX_CODEC_HEVC) {
    temp_caps = gst_pad_query_caps (GST_VIDEO_DECODER (thiz)->srcpad, NULL);
    temp_caps = gst_caps_make_writable (temp_caps);

    if (!gst_msdkdec_fixate_format (thiz, temp_caps, format)) {
      GST_ERROR_OBJECT (thiz, "Format is not negotiable");
      gst_caps_unref (temp_caps);
      return FALSE;
    }
    g_value_init (&v_width, G_TYPE_INT);
    g_value_init (&v_height, G_TYPE_INT);

    g_value_set_int (&v_width, width);
    g_value_set_int (&v_height, height);

    gst_caps_set_value (temp_caps, "width", &v_width);
    gst_caps_set_value (temp_caps, "height", &v_height);

    g_value_unset (&v_width);
    g_value_unset (&v_height);

    src_caps = gst_pad_peer_query_caps (GST_VIDEO_DECODER
        (thiz)->srcpad, temp_caps);
    gst_caps_unref (temp_caps);

    if (gst_caps_is_empty (src_caps)) {
      if (!gst_util_fraction_multiply (width, height,
              GST_VIDEO_INFO_PAR_N (&thiz->input_state->info),
              GST_VIDEO_INFO_PAR_D (&thiz->input_state->info),
              &dar_n, &dar_d)) {
        GST_ERROR_OBJECT (thiz, "Error to calculate the output scaled size");
        gst_caps_unref (src_caps);
        return FALSE;
      }

      allowed_caps =
          gst_pad_get_allowed_caps (GST_VIDEO_DECODER (thiz)->srcpad);
      outs = gst_caps_get_structure (allowed_caps, 0);

      if (gst_structure_has_field (outs, "drm-format")) {
        out_format = gst_structure_get_string (outs, "drm-format");
        has_drm_format = TRUE;
      } else {
        out_format = gst_structure_get_string (outs, "format");
      }

      if (out_format) {
        if (has_drm_format) {
#ifndef _WIN32
          guint drm_fourcc =
              gst_video_dma_drm_fourcc_from_string (out_format, NULL);
          format = gst_va_video_format_from_drm_fourcc (drm_fourcc);
#endif
        } else {
          format = gst_video_format_from_string (out_format);
        }
        thiz->sfc = TRUE;
      }

      gst_structure_get_int (outs, "width", &out_width);
      gst_structure_get_int (outs, "height", &out_height);

      if (!out_width && !out_height) {
        out_width = width;
        out_height = height;
      } else {
        /* When user does not set out_width, fill it to fit DAR */
        if (!out_width)
          out_width = gst_util_uint64_scale (out_height, dar_n, dar_d);
        /* When user does not set out_height, fill it to fit DAR */
        if (!out_height)
          out_height = gst_util_uint64_scale (out_width, dar_d, dar_n);

        if (out_width > width || out_height > height)
          goto sfc_error;
        else if (out_width < width || out_height < height) {
          width = out_width;
          height = out_height;
          thiz->sfc = TRUE;
        }
      }
      gst_caps_unref (allowed_caps);
    }
    gst_caps_unref (src_caps);
  }
#endif

  thiz->output_state =
      gst_video_decoder_set_output_state (GST_VIDEO_DECODER (thiz),
      format, width, height, thiz->input_state);

  if (!thiz->output_state)
    return FALSE;

  /* Find allocation width and height */
  alloc_w =
      GST_ROUND_UP_16 (thiz->param.mfx.FrameInfo.Width ? thiz->param.mfx.
      FrameInfo.Width : width);
  alloc_h =
      GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Height ? thiz->param.mfx.
      FrameInfo.Height : height);

  /* Ensure output_state->caps and info have same width and height
   * Also, mandate 32 bit alignment */
  vinfo = thiz->output_state->info;
  if (width == out_width || height == out_height)
    gst_msdk_set_video_alignment (&vinfo, 0, 0, &align);
  else
    gst_msdk_set_video_alignment (&vinfo, alloc_w, alloc_h, &align);
  gst_video_info_align (&vinfo, &align);

  out_caps = gst_video_info_to_caps (&vinfo);

#ifndef _WIN32
  if (pad_accept_memory (thiz, GST_CAPS_FEATURE_MEMORY_VA, &out_caps)) {
    GST_INFO_OBJECT (thiz, "Decoder src pad accepts VAMemory caps");
  } else if (pad_accept_memory (thiz, GST_CAPS_FEATURE_MEMORY_DMABUF,
          &out_caps)) {
    GST_INFO_OBJECT (thiz, "Decoder src pad accepts DMABuf caps");
  }
#else
  if (pad_accept_memory (thiz, GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, &out_caps)) {
    GST_INFO_OBJECT (thiz, "Decoder src pad accepts D3D11Memory caps");
  }
#endif
  thiz->output_state->caps = out_caps;

  if (need_allocation) {
    /* Find allocation width and height */
    width =
        GST_ROUND_UP_16 (thiz->param.mfx.FrameInfo.Width ? thiz->param.mfx.
        FrameInfo.Width : GST_VIDEO_INFO_WIDTH (&thiz->output_state->info));
    height =
        GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Height ? thiz->param.mfx.
        FrameInfo.Height : GST_VIDEO_INFO_HEIGHT (&thiz->output_state->info));

    /* set allocation width and height in allocation_caps,
     * which may or may not be similar to the output_state caps */
    allocation_caps = gst_caps_copy (thiz->output_state->caps);
    gst_caps_set_simple (allocation_caps, "width", G_TYPE_INT, width, "height",
        G_TYPE_INT, height, NULL);
    GST_INFO_OBJECT (thiz, "new alloc caps = %" GST_PTR_FORMAT,
        allocation_caps);
    gst_caps_replace (&thiz->output_state->allocation_caps, allocation_caps);
    gst_caps_unref (allocation_caps);
  } else {
    /* We keep the allocation parameters as it is to avoid pool re-negotiation.
     * For codecs like VP9, dynamic resolution change doesn't require allocation
     * reset if the new video frame resolution is lower than the
     * already configured one */
  }

  return TRUE;

sfc_error:
  GST_ERROR_OBJECT (thiz, "Decoder SFC cannot do up-scaling");
  gst_caps_unref (allowed_caps);
  gst_caps_unref (temp_caps);
  return FALSE;
}

static void
gst_msdkdec_set_latency (GstMsdkDec * thiz)
{
  GstVideoInfo *info = &thiz->input_state->info;
  gint min_delayed_frames;
  GstClockTime latency;

  min_delayed_frames = thiz->async_depth;

  if (info->fps_n) {
    latency = gst_util_uint64_scale_ceil (GST_SECOND * info->fps_d,
        min_delayed_frames, info->fps_n);
  } else {
    /* FIXME: Assume 25fps. This is better than reporting no latency at
     * all and then later failing in live pipelines
     */
    latency = gst_util_uint64_scale_ceil (GST_SECOND * 1,
        min_delayed_frames, 25);
  }

  GST_INFO_OBJECT (thiz,
      "Updating latency to %" GST_TIME_FORMAT " (%d frames)",
      GST_TIME_ARGS (latency), min_delayed_frames);

  gst_video_decoder_set_latency (GST_VIDEO_DECODER (thiz), latency, latency);
}

static gint
_find_msdk_surface (gconstpointer msdk_surface, gconstpointer comp_surface)
{
  GstMsdkSurface *cached_surface = (GstMsdkSurface *) msdk_surface;
  mfxFrameSurface1 *_surface = (mfxFrameSurface1 *) comp_surface;

  return cached_surface ? cached_surface->surface != _surface : -1;
}

static void
finish_task (GstMsdkDec * thiz, MsdkDecTask * task)
{
  GstMsdkSurface *surface = task->surface;
  if (surface) {
    thiz->locked_msdk_surfaces =
        g_list_append (thiz->locked_msdk_surfaces, surface);
  }
  task->sync_point = NULL;
  task->surface = NULL;
  task->decode_only = FALSE;
}

static void
gst_msdkdec_frame_corruption_report (GstMsdkDec * thiz, mfxU16 corruption)
{
  if (!thiz->report_error || !corruption)
    return;

  if (corruption & MFX_CORRUPTION_MINOR)
    GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
        ("[Corruption] Minor corruption detected!"), (NULL));

  if (corruption & MFX_CORRUPTION_MAJOR)
    GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
        ("[Corruption] Major corruption detected!"), (NULL));

  if (corruption & MFX_CORRUPTION_ABSENT_TOP_FIELD)
    GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
        ("[Corruption] Absent top field!"), (NULL));

  if (corruption & MFX_CORRUPTION_ABSENT_BOTTOM_FIELD)
    GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
        ("[Corruption] Absent bottom field!"), (NULL));

  if (corruption & MFX_CORRUPTION_REFERENCE_FRAME)
    GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
        ("[Corruption] Corrupted reference frame!"), (NULL));

  if (corruption & MFX_CORRUPTION_REFERENCE_LIST)
    GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
        ("[Corruption] Corrupted reference list!"), (NULL));
}

static gboolean
_copy_to_sys_mem (GstMsdkDec * thiz, GstMsdkSurface * surface,
    GstVideoCodecFrame * frame)
{
  GstBuffer *buffer = NULL;
  GstVideoFrame src_frame;
  GstVideoFrame dst_frame;
  GstVideoInfo *src_info;
  GstVideoInfo dst_info;

  src_info = &thiz->output_state->info;
  gst_video_info_set_format (&dst_info, GST_VIDEO_INFO_FORMAT (src_info),
      GST_VIDEO_INFO_WIDTH (src_info), GST_VIDEO_INFO_HEIGHT (src_info));

  if (!gst_buffer_pool_is_active (thiz->other_pool) &&
      !gst_buffer_pool_set_active (thiz->other_pool, TRUE)) {
    GST_ERROR_OBJECT (thiz, "Failed to activate buffer pool");
    goto error_active;
  }

  if (gst_buffer_pool_acquire_buffer (thiz->other_pool, &buffer, NULL)
      != GST_FLOW_OK) {
    GST_ERROR ("Failed to acquire buffer from pool");
    goto error;
  }

  if (!gst_video_frame_map (&src_frame, src_info, surface->buf, GST_MAP_READ)) {
    GST_ERROR_OBJECT (thiz, "Failed to map buf to src frame");
    goto error;
  }

  if (!gst_video_frame_map (&dst_frame, &dst_info, buffer, GST_MAP_WRITE)) {
    GST_ERROR_OBJECT (thiz, "Failed to map buf to dst frame");
    gst_video_frame_unmap (&src_frame);
    goto error;
  }

  if (!gst_video_frame_copy (&dst_frame, &src_frame)) {
    GST_ERROR_OBJECT (thiz, "Failed to copy surface data");
    gst_video_frame_unmap (&src_frame);
    gst_video_frame_unmap (&dst_frame);
    goto error;
  }

  frame->output_buffer = buffer;
  gst_video_frame_unmap (&src_frame);
  gst_video_frame_unmap (&dst_frame);

  return TRUE;

error:
  gst_buffer_unref (buffer);
  gst_buffer_pool_set_active (thiz->other_pool, FALSE);
  gst_object_unref (thiz->other_pool);

error_active:
  return FALSE;
}

static GstFlowReturn
gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task)
{
  GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
  GstFlowReturn flow;
  GstVideoCodecFrame *frame;
  GstMsdkSurface *surface;
  mfxStatus status;
  guint64 pts = MFX_TIMESTAMP_UNKNOWN;

  if (G_LIKELY (task->sync_point)) {
    status =
        MFXVideoCORE_SyncOperation (gst_msdk_context_get_session
        (thiz->context), task->sync_point, 300000);
    if (status != MFX_ERR_NONE) {
      GST_ERROR_OBJECT (thiz, "failed to do sync operation");
      return GST_FLOW_ERROR;
    }
  }

  surface = task->surface;
  if (surface) {
    gst_msdkdec_frame_corruption_report (thiz,
        surface->surface->Data.Corrupted);
    GST_DEBUG_OBJECT (thiz, "Decoded MFX TimeStamp: %" G_GUINT64_FORMAT,
        (guint64) surface->surface->Data.TimeStamp);
    pts = surface->surface->Data.TimeStamp;

    if (thiz->param.mfx.CodecId == MFX_CODEC_VP9) {
      /* detect whether the resolution change and negotiate with downstream if so */
      if ((surface->surface->Info.CropW && surface->surface->Info.CropH)
          && ((thiz->output_state->info.width != surface->surface->Info.CropW)
              || (thiz->output_state->info.height !=
                  surface->surface->Info.CropH))) {
        thiz->output_state->info.width = surface->surface->Info.CropW;
        thiz->output_state->info.height = surface->surface->Info.CropH;
        thiz->output_state->caps =
            gst_video_info_to_caps (&thiz->output_state->info);
        if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (thiz))) {
          GST_ERROR_OBJECT (thiz, "Failed to negotiate");
          return GST_FLOW_NOT_NEGOTIATED;
        }
      }
    }
  }

  if (G_LIKELY (task->sync_point || (surface && task->decode_only))) {
    gboolean decode_only = task->decode_only;

    frame = gst_msdkdec_get_oldest_frame (decoder);
    /* align decoder frame list with current decoded position */
    while (frame && MFX_TIME_IS_VALID (pts)
        && GST_CLOCK_TIME_IS_VALID (frame->pts)
        && GST_TO_MFX_TIME (frame->pts) < pts) {
      GST_INFO_OBJECT (thiz, "Discarding frame: %p PTS: %" GST_TIME_FORMAT
          " MFX TimeStamp: %" G_GUINT64_FORMAT,
          frame, GST_TIME_ARGS (frame->pts), GST_TO_MFX_TIME (frame->pts));
      gst_video_decoder_release_frame (decoder, frame);
      frame = gst_msdkdec_get_oldest_frame (decoder);
    }

    if (G_LIKELY (frame)) {
      if (!thiz->do_copy) {
        /* gst_video_decoder_finish_frame will call gst_buffer_make_writable
         * we need this to avoid copy buffer                              */
        GST_MINI_OBJECT_FLAG_SET (surface->buf, GST_MINI_OBJECT_FLAG_LOCKABLE);
        frame->output_buffer = gst_buffer_ref (surface->buf);
      } else {
        /* We need to do the copy from video memory to system memory */
        if (!_copy_to_sys_mem (thiz, surface, frame))
          return GST_FLOW_ERROR;
      }

      GST_DEBUG_OBJECT (thiz, "surface %p TimeStamp: %" G_GUINT64_FORMAT
          " frame %p TimeStamp: %" G_GUINT64_FORMAT,
          surface->surface, (guint64) surface->surface->Data.TimeStamp,
          frame, GST_TO_MFX_TIME (frame->pts));
    }

    finish_task (thiz, task);

    if (!frame)
      return GST_FLOW_FLUSHING;

    if (decode_only)
      GST_VIDEO_CODEC_FRAME_SET_DECODE_ONLY (frame);

    frame->pts = MFX_TO_GST_TIME (pts);
    flow = gst_video_decoder_finish_frame (decoder, frame);
    if (flow == GST_FLOW_ERROR)
      GST_ERROR_OBJECT (thiz, "Failed to finish frame");
    return flow;
  }
  finish_task (thiz, task);

  return GST_FLOW_OK;
}

static gboolean
gst_msdkdec_context_prepare (GstMsdkDec * thiz)
{
  /* Try to find an existing context from the pipeline. This may (indirectly)
   * invoke gst_msdkdec_set_context, which will set thiz->context. */
  if (!gst_msdk_context_find (GST_ELEMENT_CAST (thiz), &thiz->context))
    return FALSE;

  if (thiz->context == thiz->old_context) {
    GST_INFO_OBJECT (thiz, "Found old context %" GST_PTR_FORMAT
        ", reusing as-is", thiz->context);
    return TRUE;
  }

  GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
      thiz->context);

  if (!(gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_DECODER)) {
    gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_DECODER);
    return TRUE;
  }

  /* Found an existing context that's already being used as a decoder, clone
   * the MFX session inside it to create a new one */
  {
    GstMsdkContext *parent_context, *msdk_context;

    GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT " with "
        "joined session", thiz->context);
    parent_context = thiz->context;
    msdk_context = gst_msdk_context_new_with_parent (parent_context);

    if (!msdk_context) {
      GST_ERROR_OBJECT (thiz, "Failed to create a context with parent context "
          "as %" GST_PTR_FORMAT, parent_context);
      return FALSE;
    }

    thiz->context = msdk_context;
    gst_msdk_context_add_shared_async_depth (thiz->context,
        gst_msdk_context_get_shared_async_depth (parent_context));
    gst_object_unref (parent_context);
  }

  return TRUE;
}

static gboolean
gst_msdkdec_start (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  if (!gst_msdkdec_context_prepare (thiz)) {
    if (!gst_msdk_ensure_new_context (GST_ELEMENT_CAST (thiz),
            thiz->hardware, GST_MSDK_JOB_DECODER, &thiz->context))
      return FALSE;
    GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT,
        thiz->context);
  }

  /* Save the current context in a separate field so that we know whether it
   * has changed between calls to _start() */
  gst_object_replace ((GstObject **) & thiz->old_context,
      (GstObject *) thiz->context);

  gst_msdk_context_add_shared_async_depth (thiz->context, thiz->async_depth);

  return TRUE;
}

static gboolean
gst_msdkdec_close (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  gst_clear_object (&thiz->context);

  return TRUE;
}

static gboolean
gst_msdkdec_stop (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  gst_msdkdec_flush (decoder);

  if (thiz->input_state) {
    gst_video_codec_state_unref (thiz->input_state);
    thiz->input_state = NULL;
  }
  g_clear_pointer (&thiz->output_state, gst_video_codec_state_unref);
  if (thiz->pool) {
    gst_object_unref (thiz->pool);
    thiz->pool = NULL;
  }
  if (thiz->other_pool) {
    gst_object_unref (thiz->other_pool);
    thiz->other_pool = NULL;
  }
  gst_object_replace ((GstObject **) & thiz->alloc_pool, NULL);

  gst_video_info_init (&thiz->non_msdk_pool_info);

  gst_msdkdec_close_decoder (thiz, TRUE);
  return TRUE;
}

static gboolean
gst_msdkdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  if (thiz->input_state) {
    /* mark for re-negotiation if display resolution or any other video info
     * changes like framerate. */
    if (!gst_video_info_is_equal (&thiz->input_state->info, &state->info)) {
      GST_INFO_OBJECT (thiz, "Schedule renegotiation as video info changed");
      thiz->do_renego = TRUE;
    }
    gst_video_codec_state_unref (thiz->input_state);
  }
  thiz->input_state = gst_video_codec_state_ref (state);

  /* we don't set output state here to avoid caching of mismatched
   * video information if there is dynamic resolution change in the stream.
   * All negotiation code is consolidated in gst_msdkdec_negotiate() and
   * this will be invoked from handle_frame() */

  gst_msdkdec_set_latency (thiz);
  return TRUE;
}

static void
release_msdk_surfaces (GstMsdkDec * thiz)
{
  GList *l;
  GstMsdkSurface *surface;
  gint locked = 0;
  gst_msdkdec_free_unlocked_msdk_surfaces (thiz, FALSE);

  for (l = thiz->locked_msdk_surfaces; l; l = l->next) {
    surface = (GstMsdkSurface *) l->data;
    free_surface (surface);
    locked++;
  }
  if (locked)
    GST_ERROR_OBJECT (thiz, "msdk still locked %d surfaces", locked);
  g_list_free (thiz->locked_msdk_surfaces);
  thiz->locked_msdk_surfaces = NULL;
}

/* This will get invoked in the following situations:
 * 1: beginning of the stream, which requires initialization (== complete reset)
 * 2: upstream notified a resolution change and set do_renego to TRUE.
 *    new resolution may or may not requires full reset
 * 3: upstream failed to notify the resolution change but
 *    msdk detected the change (eg: vp9 stream in ivf elementary form
 *     with varying resolution frames).
 *
 * for any input configuration change, we deal with notification
 * from upstream and also use msdk APIs to handle the parameter initialization
 * efficiently
 */
static gboolean
gst_msdkdec_negotiate (GstMsdkDec * thiz, gboolean hard_reset)
{
  GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
  GST_DEBUG_OBJECT (thiz,
      "Start Negotiating caps, pool and Init the msdk decdoer subsystem");

  if (hard_reset) {
    /* Retrieve any pending frames and push them downstream */
    if (gst_msdkdec_drain (GST_VIDEO_DECODER (thiz)) != GST_FLOW_OK)
      goto error_drain;

    /* This will initiate the allocation query which will help to flush
     * all the pending buffers in the pipeline so that we can stop
     * the active bufferpool and safely invoke gst_msdk_frame_free() */
    if (thiz->initialized) {
      GstCaps *caps = gst_pad_get_current_caps (decoder->srcpad);
      GstQuery *query = NULL;
      if (caps) {
        query = gst_query_new_allocation (caps, FALSE);
        gst_pad_peer_query (decoder->srcpad, query);
        gst_query_unref (query);
        gst_caps_unref (caps);
      }
    }

    /* De-initialize the decoder if it is already active */
    /* Do not reset the mfxVideoParam since it already
     * has the required parameters for new session decode */
    gst_msdkdec_close_decoder (thiz, FALSE);

    /* request for pool re-negotiation by setting do_realloc */
    thiz->do_realloc = TRUE;
  }

  /* At this point all pending frames (if there are any) are pushed downstream
   * and we are ready to negotiate the output caps */
  if (!gst_msdkdec_set_src_caps (thiz, hard_reset))
    return FALSE;

  /* this will initiate the allocation query, we create the
   * bufferpool in decide_allocation in order to account
   * for the downstream min_buffer requirement
   * Required initializations for MediaSDK operations
   * will all be initialized from decide_allocation after considering
   * some of the downstream requirements */
  if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (thiz)))
    goto error_negotiate;

  thiz->do_renego = FALSE;
  thiz->do_realloc = FALSE;

  return TRUE;

error_drain:
  GST_ERROR_OBJECT (thiz, "Failed to Drain the queued decoded frames");
  return FALSE;

error_negotiate:
  GST_ERROR_OBJECT (thiz, "Failed to re-negotiate");
  return FALSE;
}

static inline gboolean
find_msdk_surface (GstMsdkDec * thiz, MsdkDecTask * task,
    mfxFrameSurface1 * out_surface)
{
  GList *l;
  task->surface = NULL;

  if (!out_surface)
    return TRUE;
  l = g_list_find_custom (thiz->locked_msdk_surfaces, out_surface,
      _find_msdk_surface);
  if (!l) {
    GST_ERROR_OBJECT (thiz, "msdk return an invalid surface %p", out_surface);
    return FALSE;
  }
  task->surface = (GstMsdkSurface *) l->data;
  thiz->locked_msdk_surfaces =
      g_list_delete_link (thiz->locked_msdk_surfaces, l);
  return TRUE;
}

static void
gst_msdkdec_error_report (GstMsdkDec * thiz)
{
  if (!thiz->report_error)
    return;

#if (MFX_VERSION >= 1025)
  else {
    if (thiz->error_report.ErrorTypes & MFX_ERROR_SPS)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error] SPS Error detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_PPS)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error] PPS Error detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_SLICEHEADER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error] SliceHeader Error detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_FRAME_GAP)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error] Frame Gap Error detected!"), (NULL));

#ifdef ONEVPL_EXPERIMENTAL
    if (thiz->error_report.ErrorTypes & MFX_ERROR_JPEG_APP0_MARKER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error]  APP0 unknown marker detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_JPEG_APP14_MARKER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error]  APP14 unknown marker detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_JPEG_DQT_MARKER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error]  DQT unknown marker detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_JPEG_SOF0_MARKER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error]  SOF0 unknown marker detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_JPEG_DHT_MARKER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error]  DHT unknown marker detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_JPEG_DRI_MARKER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error]  DRI unknown marker detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_JPEG_SOS_MARKER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error]  SOS unknown marker detected!"), (NULL));

    if (thiz->error_report.ErrorTypes & MFX_ERROR_JPEG_UNKNOWN_MARKER)
      GST_ELEMENT_WARNING (thiz, STREAM, DECODE,
          ("[Error]  Error unknown marker detected!"), (NULL));
#endif
  }
#endif
}

static GstFlowReturn
gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
  GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
  GstFlowReturn flow;
  GstBuffer *input_buffer = NULL;
  GstVideoInfo alloc_info;
  MsdkDecTask *task = NULL;
  mfxBitstream bitstream;
  GstMsdkSurface *surface = NULL;
  mfxFrameSurface1 *out_surface = NULL;
  mfxSession session;
  mfxStatus status;
  GstMapInfo map_info;
  guint i, retry_err_incompatible = 0;
  gsize data_size;
  gboolean hard_reset = FALSE;
  GstClockTime pts = GST_CLOCK_TIME_NONE;

  /* configure the subclass in order to fill the CodecID field of
   * mfxVideoParam and also to load the PluginID for some of the
   * codecs which is mandatory to invoke the
   * MFXVideoDECODE_DecodeHeader API.
   *
   * For non packetized formats (currently only vc1), there
   * could be headers received as codec_data which are not available
   * instream and in that case subclass implementation will
   * push it to the internal adapter. We invoke the subclass configure
   * well early to make sure the codec_data received has been correctly
   * pushed to the adapter by the subclasses before doing
   * the DecodeHeader() later on
   */
  if (!thiz->initialized || thiz->do_renego) {
    /* Clear the internal adapter in re-negotiation for non-packetized
     * formats */
    if (!gst_video_decoder_get_packetized (decoder))
      gst_adapter_clear (thiz->adapter);

    if (!klass->configure || !klass->configure (thiz)) {
      flow = GST_FLOW_OK;
      goto error;
    }
  }

  /* Current frame-codec could be pushed and released before this
   * function ends -- because msdkdec pushes the oldest frame,
   * according its PTS, and it could be this very same frame-codec
   * among others pending frame-codecs.
   *
   * Instead of copying the input data into the mfxBitstream, let's
   * keep an extra reference to frame-codec's input buffer */
  input_buffer = gst_buffer_ref (frame->input_buffer);
  if (!gst_buffer_map (input_buffer, &map_info, GST_MAP_READ)) {
    gst_buffer_unref (input_buffer);
    return GST_FLOW_ERROR;
  }

  memset (&bitstream, 0, sizeof (bitstream));

  /* Add extended buffers */
  if (thiz->num_bs_extra_params) {
    bitstream.NumExtParam = thiz->num_bs_extra_params;
    bitstream.ExtParam = thiz->bs_extra_params;
  }

  if (gst_video_decoder_get_packetized (decoder)) {
    /* Packetized stream: we prefer to have a parser as a connected upstream
     * element to the decoder */
    pts = frame->pts;
    bitstream.Data = map_info.data;
    bitstream.DataLength = map_info.size;
    bitstream.MaxLength = map_info.size;
    bitstream.TimeStamp = GST_TO_MFX_TIME (pts);

    /*
     * MFX_BITSTREAM_COMPLETE_FRAME was removed since commit df59db9, however
     * some customers still use DecodedOrder (deprecated in msdk-2017 version)
     * for low-latency streaming of non-b-frame encoded streams, which needs to
     * output the frame at once, so add it back for this case
     */
    if (thiz->param.mfx.DecodedOrder == GST_MSDKDEC_OUTPUT_ORDER_DECODE)
      bitstream.DataFlag |= MFX_BITSTREAM_COMPLETE_FRAME;
  } else {
    /* Non packetized streams: eg: vc1 advanced profile with per buffer bdu */
    gst_adapter_push (thiz->adapter, gst_buffer_ref (input_buffer));
    data_size = gst_adapter_available (thiz->adapter);

    bitstream.Data = (mfxU8 *) gst_adapter_map (thiz->adapter, data_size);
    bitstream.DataLength = (mfxU32) data_size;
    bitstream.MaxLength = bitstream.DataLength;
    bitstream.TimeStamp = GST_TO_MFX_TIME (pts);
  }
  GST_DEBUG_OBJECT (thiz,
      "mfxBitStream=> DataLength:%d DataOffset:%d MaxLength:%d "
      "PTS: %" GST_TIME_FORMAT " MFX TimeStamp %" G_GUINT64_FORMAT,
      bitstream.DataLength, bitstream.DataOffset, bitstream.MaxLength,
      GST_TIME_ARGS (pts), (guint64) bitstream.TimeStamp);

  session = gst_msdk_context_get_session (thiz->context);

  if (!thiz->initialized || thiz->do_renego) {

    /* gstreamer caps will not provide all the necessary parameters
     * required for optimal decode configuration. For example: the required number
     * of surfaces to be allocated can be calculated based on H264 SEI header
     * and this information can't be retrieved from the negotiated caps.
     * So instead of introducing a codecparser dependency to parse the headers
     * inside msdk plugin, we simply use the mfx APIs to extract header information */
#if (MFX_VERSION >= 1025)
    if (thiz->report_error)
      thiz->error_report.ErrorTypes = 0;
#endif

    status = MFXVideoDECODE_DecodeHeader (session, &bitstream, &thiz->param);
    GST_DEBUG_OBJECT (decoder, "DecodeHeader => %d", status);
    gst_msdkdec_error_report (thiz);

    if (status == MFX_ERR_MORE_DATA) {
      flow = GST_FLOW_OK;
      goto done;
    }

    if (!klass->post_configure (thiz)) {
      flow = GST_FLOW_ERROR;
      goto error;
    }

    if (!thiz->initialized)
      hard_reset = TRUE;
    else {
      if (thiz->output_state) {
        if (thiz->output_state->allocation_caps) {
          if (!gst_msdkcaps_video_info_from_caps
              (thiz->output_state->allocation_caps, &alloc_info, NULL)) {
            GST_ERROR_OBJECT (thiz, "Failed to get video info from caps");
            flow = GST_FLOW_ERROR;
            goto error;
          }

          /* Check whether we need complete reset for dynamic resolution change */
          if (thiz->param.mfx.FrameInfo.Width >
              GST_VIDEO_INFO_WIDTH (&alloc_info)
              || thiz->param.mfx.FrameInfo.Height >
              GST_VIDEO_INFO_HEIGHT (&alloc_info))
            hard_reset = TRUE;
        }
      }

    }

    /* if subclass requested for the force reset */
    if (thiz->force_reset_on_res_change)
      hard_reset = TRUE;

    if (!gst_msdkdec_negotiate (thiz, hard_reset)) {
      GST_ELEMENT_ERROR (thiz, CORE, NEGOTIATION,
          ("Could not negotiate the stream"), (NULL));
      flow = GST_FLOW_ERROR;
      goto error;
    }
  }

  /* gst_msdkdec_handle_frame owns one ref on input argument |frame|. At this
   * point this frame is not used so just unref it right away.
   * gst_msdkdec_finish_task is fetching the frames itself.  */
  gst_video_codec_frame_unref (frame);
  frame = NULL;
  for (;;) {
    task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
    flow = gst_msdkdec_finish_task (thiz, task);
    if (flow != GST_FLOW_OK) {
      if (flow == GST_FLOW_ERROR)
        GST_ERROR_OBJECT (thiz, "Failed to finish a task");
      goto error;
    }
    if (!surface) {
      surface = allocate_output_surface (thiz);
      if (!surface) {
        /* Can't get a surface for some reason; finish tasks, then see if
           a surface becomes available. */
        for (i = 0; i < thiz->tasks->len - 1; i++) {
          thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
          task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
          flow = gst_msdkdec_finish_task (thiz, task);
          if (flow != GST_FLOW_OK)
            goto error;
          surface = allocate_output_surface (thiz);
          if (surface)
            break;
        }
        if (!surface) {
          GST_ERROR_OBJECT (thiz, "Couldn't get a surface");
          flow = GST_FLOW_ERROR;
          goto error;
        }
      }
    }
#if (MFX_VERSION >= 1025)
    if (thiz->report_error)
      thiz->error_report.ErrorTypes = 0;
#endif

    status =
        MFXVideoDECODE_DecodeFrameAsync (session, &bitstream, surface->surface,
        &out_surface, &task->sync_point);

    if (!find_msdk_surface (thiz, task, out_surface)) {
      flow = GST_FLOW_ERROR;
      goto done;
    }

    GST_DEBUG_OBJECT (decoder, "DecodeFrameAsync => %d", status);
    gst_msdkdec_error_report (thiz);

    /* media-sdk requires complete reset since the surface is inadequate
     * for further decoding */
    if (status == MFX_ERR_INCOMPATIBLE_VIDEO_PARAM &&
        retry_err_incompatible++ < 1) {
      /* MFX_ERR_INCOMPATIBLE_VIDEO_PARAM means the current mfx surface is not
       * suitable for the current frame. Call MFXVideoDECODE_DecodeHeader to get
       * the current frame size, then do memory re-allocation, otherwise
       * MFXVideoDECODE_DecodeFrameAsync will still fail on next call */
#if (MFX_VERSION >= 1025)
      if (thiz->report_error)
        thiz->error_report.ErrorTypes = 0;
#endif
      status = MFXVideoDECODE_DecodeHeader (session, &bitstream, &thiz->param);
      GST_DEBUG_OBJECT (decoder, "DecodeHeader => %d", status);
      gst_msdkdec_error_report (thiz);

      if (status == MFX_ERR_MORE_DATA) {
        flow = GST_FLOW_OK;
        goto done;
      }

      /* Requires memory re-allocation, do a hard reset */
      if (!gst_msdkdec_negotiate (thiz, TRUE))
        goto error;

      /* The current surface is freed when doing a hard reset; a new surface is
       * required for the new resolution */
      surface = NULL;
      continue;
    }

    retry_err_incompatible = 0;

    if (G_LIKELY (status == MFX_ERR_NONE)
        || (status == MFX_WRN_VIDEO_PARAM_CHANGED)) {
      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;

      if (surface->surface->Data.Locked > 0)
        surface = NULL;

      if (bitstream.DataLength == 0) {
        flow = GST_FLOW_OK;

        /* Don't release it if the current surface is in use */
        if (surface && task->surface->surface == surface->surface)
          surface = NULL;

        break;
      }
    } else if (status == MFX_ERR_MORE_DATA) {
      if (task->surface) {
        task->decode_only = TRUE;
        thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
      }

      if (surface->surface->Data.Locked > 0)
        surface = NULL;
      flow = GST_VIDEO_DECODER_FLOW_NEED_DATA;
      break;
    } else if (status == MFX_ERR_MORE_SURFACE) {
      surface = NULL;
      continue;
    } else if (status == MFX_WRN_DEVICE_BUSY) {
      /* If device is busy, wait 1ms and retry, as per MSDK's recommendation */
      g_usleep (1000);

      if (surface->surface->Data.Locked > 0)
        surface = NULL;

      /* If the current surface is still busy, we should do sync operation,
       * then try to decode again
       */
      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
    } else if (status < MFX_ERR_NONE) {
      GST_ERROR_OBJECT (thiz, "DecodeFrameAsync failed (%s)",
          msdk_status_to_string (status));
      flow = GST_FLOW_ERROR;
      break;
    }
  }

  if (!gst_video_decoder_get_packetized (decoder)) {
    /* flush out the data which has already been consumed by msdk */
    gst_adapter_flush (thiz->adapter, bitstream.DataOffset);
  }

  /*
   * DecodedOrder was deprecated in msdk-2017 version, but some
   * customers still using this for low-latency streaming of non-b-frame
   * encoded streams, which needs to output the frame at once
   */
  if (thiz->param.mfx.DecodedOrder == GST_MSDKDEC_OUTPUT_ORDER_DECODE)
    gst_msdkdec_finish_task (thiz, task);

done:
  gst_buffer_unmap (input_buffer, &map_info);
  gst_buffer_unref (input_buffer);
  return flow;

error:
  if (input_buffer) {
    gst_buffer_unmap (input_buffer, &map_info);
    gst_buffer_unref (input_buffer);
  }
  if (frame)
    gst_video_decoder_drop_frame (decoder, frame);

  return flow;
}

static GstFlowReturn
gst_msdkdec_parse (GstVideoDecoder * decoder, GstVideoCodecFrame * frame,
    GstAdapter * adapter, gboolean at_eos)
{
  gsize size;
  GstFlowReturn ret;
  GstBuffer *buffer;
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  /* Don't parse the input buffer indeed, it will invoke
   * gst_msdkdec_handle_frame to handle the input buffer */
  size = gst_adapter_available (adapter);
  gst_video_decoder_add_to_frame (decoder, size);
  ret = gst_video_decoder_have_frame (decoder);
  size = gst_adapter_available (thiz->adapter);

  if (size) {
    /* The base class will set up a new frame for parsing as
     * soon as there is valid data in the buffer */
    buffer = gst_adapter_get_buffer (thiz->adapter, size);
    gst_adapter_flush (thiz->adapter, size);
    gst_adapter_push (adapter, buffer);
  }

  return ret;
}

#ifndef _WIN32
static GstBufferPool *
gst_msdk_create_va_pool (GstMsdkDec * thiz, GstVideoInfo * info,
    guint num_buffers)
{
  GstBufferPool *pool = NULL;
  GstAllocator *allocator;
  GArray *formats = NULL;
  GstAllocationParams alloc_params = { 0, 31, 0, 0 };
  GstVaDisplay *display = NULL;
  GstCaps *caps = NULL;

  display = (GstVaDisplay *) gst_msdk_context_get_va_display (thiz->context);

  if (thiz->use_dmabuf)
    allocator = gst_va_dmabuf_allocator_new (display);
  else {
    formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
    g_array_append_val (formats, GST_VIDEO_INFO_FORMAT (info));
    allocator = gst_va_allocator_new (display, formats);
  }

  gst_object_unref (display);

  if (!allocator) {
    GST_ERROR_OBJECT (thiz, "Failed to create allocator");
    if (formats)
      g_array_unref (formats);
    return NULL;
  }

  if (thiz->use_dmabuf && thiz->modifier != DRM_FORMAT_MOD_INVALID) {
    caps = gst_msdkcaps_video_info_to_drm_caps (info, thiz->modifier);
    gst_caps_set_features (caps, 0,
        gst_caps_features_new_static_str (GST_CAPS_FEATURE_MEMORY_DMABUF,
            NULL));
  } else {
    caps = gst_video_info_to_caps (info);
  }

  pool = gst_va_pool_new_with_config (caps, num_buffers, num_buffers,
      VA_SURFACE_ATTRIB_USAGE_HINT_DECODER, GST_VA_FEATURE_AUTO,
      allocator, &alloc_params);

  gst_object_unref (allocator);
  gst_caps_unref (caps);
  GST_LOG_OBJECT (thiz, "Creating va pool");
  return pool;
}
#else
static GstBufferPool *
gst_msdk_create_d3d11_pool (GstMsdkDec * thiz, GstVideoInfo * info,
    guint num_buffers)
{
  GstBufferPool *pool = NULL;
  GstD3D11Device *device;
  GstStructure *config;
  GstD3D11AllocationParams *params;

  device = gst_msdk_context_get_d3d11_device (thiz->context);

  pool = gst_d3d11_buffer_pool_new (device);
  config = gst_buffer_pool_get_config (pool);
  params = gst_d3d11_allocation_params_new (device, info,
      GST_D3D11_ALLOCATION_FLAG_DEFAULT,
      D3D11_BIND_DECODER | D3D11_BIND_SHADER_RESOURCE, 0);

  gst_buffer_pool_config_set_d3d11_allocation_params (config, params);
  gst_d3d11_allocation_params_free (params);

  return pool;
}
#endif

static GstBufferPool *
gst_msdkdec_create_buffer_pool (GstMsdkDec * thiz, GstVideoInfo * info,
    guint num_buffers)
{
  GstBufferPool *pool = NULL;
  GstStructure *config;
  GstCaps *caps;
  GstVideoAlignment align;
  GstVideoInfo vinfo = *info;

  gst_msdk_set_video_alignment (&vinfo, 0, 0, &align);
  gst_video_info_align (&vinfo, &align);

  if (thiz->do_copy)
    pool = gst_video_buffer_pool_new ();
  else {
#ifndef _WIN32
    pool = gst_msdk_create_va_pool (thiz, &vinfo, num_buffers);
#else
    pool = gst_msdk_create_d3d11_pool (thiz, &vinfo, num_buffers);
#endif
  }

  if (!pool)
    goto error_no_pool;

#ifndef _WIN32
  if (thiz->use_dmabuf) {
    caps = gst_msdkcaps_video_info_to_drm_caps (&vinfo, thiz->modifier);
    gst_caps_set_features (caps, 0,
        gst_caps_features_new_static_str (GST_CAPS_FEATURE_MEMORY_DMABUF,
            NULL));
  } else {
    caps = gst_video_info_to_caps (&vinfo);
  }
#else
  caps = gst_video_info_to_caps (&vinfo);
#endif

  config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));
  gst_buffer_pool_config_set_params (config, caps,
      GST_VIDEO_INFO_SIZE (&vinfo), num_buffers, 0);
  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
  gst_buffer_pool_config_add_option (config,
      GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
  gst_buffer_pool_config_set_video_alignment (config, &align);

  gst_caps_unref (caps);

  if (!gst_buffer_pool_set_config (pool, config))
    goto error_pool_config;

  return pool;

error_no_pool:
  {
    GST_INFO_OBJECT (thiz, "Failed to create bufferpool");
    return NULL;
  }
error_pool_config:
  {
    GST_INFO_OBJECT (thiz, "Failed to set config");
    gst_object_unref (pool);
    return NULL;
  }
}

static gboolean
gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
  GstBufferPool *pool = NULL;
  GstStructure *pool_config = NULL;
  GstCaps *pool_caps = NULL;
  guint size = 0;
  guint min_buffers = 0;
  guint max_buffers = 0;
  gboolean has_videometa, has_video_alignment;

  GstAllocator *allocator = NULL;
  GstAllocationParams params;
  gboolean update_pool = FALSE;
  gboolean update_allocator = FALSE;

  GstVideoInfo vinfo;

  if (!thiz->param.mfx.FrameInfo.Width || !thiz->param.mfx.FrameInfo.Height)
    return FALSE;

  gst_query_parse_allocation (query, &pool_caps, NULL);

  if (!gst_msdkcaps_video_info_from_caps (pool_caps, &vinfo, NULL))
    goto failed_to_parse_caps;

  if (gst_query_get_n_allocation_params (query) > 0) {
    gst_query_parse_nth_allocation_param (query, 0, &allocator, &params);
    update_allocator = TRUE;
  }
  if (!allocator)
    gst_allocation_params_init (&params);

  if (gst_query_get_n_allocation_pools (query) > 0) {
    gst_query_parse_nth_allocation_pool (query,
        0, &pool, &size, &min_buffers, &max_buffers);
    update_pool = TRUE;
  }
  if (!pool)
    pool = gst_video_buffer_pool_new ();

  size = MAX (size, vinfo.size);
  pool_config = gst_buffer_pool_get_config (pool);
  gst_buffer_pool_config_set_params (pool_config,
      pool_caps, size, min_buffers, max_buffers);
  gst_buffer_pool_config_set_allocator (pool_config, allocator, &params);

  if (!gst_buffer_pool_set_config (pool, pool_config)) {
    pool_config = gst_buffer_pool_get_config (pool);

    if (!gst_buffer_pool_config_validate_params (pool_config,
            pool_caps, size, min_buffers, max_buffers)) {
      gst_object_unref (pool);
      pool = gst_video_buffer_pool_new ();
      gst_buffer_pool_config_set_params (pool_config,
          pool_caps, size, min_buffers, max_buffers);
      gst_buffer_pool_config_set_allocator (pool_config, allocator, &params);
    }

    if (!gst_buffer_pool_set_config (pool, pool_config))
      return FALSE;
  }

  if (update_allocator)
    gst_query_set_nth_allocation_param (query, 0, allocator, &params);
  else
    gst_query_add_allocation_param (query, allocator, &params);

  if (update_pool)
    gst_query_set_nth_allocation_pool (query,
        0, pool, size, min_buffers, max_buffers);
  else
    gst_query_add_allocation_pool (query, pool, size, min_buffers, max_buffers);

  /* Get the buffer pool config decided on by the base class. The base
     class ensures that there will always be at least a 0th pool in
     the query. */

  pool_config = gst_buffer_pool_get_config (pool);

  has_videometa = gst_query_find_allocation_meta
      (query, GST_VIDEO_META_API_TYPE, NULL);
  has_video_alignment = gst_buffer_pool_has_option
      (pool, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);

  /* Get the caps of pool and increase the min and max buffers by async_depth.
   * We will always have that number of decode operations in-flight */
  min_buffers += thiz->async_depth;
  if (max_buffers)
    max_buffers += thiz->async_depth;

  /* increase the min_buffers by 1 for smooth display in render pipeline */
  min_buffers += 1;

  /* this will get updated with msdk requirement */
  thiz->min_prealloc_buffers = min_buffers;

  if (gst_msdkcaps_has_feature (pool_caps, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
    thiz->use_dmabuf = TRUE;
  }
  /* Decoder always use its own pool. So we create a pool if msdk APIs
   * previously requested for allocation (do_realloc = TRUE) */
  if (thiz->do_realloc || !thiz->pool) {
    gst_clear_object (&thiz->pool);
    GST_INFO_OBJECT (decoder, "create new MSDK bufferpool");
    thiz->pool = gst_msdkdec_create_buffer_pool
        (thiz, &thiz->output_state->info, min_buffers);
    if (!thiz->pool) {
      GST_ERROR_OBJECT (decoder, "failed to create new pool");
      goto failed_to_create_pool;
    }
  }
#ifndef _WIN32
  if (!(GST_IS_VA_ALLOCATOR (allocator) ||
          GST_IS_VA_DMABUF_ALLOCATOR (allocator)))
    thiz->ds_has_known_allocator = FALSE;
#else
  if (!GST_IS_D3D11_BUFFER_POOL (pool)) {
    thiz->ds_has_known_allocator = FALSE;
  }
#endif
  if (allocator)
    gst_object_unref (allocator);

  /* If downstream supports video meta and video alignment, or downstream
   * doesn't have known allocator (known allocator refers to va allocator
   * or d3d allocator), we replace with our own bufferpool and use it.
   */
  if ((has_videometa && has_video_alignment)
      || !thiz->ds_has_known_allocator) {
    GstStructure *config;

    gst_structure_free (pool_config);
    gst_object_unref (pool);

    pool = gst_object_ref (thiz->pool);

    /* Set the allocator of new msdk bufferpool */
    config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));

    if (gst_buffer_pool_config_get_allocator (config, &allocator, NULL))
      gst_query_set_nth_allocation_param (query, 0, allocator, NULL);
    gst_structure_free (config);
  } else {
    /* When downstream doesn't have videometa or alignment support,
     * or downstream pool is va/d3d pool,we will use downstream pool
     * and keep decoder's own pool as side-pool.
     */
    GST_INFO_OBJECT (decoder, "Keep MSDK bufferpool as a side-pool");

    /* Update params to downstream's pool */
    gst_buffer_pool_config_set_params (pool_config, pool_caps, size,
        min_buffers, max_buffers);
    if (!gst_buffer_pool_set_config (pool, pool_config))
      goto error_set_config;

    thiz->non_msdk_pool_info = vinfo;

    /* update width and height with actual negotiated values */
    GST_VIDEO_INFO_WIDTH (&thiz->non_msdk_pool_info) =
        GST_VIDEO_INFO_WIDTH (&thiz->output_state->info);
    GST_VIDEO_INFO_HEIGHT (&thiz->non_msdk_pool_info) =
        GST_VIDEO_INFO_HEIGHT (&thiz->output_state->info);
  }

  gst_msdk_context_set_alloc_pool (thiz->context, pool);

  /* Initialize MSDK decoder before new bufferpool tries to alloc each buffer,
   * which requires information about frame allocation.
   * No effect if already initialized.
   */
  if (!gst_msdkdec_init_decoder (thiz))
    return FALSE;

  /* get the updated min_buffers, which account for the msdk requirement as well */
  min_buffers = thiz->min_prealloc_buffers;

  if (!has_videometa && !thiz->ds_has_known_allocator
      && gst_msdkcaps_has_feature (pool_caps,
          GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY)) {
    /* We need to create other pool with system memory for copy use under conditions:
     * (1) downstream has no videometa; (2) downstream allocator is unknown;
     * (3) negotiated caps is raw.
     */
    thiz->do_copy = TRUE;
    thiz->other_pool =
        gst_msdkdec_create_buffer_pool (thiz, &thiz->output_state->info,
        min_buffers);
  }

  gst_query_set_nth_allocation_pool (query, 0, pool, size, min_buffers,
      max_buffers);

  gst_object_unref (pool);

  return TRUE;

failed_to_parse_caps:
  GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
  return FALSE;

failed_to_create_pool:
  GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
  gst_object_unref (pool);
  return FALSE;

error_set_config:
  GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
  gst_object_unref (pool);
  return FALSE;
}

static GstFlowReturn
gst_msdkdec_drain (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
  GstFlowReturn flow;
  MsdkDecTask *task;
  GstMsdkSurface *surface = NULL;
  mfxFrameSurface1 *out_surface;
  mfxSession session;
  mfxStatus status;
  guint i;

  if (!thiz->initialized)
    return GST_FLOW_OK;
  session = gst_msdk_context_get_session (thiz->context);

  for (;;) {
    task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
    if ((flow = gst_msdkdec_finish_task (thiz, task)) != GST_FLOW_OK) {
      if (flow != GST_FLOW_FLUSHING)
        GST_WARNING_OBJECT (decoder,
            "failed to finish the task %p, but keep draining for the remaining frames",
            task);
    }

    if (!surface) {
      surface = allocate_output_surface (thiz);
      if (!surface)
        return GST_FLOW_ERROR;
    }
#if (MFX_VERSION >= 1025)
    if (thiz->report_error)
      thiz->error_report.ErrorTypes = 0;
#endif

    status =
        MFXVideoDECODE_DecodeFrameAsync (session, NULL, surface->surface,
        &out_surface, &task->sync_point);

    if (!find_msdk_surface (thiz, task, out_surface)) {
      return GST_FLOW_ERROR;
    }

    GST_DEBUG_OBJECT (decoder, "DecodeFrameAsync => %d", status);
    gst_msdkdec_error_report (thiz);

    if (G_LIKELY (status == MFX_ERR_NONE)) {
      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
      surface = NULL;
    } else if (status == MFX_WRN_VIDEO_PARAM_CHANGED) {
      continue;
    } else if (status == MFX_WRN_DEVICE_BUSY) {
      /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */
      g_usleep (1000);

      /* If the current surface is still busy, we should do sync operation,
       * then try to decode again
       */
      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
    } else if (status == MFX_ERR_MORE_DATA) {
      break;
    } else if (status == MFX_ERR_MORE_SURFACE) {
      surface = NULL;
      continue;
    } else if (status < MFX_ERR_NONE)
      return GST_FLOW_ERROR;
  }

  for (i = 0; i < thiz->tasks->len; i++) {
    task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
    gst_msdkdec_finish_task (thiz, task);
    thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
  }

  return GST_FLOW_OK;
}

static gboolean
gst_msdkdec_flush (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
  GstFlowReturn ret;

  ret = gst_msdkdec_drain (GST_VIDEO_DECODER_CAST (thiz));

  return ret == GST_FLOW_OK;
}

static GstFlowReturn
gst_msdkdec_finish (GstVideoDecoder * decoder)
{
  return gst_msdkdec_drain (decoder);
}

static gboolean
gst_msdkdec_query (GstVideoDecoder * decoder, GstQuery * query,
    GstPadDirection dir)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
  gboolean ret = FALSE;

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_CONTEXT:{
      GstMsdkContext *msdk_context = NULL;

      gst_object_replace ((GstObject **) & msdk_context,
          (GstObject *) thiz->context);
      ret = gst_msdk_handle_context_query (GST_ELEMENT_CAST (decoder),
          query, msdk_context);
      gst_clear_object (&msdk_context);
      break;
    }
    default:
      if (dir == GST_PAD_SRC) {
        ret =
            GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query);
      } else {
        ret =
            GST_VIDEO_DECODER_CLASS (parent_class)->sink_query (decoder, query);
      }
      break;
  }

  return ret;
}

static gboolean
gst_msdkdec_src_query (GstVideoDecoder * decoder, GstQuery * query)
{
  return gst_msdkdec_query (decoder, query, GST_PAD_SRC);
}

static gboolean
gst_msdkdec_sink_query (GstVideoDecoder * decoder, GstQuery * query)
{
  return gst_msdkdec_query (decoder, query, GST_PAD_SINK);
}

static void
gst_msdkdec_set_property (GObject * object, guint prop_id, const GValue * value,
    GParamSpec * pspec)
{
  GstMsdkDec *thiz = GST_MSDKDEC (object);
  GstState state;

  GST_OBJECT_LOCK (thiz);

  state = GST_STATE (thiz);
  if ((state != GST_STATE_READY && state != GST_STATE_NULL) &&
      !(pspec->flags & GST_PARAM_MUTABLE_PLAYING))
    goto wrong_state;

  switch (prop_id) {
    case GST_MSDKDEC_PROP_HARDWARE:
      thiz->hardware = g_value_get_boolean (value);
      break;
    case GST_MSDKDEC_PROP_ASYNC_DEPTH:
      thiz->async_depth = g_value_get_uint (value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
  GST_OBJECT_UNLOCK (thiz);
  return;

  /* ERROR */
wrong_state:
  {
    GST_WARNING_OBJECT (thiz, "setting property in wrong state");
    GST_OBJECT_UNLOCK (thiz);
  }
}

static void
gst_msdkdec_get_property (GObject * object, guint prop_id, GValue * value,
    GParamSpec * pspec)
{
  GstMsdkDec *thiz = GST_MSDKDEC (object);

  GST_OBJECT_LOCK (thiz);
  switch (prop_id) {
    case GST_MSDKDEC_PROP_HARDWARE:
      g_value_set_boolean (value, thiz->hardware);
      break;
    case GST_MSDKDEC_PROP_ASYNC_DEPTH:
      g_value_set_uint (value, thiz->async_depth);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
  GST_OBJECT_UNLOCK (thiz);
}

static void
gst_msdkdec_dispose (GObject * object)
{
  GstMsdkDec *thiz = GST_MSDKDEC (object);

  g_clear_object (&thiz->adapter);
  gst_clear_object (&thiz->context);
  gst_clear_object (&thiz->old_context);

  G_OBJECT_CLASS (parent_class)->dispose (object);
}

static void
gst_msdkdec_finalize (GObject * object)
{
  GstMsdkDec *thiz = GST_MSDKDEC (object);

  g_array_unref (thiz->tasks);
  thiz->tasks = NULL;

  release_msdk_surfaces (thiz);

  G_OBJECT_CLASS (parent_class)->finalize (object);
}

static gboolean
gst_msdkdec_post_configure (GstMsdkDec * decoder)
{
  /* Do nothing */
  return TRUE;
}

static gboolean
gst_msdkdec_preinit_decoder (GstMsdkDec * decoder)
{
  decoder->param.mfx.FrameInfo.Width =
      GST_ROUND_UP_16 (decoder->param.mfx.FrameInfo.Width);
  decoder->param.mfx.FrameInfo.Height =
      GST_ROUND_UP_32 (decoder->param.mfx.FrameInfo.Height);

  decoder->param.mfx.FrameInfo.PicStruct =
      decoder->param.mfx.FrameInfo.PicStruct ? decoder->param.mfx.
      FrameInfo.PicStruct : MFX_PICSTRUCT_PROGRESSIVE;

  return TRUE;
}

static gboolean
gst_msdkdec_postinit_decoder (GstMsdkDec * decoder)
{
  /* Do nothing */
  return TRUE;
}

static gboolean
gst_msdkdec_transform_meta (GstVideoDecoder * decoder,
    GstVideoCodecFrame * frame, GstMeta * meta)
{
  const GstMetaInfo *info = meta->info;

  if (GST_VIDEO_DECODER_CLASS (parent_class)->transform_meta (decoder, frame,
          meta))
    return TRUE;

  if (!g_strcmp0 (g_type_name (info->type), "GstVideoRegionOfInterestMeta"))
    return TRUE;

  return FALSE;
}

static void
gst_msdkdec_class_init (GstMsdkDecClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *element_class;
  GstVideoDecoderClass *decoder_class;

  gobject_class = G_OBJECT_CLASS (klass);
  element_class = GST_ELEMENT_CLASS (klass);
  decoder_class = GST_VIDEO_DECODER_CLASS (klass);

  gobject_class->set_property = gst_msdkdec_set_property;
  gobject_class->get_property = gst_msdkdec_get_property;
  gobject_class->dispose = gst_msdkdec_dispose;
  gobject_class->finalize = gst_msdkdec_finalize;

  element_class->set_context = gst_msdkdec_set_context;

  decoder_class->close = GST_DEBUG_FUNCPTR (gst_msdkdec_close);
  decoder_class->start = GST_DEBUG_FUNCPTR (gst_msdkdec_start);
  decoder_class->stop = GST_DEBUG_FUNCPTR (gst_msdkdec_stop);
  decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_msdkdec_set_format);
  decoder_class->finish = GST_DEBUG_FUNCPTR (gst_msdkdec_finish);
  decoder_class->handle_frame = GST_DEBUG_FUNCPTR (gst_msdkdec_handle_frame);
  decoder_class->parse = GST_DEBUG_FUNCPTR (gst_msdkdec_parse);
  decoder_class->decide_allocation =
      GST_DEBUG_FUNCPTR (gst_msdkdec_decide_allocation);
  decoder_class->getcaps = GST_DEBUG_FUNCPTR (gst_msdkdec_getcaps);
  decoder_class->flush = GST_DEBUG_FUNCPTR (gst_msdkdec_flush);
  decoder_class->drain = GST_DEBUG_FUNCPTR (gst_msdkdec_drain);
  decoder_class->transform_meta =
      GST_DEBUG_FUNCPTR (gst_msdkdec_transform_meta);
  decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_msdkdec_src_query);
  decoder_class->sink_query = GST_DEBUG_FUNCPTR (gst_msdkdec_sink_query);

  klass->post_configure = GST_DEBUG_FUNCPTR (gst_msdkdec_post_configure);
  klass->preinit_decoder = GST_DEBUG_FUNCPTR (gst_msdkdec_preinit_decoder);
  klass->postinit_decoder = GST_DEBUG_FUNCPTR (gst_msdkdec_postinit_decoder);

  g_object_class_install_property (gobject_class, GST_MSDKDEC_PROP_HARDWARE,
      g_param_spec_boolean ("hardware", "Hardware", "Enable hardware decoders",
          PROP_HARDWARE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, GST_MSDKDEC_PROP_ASYNC_DEPTH,
      g_param_spec_uint ("async-depth", "Async Depth",
          "Depth of asynchronous pipeline",
          1, 20, PROP_ASYNC_DEPTH_DEFAULT,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
}

static void
gst_msdkdec_init (GstMsdkDec * thiz)
{
  gst_video_info_init (&thiz->non_msdk_pool_info);
  thiz->tasks = g_array_new (FALSE, TRUE, sizeof (MsdkDecTask));
  thiz->hardware = PROP_HARDWARE_DEFAULT;
  thiz->async_depth = PROP_ASYNC_DEPTH_DEFAULT;
  thiz->do_renego = TRUE;
  thiz->do_realloc = TRUE;
  thiz->force_reset_on_res_change = TRUE;
  thiz->report_error = FALSE;
  thiz->sfc = FALSE;
  thiz->ds_has_known_allocator = TRUE;
#ifndef _WIN32
  thiz->modifier = DRM_FORMAT_MOD_INVALID;
#endif
  thiz->adapter = gst_adapter_new ();
  thiz->input_state = NULL;
  thiz->output_state = NULL;
  thiz->pool = NULL;
  thiz->context = NULL;
}