gstreamer/sys/msdk/gstmsdkdec.c

/* GStreamer Intel MSDK plugin
 * Copyright (c) 2016, Intel Corporation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif

#include <stdlib.h>

#include "gstmsdkdec.h"
#include "gstmsdkbufferpool.h"
#include "gstmsdkvideomemory.h"
#include "gstmsdksystemmemory.h"
#include "gstmsdkcontextutil.h"

GST_DEBUG_CATEGORY_EXTERN (gst_msdkdec_debug);
#define GST_CAT_DEFAULT gst_msdkdec_debug

static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS (GST_MSDK_CAPS_STR ("NV12", "NV12"))
    );

#define PROP_HARDWARE_DEFAULT            TRUE
#define PROP_ASYNC_DEPTH_DEFAULT         1

#define IS_ALIGNED(i, n) (((i) & ((n)-1)) == 0)

#define gst_msdkdec_parent_class parent_class
G_DEFINE_TYPE (GstMsdkDec, gst_msdkdec, GST_TYPE_VIDEO_DECODER);

typedef struct _MsdkSurface
{
  mfxFrameSurface1 *surface;
  GstBuffer *buf;
  GstVideoFrame data;
  GstVideoFrame copy;
} MsdkSurface;

static gboolean gst_msdkdec_drain (GstVideoDecoder * decoder);
static gboolean gst_msdkdec_flush (GstVideoDecoder * decoder);
static gboolean gst_msdkdec_negotiate (GstMsdkDec * thiz, gboolean hard_reset);

static GstVideoCodecFrame *
gst_msdkdec_get_oldest_frame (GstVideoDecoder * decoder)
{
  GstVideoCodecFrame *frame = NULL, *old_frame = NULL;
  GList *frames, *l;
  gint count = 0;

  frames = gst_video_decoder_get_frames (decoder);

  for (l = frames; l != NULL; l = l->next) {
    GstVideoCodecFrame *f = l->data;

    if (!GST_CLOCK_TIME_IS_VALID (f->pts)) {
      GST_INFO
          ("Frame doesn't have a valid pts yet, Use gst_video_decoder_get_oldest_frame()"
          "with out considering the PTS for selecting the frame to be finished");
      old_frame = gst_video_decoder_get_oldest_frame (decoder);
      break;
    }

    if (!frame || frame->pts > f->pts)
      frame = f;

    count++;
  }

  if (old_frame)
    frame = old_frame;

  if (frame) {
    GST_LOG_OBJECT (decoder,
        "Oldest frame is %d %" GST_TIME_FORMAT " and %d frames left",
        frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1);
    gst_video_codec_frame_ref (frame);
  }

  if (old_frame)
    gst_video_codec_frame_unref (old_frame);

  g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref);

  return frame;
}

static GstFlowReturn
allocate_output_buffer (GstMsdkDec * thiz, GstBuffer ** buffer)
{
  GstFlowReturn flow;
  GstVideoCodecFrame *frame;
  GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);

  frame = gst_msdkdec_get_oldest_frame (decoder);
  if (!frame) {
    if (GST_PAD_IS_FLUSHING (decoder->srcpad))
      return GST_FLOW_FLUSHING;
    else
      return GST_FLOW_CUSTOM_SUCCESS;
  }

  if (!frame->output_buffer) {
    flow = gst_video_decoder_allocate_output_frame (decoder, frame);
    if (flow != GST_FLOW_OK) {
      gst_video_codec_frame_unref (frame);
      return flow;
    }
  }

  *buffer = gst_buffer_ref (frame->output_buffer);
  gst_buffer_replace (&frame->output_buffer, NULL);
  gst_video_codec_frame_unref (frame);

  if (thiz->postpone_free_surface)
    GST_MINI_OBJECT_FLAG_SET (*buffer, GST_MINI_OBJECT_FLAG_LOCKABLE);

  return GST_FLOW_OK;
}

static void
free_surface (GstMsdkDec * thiz, MsdkSurface * s)
{
  if (s->copy.buffer) {
    gst_video_frame_unmap (&s->copy);
    gst_buffer_unref (s->copy.buffer);
  }

  if (s->data.buffer)
    gst_video_frame_unmap (&s->data);

  gst_buffer_unref (s->buf);
  thiz->decoded_msdk_surfaces = g_list_remove (thiz->decoded_msdk_surfaces, s);

  g_slice_free (MsdkSurface, s);
}

static MsdkSurface *
get_surface (GstMsdkDec * thiz, GstBuffer * buffer)
{
  MsdkSurface *i;
  GstVideoCodecState *output_state = NULL;
  gboolean success;

  i = g_slice_new0 (MsdkSurface);

  if (gst_msdk_is_msdk_buffer (buffer)) {
    i->surface = gst_msdk_get_surface_from_buffer (buffer);
    i->buf = buffer;
  } else {
    /* Confirm to activate the side pool */
    if (!gst_buffer_pool_is_active (thiz->pool) &&
        !gst_buffer_pool_set_active (thiz->pool, TRUE)) {
      g_slice_free (MsdkSurface, i);
      return NULL;
    }

    if (!gst_video_frame_map (&i->copy, &thiz->non_msdk_pool_info, buffer,
            GST_MAP_WRITE))
      goto failed_unref_buffer;

    if (gst_buffer_pool_acquire_buffer (thiz->pool, &buffer,
            NULL) != GST_FLOW_OK)
      goto failed_unmap_copy;

    i->surface = gst_msdk_get_surface_from_buffer (buffer);
    i->buf = buffer;

    output_state =
        gst_video_decoder_get_output_state (GST_VIDEO_DECODER (thiz));
    success =
        gst_video_frame_map (&i->data, &output_state->info, buffer,
        GST_MAP_READWRITE);
    gst_video_codec_state_unref (output_state);
    if (!success)
      goto failed_unref_buffer2;
  }

  gst_msdk_update_mfx_frame_info_from_mfx_video_param (&i->surface->Info,
      &thiz->param);
  thiz->decoded_msdk_surfaces = g_list_append (thiz->decoded_msdk_surfaces, i);
  return i;

failed_unref_buffer2:
  gst_buffer_unref (buffer);
  buffer = i->data.buffer;
failed_unmap_copy:
  gst_video_frame_unmap (&i->copy);
failed_unref_buffer:
  gst_buffer_unref (buffer);
  g_slice_free (MsdkSurface, i);

  GST_ERROR_OBJECT (thiz, "failed to handle buffer");
  return NULL;
}

static void
gst_msdkdec_close_decoder (GstMsdkDec * thiz, gboolean reset_param)
{
  mfxStatus status;

  if (!thiz->context || !thiz->initialized)
    return;

  GST_DEBUG_OBJECT (thiz, "Closing decoder with context %" GST_PTR_FORMAT,
      thiz->context);

  if (thiz->use_video_memory)
    gst_msdk_frame_free (thiz->context, &thiz->alloc_resp);

  status = MFXVideoDECODE_Close (gst_msdk_context_get_session (thiz->context));
  if (status != MFX_ERR_NONE && status != MFX_ERR_NOT_INITIALIZED) {
    GST_WARNING_OBJECT (thiz, "Decoder close failed (%s)",
        msdk_status_to_string (status));
  }

  g_array_set_size (thiz->tasks, 0);

  if (reset_param)
    memset (&thiz->param, 0, sizeof (thiz->param));

  thiz->initialized = FALSE;
  gst_adapter_clear (thiz->adapter);
}

static void
gst_msdkdec_set_context (GstElement * element, GstContext * context)
{
  GstMsdkContext *msdk_context = NULL;
  GstMsdkDec *thiz = GST_MSDKDEC (element);

  if (gst_msdk_context_get_context (context, &msdk_context)) {
    gst_object_replace ((GstObject **) & thiz->context,
        (GstObject *) msdk_context);
    gst_object_unref (msdk_context);
  }

  GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}

static gboolean
gst_msdkdec_init_decoder (GstMsdkDec * thiz)
{
  GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
  GstVideoInfo *info;
  mfxSession session;
  mfxStatus status;
  mfxFrameAllocRequest request;

  if (thiz->initialized)
    return TRUE;

  if (!thiz->context) {
    GST_WARNING_OBJECT (thiz, "No MSDK Context");
    return FALSE;
  }

  if (!thiz->input_state) {
    GST_DEBUG_OBJECT (thiz, "Have no input state yet");
    return FALSE;
  }
  info = &thiz->input_state->info;

  GST_OBJECT_LOCK (thiz);

  if (thiz->use_video_memory) {
    gst_msdk_set_frame_allocator (thiz->context);
    thiz->param.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY;
  } else {
    thiz->param.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
  }

  GST_INFO_OBJECT (thiz, "This MSDK decoder uses %s memory",
      thiz->use_video_memory ? "video" : "system");

  thiz->param.AsyncDepth = thiz->async_depth;

  /* We expect msdk to fill the width and height values */
  g_return_val_if_fail (thiz->param.mfx.FrameInfo.Width
      && thiz->param.mfx.FrameInfo.Height, FALSE);

  klass->preinit_decoder (thiz);

  /* Set frame rate only if provided.
   * If not, frame rate will be assumed inside the driver.
   * Also we respect the upstream provided fps values */
  if (info->fps_n > 0 && info->fps_d > 0
      && info->fps_n != thiz->param.mfx.FrameInfo.FrameRateExtN
      && info->fps_d != thiz->param.mfx.FrameInfo.FrameRateExtD) {
    thiz->param.mfx.FrameInfo.FrameRateExtN = info->fps_n;
    thiz->param.mfx.FrameInfo.FrameRateExtD = info->fps_d;
  }

  if (info->par_n && info->par_d && !thiz->param.mfx.FrameInfo.AspectRatioW
      && !thiz->param.mfx.FrameInfo.AspectRatioH) {
    thiz->param.mfx.FrameInfo.AspectRatioW = info->par_n;
    thiz->param.mfx.FrameInfo.AspectRatioH = info->par_d;
  }

  thiz->param.mfx.FrameInfo.FourCC =
      thiz->param.mfx.FrameInfo.FourCC ? thiz->param.mfx.
      FrameInfo.FourCC : MFX_FOURCC_NV12;
  thiz->param.mfx.FrameInfo.ChromaFormat =
      thiz->param.mfx.FrameInfo.ChromaFormat ? thiz->param.mfx.
      FrameInfo.ChromaFormat : MFX_CHROMAFORMAT_YUV420;

  session = gst_msdk_context_get_session (thiz->context);
  /* validate parameters and allow the Media SDK to make adjustments */
  status = MFXVideoDECODE_Query (session, &thiz->param, &thiz->param);
  if (status < MFX_ERR_NONE) {
    GST_ERROR_OBJECT (thiz, "Video Decode Query failed (%s)",
        msdk_status_to_string (status));
    goto failed;
  } else if (status > MFX_ERR_NONE) {
    GST_WARNING_OBJECT (thiz, "Video Decode Query returned: %s",
        msdk_status_to_string (status));
  }

  klass->postinit_decoder (thiz);

  status = MFXVideoDECODE_QueryIOSurf (session, &thiz->param, &request);
  if (status < MFX_ERR_NONE) {
    GST_ERROR_OBJECT (thiz, "Query IO surfaces failed (%s)",
        msdk_status_to_string (status));
    goto failed;
  } else if (status > MFX_ERR_NONE) {
    GST_WARNING_OBJECT (thiz, "Query IO surfaces returned: %s",
        msdk_status_to_string (status));
  }

  if (request.NumFrameSuggested < thiz->param.AsyncDepth) {
    GST_ERROR_OBJECT (thiz, "Required %d surfaces (%d suggested), async %d",
        request.NumFrameMin, request.NumFrameSuggested, thiz->param.AsyncDepth);
    goto failed;
  }

  /* account for downstream requirement */
  if (G_LIKELY (thiz->min_prealloc_buffers))
    request.NumFrameSuggested += thiz->min_prealloc_buffers;
  else
    GST_WARNING_OBJECT (thiz,
        "Allocating resources without considering the downstream requirement"
        "or extra scratch surface count");

  if (thiz->use_video_memory) {
    gint shared_async_depth;

    shared_async_depth =
        gst_msdk_context_get_shared_async_depth (thiz->context);
    request.NumFrameSuggested += shared_async_depth;

    request.Type |= MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
    if (thiz->use_dmabuf)
      request.Type |= MFX_MEMTYPE_EXPORT_FRAME;
    gst_msdk_frame_alloc (thiz->context, &request, &thiz->alloc_resp);
  }

  /* update the prealloc_buffer count, which will be used later
   * as GstBufferPool min_buffers */
  thiz->min_prealloc_buffers = request.NumFrameSuggested;

  GST_DEBUG_OBJECT (thiz, "Required %d surfaces (%d suggested)",
      request.NumFrameMin, request.NumFrameSuggested);

  status = MFXVideoDECODE_Init (session, &thiz->param);
  if (status < MFX_ERR_NONE) {
    GST_ERROR_OBJECT (thiz, "Init failed (%s)", msdk_status_to_string (status));
    goto failed;
  } else if (status > MFX_ERR_NONE) {
    GST_WARNING_OBJECT (thiz, "Init returned: %s",
        msdk_status_to_string (status));
  }

  status = MFXVideoDECODE_GetVideoParam (session, &thiz->param);
  if (status < MFX_ERR_NONE) {
    GST_ERROR_OBJECT (thiz, "Get Video Parameters failed (%s)",
        msdk_status_to_string (status));
    goto failed;
  } else if (status > MFX_ERR_NONE) {
    GST_WARNING_OBJECT (thiz, "Get Video Parameters returned: %s",
        msdk_status_to_string (status));
  }

  g_array_set_size (thiz->tasks, 0);    /* resets array content */
  g_array_set_size (thiz->tasks, thiz->param.AsyncDepth);
  thiz->next_task = 0;

  GST_OBJECT_UNLOCK (thiz);

  thiz->initialized = TRUE;
  return TRUE;

failed:
  GST_OBJECT_UNLOCK (thiz);
  return FALSE;
}


static gboolean
_gst_caps_has_feature (const GstCaps * caps, const gchar * feature)
{
  guint i;

  for (i = 0; i < gst_caps_get_size (caps); i++) {
    GstCapsFeatures *const features = gst_caps_get_features (caps, i);
    /* Skip ANY features, we need an exact match for correct evaluation */
    if (gst_caps_features_is_any (features))
      continue;
    if (gst_caps_features_contains (features, feature))
      return TRUE;
  }

  return FALSE;
}

static gboolean
srcpad_can_dmabuf (GstMsdkDec * thiz)
{
  gboolean ret = FALSE;
  GstCaps *caps, *out_caps;
  GstPad *srcpad;

  srcpad = GST_VIDEO_DECODER_SRC_PAD (thiz);
  caps = gst_pad_get_pad_template_caps (srcpad);

  out_caps = gst_pad_peer_query_caps (srcpad, caps);
  if (!out_caps)
    goto done;

  if (gst_caps_is_any (out_caps) || gst_caps_is_empty (out_caps)
      || out_caps == caps)
    goto done;

  if (_gst_caps_has_feature (out_caps, GST_CAPS_FEATURE_MEMORY_DMABUF))
    ret = TRUE;

done:
  if (caps)
    gst_caps_unref (caps);
  if (out_caps)
    gst_caps_unref (out_caps);
  return ret;
}

static gboolean
gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation)
{
  GstVideoCodecState *output_state;
  GstVideoInfo *vinfo;
  GstVideoAlignment align;
  GstCaps *allocation_caps = NULL;
  GstVideoFormat format;
  guint width, height;
  guint alloc_w, alloc_h;
  const gchar *format_str;

  /* use display width and display height in output state, which
   * will be used for caps negotiation */
  width =
      thiz->param.mfx.FrameInfo.CropW ? thiz->param.mfx.
      FrameInfo.CropW : GST_VIDEO_INFO_WIDTH (&thiz->input_state->info);
  height =
      thiz->param.mfx.FrameInfo.CropH ? thiz->param.mfx.
      FrameInfo.CropH : GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info);

  format =
      gst_msdk_get_video_format_from_mfx_fourcc (thiz->param.mfx.
      FrameInfo.FourCC);

  if (format == GST_VIDEO_FORMAT_UNKNOWN) {
    GST_WARNING_OBJECT (thiz, "Failed to find a valid video format\n");
    return FALSE;
  }

  output_state =
      gst_video_decoder_set_output_state (GST_VIDEO_DECODER (thiz),
      format, width, height, thiz->input_state);
  if (!output_state)
    return FALSE;

  /* Find allocation width and height */
  alloc_w =
      GST_ROUND_UP_16 (thiz->param.mfx.FrameInfo.Width ? thiz->param.mfx.
      FrameInfo.Width : width);
  alloc_h =
      GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Height ? thiz->param.mfx.
      FrameInfo.Height : height);

  /* Ensure output_state->caps and info have same width and height
   * Also, mandate 32 bit alignment */
  vinfo = &output_state->info;
  gst_msdk_set_video_alignment (vinfo, alloc_w, alloc_h, &align);
  gst_video_info_align (vinfo, &align);
  output_state->caps = gst_video_info_to_caps (vinfo);
  if (srcpad_can_dmabuf (thiz))
    gst_caps_set_features (output_state->caps, 0,
        gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_DMABUF, NULL));

  if (need_allocation) {
    /* Find allocation width and height */
    width =
        GST_ROUND_UP_16 (thiz->param.mfx.FrameInfo.Width ? thiz->param.mfx.
        FrameInfo.Width : GST_VIDEO_INFO_WIDTH (&output_state->info));
    height =
        GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Height ? thiz->param.mfx.
        FrameInfo.Height : GST_VIDEO_INFO_HEIGHT (&output_state->info));

    /* set allocation width and height in allocation_caps,
     * which may or may not be similar to the output_state caps */
    allocation_caps = gst_caps_copy (output_state->caps);
    format_str =
        gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
        (&output_state->info));
    gst_caps_set_simple (allocation_caps, "width", G_TYPE_INT, width, "height",
        G_TYPE_INT, height, "format", G_TYPE_STRING, format_str, NULL);
    GST_INFO_OBJECT (thiz, "new alloc caps = %" GST_PTR_FORMAT,
        allocation_caps);
    gst_caps_replace (&output_state->allocation_caps, allocation_caps);
    gst_caps_unref (allocation_caps);
  } else {
    /* We keep the allocation parameters as it is to avoid pool re-negotiation.
     * For codecs like VP9, dynamic resolution change doesn't require allocation
     * reset if the new video frame resolution is lower than the
     * already configured one */
  }
  gst_video_codec_state_unref (output_state);

  return TRUE;
}

static void
gst_msdkdec_set_latency (GstMsdkDec * thiz)
{
  GstVideoInfo *info = &thiz->input_state->info;
  gint min_delayed_frames;
  GstClockTime latency;

  min_delayed_frames = thiz->async_depth;

  if (info->fps_n) {
    latency = gst_util_uint64_scale_ceil (GST_SECOND * info->fps_d,
        min_delayed_frames, info->fps_n);
  } else {
    /* FIXME: Assume 25fps. This is better than reporting no latency at
     * all and then later failing in live pipelines
     */
    latency = gst_util_uint64_scale_ceil (GST_SECOND * 1,
        min_delayed_frames, 25);
  }

  GST_INFO_OBJECT (thiz,
      "Updating latency to %" GST_TIME_FORMAT " (%d frames)",
      GST_TIME_ARGS (latency), min_delayed_frames);

  gst_video_decoder_set_latency (GST_VIDEO_DECODER (thiz), latency, latency);
}

static gint
_find_msdk_surface (gconstpointer msdk_surface, gconstpointer comp_surface)
{
  MsdkSurface *cached_surface = (MsdkSurface *) msdk_surface;
  mfxFrameSurface1 *_surface = (mfxFrameSurface1 *) comp_surface;

  return cached_surface ? cached_surface->surface != _surface : -1;
}

static void
gst_msdkdec_free_unlocked_msdk_surfaces (GstMsdkDec * thiz,
    MsdkSurface * curr_surface)
{
  GList *l;
  MsdkSurface *surface;

  for (l = thiz->decoded_msdk_surfaces; l;) {
    surface = l->data;
    l = l->next;

    if (surface != curr_surface && surface->surface->Data.Locked == 0)
      free_surface (thiz, surface);
  }
}

static GstFlowReturn
gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task)
{
  GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
  GstFlowReturn flow;
  GstVideoCodecFrame *frame;
  MsdkSurface *surface;
  mfxStatus status;
  GList *l;

  if (G_LIKELY (task->sync_point)) {
    status =
        MFXVideoCORE_SyncOperation (gst_msdk_context_get_session
        (thiz->context), task->sync_point, 300000);
    if (status != MFX_ERR_NONE) {
      GST_ERROR_OBJECT (thiz, "failed to do sync operation");
      return GST_FLOW_ERROR;
    }
  }

  if (G_LIKELY (task->sync_point || (task->surface && task->decode_only))) {
    gboolean decode_only = task->decode_only;

    frame = gst_msdkdec_get_oldest_frame (decoder);

    l = g_list_find_custom (thiz->decoded_msdk_surfaces, task->surface,
        _find_msdk_surface);
    if (l) {
      surface = l->data;
    } else {
      GST_ERROR_OBJECT (thiz, "Couldn't find the cached MSDK surface");
      return GST_FLOW_ERROR;
    }

    if (G_LIKELY (frame)) {
      if (G_LIKELY (surface->copy.buffer == NULL)) {
        frame->output_buffer = gst_buffer_ref (surface->buf);
      } else {
        gst_video_frame_copy (&surface->copy, &surface->data);
        frame->output_buffer = gst_buffer_ref (surface->copy.buffer);
      }
    }

    if (!thiz->postpone_free_surface)
      free_surface (thiz, surface);
    task->sync_point = NULL;
    task->surface = NULL;
    task->decode_only = FALSE;

    if (!frame)
      return GST_FLOW_FLUSHING;
    gst_video_codec_frame_unref (frame);

    if (decode_only)
      GST_VIDEO_CODEC_FRAME_SET_DECODE_ONLY (frame);
    flow = gst_video_decoder_finish_frame (decoder, frame);
    return flow;
  }
  return GST_FLOW_OK;
}

static gboolean
gst_msdkdec_start (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  if (gst_msdk_context_prepare (GST_ELEMENT_CAST (thiz), &thiz->context)) {
    GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
        thiz->context);

    /* TODO: Currently d3d allocator is not implemented.
     * So decoder uses system memory by default on Windows.
     */
#ifndef _WIN32
    thiz->use_video_memory = TRUE;
#else
    thiz->use_video_memory = FALSE;
#endif

    if (gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_DECODER) {
      GstMsdkContext *parent_context, *msdk_context;

      parent_context = thiz->context;
      msdk_context = gst_msdk_context_new_with_parent (parent_context);

      if (!msdk_context) {
        GST_ERROR_OBJECT (thiz, "Context creation failed");
        return FALSE;
      }

      thiz->context = msdk_context;

      gst_msdk_context_add_shared_async_depth (thiz->context,
          gst_msdk_context_get_shared_async_depth (parent_context));
      gst_object_unref (parent_context);

      GST_INFO_OBJECT (thiz,
          "Creating new context %" GST_PTR_FORMAT " with joined session",
          thiz->context);
    } else {
      gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_DECODER);
    }
  } else {
    if (!gst_msdk_context_ensure_context (GST_ELEMENT_CAST (thiz),
            thiz->hardware, GST_MSDK_JOB_DECODER))
      return FALSE;
    GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT,
        thiz->context);
  }

  gst_msdk_context_add_shared_async_depth (thiz->context, thiz->async_depth);

  return TRUE;
}

static gboolean
gst_msdkdec_close (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  if (thiz->context)
    gst_object_replace ((GstObject **) & thiz->context, NULL);

  return TRUE;
}

static gboolean
gst_msdkdec_stop (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  gst_msdkdec_flush (decoder);

  if (thiz->input_state) {
    gst_video_codec_state_unref (thiz->input_state);
    thiz->input_state = NULL;
  }
  if (thiz->pool) {
    gst_object_unref (thiz->pool);
    thiz->pool = NULL;
  }
  gst_video_info_init (&thiz->non_msdk_pool_info);

  gst_msdkdec_close_decoder (thiz, TRUE);
  return TRUE;
}

static gboolean
gst_msdkdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  if (thiz->input_state) {
    /* mark for re-negotiation if display resolution changes */
    if ((GST_VIDEO_INFO_WIDTH (&thiz->input_state->info) !=
            GST_VIDEO_INFO_WIDTH (&state->info)) ||
        GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info) !=
        GST_VIDEO_INFO_HEIGHT (&state->info))
      thiz->do_renego = TRUE;
    gst_video_codec_state_unref (thiz->input_state);
  }
  thiz->input_state = gst_video_codec_state_ref (state);

  /* we don't set output state here to avoid caching of mismatched
   * video information if there is dynamic resolution change in the stream.
   * All negotiation code is consolidated in gst_msdkdec_negotiate() and
   * this will be invoked from handle_frame() */

  gst_msdkdec_set_latency (thiz);
  return TRUE;
}

static void
release_msdk_surfaces (GstMsdkDec * thiz)
{
  GList *l;
  MsdkSurface *surface;

  for (l = thiz->decoded_msdk_surfaces; l;) {
    surface = l->data;
    l = l->next;
    free_surface (thiz, surface);
  }
}

/* This will get invoked in the following situations:
 * 1: beginning of the stream, which requires initialization (== complete reset)
 * 2: upstream notified a resolution change and set do_renego to TRUE.
 *    new resolution may or may not requires full reset
 * 3: upstream failed to notify the resolution change but
 *    msdk detected the change (eg: vp9 stream in ivf elementary form
 *     with varying resolution frames).
 *
 * for any input configuration change, we deal with notification
 * from upstream and also use msdk APIs to handle the parameter initialization
 * efficiently
 */
static gboolean
gst_msdkdec_negotiate (GstMsdkDec * thiz, gboolean hard_reset)
{
  GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
  GST_DEBUG_OBJECT (thiz,
      "Start Negotiating caps, pool and Init the msdk decdoer subsystem");

  if (hard_reset) {
    /* Retrieve any pending frames and push them downstream */
    if (gst_msdkdec_drain (GST_VIDEO_DECODER (thiz)) != GST_FLOW_OK)
      goto error_drain;

    /* This will initiate the allocation query which will help to flush
     * all the pending buffers in the pipeline so that we can stop
     * the active bufferpool and safely invoke gst_msdk_frame_free() */
    if (thiz->initialized) {
      GstCaps *caps = gst_pad_get_current_caps (decoder->srcpad);
      GstQuery *query = NULL;
      if (caps) {
        query = gst_query_new_allocation (caps, FALSE);
        gst_pad_peer_query (decoder->srcpad, query);
        gst_query_unref (query);
        gst_caps_unref (caps);
      }
    }

    /* De-initialize the decoder if it is already active */
    /* Do not reset the mfxVideoParam since it already
     * has the required parameters for new session decode */
    gst_msdkdec_close_decoder (thiz, FALSE);

    /* request for pool re-negotiation by setting do_realloc */
    thiz->do_realloc = TRUE;
  }

  /* At this point all pending frames (if there are any) are pushed downstream
   * and we are ready to negotiate the output caps */
  if (!gst_msdkdec_set_src_caps (thiz, hard_reset))
    return FALSE;

  /* this will initiate the allocation query, we create the
   * bufferpool in decide_allocation in order to account
   * for the downstream min_buffer requirement
   * Required initializations for MediaSDK operations
   * will all be initialized from decide_allocation after considering
   * some of the downstream requirements */
  if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (thiz)))
    goto error_negotiate;

  thiz->do_renego = FALSE;
  thiz->do_realloc = FALSE;

  return TRUE;

error_drain:
  GST_ERROR_OBJECT (thiz, "Failed to Drain the queued decoded frames");
  return FALSE;

error_negotiate:
  GST_ERROR_OBJECT (thiz, "Failed to re-negotiate");
  return FALSE;
}

static GstFlowReturn
gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
  GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
  GstFlowReturn flow;
  GstBuffer *buffer, *input_buffer = NULL;
  GstVideoInfo alloc_info;
  MsdkDecTask *task = NULL;
  mfxBitstream bitstream;
  MsdkSurface *surface = NULL;
  mfxSession session;
  mfxStatus status;
  GstMapInfo map_info;
  guint i, retry_err_incompatible = 0;
  gsize data_size;
  gboolean hard_reset = FALSE;

  /* configure the subclass in order to fill the CodecID field of
   * mfxVideoParam and also to load the PluginID for some of the
   * codecs which is mandatory to invoke the
   * MFXVideoDECODE_DecodeHeader API.
   *
   * For non packetized formats (currently only vc1), there
   * could be headers received as codec_data which are not available
   * instream and in that case subclass implementation will
   * push it to the internal adapter. We invoke the subclass configure
   * well early to make sure the codec_data received has been correctly
   * pushed to the adapter by the subclasses before doing
   * the DecodeHeader() later on
   */
  if (!thiz->initialized || thiz->do_renego) {
    /* Clear the internal adapter in re-negotiation for non-packetized
     * formats */
    if (!gst_video_decoder_get_packetized (decoder))
      gst_adapter_clear (thiz->adapter);

    if (!klass->configure || !klass->configure (thiz)) {
      flow = GST_FLOW_OK;
      goto error;
    }
  }

  /* Current frame-codec could be pushed and released before this
   * function ends -- because msdkdec pushes the oldest frame,
   * according its PTS, and it could be this very same frame-codec
   * among others pending frame-codecs.
   *
   * Instead of copying the input data into the mfxBitstream, let's
   * keep an extra reference to frame-codec's input buffer */
  input_buffer = gst_buffer_ref (frame->input_buffer);
  if (!gst_buffer_map (input_buffer, &map_info, GST_MAP_READ)) {
    gst_buffer_unref (input_buffer);
    return GST_FLOW_ERROR;
  }

  memset (&bitstream, 0, sizeof (bitstream));

  if (gst_video_decoder_get_packetized (decoder)) {
    /* Packetized stream: we prefer to have a parser as a connected upstream
     * element to the decoder */
    bitstream.Data = map_info.data;
    bitstream.DataLength = map_info.size;
    bitstream.MaxLength = map_info.size;

    /*
     * MFX_BITSTREAM_COMPLETE_FRAME was removed since commit df59db9, however
     * some customers still use DecodedOrder (deprecated in msdk-2017 version)
     * for low-latency streaming of non-b-frame encoded streams, which needs to
     * output the frame at once, so add it back for this case
     */
    if (thiz->param.mfx.DecodedOrder == GST_MSDKDEC_OUTPUT_ORDER_DECODE)
      bitstream.DataFlag |= MFX_BITSTREAM_COMPLETE_FRAME;
  } else {
    /* Non packetized streams: eg: vc1 advanced profile with per buffer bdu */
    gst_adapter_push (thiz->adapter, gst_buffer_ref (input_buffer));
    data_size = gst_adapter_available (thiz->adapter);

    bitstream.Data = (mfxU8 *) gst_adapter_map (thiz->adapter, data_size);
    bitstream.DataLength = (mfxU32) data_size;
    bitstream.MaxLength = bitstream.DataLength;
  }
  GST_INFO_OBJECT (thiz,
      "mfxBitStream=> DataLength:%d DataOffset:%d MaxLength:%d",
      bitstream.DataLength, bitstream.DataOffset, bitstream.MaxLength);

  session = gst_msdk_context_get_session (thiz->context);

  if (!thiz->initialized || thiz->do_renego) {

    /* gstreamer caps will not provide all the necessary parameters
     * required for optimal decode configuration. For example: the required number
     * of surfaces to be allocated can be calculated based on H264 SEI header
     * and this information can't be retrieved from the negotiated caps.
     * So instead of introducing a codecparser dependency to parse the headers
     * inside msdk plugin, we simply use the mfx APIs to extract header information */
    status = MFXVideoDECODE_DecodeHeader (session, &bitstream, &thiz->param);
    if (status == MFX_ERR_MORE_DATA) {
      flow = GST_FLOW_OK;
      goto done;
    }

    if (!klass->post_configure (thiz)) {
      flow = GST_FLOW_ERROR;
      goto error;
    }

    if (!thiz->initialized)
      hard_reset = TRUE;
    else {
      GstVideoCodecState *output_state =
          gst_video_decoder_get_output_state (GST_VIDEO_DECODER (thiz));
      if (output_state) {
        if (output_state->allocation_caps) {
          gst_video_info_from_caps (&alloc_info, output_state->allocation_caps);

          /* Check whether we need complete reset for dynamic resolution change */
          if (thiz->param.mfx.FrameInfo.Width >
              GST_VIDEO_INFO_WIDTH (&alloc_info)
              || thiz->param.mfx.FrameInfo.Height >
              GST_VIDEO_INFO_HEIGHT (&alloc_info))
            hard_reset = TRUE;
        }
        gst_video_codec_state_unref (output_state);
      }

    }

    /* if subclass requested for the force reset */
    if (thiz->force_reset_on_res_change)
      hard_reset = TRUE;

    /* Config changed dynamically and we are going to do a full reset,
     * this will unref the input frame which has the new configuration.
     * Keep a ref to the input_frame to keep it alive */
    if (thiz->initialized && thiz->do_renego)
      gst_video_codec_frame_ref (frame);

    if (!gst_msdkdec_negotiate (thiz, hard_reset)) {
      GST_ELEMENT_ERROR (thiz, CORE, NEGOTIATION,
          ("Could not negotiate the stream"), (NULL));
      flow = GST_FLOW_ERROR;
      goto error;
    }
  }

  for (;;) {
    if (thiz->postpone_free_surface)
      gst_msdkdec_free_unlocked_msdk_surfaces (thiz, surface);
    task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
    flow = gst_msdkdec_finish_task (thiz, task);
    if (flow != GST_FLOW_OK)
      goto error;
    if (!surface) {
      flow = allocate_output_buffer (thiz, &buffer);
      if (flow == GST_FLOW_CUSTOM_SUCCESS) {
        flow = GST_FLOW_OK;
        break;
      } else if (flow != GST_FLOW_OK)
        goto error;
      surface = get_surface (thiz, buffer);
      if (!surface) {
        /* Can't get a surface for some reason; finish tasks, then see if
           a surface becomes available. */
        for (i = 0; i < thiz->tasks->len - 1; i++) {
          thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
          task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
          flow = gst_msdkdec_finish_task (thiz, task);
          if (flow != GST_FLOW_OK)
            goto error;
          surface = get_surface (thiz, buffer);
          if (surface)
            break;
        }
        if (!surface) {
          GST_ERROR_OBJECT (thiz, "Couldn't get a surface");
          flow = GST_FLOW_ERROR;
          goto error;
        }
      }
    }

    status =
        MFXVideoDECODE_DecodeFrameAsync (session, &bitstream, surface->surface,
        &task->surface, &task->sync_point);

    /* media-sdk requires complete reset since the surface is inadequate
     * for further decoding */
    if (status == MFX_ERR_INCOMPATIBLE_VIDEO_PARAM &&
        retry_err_incompatible++ < 1) {
      /* MFX_ERR_INCOMPATIBLE_VIDEO_PARAM means the current mfx surface is not
       * suitable for the current frame. Call MFXVideoDECODE_DecodeHeader to get
       * the current frame size, then do memory re-allocation, otherwise
       * MFXVideoDECODE_DecodeFrameAsync will still fail on next call */
      status = MFXVideoDECODE_DecodeHeader (session, &bitstream, &thiz->param);
      if (status == MFX_ERR_MORE_DATA) {
        flow = GST_FLOW_OK;
        goto done;
      }

      /* Requires memory re-allocation, do a hard reset */
      if (!gst_msdkdec_negotiate (thiz, TRUE))
        goto error;

      /* The current surface is freed when doing a hard reset; a new surface is
       * required for the new resolution */
      surface = NULL;
      continue;
    }

    retry_err_incompatible = 0;

    if (G_LIKELY (status == MFX_ERR_NONE)
        || (status == MFX_WRN_VIDEO_PARAM_CHANGED)) {
      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;

      if (surface->surface->Data.Locked > 0)
        surface = NULL;
      else if (!thiz->use_video_memory) {
        /* The for loop will continue because DataLength is greater than 0 so
         * free the surface right away if not in use. */
        if (bitstream.DataLength > 0 && task->surface != surface->surface)
          free_surface (thiz, surface);
        surface = NULL;
      }

      if (bitstream.DataLength == 0) {
        flow = GST_FLOW_OK;

        /* Don't release it if the current surface is in use */
        if (surface && task->surface == surface->surface)
          surface = NULL;

        break;
      }
    } else if (status == MFX_ERR_MORE_DATA) {
      if (task->surface) {
        task->decode_only = TRUE;
        thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
      }

      if (surface->surface->Data.Locked > 0)
        surface = NULL;
      flow = GST_VIDEO_DECODER_FLOW_NEED_DATA;
      break;
    } else if (status == MFX_ERR_MORE_SURFACE) {
      surface = NULL;
      continue;
    } else if (status == MFX_WRN_DEVICE_BUSY) {
      /* If device is busy, wait 1ms and retry, as per MSDK's recommendation */
      g_usleep (1000);

      if (task->surface &&
          task->surface == surface->surface && !task->sync_point) {
        free_surface (thiz, surface);
        surface = NULL;
      }

      /* If the current surface is still busy, we should do sync operation,
       * then try to decode again
       */
      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
    } else if (status < MFX_ERR_NONE) {
      GST_ERROR_OBJECT (thiz, "DecodeFrameAsync failed (%s)",
          msdk_status_to_string (status));
      flow = GST_FLOW_ERROR;
      break;
    }
  }

  if (!gst_video_decoder_get_packetized (decoder)) {
    /* flush out the data which has already been consumed by msdk */
    gst_adapter_flush (thiz->adapter, bitstream.DataOffset);
  }

  /*
   * DecodedOrder was deprecated in msdk-2017 version, but some
   * customers still using this for low-latency streaming of non-b-frame
   * encoded streams, which needs to output the frame at once
   */
  if (thiz->param.mfx.DecodedOrder == GST_MSDKDEC_OUTPUT_ORDER_DECODE)
    gst_msdkdec_finish_task (thiz, task);

done:
  if (surface)
    free_surface (thiz, surface);

  gst_buffer_unmap (input_buffer, &map_info);
  gst_buffer_unref (input_buffer);
  return flow;

error:
  if (input_buffer) {
    gst_buffer_unmap (input_buffer, &map_info);
    gst_buffer_unref (input_buffer);
  }
  gst_video_decoder_drop_frame (decoder, frame);

  return flow;
}

static GstFlowReturn
gst_msdkdec_parse (GstVideoDecoder * decoder, GstVideoCodecFrame * frame,
    GstAdapter * adapter, gboolean at_eos)
{
  gsize size;
  GstFlowReturn ret;
  GstBuffer *buffer;
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  /* Don't parse the input buffer indeed, it will invoke
   * gst_msdkdec_handle_frame to handle the input buffer */
  size = gst_adapter_available (adapter);
  gst_video_decoder_add_to_frame (decoder, size);
  ret = gst_video_decoder_have_frame (decoder);
  size = gst_adapter_available (thiz->adapter);

  if (size) {
    /* The base class will set up a new frame for parsing as
     * soon as there is valid data in the buffer */
    buffer = gst_adapter_get_buffer (thiz->adapter, size);
    gst_adapter_flush (thiz->adapter, size);
    gst_adapter_push (adapter, buffer);
  }

  return ret;
}

static GstBufferPool *
gst_msdkdec_create_buffer_pool (GstMsdkDec * thiz, GstVideoInfo * info,
    guint num_buffers)
{
  GstBufferPool *pool = NULL;
  GstStructure *config;
  GstAllocator *allocator = NULL;
  GstVideoAlignment align;
  GstCaps *caps = NULL;
  GstAllocationParams params = { 0, 31, 0, 0, };
  mfxFrameAllocResponse *alloc_resp = NULL;

  g_return_val_if_fail (info, NULL);
  g_return_val_if_fail (GST_VIDEO_INFO_WIDTH (info)
      && GST_VIDEO_INFO_HEIGHT (info), NULL);

  alloc_resp = &thiz->alloc_resp;

  pool = gst_msdk_buffer_pool_new (thiz->context, alloc_resp);
  if (!pool)
    goto error_no_pool;

  caps = gst_video_info_to_caps (info);

  /* allocators should use the same width/height/stride/height_alignment of
   * negotiated output caps, which is what we configure in msdk_allocator */
  if (thiz->use_dmabuf)
    allocator = gst_msdk_dmabuf_allocator_new (thiz->context, info, alloc_resp);
  else if (thiz->use_video_memory)
    allocator = gst_msdk_video_allocator_new (thiz->context, info, alloc_resp);
  else
    allocator = gst_msdk_system_allocator_new (info);

  if (!allocator) {
    gst_caps_unref (caps);
    goto error_no_allocator;
  }

  config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));
  gst_buffer_pool_config_set_params (config, caps,
      GST_VIDEO_INFO_SIZE (info), num_buffers, 0);
  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
  gst_buffer_pool_config_add_option (config,
      GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
  gst_caps_unref (caps);

  if (thiz->use_video_memory) {
    gst_buffer_pool_config_add_option (config,
        GST_BUFFER_POOL_OPTION_MSDK_USE_VIDEO_MEMORY);
    if (thiz->use_dmabuf)
      gst_buffer_pool_config_add_option (config,
          GST_BUFFER_POOL_OPTION_MSDK_USE_DMABUF);
  }

  gst_buffer_pool_config_set_video_alignment (config, &align);
  gst_buffer_pool_config_set_allocator (config, allocator, &params);
  gst_object_unref (allocator);

  if (!gst_buffer_pool_set_config (pool, config))
    goto error_pool_config;

  return pool;

error_no_pool:
  {
    GST_INFO_OBJECT (thiz, "failed to create bufferpool");
    return NULL;
  }
error_no_allocator:
  {
    GST_INFO_OBJECT (thiz, "failed to create allocator");
    gst_object_unref (pool);
    return NULL;
  }
error_pool_config:
  {
    GST_INFO_OBJECT (thiz, "failed to set config");
    gst_object_unref (pool);
    gst_object_unref (allocator);
    return NULL;
  }
}

static gboolean
gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
  GstBufferPool *pool = NULL;
  GstStructure *pool_config = NULL;
  GstCaps *pool_caps /*, *negotiated_caps */ ;
  guint size, min_buffers, max_buffers;

  if (!GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation (decoder,
          query))
    return FALSE;

  /* Get the buffer pool config decided on by the base class. The base
     class ensures that there will always be at least a 0th pool in
     the query. */
  gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL);
  pool_config = gst_buffer_pool_get_config (pool);

  /* Get the caps of pool and increase the min and max buffers by async_depth.
   * We will always have that number of decode operations in-flight */
  gst_buffer_pool_config_get_params (pool_config, &pool_caps, &size,
      &min_buffers, &max_buffers);
  min_buffers += thiz->async_depth;
  if (max_buffers)
    max_buffers += thiz->async_depth;

  /* increase the min_buffers by 1 for smooth display in render pipeline */
  min_buffers += 1;

  /* this will get updated with msdk requirement */
  thiz->min_prealloc_buffers = min_buffers;

  if (_gst_caps_has_feature (pool_caps, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
    GST_INFO_OBJECT (decoder, "This MSDK decoder uses DMABuf memory");
    thiz->use_video_memory = thiz->use_dmabuf = TRUE;
  }

  /* Initialize MSDK decoder before new bufferpool tries to alloc each buffer,
   * which requires information about frame allocation.
   * No effect if already initialized.
   */
  if (!gst_msdkdec_init_decoder (thiz))
    return FALSE;

  /* get the updated min_buffers, which account for the msdk requirement as well */
  min_buffers = thiz->min_prealloc_buffers;

  /* Decoder always use its own pool. So we create a pool if msdk APIs
   * previously requested for allocation (do_realloc = TRUE) */
  if (thiz->do_realloc || !thiz->pool) {
    GstVideoCodecState *output_state =
        gst_video_decoder_get_output_state (GST_VIDEO_DECODER (thiz));
    if (thiz->pool)
      gst_object_replace ((GstObject **) & thiz->pool, NULL);
    GST_INFO_OBJECT (decoder, "create new MSDK bufferpool");
    thiz->pool =
        gst_msdkdec_create_buffer_pool (thiz, &output_state->info, min_buffers);
    gst_video_codec_state_unref (output_state);
    if (!thiz->pool) {
      GST_ERROR_OBJECT (decoder, "failed to create new pool");
      goto failed_to_create_pool;
    }
  }

  if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)
      && gst_buffer_pool_has_option (pool,
          GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT)) {
    GstStructure *config;
    GstAllocator *allocator;

    /* If downstream supports video meta and video alignment,
     * we can replace with our own msdk bufferpool and use it
     */
    /* Remove downstream's pool */
    gst_structure_free (pool_config);
    gst_object_unref (pool);

    pool = gst_object_ref (thiz->pool);

    /* Set the allocator of new msdk bufferpool */
    config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));

    if (gst_buffer_pool_config_get_allocator (config, &allocator, NULL))
      gst_query_set_nth_allocation_param (query, 0, allocator, NULL);
    gst_structure_free (config);
  } else {
    /* Unfortunately, downstream doesn't have videometa or alignment support,
     * we keep msdk pool as a side-pool that will be decoded into and
     * then copied from.
     */
    GstVideoCodecState *output_state = NULL;

    GST_INFO_OBJECT (decoder, "Keep MSDK bufferpool as a side-pool");

    /* Update params to downstream's pool */
    gst_buffer_pool_config_set_params (pool_config, pool_caps, size,
        min_buffers, max_buffers);
    if (!gst_buffer_pool_set_config (pool, pool_config))
      goto error_set_config;
    gst_video_info_from_caps (&thiz->non_msdk_pool_info, pool_caps);

    /* update width and height with actual negotiated values */
    output_state =
        gst_video_decoder_get_output_state (GST_VIDEO_DECODER (thiz));
    GST_VIDEO_INFO_WIDTH (&thiz->non_msdk_pool_info) =
        GST_VIDEO_INFO_WIDTH (&output_state->info);
    GST_VIDEO_INFO_HEIGHT (&thiz->non_msdk_pool_info) =
        GST_VIDEO_INFO_HEIGHT (&output_state->info);
    gst_video_codec_state_unref (output_state);
  }

  gst_query_set_nth_allocation_pool (query, 0, pool, size, min_buffers,
      max_buffers);

  if (pool)
    gst_object_unref (pool);


  return TRUE;

failed_to_create_pool:
  GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
  if (pool)
    gst_object_unref (pool);
  return FALSE;

error_set_config:
  GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
  if (pool)
    gst_object_unref (pool);
  return FALSE;
}

static GstFlowReturn
gst_msdkdec_drain (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
  GstFlowReturn flow;
  GstBuffer *buffer;
  MsdkDecTask *task;
  MsdkSurface *surface = NULL;
  mfxSession session;
  mfxStatus status;
  guint i;

  if (!thiz->initialized)
    return GST_FLOW_OK;
  session = gst_msdk_context_get_session (thiz->context);

  for (;;) {
    if (thiz->postpone_free_surface)
      gst_msdkdec_free_unlocked_msdk_surfaces (thiz, surface);
    task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
    if ((flow = gst_msdkdec_finish_task (thiz, task)) != GST_FLOW_OK) {
      if (flow != GST_FLOW_FLUSHING)
        GST_WARNING_OBJECT (decoder,
            "failed to finish the task %p, but keep draining for the remaining frames",
            task);
    }

    if (!surface) {
      flow = allocate_output_buffer (thiz, &buffer);
      if (flow != GST_FLOW_OK)
        return flow;
      surface = get_surface (thiz, buffer);
      if (!surface)
        return GST_FLOW_ERROR;
    }

    status =
        MFXVideoDECODE_DecodeFrameAsync (session, NULL, surface->surface,
        &task->surface, &task->sync_point);
    if (G_LIKELY (status == MFX_ERR_NONE)) {
      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;

      if (surface->surface->Data.Locked == 0)
        free_surface (thiz, surface);
      surface = NULL;
    } else if (status == MFX_WRN_VIDEO_PARAM_CHANGED) {
      continue;
    } else if (status == MFX_WRN_DEVICE_BUSY) {
      /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */
      g_usleep (1000);

      /* If the current surface is still busy, we should do sync operation,
       * then try to decode again
       */
      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
    } else if (status == MFX_ERR_MORE_DATA) {
      break;
    } else if (status == MFX_ERR_MORE_SURFACE) {
      surface = NULL;
      continue;
    } else if (status < MFX_ERR_NONE)
      return GST_FLOW_ERROR;
  }
  if (surface)
    free_surface (thiz, surface);

  for (i = 0; i < thiz->tasks->len; i++) {
    task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
    gst_msdkdec_finish_task (thiz, task);
    thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
  }

  release_msdk_surfaces (thiz);

  return GST_FLOW_OK;
}

static gboolean
gst_msdkdec_flush (GstVideoDecoder * decoder)
{
  GstMsdkDec *thiz = GST_MSDKDEC (decoder);

  return gst_msdkdec_drain (GST_VIDEO_DECODER_CAST (thiz));
}

static GstFlowReturn
gst_msdkdec_finish (GstVideoDecoder * decoder)
{
  return gst_msdkdec_drain (decoder);
}

static void
gst_msdkdec_set_property (GObject * object, guint prop_id, const GValue * value,
    GParamSpec * pspec)
{
  GstMsdkDec *thiz = GST_MSDKDEC (object);
  GstState state;

  GST_OBJECT_LOCK (thiz);

  state = GST_STATE (thiz);
  if ((state != GST_STATE_READY && state != GST_STATE_NULL) &&
      !(pspec->flags & GST_PARAM_MUTABLE_PLAYING))
    goto wrong_state;

  switch (prop_id) {
    case GST_MSDKDEC_PROP_HARDWARE:
      thiz->hardware = g_value_get_boolean (value);
      break;
    case GST_MSDKDEC_PROP_ASYNC_DEPTH:
      thiz->async_depth = g_value_get_uint (value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
  GST_OBJECT_UNLOCK (thiz);
  return;

  /* ERROR */
wrong_state:
  {
    GST_WARNING_OBJECT (thiz, "setting property in wrong state");
    GST_OBJECT_UNLOCK (thiz);
  }
}

static void
gst_msdkdec_get_property (GObject * object, guint prop_id, GValue * value,
    GParamSpec * pspec)
{
  GstMsdkDec *thiz = GST_MSDKDEC (object);

  GST_OBJECT_LOCK (thiz);
  switch (prop_id) {
    case GST_MSDKDEC_PROP_HARDWARE:
      g_value_set_boolean (value, thiz->hardware);
      break;
    case GST_MSDKDEC_PROP_ASYNC_DEPTH:
      g_value_set_uint (value, thiz->async_depth);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
  GST_OBJECT_UNLOCK (thiz);
}

static void
gst_msdkdec_finalize (GObject * object)
{
  GstMsdkDec *thiz = GST_MSDKDEC (object);

  g_array_unref (thiz->tasks);
  g_object_unref (thiz->adapter);

  /* NULL is the empty list. */
  if (G_UNLIKELY (thiz->decoded_msdk_surfaces != NULL)) {
    GST_ERROR_OBJECT (thiz, "leaking %u surfaces",
        g_list_length (thiz->decoded_msdk_surfaces));
  }
}

static gboolean
gst_msdkdec_post_configure (GstMsdkDec * decoder)
{
  /* Do nothing */
  return TRUE;
}

static gboolean
gst_msdkdec_preinit_decoder (GstMsdkDec * decoder)
{
  decoder->param.mfx.FrameInfo.Width =
      GST_ROUND_UP_16 (decoder->param.mfx.FrameInfo.Width);
  decoder->param.mfx.FrameInfo.Height =
      GST_ROUND_UP_32 (decoder->param.mfx.FrameInfo.Height);

  decoder->param.mfx.FrameInfo.PicStruct =
      decoder->param.mfx.FrameInfo.PicStruct ? decoder->param.mfx.
      FrameInfo.PicStruct : MFX_PICSTRUCT_PROGRESSIVE;

  return TRUE;
}

static gboolean
gst_msdkdec_postinit_decoder (GstMsdkDec * decoder)
{
  /* Do nothing */
  return TRUE;
}

static void
gst_msdkdec_class_init (GstMsdkDecClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *element_class;
  GstVideoDecoderClass *decoder_class;

  gobject_class = G_OBJECT_CLASS (klass);
  element_class = GST_ELEMENT_CLASS (klass);
  decoder_class = GST_VIDEO_DECODER_CLASS (klass);

  gobject_class->set_property = gst_msdkdec_set_property;
  gobject_class->get_property = gst_msdkdec_get_property;
  gobject_class->finalize = gst_msdkdec_finalize;

  element_class->set_context = gst_msdkdec_set_context;

  decoder_class->close = GST_DEBUG_FUNCPTR (gst_msdkdec_close);
  decoder_class->start = GST_DEBUG_FUNCPTR (gst_msdkdec_start);
  decoder_class->stop = GST_DEBUG_FUNCPTR (gst_msdkdec_stop);
  decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_msdkdec_set_format);
  decoder_class->finish = GST_DEBUG_FUNCPTR (gst_msdkdec_finish);
  decoder_class->handle_frame = GST_DEBUG_FUNCPTR (gst_msdkdec_handle_frame);
  decoder_class->parse = GST_DEBUG_FUNCPTR (gst_msdkdec_parse);
  decoder_class->decide_allocation =
      GST_DEBUG_FUNCPTR (gst_msdkdec_decide_allocation);
  decoder_class->flush = GST_DEBUG_FUNCPTR (gst_msdkdec_flush);
  decoder_class->drain = GST_DEBUG_FUNCPTR (gst_msdkdec_drain);

  klass->post_configure = GST_DEBUG_FUNCPTR (gst_msdkdec_post_configure);
  klass->preinit_decoder = GST_DEBUG_FUNCPTR (gst_msdkdec_preinit_decoder);
  klass->postinit_decoder = GST_DEBUG_FUNCPTR (gst_msdkdec_postinit_decoder);

  g_object_class_install_property (gobject_class, GST_MSDKDEC_PROP_HARDWARE,
      g_param_spec_boolean ("hardware", "Hardware", "Enable hardware decoders",
          PROP_HARDWARE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, GST_MSDKDEC_PROP_ASYNC_DEPTH,
      g_param_spec_uint ("async-depth", "Async Depth",
          "Depth of asynchronous pipeline",
          1, 20, PROP_ASYNC_DEPTH_DEFAULT,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  gst_element_class_add_static_pad_template (element_class, &src_factory);
}

static void
gst_msdkdec_init (GstMsdkDec * thiz)
{
  gst_video_info_init (&thiz->non_msdk_pool_info);
  thiz->tasks = g_array_new (FALSE, TRUE, sizeof (MsdkDecTask));
  thiz->hardware = PROP_HARDWARE_DEFAULT;
  thiz->async_depth = PROP_ASYNC_DEPTH_DEFAULT;
  thiz->do_renego = TRUE;
  thiz->do_realloc = TRUE;
  thiz->force_reset_on_res_change = TRUE;
  thiz->postpone_free_surface = FALSE;
  thiz->adapter = gst_adapter_new ();
  thiz->input_state = NULL;
  thiz->pool = NULL;
  thiz->context = NULL;
  thiz->decoded_msdk_surfaces = NULL;
}