d3d11: Add vp9 decoder element

Based on gstreamer-vaapi and Chromium implemetation.

sys/d3d11/gstd3d11_fwd.h

@@ -79,6 +79,9 @@ typedef struct _GstD3D11DecoderPrivate GstD3D11DecoderPrivate;
 typedef struct _GstD3D11H264Dec GstD3D11H264Dec;
 typedef struct _GstD3D11H264DecClass GstD3D11H264DecClass;
 
+typedef struct _GstD3D11Vp9Dec GstD3D11Vp9Dec;
+typedef struct _GstD3D11Vp9DecClass GstD3D11Vp9DecClass;
+
 G_END_DECLS
 
 #endif /* __GST_D3D11_FWD_H__ */

sys/d3d11/gstd3d11decoder.c

@@ -598,6 +598,11 @@ gst_d3d11_decoder_open (GstD3D11Decoder * decoder, GstD3D11Codec codec,
       best_config = &config_list[i];
       break;
     }
+
+    if (codec == GST_D3D11_CODEC_VP9 && config_list[i].ConfigBitstreamRaw == 1) {
+      best_config = &config_list[i];
+      break;
+    }
   }
 
   if (best_config == NULL) {

sys/d3d11/gstd3d11decoder.h

@@ -57,6 +57,7 @@ typedef enum
 {
   GST_D3D11_CODEC_NONE,
   GST_D3D11_CODEC_H264,
+  GST_D3D11_CODEC_VP9,
 
   /* the last of supported codec */
   GST_D3D11_CODEC_LAST

sys/d3d11/gstd3d11vp9dec.h

@@ -0,0 +1,72 @@
+/* GStreamer
+ * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_D3D11_VP9_DEC_H__
+#define __GST_D3D11_VP9_DEC_H__
+
+#include "gstvp9decoder.h"
+#include "gstvp9picture.h"
+#include "gstd3d11decoder.h"
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_D3D11_VP9_DEC \
+  (gst_d3d11_vp9_dec_get_type())
+#define GST_D3D11_VP9_DEC(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_D3D11_VP9_DEC,GstD3D11Vp9Dec))
+#define GST_D3D11_VP9_DEC_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_D3D11_VP9_DEC,GstD3D11Vp9DecClass))
+#define GST_D3D11_VP9_DEC_GET_CLASS(obj) \
+  (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_D3D11_VP9_DEC,GstD3D11Vp9DecClass))
+#define GST_IS_D3D11_VP9_DEC(obj) \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_D3D11_VP9_DEC))
+#define GST_IS_D3D11_VP9_DEC_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_D3D11_VP9_DEC))
+
+struct _GstD3D11Vp9Dec
+{
+  GstVp9Decoder parent;
+
+  GstVideoCodecState *output_state;
+
+  GstD3D11Device *device;
+  gint adapter;
+
+  GstD3D11Decoder *d3d11_decoder;
+
+  GstVp9Picture *current_picture;
+
+  guint width, height;
+  GstVP9Profile profile;
+
+  GstVideoFormat out_format;
+
+  gboolean use_d3d11_output;
+};
+
+struct _GstD3D11Vp9DecClass
+{
+  GstVp9DecoderClass parent_class;
+};
+
+GType gst_d3d11_vp9_dec_get_type (void);
+
+G_END_DECLS
+
+#endif /* __GST_D3D11_VP9_DEC_H__ */

sys/d3d11/gstvp9decoder.c

@@ -0,0 +1,434 @@
+/* GStreamer
+ * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/*
+ * Copyright 2015 The Chromium Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *    * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *    * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "gstvp9decoder.h"
+
+GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_vp9_dec_debug);
+#define GST_CAT_DEFAULT gst_d3d11_vp9_dec_debug
+
+struct _GstVp9DecoderPrivate
+{
+  gint width;
+  gint height;
+  GstVP9Profile profile;
+
+  gboolean had_sequence;
+
+  GstVp9Parser *parser;
+  GstVp9Dpb *dpb;
+
+  guint num_frames;             /* number of frames in a super frame */
+  gsize frame_sizes[8];         /* size of frames in a super frame */
+  guint frame_cnt;              /* frame count variable for super frame */
+  guint total_idx_size;         /* super frame index size (full block size) */
+  gboolean had_superframe_hdr;  /* indicate the presense of super frame */
+};
+
+#define parent_class gst_vp9_decoder_parent_class
+G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE (GstVp9Decoder, gst_vp9_decoder,
+    GST_TYPE_VIDEO_DECODER);
+
+static gboolean gst_vp9_decoder_start (GstVideoDecoder * decoder);
+static gboolean gst_vp9_decoder_stop (GstVideoDecoder * decoder);
+
+static GstFlowReturn gst_vp9_decoder_parse (GstVideoDecoder * decoder,
+    GstVideoCodecFrame * frame, GstAdapter * adapter, gboolean at_eos);
+static gboolean gst_vp9_decoder_set_format (GstVideoDecoder * decoder,
+    GstVideoCodecState * state);
+
+static GstVp9Picture *gst_vp9_decoder_duplicate_picture_default (GstVp9Decoder *
+    decoder, GstVp9Picture * picture);
+
+static void
+gst_vp9_decoder_class_init (GstVp9DecoderClass * klass)
+{
+  GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
+
+  decoder_class->start = GST_DEBUG_FUNCPTR (gst_vp9_decoder_start);
+  decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vp9_decoder_stop);
+  decoder_class->parse = GST_DEBUG_FUNCPTR (gst_vp9_decoder_parse);
+  decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_vp9_decoder_set_format);
+
+  klass->duplicate_picture =
+      GST_DEBUG_FUNCPTR (gst_vp9_decoder_duplicate_picture_default);
+}
+
+static void
+gst_vp9_decoder_init (GstVp9Decoder * self)
+{
+  gst_video_decoder_set_packetized (GST_VIDEO_DECODER (self), FALSE);
+
+  self->priv = gst_vp9_decoder_get_instance_private (self);
+}
+
+static gboolean
+gst_vp9_decoder_start (GstVideoDecoder * decoder)
+{
+  GstVp9Decoder *self = GST_VP9_DECODER (decoder);
+  GstVp9DecoderPrivate *priv = self->priv;
+
+  priv->parser = gst_vp9_parser_new ();
+  priv->dpb = gst_vp9_dpb_new ();
+
+  return TRUE;
+}
+
+static gboolean
+gst_vp9_decoder_stop (GstVideoDecoder * decoder)
+{
+  GstVp9Decoder *self = GST_VP9_DECODER (decoder);
+  GstVp9DecoderPrivate *priv = self->priv;
+
+  if (self->input_state) {
+    gst_video_codec_state_unref (self->input_state);
+    self->input_state = NULL;
+  }
+
+  if (priv->parser) {
+    gst_vp9_parser_free (priv->parser);
+    priv->parser = NULL;
+  }
+
+  if (priv->dpb) {
+    gst_vp9_dpb_free (priv->dpb);
+    priv->dpb = NULL;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_vp9_decoder_check_codec_change (GstVp9Decoder * self,
+    const GstVp9FrameHdr * frame_hdr)
+{
+  GstVp9DecoderPrivate *priv = self->priv;
+  gboolean ret = TRUE;
+  gboolean changed = FALSE;
+
+  if (priv->width != frame_hdr->width || priv->height != frame_hdr->height) {
+    GST_INFO_OBJECT (self, "resolution changed %dx%d", frame_hdr->width,
+        frame_hdr->height);
+    priv->width = frame_hdr->width;
+    priv->height = frame_hdr->height;
+    changed = TRUE;
+  }
+
+  if (priv->profile != frame_hdr->profile) {
+    GST_INFO_OBJECT (self, "profile changed %d", frame_hdr->profile);
+    priv->profile = frame_hdr->profile;
+    changed = TRUE;
+  }
+
+  if (changed || !priv->had_sequence) {
+    GstVp9DecoderClass *klass = GST_VP9_DECODER_GET_CLASS (self);
+
+    priv->had_sequence = TRUE;
+
+    if (klass->new_sequence)
+      ret = klass->new_sequence (self, frame_hdr);
+  }
+
+  return ret;
+}
+
+static gboolean
+gst_vp9_decoder_parse_super_frame (GstVp9Decoder * self, const guint8 * data,
+    gsize size, gsize * frame_sizes, guint * frame_count,
+    guint * total_idx_size)
+{
+  guint8 marker;
+  guint32 num_frames = 1, frame_size_length, total_index_size;
+  guint i, j;
+
+  if (size <= 0)
+    return FALSE;
+
+  marker = data[size - 1];
+
+  if ((marker & 0xe0) == 0xc0) {
+
+    GST_DEBUG_OBJECT (self, "Got VP9-Super Frame, size %" G_GSIZE_FORMAT, size);
+
+    num_frames = (marker & 0x7) + 1;
+    frame_size_length = ((marker >> 3) & 0x3) + 1;
+    total_index_size = 2 + num_frames * frame_size_length;
+
+    if ((size >= total_index_size)
+        && (data[size - total_index_size] == marker)) {
+      const guint8 *x = &data[size - total_index_size + 1];
+
+      for (i = 0; i < num_frames; i++) {
+        guint32 cur_frame_size = 0;
+
+        for (j = 0; j < frame_size_length; j++)
+          cur_frame_size |= (*x++) << (j * 8);
+
+        frame_sizes[i] = cur_frame_size;
+      }
+
+      *frame_count = num_frames;
+      *total_idx_size = total_index_size;
+    } else {
+      GST_ERROR_OBJECT (self, "Failed to parse Super-frame");
+      return FALSE;
+    }
+  } else {
+    *frame_count = num_frames;
+    frame_sizes[0] = size;
+    *total_idx_size = 0;
+  }
+
+  return TRUE;
+}
+
+static GstFlowReturn
+gst_vp9_decoder_parse (GstVideoDecoder * decoder,
+    GstVideoCodecFrame * frame, GstAdapter * adapter, gboolean at_eos)
+{
+  GstVp9Decoder *self = GST_VP9_DECODER (decoder);
+  GstVp9DecoderClass *klass = GST_VP9_DECODER_GET_CLASS (self);
+  GstVp9DecoderPrivate *priv = self->priv;
+  GstVp9ParserResult pres;
+  GstFlowReturn flow_ret = GST_FLOW_OK;
+  GstVp9FrameHdr frame_hdr;
+  GstVp9Picture *picture = NULL;
+  const guint8 *data;
+  gsize size;
+  guint buf_size;
+  GstClockTime pts;
+
+  size = gst_adapter_available (adapter);
+
+  if (size < 1) {
+    GST_LOG_OBJECT (self, "need more data");
+    return GST_VIDEO_DECODER_FLOW_NEED_DATA;
+  }
+
+  pts = gst_adapter_prev_pts (adapter, NULL);
+  data = (const guint8 *) gst_adapter_map (adapter, size);
+
+  if (!priv->had_superframe_hdr) {
+    if (!gst_vp9_decoder_parse_super_frame (self, data, size, priv->frame_sizes,
+            &priv->num_frames, &priv->total_idx_size)) {
+      goto unmap_and_error;
+    }
+
+    if (priv->num_frames > 1)
+      priv->had_superframe_hdr = TRUE;
+  }
+
+  buf_size = priv->frame_sizes[priv->frame_cnt++];
+
+  pres = gst_vp9_parser_parse_frame_header (priv->parser, &frame_hdr,
+      data, buf_size);
+
+  if (priv->frame_cnt == priv->num_frames) {
+    priv->num_frames = 0;
+    priv->frame_cnt = 0;
+    priv->had_superframe_hdr = FALSE;
+    buf_size += priv->total_idx_size;
+  }
+
+  if (pres != GST_VP9_PARSER_OK) {
+    GST_ERROR_OBJECT (self, "Failed to parsing frame header");
+    goto unmap_and_error;
+  }
+
+  if (frame_hdr.show_existing_frame) {
+    GstVp9Picture *pic_to_dup;
+
+    gst_adapter_unmap (adapter);
+
+    if (frame_hdr.frame_to_show >= GST_VP9_REF_FRAMES ||
+        !priv->dpb->pic_list[frame_hdr.frame_to_show]) {
+      GST_ERROR_OBJECT (self, "Invalid frame_to_show %d",
+          frame_hdr.frame_to_show);
+      goto error;
+    }
+
+    g_assert (klass->duplicate_picture);
+    pic_to_dup = priv->dpb->pic_list[frame_hdr.frame_to_show];
+    picture = klass->duplicate_picture (self, pic_to_dup);
+
+    if (!picture) {
+      GST_ERROR_OBJECT (self, "subclass didn't provide duplicated picture");
+      goto error;
+    }
+
+    picture->pts = pts;
+    picture->size = buf_size;
+
+    gst_video_decoder_add_to_frame (GST_VIDEO_DECODER (self), picture->size);
+    flow_ret = gst_video_decoder_have_frame (GST_VIDEO_DECODER (self));
+
+    if (flow_ret == GST_FLOW_OK) {
+      if (klass->output_picture)
+        flow_ret = klass->output_picture (self, picture);
+    }
+
+    gst_vp9_picture_unref (picture);
+
+    return flow_ret;
+  }
+
+  if (!gst_vp9_decoder_check_codec_change (self, &frame_hdr)) {
+    GST_ERROR_OBJECT (self, "codec change error");
+    goto unmap_and_error;
+  }
+
+  picture = gst_vp9_picture_new ();
+  picture->frame_hdr = frame_hdr;
+  picture->pts = pts;
+
+  picture->data = data;
+  picture->size = buf_size;
+
+  picture->subsampling_x = priv->parser->subsampling_x;
+  picture->subsampling_y = priv->parser->subsampling_y;
+  picture->bit_depth = priv->parser->bit_depth;
+
+  if (klass->new_picture) {
+    if (!klass->new_picture (self, picture)) {
+      GST_ERROR_OBJECT (self, "new picture error");
+      goto unmap_and_error;
+    }
+  }
+
+  if (klass->start_picture) {
+    if (!klass->start_picture (self, picture)) {
+      GST_ERROR_OBJECT (self, "start picture error");
+      goto unmap_and_error;
+    }
+  }
+
+  if (klass->decode_picture) {
+    if (!klass->decode_picture (self, picture, priv->dpb)) {
+      GST_ERROR_OBJECT (self, "decode picture error");
+      goto unmap_and_error;
+    }
+  }
+
+  if (klass->end_picture) {
+    if (!klass->end_picture (self, picture)) {
+      GST_ERROR_OBJECT (self, "end picture error");
+      goto unmap_and_error;
+    }
+  }
+
+  gst_adapter_unmap (adapter);
+
+  gst_video_decoder_add_to_frame (GST_VIDEO_DECODER (self), picture->size);
+  flow_ret = gst_video_decoder_have_frame (GST_VIDEO_DECODER (self));
+
+  if (flow_ret == GST_FLOW_OK && klass->output_picture) {
+    flow_ret = klass->output_picture (self, picture);
+  }
+
+  picture->data = NULL;
+
+  gst_vp9_dpb_add (priv->dpb, picture);
+
+  return flow_ret;
+
+unmap_and_error:
+  {
+    gst_adapter_unmap (adapter);
+    goto error;
+  }
+
+error:
+  {
+    if (picture)
+      gst_vp9_picture_unref (picture);
+
+    if (size)
+      gst_adapter_flush (adapter, size);
+
+    GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
+        ("Failed to decode data"), (NULL), flow_ret);
+
+    return flow_ret;
+  }
+}
+
+static gboolean
+gst_vp9_decoder_set_format (GstVideoDecoder * decoder,
+    GstVideoCodecState * state)
+{
+  GstVp9Decoder *self = GST_VP9_DECODER (decoder);
+  GstVp9DecoderPrivate *priv = self->priv;
+
+  GST_DEBUG_OBJECT (decoder, "Set format");
+
+  if (self->input_state)
+    gst_video_codec_state_unref (self->input_state);
+
+  self->input_state = gst_video_codec_state_ref (state);
+
+  priv->width = GST_VIDEO_INFO_WIDTH (&state->info);
+  priv->height = GST_VIDEO_INFO_HEIGHT (&state->info);
+
+  return TRUE;
+}
+
+static GstVp9Picture *
+gst_vp9_decoder_duplicate_picture_default (GstVp9Decoder * decoder,
+    GstVp9Picture * picture)
+{
+  GstVp9Picture *new_picture;
+
+  new_picture = gst_vp9_picture_new ();
+  new_picture->frame_hdr = picture->frame_hdr;
+
+  return new_picture;
+}

sys/d3d11/gstvp9decoder.h

@@ -0,0 +1,114 @@
+/* GStreamer
+ * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_VP9_DECODER_H__
+#define __GST_VP9_DECODER_H__
+
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/codecparsers/gstvp9parser.h>
+#include "gstvp9picture.h"
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_VP9_DECODER            (gst_vp9_decoder_get_type())
+#define GST_VP9_DECODER(obj)            (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VP9_DECODER,GstVp9Decoder))
+#define GST_VP9_DECODER_CLASS(klass)    (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VP9_DECODER,GstVp9DecoderClass))
+#define GST_VP9_DECODER_GET_CLASS(obj)  (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VP9_DECODER,GstVp9DecoderClass))
+#define GST_IS_VP9_DECODER(obj)         (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VP9_DECODER))
+#define GST_IS_VP9_DECODER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VP9_DECODER))
+#define GST_VP9_DECODER_CAST(obj)       ((GstVP9Decoder*)obj)
+
+typedef struct _GstVp9Decoder GstVp9Decoder;
+typedef struct _GstVp9DecoderClass GstVp9DecoderClass;
+typedef struct _GstVp9DecoderPrivate GstVp9DecoderPrivate;
+
+/**
+ * GstVp9Decoder:
+ *
+ * The opaque #GstVp9Decoder data structure.
+ */
+struct _GstVp9Decoder
+{
+  /*< private >*/
+  GstVideoDecoder parent;
+
+  /*< protected >*/
+  GstVideoCodecState * input_state;
+
+  /*< private >*/
+  GstVp9DecoderPrivate *priv;
+  gpointer padding[GST_PADDING_LARGE];
+};
+
+/**
+ * GstVp9DecoderClass:
+ * @new_sequence:      Notifies subclass of SPS update
+ * @new_picture:       Optional.
+ *                     Called whenever new #GstH264Picture is created.
+ *                     Subclass can set implementation specific user data
+ *                     on the #GstH264Picture via gst_h264_picture_set_user_data()
+ * @duplicate_picture: Duplicate the #GstVp9Picture
+ * @output_picture:    Optional.
+ *                     Called just before gst_video_decoder_have_frame().
+ *                     Subclass should be prepared for handle_frame()
+ * @start_picture:     Optional.
+ *                     Called per one #GstH264Picture to notify subclass to prepare
+ *                     decoding process for the #GstH264Picture
+ * @decode_slice:      Provides per slice data with parsed slice header and
+ *                     required raw bitstream for subclass to decode it
+ * @end_picture:       Optional.
+ *                     Called per one #GstH264Picture to notify subclass to finish
+ *                     decoding process for the #GstH264Picture
+ */
+struct _GstVp9DecoderClass
+{
+  GstVideoDecoderClass parent_class;
+
+  gboolean        (*new_sequence)      (GstVp9Decoder * decoder,
+                                        const GstVp9FrameHdr * frame_hdr);
+
+  gboolean        (*new_picture)       (GstVp9Decoder * decoder,
+                                        GstVp9Picture * picture);
+
+  GstVp9Picture * (*duplicate_picture) (GstVp9Decoder * decoder,
+                                        GstVp9Picture * picture);
+
+  GstFlowReturn   (*output_picture)    (GstVp9Decoder * decoder,
+                                        GstVp9Picture * picture);
+
+  gboolean        (*start_picture)     (GstVp9Decoder * decoder,
+                                        GstVp9Picture * picture);
+
+  gboolean        (*decode_picture)    (GstVp9Decoder * decoder,
+                                        GstVp9Picture * picture,
+                                        GstVp9Dpb * dpb);
+
+  gboolean        (*end_picture)       (GstVp9Decoder * decoder,
+                                        GstVp9Picture * picture);
+
+  /*< private >*/
+  gpointer padding[GST_PADDING_LARGE];
+};
+
+GType gst_vp9_decoder_get_type (void);
+
+G_END_DECLS
+
+#endif /* __GST_VP9_DECODER_H__ */

sys/d3d11/gstvp9picture.c

@@ -0,0 +1,189 @@
+/* GStreamer
+ * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "gstvp9picture.h"
+
+GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_vp9_dec_debug);
+#define GST_CAT_DEFAULT gst_d3d11_vp9_dec_debug
+
+GST_DEFINE_MINI_OBJECT_TYPE (GstVp9Picture, gst_vp9_picture);
+
+static void
+_gst_vp9_picture_free (GstVp9Picture * picture)
+{
+  GST_TRACE ("Free picture %p", picture);
+
+  if (picture->notify)
+    picture->notify (picture->user_data);
+
+  g_free (picture);
+}
+
+/**
+ * gst_vp9_picture_new:
+ * @buffer: (transfer none): a #GstBuffer
+ *
+ * Create new #GstVp9Picture
+ *
+ * Returns: a new #GstVp9Picture
+ */
+GstVp9Picture *
+gst_vp9_picture_new (void)
+{
+  GstVp9Picture *pic;
+
+  pic = g_new0 (GstVp9Picture, 1);
+  pic->pts = GST_CLOCK_TIME_NONE;
+
+  gst_mini_object_init (GST_MINI_OBJECT_CAST (pic), 0,
+      GST_TYPE_VP9_PICTURE, NULL, NULL,
+      (GstMiniObjectFreeFunction) _gst_vp9_picture_free);
+
+  GST_TRACE ("New picture %p", pic);
+
+  return pic;
+}
+
+/**
+ * gst_vp9_picture_set_user_data:
+ * @picture: a #GstVp9Picture
+ * @user_data: private data
+ * @notify: (closure user_data): a #GDestroyNotify
+ *
+ * Sets @user_data on the picture and the #GDestroyNotify that will be called when
+ * the picture is freed.
+ *
+ * If a @user_data was previously set, then the previous set @notify will be called
+ * before the @user_data is replaced.
+ */
+void
+gst_vp9_picture_set_user_data (GstVp9Picture * picture, gpointer user_data,
+    GDestroyNotify notify)
+{
+  g_return_if_fail (GST_IS_VP9_PICTURE (picture));
+
+  if (picture->notify)
+    picture->notify (picture->user_data);
+
+  picture->user_data = user_data;
+  picture->notify = notify;
+}
+
+/**
+ * gst_vp9_picture_get_user_data:
+ * @picture: a #GstVp9Picture
+ *
+ * Gets private data set on the picture via
+ * gst_vp9_picture_set_user_data() previously.
+ *
+ * Returns: (transfer none): The previously set user_data
+ */
+gpointer
+gst_vp9_picture_get_user_data (GstVp9Picture * picture)
+{
+  return picture->user_data;
+}
+
+/**
+ * gst_vp9_dpb_new:
+ *
+ * Create new #GstVp9Dpb
+ *
+ * Returns: a new #GstVp9Dpb
+ */
+GstVp9Dpb *
+gst_vp9_dpb_new (void)
+{
+  GstVp9Dpb *dpb;
+
+  dpb = g_new0 (GstVp9Dpb, 1);
+
+  return dpb;
+}
+
+/**
+ * gst_vp9_dpb_free:
+ * @dpb: a #GstVp9Dpb to free
+ *
+ * Free the @dpb
+ */
+void
+gst_vp9_dpb_free (GstVp9Dpb * dpb)
+{
+  g_return_if_fail (dpb != NULL);
+
+  gst_vp9_dpb_clear (dpb);
+  g_free (dpb);
+}
+
+/**
+ * gst_vp9_dpb_clear:
+ * @dpb: a #GstVp9Dpb
+ *
+ * Clear all stored #GstVp9Picture
+ */
+void
+gst_vp9_dpb_clear (GstVp9Dpb * dpb)
+{
+  gint i;
+
+  g_return_if_fail (dpb != NULL);
+
+  for (i = 0; i < GST_VP9_REF_FRAMES; i++)
+    gst_vp9_picture_clear (&dpb->pic_list[i]);
+}
+
+/**
+ * gst_vp9_dpb_add:
+ * @dpb: a #GstVp9Dpb
+ * @picture: (transfer full): a #GstVp9Picture
+ *
+ * Store the @picture
+ */
+void
+gst_vp9_dpb_add (GstVp9Dpb * dpb, GstVp9Picture * picture)
+{
+  guint8 refresh_frame_flags;
+  gint i;
+
+  g_return_if_fail (dpb != NULL);
+  g_return_if_fail (GST_IS_VP9_PICTURE (picture));
+
+  if (picture->frame_hdr.frame_type == GST_VP9_KEY_FRAME) {
+    refresh_frame_flags = (1 << GST_VP9_REF_FRAMES) - 1;
+    GST_TRACE ("keyframe, fill to all pictures");
+  } else {
+    refresh_frame_flags = picture->frame_hdr.refresh_frame_flags;
+    GST_TRACE ("non-keyframe, refresh frame flags 0x%x", refresh_frame_flags);
+  }
+
+  for (i = 0; i < GST_VP9_REF_FRAMES; i++) {
+    if (refresh_frame_flags & 0x1) {
+      gst_vp9_picture_replace (&dpb->pic_list[i], picture);
+    }
+
+    refresh_frame_flags >>= 1;
+  }
+
+  gst_vp9_picture_unref (picture);
+}

sys/d3d11/gstvp9picture.h

@@ -0,0 +1,128 @@
+/* GStreamer
+ * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_VP9_PICTURE_H__
+#define __GST_VP9_PICTURE_H__
+
+#include <gst/gst.h>
+#include <gst/codecparsers/gstvp9parser.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_VP9_PICTURE     (gst_vp9_picture_get_type())
+#define GST_IS_VP9_PICTURE(obj)  (GST_IS_MINI_OBJECT_TYPE(obj, GST_TYPE_VP9_PICTURE))
+#define GST_VP9_PICTURE(obj)     ((GstVp9Picture *)obj)
+#define GST_VP9_PICTURE_CAST(obj) (GST_VP9_PICTURE(obj))
+
+typedef struct _GstVp9Picture GstVp9Picture;
+
+struct _GstVp9Picture
+{
+  GstMiniObject parent;
+
+  GstClockTime pts;
+
+  GstVp9FrameHdr frame_hdr;
+
+  /* copied from parser */
+  gint subsampling_x;
+  gint subsampling_y;
+  guint bit_depth;
+
+  /* raw data and size (does not have ownership) */
+  const guint8 * data;
+  gsize size;
+
+  gpointer user_data;
+  GDestroyNotify notify;
+};
+
+G_GNUC_INTERNAL
+GType gst_vp9_picture_get_type (void);
+
+G_GNUC_INTERNAL
+GstVp9Picture * gst_vp9_picture_new (void);
+
+G_GNUC_INTERNAL
+static inline GstVp9Picture *
+gst_vp9_picture_ref (GstVp9Picture * picture)
+{
+  return (GstVp9Picture *) gst_mini_object_ref (GST_MINI_OBJECT_CAST (picture));
+}
+
+G_GNUC_INTERNAL
+static inline void
+gst_vp9_picture_unref (GstVp9Picture * picture)
+{
+  gst_mini_object_unref (GST_MINI_OBJECT_CAST (picture));
+}
+
+G_GNUC_INTERNAL
+static inline gboolean
+gst_vp9_picture_replace (GstVp9Picture ** old_picture,
+    GstVp9Picture * new_picture)
+{
+  return gst_mini_object_replace ((GstMiniObject **) old_picture,
+      (GstMiniObject *) new_picture);
+}
+
+G_GNUC_INTERNAL
+static inline void
+gst_vp9_picture_clear (GstVp9Picture ** picture)
+{
+  if (picture && *picture) {
+    gst_vp9_picture_unref (*picture);
+    *picture = NULL;
+  }
+}
+
+G_GNUC_INTERNAL
+void gst_vp9_picture_set_user_data (GstVp9Picture * picture,
+                                    gpointer user_data,
+                                    GDestroyNotify notify);
+
+G_GNUC_INTERNAL
+gpointer gst_vp9_picture_get_user_data (GstVp9Picture * picture);
+
+/*******************
+ * GstVp9Dpb *
+ *******************/
+typedef struct _GstVp9Dpb GstVp9Dpb;
+
+struct _GstVp9Dpb
+{
+  GstVp9Picture *pic_list[GST_VP9_REF_FRAMES];
+};
+
+G_GNUC_INTERNAL
+GstVp9Dpb * gst_vp9_dpb_new (void);
+
+G_GNUC_INTERNAL
+void  gst_vp9_dpb_free             (GstVp9Dpb * dpb);
+
+G_GNUC_INTERNAL
+void  gst_vp9_dpb_clear            (GstVp9Dpb * dpb);
+
+G_GNUC_INTERNAL
+void  gst_vp9_dpb_add              (GstVp9Dpb * dpb,
+                                    GstVp9Picture * picture);
+
+G_END_DECLS
+
+#endif /* __GST_VP9_PICTURE_H__ */

sys/d3d11/meson.build

@@ -22,6 +22,9 @@ d3d11_dec_sources = [
   'gsth264decoder.c',
   'gstd3d11decoder.c',
   'gstd3d11h264dec.c',
+  'gstvp9picture.c',
+  'gstvp9decoder.c',
+  'gstd3d11vp9dec.c',
 ]
 
 dxgi_headers = [

sys/d3d11/plugin.c

@@ -32,6 +32,7 @@
 #ifdef HAVE_DXVA_H
 #include "gstd3d11utils.h"
 #include "gstd3d11h264dec.h"
+#include "gstd3d11vp9dec.h"
 #endif
 
 GST_DEBUG_CATEGORY (gst_d3d11_shader_debug);
@@ -47,6 +48,7 @@ GST_DEBUG_CATEGORY (gst_d3d11_debug_layer_debug);
 
 #ifdef HAVE_DXVA_H
 GST_DEBUG_CATEGORY (gst_d3d11_h264_dec_debug);
+GST_DEBUG_CATEGORY (gst_d3d11_vp9_dec_debug);
 #endif
 
 static gboolean
@@ -87,9 +89,13 @@ plugin_init (GstPlugin * plugin)
   if (gst_d3d11_is_windows_8_or_greater ()) {
     GST_DEBUG_CATEGORY_INIT (gst_d3d11_h264_dec_debug,
         "d3d11h264dec", 0, "Direct3D11 H.264 Video Decoder");
+    GST_DEBUG_CATEGORY_INIT (gst_d3d11_vp9_dec_debug,
+        "d3d11vp9dec", 0, "Direct3D11 VP9 Video Decoder");
 
     gst_element_register (plugin,
         "d3d11h264dec", GST_RANK_SECONDARY, GST_TYPE_D3D11_H264_DEC);
+    gst_element_register (plugin,
+        "d3d11vp9dec", GST_RANK_SECONDARY, GST_TYPE_D3D11_VP9_DEC);
   }
 #endif