From 6f6a82d0f547dd84a93c0124248cb76844d535eb Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Mon, 11 Jan 2021 01:06:24 +0900 Subject: [PATCH] d3d11: Add support for MPEG-2 video decoding Add DXVA/Direct3D11 API based MPEG-2 decoder element Part-of: --- sys/d3d11/gstd3d11decoder.c | 5 + sys/d3d11/gstd3d11decoder.h | 1 + sys/d3d11/gstd3d11mpeg2dec.c | 1028 ++++++++++++++++++++++++++++++++++ sys/d3d11/gstd3d11mpeg2dec.h | 34 ++ sys/d3d11/meson.build | 1 + sys/d3d11/plugin.c | 6 + 6 files changed, 1075 insertions(+) create mode 100644 sys/d3d11/gstd3d11mpeg2dec.c create mode 100644 sys/d3d11/gstd3d11mpeg2dec.h diff --git a/sys/d3d11/gstd3d11decoder.c b/sys/d3d11/gstd3d11decoder.c index 403b0d2dc7..7856c96571 100644 --- a/sys/d3d11/gstd3d11decoder.c +++ b/sys/d3d11/gstd3d11decoder.c @@ -627,6 +627,10 @@ gst_d3d11_decoder_open (GstD3D11Decoder * decoder, GstD3D11Codec codec, else alignment = 16; break; + case GST_D3D11_CODEC_MPEG2: + /* XXX: ffmpeg does this */ + alignment = 32; + break; default: alignment = 16; break; @@ -676,6 +680,7 @@ gst_d3d11_decoder_open (GstD3D11Decoder * decoder, GstD3D11Codec codec, case GST_D3D11_CODEC_H265: case GST_D3D11_CODEC_VP9: case GST_D3D11_CODEC_VP8: + case GST_D3D11_CODEC_MPEG2: if (config_list[i].ConfigBitstreamRaw == 1) best_config = &config_list[i]; break; diff --git a/sys/d3d11/gstd3d11decoder.h b/sys/d3d11/gstd3d11decoder.h index ca1732ff1d..15a489705d 100644 --- a/sys/d3d11/gstd3d11decoder.h +++ b/sys/d3d11/gstd3d11decoder.h @@ -39,6 +39,7 @@ typedef enum GST_D3D11_CODEC_VP9, GST_D3D11_CODEC_H265, GST_D3D11_CODEC_VP8, + GST_D3D11_CODEC_MPEG2, /* the last of supported codec */ GST_D3D11_CODEC_LAST diff --git a/sys/d3d11/gstd3d11mpeg2dec.c b/sys/d3d11/gstd3d11mpeg2dec.c new file mode 100644 index 0000000000..daf049febd --- /dev/null +++ b/sys/d3d11/gstd3d11mpeg2dec.c @@ -0,0 +1,1028 @@ +/* GStreamer + * Copyright (C) 2021 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "gstd3d11mpeg2dec.h" + +#include +#include + +/* HACK: to expose dxva data structure on UWP */ +#ifdef WINAPI_PARTITION_DESKTOP +#undef WINAPI_PARTITION_DESKTOP +#endif +#define WINAPI_PARTITION_DESKTOP 1 +#include +#include + +GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_mpeg2_dec_debug); +#define GST_CAT_DEFAULT gst_d3d11_mpeg2_dec_debug + +enum +{ + PROP_0, + PROP_ADAPTER, + PROP_DEVICE_ID, + PROP_VENDOR_ID, +}; + +/* copied from d3d11.h since mingw header doesn't define them */ +DEFINE_GUID (GST_GUID_D3D11_DECODER_PROFILE_MPEG2_VLD, 0xee27417f, 0x5e28, + 0x4e65, 0xbe, 0xea, 0x1d, 0x26, 0xb5, 0x08, 0xad, 0xc9); +DEFINE_GUID (GST_GUID_D3D11_DECODER_PROFILE_MPEG2and1_VLD, 0x86695f12, 0x340e, + 0x4f04, 0x9f, 0xd3, 0x92, 0x53, 0xdd, 0x32, 0x74, 0x60); + +/* reference list 2 + 4 margin */ +#define NUM_OUTPUT_VIEW 6 + +typedef struct _GstD3D11Mpeg2Dec +{ + GstMpeg2Decoder parent; + + GstVideoCodecState *output_state; + GstD3D11Device *device; + GstD3D11Decoder *d3d11_decoder; + + guint width, height; + guint width_in_mb, height_in_mb; + GstVideoFormat out_format; + GstMpegVideoSequenceHdr seq; + GstMpegVideoProfile profile; + gboolean interlaced; + + /* Array of DXVA_SliceInfo */ + GArray *slice_list; + gboolean submit_iq_data; + + /* Pointing current bitstream buffer */ + guint written_buffer_size; + guint remaining_buffer_size; + guint8 *bitstream_buffer_data; + + gboolean use_d3d11_output; +} GstD3D11Mpeg2Dec; + +typedef struct _GstD3D11Mpeg2DecClass +{ + GstMpeg2DecoderClass parent_class; + guint adapter; + guint device_id; + guint vendor_id; +} GstD3D11Mpeg2DecClass; + +static GstElementClass *parent_class = NULL; + +#define GST_D3D11_MPEG2_DEC(object) ((GstD3D11Mpeg2Dec *) (object)) +#define GST_D3D11_MPEG2_DEC_GET_CLASS(object) \ + (G_TYPE_INSTANCE_GET_CLASS ((object),G_TYPE_FROM_INSTANCE (object),GstD3D11Mpeg2DecClass)) + +static void gst_d3d11_mpeg2_dec_finalize (GObject * object); +static void gst_d3d11_mpeg2_dec_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_d3d11_mpeg2_dec_set_context (GstElement * element, + GstContext * context); + +static gboolean gst_d3d11_mpeg2_dec_open (GstVideoDecoder * decoder); +static gboolean gst_d3d11_mpeg2_dec_close (GstVideoDecoder * decoder); +static gboolean gst_d3d11_mpeg2_dec_negotiate (GstVideoDecoder * decoder); +static gboolean gst_d3d11_mpeg2_dec_decide_allocation (GstVideoDecoder * + decoder, GstQuery * query); +static gboolean gst_d3d11_mpeg2_dec_src_query (GstVideoDecoder * decoder, + GstQuery * query); + +/* GstMpeg2Decoder */ +static gboolean gst_d3d11_mpeg2_dec_new_sequence (GstMpeg2Decoder * decoder, + const GstMpegVideoSequenceHdr * seq, + const GstMpegVideoSequenceExt * seq_ext, + const GstMpegVideoSequenceDisplayExt * seq_display_ext, + const GstMpegVideoSequenceScalableExt * seq_scalable_ext); +static gboolean gst_d3d11_mpeg2_dec_new_picture (GstMpeg2Decoder * decoder, + GstVideoCodecFrame * frame, GstMpeg2Picture * picture); +static gboolean gst_d3d11_mpeg2_dec_new_field_picture (GstMpeg2Decoder * + decoder, const GstMpeg2Picture * first_field, + GstMpeg2Picture * second_field); +static gboolean gst_d3d11_mpeg2_dec_start_picture (GstMpeg2Decoder * decoder, + GstMpeg2Picture * picture, GstMpeg2Slice * slice, + GstMpeg2Picture * prev_picture, GstMpeg2Picture * next_picture); +static gboolean gst_d3d11_mpeg2_dec_decode_slice (GstMpeg2Decoder * decoder, + GstMpeg2Picture * picture, GstMpeg2Slice * slice); +static gboolean gst_d3d11_mpeg2_dec_end_picture (GstMpeg2Decoder * decoder, + GstMpeg2Picture * picture); +static GstFlowReturn gst_d3d11_mpeg2_dec_output_picture (GstMpeg2Decoder * + decoder, GstVideoCodecFrame * frame, GstMpeg2Picture * picture); + +static void +gst_d3d11_mpeg2_dec_class_init (GstD3D11Mpeg2DecClass * klass, gpointer data) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass); + GstMpeg2DecoderClass *mpeg2decoder_class = GST_MPEG2_DECODER_CLASS (klass); + GstD3D11DecoderClassData *cdata = (GstD3D11DecoderClassData *) data; + gchar *long_name; + + gobject_class->get_property = gst_d3d11_mpeg2_dec_get_property; + gobject_class->finalize = gst_d3d11_mpeg2_dec_finalize; + + g_object_class_install_property (gobject_class, PROP_ADAPTER, + g_param_spec_uint ("adapter", "Adapter", + "DXGI Adapter index for creating device", + 0, G_MAXUINT32, cdata->adapter, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_DEVICE_ID, + g_param_spec_uint ("device-id", "Device Id", + "DXGI Device ID", 0, G_MAXUINT32, 0, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_VENDOR_ID, + g_param_spec_uint ("vendor-id", "Vendor Id", + "DXGI Vendor ID", 0, G_MAXUINT32, 0, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + parent_class = g_type_class_peek_parent (klass); + + klass->adapter = cdata->adapter; + klass->device_id = cdata->device_id; + klass->vendor_id = cdata->vendor_id; + + element_class->set_context = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_set_context); + + long_name = + g_strdup_printf ("Direct3D11 MPEG2 %s Decoder", cdata->description); + gst_element_class_set_metadata (element_class, long_name, + "Codec/Decoder/Video/Hardware", "A Direct3D11 based MPEG2 video decoder", + "Seungha Yang "); + g_free (long_name); + + gst_element_class_add_pad_template (element_class, + gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + cdata->sink_caps)); + gst_element_class_add_pad_template (element_class, + gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, + cdata->src_caps)); + gst_d3d11_decoder_class_data_free (cdata); + + decoder_class->open = GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_open); + decoder_class->close = GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_close); + decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_negotiate); + decoder_class->decide_allocation = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_decide_allocation); + decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_src_query); + + mpeg2decoder_class->new_sequence = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_new_sequence); + mpeg2decoder_class->new_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_new_picture); + mpeg2decoder_class->new_field_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_new_field_picture); + mpeg2decoder_class->start_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_start_picture); + mpeg2decoder_class->decode_slice = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_decode_slice); + mpeg2decoder_class->end_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_end_picture); + mpeg2decoder_class->output_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_mpeg2_dec_output_picture); +} + +static void +gst_d3d11_mpeg2_dec_init (GstD3D11Mpeg2Dec * self) +{ + self->slice_list = g_array_new (FALSE, TRUE, sizeof (DXVA_SliceInfo)); + self->profile = GST_MPEG_VIDEO_PROFILE_MAIN; +} + +static void +gst_d3d11_mpeg2_dec_finalize (GObject * object) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (object); + + g_array_unref (self->slice_list); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_d3d11_mpeg2_dec_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstD3D11Mpeg2DecClass *klass = GST_D3D11_MPEG2_DEC_GET_CLASS (object); + + switch (prop_id) { + case PROP_ADAPTER: + g_value_set_uint (value, klass->adapter); + break; + case PROP_DEVICE_ID: + g_value_set_uint (value, klass->device_id); + break; + case PROP_VENDOR_ID: + g_value_set_uint (value, klass->vendor_id); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d11_mpeg2_dec_set_context (GstElement * element, GstContext * context) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (element); + GstD3D11Mpeg2DecClass *klass = GST_D3D11_MPEG2_DEC_GET_CLASS (self); + + gst_d3d11_handle_set_context (element, context, klass->adapter, + &self->device); + + GST_ELEMENT_CLASS (parent_class)->set_context (element, context); +} + +static gboolean +gst_d3d11_mpeg2_dec_open (GstVideoDecoder * decoder) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + GstD3D11Mpeg2DecClass *klass = GST_D3D11_MPEG2_DEC_GET_CLASS (self); + + if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self), klass->adapter, + &self->device)) { + GST_ERROR_OBJECT (self, "Cannot create d3d11device"); + return FALSE; + } + + self->d3d11_decoder = gst_d3d11_decoder_new (self->device); + + if (!self->d3d11_decoder) { + GST_ERROR_OBJECT (self, "Cannot create d3d11 decoder"); + gst_clear_object (&self->device); + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_d3d11_mpeg2_dec_close (GstVideoDecoder * decoder) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + + gst_clear_object (&self->d3d11_decoder); + gst_clear_object (&self->device); + + return TRUE; +} + +static gboolean +gst_d3d11_mpeg2_dec_negotiate (GstVideoDecoder * decoder) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + GstMpeg2Decoder *mpeg2dec = GST_MPEG2_DECODER (decoder); + + if (!gst_d3d11_decoder_negotiate (decoder, mpeg2dec->input_state, + self->out_format, self->width, self->height, + self->interlaced ? GST_VIDEO_INTERLACE_MODE_MIXED : + GST_VIDEO_INTERLACE_MODE_PROGRESSIVE, + &self->output_state, &self->use_d3d11_output)) + return FALSE; + + return GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder); +} + +static gboolean +gst_d3d11_mpeg2_dec_decide_allocation (GstVideoDecoder * decoder, + GstQuery * query) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + + if (!gst_d3d11_decoder_decide_allocation (decoder, query, self->device, + GST_D3D11_CODEC_MPEG2, self->use_d3d11_output)) + return FALSE; + + return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation + (decoder, query); +} + +static gboolean +gst_d3d11_mpeg2_dec_src_query (GstVideoDecoder * decoder, GstQuery * query) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + if (gst_d3d11_handle_context_query (GST_ELEMENT (decoder), + query, self->device)) { + return TRUE; + } + break; + default: + break; + } + + return GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query); +} + +static gboolean +gst_d3d11_mpeg2_dec_new_sequence (GstMpeg2Decoder * decoder, + const GstMpegVideoSequenceHdr * seq, + const GstMpegVideoSequenceExt * seq_ext, + const GstMpegVideoSequenceDisplayExt * seq_display_ext, + const GstMpegVideoSequenceScalableExt * seq_scalable_ext) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + gboolean interlaced; + gboolean modified = FALSE; + static const GUID *supported_profiles[] = { + &GST_GUID_D3D11_DECODER_PROFILE_MPEG2_VLD, + &GST_GUID_D3D11_DECODER_PROFILE_MPEG2and1_VLD, + }; + gint width, height; + GstMpegVideoProfile mpeg_profile; + + GST_LOG_OBJECT (self, "new sequence"); + + interlaced = seq_ext ? !seq_ext->progressive : FALSE; + if (self->interlaced != interlaced) { + GST_INFO_OBJECT (self, "interlaced sequence change"); + self->interlaced = interlaced; + modified = TRUE; + } + + width = seq->width; + height = seq->height; + if (seq_ext) { + width = (width & 0x0fff) | ((guint32) seq_ext->horiz_size_ext << 12); + height = (height & 0x0fff) | ((guint32) seq_ext->vert_size_ext << 12); + } + + if (self->width != width || self->height != height) { + GST_INFO_OBJECT (self, "resolution change %dx%d -> %dx%d", + self->width, self->height, width, height); + self->width = width; + self->height = height; + self->width_in_mb = GST_ROUND_UP_16 (width) >> 4; + self->height_in_mb = GST_ROUND_UP_16 (height) >> 4; + modified = TRUE; + } + + mpeg_profile = GST_MPEG_VIDEO_PROFILE_MAIN; + if (seq_ext) + mpeg_profile = seq_ext->profile; + + if (mpeg_profile != GST_MPEG_VIDEO_PROFILE_MAIN && + mpeg_profile != GST_MPEG_VIDEO_PROFILE_SIMPLE) { + GST_ERROR_OBJECT (self, "Cannot support profile %d", mpeg_profile); + return FALSE; + } + + if (self->profile != mpeg_profile) { + GST_INFO_OBJECT (self, "Profile change %d -> %d", + self->profile, mpeg_profile); + self->profile = mpeg_profile; + modified = TRUE; + } + + if (modified || !self->d3d11_decoder->opened) { + GstVideoInfo info; + + /* FIXME: support I420 */ + self->out_format = GST_VIDEO_FORMAT_NV12; + + gst_video_info_set_format (&info, + self->out_format, self->width, self->height); + + gst_d3d11_decoder_reset (self->d3d11_decoder); + if (!gst_d3d11_decoder_open (self->d3d11_decoder, GST_D3D11_CODEC_MPEG2, + &info, self->width, self->height, NUM_OUTPUT_VIEW, + supported_profiles, G_N_ELEMENTS (supported_profiles))) { + GST_ERROR_OBJECT (self, "Failed to create decoder"); + return FALSE; + } + + if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) { + GST_ERROR_OBJECT (self, "Failed to negotiate with downstream"); + return FALSE; + } + } + + return TRUE; +} + +static gboolean +gst_d3d11_mpeg2_dec_new_picture (GstMpeg2Decoder * decoder, + GstVideoCodecFrame * frame, GstMpeg2Picture * picture) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + GstBuffer *view_buffer; + + view_buffer = gst_d3d11_decoder_get_output_view_buffer (self->d3d11_decoder); + if (!view_buffer) { + GST_ERROR_OBJECT (self, "No available output view buffer"); + return FALSE; + } + + GST_LOG_OBJECT (self, "New output view buffer %" GST_PTR_FORMAT, view_buffer); + + gst_mpeg2_picture_set_user_data (picture, + view_buffer, (GDestroyNotify) gst_buffer_unref); + + GST_LOG_OBJECT (self, "New MPEG2 picture %p", picture); + + return TRUE; +} + +static gboolean +gst_d3d11_mpeg2_dec_new_field_picture (GstMpeg2Decoder * decoder, + const GstMpeg2Picture * first_field, GstMpeg2Picture * second_field) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + GstBuffer *view_buffer; + + view_buffer = + gst_mpeg2_picture_get_user_data ((GstMpeg2Picture *) first_field); + + if (!view_buffer) { + GST_WARNING_OBJECT (self, "First picture does not have output view buffer"); + return TRUE; + } + + GST_LOG_OBJECT (self, "New field picture with buffer %" GST_PTR_FORMAT, + view_buffer); + + gst_mpeg2_picture_set_user_data (second_field, + gst_buffer_ref (view_buffer), (GDestroyNotify) gst_buffer_unref); + + return TRUE; +} + +static gboolean +gst_d3d11_mpeg2_dec_get_bitstream_buffer (GstD3D11Mpeg2Dec * self) +{ + GST_TRACE_OBJECT (self, "Getting bitstream buffer"); + + self->written_buffer_size = 0; + self->remaining_buffer_size = 0; + self->bitstream_buffer_data = NULL; + + if (!gst_d3d11_decoder_get_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_BITSTREAM, &self->remaining_buffer_size, + (gpointer *) & self->bitstream_buffer_data)) { + GST_ERROR_OBJECT (self, "Faild to get bitstream buffer"); + return FALSE; + } + + GST_TRACE_OBJECT (self, "Got bitstream buffer %p with size %d", + self->bitstream_buffer_data, self->remaining_buffer_size); + self->written_buffer_size = 0; + + return TRUE; +} + +static ID3D11VideoDecoderOutputView * +gst_d3d11_mpeg2_dec_get_output_view_from_picture (GstD3D11Mpeg2Dec * self, + GstMpeg2Picture * picture) +{ + GstBuffer *view_buffer; + ID3D11VideoDecoderOutputView *view; + + if (!picture) + return NULL; + + view_buffer = (GstBuffer *) gst_mpeg2_picture_get_user_data (picture); + if (!view_buffer) { + GST_DEBUG_OBJECT (self, "current picture does not have output view buffer"); + return NULL; + } + + view = + gst_d3d11_decoder_get_output_view_from_buffer (self->d3d11_decoder, + view_buffer); + if (!view) { + GST_DEBUG_OBJECT (self, "current picture does not have output view handle"); + return NULL; + } + + return view; +} + +static inline WORD +_pack_f_codes (guint8 f_code[2][2]) +{ + return (((WORD) f_code[0][0] << 12) + | ((WORD) f_code[0][1] << 8) + | ((WORD) f_code[1][0] << 4) + | (f_code[1][1])); +} + +static inline WORD +_pack_pce_elements (GstMpeg2Slice * slice) +{ + return (((WORD) slice->pic_ext->intra_dc_precision << 14) + | ((WORD) slice->pic_ext->picture_structure << 12) + | ((WORD) slice->pic_ext->top_field_first << 11) + | ((WORD) slice->pic_ext->frame_pred_frame_dct << 10) + | ((WORD) slice->pic_ext->concealment_motion_vectors << 9) + | ((WORD) slice->pic_ext->q_scale_type << 8) + | ((WORD) slice->pic_ext->intra_vlc_format << 7) + | ((WORD) slice->pic_ext->alternate_scan << 6) + | ((WORD) slice->pic_ext->repeat_first_field << 5) + | ((WORD) slice->pic_ext->chroma_420_type << 4) + | ((WORD) slice->pic_ext->progressive_frame << 3)); +} + +static gboolean +gst_d3d11_mpeg2_dec_start_picture (GstMpeg2Decoder * decoder, + GstMpeg2Picture * picture, GstMpeg2Slice * slice, + GstMpeg2Picture * prev_picture, GstMpeg2Picture * next_picture) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + ID3D11VideoDecoderOutputView *view; + ID3D11VideoDecoderOutputView *other_view; + DXVA_PictureParameters pic_params = { 0, }; + DXVA_QmatrixData iq_matrix = { 0, }; + guint d3d11_buffer_size = 0; + gpointer d3d11_buffer = NULL; + gboolean is_field = + picture->structure != GST_MPEG_VIDEO_PICTURE_STRUCTURE_FRAME; + + view = gst_d3d11_mpeg2_dec_get_output_view_from_picture (self, picture); + if (!view) { + GST_ERROR_OBJECT (self, "current picture does not have output view handle"); + return FALSE; + } + + GST_TRACE_OBJECT (self, "Begin frame"); + if (!gst_d3d11_decoder_begin_frame (self->d3d11_decoder, view, 0, NULL)) { + GST_ERROR_OBJECT (self, "Failed to begin frame"); + return FALSE; + } + + /* Fill DXVA_PictureParameters */ + pic_params.wDecodedPictureIndex = + gst_d3d11_decoder_get_output_view_index (view); + pic_params.wForwardRefPictureIndex = 0xffff; + pic_params.wBackwardRefPictureIndex = 0xffff; + + switch (picture->type) { + case GST_MPEG_VIDEO_PICTURE_TYPE_B:{ + if (next_picture) { + other_view = + gst_d3d11_mpeg2_dec_get_output_view_from_picture (self, + next_picture); + if (other_view) + pic_params.wBackwardRefPictureIndex = + gst_d3d11_decoder_get_output_view_index (other_view); + } + } + /* fall-through */ + case GST_MPEG_VIDEO_PICTURE_TYPE_P:{ + if (prev_picture) { + other_view = + gst_d3d11_mpeg2_dec_get_output_view_from_picture (self, + prev_picture); + if (other_view) + pic_params.wForwardRefPictureIndex = + gst_d3d11_decoder_get_output_view_index (other_view); + } + } + default: + break; + } + + pic_params.wPicWidthInMBminus1 = self->width_in_mb - 1; + pic_params.wPicHeightInMBminus1 = (self->height_in_mb >> is_field) - 1; + pic_params.bMacroblockWidthMinus1 = 15; + pic_params.bMacroblockHeightMinus1 = 15; + pic_params.bBlockWidthMinus1 = 7; + pic_params.bBlockHeightMinus1 = 7; + pic_params.bBPPminus1 = 7; + pic_params.bPicStructure = (BYTE) picture->structure; + pic_params.bSecondField = is_field && ! !picture->first_field; + pic_params.bPicIntra = picture->type == GST_MPEG_VIDEO_PICTURE_TYPE_I; + pic_params.bPicBackwardPrediction = + picture->type == GST_MPEG_VIDEO_PICTURE_TYPE_B; + /* FIXME: 1 -> 4:2:0, 2 -> 4:2:2, 3 -> 4:4:4 */ + pic_params.bChromaFormat = 1; + pic_params.bPicScanFixed = 1; + pic_params.bPicScanMethod = slice->pic_ext->alternate_scan; + pic_params.wBitstreamFcodes = _pack_f_codes (slice->pic_ext->f_code); + pic_params.wBitstreamPCEelements = _pack_pce_elements (slice); + + GST_TRACE_OBJECT (self, "Getting picture param decoder buffer"); + if (!gst_d3d11_decoder_get_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_PICTURE_PARAMETERS, &d3d11_buffer_size, + &d3d11_buffer)) { + GST_ERROR_OBJECT (self, + "Failed to get decoder buffer for picture parameters"); + return FALSE; + } + + memcpy (d3d11_buffer, &pic_params, sizeof (pic_params)); + + GST_TRACE_OBJECT (self, "Release picture param decoder buffer"); + if (!gst_d3d11_decoder_release_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_PICTURE_PARAMETERS)) { + GST_ERROR_OBJECT (self, "Failed to release decoder buffer"); + return FALSE; + } + + /* Fill DXVA_QmatrixData */ + if (slice->quant_matrix && + /* The value in bNewQmatrix[0] and bNewQmatrix[1] must not both be zero. + * https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/dxva/ns-dxva-_dxva_qmatrixdata + */ + (slice->quant_matrix->load_intra_quantiser_matrix || + slice->quant_matrix->load_non_intra_quantiser_matrix)) { + GstMpegVideoQuantMatrixExt *quant_matrix = slice->quant_matrix; + self->submit_iq_data = TRUE; + + if (quant_matrix->load_intra_quantiser_matrix) { + iq_matrix.bNewQmatrix[0] = 1; + memcpy (iq_matrix.Qmatrix[0], quant_matrix->intra_quantiser_matrix, + sizeof (quant_matrix->intra_quantiser_matrix)); + } + + if (quant_matrix->load_non_intra_quantiser_matrix) { + iq_matrix.bNewQmatrix[1] = 1; + memcpy (iq_matrix.Qmatrix[1], quant_matrix->non_intra_quantiser_matrix, + sizeof (quant_matrix->non_intra_quantiser_matrix)); + } + + if (quant_matrix->load_chroma_intra_quantiser_matrix) { + iq_matrix.bNewQmatrix[2] = 1; + memcpy (iq_matrix.Qmatrix[2], quant_matrix->chroma_intra_quantiser_matrix, + sizeof (quant_matrix->chroma_intra_quantiser_matrix)); + } + + if (quant_matrix->load_chroma_non_intra_quantiser_matrix) { + iq_matrix.bNewQmatrix[3] = 1; + memcpy (iq_matrix.Qmatrix[3], + quant_matrix->chroma_non_intra_quantiser_matrix, + sizeof (quant_matrix->chroma_non_intra_quantiser_matrix)); + } + + GST_TRACE_OBJECT (self, "Getting inverse quantization matrix buffer"); + if (!gst_d3d11_decoder_get_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_INVERSE_QUANTIZATION_MATRIX, + &d3d11_buffer_size, &d3d11_buffer)) { + GST_ERROR_OBJECT (self, + "Failed to get decoder buffer for inv. quantization matrix"); + return FALSE; + } + + memcpy (d3d11_buffer, &iq_matrix, sizeof (DXVA_QmatrixData)); + + GST_TRACE_OBJECT (self, "Release inverse quantization matrix buffer"); + if (!gst_d3d11_decoder_release_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_INVERSE_QUANTIZATION_MATRIX)) { + GST_ERROR_OBJECT (self, "Failed to release decoder buffer"); + return FALSE; + } + } else { + self->submit_iq_data = FALSE; + } + + g_array_set_size (self->slice_list, 0); + + return gst_d3d11_mpeg2_dec_get_bitstream_buffer (self); +} + +static gboolean +gst_d3d11_mpeg2_dec_submit_slice_data (GstD3D11Mpeg2Dec * self, + GstMpeg2Picture * picture) +{ + guint buffer_size; + gpointer buffer; + guint8 *data; + gsize offset = 0; + gint i; + D3D11_VIDEO_DECODER_BUFFER_DESC buffer_desc[4] = { 0, }; + gboolean ret; + guint buffer_count = 0; + DXVA_SliceInfo *slice_data; + gboolean is_field = + picture->structure != GST_MPEG_VIDEO_PICTURE_STRUCTURE_FRAME; + guint mb_count = self->width_in_mb * (self->height_in_mb >> is_field); + + if (self->slice_list->len < 1) { + GST_WARNING_OBJECT (self, "Nothing to submit"); + return FALSE; + } + + GST_TRACE_OBJECT (self, "Getting slice control buffer"); + + if (!gst_d3d11_decoder_get_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_SLICE_CONTROL, &buffer_size, &buffer)) { + GST_ERROR_OBJECT (self, "Couldn't get slice control buffer"); + return FALSE; + } + + data = buffer; + for (i = 0; i < self->slice_list->len; i++) { + slice_data = &g_array_index (self->slice_list, DXVA_SliceInfo, i); + + /* Update the number of MBs per slice */ + if (i == self->slice_list->len - 1) { + slice_data->wNumberMBsInSlice = mb_count - slice_data->wNumberMBsInSlice; + } else { + DXVA_SliceInfo *next = + &g_array_index (self->slice_list, DXVA_SliceInfo, i + 1); + slice_data->wNumberMBsInSlice = + next->wNumberMBsInSlice - slice_data->wNumberMBsInSlice; + } + + memcpy (data + offset, slice_data, sizeof (DXVA_SliceInfo)); + offset += sizeof (DXVA_SliceInfo); + } + + GST_TRACE_OBJECT (self, "Release slice control buffer"); + if (!gst_d3d11_decoder_release_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_SLICE_CONTROL)) { + GST_ERROR_OBJECT (self, "Failed to release slice control buffer"); + return FALSE; + } + + if (!gst_d3d11_decoder_release_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_BITSTREAM)) { + GST_ERROR_OBJECT (self, "Failed to release bitstream buffer"); + return FALSE; + } + + buffer_desc[buffer_count].BufferType = + D3D11_VIDEO_DECODER_BUFFER_PICTURE_PARAMETERS; + buffer_desc[buffer_count].DataOffset = 0; + buffer_desc[buffer_count].DataSize = sizeof (DXVA_PictureParameters); + buffer_count++; + + if (self->submit_iq_data) { + buffer_desc[buffer_count].BufferType = + D3D11_VIDEO_DECODER_BUFFER_INVERSE_QUANTIZATION_MATRIX; + buffer_desc[buffer_count].DataOffset = 0; + buffer_desc[buffer_count].DataSize = sizeof (DXVA_QmatrixData); + buffer_count++; + } + + buffer_desc[buffer_count].BufferType = + D3D11_VIDEO_DECODER_BUFFER_SLICE_CONTROL; + buffer_desc[buffer_count].DataOffset = 0; + buffer_desc[buffer_count].DataSize = + sizeof (DXVA_SliceInfo) * self->slice_list->len; + buffer_count++; + + buffer_desc[buffer_count].BufferType = D3D11_VIDEO_DECODER_BUFFER_BITSTREAM; + buffer_desc[buffer_count].DataOffset = 0; + buffer_desc[buffer_count].DataSize = self->written_buffer_size; + buffer_count++; + + ret = gst_d3d11_decoder_submit_decoder_buffers (self->d3d11_decoder, + buffer_count, buffer_desc); + + self->written_buffer_size = 0; + self->bitstream_buffer_data = NULL; + self->remaining_buffer_size = 0; + g_array_set_size (self->slice_list, 0); + + return ret; +} + +static gboolean +gst_d3d11_mpeg2_dec_decode_slice (GstMpeg2Decoder * decoder, + GstMpeg2Picture * picture, GstMpeg2Slice * slice) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + GstMpegVideoSliceHdr *header = &slice->header; + GstMpegVideoPacket *packet = &slice->packet; + /* including start code 4 bytes */ + guint to_write = packet->size + 4; + DXVA_SliceInfo slice_info = { 0, }; + + g_assert (packet->offset >= 4); + + /* FIXME: DXVA wants to know the number of MBs per slice + * (not sure whether it's actually used by driver). But in case that + * one slice is splitted into two bitstream buffer, it's almost impossible + * to know the number of MBs per splitted bitstream buffer. + * So, we will not support too large bitstream buffer which requires multiple + * hardware bitstream buffer at this moment. + */ + if (self->remaining_buffer_size < to_write) { + /* Submit slice data we have so that release acquired bitstream buffers */ + if (self->bitstream_buffer_data) + gst_d3d11_mpeg2_dec_submit_slice_data (self, picture); + self->bitstream_buffer_data = 0; + + GST_ERROR_OBJECT (self, "Slice data is too large"); + + return FALSE; + } + + slice_info.wHorizontalPosition = header->mb_column; + slice_info.wVerticalPosition = header->mb_row; + slice_info.dwSliceBitsInBuffer = 8 * to_write; + slice_info.dwSliceDataLocation = self->written_buffer_size; + /* XXX: We don't have information about the number of MBs in this slice. + * Just store offset here, and actual number will be calculated later */ + slice_info.wNumberMBsInSlice = + (header->mb_row * self->width_in_mb) + header->mb_column; + slice_info.wQuantizerScaleCode = header->quantiser_scale_code; + slice_info.wMBbitOffset = header->header_size + 32; + memcpy (self->bitstream_buffer_data, packet->data + packet->offset - 4, + to_write); + + g_array_append_val (self->slice_list, slice_info); + self->remaining_buffer_size -= to_write; + self->written_buffer_size += to_write; + self->bitstream_buffer_data += to_write; + + return TRUE; +} + +static GstFlowReturn +gst_d3d11_mpeg2_dec_output_picture (GstMpeg2Decoder * decoder, + GstVideoCodecFrame * frame, GstMpeg2Picture * picture) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder); + GstBuffer *output_buffer = NULL; + GstBuffer *view_buffer; + + GST_LOG_OBJECT (self, "Outputting picture %p", picture); + + view_buffer = (GstBuffer *) gst_mpeg2_picture_get_user_data (picture); + + if (!view_buffer) { + GST_ERROR_OBJECT (self, "Could not get output view"); + goto error; + } + + /* if downstream is d3d11 element and forward playback case, + * expose our decoder view without copy. In case of reverse playback, however, + * we cannot do that since baseclass will store the decoded buffer + * up to gop size but our dpb pool cannot be increased */ + if (self->use_d3d11_output && + gst_d3d11_decoder_supports_direct_rendering (self->d3d11_decoder) && + vdec->input_segment.rate > 0) { + GstMemory *mem; + + output_buffer = gst_buffer_ref (view_buffer); + mem = gst_buffer_peek_memory (output_buffer, 0); + GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD); + } else { + output_buffer = gst_video_decoder_allocate_output_buffer (vdec); + } + + if (!output_buffer) { + GST_ERROR_OBJECT (self, "Couldn't allocate output buffer"); + goto error; + } + + frame->output_buffer = output_buffer; + + if (!gst_d3d11_decoder_process_output (self->d3d11_decoder, + &self->output_state->info, + GST_VIDEO_INFO_WIDTH (&self->output_state->info), + GST_VIDEO_INFO_HEIGHT (&self->output_state->info), + view_buffer, output_buffer)) { + GST_ERROR_OBJECT (self, "Failed to copy buffer"); + goto error; + } + + if (picture->buffer_flags != 0) { + gboolean interlaced = + (picture->buffer_flags & GST_VIDEO_BUFFER_FLAG_INTERLACED) != 0; + gboolean tff = (picture->buffer_flags & GST_VIDEO_BUFFER_FLAG_TFF) != 0; + + GST_TRACE_OBJECT (self, + "apply buffer flags 0x%x (interlaced %d, top-field-first %d)", + picture->buffer_flags, interlaced, tff); + GST_BUFFER_FLAG_SET (frame->output_buffer, picture->buffer_flags); + } + + gst_mpeg2_picture_unref (picture); + + return gst_video_decoder_finish_frame (vdec, frame); + +error: + gst_video_decoder_drop_frame (vdec, frame); + gst_mpeg2_picture_unref (picture); + + return GST_FLOW_ERROR; +} + +static gboolean +gst_d3d11_mpeg2_dec_end_picture (GstMpeg2Decoder * decoder, + GstMpeg2Picture * picture) +{ + GstD3D11Mpeg2Dec *self = GST_D3D11_MPEG2_DEC (decoder); + + if (!gst_d3d11_mpeg2_dec_submit_slice_data (self, picture)) { + GST_ERROR_OBJECT (self, "Failed to submit slice data"); + return FALSE; + } + + if (!gst_d3d11_decoder_end_frame (self->d3d11_decoder)) { + GST_ERROR_OBJECT (self, "Failed to EndFrame"); + return FALSE; + } + + return TRUE; +} + +typedef struct +{ + guint width; + guint height; +} GstD3D11Mpeg2DecResolution; + +void +gst_d3d11_mpeg2_dec_register (GstPlugin * plugin, GstD3D11Device * device, + GstD3D11Decoder * decoder, guint rank) +{ + GType type; + gchar *type_name; + gchar *feature_name; + guint index = 0; + GUID profile; + GTypeInfo type_info = { + sizeof (GstD3D11Mpeg2DecClass), + NULL, + NULL, + (GClassInitFunc) gst_d3d11_mpeg2_dec_class_init, + NULL, + NULL, + sizeof (GstD3D11Mpeg2Dec), + 0, + (GInstanceInitFunc) gst_d3d11_mpeg2_dec_init, + }; + static const GUID *supported_profiles[] = { + &GST_GUID_D3D11_DECODER_PROFILE_MPEG2_VLD, + &GST_GUID_D3D11_DECODER_PROFILE_MPEG2and1_VLD + }; + GstCaps *sink_caps = NULL; + GstCaps *src_caps = NULL; + + if (!gst_d3d11_decoder_get_supported_decoder_profile (decoder, + supported_profiles, G_N_ELEMENTS (supported_profiles), &profile)) { + GST_INFO_OBJECT (device, "device does not support MPEG-2 video decoding"); + return; + } + + sink_caps = gst_caps_from_string ("video/mpeg, " + "mpegversion = (int)2, systemstream = (boolean) false, " + "profile = (string) { main, simple }"); + src_caps = gst_caps_from_string ("video/x-raw(" + GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY "); video/x-raw"); + + /* NOTE: We are supporting only 4:2:0, main or simple profiles */ + gst_caps_set_simple (src_caps, "format", G_TYPE_STRING, "NV12", NULL); + + gst_caps_set_simple (sink_caps, + "width", GST_TYPE_INT_RANGE, 64, 1920, + "height", GST_TYPE_INT_RANGE, 64, 1920, NULL); + gst_caps_set_simple (src_caps, + "width", GST_TYPE_INT_RANGE, 64, 1920, + "height", GST_TYPE_INT_RANGE, 64, 1920, NULL); + + type_info.class_data = + gst_d3d11_decoder_class_data_new (device, sink_caps, src_caps); + + type_name = g_strdup ("GstD3D11Mpeg2Dec"); + feature_name = g_strdup ("d3d11mpeg2dec"); + + while (g_type_from_name (type_name)) { + index++; + g_free (type_name); + g_free (feature_name); + type_name = g_strdup_printf ("GstD3D11Mpeg2Device%dDec", index); + feature_name = g_strdup_printf ("d3d11mpeg2device%ddec", index); + } + + type = g_type_register_static (GST_TYPE_MPEG2_DECODER, + type_name, &type_info, 0); + + /* make lower rank than default device */ + if (rank > 0 && index != 0) + rank--; + + if (!gst_element_register (plugin, feature_name, rank, type)) + GST_WARNING ("Failed to register plugin '%s'", type_name); + + g_free (type_name); + g_free (feature_name); +} diff --git a/sys/d3d11/gstd3d11mpeg2dec.h b/sys/d3d11/gstd3d11mpeg2dec.h new file mode 100644 index 0000000000..2c2ce5441d --- /dev/null +++ b/sys/d3d11/gstd3d11mpeg2dec.h @@ -0,0 +1,34 @@ +/* GStreamer + * Copyright (C) 2021 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_D3D11_MPEG2_DEC_H__ +#define __GST_D3D11_MPEG2_DEC_H__ + +#include "gstd3d11decoder.h" + +G_BEGIN_DECLS + +void gst_d3d11_mpeg2_dec_register (GstPlugin * plugin, + GstD3D11Device * device, + GstD3D11Decoder * decoder, + guint rank); + +G_END_DECLS + +#endif /* __GST_D3D11_MPEG2_DEC_H__ */ diff --git a/sys/d3d11/meson.build b/sys/d3d11/meson.build index 4a987da52c..c6bbbc7364 100644 --- a/sys/d3d11/meson.build +++ b/sys/d3d11/meson.build @@ -21,6 +21,7 @@ d3d11_dec_sources = [ 'gstd3d11h264dec.c', 'gstd3d11vp9dec.c', 'gstd3d11h265dec.c', + 'gstd3d11mpeg2dec.c', 'gstd3d11vp8dec.c', ] diff --git a/sys/d3d11/plugin.c b/sys/d3d11/plugin.c index 03b44978c4..dee560b720 100644 --- a/sys/d3d11/plugin.c +++ b/sys/d3d11/plugin.c @@ -36,6 +36,7 @@ #include "gstd3d11h265dec.h" #include "gstd3d11vp9dec.h" #include "gstd3d11vp8dec.h" +#include "gstd3d11mpeg2dec.h" #endif #ifdef HAVE_DXGI_DESKTOP_DUP #include "gstd3d11desktopdupsrc.h" @@ -57,6 +58,7 @@ GST_DEBUG_CATEGORY (gst_d3d11_h264_dec_debug); GST_DEBUG_CATEGORY (gst_d3d11_h265_dec_debug); GST_DEBUG_CATEGORY (gst_d3d11_vp9_dec_debug); GST_DEBUG_CATEGORY (gst_d3d11_vp8_dec_debug); +GST_DEBUG_CATEGORY (gst_d3d11_mpeg2_dec_debug); #endif #ifdef HAVE_DXGI_DESKTOP_DUP @@ -109,6 +111,8 @@ plugin_init (GstPlugin * plugin) "d3d11h265dec", 0, "Direct3D11 H.265 Video Decoder"); GST_DEBUG_CATEGORY_INIT (gst_d3d11_vp8_dec_debug, "d3d11vp8dec", 0, "Direct3D11 VP8 Decoder"); + GST_DEBUG_CATEGORY_INIT (gst_d3d11_mpeg2_dec_debug, + "d3d11mpeg2dec", 0, "Direct3D11 MPEG2 Decoder"); } #endif @@ -156,6 +160,8 @@ plugin_init (GstPlugin * plugin) GST_RANK_SECONDARY); gst_d3d11_vp8_dec_register (plugin, device, decoder, GST_RANK_SECONDARY); + gst_d3d11_mpeg2_dec_register (plugin, device, decoder, + GST_RANK_SECONDARY); } done: