From 75f6f79e571dbc3aee66c4ff4e55564c527e9c71 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Thu, 19 Aug 2021 21:56:05 +0900 Subject: [PATCH] mfvideosrc: Fix for negative MF stride Negative stride value can be used in MediaFoundation to inform whether memory layout is top-down or bottom-up manner. Note that negative stride is allowed only for RGB, system memory. See also https://docs.microsoft.com/en-us/windows/win32/medfound/image-stride Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/issues/1646 Part-of: --- sys/mediafoundation/gstmfsourcereader.cpp | 80 +++++++++++--- sys/mediafoundation/gstmfutils.cpp | 123 ++++++++++++++++++++-- sys/mediafoundation/gstmfutils.h | 3 + 3 files changed, 186 insertions(+), 20 deletions(-) diff --git a/sys/mediafoundation/gstmfsourcereader.cpp b/sys/mediafoundation/gstmfsourcereader.cpp index 57a0a5a3b1..ef53fdc132 100644 --- a/sys/mediafoundation/gstmfsourcereader.cpp +++ b/sys/mediafoundation/gstmfsourcereader.cpp @@ -85,6 +85,8 @@ struct _GstMFSourceReader GstMFStreamMediaType *cur_type; GstVideoInfo info; + gboolean top_down_image; + gboolean flushing; }; @@ -394,12 +396,38 @@ gst_mf_source_reader_start (GstMFSourceObject * object) } type = self->cur_type; + self->top_down_image = TRUE; if (GST_VIDEO_INFO_FORMAT (&self->info) != GST_VIDEO_FORMAT_ENCODED) { - hr = type->media_type->SetUINT32 (MF_MT_DEFAULT_STRIDE, - GST_VIDEO_INFO_PLANE_STRIDE (&self->info, 0)); - if (!gst_mf_result (hr)) - return FALSE; + UINT32 stride; + INT32 actual_stride = GST_VIDEO_INFO_PLANE_STRIDE (&self->info, 0); + + /* This MF_MT_DEFAULT_STRIDE uses UINT32 type but actual value is + * INT32, which can be negative in case of RGB image, and negative means + * its stored as bottom-up manner */ + hr = type->media_type->GetUINT32 (MF_MT_DEFAULT_STRIDE, &stride); + if (gst_mf_result (hr)) { + actual_stride = (INT32) stride; + if (actual_stride < 0) { + if (!GST_VIDEO_INFO_IS_RGB (&self->info)) { + GST_ERROR_OBJECT (self, + "Bottom-up image is allowed only for RGB format"); + return FALSE; + } + + GST_DEBUG_OBJECT (self, + "Detected bottom-up image, stride %d", actual_stride); + + self->top_down_image = FALSE; + } + } else { + /* If MF_MT_DEFAULT_STRIDE attribute is not specified, we can use our + * value */ + type->media_type->SetUINT32 (MF_MT_DEFAULT_STRIDE, + (UINT32) actual_stride); + } + gst_mf_update_video_info_with_stride (&self->info, + std::abs (actual_stride)); } hr = self->reader->SetStreamSelection (type->stream_index, TRUE); @@ -568,24 +596,48 @@ gst_mf_source_reader_fill (GstMFSourceObject * object, GstBuffer * buffer) return GST_FLOW_ERROR; } - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->info); i++) { + if (!self->top_down_image) { guint8 *src, *dst; gint src_stride, dst_stride; - gint width; + gint width, height; - src = data + GST_VIDEO_INFO_PLANE_OFFSET (&self->info, i); - dst = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, i); + /* must be single plane RGB */ + width = GST_VIDEO_INFO_COMP_WIDTH (&self->info, 0) + * GST_VIDEO_INFO_COMP_PSTRIDE (&self->info, 0); + height = GST_VIDEO_INFO_HEIGHT (&self->info); - src_stride = GST_VIDEO_INFO_PLANE_STRIDE (&self->info, i); - dst_stride = GST_VIDEO_FRAME_PLANE_STRIDE (&frame, i); - width = GST_VIDEO_INFO_COMP_WIDTH (&self->info, i) - * GST_VIDEO_INFO_COMP_PSTRIDE (&self->info, i); + src_stride = GST_VIDEO_INFO_PLANE_STRIDE (&self->info, 0); + dst_stride = GST_VIDEO_FRAME_PLANE_STRIDE (&frame, 0); - for (j = 0; j < GST_VIDEO_INFO_COMP_HEIGHT (&self->info, i); j++) { + /* This is bottom up image, should copy lines in reverse order */ + src = data + src_stride * (height - 1); + dst = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + + for (j = 0; j < height; j++) { memcpy (dst, src, width); - src += src_stride; + src -= src_stride; dst += dst_stride; } + } else { + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->info); i++) { + guint8 *src, *dst; + gint src_stride, dst_stride; + gint width; + + src = data + GST_VIDEO_INFO_PLANE_OFFSET (&self->info, i); + dst = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, i); + + src_stride = GST_VIDEO_INFO_PLANE_STRIDE (&self->info, i); + dst_stride = GST_VIDEO_FRAME_PLANE_STRIDE (&frame, i); + width = GST_VIDEO_INFO_COMP_WIDTH (&self->info, i) + * GST_VIDEO_INFO_COMP_PSTRIDE (&self->info, i); + + for (j = 0; j < GST_VIDEO_INFO_COMP_HEIGHT (&self->info, i); j++) { + memcpy (dst, src, width); + src += src_stride; + dst += dst_stride; + } + } } gst_video_frame_unmap (&frame); diff --git a/sys/mediafoundation/gstmfutils.cpp b/sys/mediafoundation/gstmfutils.cpp index a3ab9b1930..bbb69d46e9 100644 --- a/sys/mediafoundation/gstmfutils.cpp +++ b/sys/mediafoundation/gstmfutils.cpp @@ -49,23 +49,35 @@ static struct const gchar *caps_string; GstVideoFormat format; } raw_video_format_map[] = { + /* NOTE: when adding new format, gst_mf_update_video_info_with_stride() must + * be updated as well */ {MFVideoFormat_RGB32, MAKE_RAW_FORMAT_CAPS ("BGRx"), GST_VIDEO_FORMAT_BGRx}, {MFVideoFormat_ARGB32, MAKE_RAW_FORMAT_CAPS ("BGRA"), GST_VIDEO_FORMAT_BGRA}, - {MFVideoFormat_RGB24, MAKE_RAW_FORMAT_CAPS ("BGR"), GST_VIDEO_FORMAT_BGR}, - {MFVideoFormat_RGB555, MAKE_RAW_FORMAT_CAPS ("RGB15"), GST_VIDEO_FORMAT_RGB15}, {MFVideoFormat_RGB565, MAKE_RAW_FORMAT_CAPS ("RGB16"), GST_VIDEO_FORMAT_RGB16}, - {MFVideoFormat_AYUV, MAKE_RAW_FORMAT_CAPS ("VUYA"), GST_VIDEO_FORMAT_VUYA}, + {MFVideoFormat_RGB555, MAKE_RAW_FORMAT_CAPS ("RGB15"), GST_VIDEO_FORMAT_RGB15}, + {MFVideoFormat_RGB24, MAKE_RAW_FORMAT_CAPS ("BGR"), GST_VIDEO_FORMAT_BGR}, + + /* packed YUV */ {MFVideoFormat_YUY2, MAKE_RAW_FORMAT_CAPS ("YUY2"), GST_VIDEO_FORMAT_YUY2}, {MFVideoFormat_YVYU, MAKE_RAW_FORMAT_CAPS ("YVYU"), GST_VIDEO_FORMAT_YVYU}, {MFVideoFormat_UYVY, MAKE_RAW_FORMAT_CAPS ("UYVY"), GST_VIDEO_FORMAT_UYVY}, + {MFVideoFormat_AYUV, MAKE_RAW_FORMAT_CAPS ("VUYA"), GST_VIDEO_FORMAT_VUYA}, + + /* semi-planar */ {MFVideoFormat_NV12, MAKE_RAW_FORMAT_CAPS ("NV12"), GST_VIDEO_FORMAT_NV12}, - {MFVideoFormat_YV12, MAKE_RAW_FORMAT_CAPS ("YV12"), GST_VIDEO_FORMAT_YV12}, - {MFVideoFormat_I420, MAKE_RAW_FORMAT_CAPS ("I420"), GST_VIDEO_FORMAT_I420}, - {MFVideoFormat_IYUV, MAKE_RAW_FORMAT_CAPS ("I420"), GST_VIDEO_FORMAT_I420}, {MFVideoFormat_P010, MAKE_RAW_FORMAT_CAPS ("P010_10LE"), GST_VIDEO_FORMAT_P010_10LE}, {MFVideoFormat_P016, MAKE_RAW_FORMAT_CAPS ("P016_LE"), GST_VIDEO_FORMAT_P016_LE}, + + /* planar */ + {MFVideoFormat_I420, MAKE_RAW_FORMAT_CAPS ("I420"), GST_VIDEO_FORMAT_I420}, + {MFVideoFormat_IYUV, MAKE_RAW_FORMAT_CAPS ("I420"), GST_VIDEO_FORMAT_I420}, + {MFVideoFormat_YV12, MAKE_RAW_FORMAT_CAPS ("YV12"), GST_VIDEO_FORMAT_YV12}, + + /* complex format */ {MFVideoFormat_v210, MAKE_RAW_FORMAT_CAPS ("v210"), GST_VIDEO_FORMAT_v210}, {MFVideoFormat_v216, MAKE_RAW_FORMAT_CAPS ("v216"), GST_VIDEO_FORMAT_v216}, + + /* gray */ {MFVideoFormat_Y16, MAKE_RAW_FORMAT_CAPS ("GRAY16_LE"), GST_VIDEO_FORMAT_GRAY16_LE}, }; @@ -361,6 +373,105 @@ gst_mf_media_type_release (IMFMediaType * media_type) media_type->Release (); } +gboolean +gst_mf_update_video_info_with_stride (GstVideoInfo * info, gint stride) +{ + guint width, height, cr_h; + + g_return_val_if_fail (info != nullptr, FALSE); + g_return_val_if_fail (stride > 0, FALSE); + g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (info) + != GST_VIDEO_FORMAT_UNKNOWN, FALSE); + + if (GST_VIDEO_INFO_FORMAT (info) == GST_VIDEO_FORMAT_ENCODED) + return TRUE; + + width = GST_VIDEO_INFO_WIDTH (info); + height = GST_VIDEO_INFO_HEIGHT (info); + + /* copied from video-info */ + switch (GST_VIDEO_INFO_FORMAT (info)) { + /* RGB */ + case GST_VIDEO_FORMAT_RGBx: + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_RGB16: + case GST_VIDEO_FORMAT_BGR15: + case GST_VIDEO_FORMAT_BGR: + info->stride[0] = stride; + info->offset[0] = 0; + info->size = info->stride[0] * height; + break; + /* packed YUV */ + case GST_VIDEO_FORMAT_YUY2: + case GST_VIDEO_FORMAT_YVYU: + case GST_VIDEO_FORMAT_UYVY: + case GST_VIDEO_FORMAT_VUYA: + info->stride[0] = stride; + info->offset[0] = 0; + info->size = info->stride[0] * height; + break; + /* semi-planar */ + case GST_VIDEO_FORMAT_NV12: + case GST_VIDEO_FORMAT_P010_10LE: + case GST_VIDEO_FORMAT_P016_LE: + if (height % 2) { + GST_ERROR ("Height must be even number"); + return FALSE; + } + + cr_h = height / 2; + + info->stride[0] = stride; + info->stride[1] = info->stride[0]; + info->offset[0] = 0; + info->offset[1] = info->stride[0] * height; + info->size = info->offset[1] + info->stride[0] * cr_h; + break; + /* planar */ + case GST_VIDEO_FORMAT_I420: + case GST_VIDEO_FORMAT_YV12: + if (stride % 2) { + GST_ERROR ("Stride must be even number"); + return FALSE; + } + + if (height % 2) { + GST_ERROR ("Height must be even number"); + return FALSE; + } + + cr_h = height / 2; + + info->stride[0] = stride; + info->stride[1] = stride / 2; + info->stride[2] = info->stride[1]; + info->offset[0] = 0; + info->offset[1] = info->stride[0] * height; + info->offset[2] = info->offset[1] + info->stride[1] * cr_h; + info->size = info->offset[2] + info->stride[2] * cr_h; + break; + /* complex */ + case GST_VIDEO_FORMAT_v210: + case GST_VIDEO_FORMAT_v216: + info->stride[0] = stride; + info->offset[0] = 0; + info->size = info->stride[0] * height; + break; + /* gray */ + case GST_VIDEO_FORMAT_GRAY16_LE: + info->stride[0] = stride; + info->offset[0] = 0; + info->size = info->stride[0] * height; + break; + default: + GST_ERROR ("Unhandled format %s", + gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (info))); + return FALSE; + } + + return TRUE; +} + gboolean _gst_mf_result (HRESULT hr, GstDebugCategory * cat, const gchar * file, const gchar * function, gint line) diff --git a/sys/mediafoundation/gstmfutils.h b/sys/mediafoundation/gstmfutils.h index 79caae6090..d5e247cb63 100644 --- a/sys/mediafoundation/gstmfutils.h +++ b/sys/mediafoundation/gstmfutils.h @@ -50,6 +50,9 @@ GstCaps * gst_mf_media_type_to_caps (IMFMediaType * media_type); void gst_mf_media_type_release (IMFMediaType * media_type); +gboolean gst_mf_update_video_info_with_stride (GstVideoInfo * info, + gint stride); + gboolean _gst_mf_result (HRESULT hr, GstDebugCategory * cat, const gchar * file,