From c9b127dae3af4b3fe78468e697cb491902425254 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 2 Dec 2021 21:46:59 -0500 Subject: [PATCH] video: Add NV12_16L32S aka Mediatek MM21 support Unlike other simple tiled formats, the Mediatek HW use different tile size per-plane. The tile size is scaled according to the subsampling. Effectively, using the name 16L32S to represent linearly layout tiles of size 16x32 bytes in the Y plane, and 16x16 in the UV plane. In order to make this specificity discoverable, a new SUBTILES flags have been added. Part-of: --- .../gst-libs/gst/video/video-converter.c | 1 + .../gst-libs/gst/video/video-format.c | 32 ++++++++++++--- .../gst-libs/gst/video/video-format.h | 39 +++++++++++++++++-- .../gst-libs/gst/video/video-frame.c | 25 ++++++++++++ .../gst-libs/gst/video/video-info.c | 19 +++++++++ .../gst-libs/gst/video/video-tile.h | 2 + .../tests/check/elements/videoscale.c | 1 + .../gst-plugins-base/tests/check/libs/video.c | 1 + 8 files changed, 111 insertions(+), 9 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/video-converter.c b/subprojects/gst-plugins-base/gst-libs/gst/video/video-converter.c index 50bb7d60f6..cc35eb75b5 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/video-converter.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/video-converter.c @@ -7331,6 +7331,7 @@ get_scale_format (GstVideoFormat format, gint plane) case GST_VIDEO_FORMAT_NV12_64Z32: case GST_VIDEO_FORMAT_NV12_4L4: case GST_VIDEO_FORMAT_NV12_32L32: + case GST_VIDEO_FORMAT_NV12_16L32S: case GST_VIDEO_FORMAT_A420_10BE: case GST_VIDEO_FORMAT_A420_10LE: case GST_VIDEO_FORMAT_A422_10BE: diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/video-format.c b/subprojects/gst-plugins-base/gst-libs/gst/video/video-format.c index 350d1b7af9..b90d53439e 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/video-format.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/video-format.c @@ -4764,12 +4764,26 @@ get_tile_NV12 (gint tile_width, gint ts, gint tx, gint ty, tile_data[0] = ((guint8 *) data[0]) + offset; /* index of UV tile */ - offset = gst_video_tile_get_index (mode, - tx, ty >> 1, GST_VIDEO_TILE_X_TILES (stride[1]), - GST_VIDEO_TILE_Y_TILES (stride[1])); - offset <<= ts; - /* On odd rows we return the second part of the UV tile */ - offset |= (ty & 1) << (ts - 1); + if (stride[0] != stride[1]) { + offset = gst_video_tile_get_index (mode, + tx, ty >> 1, GST_VIDEO_TILE_X_TILES (stride[1]), + GST_VIDEO_TILE_Y_TILES (stride[1])); + + offset <<= ts; + + /* On odd rows we return the second part of the UV tile */ + offset |= (ty & 1) << (ts - 1); + } else { + /* handle subsampled tiles, with type of tiles will have the same number + * of tiles on both planes, but the height of the tiles are half. */ + offset = gst_video_tile_get_index (mode, + tx, ty, GST_VIDEO_TILE_X_TILES (stride[1]), + GST_VIDEO_TILE_Y_TILES (stride[1])); + + /* For subsampled tile Subsampled tile size */ + offset <<= (ts - 1); + } + tile_data[1] = ((guint8 *) data[1]) + offset; tile_stride[0] = tile_stride[1] = tile_width; @@ -6848,6 +6862,7 @@ typedef struct /* tile_mode, tile_ws (width shift), tile_hs (height shift) */ #define TILE_4x4(mode) GST_VIDEO_TILE_MODE_ ##mode, 2, 2 +#define TILE_16x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 4, 5 #define TILE_32x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 5, 5 #define TILE_64x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 6, 5 @@ -6869,6 +6884,8 @@ typedef struct { fourcc, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_COMPLEX | GST_VIDEO_FORMAT_FLAG_LE, depth, pstride, plane, offs, sub, pack } } #define MAKE_YUV_T_FORMAT(name, desc, fourcc, depth, pstride, plane, offs, sub, pack, tile) \ { fourcc, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_COMPLEX | GST_VIDEO_FORMAT_FLAG_TILED, depth, pstride, plane, offs, sub, pack, tile } } +#define MAKE_YUV_ST_FORMAT(name, desc, fourcc, depth, pstride, plane, offs, sub, pack, tile) \ + { fourcc, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_COMPLEX | GST_VIDEO_FORMAT_FLAG_TILED | GST_VIDEO_FORMAT_FLAG_SUBTILES, depth, pstride, plane, offs, sub, pack, tile } } #define MAKE_RGB_FORMAT(name, desc, depth, pstride, plane, offs, sub, pack) \ { 0x00000000, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_RGB, depth, pstride, plane, offs, sub, pack } } @@ -7145,6 +7162,9 @@ static const VideoFormat formats[] = { PLANE0, OFFS6420, SUB444, PACK_ABGR64_LE), MAKE_RGBA_FORMAT (ABGR64_BE, "raw video", DPTH16_16_16_16, PSTR8888, PLANE0, OFFS6420, SUB4444, PACK_ABGR64_BE), + MAKE_YUV_ST_FORMAT (NV12_16L32S, "raw video", + GST_MAKE_FOURCC ('M', 'M', '2', '1'), DPTH888, PSTR122, PLANE011, + OFFS001, SUB420, PACK_NV12_TILED, TILE_16x32 (LINEAR)), }; static GstVideoFormat diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/video-format.h b/subprojects/gst-plugins-base/gst-libs/gst/video/video-format.h index 9c701c895c..8ce34c637b 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/video-format.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/video-format.h @@ -142,6 +142,7 @@ G_BEGIN_DECLS * @GST_VIDEO_FORMAT_BGRA64_BE: reverse RGB with alpha channel last, 16 bits per channel * @GST_VIDEO_FORMAT_ABGR64_LE: reverse RGB with alpha channel first, 16 bits per channel * @GST_VIDEO_FORMAT_ABGR64_BE: reverse RGB with alpha channel first, 16 bits per channel + * @GST_VIDEO_FORMAT_NV12_16L32S: NV12 with 16x32 Y tiles and 16x16 UV tiles. (Since: 1.22) * * Enum value describing the most common video formats. * @@ -370,6 +371,15 @@ typedef enum { * Since: 1.20 */ GST_VIDEO_FORMAT_ABGR64_BE, + /** + * GST_VIDEO_FORMAT_NV12_16L32S: + * + * NV12 with 16x32 Y tiles and 16x16 UV tiles. + * + * Since: 1.22 + */ + GST_VIDEO_FORMAT_NV12_16L32S, + } GstVideoFormat; #define GST_VIDEO_MAX_PLANES 4 @@ -397,6 +407,8 @@ typedef struct _GstVideoFormatInfo GstVideoFormatInfo; * #GstVideoFormatUnpack and #GstVideoFormatPack function. * @GST_VIDEO_FORMAT_FLAG_TILED: The format is tiled, there is tiling information * in the last plane. + * @GST_VIDEO_FORMAT_FLAG_SUBTILES: The tile size varies per plane + * according to the subsampling. (Since: 1.22) * * The different video flags that a format info can have. */ @@ -410,7 +422,15 @@ typedef enum GST_VIDEO_FORMAT_FLAG_PALETTE = (1 << 5), GST_VIDEO_FORMAT_FLAG_COMPLEX = (1 << 6), GST_VIDEO_FORMAT_FLAG_UNPACK = (1 << 7), - GST_VIDEO_FORMAT_FLAG_TILED = (1 << 8) + GST_VIDEO_FORMAT_FLAG_TILED = (1 << 8), + /** + * GST_VIDEO_FORMAT_FLAG_SUBTILES: + * + * The tile size varies per plane according to the subsampling. + * + * Since: 1.22 + */ + GST_VIDEO_FORMAT_FLAG_SUBTILES = (1 << 9) } GstVideoFormatFlags; /* YUV components */ @@ -602,6 +622,19 @@ struct _GstVideoFormatInfo { #define GST_VIDEO_FORMAT_INFO_HAS_PALETTE(info) (((info)->flags & GST_VIDEO_FORMAT_FLAG_PALETTE) != 0) #define GST_VIDEO_FORMAT_INFO_IS_COMPLEX(info) (((info)->flags & GST_VIDEO_FORMAT_FLAG_COMPLEX) != 0) #define GST_VIDEO_FORMAT_INFO_IS_TILED(info) (((info)->flags & GST_VIDEO_FORMAT_FLAG_TILED) != 0) +/** + * GST_VIDEO_FORMAT_INFO_HAS_SUBTILES: + * @info: a #GstVideoFormatInfo + * + * This macro checks if %GST_VIDEO_FORMAT_FLAG_SUBTILES is set. When this + * flag is set, it means that the tile sizes must be scaled as per the + * subsampling. + * + * Returns: %TRUE if the format uses subsampled tile sizes. + * + * Since: 1.22 + */ +#define GST_VIDEO_FORMAT_INFO_HAS_SUBTILES(info) (((info)->flags & GST_VIDEO_FORMAT_FLAG_SUBTILES) != 0) #define GST_VIDEO_FORMAT_INFO_BITS(info) ((info)->bits) #define GST_VIDEO_FORMAT_INFO_N_COMPONENTS(info) ((info)->n_components) @@ -745,7 +778,7 @@ gconstpointer gst_video_format_get_palette (GstVideoFormat format, gsi "I422_10BE, I422_10LE, NV16_10LE32, Y210, v210, UYVP, I420_10BE, I420_10LE, " \ "P010_10BE, P010_10LE, NV12_10LE32, NV12_10LE40, Y444, RGBP, GBR, BGRP, NV24, xBGR, BGRx, " \ "xRGB, RGBx, BGR, IYU2, v308, RGB, Y42B, NV61, NV16, VYUY, UYVY, YVYU, YUY2, I420, " \ - "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, Y41B, IYU1, YVU9, YUV9, RGB16, " \ + "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, NV12_16L32S, Y41B, IYU1, YVU9, YUV9, RGB16, " \ "BGR16, RGB15, BGR15, RGB8P, GRAY16_BE, GRAY16_LE, GRAY10_LE32, GRAY8 }" #elif G_BYTE_ORDER == G_LITTLE_ENDIAN #define GST_VIDEO_FORMATS_ALL "{ ABGR64_LE, BGRA64_LE, AYUV64, ARGB64_LE, ARGB64, " \ @@ -758,7 +791,7 @@ gconstpointer gst_video_format_get_palette (GstVideoFormat format, gsi "I422_10LE, I422_10BE, NV16_10LE32, Y210, v210, UYVP, I420_10LE, I420_10BE, " \ "P010_10LE, NV12_10LE32, NV12_10LE40, P010_10BE, Y444, RGBP, GBR, BGRP, NV24, xBGR, BGRx, " \ "xRGB, RGBx, BGR, IYU2, v308, RGB, Y42B, NV61, NV16, VYUY, UYVY, YVYU, YUY2, I420, " \ - "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, Y41B, IYU1, YVU9, YUV9, RGB16, " \ + "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, NV12_16L32S, Y41B, IYU1, YVU9, YUV9, RGB16, " \ "BGR16, RGB15, BGR15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }" #endif diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/video-frame.c b/subprojects/gst-plugins-base/gst-libs/gst/video/video-frame.c index 0b239894ea..d92091294d 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/video-frame.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/video-frame.c @@ -278,6 +278,27 @@ gst_video_frame_unmap (GstVideoFrame * frame) gst_buffer_unref (frame->buffer); } +static void +scale_tile_shifts (const GstVideoFormatInfo * finfo, gint plane, guint * ws, + guint * hs) +{ + gint comp[GST_VIDEO_MAX_COMPONENTS]; + gint i; + + gst_video_format_info_component (finfo, plane, comp); + + /* scale the tile size according to the subsampling */ + *ws -= finfo->w_sub[comp[0]]; + *hs -= finfo->h_sub[comp[0]]; + + /* for each additional component in the same plane, double the tile width, + * this should provide the appropriate tile size when the tile size varies + * base on the subsampling. */ + for (i = 1; comp[i] >= 0; i++) + *ws += 1; +} + + /** * gst_video_frame_copy_plane: * @dest: a #GstVideoFrame @@ -348,6 +369,10 @@ gst_video_frame_copy_plane (GstVideoFrame * dest, const GstVideoFrame * src, ws = GST_VIDEO_FORMAT_INFO_TILE_WS (finfo); hs = GST_VIDEO_FORMAT_INFO_TILE_HS (finfo); + + if (GST_VIDEO_FORMAT_INFO_HAS_SUBTILES (finfo)) + scale_tile_shifts (finfo, plane, &ws, &hs); + ts = ws + hs; tile_size = 1 << ts; diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c b/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c index 9cf7821f9f..8b3064d88c 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c @@ -1112,6 +1112,25 @@ fill_planes (GstVideoInfo * info, gsize plane_size[GST_VIDEO_MAX_PLANES]) (GST_ROUND_UP_N (height, 1 << (hs + 1)) / 2); break; } + case GST_VIDEO_FORMAT_NV12_16L32S: + { + gint ws = GST_VIDEO_FORMAT_INFO_TILE_WS (info->finfo); + gint hs = GST_VIDEO_FORMAT_INFO_TILE_HS (info->finfo); + info->stride[0] = + GST_VIDEO_TILE_MAKE_STRIDE (GST_ROUND_UP_N (width, 1 << ws) >> ws, + GST_ROUND_UP_N (height, 1 << hs) >> hs); + /* + * size of UV plane tiles is subsample, hence have the same number of + * tiles in both directions. + */ + info->stride[1] = info->stride[0]; + info->offset[0] = 0; + info->offset[1] = + GST_ROUND_UP_N (width, 1 << ws) * GST_ROUND_UP_N (height, 1 << hs); + info->size = info->offset[1] + GST_ROUND_UP_N (width, 1 << ws) * + GST_ROUND_UP_N (height, 1 << (hs - 1)); + break; + } case GST_VIDEO_FORMAT_A420_10LE: case GST_VIDEO_FORMAT_A420_10BE: info->stride[0] = GST_ROUND_UP_4 (width * 2); diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/video-tile.h b/subprojects/gst-plugins-base/gst-libs/gst/video/video-tile.h index 8992bb351d..f1000e8dfa 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/video-tile.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/video-tile.h @@ -113,6 +113,8 @@ typedef enum * in memory in Z or flipped Z order. In case of odd rows, the last row * of blocks is arranged in linear order. * @GST_VIDEO_TILE_MODE_LINEAR: Tiles are in row order. (Since: 1.18) + * @GST_VIDEO_TILE_MODE_LINEAR_SUBSAMPLED: Tiles are in row order, with + * variable tile size according to subsampling. (Since: 1.20) * * Enum value describing the available tiling modes. */ diff --git a/subprojects/gst-plugins-base/tests/check/elements/videoscale.c b/subprojects/gst-plugins-base/tests/check/elements/videoscale.c index 71bb35509c..976f21c96e 100644 --- a/subprojects/gst-plugins-base/tests/check/elements/videoscale.c +++ b/subprojects/gst-plugins-base/tests/check/elements/videoscale.c @@ -126,6 +126,7 @@ check_pad_template (GstPadTemplate * tmpl) case GST_VIDEO_FORMAT_NV12_64Z32: case GST_VIDEO_FORMAT_NV12_4L4: case GST_VIDEO_FORMAT_NV12_32L32: + case GST_VIDEO_FORMAT_NV12_16L32S: GST_LOG ("Ignoring lack of support for format %s", fmt_str); break; default: diff --git a/subprojects/gst-plugins-base/tests/check/libs/video.c b/subprojects/gst-plugins-base/tests/check/libs/video.c index 923e73d0f7..8dea48c1b4 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/video.c +++ b/subprojects/gst-plugins-base/tests/check/libs/video.c @@ -3203,6 +3203,7 @@ GST_START_TEST (test_video_formats_pstrides) || fmt == GST_VIDEO_FORMAT_NV12_64Z32 || fmt == GST_VIDEO_FORMAT_NV12_4L4 || fmt == GST_VIDEO_FORMAT_NV12_32L32 + || fmt == GST_VIDEO_FORMAT_NV12_16L32S || fmt == GST_VIDEO_FORMAT_NV12_10LE32 || fmt == GST_VIDEO_FORMAT_NV16_10LE32 || fmt == GST_VIDEO_FORMAT_NV12_10LE40