video: Add NV12_16L32S aka Mediatek MM21 support

Unlike other simple tiled formats, the Mediatek HW use different tile size
per-plane. The tile size is scaled according to the subsampling. Effectively,
using the name 16L32S to represent linearly layout tiles of size 16x32 bytes
in the Y plane, and 16x16 in the UV plane. In order to make this specificity
discoverable, a new SUBTILES flags have been added.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1567>

subprojects/gst-plugins-base/gst-libs/gst/video/video-converter.c

@@ -7331,6 +7331,7 @@ get_scale_format (GstVideoFormat format, gint plane)
     case GST_VIDEO_FORMAT_NV12_64Z32:
     case GST_VIDEO_FORMAT_NV12_4L4:
     case GST_VIDEO_FORMAT_NV12_32L32:
+    case GST_VIDEO_FORMAT_NV12_16L32S:
     case GST_VIDEO_FORMAT_A420_10BE:
     case GST_VIDEO_FORMAT_A420_10LE:
     case GST_VIDEO_FORMAT_A422_10BE:

subprojects/gst-plugins-base/gst-libs/gst/video/video-format.c

@@ -4764,12 +4764,26 @@ get_tile_NV12 (gint tile_width, gint ts, gint tx, gint ty,
   tile_data[0] = ((guint8 *) data[0]) + offset;
 
   /* index of UV tile */
+  if (stride[0] != stride[1]) {
     offset = gst_video_tile_get_index (mode,
         tx, ty >> 1, GST_VIDEO_TILE_X_TILES (stride[1]),
         GST_VIDEO_TILE_Y_TILES (stride[1]));
+
     offset <<= ts;
+
     /* On odd rows we return the second part of the UV tile */
     offset |= (ty & 1) << (ts - 1);
+  } else {
+    /* handle subsampled tiles, with type of tiles will have the same number
+     * of tiles on both planes, but the height of the tiles are half. */
+    offset = gst_video_tile_get_index (mode,
+        tx, ty, GST_VIDEO_TILE_X_TILES (stride[1]),
+        GST_VIDEO_TILE_Y_TILES (stride[1]));
+
+    /* For subsampled tile Subsampled tile size */
+    offset <<= (ts - 1);
+  }
+
   tile_data[1] = ((guint8 *) data[1]) + offset;
 
   tile_stride[0] = tile_stride[1] = tile_width;
@@ -6848,6 +6862,7 @@ typedef struct
 
 /* tile_mode, tile_ws (width shift), tile_hs (height shift) */
 #define TILE_4x4(mode) GST_VIDEO_TILE_MODE_ ##mode, 2, 2
+#define TILE_16x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 4, 5
 #define TILE_32x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 5, 5
 #define TILE_64x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 6, 5
 
@@ -6869,6 +6884,8 @@ typedef struct
  { fourcc, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_COMPLEX | GST_VIDEO_FORMAT_FLAG_LE, depth, pstride, plane, offs, sub, pack } }
 #define MAKE_YUV_T_FORMAT(name, desc, fourcc, depth, pstride, plane, offs, sub, pack, tile) \
  { fourcc, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_COMPLEX | GST_VIDEO_FORMAT_FLAG_TILED, depth, pstride, plane, offs, sub, pack, tile } }
+#define MAKE_YUV_ST_FORMAT(name, desc, fourcc, depth, pstride, plane, offs, sub, pack, tile) \
+ { fourcc, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_COMPLEX | GST_VIDEO_FORMAT_FLAG_TILED | GST_VIDEO_FORMAT_FLAG_SUBTILES, depth, pstride, plane, offs, sub, pack, tile } }
 
 #define MAKE_RGB_FORMAT(name, desc, depth, pstride, plane, offs, sub, pack) \
  { 0x00000000, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_RGB, depth, pstride, plane, offs, sub, pack } }
@@ -7145,6 +7162,9 @@ static const VideoFormat formats[] = {
       PLANE0, OFFS6420, SUB444, PACK_ABGR64_LE),
   MAKE_RGBA_FORMAT (ABGR64_BE, "raw video", DPTH16_16_16_16, PSTR8888, PLANE0,
       OFFS6420, SUB4444, PACK_ABGR64_BE),
+  MAKE_YUV_ST_FORMAT (NV12_16L32S, "raw video",
+      GST_MAKE_FOURCC ('M', 'M', '2', '1'), DPTH888, PSTR122, PLANE011,
+      OFFS001, SUB420, PACK_NV12_TILED, TILE_16x32 (LINEAR)),
 };
 
 static GstVideoFormat

subprojects/gst-plugins-base/gst-libs/gst/video/video-format.h

@@ -142,6 +142,7 @@ G_BEGIN_DECLS
  * @GST_VIDEO_FORMAT_BGRA64_BE: reverse RGB with alpha channel last, 16 bits per channel
  * @GST_VIDEO_FORMAT_ABGR64_LE: reverse RGB with alpha channel first, 16 bits per channel
  * @GST_VIDEO_FORMAT_ABGR64_BE: reverse RGB with alpha channel first, 16 bits per channel
+ * @GST_VIDEO_FORMAT_NV12_16L32S: NV12 with 16x32 Y tiles and 16x16 UV tiles. (Since: 1.22)
  *
  * Enum value describing the most common video formats.
  *
@@ -370,6 +371,15 @@ typedef enum {
    * Since: 1.20
    */
   GST_VIDEO_FORMAT_ABGR64_BE,
+  /**
+   * GST_VIDEO_FORMAT_NV12_16L32S:
+   *
+   * NV12 with 16x32 Y tiles and 16x16 UV tiles.
+   *
+   * Since: 1.22
+   */
+  GST_VIDEO_FORMAT_NV12_16L32S,
+
 } GstVideoFormat;
 
 #define GST_VIDEO_MAX_PLANES 4
@@ -397,6 +407,8 @@ typedef struct _GstVideoFormatInfo GstVideoFormatInfo;
  *   #GstVideoFormatUnpack and #GstVideoFormatPack function.
  * @GST_VIDEO_FORMAT_FLAG_TILED: The format is tiled, there is tiling information
  *   in the last plane.
+ * @GST_VIDEO_FORMAT_FLAG_SUBTILES: The tile size varies per plane
+ *   according to the subsampling. (Since: 1.22)
  *
  * The different video flags that a format info can have.
  */
@@ -410,7 +422,15 @@ typedef enum
   GST_VIDEO_FORMAT_FLAG_PALETTE  = (1 << 5),
   GST_VIDEO_FORMAT_FLAG_COMPLEX  = (1 << 6),
   GST_VIDEO_FORMAT_FLAG_UNPACK   = (1 << 7),
-  GST_VIDEO_FORMAT_FLAG_TILED    = (1 << 8)
+  GST_VIDEO_FORMAT_FLAG_TILED    = (1 << 8),
+  /**
+   * GST_VIDEO_FORMAT_FLAG_SUBTILES:
+   *
+   * The tile size varies per plane according to the subsampling.
+   *
+   * Since: 1.22
+   */
+  GST_VIDEO_FORMAT_FLAG_SUBTILES = (1 << 9)
 } GstVideoFormatFlags;
 
 /* YUV components */
@@ -602,6 +622,19 @@ struct _GstVideoFormatInfo {
 #define GST_VIDEO_FORMAT_INFO_HAS_PALETTE(info)  (((info)->flags & GST_VIDEO_FORMAT_FLAG_PALETTE) != 0)
 #define GST_VIDEO_FORMAT_INFO_IS_COMPLEX(info)   (((info)->flags & GST_VIDEO_FORMAT_FLAG_COMPLEX) != 0)
 #define GST_VIDEO_FORMAT_INFO_IS_TILED(info)     (((info)->flags & GST_VIDEO_FORMAT_FLAG_TILED) != 0)
+/**
+ * GST_VIDEO_FORMAT_INFO_HAS_SUBTILES:
+ * @info: a #GstVideoFormatInfo
+ *
+ * This macro checks if %GST_VIDEO_FORMAT_FLAG_SUBTILES is set. When this
+ * flag is set, it means that the tile sizes must be scaled as per the
+ * subsampling.
+ *
+ * Returns: %TRUE if the format uses subsampled tile sizes.
+ *
+ * Since: 1.22
+ */
+#define GST_VIDEO_FORMAT_INFO_HAS_SUBTILES(info) (((info)->flags & GST_VIDEO_FORMAT_FLAG_SUBTILES) != 0)
 
 #define GST_VIDEO_FORMAT_INFO_BITS(info)         ((info)->bits)
 #define GST_VIDEO_FORMAT_INFO_N_COMPONENTS(info) ((info)->n_components)
@@ -745,7 +778,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "I422_10BE, I422_10LE, NV16_10LE32, Y210, v210, UYVP, I420_10BE, I420_10LE, " \
     "P010_10BE, P010_10LE, NV12_10LE32, NV12_10LE40, Y444, RGBP, GBR, BGRP, NV24, xBGR, BGRx, " \
     "xRGB, RGBx, BGR, IYU2, v308, RGB, Y42B, NV61, NV16, VYUY, UYVY, YVYU, YUY2, I420, " \
-    "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, Y41B, IYU1, YVU9, YUV9, RGB16, " \
+    "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, NV12_16L32S, Y41B, IYU1, YVU9, YUV9, RGB16, " \
     "BGR16, RGB15, BGR15, RGB8P, GRAY16_BE, GRAY16_LE, GRAY10_LE32, GRAY8 }"
 #elif G_BYTE_ORDER == G_LITTLE_ENDIAN
 #define GST_VIDEO_FORMATS_ALL "{ ABGR64_LE, BGRA64_LE, AYUV64, ARGB64_LE, ARGB64, " \
@@ -758,7 +791,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "I422_10LE, I422_10BE, NV16_10LE32, Y210, v210, UYVP, I420_10LE, I420_10BE, " \
     "P010_10LE, NV12_10LE32, NV12_10LE40, P010_10BE, Y444, RGBP, GBR, BGRP, NV24, xBGR, BGRx, " \
     "xRGB, RGBx, BGR, IYU2, v308, RGB, Y42B, NV61, NV16, VYUY, UYVY, YVYU, YUY2, I420, " \
-    "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, Y41B, IYU1, YVU9, YUV9, RGB16, " \
+    "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, NV12_16L32S, Y41B, IYU1, YVU9, YUV9, RGB16, " \
     "BGR16, RGB15, BGR15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }"
 #endif
 

subprojects/gst-plugins-base/gst-libs/gst/video/video-frame.c

@@ -278,6 +278,27 @@ gst_video_frame_unmap (GstVideoFrame * frame)
     gst_buffer_unref (frame->buffer);
 }
 
+static void
+scale_tile_shifts (const GstVideoFormatInfo * finfo, gint plane, guint * ws,
+    guint * hs)
+{
+  gint comp[GST_VIDEO_MAX_COMPONENTS];
+  gint i;
+
+  gst_video_format_info_component (finfo, plane, comp);
+
+  /* scale the tile size according to the subsampling */
+  *ws -= finfo->w_sub[comp[0]];
+  *hs -= finfo->h_sub[comp[0]];
+
+  /* for each additional component in the same plane, double the tile width,
+   * this should provide the appropriate tile size when the tile size varies
+   * base on the subsampling. */
+  for (i = 1; comp[i] >= 0; i++)
+    *ws += 1;
+}
+
+
 /**
  * gst_video_frame_copy_plane:
  * @dest: a #GstVideoFrame
@@ -348,6 +369,10 @@ gst_video_frame_copy_plane (GstVideoFrame * dest, const GstVideoFrame * src,
 
     ws = GST_VIDEO_FORMAT_INFO_TILE_WS (finfo);
     hs = GST_VIDEO_FORMAT_INFO_TILE_HS (finfo);
+
+    if (GST_VIDEO_FORMAT_INFO_HAS_SUBTILES (finfo))
+      scale_tile_shifts (finfo, plane, &ws, &hs);
+
     ts = ws + hs;
 
     tile_size = 1 << ts;

subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c

@@ -1112,6 +1112,25 @@ fill_planes (GstVideoInfo * info, gsize plane_size[GST_VIDEO_MAX_PLANES])
           (GST_ROUND_UP_N (height, 1 << (hs + 1)) / 2);
       break;
     }
+    case GST_VIDEO_FORMAT_NV12_16L32S:
+    {
+      gint ws = GST_VIDEO_FORMAT_INFO_TILE_WS (info->finfo);
+      gint hs = GST_VIDEO_FORMAT_INFO_TILE_HS (info->finfo);
+      info->stride[0] =
+          GST_VIDEO_TILE_MAKE_STRIDE (GST_ROUND_UP_N (width, 1 << ws) >> ws,
+          GST_ROUND_UP_N (height, 1 << hs) >> hs);
+      /*
+       * size of UV plane tiles is subsample, hence have the same number of
+       * tiles in both directions.
+       */
+      info->stride[1] = info->stride[0];
+      info->offset[0] = 0;
+      info->offset[1] =
+          GST_ROUND_UP_N (width, 1 << ws) * GST_ROUND_UP_N (height, 1 << hs);
+      info->size = info->offset[1] + GST_ROUND_UP_N (width, 1 << ws) *
+          GST_ROUND_UP_N (height, 1 << (hs - 1));
+      break;
+    }
     case GST_VIDEO_FORMAT_A420_10LE:
     case GST_VIDEO_FORMAT_A420_10BE:
       info->stride[0] = GST_ROUND_UP_4 (width * 2);

subprojects/gst-plugins-base/gst-libs/gst/video/video-tile.h

@@ -113,6 +113,8 @@ typedef enum
  *    in memory in Z or flipped Z order. In case of odd rows, the last row
  *    of blocks is arranged in linear order.
  * @GST_VIDEO_TILE_MODE_LINEAR: Tiles are in row order. (Since: 1.18)
+ * @GST_VIDEO_TILE_MODE_LINEAR_SUBSAMPLED: Tiles are in row order, with
+ *   variable tile size according to subsampling. (Since: 1.20)
  *
  * Enum value describing the available tiling modes.
  */

subprojects/gst-plugins-base/tests/check/elements/videoscale.c

@@ -126,6 +126,7 @@ check_pad_template (GstPadTemplate * tmpl)
         case GST_VIDEO_FORMAT_NV12_64Z32:
         case GST_VIDEO_FORMAT_NV12_4L4:
         case GST_VIDEO_FORMAT_NV12_32L32:
+        case GST_VIDEO_FORMAT_NV12_16L32S:
           GST_LOG ("Ignoring lack of support for format %s", fmt_str);
           break;
         default:

subprojects/gst-plugins-base/tests/check/libs/video.c

@@ -3203,6 +3203,7 @@ GST_START_TEST (test_video_formats_pstrides)
         || fmt == GST_VIDEO_FORMAT_NV12_64Z32
         || fmt == GST_VIDEO_FORMAT_NV12_4L4
         || fmt == GST_VIDEO_FORMAT_NV12_32L32
+        || fmt == GST_VIDEO_FORMAT_NV12_16L32S
         || fmt == GST_VIDEO_FORMAT_NV12_10LE32
         || fmt == GST_VIDEO_FORMAT_NV16_10LE32
         || fmt == GST_VIDEO_FORMAT_NV12_10LE40