video: Add support for linear 8x128 NV12 tiles and 10bit BE tiles

This adds linear 8x128 NV12 based tiles and NV12 10bit big endian tiles.
These formats are used by i.MX 8QXP/8QM VPU and exposed in V4L2.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1379>

subprojects/gst-plugins-base/gst-libs/gst/video/video-converter.c

@@ -7356,6 +7356,8 @@ get_scale_format (GstVideoFormat format, gint plane)
     case GST_VIDEO_FORMAT_Y212_LE:
     case GST_VIDEO_FORMAT_Y412_BE:
     case GST_VIDEO_FORMAT_Y412_LE:
+    case GST_VIDEO_FORMAT_NV12_8L128:
+    case GST_VIDEO_FORMAT_NV12_10BE_8L128:
       res = format;
       g_assert_not_reached ();
       break;

subprojects/gst-plugins-base/gst-libs/gst/video/video-format.c

@@ -6767,6 +6767,145 @@ pack_BGRP (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
   video_orc_pack_Y444 (dr, dg, db, src, width);
 }
 
+#define PACK_NV12_10BE_8L128  GST_VIDEO_FORMAT_AYUV64, unpack_NV12_10BE_8L128, 1, pack_NV12_10BE_8L128
+#define GST_SET_U8_BITS(__pdata, __val, __mask, __shift)  \
+  *(guint8 *)(__pdata) = (guint8)(((*(guint8 *)(__pdata)) & (~((__mask) << (__shift)))) | (((__val) & (__mask)) << (__shift)))
+#define GST_GET_U8_BITS(__pdata, __mask, __shift)   (((*(guint8 *)(__pdata)) >> (__shift)) & (__mask))
+
+static void
+get_tile_NV12_10BE (const gint bit_depth,
+    gint ts, gint ws, gint hs, gint ty, gint y, gint x,
+    const gpointer data, const gint stride,
+    gpointer restrict pdata[2], gint bits[2], guint16 mask[2])
+{
+  gint bit_index = bit_depth * x;
+  gint line_size = GST_VIDEO_TILE_X_TILES (stride) << ts;
+  gint tile_size = 1 << ts;
+  gint tile_width = 1 << ws;
+  gint pos[2];
+  gsize offset;
+  gint i;
+
+  pos[0] = bit_index / 8;
+  bits[0] = 8 - (bit_index % 8);
+  if (bits[0] > bit_depth)
+    bits[0] = bit_depth;
+  bits[1] = bit_depth > bits[0] ? (bit_depth - bits[0]) : 0;
+  pos[1] = bits[1] ? (pos[0] + 1) : 0;
+  mask[0] = (1 << bits[0]) - 1;
+  mask[1] = (1 << bits[1]) - 1;
+
+  for (i = 0; i < 2; i++) {
+    offset =
+        line_size * ty + tile_size * (pos[i] >> ws) + tile_width * y +
+        (pos[i] % tile_width);
+    pdata[i] = ((guint8 *) data) + offset;
+  }
+}
+
+static void
+unpack_NV12_10BE_8L128 (const GstVideoFormatInfo * info,
+    GstVideoPackFlags flags, gpointer dest,
+    const gpointer data[GST_VIDEO_MAX_PLANES],
+    const gint stride[GST_VIDEO_MAX_PLANES], gint x, gint y, gint width)
+{
+  const gint bit_depth = 10;
+  guint16 *restrict d = dest;
+  gint ws, hs, ts;
+  gint ty, uv_ty, uv_y;
+  guint16 Y = 0, U = 0, V = 0;
+  gpointer restrict pdata[2];
+  gint bits[2];
+  guint16 mask[2];
+  int i;
+  int j;
+
+  ws = GST_VIDEO_FORMAT_INFO_TILE_WS (info);
+  hs = GST_VIDEO_FORMAT_INFO_TILE_HS (info);
+  ts = ws + hs;
+  ty = y >> hs;
+  y = y & ((1 << hs) - 1);
+  uv_ty = ty >> 1;
+  uv_y = (((ty % 2) << hs) + y) / 2;
+
+  for (i = 0; i < width; i++) {
+    j = x + i;
+
+    get_tile_NV12_10BE (bit_depth, ts, ws, hs, ty, y, j,
+        data[0], stride[0], pdata, bits, mask);
+    Y = (GST_GET_U8_BITS (pdata[0], mask[0], 0) << bits[1]) |
+        GST_GET_U8_BITS (pdata[1], mask[1], 8 - bits[1]);
+
+    if ((i == 0) && (j % 2))
+      j--;
+    if (j % 2 == 0) {
+      get_tile_NV12_10BE (bit_depth, ts, ws, hs, uv_ty, uv_y, j,
+          data[1], stride[1], pdata, bits, mask);
+      U = (GST_GET_U8_BITS (pdata[0], mask[0], 0) << bits[1]) |
+          GST_GET_U8_BITS (pdata[1], mask[1], 8 - bits[1]);
+
+      get_tile_NV12_10BE (bit_depth, ts, ws, hs, uv_ty, uv_y, j + 1,
+          data[1], stride[1], pdata, bits, mask);
+      V = (GST_GET_U8_BITS (pdata[0], mask[0], 0) << bits[1]) |
+          GST_GET_U8_BITS (pdata[1], mask[1], 8 - bits[1]);
+    }
+
+    d[0] = 0xffff;
+    d[1] = Y << 6;
+    d[2] = U << 6;
+    d[3] = V << 6;
+    d += 4;
+  }
+}
+
+static void
+pack_NV12_10BE_8L128 (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
+    const gpointer src, gint sstride, gpointer data[GST_VIDEO_MAX_PLANES],
+    const gint stride[GST_VIDEO_MAX_PLANES], GstVideoChromaSite chroma_site,
+    gint y, gint width)
+{
+  const guint16 *restrict s = src;
+  const gint bit_depth = 10;
+  gint ws, hs, ts;
+  gint ty, uv_ty, uv_y;
+  guint16 Y, U, V;
+  gpointer restrict pdata[2];
+  gint bits[2];
+  guint16 mask[2];
+  int i;
+
+  ws = GST_VIDEO_FORMAT_INFO_TILE_WS (info);
+  hs = GST_VIDEO_FORMAT_INFO_TILE_HS (info);
+  ts = ws + hs;
+  ty = y >> hs;
+  y = y & ((1 << hs) - 1);
+  uv_ty = ty >> 1;
+  uv_y = (((ty % 2) << hs) + y) / 2;
+
+  for (i = 0; i < width; i++) {
+    Y = s[i * 4 + 1] >> 6;
+    U = s[i * 4 + 2] >> 6;
+    V = s[i * 4 + 3] >> 6;
+
+    get_tile_NV12_10BE (bit_depth, ts, ws, hs, ty, y, i,
+        data[0], stride[0], pdata, bits, mask);
+    GST_SET_U8_BITS (pdata[0], Y >> bits[1], mask[0], 0);
+    GST_SET_U8_BITS (pdata[1], Y & mask[1], mask[1], 8 - bits[1]);
+
+    if ((y % 2 == 0) && (i % 2 == 0)) {
+      get_tile_NV12_10BE (bit_depth, ts, ws, hs, uv_ty, uv_y, i,
+          data[1], stride[1], pdata, bits, mask);
+      GST_SET_U8_BITS (pdata[0], U >> bits[1], mask[0], 0);
+      GST_SET_U8_BITS (pdata[1], U & mask[1], mask[1], 8 - bits[1]);
+
+      get_tile_NV12_10BE (bit_depth, ts, ws, hs, uv_ty, uv_y, i + 1,
+          data[1], stride[1], pdata, bits, mask);
+      GST_SET_U8_BITS (pdata[0], V >> bits[1], mask[0], 0);
+      GST_SET_U8_BITS (pdata[1], V & mask[1], mask[1], 8 - bits[1]);
+    }
+  }
+}
+
 typedef struct
 {
   guint32 fourcc;
@@ -6865,6 +7004,7 @@ typedef struct
 #define TILE_16x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 4, 5
 #define TILE_32x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 5, 5
 #define TILE_64x32(mode) GST_VIDEO_TILE_MODE_ ##mode, 6, 5
+#define TILE_8x128(mode) GST_VIDEO_TILE_MODE_ ##mode, 3, 7
 
 #define MAKE_YUV_FORMAT(name, desc, fourcc, depth, pstride, plane, offs, sub, pack ) \
  { fourcc, {GST_VIDEO_FORMAT_ ##name, G_STRINGIFY(name), desc, GST_VIDEO_FORMAT_FLAG_YUV, depth, pstride, plane, offs, sub, pack } }
@@ -7165,6 +7305,12 @@ static const VideoFormat formats[] = {
   MAKE_YUV_ST_FORMAT (NV12_16L32S, "raw video",
       GST_MAKE_FOURCC ('M', 'M', '2', '1'), DPTH888, PSTR122, PLANE011,
       OFFS001, SUB420, PACK_NV12_TILED, TILE_16x32 (LINEAR)),
+  MAKE_YUV_T_FORMAT (NV12_8L128, "raw video",
+      GST_MAKE_FOURCC ('N', 'A', '1', '2'), DPTH888, PSTR122, PLANE011,
+      OFFS001, SUB420, PACK_NV12_TILED, TILE_8x128 (LINEAR)),
+  MAKE_YUV_T_FORMAT (NV12_10BE_8L128, "raw video",
+      GST_MAKE_FOURCC ('N', 'T', '1', '2'), DPTH10_10_10, PSTR122, PLANE011,
+      OFFS001, SUB420, PACK_NV12_10BE_8L128, TILE_8x128 (LINEAR)),
 };
 
 static GstVideoFormat

subprojects/gst-plugins-base/gst-libs/gst/video/video-format.h

@@ -143,6 +143,8 @@ G_BEGIN_DECLS
  * @GST_VIDEO_FORMAT_ABGR64_LE: reverse RGB with alpha channel first, 16 bits per channel
  * @GST_VIDEO_FORMAT_ABGR64_BE: reverse RGB with alpha channel first, 16 bits per channel
  * @GST_VIDEO_FORMAT_NV12_16L32S: NV12 with 16x32 Y tiles and 16x16 UV tiles. (Since: 1.22)
+ * @GST_VIDEO_FORMAT_NV12_8L128 : NV12 with 8x128 tiles in linear order (Since: 1.22)
+ * @GST_VIDEO_FORMAT_NV12_10BE_8L128 : NV12 10bit big endian with 8x128 tiles in linear order (Since: 1.22)
  *
  * Enum value describing the most common video formats.
  *
@@ -371,6 +373,7 @@ typedef enum {
    * Since: 1.20
    */
   GST_VIDEO_FORMAT_ABGR64_BE,
+
   /**
    * GST_VIDEO_FORMAT_NV12_16L32S:
    *
@@ -380,6 +383,23 @@ typedef enum {
    */
   GST_VIDEO_FORMAT_NV12_16L32S,
 
+  /**
+   * GST_VIDEO_FORMAT_NV12_8L128:
+   *
+   * NV12 with 8x128 tiles in linear order.
+   *
+   * Since: 1.22
+   */
+  GST_VIDEO_FORMAT_NV12_8L128,
+
+  /**
+   * GST_VIDEO_FORMAT_NV12_10BE_8L128:
+   *
+   * NV12 10bit big endian with 8x128 tiles in linear order.
+   *
+   * Since: 1.22
+   */
+  GST_VIDEO_FORMAT_NV12_10BE_8L128,
 } GstVideoFormat;
 
 #define GST_VIDEO_MAX_PLANES 4
@@ -780,9 +800,9 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "GBR_12BE, Y444_12LE, GBR_12LE, I422_12BE, I422_12LE, Y212_BE, Y212_LE, I420_12BE, " \
     "I420_12LE, P012_BE, P012_LE, Y444_10BE, GBR_10BE, Y444_10LE, GBR_10LE, r210, " \
     "I422_10BE, I422_10LE, NV16_10LE32, Y210, v210, UYVP, I420_10BE, I420_10LE, " \
-    "P010_10BE, P010_10LE, NV12_10LE32, NV12_10LE40, Y444, RGBP, GBR, BGRP, NV24, xBGR, BGRx, " \
+    "P010_10BE, P010_10LE, NV12_10LE32, NV12_10LE40, NV12_10BE_8L128, Y444, RGBP, GBR, BGRP, NV24, xBGR, BGRx, " \
     "xRGB, RGBx, BGR, IYU2, v308, RGB, Y42B, NV61, NV16, VYUY, UYVY, YVYU, YUY2, I420, " \
-    "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, NV12_16L32S, Y41B, IYU1, YVU9, YUV9, RGB16, " \
+    "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, NV12_16L32S, NV12_8L128, Y41B, IYU1, YVU9, YUV9, RGB16, " \
     "BGR16, RGB15, BGR15, RGB8P, GRAY16_BE, GRAY16_LE, GRAY10_LE32, GRAY8 }"
 #elif G_BYTE_ORDER == G_LITTLE_ENDIAN
 #define GST_VIDEO_FORMATS_ALL "{ ABGR64_LE, BGRA64_LE, AYUV64, ARGB64_LE, ARGB64, " \
@@ -792,10 +812,10 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "AYUV, ARGB, RGBA, A420, AV12, Y444_16LE, Y444_16BE, v216, P016_LE, P016_BE, Y444_12LE, " \
     "GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y212_LE, Y212_BE, I420_12LE, " \
     "I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, Y444_10BE, GBR_10BE, r210, " \
-    "I422_10LE, I422_10BE, NV16_10LE32, Y210, v210, UYVP, I420_10LE, I420_10BE, " \
+    "I422_10LE, I422_10BE, NV16_10LE32, NV12_10BE_8L128, Y210, v210, UYVP, I420_10LE, I420_10BE, " \
     "P010_10LE, NV12_10LE32, NV12_10LE40, P010_10BE, Y444, RGBP, GBR, BGRP, NV24, xBGR, BGRx, " \
     "xRGB, RGBx, BGR, IYU2, v308, RGB, Y42B, NV61, NV16, VYUY, UYVY, YVYU, YUY2, I420, " \
-    "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, NV12_16L32S, Y41B, IYU1, YVU9, YUV9, RGB16, " \
+    "YV12, NV21, NV12, NV12_64Z32, NV12_4L4, NV12_32L32, NV12_16L32S, NV12_8L128, Y41B, IYU1, YVU9, YUV9, RGB16, " \
     "BGR16, RGB15, BGR15, RGB8P, GRAY16_LE, GRAY16_BE, GRAY10_LE32, GRAY8 }"
 #endif
 

subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c

@@ -1218,7 +1218,51 @@ fill_planes (GstVideoInfo * info, gsize plane_size[GST_VIDEO_MAX_PLANES])
         cr_h = GST_ROUND_UP_2 (cr_h);
       info->size = info->offset[1] + info->stride[0] * cr_h;
       break;
+    case GST_VIDEO_FORMAT_NV12_8L128:
+    {
+      gint ws = GST_VIDEO_FORMAT_INFO_TILE_WS (info->finfo);
+      gint hs = GST_VIDEO_FORMAT_INFO_TILE_HS (info->finfo);
+      gint ALIGN_W = 1 << ws;
+      gint ALIGN_H = 1 << hs;
 
+      if (GST_VIDEO_INFO_IS_INTERLACED (info))
+        ALIGN_H = (ALIGN_H << 1);
+      info->stride[0] =
+          GST_VIDEO_TILE_MAKE_STRIDE (GST_ROUND_UP_N (width, ALIGN_W) >> ws,
+          GST_ROUND_UP_N (height, ALIGN_H) >> hs);
+      info->stride[1] =
+          GST_VIDEO_TILE_MAKE_STRIDE (GST_ROUND_UP_N (width, ALIGN_W) >> ws,
+          GST_ROUND_UP_N (height, ALIGN_H << 1) >> (hs + 1));
+      info->offset[0] = 0;
+      info->offset[1] =
+          GST_ROUND_UP_N (width, ALIGN_W) * GST_ROUND_UP_N (height, ALIGN_H);
+      cr_h = GST_ROUND_UP_N (height, ALIGN_H << 1) / 2;
+      info->size = info->offset[1] + GST_ROUND_UP_N (width, ALIGN_W) * cr_h;
+      break;
+    }
+    case GST_VIDEO_FORMAT_NV12_10BE_8L128:
+    {
+      gint ws = GST_VIDEO_FORMAT_INFO_TILE_WS (info->finfo);
+      gint hs = GST_VIDEO_FORMAT_INFO_TILE_HS (info->finfo);
+      gint ALIGN_W = 1 << ws;
+      gint ALIGN_H = 1 << hs;
+      gint stride = GST_ROUND_UP_8 (width * 10) >> 3;
+
+      if (GST_VIDEO_INFO_IS_INTERLACED (info))
+        ALIGN_H = (ALIGN_H << 1);
+      info->stride[0] =
+          GST_VIDEO_TILE_MAKE_STRIDE (GST_ROUND_UP_N (stride, ALIGN_W) >> ws,
+          GST_ROUND_UP_N (height, ALIGN_H) >> hs);
+      info->stride[1] =
+          GST_VIDEO_TILE_MAKE_STRIDE (GST_ROUND_UP_N (stride, ALIGN_W) >> ws,
+          GST_ROUND_UP_N (height, ALIGN_H << 1) >> (hs + 1));
+      info->offset[0] = 0;
+      info->offset[1] =
+          GST_ROUND_UP_N (stride, ALIGN_W) * GST_ROUND_UP_N (height, ALIGN_H);
+      cr_h = GST_ROUND_UP_N (height, ALIGN_H << 1) / 2;
+      info->size = info->offset[1] + GST_ROUND_UP_N (stride, ALIGN_W) * cr_h;
+      break;
+    }
     case GST_VIDEO_FORMAT_ENCODED:
       break;
     case GST_VIDEO_FORMAT_UNKNOWN:

subprojects/gst-plugins-base/tests/check/elements/videoscale.c

@@ -127,6 +127,8 @@ check_pad_template (GstPadTemplate * tmpl)
         case GST_VIDEO_FORMAT_NV12_4L4:
         case GST_VIDEO_FORMAT_NV12_32L32:
         case GST_VIDEO_FORMAT_NV12_16L32S:
+        case GST_VIDEO_FORMAT_NV12_8L128:
+        case GST_VIDEO_FORMAT_NV12_10BE_8L128:
           GST_LOG ("Ignoring lack of support for format %s", fmt_str);
           break;
         default:

subprojects/gst-plugins-base/tests/check/libs/video.c

@@ -3207,7 +3207,9 @@ GST_START_TEST (test_video_formats_pstrides)
         || fmt == GST_VIDEO_FORMAT_NV12_10LE32
         || fmt == GST_VIDEO_FORMAT_NV16_10LE32
         || fmt == GST_VIDEO_FORMAT_NV12_10LE40
-        || fmt == GST_VIDEO_FORMAT_Y410) {
+        || fmt == GST_VIDEO_FORMAT_Y410
+        || fmt == GST_VIDEO_FORMAT_NV12_8L128
+        || fmt == GST_VIDEO_FORMAT_NV12_10BE_8L128) {
       fmt++;
       continue;
     }