gl: expose calculating swizzle indices to/from RGBA/YUVA

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5109>

girs/GstGL-1.0.gir

@@ -11213,6 +11213,31 @@ NDC coordinates in @matrix.</doc>
         <type name="GType" c:type="GType"/>
       </return-value>
     </function>
+    <function name="gl_swizzle_invert" c:identifier="gst_gl_swizzle_invert" version="1.24">
+      <doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c">Given @swizzle, produce @inversion such that:
+
+@swizzle[@inversion[i]] == identity[i] where:
+- identity = {0, 1, 2,...}
+- unset fields are marked by -1</doc>
+      <source-position filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.h"/>
+      <return-value transfer-ownership="none">
+        <type name="none" c:type="void"/>
+      </return-value>
+      <parameters>
+        <parameter name="swizzle" transfer-ownership="none">
+          <doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c">input swizzle</doc>
+          <array zero-terminated="0" c:type="gint*" fixed-size="4">
+            <type name="gint" c:type="gint"/>
+          </array>
+        </parameter>
+        <parameter name="inversion" direction="out" caller-allocates="0" transfer-ownership="full">
+          <doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c">resulting inversion</doc>
+          <array zero-terminated="0" c:type="gint*" fixed-size="4">
+            <type name="gint" c:type="gint"/>
+          </array>
+        </parameter>
+      </parameters>
+    </function>
     <function name="gl_sync_meta_api_get_type" c:identifier="gst_gl_sync_meta_api_get_type">
       <source-position filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglsyncmeta.h"/>
       <return-value transfer-ownership="none">
@@ -11365,6 +11390,35 @@ NDC coordinates in @matrix.</doc>
         </parameter>
       </parameters>
     </function>
+    <function name="gl_video_format_swizzle" c:identifier="gst_gl_video_format_swizzle" version="1.24">
+      <doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c">Calculates the swizzle indices for @video_format and @gl_format in order to
+access a texture such that accessing a texel from a texture through the swizzle
+index produces values in the order (R, G, B, A) or (Y, U, V, A).
+
+For multi-planer formats, the swizzle index uses the same component order (RGBA/YUVA)
+and should be applied after combining multiple planes into a single rgba/yuva value.
+e.g. sampling from a NV12 format would have Y from one texture and UV from
+another texture into a (Y, U, V) value.  Add an Aplha component and then
+perform swizzling.  Sampling from NV21 would produce (Y, V, U) which is then
+swizzled to (Y, U, V).</doc>
+      <source-position filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.h"/>
+      <return-value transfer-ownership="none">
+        <doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c">whether valid swizzle indices could be found</doc>
+        <type name="gboolean" c:type="gboolean"/>
+      </return-value>
+      <parameters>
+        <parameter name="video_format" transfer-ownership="none">
+          <doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c">the #GstVideoFormat in use</doc>
+          <type name="GstVideo.VideoFormat" c:type="GstVideoFormat"/>
+        </parameter>
+        <parameter name="swizzle" direction="out" caller-allocates="0" transfer-ownership="full">
+          <doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c">the returned swizzle indices</doc>
+          <array zero-terminated="0" c:type="int*" fixed-size="4">
+            <type name="gint" c:type="int"/>
+          </array>
+        </parameter>
+      </parameters>
+    </function>
     <function name="gl_window_error_quark" c:identifier="gst_gl_window_error_quark" moved-to="GLWindowError.quark">
       <return-value transfer-ownership="none">
         <doc xml:space="preserve" filename="../subprojects/gst-plugins-base/gst-libs/gst/gl/gstglwindow.c">the quark used for #GstGLWindow in #GError's</doc>

subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c

@@ -1624,65 +1624,11 @@ _index_to_shader_swizzle (int idx)
   }
 }
 
-static void
-get_rgb_format_gl_swizzle_order (GstVideoFormat format,
-    gint swizzle[GST_VIDEO_MAX_COMPONENTS])
-{
-  const GstVideoFormatInfo *finfo = gst_video_format_get_info (format);
-  int c_i = 0, i;
-
-  g_return_if_fail (finfo->flags & GST_VIDEO_FORMAT_FLAG_RGB
-      || format == GST_VIDEO_FORMAT_AYUV || format == GST_VIDEO_FORMAT_VUYA);
-
-  for (i = 0; i < finfo->n_components; i++) {
-    swizzle[c_i++] = finfo->poffset[i];
-  }
-
-  /* special case spaced RGB formats as the space does not contain a poffset
-   * value and we need all four components to be valid in order to swizzle
-   * correctly */
-  if (format == GST_VIDEO_FORMAT_xRGB || format == GST_VIDEO_FORMAT_xBGR) {
-    swizzle[c_i++] = 0;
-  } else if (format == GST_VIDEO_FORMAT_RGBx || format == GST_VIDEO_FORMAT_BGRx) {
-    swizzle[c_i++] = 3;
-  } else {
-    for (i = finfo->n_components; i < GST_VIDEO_MAX_COMPONENTS; i++) {
-      swizzle[c_i++] = -1;
-    }
-  }
-}
-
 static void
 video_format_to_gl_reorder (GstVideoFormat v_format, gint * reorder,
     gboolean input)
 {
   switch (v_format) {
-    case GST_VIDEO_FORMAT_RGBA:
-    case GST_VIDEO_FORMAT_RGBx:
-    case GST_VIDEO_FORMAT_BGRA:
-    case GST_VIDEO_FORMAT_BGRx:
-    case GST_VIDEO_FORMAT_ARGB:
-    case GST_VIDEO_FORMAT_xRGB:
-    case GST_VIDEO_FORMAT_ABGR:
-    case GST_VIDEO_FORMAT_xBGR:
-    case GST_VIDEO_FORMAT_AYUV:
-    case GST_VIDEO_FORMAT_VUYA:
-      get_rgb_format_gl_swizzle_order (v_format, reorder);
-      break;
-    case GST_VIDEO_FORMAT_BGR:
-    case GST_VIDEO_FORMAT_BGRP:
-      reorder[0] = 2;
-      reorder[1] = 1;
-      reorder[2] = 0;
-      reorder[3] = 3;
-      break;
-    case GST_VIDEO_FORMAT_RGB:
-    case GST_VIDEO_FORMAT_RGBP:
-      reorder[0] = 0;
-      reorder[1] = 1;
-      reorder[2] = 2;
-      reorder[3] = 3;
-      break;
     case GST_VIDEO_FORMAT_UYVY:
       reorder[0] = 1;
       reorder[1] = 0;
@@ -1698,54 +1644,8 @@ video_format_to_gl_reorder (GstVideoFormat v_format, gint * reorder,
       reorder[2] = 0;
       reorder[3] = input ? 3 : 2;
       break;
-    case GST_VIDEO_FORMAT_AV12:
-    case GST_VIDEO_FORMAT_NV12:
-    case GST_VIDEO_FORMAT_NV16:
-    case GST_VIDEO_FORMAT_P010_10LE:
-    case GST_VIDEO_FORMAT_P010_10BE:
-    case GST_VIDEO_FORMAT_P012_LE:
-    case GST_VIDEO_FORMAT_P012_BE:
-    case GST_VIDEO_FORMAT_P016_LE:
-    case GST_VIDEO_FORMAT_P016_BE:
-    case GST_VIDEO_FORMAT_NV12_16L32S:
-    case GST_VIDEO_FORMAT_NV12_4L4:
-      reorder[0] = 0;
-      reorder[1] = 1;
-      reorder[2] = 2;
-      reorder[3] = 3;
-      break;
-    case GST_VIDEO_FORMAT_NV21:
-    case GST_VIDEO_FORMAT_NV61:
-      reorder[0] = 0;
-      reorder[1] = 2;
-      reorder[2] = 1;
-      reorder[3] = 3;
-      break;
-    case GST_VIDEO_FORMAT_Y410:
-    case GST_VIDEO_FORMAT_Y412_LE:
-    case GST_VIDEO_FORMAT_Y412_BE:
-      reorder[0] = 1;
-      reorder[1] = 0;
-      reorder[2] = 2;
-      reorder[3] = 0;
-      break;
-    case GST_VIDEO_FORMAT_I420:
-    case GST_VIDEO_FORMAT_Y444:
-    case GST_VIDEO_FORMAT_Y42B:
-    case GST_VIDEO_FORMAT_Y41B:
-    case GST_VIDEO_FORMAT_A420:
-      reorder[0] = 0;
-      reorder[1] = 1;
-      reorder[2] = 2;
-      reorder[3] = 3;
-      break;
-    case GST_VIDEO_FORMAT_YV12:
-      reorder[0] = 0;
-      reorder[1] = 2;
-      reorder[2] = 1;
-      reorder[3] = 3;
-      break;
     default:
+      if (!gst_gl_video_format_swizzle (v_format, reorder))
         g_assert_not_reached ();
       break;
   }
@@ -1754,30 +1654,6 @@ video_format_to_gl_reorder (GstVideoFormat v_format, gint * reorder,
       reorder[3]);
 }
 
-/* given a swizzle index, produce an index such that:
- *
- * swizzle[idx[i]] == identity[i] where:
- * - swizzle is the original swizzle
- * - idx is the result
- * - identity = {0, 1, 2,...}
- * - unset fields are marked by -1
- */
-static void
-swizzle_identity_order (gint * swizzle, gint * idx)
-{
-  int i;
-
-  for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
-    idx[i] = -1;
-  }
-
-  for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
-    if (swizzle[i] >= 0 && swizzle[i] < 4 && idx[swizzle[i]] == -1) {
-      idx[swizzle[i]] = i;
-    }
-  }
-}
-
 static void
 calculate_reorder_indexes (GstVideoFormat in_format,
     GstVideoFormat out_format,
@@ -1797,7 +1673,7 @@ calculate_reorder_indexes (GstVideoFormat in_format,
     for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
       ret_out[i] = out_reorder[i];
   } else {
-    swizzle_identity_order (out_reorder, ret_out);
+    gst_gl_swizzle_invert (out_reorder, ret_out);
   }
 
   for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)

subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c

@@ -456,6 +456,144 @@ gst_gl_format_n_components (GstGLFormat gl_format)
   }
 }
 
+static void
+get_single_planar_format_gl_swizzle_order (GstVideoFormat format,
+    gint swizzle[GST_VIDEO_MAX_COMPONENTS])
+{
+  const GstVideoFormatInfo *finfo = gst_video_format_get_info (format);
+  int c_i = 0, i;
+
+  g_return_if_fail (finfo->flags & GST_VIDEO_FORMAT_FLAG_RGB
+      || format == GST_VIDEO_FORMAT_AYUV || format == GST_VIDEO_FORMAT_VUYA);
+
+  for (i = 0; i < finfo->n_components; i++) {
+    swizzle[c_i++] = finfo->poffset[i];
+  }
+
+  /* special case spaced RGB formats as the space does not contain a poffset
+   * value and we need all four components to be valid in order to swizzle
+   * correctly */
+  if (format == GST_VIDEO_FORMAT_xRGB || format == GST_VIDEO_FORMAT_xBGR) {
+    swizzle[c_i++] = 0;
+  } else if (format == GST_VIDEO_FORMAT_RGBx || format == GST_VIDEO_FORMAT_BGRx
+      || format == GST_VIDEO_FORMAT_RGB || format == GST_VIDEO_FORMAT_BGR) {
+    swizzle[c_i++] = 3;
+  } else {
+    for (i = finfo->n_components; i < GST_VIDEO_MAX_COMPONENTS; i++) {
+      swizzle[c_i++] = -1;
+    }
+  }
+}
+
+/**
+ * gst_gl_video_format_swizzle:
+ * @video_format: the #GstVideoFormat in use
+ * @swizzle: (out) (array fixed-size=4): the returned swizzle indices
+ *
+ * Calculates the swizzle indices for @video_format and @gl_format in order to
+ * access a texture such that accessing a texel from a texture through the swizzle
+ * index produces values in the order (R, G, B, A) or (Y, U, V, A).
+ *
+ * For multi-planer formats, the swizzle index uses the same component order (RGBA/YUVA)
+ * and should be applied after combining multiple planes into a single rgba/yuva value.
+ * e.g. sampling from a NV12 format would have Y from one texture and UV from
+ * another texture into a (Y, U, V) value.  Add an Aplha component and then
+ * perform swizzling.  Sampling from NV21 would produce (Y, V, U) which is then
+ * swizzled to (Y, U, V).
+ *
+ * Returns: whether valid swizzle indices could be found
+ *
+ * Since: 1.24
+ */
+gboolean
+gst_gl_video_format_swizzle (GstVideoFormat video_format, int *swizzle)
+{
+  const GstVideoFormatInfo *finfo = gst_video_format_get_info (video_format);
+
+  if (finfo->n_planes == 1) {
+    get_single_planar_format_gl_swizzle_order (video_format, swizzle);
+    return TRUE;
+  }
+
+  switch (video_format) {
+    case GST_VIDEO_FORMAT_BGRP:
+      get_single_planar_format_gl_swizzle_order (GST_VIDEO_FORMAT_BGR, swizzle);
+      return TRUE;
+    case GST_VIDEO_FORMAT_RGBP:
+      get_single_planar_format_gl_swizzle_order (GST_VIDEO_FORMAT_RGB, swizzle);
+      return TRUE;
+    case GST_VIDEO_FORMAT_AV12:
+    case GST_VIDEO_FORMAT_NV12:
+    case GST_VIDEO_FORMAT_NV16:
+    case GST_VIDEO_FORMAT_P010_10LE:
+    case GST_VIDEO_FORMAT_P010_10BE:
+    case GST_VIDEO_FORMAT_P012_LE:
+    case GST_VIDEO_FORMAT_P012_BE:
+    case GST_VIDEO_FORMAT_P016_LE:
+    case GST_VIDEO_FORMAT_P016_BE:
+    case GST_VIDEO_FORMAT_NV12_16L32S:
+    case GST_VIDEO_FORMAT_NV12_4L4:
+    case GST_VIDEO_FORMAT_I420:
+    case GST_VIDEO_FORMAT_Y444:
+    case GST_VIDEO_FORMAT_Y42B:
+    case GST_VIDEO_FORMAT_Y41B:
+    case GST_VIDEO_FORMAT_A420:
+      swizzle[0] = 0;
+      swizzle[1] = 1;
+      swizzle[2] = 2;
+      swizzle[3] = 3;
+      return TRUE;
+    case GST_VIDEO_FORMAT_NV21:
+    case GST_VIDEO_FORMAT_NV61:
+    case GST_VIDEO_FORMAT_YV12:
+      swizzle[0] = 0;
+      swizzle[1] = 2;
+      swizzle[2] = 1;
+      swizzle[3] = 3;
+      return TRUE;
+    case GST_VIDEO_FORMAT_Y410:
+    case GST_VIDEO_FORMAT_Y412_LE:
+    case GST_VIDEO_FORMAT_Y412_BE:
+      swizzle[0] = 1;
+      swizzle[1] = 0;
+      swizzle[2] = 2;
+      swizzle[3] = 3;
+      return TRUE;
+      /* TODO: deal with YUY2 variants */
+    default:
+      return FALSE;
+  }
+}
+
+/**
+ * gst_gl_swizzle_invert:
+ * @swizzle: (array fixed-size=4): input swizzle
+ * @inversion: (out) (array fixed-size=4): resulting inversion
+ *
+ * Given @swizzle, produce @inversion such that:
+ *
+ * @swizzle[@inversion[i]] == identity[i] where:
+ * - identity = {0, 1, 2,...}
+ * - unset fields are marked by -1
+ *
+ * Since: 1.24
+ */
+void
+gst_gl_swizzle_invert (gint * swizzle, gint * inversion)
+{
+  int i;
+
+  for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
+    inversion[i] = -1;
+  }
+
+  for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
+    if (swizzle[i] >= 0 && swizzle[i] < 4 && inversion[swizzle[i]] == -1) {
+      inversion[swizzle[i]] = i;
+    }
+  }
+}
+
 /**
  * gst_gl_format_is_supported:
  * @context: a #GstGLContext

subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.h

@@ -151,11 +151,18 @@ void                    gst_gl_format_type_from_sized_gl_format     (GstGLFormat
                                                                      GstGLFormat * unsized_format,
                                                                      guint * gl_type);
 
+GST_GL_API
+gboolean                gst_gl_video_format_swizzle                 (GstVideoFormat video_format,
+                                                                     int * swizzle);
+
 GST_GL_API
 gboolean                gst_gl_format_is_supported                  (GstGLContext * context,
                                                                      GstGLFormat format);
 GST_GL_API
 guint                   gst_gl_format_n_components                  (GstGLFormat gl_format);
+GST_GL_API
+void                    gst_gl_swizzle_invert                       (gint * swizzle,
+                                                                     gint * inversion);
 
 GST_GL_API
 GstGLTextureTarget      gst_gl_texture_target_from_string           (const gchar * str);