From 22c8d1890ef27b4e8b279ddf0645b9f84340b7e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?=
 <vjaquez@igalia.com>
Date: Tue, 29 Aug 2023 12:36:34 +0200
Subject: [PATCH] va: retry if surface creation fails

Old versions of mesa doesn't support VASurfaceAttribDRMFormatModifiers. To
solve it, by just ignoring the modifiers assuming that linear is accepted and
produced, the creation of frames will be tried again without that attribute.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5256>
---
 .../gst-libs/gst/va/vasurfaceimage.c             | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/va/vasurfaceimage.c b/subprojects/gst-plugins-bad/gst-libs/gst/va/vasurfaceimage.c
index a73b9a5ac3..0a1d564287 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/va/vasurfaceimage.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/va/vasurfaceimage.c
@@ -115,8 +115,24 @@ va_create_surfaces (GstVaDisplay * display, guint rt_format, guint fourcc,
     /* *INDENT-ON* */
   }
 
+retry:
   status = vaCreateSurfaces (dpy, rt_format, width, height, surfaces,
       num_surfaces, attrs, num_attrs);
+
+  if (status == VA_STATUS_ERROR_ATTR_NOT_SUPPORTED
+      && attrs[num_attrs - 1].type == VASurfaceAttribDRMFormatModifiers) {
+    int i;
+
+    /* if requested modifiers contain linear, let's remove the attribute and
+     * "hope" the driver will create linear dmabufs */
+    for (i = 0; i < num_modifiers; ++i) {
+      if (modifiers[i] == DRM_FORMAT_MOD_LINEAR) {
+        num_attrs--;
+        goto retry;
+      }
+    }
+  }
+
   if (status != VA_STATUS_SUCCESS) {
     GST_ERROR ("vaCreateSurfaces: %s", vaErrorStr (status));
     return FALSE;