Handle I420 formats internally in GstVaapiImage.

gst-libs/gst/vaapi/gstvaapidisplay.c

@@ -54,15 +54,53 @@ enum {
 };
 
 static void
-filter_formats(VAImageFormat *va_formats, unsigned int *pnum_va_formats)
+append_format(
+    VAImageFormat     **pva_formats,
+    unsigned int       *pnum_va_formats,
+    GstVaapiImageFormat format
+)
+{
+    const VAImageFormat *va_format;
+    VAImageFormat *new_va_formats;
+
+    va_format = gst_vaapi_image_format_get_va_format(format);
+    if (!va_format)
+        return;
+
+    new_va_formats = realloc(
+        *pva_formats,
+        sizeof(new_va_formats[0]) * (1 + *pnum_va_formats)
+    );
+    if (!new_va_formats)
+        return;
+
+    new_va_formats[(*pnum_va_formats)++] = *va_format;
+    *pva_formats = new_va_formats;
+}
+
+static void
+filter_formats(VAImageFormat **pva_formats, unsigned int *pnum_va_formats)
 {
     unsigned int i = 0;
+    gboolean has_YV12 = FALSE;
+    gboolean has_I420 = FALSE;
 
     while (i < *pnum_va_formats) {
-        VAImageFormat * const va_format = &va_formats[i];
+        VAImageFormat * const va_format = &(*pva_formats)[i];
         const GstVaapiImageFormat format = gst_vaapi_image_format(va_format);
-        if (format)
+        if (format) {
             ++i;
+            switch (format) {
+            case GST_VAAPI_IMAGE_YV12:
+                has_YV12 = TRUE;
+                break;
+            case GST_VAAPI_IMAGE_I420:
+                has_I420 = TRUE;
+                break;
+            default:
+                break;
+            }
+        }
         else {
             /* Remove any format that is not supported by libgstvaapi */
             GST_DEBUG("unsupported format %c%c%c%c",
@@ -70,9 +108,16 @@ filter_formats(VAImageFormat *va_formats, unsigned int *pnum_va_formats)
                       (va_format->fourcc >> 8) & 0xff,
                       (va_format->fourcc >> 16) & 0xff,
                       (va_format->fourcc >> 24) & 0xff);
-            *va_format = va_formats[--(*pnum_va_formats)];
+            *va_format = (*pva_formats)[--(*pnum_va_formats)];
         }
     }
+
+    /* Append I420 (resp. YV12) format if YV12 (resp. I420) is not
+       supported by the underlying driver */
+    if (has_YV12 && !has_I420)
+        append_format(pva_formats, pnum_va_formats, GST_VAAPI_IMAGE_I420);
+    else if (has_I420 && !has_YV12)
+        append_format(pva_formats, pnum_va_formats, GST_VAAPI_IMAGE_YV12);
 }
 
 /* Sort image formats. Prefer YUV formats first */
@@ -207,7 +252,7 @@ gst_vaapi_display_create(GstVaapiDisplay *display)
     for (i = 0; i < priv->num_image_formats; i++)
         GST_DEBUG("  %s", string_of_FOURCC(priv->image_formats[i].fourcc));
 
-    filter_formats(priv->image_formats, &priv->num_image_formats);
+    filter_formats(&priv->image_formats, &priv->num_image_formats);
     qsort(
         priv->image_formats,
         priv->num_image_formats,
@@ -232,7 +277,7 @@ gst_vaapi_display_create(GstVaapiDisplay *display)
     if (!vaapi_check_status(status, "vaQuerySubpictureFormats()"))
         return FALSE;
 
-    filter_formats(priv->subpicture_formats, &priv->num_subpicture_formats);
+    filter_formats(&priv->subpicture_formats, &priv->num_subpicture_formats);
     qsort(
         priv->subpicture_formats,
         priv->num_subpicture_formats,

gst-libs/gst/vaapi/gstvaapiimage.c

@@ -39,6 +39,7 @@ struct _GstVaapiImagePrivate {
     gboolean            is_constructed;
     VAImage             image;
     guchar             *image_data;
+    GstVaapiImageFormat internal_format;
     GstVaapiImageFormat format;
     guint               width;
     guint               height;
@@ -77,29 +78,57 @@ gst_vaapi_image_destroy(GstVaapiImage *image)
 }
 
 static gboolean
-gst_vaapi_image_create(GstVaapiImage *image)
+_gst_vaapi_image_create(GstVaapiImage *image, GstVaapiImageFormat format)
 {
     GstVaapiImagePrivate * const priv = image->priv;
-    const VAImageFormat *format;
+    const VAImageFormat *va_format;
     VAStatus status;
 
-    if (!gst_vaapi_display_has_image_format(priv->display, priv->format))
+    if (!gst_vaapi_display_has_image_format(priv->display, format))
         return FALSE;
 
-    format = gst_vaapi_image_format_get_va_format(priv->format);
+    va_format = gst_vaapi_image_format_get_va_format(format);
-
+    if (!va_format)
-    g_return_val_if_fail(format, FALSE);
+        return FALSE;
 
     status = vaCreateImage(
         gst_vaapi_display_get_display(priv->display),
-        (VAImageFormat *)format,
+        (VAImageFormat *)va_format,
         priv->width,
         priv->height,
         &priv->image
     );
-    if (!vaapi_check_status(status, "vaCreateImage()"))
+    return (status == VA_STATUS_SUCCESS &&
+            priv->image.format.fourcc == va_format->fourcc);
+}
+
+static gboolean
+gst_vaapi_image_create(GstVaapiImage *image)
+{
+    GstVaapiImagePrivate * const priv = image->priv;
+
+    if (_gst_vaapi_image_create(image, priv->format)) {
+        priv->internal_format = priv->format;
+        return TRUE;
+    }
+
+    switch (priv->format) {
+    case GST_VAAPI_IMAGE_I420:
+        priv->internal_format = GST_VAAPI_IMAGE_YV12;
+        break;
+    case GST_VAAPI_IMAGE_YV12:
+        priv->internal_format = GST_VAAPI_IMAGE_I420;
+        break;
+    default:
+        priv->internal_format = 0;
+        break;
+    }
+    if (!priv->internal_format)
+        return FALSE;
+    if (!_gst_vaapi_image_create(image, priv->internal_format))
         return FALSE;
 
+    GST_DEBUG("image 0x%08x", priv->image.image_id);
     return TRUE;
 }
 
@@ -462,13 +491,15 @@ gst_vaapi_image_update_from_buffer(GstVaapiImage *image, GstBuffer *buffer)
         return FALSE;
 
     format = gst_vaapi_image_format_from_caps(caps);
-    swap_YUV = ((format == GST_VAAPI_IMAGE_I420 &&
+    if (format != priv->format)
-                 priv->format == GST_VAAPI_IMAGE_YV12) ||
-                (format == GST_VAAPI_IMAGE_YV12 &&
-                 priv->format == GST_VAAPI_IMAGE_I420));
-    if (format != priv->format && !swap_YUV)
         return FALSE;
 
+    swap_YUV = (priv->format != priv->internal_format &&
+                ((priv->format == GST_VAAPI_IMAGE_I420 &&
+                  priv->internal_format == GST_VAAPI_IMAGE_YV12) ||
+                 (priv->format == GST_VAAPI_IMAGE_YV12 &&
+                  priv->internal_format == GST_VAAPI_IMAGE_I420)));
+
     structure = gst_caps_get_structure(caps, 0);
     gst_structure_get_int(structure, "width",  &width);
     gst_structure_get_int(structure, "height", &height);
@@ -478,7 +509,7 @@ gst_vaapi_image_update_from_buffer(GstVaapiImage *image, GstBuffer *buffer)
     if (!gst_vaapi_image_map(image))
         return FALSE;
 
-    if (format == priv->format && data_size == priv->image.data_size)
+    if (format == priv->internal_format && data_size == priv->image.data_size)
         memcpy(priv->image_data, data, data_size);
     else {
         /* XXX: copied from gst_video_format_get_row_stride() -- no NV12? */

sys/vaapiconvert/gstvaapiconvert.c

@@ -292,41 +292,14 @@ gst_vaapiconvert_transform_caps(
             return NULL;
         out_caps = gst_caps_from_string(gst_vaapiconvert_yuv_caps_str);
         if (convert->display) {
-            GstVaapiImageFormat fixup_format;
+            GstCaps *allowed_caps, *inter_caps;
-            GstCaps *allowed_caps, *new_caps;
-
             allowed_caps = gst_vaapi_display_get_image_caps(convert->display);
             if (!allowed_caps)
                 return NULL;
-
+            inter_caps = gst_caps_intersect(out_caps, allowed_caps);
-            new_caps = gst_caps_intersect(out_caps, allowed_caps);
             gst_caps_unref(allowed_caps);
             gst_caps_unref(out_caps);
-            out_caps = new_caps;
+            out_caps = inter_caps;
-
-            convert->has_YV12 = gst_vaapi_display_has_image_format(
-                convert->display,
-                GST_VAAPI_IMAGE_YV12
-            );
-            convert->has_I420 = gst_vaapi_display_has_image_format(
-                convert->display,
-                GST_VAAPI_IMAGE_I420
-            );
-            if (convert->has_YV12 && !convert->has_I420)
-                fixup_format = GST_VAAPI_IMAGE_I420;
-            else if (convert->has_I420 && !convert->has_YV12)
-                fixup_format = GST_VAAPI_IMAGE_YV12;
-            else
-                fixup_format = 0;
-            if (fixup_format) {
-                allowed_caps = gst_vaapi_image_format_get_caps(fixup_format);
-                if (allowed_caps) {
-                    new_caps = gst_caps_union(out_caps, allowed_caps);
-                    gst_caps_unref(allowed_caps);
-                    gst_caps_unref(out_caps);
-                    out_caps = new_caps;
-                }
-            }
         }
     }
 
@@ -348,34 +321,13 @@ gst_vaapiconvert_set_caps(
 )
 {
     GstVaapiConvert * const convert = GST_VAAPICONVERT(trans);
-    GstCaps *fixed_incaps = NULL;
     GstStructure *structure;
     gint width, height;
-    guint32 format;
 
     structure = gst_caps_get_structure(incaps, 0);
     gst_structure_get_int(structure, "width",  &width);
     gst_structure_get_int(structure, "height", &height);
 
-#define GST_FORMAT_YV12 GST_MAKE_FOURCC ('Y', 'V', '1', '2')
-#define GST_FORMAT_I420 GST_MAKE_FOURCC ('I', '4', '2', '0')
-
-    /* Fix I420 and YV12 formats */
-    if (gst_structure_get_fourcc(structure, "format", &format)) {
-        if (format == GST_FORMAT_I420 && !convert->has_I420)
-            format = GST_FORMAT_YV12;
-        else if (format == GST_FORMAT_YV12 && !convert->has_YV12)
-            format = GST_FORMAT_I420;
-        else
-            format = 0;
-        if (format) {
-            fixed_incaps = gst_caps_copy(incaps);
-            structure = gst_caps_get_structure(fixed_incaps, 0);
-            gst_structure_set(structure, "format", GST_TYPE_FOURCC, format, NULL);
-            incaps = fixed_incaps;
-        }
-    }
-
     if (width != convert->image_width || height != convert->image_height) {
         if (convert->images)
             g_object_unref(convert->images);
@@ -395,9 +347,6 @@ gst_vaapiconvert_set_caps(
         if (!convert->surfaces)
             return FALSE;
     }
-
-    if (fixed_incaps)
-        gst_caps_unref(fixed_incaps);
     return TRUE;
 }
 

sys/vaapiconvert/gstvaapiconvert.h

@@ -65,17 +65,12 @@ struct _GstVaapiConvert {
     GstBaseTransform    parent_instance;
 
     GstVaapiDisplay    *display;
-    GstVaapiImageFormat image_format;
     GstVaapiVideoPool  *images;
     guint               image_width;
     guint               image_height;
     GstVaapiVideoPool  *surfaces;
     guint               surface_width;
     guint               surface_height;
-
-    /* XXX: implement YV12 or I420 formats ourselves */
-    guint               has_YV12 : 1;
-    guint               has_I420 : 1;
 };
 
 struct _GstVaapiConvertClass {