waylandsink: Rework dmabuf support

Simplify and fix some of the show_frame logic.

https://bugzilla.gnome.org/show_bug.cgi?id=711155

ext/wayland/gstwaylandsink.c

@@ -472,7 +472,6 @@ static gboolean
 gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
 {
   GstWaylandSink *sink;
-  GstBufferPool *newpool;
   gboolean use_dmabuf;
   GstVideoFormat format;
 
@@ -487,35 +486,20 @@ gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
   format = GST_VIDEO_INFO_FORMAT (&sink->video_info);
   sink->video_info_changed = TRUE;
 
+  /* create a new pool for the new caps */
+  if (sink->pool)
+    gst_object_unref (sink->pool);
+  sink->pool = gst_wayland_create_pool (sink, caps);
+
   use_dmabuf = gst_caps_features_contains (gst_caps_get_features (caps, 0),
       GST_CAPS_FEATURE_MEMORY_DMABUF);
 
-  if (!use_dmabuf) {
-    /* shm - verify format support */
-    enum wl_shm_format format_shm = gst_video_format_to_wl_shm_format (format);
-    if ((gint) format_shm == -1)
-      goto invalid_format;
-
-    if (!is_shm_format_supported (format_shm, sink->display))
+  /* validate the format base on the memory type. */
+  if (use_dmabuf) {
+    if (!gst_wl_display_check_format_for_dmabuf (sink->display, format))
       goto unsupported_format;
-
-    /* create a new pool for the new configuration */
-    newpool = gst_wayland_create_pool (sink, caps);
-    if (!newpool)
-      goto pool_failed;
-
-    gst_object_replace ((GstObject **) & sink->pool, (GstObject *) newpool);
-    gst_object_unref (newpool);
-  } else {
-    /* dmabuf - verify format support */
-    gint format_dmabuf = gst_video_format_to_wl_dmabuf_format (format);
-    if (format_dmabuf == -1)
-      goto invalid_format;
-
-    if (!is_dmabuf_format_supported (format_dmabuf, sink->display))
-      goto unsupported_format;
-
-    /* No pool for dmabuf, this is under upstream element responsability */
+  } else if (!gst_wl_display_check_format_for_shm (sink->display, format)) {
+    goto unsupported_format;
   }
 
   sink->use_dmabuf = use_dmabuf;
@@ -534,11 +518,6 @@ unsupported_format:
         gst_video_format_to_string (format));
     return FALSE;
   }
-pool_failed:
-  {
-    GST_ERROR_OBJECT (sink, "Failed to create new pool");
-    return FALSE;
-  }
 }
 
 static gboolean
@@ -612,6 +591,11 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer)
   GstWaylandSink *sink = GST_WAYLAND_SINK (vsink);
   GstBuffer *to_render;
   GstWlBuffer *wlbuffer;
+  GstVideoMeta *vmeta;
+  GstVideoFormat format;
+  GstMemory *mem;
+  struct wl_buffer *wbuf = NULL;
+
   GstFlowReturn ret = GST_FLOW_OK;
 
   g_mutex_lock (&sink->render_lock);
@@ -646,45 +630,42 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer)
     GST_LOG_OBJECT (sink, "buffer %p has a wl_buffer from our display, "
         "writing directly", buffer);
     to_render = buffer;
-  } else {
-    GstVideoMeta *vmeta;
-    GstMemory *mem;
-    struct wl_buffer *wbuf = NULL;
+    goto render;
+  }
 
-    /* update video info from video meta */
-    vmeta = gst_buffer_get_video_meta (buffer);
-    if (vmeta) {
-      gint i;
+  /* update video info from video meta */
+  vmeta = gst_buffer_get_video_meta (buffer);
+  if (vmeta) {
+    gint i;
 
-      for (i = 0; i < vmeta->n_planes; i++) {
-        sink->video_info.offset[i] = vmeta->offset[i];
-        sink->video_info.stride[i] = vmeta->stride[i];
-      }
+    for (i = 0; i < vmeta->n_planes; i++) {
+      sink->video_info.offset[i] = vmeta->offset[i];
+      sink->video_info.stride[i] = vmeta->stride[i];
     }
+  }
 
-    GST_LOG_OBJECT (sink, "buffer %p does not have a wl_buffer from our "
-        "display, creating it", buffer);
+  GST_LOG_OBJECT (sink, "buffer %p does not have a wl_buffer from our "
+      "display, creating it", buffer);
 
-    mem = gst_buffer_peek_memory (buffer, 0);
+  mem = gst_buffer_peek_memory (buffer, 0);
 
+  format = GST_VIDEO_INFO_FORMAT (&sink->video_info);
+  if (gst_wl_display_check_format_for_dmabuf (sink->display, format)) {
+    guint i, nb_dmabuf = 0;
+
+    for (i = 0; i < gst_buffer_n_memory (buffer); i++)
+      if (gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, i)))
+        nb_dmabuf++;
+
+    if (nb_dmabuf && (nb_dmabuf == gst_buffer_n_memory (buffer)))
+      wbuf = gst_wl_linux_dmabuf_construct_wl_buffer (buffer, sink->display,
+          &sink->video_info);
+  }
+
+  if (!wbuf && gst_wl_display_check_format_for_shm (sink->display, format)) {
     if (gst_is_fd_memory (mem)) {
       wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
           &sink->video_info);
-    } else if (sink->display->dmabuf) {
-      guint i, nb_dmabuf = 0;
-
-      for (i = 0; i < gst_buffer_n_memory (buffer); i++)
-        if (gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, i)))
-          nb_dmabuf++;
-
-      if (nb_dmabuf && (nb_dmabuf == gst_buffer_n_memory (buffer)))
-        wbuf = gst_wl_linux_dmabuf_construct_wl_buffer (buffer, sink->display,
-            &sink->video_info);
-    }
-
-    if (wbuf) {
-      gst_buffer_add_wl_buffer (buffer, wbuf, sink->display);
-      to_render = buffer;
     } else {
       GstVideoFrame src, dst;
       GstVideoInfo src_info = sink->video_info;
@@ -717,21 +698,18 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer)
       if (ret != GST_FLOW_OK)
         goto no_buffer;
 
-      /* the first time we acquire a buffer,
-       * we need to attach a wl_buffer on it */
       wlbuffer = gst_buffer_get_wl_buffer (to_render);
+
+      /* attach a wl_buffer if there isn't one yet */
       if (G_UNLIKELY (!wlbuffer)) {
         mem = gst_buffer_peek_memory (to_render, 0);
         wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
             &sink->video_info);
+
         if (G_UNLIKELY (!wbuf))
           goto no_wl_buffer_shm;
 
         gst_buffer_add_wl_buffer (to_render, wbuf, sink->display);
-      } else {
-        /* we cannot get a wl_buffer and if we do not have any pool to create
-         * one by ourself */
-        goto no_wl_buffer;
       }
 
       if (!gst_video_frame_map (&dst, &sink->video_info, to_render,
@@ -747,9 +725,18 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer)
 
       gst_video_frame_unmap (&src);
       gst_video_frame_unmap (&dst);
+
+      goto render;
     }
   }
 
+  if (!wbuf)
+    goto no_wl_buffer;
+
+  gst_buffer_add_wl_buffer (buffer, wbuf, sink->display);
+  to_render = buffer;
+
+render:
   /* drop double rendering */
   if (G_UNLIKELY (to_render == sink->last_buffer)) {
     GST_LOG_OBJECT (sink, "Buffer already being rendered");

ext/wayland/wldisplay.c

@@ -168,14 +168,20 @@ static const struct zwp_linux_dmabuf_v1_listener dmabuf_listener = {
 };
 
 gboolean
-is_shm_format_supported (enum wl_shm_format format_shm, GstWlDisplay * display)
+gst_wl_display_check_format_for_shm (GstWlDisplay * display,
+    GstVideoFormat format)
 {
+  enum wl_shm_format shm_fmt;
   GArray *formats;
   guint i;
 
+  shm_fmt = gst_video_format_to_wl_shm_format (format);
+  if (shm_fmt < 0)
+    return FALSE;
+
   formats = display->shm_formats;
   for (i = 0; i < formats->len; i++) {
-    if (g_array_index (formats, uint32_t, i) == format_shm)
+    if (g_array_index (formats, uint32_t, i) == shm_fmt)
       return TRUE;
   }
 
@@ -183,14 +189,22 @@ is_shm_format_supported (enum wl_shm_format format_shm, GstWlDisplay * display)
 }
 
 gboolean
-is_dmabuf_format_supported (guint format_dmabuf, GstWlDisplay * display)
+gst_wl_display_check_format_for_dmabuf (GstWlDisplay * display,
+    GstVideoFormat format)
 {
   GArray *formats;
-  guint i;
+  guint i, dmabuf_fmt;
+
+  if (!display->dmabuf)
+    return FALSE;
+
+  dmabuf_fmt = gst_video_format_to_wl_dmabuf_format (format);
+  if (dmabuf_fmt < 0)
+    return FALSE;
 
   formats = display->dmabuf_formats;
   for (i = 0; i < formats->len; i++) {
-    if (g_array_index (formats, uint32_t, i) == format_dmabuf)
+    if (g_array_index (formats, uint32_t, i) == dmabuf_fmt)
       return TRUE;
   }
 

ext/wayland/wldisplay.h

@@ -22,6 +22,7 @@
 #define __GST_WL_DISPLAY_H__
 
 #include <gst/gst.h>
+#include <gst/video/video.h>
 #include <wayland-client.h>
 #include "viewporter-client-protocol.h"
 #include "linux-dmabuf-unstable-v1-client-protocol.h"
@@ -82,10 +83,10 @@ GstWlDisplay *gst_wl_display_new_existing (struct wl_display * display,
 void gst_wl_display_register_buffer (GstWlDisplay * self, gpointer buf);
 void gst_wl_display_unregister_buffer (GstWlDisplay * self, gpointer buf);
 
-gboolean is_shm_format_supported (enum wl_shm_format format_shm,
-    GstWlDisplay * display);
-gboolean is_dmabuf_format_supported (guint format_dmabuf,
-    GstWlDisplay * display);
+gboolean gst_wl_display_check_format_for_shm (GstWlDisplay * display,
+    GstVideoFormat format);
+gboolean gst_wl_display_check_format_for_dmabuf (GstWlDisplay * display,
+    GstVideoFormat format);
 
 G_END_DECLS
 

ext/wayland/wllinuxdmabuf.c

@@ -80,9 +80,11 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf,
   gint64 timeout;
   ConstructBufferData data;
 
+  g_return_val_if_fail (gst_wl_display_check_format_for_dmabuf (display,
+          GST_VIDEO_INFO_FORMAT (info)), NULL);
+
   mem = gst_buffer_peek_memory (buf, 0);
   format = gst_video_format_to_wl_dmabuf_format (GST_VIDEO_INFO_FORMAT (info));
-  g_return_val_if_fail (is_dmabuf_format_supported (format, display), NULL);
 
   g_cond_init (&data.cond);
   g_mutex_init (&data.lock);
@@ -93,6 +95,10 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf,
   nplanes = GST_VIDEO_INFO_N_PLANES (info);
   nmem = gst_buffer_n_memory (buf);
 
+  GST_DEBUG_OBJECT (display, "Creating wl_buffer from DMABuf of size %"
+      G_GSSIZE_FORMAT " (%d x %d), format %s", info->size, width, height,
+      gst_wl_dmabuf_format_to_string (format));
+
   /* Creation and configuration of planes  */
   params = zwp_linux_dmabuf_v1_create_params (display->dmabuf);
 

ext/wayland/wlshmallocator.c

@@ -146,9 +146,10 @@ gst_wl_shm_memory_construct_wl_buffer (GstMemory * mem, GstWlDisplay * display,
 
   g_return_val_if_fail (gst_is_fd_memory (mem), NULL);
   g_return_val_if_fail (size <= memsize, NULL);
-  g_return_val_if_fail (is_shm_format_supported (format, display), NULL);
+  g_return_val_if_fail (gst_wl_display_check_format_for_shm (display,
+          GST_VIDEO_INFO_FORMAT (info)), NULL);
 
-  GST_DEBUG_OBJECT (mem->allocator, "Creating wl_buffer of size %"
+  GST_DEBUG_OBJECT (display, "Creating wl_buffer from SHM of size %"
       G_GSSIZE_FORMAT " (%d x %d, stride %d), format %s", size, width, height,
       stride, gst_wl_shm_format_to_string (format));