qmlglsink: Keep old buffers around a bit longer if they were bound by QML

We don't know exactly when QML will stop using them but it should be
safe to unref them after at least 2 more buffers were bound.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/810>

ext/qt/gstqsgtexture.cc

@@ -47,6 +47,7 @@ GstQSGTexture::GstQSGTexture ()
 
   gst_video_info_init (&this->v_info);
   this->buffer_ = NULL;
+  this->buffer_was_bound = FALSE;
   this->qt_context_ = NULL;
   this->sync_buffer_ = gst_buffer_new ();
   this->dummy_tex_id_ = 0;
@@ -56,6 +57,7 @@ GstQSGTexture::~GstQSGTexture ()
 {
   gst_buffer_replace (&this->buffer_, NULL);
   gst_buffer_replace (&this->sync_buffer_, NULL);
+  this->buffer_was_bound = FALSE;
   if (this->dummy_tex_id_ && QOpenGLContext::currentContext ()) {
     QOpenGLContext::currentContext ()->functions ()->glDeleteTextures (1,
         &this->dummy_tex_id_);
@@ -80,11 +82,26 @@ GstQSGTexture::setBuffer (GstBuffer * buffer)
   if (!gst_buffer_replace (&this->buffer_, buffer))
     return FALSE;
 
+  this->buffer_was_bound = FALSE;
   this->qt_context_ = gst_gl_context_get_current ();
 
   return TRUE;
 }
 
+/* only called from the streaming thread with scene graph thread blocked */
+GstBuffer *
+GstQSGTexture::getBuffer (gboolean * was_bound)
+{
+  GstBuffer *buffer = NULL;
+
+  if (this->buffer_)
+    buffer = gst_buffer_ref (this->buffer_);
+  if (was_bound)
+    *was_bound = this->buffer_was_bound;
+
+  return buffer;
+}
+
 /* only called from qt's scene graph render thread */
 void
 GstQSGTexture::bind ()
@@ -142,6 +159,8 @@ GstQSGTexture::bind ()
    * to use the dummy texture */
   use_dummy_tex = FALSE;
 
+  this->buffer_was_bound = TRUE;
+
 out:
   if (G_UNLIKELY (use_dummy_tex)) {
     QOpenGLContext *qglcontext = QOpenGLContext::currentContext ();

ext/qt/gstqsgtexture.h

@@ -38,6 +38,7 @@ public:
 
     void setCaps (GstCaps * caps);
     gboolean setBuffer (GstBuffer * buffer);
+    GstBuffer * getBuffer (gboolean * was_bound);
 
     /* QSGTexture */
     void bind ();
@@ -48,6 +49,7 @@ public:
 
 private:
     GstBuffer * buffer_;
+    gboolean buffer_was_bound;
     GstBuffer * sync_buffer_;
     GstGLContext * qt_context_;
     GstMemory * mem_;

ext/qt/qtitem.cc

@@ -79,6 +79,14 @@ struct _QtGLVideoItemPrivate
   QOpenGLContext *qt_context;
   GstGLContext *other_context;
   GstGLContext *context;
+
+  /* buffers with textures that were bound by QML */
+  GQueue bound_buffers;
+  /* buffers that were previously bound but in the meantime a new one was
+   * bound so this one is most likely not used anymore
+   * FIXME: Ideally we would use fences for this but there seems to be no
+   * way to reliably "try wait" on a fence */
+  GQueue potentially_unbound_buffers;
 };
 
 class InitializeSceneGraph : public QRunnable
@@ -194,6 +202,9 @@ QSGNode *
 QtGLVideoItem::updatePaintNode(QSGNode * oldNode,
     UpdatePaintNodeData * updatePaintNodeData)
 {
+  GstBuffer *old_buffer;
+  gboolean was_bound = FALSE;
+
   if (!m_openGlContextInitialized) {
     return oldNode;
   }
@@ -221,6 +232,38 @@ QtGLVideoItem::updatePaintNode(QSGNode * oldNode,
   }
 
   tex = static_cast<GstQSGTexture *> (texNode->texture());
+
+  if ((old_buffer = tex->getBuffer(&was_bound))) {
+    if (old_buffer == this->priv->buffer) {
+      /* same buffer */
+      gst_buffer_unref (old_buffer);
+    } else if (!was_bound) {
+      GST_TRACE ("old buffer %p was not bound yet, unreffing", old_buffer);
+      gst_buffer_unref (old_buffer);
+    } else {
+      GstBuffer *tmp_buffer;
+
+      GST_TRACE ("old buffer %p was bound, queueing up for later", old_buffer);
+      /* Unref all buffers that were previously not bound anymore. At least
+       * one more buffer was bound in the meantime so this one is most likely
+       * not in use anymore. */
+      while ((tmp_buffer = (GstBuffer*) g_queue_pop_head (&this->priv->potentially_unbound_buffers))) {
+        GST_TRACE ("old buffer %p should be unbound now, unreffing", tmp_buffer);
+        gst_buffer_unref (tmp_buffer);
+      }
+
+      /* Move previous bound buffers to the next queue. We now know that
+       * another buffer was bound in the meantime and will free them on
+       * the next iteration above. */
+      while ((tmp_buffer = (GstBuffer*) g_queue_pop_head (&this->priv->bound_buffers))) {
+        GST_TRACE ("old buffer %p is potentially unbound now", tmp_buffer);
+        g_queue_push_tail (&this->priv->potentially_unbound_buffers, tmp_buffer);
+      }
+      g_queue_push_tail (&this->priv->bound_buffers, old_buffer);
+    }
+    old_buffer = NULL;
+  }
+
   tex->setCaps (this->priv->caps);
   tex->setBuffer (this->priv->buffer);
   texNode->markDirty(QSGNode::DirtyMaterial);
@@ -252,12 +295,23 @@ QtGLVideoItem::updatePaintNode(QSGNode * oldNode,
 static void
 _reset (QtGLVideoItem * qt_item)
 {
+  GstBuffer *tmp_buffer;
+
   gst_buffer_replace (&qt_item->priv->buffer, NULL);
 
   gst_caps_replace (&qt_item->priv->caps, NULL);
 
   qt_item->priv->negotiated = FALSE;
   qt_item->priv->initted = FALSE;
+
+  while ((tmp_buffer = (GstBuffer*) g_queue_pop_head (&qt_item->priv->potentially_unbound_buffers))) {
+    GST_TRACE ("old buffer %p should be unbound now, unreffing", tmp_buffer);
+    gst_buffer_unref (tmp_buffer);
+  }
+  while ((tmp_buffer = (GstBuffer*) g_queue_pop_head (&qt_item->priv->bound_buffers))) {
+    GST_TRACE ("old buffer %p should be unbound now, unreffing", tmp_buffer);
+    gst_buffer_unref (tmp_buffer);
+  }
 }
 
 void