vapostproc, vadeinterlace: don't transform caps if no intersection.

If caps to transform don't intersect with those supported by the VA
filter (VAEntrypointVideoProc) then return them as is, because only
pass-through mode is the only possibility.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1369>

subprojects/gst-plugins-bad/sys/va/gstvabasetransform.c

@@ -41,6 +41,8 @@ struct _GstVaBaseTransformPrivate
   GstCaps *sinkpad_caps;
   GstVideoInfo sinkpad_info;
   GstBufferPool *sinkpad_pool;
+
+  GstCaps *filter_caps;
 };
 
 /**
@@ -72,6 +74,8 @@ gst_va_base_transform_dispose (GObject * object)
   gst_clear_caps (&self->out_caps);
   gst_clear_caps (&self->in_caps);
 
+  gst_clear_caps (&self->priv->filter_caps);
+
   gst_clear_object (&self->filter);
   gst_clear_object (&self->display);
 
@@ -501,6 +505,7 @@ gst_va_base_transform_change_state (GstElement * element,
       if (!gst_va_ensure_element_data (element, klass->render_device_path,
               &self->display))
         goto open_failed;
+      gst_clear_caps (&self->priv->filter_caps);
       gst_clear_object (&self->filter);
       self->filter = gst_va_filter_new (self->display);
       if (!gst_va_filter_open (self->filter))
@@ -519,6 +524,7 @@ gst_va_base_transform_change_state (GstElement * element,
       gst_va_filter_close (self->filter);
       break;
     case GST_STATE_CHANGE_READY_TO_NULL:
+      gst_clear_caps (&self->priv->filter_caps);
       gst_clear_object (&self->filter);
       gst_clear_object (&self->display);
       break;
@@ -765,6 +771,8 @@ gst_va_base_transform_import_buffer (GstVaBaseTransform * self,
   GstVideoFrame in_frame, out_frame;
   gboolean imported, copied;
 
+  g_return_val_if_fail (GST_IS_VA_BASE_TRANSFORM (self), GST_FLOW_ERROR);
+
   imported = _try_import_buffer (self, inbuf);
   if (imported) {
     *buf = gst_buffer_ref (inbuf);
@@ -818,3 +826,24 @@ invalid_buffer:
     return GST_FLOW_OK;
   }
 }
+
+GstCaps *
+gst_va_base_transform_get_filter_caps (GstVaBaseTransform * self)
+{
+  g_return_val_if_fail (GST_IS_VA_BASE_TRANSFORM (self), NULL);
+
+  GST_OBJECT_LOCK (self);
+  if (self->priv->filter_caps) {
+    GST_OBJECT_UNLOCK (self);
+    return self->priv->filter_caps;
+  }
+  GST_OBJECT_UNLOCK (self);
+
+  if (!self->filter)
+    return NULL;
+
+  GST_OBJECT_LOCK (self);
+  self->priv->filter_caps = gst_va_filter_get_caps (self->filter);
+  GST_OBJECT_UNLOCK (self);
+  return self->priv->filter_caps;
+}

subprojects/gst-plugins-bad/sys/va/gstvabasetransform.h

@@ -87,6 +87,9 @@ GstFlowReturn         gst_va_base_transform_import_buffer (GstVaBaseTransform *
                                                            GstBuffer * inbuf,
                                                            GstBuffer ** buf);
 
+GstCaps *             gst_va_base_transform_get_filter_caps
+                                                          (GstVaBaseTransform * self);
+
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstVaBaseTransform, gst_object_unref)
 
 G_END_DECLS

subprojects/gst-plugins-bad/sys/va/gstvadeinterlace.c

@@ -541,14 +541,22 @@ gst_va_deinterlace_transform_caps (GstBaseTransform * trans,
     GstPadDirection direction, GstCaps * caps, GstCaps * filter)
 {
   GstVaDeinterlace *self = GST_VA_DEINTERLACE (trans);
-  GstCaps *ret;
+  GstVaBaseTransform *btrans = GST_VA_BASE_TRANSFORM (trans);
+  GstCaps *ret, *filter_caps;
 
   GST_DEBUG_OBJECT (self,
       "Transforming caps %" GST_PTR_FORMAT " in direction %s", caps,
       (direction == GST_PAD_SINK) ? "sink" : "src");
 
+  filter_caps = gst_va_base_transform_get_filter_caps (btrans);
+  if (filter_caps && !gst_caps_can_intersect (caps, filter_caps)) {
+    ret = gst_caps_ref (caps);
+    goto bail;
+  }
+
   ret = gst_va_deinterlace_remove_interlace (caps);
 
+bail:
   if (filter) {
     GstCaps *intersection;
 

subprojects/gst-plugins-bad/sys/va/gstvavpp.c

@@ -884,12 +884,19 @@ gst_va_vpp_transform_caps (GstBaseTransform * trans, GstPadDirection direction,
     GstCaps * caps, GstCaps * filter)
 {
   GstVaVpp *self = GST_VA_VPP (trans);
-  GstCaps *ret, *tmp;
+  GstVaBaseTransform *btrans = GST_VA_BASE_TRANSFORM (trans);
+  GstCaps *ret, *tmp, *filter_caps;
 
   GST_DEBUG_OBJECT (self,
       "Transforming caps %" GST_PTR_FORMAT " in direction %s", caps,
       (direction == GST_PAD_SINK) ? "sink" : "src");
 
+  filter_caps = gst_va_base_transform_get_filter_caps (btrans);
+  if (filter_caps && !gst_caps_can_intersect (caps, filter_caps)) {
+    ret = gst_caps_ref (caps);
+    goto bail;
+  }
+
   ret = gst_va_vpp_caps_remove_fields (caps);
 
   tmp = gst_va_vpp_complete_caps_features (ret, GST_CAPS_FEATURE_MEMORY_VA);
@@ -905,6 +912,7 @@ gst_va_vpp_transform_caps (GstBaseTransform * trans, GstPadDirection direction,
   if (!gst_caps_is_subset (tmp, ret))
     gst_caps_append (ret, tmp);
 
+bail:
   if (filter) {
     GstCaps *intersection;