vaapipostproc: Enable to use DMABuf mem at sink and src pad

Enable DMABuf mem at both sink and src pad for vaapipostproc caps which
helps to apply DMABuf for more generic use cases: such as "! vaapih264dec !
video/x-raw(memory:DMABuf) ! vaapipostproc ! video/x-raw (memory:DMABuf)
! vaapih265enc"  . Or other mem converions like from DMABuf to VASurface
through vaapipostproc.

This patch is a portion from Junyan.He@intel.com
https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/393

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1287>

subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapifilter.c

@@ -1845,6 +1845,10 @@ gst_vaapi_filter_process (GstVaapiFilter * filter,
 /**
  * gst_vaapi_filter_get_formats:
  * @filter: a #GstVaapiFilter
+ * @min_width: the min width can be supported.
+ * @min_height: the min height can be supported.
+ * @max_width: the max width can be supported.
+ * @max_height: the max height can be supported.
  *
  * Determines the set of supported source or target formats for video
  * processing.  The caller owns an extra reference to the resulting
@@ -1854,12 +1858,31 @@ gst_vaapi_filter_process (GstVaapiFilter * filter,
  * Return value: the set of supported target formats for video processing.
  */
 GArray *
-gst_vaapi_filter_get_formats (GstVaapiFilter * filter)
+gst_vaapi_filter_get_formats (GstVaapiFilter * filter, gint * min_width,
+    gint * min_height, gint * max_width, gint * max_height)
 {
+  GstVaapiConfigSurfaceAttributes *attribs;
+
   g_return_val_if_fail (filter != NULL, NULL);
 
   if (!ensure_attributes (filter))
     return NULL;
+
+  attribs = filter->attribs;
+
+  if (attribs->min_width >= attribs->max_width ||
+      attribs->min_height >= attribs->max_height)
+    return NULL;
+
+  if (min_width)
+    *min_width = attribs->min_width;
+  if (min_height)
+    *min_height = attribs->min_height;
+  if (max_width)
+    *max_width = attribs->max_width;
+  if (max_height)
+    *max_height = attribs->max_height;
+
   if (filter->attribs->formats)
     return g_array_ref (filter->attribs->formats);
   return NULL;

subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapifilter.h

@@ -211,7 +211,8 @@ gst_vaapi_filter_process (GstVaapiFilter * filter,
     GstVaapiSurface * src_surface, GstVaapiSurface * dst_surface, guint flags);
 
 GArray *
-gst_vaapi_filter_get_formats (GstVaapiFilter * filter);
+gst_vaapi_filter_get_formats (GstVaapiFilter * filter, gint * min_width,
+    gint * min_height, gint * max_width, gint * max_height);
 
 gboolean
 gst_vaapi_filter_set_format (GstVaapiFilter * filter, GstVideoFormat format);

subprojects/gstreamer-vaapi/gst/vaapi/gstvaapipostproc.c

@@ -62,6 +62,9 @@ static const char gst_vaapipostproc_sink_caps_str[] =
   GST_VAAPI_MAKE_SURFACE_CAPS ", "
   GST_CAPS_INTERLACED_MODES "; "
   GST_VIDEO_CAPS_MAKE (GST_VAAPI_FORMATS_ALL) ", "
+  GST_CAPS_INTERLACED_MODES "; "
+  GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF,
+      GST_VAAPI_FORMATS_ALL) ", "
   GST_CAPS_INTERLACED_MODES;
 /* *INDENT-ON* */
 
@@ -74,7 +77,8 @@ static const char gst_vaapipostproc_src_caps_str[] =
 #endif
   GST_VIDEO_CAPS_MAKE (GST_VAAPI_FORMATS_ALL) ", "
   GST_CAPS_INTERLACED_MODES "; "
-  GST_VAAPI_MAKE_DMABUF_CAPS;
+  GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF,
+      GST_VAAPI_FORMATS_ALL);
 /* *INDENT-ON* */
 
 /* *INDENT-OFF* */
@@ -287,7 +291,8 @@ gst_vaapipostproc_ensure_filter_caps (GstVaapiPostproc * postproc)
   }
 
   if (!postproc->filter_formats) {
-    postproc->filter_formats = gst_vaapi_filter_get_formats (postproc->filter);
+    postproc->filter_formats = gst_vaapi_filter_get_formats
+        (postproc->filter, NULL, NULL, NULL, NULL);
     if (!postproc->filter_formats)
       return FALSE;
   }
@@ -1292,50 +1297,46 @@ gst_vaapipostproc_update_src_caps (GstVaapiPostproc * postproc, GstCaps * caps,
 static gboolean
 ensure_allowed_sinkpad_caps (GstVaapiPostproc * postproc)
 {
-  GstCaps *out_caps, *raw_caps;
-  guint i, num_structures;
+  GstCaps *out_caps = NULL;
+  guint mem_types;
+  gint min_width, min_height, max_width, max_height;
+  GArray *mem_formats = NULL;
+  gboolean ret = TRUE;
 
-  if (postproc->allowed_sinkpad_caps)
-    return TRUE;
+  if (postproc->allowed_sinkpad_caps) {
+    ret = TRUE;
+    goto out;
+  }
 
-  if (!GST_VAAPI_PLUGIN_BASE_DISPLAY (postproc))
-    return FALSE;
+  if (!GST_VAAPI_PLUGIN_BASE_DISPLAY (postproc) ||
+      !gst_vaapipostproc_ensure_filter_caps (postproc)) {
+    ret = FALSE;
+    goto out;
+  }
 
-  /* Create VA caps */
-  out_caps = gst_caps_from_string (GST_VAAPI_MAKE_SURFACE_CAPS ", "
-      GST_CAPS_INTERLACED_MODES);
+  mem_types = gst_vaapi_filter_get_memory_types (postproc->filter);
+  mem_formats = gst_vaapi_filter_get_formats (postproc->filter, &min_width,
+      &min_height, &max_width, &max_height);
+
+  out_caps = gst_vaapi_build_caps_from_formats (mem_formats, min_width,
+      min_height, max_width, max_height, mem_types);
   if (!out_caps) {
     GST_WARNING_OBJECT (postproc, "failed to create VA sink caps");
-    return FALSE;
-  }
-
-  raw_caps = gst_vaapi_plugin_base_get_allowed_sinkpad_raw_caps
-      (GST_VAAPI_PLUGIN_BASE (postproc));
-  if (!raw_caps) {
-    gst_caps_unref (out_caps);
-    GST_WARNING_OBJECT (postproc, "failed to create YUV sink caps");
-    return FALSE;
-  }
-
-  out_caps = gst_caps_make_writable (out_caps);
-  gst_caps_append (out_caps, gst_caps_copy (raw_caps));
-
-  num_structures = gst_caps_get_size (out_caps);
-  for (i = 0; i < num_structures; i++) {
-    GstStructure *structure;
-
-    structure = gst_caps_get_structure (out_caps, i);
-    if (!structure)
-      continue;
-
-    if (postproc->filter)
-      gst_vaapi_filter_append_caps (postproc->filter, structure);
+    ret = FALSE;
+    goto out;
   }
 
   postproc->allowed_sinkpad_caps = out_caps;
+  out_caps = NULL;
+  GST_INFO_OBJECT (postproc, "postproc sink allowed caps is %" GST_PTR_FORMAT,
+      postproc->allowed_sinkpad_caps);
+out:
+  if (out_caps)
+    gst_caps_unref (out_caps);
+  if (mem_formats)
+    g_array_unref (mem_formats);
 
-  /* XXX: append VA/VPP filters */
-  return TRUE;
+  return ret;
 }
 
 /* Fixup output caps so that to reflect the supported set of pixel formats */

subprojects/gstreamer-vaapi/tests/internal/test-filter.c

@@ -188,7 +188,8 @@ dump_operations (GstVaapiFilter * filter)
 static void
 dump_formats (GstVaapiFilter * filter)
 {
-  GArray *const formats = gst_vaapi_filter_get_formats (filter);
+  GArray *const formats = gst_vaapi_filter_get_formats (filter,
+      NULL, NULL, NULL, NULL);
   guint i;
 
   if (!formats)