vaapiencode: DMABuf only if PRIME is available

Add DMABuf capsfeature in encoders' allowed sinkcaps only if PRIME
memory type is available in the VA surface attributes of codec
context.

gst-libs/gst/vaapi/gstvaapiencoder.c

@@ -1533,6 +1533,7 @@ merge_profile_surface_attributes (GstVaapiEncoder * encoder,
   attribs->min_height = MIN (attribs->min_height, attr.min_height);
   attribs->max_width = MAX (attribs->max_width, attr.max_width);
   attribs->max_height = MAX (attribs->max_height, attr.max_height);
+  attribs->mem_types &= attr.mem_types;
 
   return TRUE;
 }
@@ -1553,10 +1554,10 @@ merge_profile_surface_attributes (GstVaapiEncoder * encoder,
 GArray *
 gst_vaapi_encoder_get_surface_attributes (GstVaapiEncoder * encoder,
     GArray * profiles, gint * min_width, gint * min_height,
-    gint * max_width, gint * max_height)
+    gint * max_width, gint * max_height, guint * mem_types)
 {
   GstVaapiConfigSurfaceAttributes attribs = {
-    G_MAXINT, G_MAXINT, 1, 1, 0, NULL
+    G_MAXINT, G_MAXINT, 1, 1, G_MAXUINT, NULL
   };
   GstVaapiProfile profile;
   guint i;
@@ -1586,6 +1587,8 @@ gst_vaapi_encoder_get_surface_attributes (GstVaapiEncoder * encoder,
     *max_width = attribs.max_width;
   if (max_height)
     *max_height = attribs.max_height;
+  if (mem_types)
+    *mem_types = attribs.mem_types;
   return attribs.formats;
 }
 

gst-libs/gst/vaapi/gstvaapiencoder.h

@@ -183,7 +183,7 @@ gst_vaapi_encoder_flush (GstVaapiEncoder * encoder);
 GArray *
 gst_vaapi_encoder_get_surface_attributes (GstVaapiEncoder * encoder,
     GArray * profiles, gint * min_width, gint * min_height,
-    gint * max_width, gint * max_height);
+    gint * max_width, gint * max_height, guint * mem_types);
 
 GstVaapiProfile
 gst_vaapi_encoder_get_profile (GstVaapiEncoder * encoder);

gst-libs/gst/vaapi/gstvaapiprofilecaps.c

@@ -117,3 +117,19 @@ gst_vaapi_profile_caps_append_decoder (GstVaapiDisplay * display,
 
   return append_caps_with_context_info (display, &cip, structure);
 }
+
+/**
+ * gst_vaapi_mem_type_supports:
+ * @va_mem_types:  memory types from VA surface attributes
+ * @mem_type: the #GstVaapiBufferMemoryType to test
+ *
+ * Test if @va_mem_types handles @mem_type
+ *
+ * Returns: %TRUE if @mem_type is supported in @va_mem_types;
+ *    otherwise %FALSE
+ **/
+gboolean
+gst_vaapi_mem_type_supports (guint va_mem_types, guint mem_type)
+{
+  return ((va_mem_types & from_GstVaapiBufferMemoryType (mem_type)) != 0);
+}

gst-libs/gst/vaapi/gstvaapiprofilecaps.h

@@ -33,6 +33,9 @@ gboolean
 gst_vaapi_profile_caps_append_decoder (GstVaapiDisplay * display,
     GstVaapiProfile profile, GstStructure * structure);
 
+gboolean
+gst_vaapi_mem_type_supports (guint va_mem_types, guint mem_type);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_PROFILE_CAPS_H */

gst/vaapi/gstvaapiencode.c

@@ -366,6 +366,7 @@ ensure_allowed_sinkpad_caps (GstVaapiEncode * encode)
   guint i, size;
   GstStructure *structure;
   gint min_width, min_height, max_width, max_height;
+  guint mem_types;
 
   if (encode->allowed_sinkpad_caps)
     return TRUE;
@@ -380,7 +381,7 @@ ensure_allowed_sinkpad_caps (GstVaapiEncode * encode)
   /* Then get all supported formats, all these formats should be recognized
      in video-format map. */
   formats = gst_vaapi_encoder_get_surface_attributes (encode->encoder, profiles,
-      &min_width, &min_height, &max_width, &max_height);
+      &min_width, &min_height, &max_width, &max_height, &mem_types);
   if (!formats)
     goto failed_get_attributes;
 
@@ -398,19 +399,20 @@ ensure_allowed_sinkpad_caps (GstVaapiEncode * encode)
         max_width, "height", GST_TYPE_INT_RANGE, min_height, max_height, NULL);
   }
 
+  out_caps = gst_caps_copy (raw_caps);
+
   va_caps = gst_caps_copy (raw_caps);
   gst_caps_set_features_simple (va_caps,
       gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE));
+  gst_caps_append (out_caps, va_caps);
 
+  if (gst_vaapi_mem_type_supports (mem_types,
+          GST_VAAPI_BUFFER_MEMORY_TYPE_DMA_BUF)) {
     dma_caps = gst_caps_copy (raw_caps);
     gst_caps_set_features_simple (dma_caps,
         gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_DMABUF));
-
-  /* collect all caps together. */
-  out_caps = raw_caps;
-  raw_caps = NULL;
-  gst_caps_append (out_caps, va_caps);
     gst_caps_append (out_caps, dma_caps);
+  }
 
   gst_caps_replace (&encode->allowed_sinkpad_caps, out_caps);
   GST_INFO_OBJECT (encode, "Allowed sink caps %" GST_PTR_FORMAT,