glupload: enable drm kind caps in glupload plugin for DMA buf

Most of the time, the RGB kind formats are OpenGL native supported
format which has only one plane. They can be imported at one shot
using no matter DIRECT or INDIRECT mode.
While YUV kind formats which have multi planes have two ways to import.
They can be DIRECT imported, which requires GL_OES_EGL_image_external
extension. The output format should be RGBA and TARGET should be set
as OES after imported. The other way, they can be INDIRECT imported,
which makes each plane as a texture. In this mode, the imported textures
have different fourcc from the original format. For example, the NV12
format can be imported as a R8 texture for the first plane and RG88
texture for the second plane. The output TARGET should be sets as 2D
in this mode.

When converting sink caps to src caps, we first filter the feature of
"video/x-raw(memory:DMABuf)" and system memory. Then Based on the
external_only flag (INDIRECT mode does not care while DIRECT mode cares),
we transform the drm-format into the gst video format.

When converting src caps into sink caps, we first filter the correct
TARGET(INDIRECT mode contains 2D only while DIRECT mode contains 2D,
OES or both of them) gstructure. Then Based on the include_external flag
(INDIRECT mode always true while DIRECT mode depends on TARGET), we
transform the gst video format into drm-format.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3524>
This commit is contained in:
He Junyan 2023-05-19 22:44:52 +08:00 committed by GStreamer Marge Bot
parent 96ea1b8da9
commit 21faad5eca

View file

@ -681,9 +681,7 @@ gst_egl_image_cache_new (void)
}
static GstStaticCaps _dma_buf_upload_caps =
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_DMABUF,
GST_GL_MEMORY_VIDEO_FORMATS_STR) ";"
GST_STATIC_CAPS (GST_VIDEO_DMA_DRM_CAPS_MAKE ";"
GST_VIDEO_CAPS_MAKE (GST_GL_MEMORY_VIDEO_FORMATS_STR));
static gpointer
@ -829,6 +827,16 @@ _check_modifier (GstGLContext * context, guint32 fourcc,
return FALSE;
}
static void
_set_default_formats_list (GstStructure * structure)
{
GValue formats = G_VALUE_INIT;
g_value_init (&formats, GST_TYPE_LIST);
gst_value_deserialize (&formats, GST_GL_MEMORY_VIDEO_FORMATS_STR);
gst_structure_take_value (structure, "format", &formats);
}
static GstVideoFormat
_get_video_format_from_drm_format (GstGLContext * context,
const gchar * drm_format, gboolean include_external)
@ -912,13 +920,289 @@ _dma_buf_transform_drm_formats_to_gst_formats (GstGLContext * context,
return TRUE;
}
static gboolean
_dma_buf_convert_format_field_in_structure (GstGLContext * context,
GstStructure * structure, GstPadDirection direction,
gboolean include_external)
{
const GValue *val;
if (direction == GST_PAD_SRC) {
GValue drm_formats = G_VALUE_INIT;
/* No context available, we can not know the real modifiers.
Just leaving all format related fields blank. */
if (!context) {
gst_structure_set (structure, "format", G_TYPE_STRING, "DMA_DRM", NULL);
gst_structure_remove_field (structure, "drm-format");
return TRUE;
}
/* When no format provided, just list all supported formats
and find all the possible drm-format. */
if (!(val = gst_structure_get_value (structure, "format"))) {
_set_default_formats_list (structure);
val = gst_structure_get_value (structure, "format");
}
if (_dma_buf_transform_gst_formats_to_drm_formats (context,
val, include_external, &drm_formats)) {
gst_structure_take_value (structure, "drm-format", &drm_formats);
} else {
return FALSE;
}
gst_structure_set (structure, "format", G_TYPE_STRING, "DMA_DRM", NULL);
} else {
GValue gst_formats = G_VALUE_INIT;
/* Reject the traditional "format" field directly. */
if (g_strcmp0 (gst_structure_get_string (structure, "format"),
"DMA_DRM") != 0)
return FALSE;
/* If no drm-field in the src, we just list all
supported formats in dst. */
if (!(val = gst_structure_get_value (structure, "drm-format"))) {
gst_structure_remove_field (structure, "format");
gst_structure_remove_field (structure, "drm-format");
_set_default_formats_list (structure);
return TRUE;
}
if (_dma_buf_transform_drm_formats_to_gst_formats (context,
val, include_external, &gst_formats)) {
gst_structure_take_value (structure, "format", &gst_formats);
} else {
return FALSE;
}
gst_structure_remove_field (structure, "drm-format");
}
return TRUE;
}
static gboolean
_dma_buf_check_target (GstStructure * structure, GstGLTextureTarget target_mask)
{
const GValue *target_val;
const gchar *target_str;
GstGLTextureTarget target;
guint i;
target_val = gst_structure_get_value (structure, "texture-target");
/* If no texture-target set, it means a default of 2D. */
if (!target_val)
return (1 << GST_GL_TEXTURE_TARGET_2D) & target_mask;
if (G_VALUE_HOLDS_STRING (target_val)) {
target_str = g_value_get_string (target_val);
target = gst_gl_texture_target_from_string (target_str);
return (1 << target) & target_mask;
} else if (GST_VALUE_HOLDS_LIST (target_val)) {
guint num_values = gst_value_list_get_size (target_val);
for (i = 0; i < num_values; i++) {
const GValue *val = gst_value_list_get_value (target_val, i);
target_str = g_value_get_string (val);
target = gst_gl_texture_target_from_string (target_str);
if ((1 << target) & target_mask)
return TRUE;
}
}
return FALSE;
}
static gboolean
_dma_buf_check_formats_in_structure (GstGLContext * context,
GstStructure * structure, gboolean include_external)
{
const GValue *all_formats;
GstVideoFormat gst_format;
guint32 fourcc;
all_formats = gst_structure_get_value (structure, "format");
if (!all_formats)
return FALSE;
if (G_VALUE_HOLDS_STRING (all_formats)) {
gst_format =
gst_video_format_from_string (g_value_get_string (all_formats));
if (gst_format == GST_VIDEO_FORMAT_UNKNOWN)
return FALSE;
fourcc = gst_video_dma_drm_fourcc_from_format (gst_format);
if (fourcc == DRM_FORMAT_INVALID)
return FALSE;
if (!_check_modifier (context, fourcc,
DRM_FORMAT_MOD_LINEAR, include_external))
return FALSE;
return TRUE;
} else if (GST_VALUE_HOLDS_LIST (all_formats)) {
GValue video_value = G_VALUE_INIT;
guint num_values = gst_value_list_get_size (all_formats);
GArray *gst_formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
guint i;
for (i = 0; i < num_values; i++) {
const GValue *val = gst_value_list_get_value (all_formats, i);
gst_format = gst_video_format_from_string (g_value_get_string (val));
if (gst_format == GST_VIDEO_FORMAT_UNKNOWN)
continue;
fourcc = gst_video_dma_drm_fourcc_from_format (gst_format);
if (fourcc == DRM_FORMAT_INVALID)
continue;
if (!_check_modifier (context, fourcc,
DRM_FORMAT_MOD_LINEAR, include_external))
continue;
g_array_append_val (gst_formats, gst_format);
}
if (gst_formats->len == 0) {
g_array_unref (gst_formats);
return FALSE;
}
if (gst_formats->len == 1) {
g_value_init (&video_value, G_TYPE_STRING);
gst_format = g_array_index (gst_formats, GstVideoFormat, 0);
g_value_set_string (&video_value,
gst_video_format_to_string (gst_format));
} else {
GValue item = G_VALUE_INIT;
gst_value_list_init (&video_value, gst_formats->len);
for (i = 0; i < gst_formats->len; i++) {
g_value_init (&item, G_TYPE_STRING);
gst_format = g_array_index (gst_formats, GstVideoFormat, i);
g_value_set_string (&item, gst_video_format_to_string (gst_format));
gst_value_list_append_value (&video_value, &item);
g_value_unset (&item);
}
}
g_array_unref (gst_formats);
gst_structure_take_value (structure, "format", &video_value);
return TRUE;
}
return FALSE;
}
static GstCaps *
_dma_buf_upload_transform_caps_common (GstCaps * caps,
GstGLContext * context, GstPadDirection direction,
gboolean include_external, GstGLTextureTarget target_mask,
const gchar * from_feature, const gchar * to_feature)
{
guint i, n;
GstCaps *ret_caps, *tmp_caps, *caps_to_transform;
GstCapsFeatures *passthrough, *features;
if (direction == GST_PAD_SINK) {
g_return_val_if_fail
(!g_strcmp0 (from_feature, GST_CAPS_FEATURE_MEMORY_DMABUF) ||
!g_strcmp0 (from_feature, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY), NULL);
g_return_val_if_fail
(!g_strcmp0 (to_feature, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), NULL);
} else {
g_return_val_if_fail
(!g_strcmp0 (to_feature, GST_CAPS_FEATURE_MEMORY_DMABUF) ||
!g_strcmp0 (to_feature, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY), NULL);
g_return_val_if_fail
(!g_strcmp0 (from_feature, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), NULL);
}
features = gst_caps_features_new (from_feature, NULL);
if (!_filter_caps_with_features (caps, features, &caps_to_transform)) {
gst_caps_features_free (features);
return NULL;
}
gst_caps_features_free (features);
if (gst_caps_is_any (caps_to_transform)) {
tmp_caps = caps_to_transform;
goto passthrough;
}
tmp_caps = gst_caps_new_empty ();
n = gst_caps_get_size (caps_to_transform);
for (i = 0; i < n; i++) {
GstStructure *s;
GstCapsFeatures *features;
features = gst_caps_get_features (caps_to_transform, i);
g_assert (gst_caps_features_contains (features, from_feature));
s = gst_caps_get_structure (caps_to_transform, i);
if (direction == GST_PAD_SRC && !_dma_buf_check_target (s, target_mask))
continue;
s = gst_structure_copy (s);
if (!g_strcmp0 (from_feature, GST_CAPS_FEATURE_MEMORY_DMABUF) ||
!g_strcmp0 (to_feature, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
/* Convert drm-format/format fields for DMABuf */
if (!_dma_buf_convert_format_field_in_structure (context, s,
direction, include_external)) {
gst_structure_free (s);
continue;
}
} else {
if (!_dma_buf_check_formats_in_structure (context, s, include_external)) {
gst_structure_free (s);
continue;
}
}
gst_caps_append_structure_full (tmp_caps, s,
gst_caps_features_copy (features));
}
gst_caps_unref (caps_to_transform);
if (gst_caps_is_empty (tmp_caps)) {
gst_caps_unref (tmp_caps);
return NULL;
}
passthrough:
/* Change the feature name. */
passthrough = gst_caps_features_from_string
(GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
ret_caps = _set_caps_features_with_passthrough (tmp_caps,
to_feature, passthrough);
gst_caps_features_free (passthrough);
gst_caps_unref (tmp_caps);
return ret_caps;
}
static GstCaps *
_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
GstPadDirection direction, GstCaps * caps)
{
struct DmabufUpload *dmabuf = impl;
GstCapsFeatures *passthrough;
GstCaps *ret;
GstCaps *ret, *tmp;
if (context) {
const GstGLFuncs *gl = context->gl_vtable;
@ -932,17 +1216,33 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
if (!gst_gl_context_check_feature (context, "EGL_KHR_image_base"))
return NULL;
if (!gst_gl_context_egl_supports_modifier (context))
return NULL;
}
passthrough = gst_caps_features_from_string
(GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
g_assert (dmabuf->target == GST_GL_TEXTURE_TARGET_2D);
if (direction == GST_PAD_SINK) {
GstCaps *tmp;
ret = _dma_buf_upload_transform_caps_common (caps, context, direction,
TRUE, 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_DMABUF,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
tmp = _dma_buf_upload_transform_caps_common (caps, context, direction,
TRUE, 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
if (!ret) {
ret = tmp;
tmp = NULL;
}
if (tmp)
ret = gst_caps_merge (ret, tmp);
ret =
_set_caps_features_with_passthrough (caps,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
if (!ret) {
GST_DEBUG_OBJECT (dmabuf->upload,
"direction %s, fails to transformed DMA caps %" GST_PTR_FORMAT,
"sink", caps);
return NULL;
}
tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
gst_caps_unref (ret);
@ -950,9 +1250,25 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
} else {
gint i, n;
ret =
_set_caps_features_with_passthrough (caps,
GST_CAPS_FEATURE_MEMORY_DMABUF, passthrough);
ret = _dma_buf_upload_transform_caps_common (caps, context, direction,
TRUE, 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
GST_CAPS_FEATURE_MEMORY_DMABUF);
tmp = _dma_buf_upload_transform_caps_common (caps, context, direction,
TRUE, 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
if (!ret) {
ret = tmp;
tmp = NULL;
}
if (tmp)
ret = gst_caps_merge (ret, tmp);
if (!ret) {
GST_DEBUG_OBJECT (dmabuf->upload,
"direction %s, fails to transformed DMA caps %" GST_PTR_FORMAT,
"src", caps);
return NULL;
}
n = gst_caps_get_size (ret);
for (i = 0; i < n; i++) {
@ -962,10 +1278,9 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
}
}
gst_caps_features_free (passthrough);
GST_DEBUG_OBJECT (dmabuf->upload, "transformed %" GST_PTR_FORMAT " into %"
GST_PTR_FORMAT, caps, ret);
GST_DEBUG_OBJECT (dmabuf->upload, "direction %s, transformed %"
GST_PTR_FORMAT " into %" GST_PTR_FORMAT,
direction == GST_PAD_SRC ? "src" : "sink", caps, ret);
return ret;
}
@ -1237,8 +1552,7 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
GstPadDirection direction, GstCaps * caps)
{
struct DmabufUpload *dmabuf = impl;
GstCapsFeatures *passthrough;
GstCaps *ret;
GstCaps *ret, *tmp;
if (context) {
const GstGLFuncs *gl = context->gl_vtable;
@ -1253,20 +1567,42 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
if (dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES &&
!gst_gl_context_check_feature (context, "GL_OES_EGL_image_external"))
return NULL;
}
passthrough = gst_caps_features_from_string
(GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
if (!gst_gl_context_egl_supports_modifier (context))
return NULL;
}
if (direction == GST_PAD_SINK) {
gint i, n;
GstCaps *tmp;
GstGLTextureTarget target_mask;
ret =
_set_caps_features_with_passthrough (caps,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
ret = _dma_buf_upload_transform_caps_common (caps, context, direction,
dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES,
1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_DMABUF,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
tmp = _dma_buf_upload_transform_caps_common (caps, context, direction,
dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES,
1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
if (!ret) {
ret = tmp;
tmp = NULL;
}
if (tmp)
ret = gst_caps_merge (ret, tmp);
if (!ret) {
GST_DEBUG_OBJECT (dmabuf->upload,
"direction %s, fails to transformed DMA caps %" GST_PTR_FORMAT,
"sink", caps);
return NULL;
}
/* The direct mode, sampling an imported texture will return an RGBA
vector in the same colorspace as the source image. If the source
image is stored in YUV(or some other basis) then the YUV values will
be transformed to RGB values. So, any input format is transformed to:
"video/x-raw(memory:GLMemory), format=(string)RGBA" as output. */
gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
n = gst_caps_get_size (ret);
@ -1283,21 +1619,39 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
ret = tmp;
} else {
gint i, n;
GstCaps *tmp;
GValue formats = G_VALUE_INIT;
gchar *format_str = g_strdup (GST_GL_MEMORY_VIDEO_FORMATS_STR);
GstCaps *tmp_caps;
ret =
_set_caps_features_with_passthrough (caps,
GST_CAPS_FEATURE_MEMORY_DMABUF, passthrough);
/* The src caps may only contain RGBA format, and we should list
all possible supported formats to detect the conversion for
DMABuf kind memory. */
tmp_caps = gst_caps_copy (caps);
for (i = 0; i < gst_caps_get_size (tmp_caps); i++)
_set_default_formats_list (gst_caps_get_structure (tmp_caps, i));
g_value_init (&formats, GST_TYPE_LIST);
gst_value_deserialize (&formats, format_str);
tmp = gst_caps_copy (ret);
gst_caps_set_value (tmp, "format", &formats);
gst_caps_append (ret, tmp);
g_free (format_str);
g_value_unset (&formats);
ret = _dma_buf_upload_transform_caps_common (tmp_caps, context, direction,
dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES,
1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
GST_CAPS_FEATURE_MEMORY_DMABUF);
gst_caps_unref (tmp_caps);
tmp = _dma_buf_upload_transform_caps_common (caps, context, direction,
dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES,
1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
if (!ret) {
ret = tmp;
tmp = NULL;
}
if (tmp)
ret = gst_caps_merge (ret, tmp);
if (!ret) {
GST_DEBUG_OBJECT (dmabuf->upload,
"direction %s, fails to transformed DMA caps %" GST_PTR_FORMAT,
"src", caps);
return NULL;
}
n = gst_caps_get_size (ret);
for (i = 0; i < n; i++) {
@ -1307,10 +1661,9 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
}
}
gst_caps_features_free (passthrough);
GST_DEBUG_OBJECT (dmabuf->upload, "transformed %" GST_PTR_FORMAT " into %"
GST_PTR_FORMAT, caps, ret);
GST_DEBUG_OBJECT (dmabuf->upload, "direction %s, transformed %"
GST_PTR_FORMAT " into %" GST_PTR_FORMAT,
direction == GST_PAD_SRC ? "src" : "sink", caps, ret);
return ret;
}