diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c index 7c2837e4c5..24beff17d5 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c @@ -681,9 +681,7 @@ gst_egl_image_cache_new (void) } static GstStaticCaps _dma_buf_upload_caps = - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_DMABUF, - GST_GL_MEMORY_VIDEO_FORMATS_STR) ";" + GST_STATIC_CAPS (GST_VIDEO_DMA_DRM_CAPS_MAKE ";" GST_VIDEO_CAPS_MAKE (GST_GL_MEMORY_VIDEO_FORMATS_STR)); static gpointer @@ -829,6 +827,16 @@ _check_modifier (GstGLContext * context, guint32 fourcc, return FALSE; } +static void +_set_default_formats_list (GstStructure * structure) +{ + GValue formats = G_VALUE_INIT; + + g_value_init (&formats, GST_TYPE_LIST); + gst_value_deserialize (&formats, GST_GL_MEMORY_VIDEO_FORMATS_STR); + gst_structure_take_value (structure, "format", &formats); +} + static GstVideoFormat _get_video_format_from_drm_format (GstGLContext * context, const gchar * drm_format, gboolean include_external) @@ -912,13 +920,289 @@ _dma_buf_transform_drm_formats_to_gst_formats (GstGLContext * context, return TRUE; } +static gboolean +_dma_buf_convert_format_field_in_structure (GstGLContext * context, + GstStructure * structure, GstPadDirection direction, + gboolean include_external) +{ + const GValue *val; + + if (direction == GST_PAD_SRC) { + GValue drm_formats = G_VALUE_INIT; + + /* No context available, we can not know the real modifiers. + Just leaving all format related fields blank. */ + if (!context) { + gst_structure_set (structure, "format", G_TYPE_STRING, "DMA_DRM", NULL); + gst_structure_remove_field (structure, "drm-format"); + + return TRUE; + } + + /* When no format provided, just list all supported formats + and find all the possible drm-format. */ + if (!(val = gst_structure_get_value (structure, "format"))) { + _set_default_formats_list (structure); + val = gst_structure_get_value (structure, "format"); + } + + if (_dma_buf_transform_gst_formats_to_drm_formats (context, + val, include_external, &drm_formats)) { + gst_structure_take_value (structure, "drm-format", &drm_formats); + } else { + return FALSE; + } + + gst_structure_set (structure, "format", G_TYPE_STRING, "DMA_DRM", NULL); + } else { + GValue gst_formats = G_VALUE_INIT; + + /* Reject the traditional "format" field directly. */ + if (g_strcmp0 (gst_structure_get_string (structure, "format"), + "DMA_DRM") != 0) + return FALSE; + + /* If no drm-field in the src, we just list all + supported formats in dst. */ + if (!(val = gst_structure_get_value (structure, "drm-format"))) { + gst_structure_remove_field (structure, "format"); + gst_structure_remove_field (structure, "drm-format"); + _set_default_formats_list (structure); + return TRUE; + } + + if (_dma_buf_transform_drm_formats_to_gst_formats (context, + val, include_external, &gst_formats)) { + gst_structure_take_value (structure, "format", &gst_formats); + } else { + return FALSE; + } + + gst_structure_remove_field (structure, "drm-format"); + } + + return TRUE; +} + +static gboolean +_dma_buf_check_target (GstStructure * structure, GstGLTextureTarget target_mask) +{ + const GValue *target_val; + const gchar *target_str; + GstGLTextureTarget target; + guint i; + + target_val = gst_structure_get_value (structure, "texture-target"); + + /* If no texture-target set, it means a default of 2D. */ + if (!target_val) + return (1 << GST_GL_TEXTURE_TARGET_2D) & target_mask; + + if (G_VALUE_HOLDS_STRING (target_val)) { + target_str = g_value_get_string (target_val); + target = gst_gl_texture_target_from_string (target_str); + + return (1 << target) & target_mask; + } else if (GST_VALUE_HOLDS_LIST (target_val)) { + guint num_values = gst_value_list_get_size (target_val); + + for (i = 0; i < num_values; i++) { + const GValue *val = gst_value_list_get_value (target_val, i); + + target_str = g_value_get_string (val); + target = gst_gl_texture_target_from_string (target_str); + if ((1 << target) & target_mask) + return TRUE; + } + } + + return FALSE; +} + +static gboolean +_dma_buf_check_formats_in_structure (GstGLContext * context, + GstStructure * structure, gboolean include_external) +{ + const GValue *all_formats; + GstVideoFormat gst_format; + guint32 fourcc; + + all_formats = gst_structure_get_value (structure, "format"); + if (!all_formats) + return FALSE; + + if (G_VALUE_HOLDS_STRING (all_formats)) { + gst_format = + gst_video_format_from_string (g_value_get_string (all_formats)); + if (gst_format == GST_VIDEO_FORMAT_UNKNOWN) + return FALSE; + + fourcc = gst_video_dma_drm_fourcc_from_format (gst_format); + if (fourcc == DRM_FORMAT_INVALID) + return FALSE; + + if (!_check_modifier (context, fourcc, + DRM_FORMAT_MOD_LINEAR, include_external)) + return FALSE; + + return TRUE; + } else if (GST_VALUE_HOLDS_LIST (all_formats)) { + GValue video_value = G_VALUE_INIT; + guint num_values = gst_value_list_get_size (all_formats); + GArray *gst_formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat)); + guint i; + + for (i = 0; i < num_values; i++) { + const GValue *val = gst_value_list_get_value (all_formats, i); + + gst_format = gst_video_format_from_string (g_value_get_string (val)); + if (gst_format == GST_VIDEO_FORMAT_UNKNOWN) + continue; + + fourcc = gst_video_dma_drm_fourcc_from_format (gst_format); + if (fourcc == DRM_FORMAT_INVALID) + continue; + + if (!_check_modifier (context, fourcc, + DRM_FORMAT_MOD_LINEAR, include_external)) + continue; + + g_array_append_val (gst_formats, gst_format); + } + + if (gst_formats->len == 0) { + g_array_unref (gst_formats); + return FALSE; + } + + if (gst_formats->len == 1) { + g_value_init (&video_value, G_TYPE_STRING); + gst_format = g_array_index (gst_formats, GstVideoFormat, 0); + g_value_set_string (&video_value, + gst_video_format_to_string (gst_format)); + } else { + GValue item = G_VALUE_INIT; + + gst_value_list_init (&video_value, gst_formats->len); + + for (i = 0; i < gst_formats->len; i++) { + g_value_init (&item, G_TYPE_STRING); + + gst_format = g_array_index (gst_formats, GstVideoFormat, i); + g_value_set_string (&item, gst_video_format_to_string (gst_format)); + gst_value_list_append_value (&video_value, &item); + g_value_unset (&item); + } + } + + g_array_unref (gst_formats); + + gst_structure_take_value (structure, "format", &video_value); + + return TRUE; + } + + return FALSE; +} + +static GstCaps * +_dma_buf_upload_transform_caps_common (GstCaps * caps, + GstGLContext * context, GstPadDirection direction, + gboolean include_external, GstGLTextureTarget target_mask, + const gchar * from_feature, const gchar * to_feature) +{ + guint i, n; + GstCaps *ret_caps, *tmp_caps, *caps_to_transform; + GstCapsFeatures *passthrough, *features; + + if (direction == GST_PAD_SINK) { + g_return_val_if_fail + (!g_strcmp0 (from_feature, GST_CAPS_FEATURE_MEMORY_DMABUF) || + !g_strcmp0 (from_feature, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY), NULL); + g_return_val_if_fail + (!g_strcmp0 (to_feature, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), NULL); + } else { + g_return_val_if_fail + (!g_strcmp0 (to_feature, GST_CAPS_FEATURE_MEMORY_DMABUF) || + !g_strcmp0 (to_feature, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY), NULL); + g_return_val_if_fail + (!g_strcmp0 (from_feature, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), NULL); + } + + features = gst_caps_features_new (from_feature, NULL); + if (!_filter_caps_with_features (caps, features, &caps_to_transform)) { + gst_caps_features_free (features); + return NULL; + } + gst_caps_features_free (features); + + if (gst_caps_is_any (caps_to_transform)) { + tmp_caps = caps_to_transform; + goto passthrough; + } + + tmp_caps = gst_caps_new_empty (); + n = gst_caps_get_size (caps_to_transform); + + for (i = 0; i < n; i++) { + GstStructure *s; + GstCapsFeatures *features; + + features = gst_caps_get_features (caps_to_transform, i); + g_assert (gst_caps_features_contains (features, from_feature)); + + s = gst_caps_get_structure (caps_to_transform, i); + + if (direction == GST_PAD_SRC && !_dma_buf_check_target (s, target_mask)) + continue; + + s = gst_structure_copy (s); + + if (!g_strcmp0 (from_feature, GST_CAPS_FEATURE_MEMORY_DMABUF) || + !g_strcmp0 (to_feature, GST_CAPS_FEATURE_MEMORY_DMABUF)) { + /* Convert drm-format/format fields for DMABuf */ + if (!_dma_buf_convert_format_field_in_structure (context, s, + direction, include_external)) { + gst_structure_free (s); + continue; + } + } else { + if (!_dma_buf_check_formats_in_structure (context, s, include_external)) { + gst_structure_free (s); + continue; + } + } + + gst_caps_append_structure_full (tmp_caps, s, + gst_caps_features_copy (features)); + } + + gst_caps_unref (caps_to_transform); + + if (gst_caps_is_empty (tmp_caps)) { + gst_caps_unref (tmp_caps); + return NULL; + } + +passthrough: + /* Change the feature name. */ + passthrough = gst_caps_features_from_string + (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + ret_caps = _set_caps_features_with_passthrough (tmp_caps, + to_feature, passthrough); + + gst_caps_features_free (passthrough); + gst_caps_unref (tmp_caps); + + return ret_caps; +} + static GstCaps * _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context, GstPadDirection direction, GstCaps * caps) { struct DmabufUpload *dmabuf = impl; - GstCapsFeatures *passthrough; - GstCaps *ret; + GstCaps *ret, *tmp; if (context) { const GstGLFuncs *gl = context->gl_vtable; @@ -932,17 +1216,33 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context, if (!gst_gl_context_check_feature (context, "EGL_KHR_image_base")) return NULL; + + if (!gst_gl_context_egl_supports_modifier (context)) + return NULL; } - passthrough = gst_caps_features_from_string - (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + g_assert (dmabuf->target == GST_GL_TEXTURE_TARGET_2D); if (direction == GST_PAD_SINK) { - GstCaps *tmp; + ret = _dma_buf_upload_transform_caps_common (caps, context, direction, + TRUE, 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_DMABUF, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + tmp = _dma_buf_upload_transform_caps_common (caps, context, direction, + TRUE, 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + if (!ret) { + ret = tmp; + tmp = NULL; + } + if (tmp) + ret = gst_caps_merge (ret, tmp); - ret = - _set_caps_features_with_passthrough (caps, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough); + if (!ret) { + GST_DEBUG_OBJECT (dmabuf->upload, + "direction %s, fails to transformed DMA caps %" GST_PTR_FORMAT, + "sink", caps); + return NULL; + } tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D); gst_caps_unref (ret); @@ -950,9 +1250,25 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context, } else { gint i, n; - ret = - _set_caps_features_with_passthrough (caps, - GST_CAPS_FEATURE_MEMORY_DMABUF, passthrough); + ret = _dma_buf_upload_transform_caps_common (caps, context, direction, + TRUE, 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + GST_CAPS_FEATURE_MEMORY_DMABUF); + tmp = _dma_buf_upload_transform_caps_common (caps, context, direction, + TRUE, 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); + if (!ret) { + ret = tmp; + tmp = NULL; + } + if (tmp) + ret = gst_caps_merge (ret, tmp); + + if (!ret) { + GST_DEBUG_OBJECT (dmabuf->upload, + "direction %s, fails to transformed DMA caps %" GST_PTR_FORMAT, + "src", caps); + return NULL; + } n = gst_caps_get_size (ret); for (i = 0; i < n; i++) { @@ -962,10 +1278,9 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context, } } - gst_caps_features_free (passthrough); - - GST_DEBUG_OBJECT (dmabuf->upload, "transformed %" GST_PTR_FORMAT " into %" - GST_PTR_FORMAT, caps, ret); + GST_DEBUG_OBJECT (dmabuf->upload, "direction %s, transformed %" + GST_PTR_FORMAT " into %" GST_PTR_FORMAT, + direction == GST_PAD_SRC ? "src" : "sink", caps, ret); return ret; } @@ -1237,8 +1552,7 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context, GstPadDirection direction, GstCaps * caps) { struct DmabufUpload *dmabuf = impl; - GstCapsFeatures *passthrough; - GstCaps *ret; + GstCaps *ret, *tmp; if (context) { const GstGLFuncs *gl = context->gl_vtable; @@ -1253,20 +1567,42 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context, if (dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES && !gst_gl_context_check_feature (context, "GL_OES_EGL_image_external")) return NULL; - } - passthrough = gst_caps_features_from_string - (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + if (!gst_gl_context_egl_supports_modifier (context)) + return NULL; + } if (direction == GST_PAD_SINK) { gint i, n; - GstCaps *tmp; GstGLTextureTarget target_mask; - ret = - _set_caps_features_with_passthrough (caps, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough); + ret = _dma_buf_upload_transform_caps_common (caps, context, direction, + dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES, + 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_DMABUF, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + tmp = _dma_buf_upload_transform_caps_common (caps, context, direction, + dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES, + 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + if (!ret) { + ret = tmp; + tmp = NULL; + } + if (tmp) + ret = gst_caps_merge (ret, tmp); + if (!ret) { + GST_DEBUG_OBJECT (dmabuf->upload, + "direction %s, fails to transformed DMA caps %" GST_PTR_FORMAT, + "sink", caps); + return NULL; + } + + /* The direct mode, sampling an imported texture will return an RGBA + vector in the same colorspace as the source image. If the source + image is stored in YUV(or some other basis) then the YUV values will + be transformed to RGB values. So, any input format is transformed to: + "video/x-raw(memory:GLMemory), format=(string)RGBA" as output. */ gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL); n = gst_caps_get_size (ret); @@ -1283,21 +1619,39 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context, ret = tmp; } else { gint i, n; - GstCaps *tmp; - GValue formats = G_VALUE_INIT; - gchar *format_str = g_strdup (GST_GL_MEMORY_VIDEO_FORMATS_STR); + GstCaps *tmp_caps; - ret = - _set_caps_features_with_passthrough (caps, - GST_CAPS_FEATURE_MEMORY_DMABUF, passthrough); + /* The src caps may only contain RGBA format, and we should list + all possible supported formats to detect the conversion for + DMABuf kind memory. */ + tmp_caps = gst_caps_copy (caps); + for (i = 0; i < gst_caps_get_size (tmp_caps); i++) + _set_default_formats_list (gst_caps_get_structure (tmp_caps, i)); - g_value_init (&formats, GST_TYPE_LIST); - gst_value_deserialize (&formats, format_str); - tmp = gst_caps_copy (ret); - gst_caps_set_value (tmp, "format", &formats); - gst_caps_append (ret, tmp); - g_free (format_str); - g_value_unset (&formats); + ret = _dma_buf_upload_transform_caps_common (tmp_caps, context, direction, + dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES, + 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + GST_CAPS_FEATURE_MEMORY_DMABUF); + gst_caps_unref (tmp_caps); + + tmp = _dma_buf_upload_transform_caps_common (caps, context, direction, + dmabuf->target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES, + 1 << dmabuf->target, GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); + + if (!ret) { + ret = tmp; + tmp = NULL; + } + if (tmp) + ret = gst_caps_merge (ret, tmp); + + if (!ret) { + GST_DEBUG_OBJECT (dmabuf->upload, + "direction %s, fails to transformed DMA caps %" GST_PTR_FORMAT, + "src", caps); + return NULL; + } n = gst_caps_get_size (ret); for (i = 0; i < n; i++) { @@ -1307,10 +1661,9 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context, } } - gst_caps_features_free (passthrough); - - GST_DEBUG_OBJECT (dmabuf->upload, "transformed %" GST_PTR_FORMAT " into %" - GST_PTR_FORMAT, caps, ret); + GST_DEBUG_OBJECT (dmabuf->upload, "direction %s, transformed %" + GST_PTR_FORMAT " into %" GST_PTR_FORMAT, + direction == GST_PAD_SRC ? "src" : "sink", caps, ret); return ret; }