gstreamer/subprojects/gst-plugins-bad/sys/va/gstvabasedec.c
Víctor Manuel Jáquez Leal 301e281777 vabasedec: refactor format and capsfeature selection
This is a simplification of the venerable
gst_va_base_dec_get_preferred_format_and_caps_features() function, which
predates since gstreamer-vaapi. It's used to select the format and the
capsfeature to use when setting the output state. It was complex and hard to
follow. This refactor simplifies a lot the algorithm.

The first thing to remove _downstream_has_video_meta() since, most of the time
it will be called before the caps negotiation, and allocation queries make sense
only after caps negotiation. It might work during renegotiation but, in that
case, caps feature change is uncommon. Better a simple and common approach.

Also, for performance, instead of dealing with caps features as strings, GQuarks
are used.

The refactor works like this:

1. If peer pad returns any caps, the returned caps feature is system memory and
   looks for a proper format in the allowed caps.

2. The allowed caps are traversed at most 3 times: one per each valid caps
   feature. First VAMemory, later DMABuf, and last system memory. The first to
   match in allowed caps is picked, and the first format matching with the
   chroma is picked too.

Notice that, right now, using playbin videoconvert never return any.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6154>
2024-02-22 20:59:13 +00:00

1149 lines
36 KiB
C

/* GStreamer
* Copyright (C) 2020 Igalia, S.L.
* Author: Víctor Jáquez <vjaquez@igalia.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the0
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include "gstvabasedec.h"
#include <gst/va/gstva.h>
#include <gst/va/gstvavideoformat.h>
#include <gst/va/vasurfaceimage.h>
#include "gstvacaps.h"
#include "gstvapluginutils.h"
#define GST_CAT_DEFAULT (base->debug_category)
#define GST_VA_BASE_DEC_GET_PARENT_CLASS(obj) (GST_VA_BASE_DEC_GET_CLASS(obj)->parent_decoder_class)
static void
gst_va_base_dec_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstVaBaseDec *self = GST_VA_BASE_DEC (object);
GstVaBaseDecClass *klass = GST_VA_BASE_DEC_GET_CLASS (self);
switch (prop_id) {
case GST_VA_DEC_PROP_DEVICE_PATH:{
if (!self->display)
g_value_set_string (value, klass->render_device_path);
else if (GST_IS_VA_DISPLAY_PLATFORM (self->display))
g_object_get_property (G_OBJECT (self->display), "path", value);
else
g_value_set_string (value, NULL);
break;
}
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
}
}
static gboolean
gst_va_base_dec_open (GstVideoDecoder * decoder)
{
GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
GstVaBaseDecClass *klass = GST_VA_BASE_DEC_GET_CLASS (decoder);
gboolean ret = FALSE;
if (!gst_va_ensure_element_data (decoder, klass->render_device_path,
&base->display))
return FALSE;
g_object_notify (G_OBJECT (decoder), "device-path");
if (!g_atomic_pointer_get (&base->decoder)) {
GstVaDecoder *va_decoder;
va_decoder = gst_va_decoder_new (base->display, klass->codec);
if (va_decoder)
ret = TRUE;
gst_object_replace ((GstObject **) (&base->decoder),
(GstObject *) va_decoder);
gst_clear_object (&va_decoder);
} else {
ret = TRUE;
}
base->apply_video_crop = FALSE;
return ret;
}
gboolean
gst_va_base_dec_close (GstVideoDecoder * decoder)
{
GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
gst_clear_object (&base->decoder);
gst_clear_object (&base->display);
g_object_notify (G_OBJECT (decoder), "device-path");
return TRUE;
}
static gboolean
gst_va_base_dec_stop (GstVideoDecoder * decoder)
{
GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
if (!gst_va_decoder_close (base->decoder))
return FALSE;
g_clear_pointer (&base->output_state, gst_video_codec_state_unref);
g_clear_pointer (&base->input_state, gst_video_codec_state_unref);
if (base->other_pool)
gst_buffer_pool_set_active (base->other_pool, FALSE);
gst_clear_object (&base->other_pool);
g_clear_pointer (&base->convert, gst_video_converter_free);
return GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS
(decoder))->stop (decoder);
}
static GstCaps *
gst_va_base_dec_getcaps (GstVideoDecoder * decoder, GstCaps * filter)
{
GstCaps *caps = NULL, *tmp;
GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
GstVaDecoder *va_decoder = NULL;
gst_object_replace ((GstObject **) & va_decoder, (GstObject *) base->decoder);
if (va_decoder) {
caps = gst_va_decoder_get_sinkpad_caps (va_decoder);
gst_object_unref (va_decoder);
}
if (caps) {
if (filter) {
tmp = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (caps);
caps = tmp;
}
GST_LOG_OBJECT (base, "Returning caps %" GST_PTR_FORMAT, caps);
} else {
caps = gst_video_decoder_proxy_getcaps (decoder, NULL, filter);
}
return caps;
}
static gboolean
_query_context (GstVaBaseDec * self, GstQuery * query)
{
GstVaDisplay *display = NULL;
gboolean ret;
gst_object_replace ((GstObject **) & display, (GstObject *) self->display);
ret = gst_va_handle_context_query (GST_ELEMENT_CAST (self), query, display);
gst_clear_object (&display);
return ret;
}
static gboolean
gst_va_base_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
{
GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
gboolean ret = FALSE;
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:{
ret = _query_context (base, query);
break;
}
case GST_QUERY_CAPS:{
GstCaps *caps = NULL, *tmp, *filter = NULL;
GstVaDecoder *va_decoder = NULL;
gboolean fixed_caps;
gst_object_replace ((GstObject **) & va_decoder,
(GstObject *) base->decoder);
gst_query_parse_caps (query, &filter);
fixed_caps = GST_PAD_IS_FIXED_CAPS (GST_VIDEO_DECODER_SRC_PAD (decoder));
if (!fixed_caps && va_decoder)
caps = gst_va_decoder_get_srcpad_caps (va_decoder);
gst_clear_object (&va_decoder);
if (caps) {
if (filter) {
tmp =
gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (caps);
caps = tmp;
}
GST_LOG_OBJECT (base, "Returning caps %" GST_PTR_FORMAT, caps);
gst_query_set_caps_result (query, caps);
gst_caps_unref (caps);
ret = TRUE;
break;
}
/* else jump to default */
}
default:
ret = GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS
(decoder))->src_query (decoder, query);
break;
}
return ret;
}
static gboolean
gst_va_base_dec_sink_query (GstVideoDecoder * decoder, GstQuery * query)
{
if (GST_QUERY_TYPE (query) == GST_QUERY_CONTEXT)
return _query_context (GST_VA_BASE_DEC (decoder), query);
return GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS
(decoder))->sink_query (decoder, query);
}
static GstAllocator *
_create_allocator (GstVaBaseDec * base, GstCaps * caps)
{
GstAllocator *allocator = NULL;
if (gst_caps_is_dmabuf (caps))
allocator = gst_va_dmabuf_allocator_new (base->display);
else {
GArray *surface_formats =
gst_va_decoder_get_surface_formats (base->decoder);
allocator = gst_va_allocator_new (base->display, surface_formats);
gst_va_allocator_set_hacks (allocator, base->hacks);
}
return allocator;
}
static gboolean
_need_video_crop (GstVaBaseDec * base)
{
if (base->need_valign &&
(base->valign.padding_left > 0 || base->valign.padding_top > 0))
return TRUE;
return FALSE;
}
/* This path for pool setting is a little complicated but not commonly
used. We deliberately separate it from the main path of pool setting. */
static gboolean
_decide_allocation_for_video_crop (GstVideoDecoder * decoder,
GstQuery * query, GstCaps * caps)
{
GstAllocator *allocator = NULL, *other_allocator = NULL;
GstAllocationParams other_params, params;
gboolean update_pool = FALSE, update_allocator = FALSE;
GstBufferPool *pool = NULL, *other_pool = NULL;
guint size = 0, min, max, usage_hint;
GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
gboolean ret = FALSE;
gboolean dont_use_other_pool = FALSE;
GstCaps *va_caps = NULL;
/* If others provide a valid allocator, just use it. */
if (gst_query_get_n_allocation_params (query) > 0) {
gst_query_parse_nth_allocation_param (query, 0, &other_allocator,
&other_params);
GstVaDisplay *display;
display = gst_va_allocator_peek_display (other_allocator);
/* We should not use allocator and pool from other display. */
if (display != base->display) {
gst_clear_object (&other_allocator);
dont_use_other_pool = TRUE;
}
update_allocator = TRUE;
} else {
gst_allocation_params_init (&other_params);
}
/* If others provide a valid pool, just use it. */
if (gst_query_get_n_allocation_pools (query) > 0) {
gst_query_parse_nth_allocation_pool (query, 0, &other_pool, &size, &min,
&max);
if (dont_use_other_pool)
gst_clear_object (&other_pool);
min += base->min_buffers;
update_pool = TRUE;
} else {
min = base->min_buffers;
max = 0;
}
/* Ensure that the other pool is ready */
if (gst_caps_is_raw (caps)) {
if (GST_IS_VA_POOL (other_pool)) {
gst_clear_object (&other_pool);
size = 0;
}
if (!other_pool) {
if (other_allocator && (GST_IS_VA_DMABUF_ALLOCATOR (other_allocator)
|| GST_IS_VA_ALLOCATOR (other_allocator)))
gst_clear_object (&other_allocator);
GST_DEBUG_OBJECT (base, "making new other pool for copy");
base->other_pool =
gst_va_create_other_pool (other_allocator, &other_params, caps, size);
} else {
gst_object_replace ((GstObject **) & base->other_pool,
(GstObject *) other_pool);
}
if (!base->other_pool) {
GST_ERROR_OBJECT (base, "Couldn't configure other pool for copy");
goto cleanup;
}
} else {
GstStructure *other_config;
if (!GST_IS_VA_POOL (other_pool))
gst_clear_object (&other_pool);
if (!other_pool)
other_pool = gst_va_pool_new ();
if (other_allocator && !(GST_IS_VA_DMABUF_ALLOCATOR (other_allocator)
|| GST_IS_VA_ALLOCATOR (other_allocator)))
gst_clear_object (&other_allocator);
if (!other_allocator) {
other_allocator = _create_allocator (base, caps);
if (!other_allocator)
goto cleanup;
}
other_config = gst_buffer_pool_get_config (other_pool);
gst_buffer_pool_config_set_params (other_config, caps, 0, min, max);
gst_buffer_pool_config_set_allocator (other_config, other_allocator,
&other_params);
/* Always support VideoMeta but no VideoCropMeta here. */
gst_buffer_pool_config_add_option (other_config,
GST_BUFFER_POOL_OPTION_VIDEO_META);
gst_buffer_pool_config_set_va_allocation_params (other_config,
VA_SURFACE_ATTRIB_USAGE_HINT_GENERIC, GST_VA_FEATURE_AUTO);
if (!gst_buffer_pool_set_config (other_pool, other_config))
goto cleanup;
gst_object_replace ((GstObject **) & base->other_pool,
(GstObject *) other_pool);
}
/* Now setup the buffer pool for decoder */
va_caps = gst_caps_copy (caps);
gst_caps_set_features_simple (va_caps,
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_VA));
if (!(allocator = _create_allocator (base, va_caps)))
goto cleanup;
gst_allocation_params_init (&params);
pool = gst_va_pool_new ();
{
GstStructure *config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, 0, min, max);
gst_buffer_pool_config_set_allocator (config, allocator, &params);
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_META);
if (_need_video_crop (base))
gst_buffer_pool_config_set_va_alignment (config, &base->valign);
usage_hint = va_get_surface_usage_hint (base->display,
VAEntrypointVLD, GST_PAD_SRC, gst_video_is_dma_drm_caps (caps));
gst_buffer_pool_config_set_va_allocation_params (config,
usage_hint, GST_VA_FEATURE_AUTO);
if (!gst_buffer_pool_set_config (pool, config))
goto cleanup;
if (!gst_va_pool_get_buffer_size (pool, &size))
goto cleanup;
}
if (update_allocator)
gst_query_set_nth_allocation_param (query, 0, allocator, &params);
else
gst_query_add_allocation_param (query, allocator, &params);
if (update_pool)
gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
else
gst_query_add_allocation_pool (query, pool, size, min, max);
GST_WARNING_OBJECT (base, "We need to copy the output buffer manually "
"because of the top/left alignment, which may have low performance. "
"The element which supports VideoCropMeta such as 'vapostproc' can "
"avoid this.");
base->copy_frames = TRUE;
base->apply_video_crop = TRUE;
ret = TRUE;
cleanup:
gst_clear_object (&allocator);
gst_clear_object (&other_allocator);
gst_clear_object (&pool);
gst_clear_object (&other_pool);
gst_clear_caps (&va_caps);
return ret;
}
/* We only support system pool and va pool. For va pool, its allocator
* should be va allocator or dma allocator.
* If output caps is memory:VAMemory, the pool should be a va pool
* with va allocator.
* If output caps is memory:DMABuf, the pool should be a va pool
* with dma allocator.
* We may need the other_pool to copy the decoder picture to the
* output buffer. We need to do this copy when:
* 1). The output caps is raw(system mem), but the downstream does
* not support VideoMeta and the strides and offsets of the va pool
* are different from the system memory pool, which means that the
* gst_video_frame_map() can not map the buffer correctly. Then we
* need a va pool with va allocator as an the internal pool and create
* a system pool as the other_pool to copy frames to system mem and
* output it.
* 2). The decoder has crop_top/left value > 0(e.g. the conformance
* window in the H265). Which means that the real output picture
* locates in the middle of the decoded buffer. If the downstream can
* support VideoCropMeta, a VideoCropMeta is added to notify the
* real picture's coordinate and size. But if not, we need to copy
* it manually and the other_pool is needed. We always assume that
* decoded picture starts from top-left corner, and so there is no
* need to do this if crop_bottom/right value > 0.
*
* 1. if crop_top/left value > 0 and the downstream does not support the
* VideoCropMeta, we always have the other_pool to do the copy(The pool
* may be provided by the downstream element, or created by ourself if
* no suitable one found).
* 2. get allocator in query
* 2.1 if allocator is not ours and caps is raw, keep it for other_pool.
* 3. get pool in query
* 3.1 if pool is not va, downstream doesn't support video meta and
* caps are raw, keep it as other_pool.
* 3.2 if there's no pool in query and and caps is raw, create other_pool
* as GstVideoPool with the non-va from query and query's params.
* 4. create our allocator and pool if they aren't in query
* 5. add or update pool and allocator in query
* 6. set our custom pool configuration
*/
static gboolean
gst_va_base_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
{
GstAllocator *allocator = NULL, *other_allocator = NULL;
GstAllocationParams other_params, params;
GstBufferPool *pool = NULL, *other_pool = NULL;
GstCaps *caps = NULL;
GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
guint other_size = 0, size = 0, min, max, usage_hint;
gboolean update_pool = FALSE, update_allocator = FALSE;
gboolean has_videometa, has_video_crop_meta;
gboolean dont_use_other_pool = FALSE, ret = FALSE;
g_assert (base->min_buffers > 0);
gst_query_parse_allocation (query, &caps, NULL);
if (!caps)
goto wrong_caps;
has_videometa = gst_query_find_allocation_meta (query,
GST_VIDEO_META_API_TYPE, NULL);
has_video_crop_meta = has_videometa && gst_query_find_allocation_meta (query,
GST_VIDEO_CROP_META_API_TYPE, NULL);
/* 1. The output picture locates in the middle of the decoded buffer,
but the downstream element does not support VideoCropMeta, we
definitely need a copy.
2. Some codec such as H265, it does not clean the DPB when new SPS
comes. The new SPS may set the crop window to top-left corner and
so no video crop is needed here. But we may still have cached frames
in DPB which need a copy.
3. For DMA kind memory, because we may not be able to map this buffer,
just disable the copy for crop. This may cause some alignment garbage. */
if (!gst_video_is_dma_drm_caps (caps) &&
((_need_video_crop (base) && !has_video_crop_meta) ||
base->apply_video_crop)) {
return _decide_allocation_for_video_crop (decoder, query, caps);
}
if (gst_query_get_n_allocation_params (query) > 0) {
GstVaDisplay *display;
gst_query_parse_nth_allocation_param (query, 0, &allocator, &other_params);
display = gst_va_allocator_peek_display (allocator);
if (!display) {
/* save the allocator for the other pool */
other_allocator = allocator;
allocator = NULL;
} else if (display != base->display) {
/* The allocator and pool belong to other display, we should not use. */
gst_clear_object (&allocator);
dont_use_other_pool = TRUE;
}
update_allocator = TRUE;
} else {
gst_allocation_params_init (&other_params);
}
gst_allocation_params_init (&params);
if (gst_query_get_n_allocation_pools (query) > 0) {
gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
if (pool) {
if (!GST_IS_VA_POOL (pool)) {
GST_DEBUG_OBJECT (base,
"may need other pool for copy frames %" GST_PTR_FORMAT, pool);
other_pool = pool;
pool = NULL;
other_size = size;
} else if (dont_use_other_pool) {
gst_clear_object (&pool);
}
}
min += base->min_buffers;
update_pool = TRUE;
} else {
min = base->min_buffers;
max = 0;
}
if (!allocator) {
if (!(allocator = _create_allocator (base, caps)))
goto cleanup;
}
if (!pool)
pool = gst_va_pool_new ();
{
GstStructure *config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, 0, min, max);
gst_buffer_pool_config_set_allocator (config, allocator, &params);
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_META);
if (base->need_valign)
gst_buffer_pool_config_set_va_alignment (config, &base->valign);
usage_hint = va_get_surface_usage_hint (base->display,
VAEntrypointVLD, GST_PAD_SRC, gst_video_is_dma_drm_caps (caps));
gst_buffer_pool_config_set_va_allocation_params (config,
usage_hint, GST_VA_FEATURE_AUTO);
if (!gst_buffer_pool_set_config (pool, config))
goto cleanup;
if (!gst_va_pool_get_buffer_size (pool, &size))
goto cleanup;
}
if (update_allocator)
gst_query_set_nth_allocation_param (query, 0, allocator, &params);
else
gst_query_add_allocation_param (query, allocator, &params);
if (update_pool)
gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
else
gst_query_add_allocation_pool (query, pool, size, min, max);
base->copy_frames = (!has_videometa && gst_va_pool_requires_video_meta (pool)
&& gst_caps_is_raw (caps));
if (base->copy_frames) {
if (other_pool) {
gst_object_replace ((GstObject **) & base->other_pool,
(GstObject *) other_pool);
} else {
gst_clear_object (&base->other_pool);
base->other_pool =
gst_va_create_other_pool (other_allocator, &other_params, caps,
other_size);
}
if (!base->other_pool) {
GST_ERROR_OBJECT (base, "Couldn't configure other pool for copy");
goto cleanup;
}
GST_DEBUG_OBJECT (base, "Use the other pool for copy %" GST_PTR_FORMAT,
base->other_pool);
} else {
gst_clear_object (&base->other_pool);
}
ret = TRUE;
cleanup:
gst_clear_object (&allocator);
gst_clear_object (&other_allocator);
gst_clear_object (&pool);
gst_clear_object (&other_pool);
/* There's no need to chain decoder's method since all what is
* needed is done. */
return ret;
wrong_caps:
{
GST_WARNING_OBJECT (base, "No valid caps");
return FALSE;
}
}
static void
gst_va_base_dec_set_context (GstElement * element, GstContext * context)
{
GstVaDisplay *old_display, *new_display;
GstVaBaseDec *base = GST_VA_BASE_DEC (element);
GstVaBaseDecClass *klass = GST_VA_BASE_DEC_GET_CLASS (base);
gboolean ret;
old_display = base->display ? gst_object_ref (base->display) : NULL;
ret = gst_va_handle_set_context (element, context, klass->render_device_path,
&base->display);
new_display = base->display ? gst_object_ref (base->display) : NULL;
if (!ret
|| (old_display && new_display && old_display != new_display
&& base->decoder)) {
GST_ELEMENT_WARNING (base, RESOURCE, BUSY,
("Can't replace VA display while operating"), (NULL));
}
gst_clear_object (&old_display);
gst_clear_object (&new_display);
GST_ELEMENT_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS
(element))->set_context (element, context);
}
static gboolean
gst_va_base_dec_negotiate (GstVideoDecoder * decoder)
{
GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
/* Ignore downstream renegotiation request. */
if (!base->need_negotiation)
return TRUE;
base->need_negotiation = FALSE;
if (!gst_va_decoder_config_is_equal (base->decoder, base->profile,
base->rt_format, base->width, base->height)) {
if (gst_va_decoder_is_open (base->decoder) &&
!gst_va_decoder_close (base->decoder))
return FALSE;
if (!gst_va_decoder_open (base->decoder, base->profile, base->rt_format))
return FALSE;
if (!gst_va_decoder_set_frame_size (base->decoder, base->width,
base->height))
return FALSE;
}
if (!gst_va_base_dec_set_output_state (base))
return FALSE;
return GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS (decoder))
->negotiate (decoder);
}
void
gst_va_base_dec_init (GstVaBaseDec * base, GstDebugCategory * cat)
{
base->debug_category = cat;
gst_video_info_init (&base->output_info);
}
void
gst_va_base_dec_class_init (GstVaBaseDecClass * klass, GstVaCodecs codec,
const gchar * render_device_path, GstCaps * sink_caps, GstCaps * src_caps,
GstCaps * doc_src_caps, GstCaps * doc_sink_caps)
{
GstPadTemplate *sink_pad_templ, *src_pad_templ;
GObjectClass *object_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
klass->parent_decoder_class = g_type_class_peek_parent (klass);
klass->codec = codec;
klass->render_device_path = g_strdup (render_device_path);
sink_pad_templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
sink_caps);
gst_element_class_add_pad_template (element_class, sink_pad_templ);
if (doc_sink_caps) {
gst_pad_template_set_documentation_caps (sink_pad_templ, doc_sink_caps);
gst_caps_unref (doc_sink_caps);
}
src_pad_templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
src_caps);
gst_element_class_add_pad_template (element_class, src_pad_templ);
if (doc_src_caps) {
gst_pad_template_set_documentation_caps (src_pad_templ, doc_src_caps);
gst_caps_unref (doc_src_caps);
}
object_class->get_property = gst_va_base_dec_get_property;
element_class->set_context = GST_DEBUG_FUNCPTR (gst_va_base_dec_set_context);
decoder_class->open = GST_DEBUG_FUNCPTR (gst_va_base_dec_open);
decoder_class->close = GST_DEBUG_FUNCPTR (gst_va_base_dec_close);
decoder_class->stop = GST_DEBUG_FUNCPTR (gst_va_base_dec_stop);
decoder_class->getcaps = GST_DEBUG_FUNCPTR (gst_va_base_dec_getcaps);
decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_va_base_dec_src_query);
decoder_class->sink_query = GST_DEBUG_FUNCPTR (gst_va_base_dec_sink_query);
decoder_class->decide_allocation =
GST_DEBUG_FUNCPTR (gst_va_base_dec_decide_allocation);
decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_va_base_dec_negotiate);
g_object_class_install_property (object_class, GST_VA_DEC_PROP_DEVICE_PATH,
g_param_spec_string ("device-path", "Device Path",
GST_VA_DEVICE_PATH_PROP_DESC, NULL, GST_PARAM_DOC_SHOW_DEFAULT |
G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
}
static inline GstVideoFormat
_get_video_format_from_value (const GValue * format, gboolean drm_format,
guint64 * modifier)
{
guint32 fourcc;
const char *fmt;
g_assert (G_VALUE_HOLDS_STRING (format));
fmt = g_value_get_string (format);
if (drm_format) {
fourcc = gst_video_dma_drm_fourcc_from_string (fmt, modifier);
return gst_va_video_format_from_drm_fourcc (fourcc);
}
return gst_video_format_from_string (fmt);
}
static GstVideoFormat
_find_video_format_from_chroma (const GValue * formats, guint chroma_type,
gboolean drm_format, guint64 * modifier)
{
GstVideoFormat fmt = GST_VIDEO_FORMAT_UNKNOWN;
guint i, num_values;
if (!formats)
return GST_VIDEO_FORMAT_UNKNOWN;
if (G_VALUE_HOLDS_STRING (formats)) {
fmt = _get_video_format_from_value (formats, drm_format, modifier);
if (gst_va_chroma_from_video_format (fmt) == chroma_type)
return fmt;
} else if (GST_VALUE_HOLDS_LIST (formats)) {
const GValue *format;
num_values = gst_value_list_get_size (formats);
for (i = 0; i < num_values; i++) {
format = gst_value_list_get_value (formats, i);
fmt = _get_video_format_from_value (format, drm_format, modifier);
if (gst_va_chroma_from_video_format (fmt) == chroma_type)
return fmt;
}
}
return GST_VIDEO_FORMAT_UNKNOWN;
}
static GstVideoFormat
_caps_video_format_from_chroma (GstCaps * caps, guint chroma_type)
{
guint i, num_structures;
GstCapsFeatures *feats;
GstStructure *structure;
const GValue *format;
GstVideoFormat fmt;
num_structures = gst_caps_get_size (caps);
for (i = 0; i < num_structures; i++) {
feats = gst_caps_get_features (caps, i);
if (!gst_caps_features_is_equal (feats,
GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY))
continue;
structure = gst_caps_get_structure (caps, i);
format = gst_structure_get_value (structure, "format");
fmt = _find_video_format_from_chroma (format, chroma_type, FALSE, NULL);
if (fmt == GST_VIDEO_FORMAT_UNKNOWN)
continue;
if (gst_va_chroma_from_video_format (fmt) == chroma_type)
return fmt;
}
return GST_VIDEO_FORMAT_UNKNOWN;
}
/* ordered list of capsfeature preference */
enum
{ VA, DMABUF, SYSMEM };
void
gst_va_base_dec_get_preferred_format_and_caps_features (GstVaBaseDec * base,
GstVideoFormat * format, GstCapsFeatures ** capsfeatures,
guint64 * modifier)
{
GstCaps *peer_caps, *allowed_caps;
GstCapsFeatures *features;
guint num_structures, i, j;
gboolean is_any;
/* array designators might not be supported by some compilers */
const GQuark feats[] = {
/* [VA] = */ g_quark_from_string (GST_CAPS_FEATURE_MEMORY_VA),
/* [DMABUF] = */ g_quark_from_string (GST_CAPS_FEATURE_MEMORY_DMABUF),
/* [SYSMEM] = */
g_quark_from_string (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY),
};
g_return_if_fail (base);
/* verify if peer caps is any */
{
peer_caps =
gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD (base), NULL);
is_any = gst_caps_is_any (peer_caps);
gst_clear_caps (&peer_caps);
}
allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (base));
GST_DEBUG_OBJECT (base, "Allowed caps %" GST_PTR_FORMAT, allowed_caps);
/* if peer caps is any, returns format according memory to system caps in
* allowed caps
*
* if downstream doesn't support system memory negotiation will fail later.
*/
if (is_any) {
features = GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY;
if (format)
*format = _caps_video_format_from_chroma (allowed_caps, base->rt_format);
if (capsfeatures && *format != GST_VIDEO_FORMAT_UNKNOWN)
*capsfeatures = gst_caps_features_copy (features);
goto bail;
}
/* iterate allowed caps to find the first "capable" capability according our
* ordered list of preferred caps features */
num_structures = gst_caps_get_size (allowed_caps);
for (i = 0; i < G_N_ELEMENTS (feats); i++) {
for (j = 0; j < num_structures; j++) {
GstStructure *structure;
const GValue *formats;
GstVideoFormat fmt;
guint64 mod = 0;
features = gst_caps_get_features (allowed_caps, j);
if (!gst_caps_features_contains_id (features, feats[i]))
continue;
structure = gst_caps_get_structure (allowed_caps, j);
if (i == DMABUF)
formats = gst_structure_get_value (structure, "drm-format");
else
formats = gst_structure_get_value (structure, "format");
fmt = _find_video_format_from_chroma (formats, base->rt_format,
i == DMABUF, &mod);
/* if doesn't found a proper format let's try other structure */
if (fmt == GST_VIDEO_FORMAT_UNKNOWN)
continue;
if (format)
*format = fmt;
if (i == DMABUF && modifier)
*modifier = mod;
if (capsfeatures)
*capsfeatures = gst_caps_features_new_id (feats[i], NULL);
goto bail;
}
}
/* no matching format, let's fail */
if (i == G_N_ELEMENTS (feats))
*format = GST_VIDEO_FORMAT_UNKNOWN;
bail:
gst_caps_unref (allowed_caps);
}
static gboolean
_copy_buffer_and_apply_video_crop (GstVaBaseDec * base,
GstVideoFrame * src_frame, GstVideoFrame * dest_frame,
GstVideoCropMeta * video_crop)
{
GstVideoInfo dst_info = dest_frame->info;
dst_info.fps_n = src_frame->info.fps_n;
dst_info.fps_d = src_frame->info.fps_d;
if (base->convert) {
gboolean new_convert = FALSE;
gint x = 0, y = 0, width = 0, height = 0;
const GstStructure *config = gst_video_converter_get_config (base->convert);
if (!gst_structure_get_int (config, GST_VIDEO_CONVERTER_OPT_SRC_X, &x)
|| !gst_structure_get_int (config, GST_VIDEO_CONVERTER_OPT_SRC_Y, &y)
|| !gst_structure_get_int (config, GST_VIDEO_CONVERTER_OPT_SRC_WIDTH,
&width)
|| !gst_structure_get_int (config, GST_VIDEO_CONVERTER_OPT_SRC_HEIGHT,
&height))
new_convert = TRUE;
new_convert |= (video_crop->x != x);
new_convert |= (video_crop->y != y);
new_convert |= (video_crop->width != width);
new_convert |= (video_crop->height != height);
/* No need to check dest, it always has (0,0) -> (width, height) */
if (new_convert)
g_clear_pointer (&base->convert, gst_video_converter_free);
}
if (!base->convert) {
base->convert = gst_video_converter_new (&src_frame->info, &dst_info,
gst_structure_new ("options",
GST_VIDEO_CONVERTER_OPT_DITHER_METHOD,
GST_TYPE_VIDEO_DITHER_METHOD, GST_VIDEO_DITHER_NONE,
GST_VIDEO_CONVERTER_OPT_DITHER_QUANTIZATION,
G_TYPE_UINT, 0,
GST_VIDEO_CONVERTER_OPT_CHROMA_MODE,
GST_TYPE_VIDEO_CHROMA_MODE, GST_VIDEO_CHROMA_MODE_NONE,
GST_VIDEO_CONVERTER_OPT_MATRIX_MODE,
GST_TYPE_VIDEO_MATRIX_MODE, GST_VIDEO_MATRIX_MODE_NONE,
GST_VIDEO_CONVERTER_OPT_SRC_X, G_TYPE_INT, video_crop->x,
GST_VIDEO_CONVERTER_OPT_SRC_Y, G_TYPE_INT, video_crop->y,
GST_VIDEO_CONVERTER_OPT_SRC_WIDTH, G_TYPE_INT, video_crop->width,
GST_VIDEO_CONVERTER_OPT_SRC_HEIGHT, G_TYPE_INT, video_crop->height,
GST_VIDEO_CONVERTER_OPT_DEST_X, G_TYPE_INT, 0,
GST_VIDEO_CONVERTER_OPT_DEST_Y, G_TYPE_INT, 0,
GST_VIDEO_CONVERTER_OPT_DEST_WIDTH, G_TYPE_INT, video_crop->width,
GST_VIDEO_CONVERTER_OPT_DEST_HEIGHT, G_TYPE_INT, video_crop->height,
NULL));
if (!base->convert) {
GST_WARNING_OBJECT (base, "failed to create a video convert");
return FALSE;
}
}
gst_video_converter_frame (base->convert, src_frame, dest_frame);
return TRUE;
}
gboolean
gst_va_base_dec_copy_output_buffer (GstVaBaseDec * base,
GstVideoCodecFrame * codec_frame)
{
GstVideoFrame src_frame;
GstVideoFrame dest_frame;
GstVideoInfo dest_vinfo;
GstVideoInfo *src_vinfo;
GstBuffer *buffer = NULL;
GstVideoCropMeta *video_crop;
GstFlowReturn ret;
g_return_val_if_fail (base && base->output_state, FALSE);
if (!base->other_pool)
return FALSE;
if (!gst_buffer_pool_set_active (base->other_pool, TRUE))
return FALSE;
src_vinfo = &base->output_state->info;
gst_video_info_set_format (&dest_vinfo, GST_VIDEO_INFO_FORMAT (src_vinfo),
GST_VIDEO_INFO_WIDTH (src_vinfo), GST_VIDEO_INFO_HEIGHT (src_vinfo));
ret = gst_buffer_pool_acquire_buffer (base->other_pool, &buffer, NULL);
if (ret != GST_FLOW_OK)
goto fail;
if (!gst_video_frame_map (&src_frame, src_vinfo, codec_frame->output_buffer,
GST_MAP_READ))
goto fail;
if (!gst_video_frame_map (&dest_frame, &dest_vinfo, buffer, GST_MAP_WRITE)) {
gst_video_frame_unmap (&src_frame);
goto fail;
}
video_crop = gst_buffer_get_video_crop_meta (codec_frame->output_buffer);
if (video_crop) {
if (!_copy_buffer_and_apply_video_crop (base,
&src_frame, &dest_frame, video_crop)) {
gst_video_frame_unmap (&src_frame);
gst_video_frame_unmap (&dest_frame);
GST_ERROR_OBJECT (base, "fail to apply the video crop.");
goto fail;
}
} else {
/* gst_video_frame_copy can crop this, but does not know, so let
* make it think it's all right */
GST_VIDEO_INFO_WIDTH (&src_frame.info) = GST_VIDEO_INFO_WIDTH (src_vinfo);
GST_VIDEO_INFO_HEIGHT (&src_frame.info) = GST_VIDEO_INFO_HEIGHT (src_vinfo);
if (!gst_video_frame_copy (&dest_frame, &src_frame)) {
gst_video_frame_unmap (&src_frame);
gst_video_frame_unmap (&dest_frame);
goto fail;
}
}
gst_video_frame_unmap (&src_frame);
gst_video_frame_unmap (&dest_frame);
gst_buffer_replace (&codec_frame->output_buffer, buffer);
gst_buffer_unref (buffer);
return TRUE;
fail:
if (buffer)
gst_buffer_unref (buffer);
GST_ERROR_OBJECT (base, "Failed copy output buffer.");
return FALSE;
}
gboolean
gst_va_base_dec_process_output (GstVaBaseDec * base, GstVideoCodecFrame * frame,
GstVideoCodecState * input_state, GstVideoBufferFlags buffer_flags)
{
GstVideoDecoder *vdec = GST_VIDEO_DECODER (base);
if (input_state) {
g_clear_pointer (&base->input_state, gst_video_codec_state_unref);
base->input_state = gst_video_codec_state_ref (input_state);
base->need_negotiation = TRUE;
if (!gst_video_decoder_negotiate (vdec)) {
GST_ERROR_OBJECT (base, "Could not re-negotiate with updated state");
return FALSE;
}
}
if (base->copy_frames)
gst_va_base_dec_copy_output_buffer (base, frame);
if (buffer_flags != 0) {
#ifndef GST_DISABLE_GST_DEBUG
gboolean interlaced =
(buffer_flags & GST_VIDEO_BUFFER_FLAG_INTERLACED) != 0;
gboolean tff = (buffer_flags & GST_VIDEO_BUFFER_FLAG_TFF) != 0;
GST_TRACE_OBJECT (base,
"apply buffer flags 0x%x (interlaced %d, top-field-first %d)",
buffer_flags, interlaced, tff);
#endif
GST_BUFFER_FLAG_SET (frame->output_buffer, buffer_flags);
}
return TRUE;
}
GstFlowReturn
gst_va_base_dec_prepare_output_frame (GstVaBaseDec * base,
GstVideoCodecFrame * frame)
{
GstVideoDecoder *vdec = GST_VIDEO_DECODER (base);
if (base->need_negotiation) {
if (!gst_video_decoder_negotiate (vdec)) {
GST_ERROR_OBJECT (base, "Failed to negotiate with downstream");
return GST_FLOW_NOT_NEGOTIATED;
}
}
if (frame)
return gst_video_decoder_allocate_output_frame (vdec, frame);
return GST_FLOW_OK;
}
gboolean
gst_va_base_dec_set_output_state (GstVaBaseDec * base)
{
GstVideoDecoder *decoder = GST_VIDEO_DECODER (base);
GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN;
guint64 modifier;
GstCapsFeatures *capsfeatures = NULL;
GstVideoInfo *info = &base->output_info;
if (base->output_state)
gst_video_codec_state_unref (base->output_state);
gst_va_base_dec_get_preferred_format_and_caps_features (base, &format,
&capsfeatures, &modifier);
if (format == GST_VIDEO_FORMAT_UNKNOWN)
return FALSE;
base->output_state =
gst_video_decoder_set_interlaced_output_state (decoder, format,
GST_VIDEO_INFO_INTERLACE_MODE (info), GST_VIDEO_INFO_WIDTH (info),
GST_VIDEO_INFO_HEIGHT (info), base->input_state);
/* set caps feature */
if (capsfeatures && gst_caps_features_contains (capsfeatures,
GST_CAPS_FEATURE_MEMORY_DMABUF)) {
base->output_state->caps =
gst_va_video_info_to_dma_caps (&base->output_state->info, modifier);
} else {
base->output_state->caps =
gst_video_info_to_caps (&base->output_state->info);
}
if (capsfeatures)
gst_caps_set_features_simple (base->output_state->caps, capsfeatures);
GST_INFO_OBJECT (base, "Negotiated caps %" GST_PTR_FORMAT,
base->output_state->caps);
return TRUE;
}