gstreamer/gst/vaapi/gstvaapiuploader.c

/*
 *  gstvaapiuploader.c - VA-API video upload helper
 *
 *  Copyright (C) 2010-2011 Splitted-Desktop Systems
 *    Author: Gwenole Beauchesne <gwenole.beauchesne@splitted-desktop.com>
 *  Copyright (C) 2011-2013 Intel Corporation
 *    Author: Gwenole Beauchesne <gwenole.beauchesne@intel.com>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public License
 *  as published by the Free Software Foundation; either version 2.1
 *  of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free
 *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 *  Boston, MA 02110-1301 USA
 */

#include "gst/vaapi/sysdeps.h"
#include <string.h>
#include <gst/video/video.h>
#include <gst/vaapi/gstvaapisurface.h>
#include <gst/vaapi/gstvaapiimagepool.h>
#include <gst/vaapi/gstvaapisurfacepool.h>

#include "gstvaapiuploader.h"
#include "gstvaapivideobuffer.h"

#define GST_HELPER_NAME "vaapiupload"
#define GST_HELPER_DESC "VA-API video uploader"

GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapi_uploader);
#define GST_CAT_DEFAULT gst_debug_vaapi_uploader

G_DEFINE_TYPE(GstVaapiUploader, gst_vaapi_uploader, G_TYPE_OBJECT)

#define GST_VAAPI_UPLOADER_CAST(obj) \
    ((GstVaapiUploader *)(obj))

#define GST_VAAPI_UPLOADER_GET_PRIVATE(obj)                     \
    (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
                                 GST_VAAPI_TYPE_UPLOADER,	\
                                 GstVaapiUploaderPrivate))

struct _GstVaapiUploaderPrivate {
    GstVaapiDisplay    *display;
    GstCaps            *allowed_caps;
    GstVaapiVideoPool  *images;
    GstVideoInfo        image_info;
    GstVaapiVideoPool  *surfaces;
    GstVideoInfo        surface_info;
    guint               direct_rendering;
};

enum {
    PROP_0,

    PROP_DISPLAY,
};

static void
gst_vaapi_uploader_destroy(GstVaapiUploader *uploader)
{
    GstVaapiUploaderPrivate * const priv = uploader->priv;

    gst_caps_replace(&priv->allowed_caps, NULL);
    gst_vaapi_video_pool_replace(&priv->images, NULL);
    gst_vaapi_video_pool_replace(&priv->surfaces, NULL);
    gst_vaapi_display_replace(&priv->display, NULL);
}

static gboolean
ensure_display(GstVaapiUploader *uploader, GstVaapiDisplay *display)
{
    GstVaapiUploaderPrivate * const priv = uploader->priv;

    gst_vaapi_display_replace(&priv->display, display);
    return TRUE;
}

static gboolean
ensure_image(GstVaapiImage *image)
{
    guint i, num_planes, width, height;

    /* Make the image fully dirty */
    if (!gst_vaapi_image_map(image))
        return FALSE;

    gst_vaapi_image_get_size(image, &width, &height);

    num_planes = gst_vaapi_image_get_plane_count(image);
    for (i = 0; i < num_planes; i++) {
        guchar * const plane = gst_vaapi_image_get_plane(image, i);
        if (plane)
            memset(plane, 0, gst_vaapi_image_get_pitch(image, i));
    }

    if (!gst_vaapi_image_unmap(image))
        gst_vaapi_image_unmap(image);
    return TRUE;
}

static gboolean
ensure_allowed_caps(GstVaapiUploader *uploader)
{
    GstVaapiUploaderPrivate * const priv = uploader->priv;
    GstVaapiSurface *surface = NULL;
    GArray *image_formats = NULL;
    GstCaps *out_caps;
    guint i;
    gboolean success = FALSE;

    enum { WIDTH = 64, HEIGHT = 64 };

    if (priv->allowed_caps)
        return TRUE;

    out_caps = gst_caps_new_empty();
    if (!out_caps)
        return FALSE;

    image_formats = gst_vaapi_display_get_image_formats(priv->display);
    if (!image_formats)
        goto end;

    surface = gst_vaapi_surface_new(priv->display,
        GST_VAAPI_CHROMA_TYPE_YUV420, WIDTH, HEIGHT);
    if (!surface)
        goto end;

    for (i = 0; i < image_formats->len; i++) {
        const GstVideoFormat format =
            g_array_index(image_formats, GstVideoFormat, i);
        GstVaapiImage *image;

        if (format == GST_VIDEO_FORMAT_UNKNOWN)
            continue;
        image = gst_vaapi_image_new(priv->display, format, WIDTH, HEIGHT);
        if (!image)
            continue;
        if (ensure_image(image) && gst_vaapi_surface_put_image(surface, image))
            gst_caps_append(out_caps, gst_vaapi_video_format_to_caps(format));
        gst_vaapi_object_unref(image);
    }

    gst_caps_replace(&priv->allowed_caps, out_caps);
    success = TRUE;

end:
    gst_caps_unref(out_caps);
    if (image_formats)
        g_array_unref(image_formats);
    if (surface)
        gst_vaapi_object_unref(surface);
    return success;
}

static gboolean
ensure_image_pool(GstVaapiUploader *uploader, GstCaps *caps,
    gboolean *caps_changed_ptr)
{
    GstVaapiUploaderPrivate * const priv = uploader->priv;
    GstVaapiVideoPool *pool;
    GstVideoInfo vi;
    GstVideoFormat format;
    guint width, height;

    if (!gst_video_info_from_caps(&vi, caps))
        return FALSE;

    format = GST_VIDEO_INFO_FORMAT(&vi);
    width  = GST_VIDEO_INFO_WIDTH(&vi);
    height = GST_VIDEO_INFO_HEIGHT(&vi);

    *caps_changed_ptr =
        format != GST_VIDEO_INFO_FORMAT(&priv->image_info) ||
        width  != GST_VIDEO_INFO_WIDTH(&priv->image_info)  ||
        height != GST_VIDEO_INFO_HEIGHT(&priv->image_info);
    if (!*caps_changed_ptr)
        return TRUE;

    pool = gst_vaapi_image_pool_new(priv->display, &vi);
    if (!pool)
        return FALSE;

    gst_video_info_set_format(&priv->image_info, format, width, height);
    gst_vaapi_video_pool_replace(&priv->images, pool);
    gst_vaapi_video_pool_unref(pool);
    return TRUE;
}

static gboolean
ensure_surface_pool(GstVaapiUploader *uploader, GstCaps *caps,
    gboolean *caps_changed_ptr)
{
    GstVaapiUploaderPrivate * const priv = uploader->priv;
    GstVaapiVideoPool *pool;
    GstVideoInfo vi;
    GstVideoFormat format;
    guint width, height;

    if (!gst_video_info_from_caps(&vi, caps))
        return FALSE;

    format = GST_VIDEO_INFO_FORMAT(&vi);
    width  = GST_VIDEO_INFO_WIDTH(&vi);
    height = GST_VIDEO_INFO_HEIGHT(&vi);

    *caps_changed_ptr =
        format != GST_VIDEO_INFO_FORMAT(&priv->surface_info) ||
        width  != GST_VIDEO_INFO_WIDTH(&priv->surface_info)  ||
        height != GST_VIDEO_INFO_HEIGHT(&priv->surface_info);
    if (!*caps_changed_ptr)
        return TRUE;

    /* Always try to downsample source buffers to YUV 4:2:0 format as
       this saves memory bandwidth for further rendering */
    /* XXX: this also means that visual quality is not preserved */
    if (format != GST_VIDEO_FORMAT_ENCODED) {
        const GstVaapiChromaType chroma_type =
            gst_vaapi_video_format_get_chroma_type(format);
        if (chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
            const GstVideoFormat image_format =
                GST_VIDEO_INFO_FORMAT(&priv->image_info);
            GST_INFO("use implicit conversion of %s buffers to NV12 surfaces",
                     gst_video_format_to_string(image_format));
            gst_video_info_set_format(&vi, GST_VIDEO_FORMAT_NV12, width, height);
        }
    }

    pool = gst_vaapi_surface_pool_new(priv->display, &vi);
    if (!pool)
        return FALSE;

    gst_video_info_set_format(&priv->surface_info, format, width, height);
    gst_vaapi_video_pool_replace(&priv->surfaces, pool);
    gst_vaapi_video_pool_unref(pool);
    return TRUE;
}

static void
gst_vaapi_uploader_finalize(GObject *object)
{
    gst_vaapi_uploader_destroy(GST_VAAPI_UPLOADER_CAST(object));

    G_OBJECT_CLASS(gst_vaapi_uploader_parent_class)->finalize(object);
}

static void
gst_vaapi_uploader_set_property(GObject *object, guint prop_id,
    const GValue *value, GParamSpec *pspec)
{
    GstVaapiUploader * const uploader = GST_VAAPI_UPLOADER_CAST(object);

    switch (prop_id) {
    case PROP_DISPLAY:
        ensure_display(uploader, g_value_get_pointer(value));
        break;
    default:
        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
        break;
    }
}

static void
gst_vaapi_uploader_get_property(GObject *object, guint prop_id,
    GValue *value, GParamSpec *pspec)
{
    GstVaapiUploader * const uploader = GST_VAAPI_UPLOADER_CAST(object);

    switch (prop_id) {
    case PROP_DISPLAY:
        g_value_set_pointer(value, uploader->priv->display);
        break;
    default:
        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
        break;
    }
}

static void
gst_vaapi_uploader_class_init(GstVaapiUploaderClass *klass)
{
    GObjectClass * const object_class = G_OBJECT_CLASS(klass);

    GST_DEBUG_CATEGORY_INIT(gst_debug_vaapi_uploader,
        GST_HELPER_NAME, 0, GST_HELPER_DESC);

    g_type_class_add_private(klass, sizeof(GstVaapiUploaderPrivate));

    object_class->finalize      = gst_vaapi_uploader_finalize;
    object_class->set_property = gst_vaapi_uploader_set_property;
    object_class->get_property = gst_vaapi_uploader_get_property;

    g_object_class_install_property(
        object_class,
        PROP_DISPLAY,
        g_param_spec_pointer(
            "display",
            "Display",
            "The GstVaapiDisplay this object is bound to",
            G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
}

static void
gst_vaapi_uploader_init(GstVaapiUploader *uploader)
{
    GstVaapiUploaderPrivate *priv;

    priv                = GST_VAAPI_UPLOADER_GET_PRIVATE(uploader);
    uploader->priv      = priv;

    gst_video_info_init(&priv->image_info);
    gst_video_info_init(&priv->surface_info);
}

GstVaapiUploader *
gst_vaapi_uploader_new(GstVaapiDisplay *display)
{
    return g_object_new(GST_VAAPI_TYPE_UPLOADER, "display", display, NULL);
}

gboolean
gst_vaapi_uploader_ensure_display(
    GstVaapiUploader *uploader,
    GstVaapiDisplay  *display
)
{
    g_return_val_if_fail(GST_VAAPI_IS_UPLOADER(uploader), FALSE);
    g_return_val_if_fail(display != NULL, FALSE);

    return ensure_display(uploader,display);
}

gboolean
gst_vaapi_uploader_ensure_caps(
    GstVaapiUploader *uploader,
    GstCaps          *src_caps,
    GstCaps          *out_caps
)
{
    GstVaapiUploaderPrivate *priv;
    GstVaapiImage *image;
    gboolean image_caps_changed, surface_caps_changed;

    g_return_val_if_fail(GST_VAAPI_IS_UPLOADER(uploader), FALSE);
    g_return_val_if_fail(src_caps != NULL, FALSE);

    if (!out_caps)
        out_caps = src_caps;

    if (!ensure_image_pool(uploader, src_caps, &image_caps_changed))
        return FALSE;
    if (!ensure_surface_pool(uploader, out_caps, &surface_caps_changed))
        return FALSE;
    if (!image_caps_changed && !surface_caps_changed)
        return TRUE;

    priv = uploader->priv;
    priv->direct_rendering = 0;

    /* Check if we can alias source and output buffers (same data_size) */
    image = gst_vaapi_video_pool_get_object(priv->images);
    if (image) {
        if ((gst_vaapi_image_get_format(image) ==
             GST_VIDEO_INFO_FORMAT(&priv->image_info)) &&
            gst_vaapi_image_is_linear(image) &&
            (gst_vaapi_image_get_data_size(image) ==
             GST_VIDEO_INFO_SIZE(&priv->image_info)))
            priv->direct_rendering = 1;
        gst_vaapi_video_pool_put_object(priv->images, image);
    }

    GST_INFO("direct-rendering: level %u", priv->direct_rendering);
    return TRUE;
}

gboolean
gst_vaapi_uploader_process(
    GstVaapiUploader *uploader,
    GstBuffer        *src_buffer,
    GstBuffer        *out_buffer
)
{
    GstVaapiVideoMeta *src_meta, *out_meta;
    GstVaapiSurface *surface;
    GstVaapiImage *image;

    g_return_val_if_fail(GST_VAAPI_IS_UPLOADER(uploader), FALSE);

    out_meta = gst_buffer_get_vaapi_video_meta(out_buffer);
    if (!out_meta) {
        GST_WARNING("expected an output video buffer");
        return FALSE;
    }

    surface = gst_vaapi_video_meta_get_surface(out_meta);
    g_return_val_if_fail(surface != NULL, FALSE);

    src_meta = gst_buffer_get_vaapi_video_meta(src_buffer);
    if (src_meta) {
        /* GstVaapiVideoBuffer with mapped VA image */
        image = gst_vaapi_video_meta_get_image(src_meta);
        if (!image || !gst_vaapi_image_unmap(image))
            return FALSE;
    }
    else {
        /* Regular GstBuffer that needs to be uploaded to a VA image */
        image = gst_vaapi_video_meta_get_image(out_meta);
        if (!image) {
            image = gst_vaapi_video_pool_get_object(uploader->priv->images);
            if (!image)
                return FALSE;
            gst_vaapi_video_meta_set_image(out_meta, image);
        }
        if (!gst_vaapi_image_update_from_buffer(image, src_buffer, NULL))
            return FALSE;
    }
    g_return_val_if_fail(image != NULL, FALSE);

    if (!gst_vaapi_surface_put_image(surface, image)) {
        GST_WARNING("failed to upload YUV buffer to VA surface");
        return FALSE;
    }

    /* Map again for next uploads */
    if (!gst_vaapi_image_map(image))
        return FALSE;
    return TRUE;
}

GstCaps *
gst_vaapi_uploader_get_caps(GstVaapiUploader *uploader)
{
    g_return_val_if_fail(GST_VAAPI_IS_UPLOADER(uploader), NULL);

    if (!ensure_allowed_caps(uploader))
        return NULL;
    return uploader->priv->allowed_caps;
}

GstBuffer *
gst_vaapi_uploader_get_buffer(GstVaapiUploader *uploader)
{
    GstVaapiUploaderPrivate *priv;
    GstVaapiImage *image;
    GstVaapiVideoMeta *meta;
    GstVaapiSurfaceProxy *proxy;
    GstBuffer *buffer;

    g_return_val_if_fail(GST_VAAPI_IS_UPLOADER(uploader), NULL);

    priv = uploader->priv;

    buffer = gst_vaapi_video_buffer_new_from_pool(priv->images);
    if (!buffer) {
        GST_WARNING("failed to allocate video buffer");
        goto error;
    }

    proxy = gst_vaapi_surface_proxy_new_from_pool(
        GST_VAAPI_SURFACE_POOL(priv->surfaces));
    if (!proxy) {
        GST_WARNING("failed to allocate VA surface");
        goto error;
    }

    meta = gst_buffer_get_vaapi_video_meta(buffer);
    gst_vaapi_video_meta_set_surface_proxy(meta, proxy);
    gst_vaapi_surface_proxy_unref(proxy);

    image = gst_vaapi_video_meta_get_image(meta);
    if (!gst_vaapi_image_map(image)) {
        GST_WARNING("failed to map VA image");
        goto error;
    }

#if !GST_CHECK_VERSION(1,0,0)
    GST_BUFFER_DATA(buffer) = gst_vaapi_image_get_plane(image, 0);
    GST_BUFFER_SIZE(buffer) = gst_vaapi_image_get_data_size(image);
#endif
    return buffer;

error:
    gst_buffer_unref(buffer);
    return buffer;
}

gboolean
gst_vaapi_uploader_has_direct_rendering(GstVaapiUploader *uploader)
{
    g_return_val_if_fail(GST_VAAPI_IS_UPLOADER(uploader), FALSE);

    return uploader->priv->direct_rendering;
}