/* GStreamer Intel MSDK plugin * Copyright (c) 2018, Intel Corporation * Copyright (c) 2018, Igalia S.L. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGDECE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "gstmsdkcontext.h" #ifndef _WIN32 #include #include #include #include #endif GST_DEBUG_CATEGORY_STATIC (gst_debug_msdkcontext); #define GST_CAT_DEFAULT gst_debug_msdkcontext struct _GstMsdkContextPrivate { mfxSession session; GList *cached_alloc_responses; gboolean hardware; gboolean is_joined; gboolean has_frame_allocator; GstMsdkContextJobType job_type; gint shared_async_depth; GMutex mutex; GList *child_session_list; #ifndef _WIN32 gint fd; VADisplay dpy; #endif }; #define gst_msdk_context_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstMsdkContext, gst_msdk_context, GST_TYPE_OBJECT, G_ADD_PRIVATE (GstMsdkContext) GST_DEBUG_CATEGORY_INIT (gst_debug_msdkcontext, "msdkcontext", 0, "MSDK Context")); #ifndef _WIN32 static gint get_device_id (void) { GUdevClient *client = NULL; GUdevEnumerator *e = NULL; GList *devices, *l; GUdevDevice *dev, *parent; const gchar *devnode_path; const gchar *devnode_files[2] = { "renderD[0-9]*", "card[0-9]*" }; int fd = -1, i; client = g_udev_client_new (NULL); if (!client) goto done; e = g_udev_enumerator_new (client); if (!e) goto done; g_udev_enumerator_add_match_subsystem (e, "drm"); for (i = 0; i < 2; i++) { g_udev_enumerator_add_match_name (e, devnode_files[i]); devices = g_udev_enumerator_execute (e); for (l = devices; l != NULL; l = l->next) { dev = (GUdevDevice *) l->data; parent = g_udev_device_get_parent (dev); if (strcmp (g_udev_device_get_subsystem (parent), "pci") != 0 || strcmp (g_udev_device_get_driver (parent), "i915") != 0) { g_object_unref (parent); continue; } g_object_unref (parent); devnode_path = g_udev_device_get_device_file (dev); fd = open (devnode_path, O_RDWR | O_CLOEXEC); if (fd < 0) continue; GST_DEBUG ("Opened the drm device node %s", devnode_path); break; } g_list_foreach (devices, (GFunc) gst_object_unref, NULL); g_list_free (devices); if (fd >= 0) goto done; } done: if (e) g_object_unref (e); if (client) g_object_unref (client); return fd; } static gboolean gst_msdk_context_use_vaapi (GstMsdkContext * context) { gint fd; gint maj_ver, min_ver; VADisplay va_dpy = NULL; VAStatus va_status; mfxStatus status; GstMsdkContextPrivate *priv = context->priv; fd = get_device_id (); if (fd < 0) { GST_ERROR ("Couldn't find a drm device node to open"); return FALSE; } va_dpy = vaGetDisplayDRM (fd); if (!va_dpy) { GST_ERROR ("Couldn't get a VA DRM display"); goto failed; } va_status = vaInitialize (va_dpy, &maj_ver, &min_ver); if (va_status != VA_STATUS_SUCCESS) { GST_ERROR ("Couldn't initialize VA DRM display"); goto failed; } status = MFXVideoCORE_SetHandle (priv->session, MFX_HANDLE_VA_DISPLAY, (mfxHDL) va_dpy); if (status != MFX_ERR_NONE) { GST_ERROR ("Setting VAAPI handle failed (%s)", msdk_status_to_string (status)); goto failed; } priv->fd = fd; priv->dpy = va_dpy; return TRUE; failed: if (va_dpy) vaTerminate (va_dpy); close (fd); return FALSE; } #endif static gboolean gst_msdk_context_open (GstMsdkContext * context, gboolean hardware, GstMsdkContextJobType job_type) { GstMsdkContextPrivate *priv = context->priv; priv->job_type = job_type; priv->hardware = hardware; priv->session = msdk_open_session (hardware ? MFX_IMPL_HARDWARE_ANY : MFX_IMPL_SOFTWARE); if (!priv->session) goto failed; #ifndef _WIN32 priv->fd = -1; if (hardware) { if (!gst_msdk_context_use_vaapi (context)) goto failed; } #endif return TRUE; failed: msdk_close_session (priv->session); return FALSE; } static void gst_msdk_context_init (GstMsdkContext * context) { GstMsdkContextPrivate *priv = gst_msdk_context_get_instance_private (context); context->priv = priv; g_mutex_init (&priv->mutex); } static void release_child_session (gpointer session) { mfxStatus status; mfxSession _session = session; status = MFXDisjoinSession (_session); if (status != MFX_ERR_NONE) GST_WARNING ("failed to disjoin (%s)", msdk_status_to_string (status)); msdk_close_session (_session); } static void gst_msdk_context_finalize (GObject * obj) { GstMsdkContext *context = GST_MSDK_CONTEXT_CAST (obj); GstMsdkContextPrivate *priv = context->priv; /* child sessions will be closed when the parent session is closed */ if (priv->is_joined) goto done; else g_list_free_full (priv->child_session_list, release_child_session); msdk_close_session (priv->session); g_mutex_clear (&priv->mutex); #ifndef _WIN32 if (priv->dpy) vaTerminate (priv->dpy); if (priv->fd >= 0) close (priv->fd); #endif done: G_OBJECT_CLASS (parent_class)->finalize (obj); } static void gst_msdk_context_class_init (GstMsdkContextClass * klass) { GObjectClass *const g_object_class = G_OBJECT_CLASS (klass); g_object_class->finalize = gst_msdk_context_finalize; } GstMsdkContext * gst_msdk_context_new (gboolean hardware, GstMsdkContextJobType job_type) { GstMsdkContext *obj = g_object_new (GST_TYPE_MSDK_CONTEXT, NULL); if (obj && !gst_msdk_context_open (obj, hardware, job_type)) { if (obj) gst_object_unref (obj); return NULL; } return obj; } GstMsdkContext * gst_msdk_context_new_with_parent (GstMsdkContext * parent) { mfxStatus status; GstMsdkContext *obj = g_object_new (GST_TYPE_MSDK_CONTEXT, NULL); GstMsdkContextPrivate *priv = obj->priv; GstMsdkContextPrivate *parent_priv = parent->priv; status = MFXCloneSession (parent_priv->session, &priv->session); if (status != MFX_ERR_NONE) { GST_ERROR ("Failed to clone mfx session"); g_object_unref (obj); return NULL; } priv->is_joined = TRUE; priv->hardware = parent_priv->hardware; priv->job_type = parent_priv->job_type; parent_priv->child_session_list = g_list_prepend (parent_priv->child_session_list, priv->session); #ifndef _WIN32 priv->dpy = parent_priv->dpy; priv->fd = parent_priv->fd; if (priv->hardware) { status = MFXVideoCORE_SetHandle (priv->session, MFX_HANDLE_VA_DISPLAY, (mfxHDL) parent_priv->dpy); if (status != MFX_ERR_NONE) { GST_ERROR ("Setting VA handle failed (%s)", msdk_status_to_string (status)); g_object_unref (obj); return NULL; } } #endif return obj; } mfxSession gst_msdk_context_get_session (GstMsdkContext * context) { return context->priv->session; } gpointer gst_msdk_context_get_handle (GstMsdkContext * context) { #ifndef _WIN32 return context->priv->dpy; #else return NULL; #endif } gint gst_msdk_context_get_fd (GstMsdkContext * context) { #ifndef _WIN32 return context->priv->fd; #else return -1; #endif } static gint _find_response (gconstpointer resp, gconstpointer comp_resp) { GstMsdkAllocResponse *cached_resp = (GstMsdkAllocResponse *) resp; mfxFrameAllocResponse *_resp = (mfxFrameAllocResponse *) comp_resp; return cached_resp ? cached_resp->response.mids != _resp->mids : -1; } static inline gboolean _requested_frame_size_is_equal_or_lower (mfxFrameAllocRequest * _req, GstMsdkAllocResponse * cached_resp) { if (((_req->Type & MFX_MEMTYPE_EXPORT_FRAME) && _req->Info.Width == cached_resp->request.Info.Width && _req->Info.Height == cached_resp->request.Info.Height) || (!(_req->Type & MFX_MEMTYPE_EXPORT_FRAME) && _req->Info.Width <= cached_resp->request.Info.Width && _req->Info.Height <= cached_resp->request.Info.Height)) return TRUE; return FALSE; } static gint _find_request (gconstpointer resp, gconstpointer req) { GstMsdkAllocResponse *cached_resp = (GstMsdkAllocResponse *) resp; mfxFrameAllocRequest *_req = (mfxFrameAllocRequest *) req; /* Confirm if it's under the size of the cached response */ if (_req->NumFrameSuggested <= cached_resp->request.NumFrameSuggested && _requested_frame_size_is_equal_or_lower (_req, cached_resp)) return _req->Type & cached_resp-> request.Type & MFX_MEMTYPE_FROM_DECODE ? 0 : -1; return -1; } GstMsdkAllocResponse * gst_msdk_context_get_cached_alloc_responses (GstMsdkContext * context, mfxFrameAllocResponse * resp) { GstMsdkContextPrivate *priv = context->priv; GList *l = g_list_find_custom (priv->cached_alloc_responses, resp, _find_response); if (l) return l->data; else return NULL; } GstMsdkAllocResponse * gst_msdk_context_get_cached_alloc_responses_by_request (GstMsdkContext * context, mfxFrameAllocRequest * req) { GstMsdkContextPrivate *priv = context->priv; GList *l = g_list_find_custom (priv->cached_alloc_responses, req, _find_request); if (l) return l->data; else return NULL; } static void create_surfaces (GstMsdkContext * context, GstMsdkAllocResponse * resp) { gint i; mfxMemId *mem_id; mfxFrameSurface1 *surface; for (i = 0; i < resp->response.NumFrameActual; i++) { mem_id = resp->response.mids[i]; surface = (mfxFrameSurface1 *) g_slice_new0 (mfxFrameSurface1); if (!surface) { GST_ERROR ("failed to allocate surface"); break; } surface->Data.MemId = mem_id; resp->surfaces_avail = g_list_prepend (resp->surfaces_avail, surface); } } static void free_surface (gpointer surface) { g_slice_free1 (sizeof (mfxFrameSurface1), surface); } static void remove_surfaces (GstMsdkContext * context, GstMsdkAllocResponse * resp) { g_list_free_full (resp->surfaces_used, free_surface); g_list_free_full (resp->surfaces_avail, free_surface); g_list_free_full (resp->surfaces_locked, free_surface); } void gst_msdk_context_add_alloc_response (GstMsdkContext * context, GstMsdkAllocResponse * resp) { context->priv->cached_alloc_responses = g_list_prepend (context->priv->cached_alloc_responses, resp); create_surfaces (context, resp); } gboolean gst_msdk_context_remove_alloc_response (GstMsdkContext * context, mfxFrameAllocResponse * resp) { GstMsdkAllocResponse *msdk_resp; GstMsdkContextPrivate *priv = context->priv; GList *l = g_list_find_custom (priv->cached_alloc_responses, resp, _find_response); if (!l) return FALSE; msdk_resp = l->data; remove_surfaces (context, msdk_resp); g_slice_free1 (sizeof (GstMsdkAllocResponse), msdk_resp); priv->cached_alloc_responses = g_list_delete_link (priv->cached_alloc_responses, l); return TRUE; } static gboolean check_surfaces_available (GstMsdkContext * context, GstMsdkAllocResponse * resp) { GList *l; mfxFrameSurface1 *surface = NULL; GstMsdkContextPrivate *priv = context->priv; gboolean ret = FALSE; g_mutex_lock (&priv->mutex); for (l = resp->surfaces_locked; l;) { surface = l->data; l = l->next; if (!surface->Data.Locked) { resp->surfaces_locked = g_list_remove (resp->surfaces_locked, surface); resp->surfaces_avail = g_list_prepend (resp->surfaces_avail, surface); ret = TRUE; } } g_mutex_unlock (&priv->mutex); return ret; } /* * There are 3 lists here in GstMsdkContext as the following: * 1. surfaces_avail : surfaces which are free and unused anywhere * 2. surfaces_used : surfaces coupled with a gst buffer and being used now. * 3. surfaces_locked : surfaces still locked even after the gst buffer is released. * * Note that they need to be protected by mutex to be thread-safe. */ mfxFrameSurface1 * gst_msdk_context_get_surface_available (GstMsdkContext * context, mfxFrameAllocResponse * resp) { GList *l; mfxFrameSurface1 *surface = NULL; GstMsdkAllocResponse *msdk_resp = gst_msdk_context_get_cached_alloc_responses (context, resp); gint retry = 0; GstMsdkContextPrivate *priv = context->priv; retry: g_mutex_lock (&priv->mutex); for (l = msdk_resp->surfaces_avail; l;) { surface = l->data; l = l->next; if (!surface->Data.Locked) { msdk_resp->surfaces_avail = g_list_remove (msdk_resp->surfaces_avail, surface); msdk_resp->surfaces_used = g_list_prepend (msdk_resp->surfaces_used, surface); break; } } g_mutex_unlock (&priv->mutex); /* * If a msdk context is shared by multiple msdk elements, * upstream msdk element sometimes needs to wait for a gst buffer * to be released in downstream. * * Poll the pool for a maximum of 20 milisecnds. * * FIXME: Is there any better way to handle this case? */ if (!surface && retry < 20) { /* If there's no surface available, find unlocked surfaces in the locked list, * take it back to the available list and then search again. */ check_surfaces_available (context, msdk_resp); retry++; g_usleep (1000); goto retry; } return surface; } void gst_msdk_context_put_surface_locked (GstMsdkContext * context, mfxFrameAllocResponse * resp, mfxFrameSurface1 * surface) { GstMsdkContextPrivate *priv = context->priv; GstMsdkAllocResponse *msdk_resp = gst_msdk_context_get_cached_alloc_responses (context, resp); g_mutex_lock (&priv->mutex); if (!g_list_find (msdk_resp->surfaces_locked, surface)) { msdk_resp->surfaces_used = g_list_remove (msdk_resp->surfaces_used, surface); msdk_resp->surfaces_locked = g_list_prepend (msdk_resp->surfaces_locked, surface); } g_mutex_unlock (&priv->mutex); } void gst_msdk_context_put_surface_available (GstMsdkContext * context, mfxFrameAllocResponse * resp, mfxFrameSurface1 * surface) { GstMsdkContextPrivate *priv = context->priv; GstMsdkAllocResponse *msdk_resp = gst_msdk_context_get_cached_alloc_responses (context, resp); g_mutex_lock (&priv->mutex); if (!g_list_find (msdk_resp->surfaces_avail, surface)) { msdk_resp->surfaces_used = g_list_remove (msdk_resp->surfaces_used, surface); msdk_resp->surfaces_avail = g_list_prepend (msdk_resp->surfaces_avail, surface); } g_mutex_unlock (&priv->mutex); } GstMsdkContextJobType gst_msdk_context_get_job_type (GstMsdkContext * context) { return context->priv->job_type; } void gst_msdk_context_add_job_type (GstMsdkContext * context, GstMsdkContextJobType job_type) { context->priv->job_type |= job_type; } gint gst_msdk_context_get_shared_async_depth (GstMsdkContext * context) { return context->priv->shared_async_depth; } void gst_msdk_context_add_shared_async_depth (GstMsdkContext * context, gint async_depth) { context->priv->shared_async_depth += async_depth; } void gst_msdk_context_set_frame_allocator (GstMsdkContext * context, mfxFrameAllocator * allocator) { GstMsdkContextPrivate *priv = context->priv; g_mutex_lock (&priv->mutex); if (!priv->has_frame_allocator) { mfxStatus status; status = MFXVideoCORE_SetFrameAllocator (priv->session, allocator); if (status != MFX_ERR_NONE) GST_ERROR ("Failed to set frame allocator"); else priv->has_frame_allocator = 1; } g_mutex_unlock (&priv->mutex); }