wpe: Base wpe audio implementation on a web extension

This makes the implementation simpler and enable us to map
webviews and audio stream much more easily

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2252>
This commit is contained in:
Thibault Saunier 2021-04-19 20:46:46 -04:00 committed by GStreamer Marge Bot
parent 81ced7932f
commit a92d4373ad
12 changed files with 578 additions and 101 deletions

View file

@ -22,6 +22,8 @@
#endif
#include "WPEThreadedView.h"
#include "gstwpe.h"
#include "gstwpesrcbin.h"
#include <gst/gl/gl.h>
#include <gst/gl/egl/gsteglimage.h>
@ -165,6 +167,56 @@ gpointer WPEContextThread::s_viewThread(gpointer data)
return nullptr;
}
#ifdef G_OS_UNIX
static void
initialize_web_extensions (WebKitWebContext *context)
{
webkit_web_context_set_web_extensions_directory (context, gst_wpe_get_extension_path ());
}
static gboolean
webkit_extension_msg_received (WebKitWebContext *context,
WebKitUserMessage *message,
GstWpeSrc *src)
{
const gchar *name = webkit_user_message_get_name (message);
GVariant *params = webkit_user_message_get_parameters (message);
gboolean res = TRUE;
if (!g_strcmp0(name, "gstwpe.new_stream")) {
guint32 id = g_variant_get_uint32 (g_variant_get_child_value (params, 0));
const gchar *capsstr = g_variant_get_string (g_variant_get_child_value (params, 1), NULL);
GstCaps *caps = gst_caps_from_string (capsstr);
const gchar *stream_id = g_variant_get_string (g_variant_get_child_value (params, 2), NULL);
gst_wpe_src_new_audio_stream(src, id, caps, stream_id);
gst_caps_unref (caps);
} else if (!g_strcmp0(name, "gstwpe.set_shm")) {
auto fdlist = webkit_user_message_get_fd_list (message);
gint id = g_variant_get_uint32 (g_variant_get_child_value (params, 0));
gst_wpe_src_set_audio_shm (src, fdlist, id);
} else if (!g_strcmp0(name, "gstwpe.new_buffer")) {
guint32 id = g_variant_get_uint32 (g_variant_get_child_value (params, 0));
guint64 size = g_variant_get_uint64 (g_variant_get_child_value (params, 1));
gst_wpe_src_push_audio_buffer (src, id, size);
webkit_user_message_send_reply(message, webkit_user_message_new ("gstwpe.buffer_processed", NULL));
} else if (!g_strcmp0(name, "gstwpe.pause")) {
guint32 id = g_variant_get_uint32 (params);
gst_wpe_src_pause_audio_stream (src, id);
} else if (!g_strcmp0(name, "gstwpe.stop")) {
guint32 id = g_variant_get_uint32 (params);
gst_wpe_src_stop_audio_stream (src, id);
} else {
res = FALSE;
g_error("Unknown event: %s", name);
}
return res;
}
#endif
WPEView* WPEContextThread::createWPEView(GstWpeVideoSrc* src, GstGLContext* context, GstGLDisplay* display, int width, int height)
{
GST_DEBUG("context %p display %p, size (%d,%d)", context, display, width, height);
@ -179,13 +231,11 @@ WPEView* WPEContextThread::createWPEView(GstWpeVideoSrc* src, GstGLContext* cont
WPEView* view = nullptr;
dispatch([&]() mutable {
if (!glib.web_context) {
auto* manager = webkit_website_data_manager_new_ephemeral();
glib.web_context = webkit_web_context_new_with_website_data_manager(manager);
g_object_unref(manager);
}
auto* manager = webkit_website_data_manager_new_ephemeral();
auto web_context = webkit_web_context_new_with_website_data_manager(manager);
g_object_unref(manager);
view = new WPEView(glib.web_context, src, context, display, width, height);
view = new WPEView(web_context, src, context, display, width, height);
});
if (view && view->hasUri()) {
@ -233,6 +283,26 @@ static void s_loadProgressChaned(GObject* object, GParamSpec*, gpointer data)
WPEView::WPEView(WebKitWebContext* web_context, GstWpeVideoSrc* src, GstGLContext* context, GstGLDisplay* display, int width, int height)
{
#ifdef G_OS_UNIX
{
GstObject *parent = gst_object_get_parent (GST_OBJECT (src));
if (parent && GST_IS_WPE_SRC (parent)) {
audio.init_ext_sigid = g_signal_connect (web_context,
"initialize-web-extensions",
G_CALLBACK (initialize_web_extensions),
NULL);
audio.extension_msg_sigid = g_signal_connect (web_context,
"user-message-received",
G_CALLBACK (webkit_extension_msg_received),
parent);
GST_INFO_OBJECT (parent, "Enabled audio");
}
gst_clear_object (&parent);
}
#endif // G_OS_UNIX
g_mutex_init(&threading.ready_mutex);
g_cond_init(&threading.ready_cond);
threading.ready = FALSE;
@ -353,6 +423,15 @@ WPEView::~WPEView()
if (shm_committed)
gst_buffer_unref (shm_committed);
if (audio.init_ext_sigid) {
WebKitWebContext* web_context = webkit_web_view_get_context (webkit.view);
g_signal_handler_disconnect(web_context, audio.init_ext_sigid);
g_signal_handler_disconnect(web_context, audio.extension_msg_sigid);
audio.init_ext_sigid = 0;
audio.extension_msg_sigid = 0;
}
WPEContextThread::singleton().dispatch([&]() {
if (webkit.view) {
g_object_unref(webkit.view);
@ -523,11 +602,6 @@ void WPEView::setDrawBackground(gboolean drawsBackground)
webkit_web_view_set_background_color(webkit.view, &color);
}
void WPEView::registerAudioReceiver(const struct wpe_audio_receiver* audioReceiver, gpointer userData)
{
wpe_audio_register_receiver(audioReceiver, userData);
}
void WPEView::releaseImage(gpointer imagePointer)
{
s_view->dispatch([&]() {

View file

@ -23,7 +23,6 @@
#include <glib.h>
#include <gst/gl/gstglfuncs.h>
#include <gst/gl/egl/gstgldisplay_egl.h>
#include <wpe/extensions/audio.h>
#include <wpe/fdo.h>
#include <wpe/fdo-egl.h>
#include <wpe/webkit.h>
@ -52,8 +51,6 @@ public:
void loadData(GBytes*);
void setDrawBackground(gboolean);
void registerAudioReceiver(const struct wpe_audio_receiver*, gpointer);
GstEGLImage* image();
GstBuffer* buffer();
@ -129,6 +126,11 @@ private:
GstBuffer* committed;
} shm { nullptr, nullptr };
struct {
gulong init_ext_sigid;
gulong extension_msg_sigid;
} audio {0, 0};
};
class WPEContextThread {

View file

@ -65,19 +65,32 @@
#include "gstwpevideosrc.h"
#include "gstwpesrcbin.h"
#include "gstwpe.h"
static gchar *extension_path = NULL;
GST_DEBUG_CATEGORY (wpe_video_src_debug);
GST_DEBUG_CATEGORY (wpe_view_debug);
GST_DEBUG_CATEGORY (wpe_src_debug);
const gchar *gst_wpe_get_extension_path (void)
{
return extension_path;
}
static gboolean
plugin_init (GstPlugin * plugin)
{
gboolean result;
gchar *dirname = g_path_get_dirname (gst_plugin_get_filename (plugin));
GST_DEBUG_CATEGORY_INIT (wpe_video_src_debug, "wpevideosrc", 0, "WPE Video Source");
GST_DEBUG_CATEGORY_INIT (wpe_view_debug, "wpeview", 0, "WPE Threaded View");
GST_DEBUG_CATEGORY_INIT (wpe_src_debug, "wpesrc", 0, "WPE Source");
gboolean result = gst_element_register (plugin, "wpevideosrc", GST_RANK_NONE,
extension_path = g_build_filename (dirname, "wpe-extension", NULL);
g_free (dirname);
result = gst_element_register (plugin, "wpevideosrc", GST_RANK_NONE,
GST_TYPE_WPE_VIDEO_SRC);
result &= gst_element_register(plugin, "wpesrc", GST_RANK_NONE, GST_TYPE_WPE_SRC);
return result;

View file

@ -21,4 +21,4 @@
#include <gst/gst.h>
void gst_wpe_video_src_register_audio_receiver(GstElement*, const struct wpe_audio_receiver*, gpointer);
const gchar *gst_wpe_get_extension_path (void);

View file

@ -36,13 +36,16 @@
#include "gstwpesrcbin.h"
#include "gstwpevideosrc.h"
#include "gstwpe-private.h"
#include "gstwpe.h"
#include "WPEThreadedView.h"
#include <gst/allocators/allocators.h>
#include <gst/base/gstflowcombiner.h>
#include <wpe/extensions/audio.h>
#include <sys/mman.h>
#include <unistd.h>
G_DEFINE_TYPE (GstWpeAudioPad, gst_wpe_audio_pad, GST_TYPE_GHOST_PAD);
static void
@ -106,6 +109,11 @@ GST_DEBUG_CATEGORY_EXTERN (wpe_src_debug);
G_DEFINE_TYPE_WITH_CODE (GstWpeSrc, gst_wpe_src, GST_TYPE_BIN,
G_IMPLEMENT_INTERFACE (GST_TYPE_URI_HANDLER, gst_wpe_src_uri_handler_init));
/**
* GstWpeSrc!video
*
* Since: 1.20
*/
static GstStaticPadTemplate video_src_factory =
GST_STATIC_PAD_TEMPLATE ("video", GST_PAD_SRC, GST_PAD_SOMETIMES,
GST_STATIC_CAPS ("video/x-raw(memory:GLMemory), "
@ -125,6 +133,14 @@ GST_STATIC_PAD_TEMPLATE ("video", GST_PAD_SRC, GST_PAD_SOMETIMES,
#endif
));
/**
* GstWpeSrc!audio_%u
*
* Each audio stream in the renderer web page will expose the and `audio_%u`
* #GstPad.
*
* Since: 1.20
*/
static GstStaticPadTemplate audio_src_factory =
GST_STATIC_PAD_TEMPLATE ("audio_%u", GST_PAD_SRC, GST_PAD_SOMETIMES,
GST_STATIC_CAPS ( \
@ -149,18 +165,15 @@ gst_wpe_src_chain_buffer (GstPad * pad, GstObject * parent, GstBuffer * buffer)
return result;
}
static void
on_audio_receiver_handle_start(void* data, uint32_t id, int32_t channels, const char* format, int32_t sampleRate)
void
gst_wpe_src_new_audio_stream(GstWpeSrc *src, guint32 id, GstCaps *caps, const gchar *stream_id)
{
GstWpeSrc* src = GST_WPE_SRC (data);
GstWpeAudioPad *audio_pad;
GstPad *pad;
gchar *name;
GstEvent *stream_start;
GstSegment segment;
GstCaps *caps;
GST_DEBUG_OBJECT (src, "Exposing audio pad for stream %u", id);
name = g_strdup_printf ("audio_%u", id);
audio_pad = gst_wpe_audio_pad_new (name);
pad = GST_PAD_CAST (audio_pad);
@ -170,19 +183,13 @@ on_audio_receiver_handle_start(void* data, uint32_t id, int32_t channels, const
gst_element_add_pad (GST_ELEMENT_CAST (src), pad);
gst_flow_combiner_add_pad (src->flow_combiner, pad);
name = gst_pad_create_stream_id_printf(pad, GST_ELEMENT_CAST (src), "%03u", id);
stream_start = gst_event_new_stream_start (name);
gst_pad_push_event (pad, stream_start);
g_free (name);
GST_DEBUG_OBJECT (src, "Adding pad: %" GST_PTR_FORMAT, pad);
stream_start = gst_event_new_stream_start (stream_id);
gst_pad_push_event (pad, stream_start);
caps = gst_caps_new_simple ("audio/x-raw", "format", G_TYPE_STRING, format,
"rate", G_TYPE_INT, sampleRate,
"channels", G_TYPE_INT, channels,
"channel-mask", GST_TYPE_BITMASK, gst_audio_channel_get_fallback_mask (channels),
"layout", G_TYPE_STRING, "interleaved", NULL);
gst_audio_info_from_caps (&audio_pad->info, caps);
gst_pad_push_event (pad, gst_event_new_caps (caps));
gst_caps_unref (caps);
gst_segment_init (&segment, GST_FORMAT_TIME);
gst_pad_push_event (pad, gst_event_new_segment (&segment));
@ -190,23 +197,40 @@ on_audio_receiver_handle_start(void* data, uint32_t id, int32_t channels, const
g_hash_table_insert (src->audio_src_pads, GUINT_TO_POINTER (id), audio_pad);
}
static void
on_audio_receiver_handle_packet(void* data, struct wpe_audio_packet_export* packet_export, uint32_t id, int32_t fd, uint32_t size)
void
gst_wpe_src_set_audio_shm (GstWpeSrc* src, GUnixFDList *fds, guint32 id)
{
gint fd;
GstWpeAudioPad *audio_pad = GST_WPE_AUDIO_PAD (g_hash_table_lookup (src->audio_src_pads, GUINT_TO_POINTER (id)));
g_return_if_fail (GST_IS_WPE_SRC (src));
g_return_if_fail (fds);
g_return_if_fail (g_unix_fd_list_get_length (fds) == 1);
g_return_if_fail (audio_pad->fd <= 0);
fd = g_unix_fd_list_get (fds, 0, NULL);
g_return_if_fail (fd >= 0);
if (audio_pad->fd > 0)
close(audio_pad->fd);
audio_pad->fd = dup(fd);
}
void
gst_wpe_src_push_audio_buffer (GstWpeSrc* src, guint32 id, guint64 size)
{
GstWpeSrc* src = GST_WPE_SRC (data);
GstWpeAudioPad *audio_pad = GST_WPE_AUDIO_PAD (g_hash_table_lookup (src->audio_src_pads, GUINT_TO_POINTER (id)));
GstPad *pad = GST_PAD_CAST (audio_pad);
GstBuffer *buffer;
GstClock *clock;
g_return_if_fail (GST_IS_PAD (pad));
g_return_if_fail (fd >= 0);
g_return_if_fail (audio_pad->fd > 0);
GST_TRACE_OBJECT (pad, "Handling incoming audio packet");
buffer = gst_buffer_new ();
GST_TRACE_OBJECT (audio_pad, "Handling incoming audio packet");
GstMemory *mem = gst_fd_allocator_alloc (src->fd_allocator, dup (fd), size, GST_FD_MEMORY_FLAG_KEEP_MAPPED);
gst_buffer_append_memory (buffer, mem);
gpointer data = mmap (0, size, PROT_READ, MAP_PRIVATE, audio_pad->fd, 0);
buffer = gst_buffer_new_wrapped (g_memdup(data, size), size);
munmap (data, size);
gst_buffer_add_audio_meta (buffer, &audio_pad->info, size, NULL);
clock = gst_element_get_clock (GST_ELEMENT_CAST (src));
@ -231,30 +255,33 @@ on_audio_receiver_handle_packet(void* data, struct wpe_audio_packet_export* pack
audio_pad->discont_pending = FALSE;
}
gst_flow_combiner_update_pad_flow (src->flow_combiner, pad, gst_pad_push (pad, buffer));
wpe_audio_packet_export_release (packet_export);
close (fd);
gst_flow_combiner_update_pad_flow (src->flow_combiner, GST_PAD (audio_pad),
gst_pad_push (GST_PAD_CAST (audio_pad), buffer));
}
static void
on_audio_receiver_handle_stop(void* data, uint32_t id)
gst_wpe_src_remove_audio_pad (GstWpeSrc *src, GstPad *pad)
{
GstWpeSrc* src = GST_WPE_SRC (data);
GstWpeAudioPad *audio_pad = GST_WPE_AUDIO_PAD (g_hash_table_lookup (src->audio_src_pads, GUINT_TO_POINTER (id)));
GstPad *pad = GST_PAD_CAST (audio_pad);
GST_DEBUG_OBJECT (src, "Removing pad: %" GST_PTR_FORMAT, pad);
gst_element_remove_pad (GST_ELEMENT_CAST (src), pad);
gst_flow_combiner_remove_pad (src->flow_combiner, pad);
}
void
gst_wpe_src_stop_audio_stream(GstWpeSrc* src, guint32 id)
{
GstPad *pad = GST_PAD (g_hash_table_lookup (src->audio_src_pads, GUINT_TO_POINTER (id)));
g_return_if_fail (GST_IS_PAD (pad));
GST_INFO_OBJECT(pad, "Stopping");
gst_pad_push_event (pad, gst_event_new_eos ());
gst_element_remove_pad (GST_ELEMENT_CAST (src), pad);
gst_flow_combiner_remove_pad (src->flow_combiner, pad);
gst_wpe_src_remove_audio_pad (src, pad);
g_hash_table_remove (src->audio_src_pads, GUINT_TO_POINTER (id));
}
static void
on_audio_receiver_handle_pause(void* data, uint32_t id)
void
gst_wpe_src_pause_audio_stream(GstWpeSrc* src, guint32 id)
{
GstWpeSrc* src = GST_WPE_SRC (data);
GstWpeAudioPad *audio_pad = GST_WPE_AUDIO_PAD (g_hash_table_lookup (src->audio_src_pads, GUINT_TO_POINTER (id)));
GstPad *pad = GST_PAD_CAST (audio_pad);
g_return_if_fail (GST_IS_PAD (pad));
@ -265,26 +292,6 @@ on_audio_receiver_handle_pause(void* data, uint32_t id)
audio_pad->discont_pending = TRUE;
}
static void
on_audio_receiver_handle_resume(void* data, uint32_t id)
{
GstWpeSrc* src = GST_WPE_SRC (data);
GstWpeAudioPad *audio_pad = GST_WPE_AUDIO_PAD (g_hash_table_lookup (src->audio_src_pads, GUINT_TO_POINTER (id)));
GstPad *pad = GST_PAD_CAST (audio_pad);
g_return_if_fail (GST_IS_PAD (pad));
GST_INFO_OBJECT(pad, "Resuming");
}
static const struct wpe_audio_receiver audio_receiver = {
.handle_start = on_audio_receiver_handle_start,
.handle_packet = on_audio_receiver_handle_packet,
.handle_stop = on_audio_receiver_handle_stop,
.handle_pause = on_audio_receiver_handle_pause,
.handle_resume = on_audio_receiver_handle_resume
};
static void
gst_wpe_src_load_bytes (GstWpeVideoSrc * src, GBytes * bytes)
{
@ -366,9 +373,14 @@ static gchar *
gst_wpe_src_get_uri (GstURIHandler * handler)
{
GstWpeSrc *src = GST_WPE_SRC (handler);
const gchar *location;
gchar *location;
gchar *res;
g_object_get (src->video_src, "location", &location, NULL);
return g_strdup_printf ("wpe://%s", location);
res = g_strdup_printf ("wpe://%s", location);
g_free (location);
return res;
}
static gboolean
@ -403,8 +415,14 @@ gst_wpe_src_init (GstWpeSrc * src)
src->audio_src_pads = g_hash_table_new (g_direct_hash, g_direct_equal);
src->flow_combiner = gst_flow_combiner_new ();
src->video_src = gst_element_factory_make ("wpevideosrc", NULL);
}
gst_wpe_video_src_register_audio_receiver (src->video_src, &audio_receiver, src);
static gboolean
gst_wpe_audio_remove_audio_pad (gint32 *id, GstPad *pad, GstWpeSrc *self)
{
gst_wpe_src_remove_audio_pad (self, pad);
return TRUE;
}
static GstStateChangeReturn
@ -418,6 +436,7 @@ gst_wpe_src_change_state (GstElement * element, GstStateChange transition)
switch (transition) {
case GST_STATE_CHANGE_PAUSED_TO_READY:{
g_hash_table_foreach_remove (src->audio_src_pads, (GHRFunc) gst_wpe_audio_remove_audio_pad, src);
gst_flow_combiner_reset (src->flow_combiner);
break;
}
@ -459,7 +478,8 @@ gst_wpe_src_class_init (GstWpeSrcClass * klass)
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
gst_element_class_set_static_metadata (element_class, "WPE source",
"Source/Video/Audio", "Creates a video stream from a WPE browser",
"Source/Video/Audio", "Creates Audio/Video streams from a web"
" page using WPE web engine",
"Philippe Normand <philn@igalia.com>, Žan Doberšek "
"<zdobersek@igalia.com>");

View file

@ -23,6 +23,10 @@
#include <gst/gst.h>
#include <gst/audio/audio.h>
#ifdef G_OS_UNIX
#include <gio/gunixfdlist.h>
#endif
G_BEGIN_DECLS
GType gst_wpe_audio_pad_get_type(void);
@ -43,6 +47,7 @@ struct _GstWpeAudioPad
GstAudioInfo info;
GstClockTime buffer_time;
gboolean discont_pending;
gint fd;
};
struct _GstWpeAudioPadClass
@ -67,4 +72,10 @@ struct _GstWpeSrcClass
GType gst_wpe_src_get_type (void);
void gst_wpe_src_new_audio_stream(GstWpeSrc *src, guint32 id, GstCaps *caps, const gchar *stream_id);
void gst_wpe_src_set_audio_shm (GstWpeSrc* src, GUnixFDList *fds, guint32 id);
void gst_wpe_src_push_audio_buffer (GstWpeSrc* src, guint32 id, guint64 size);
void gst_wpe_src_pause_audio_stream (GstWpeSrc* src, guint32 id);
void gst_wpe_src_stop_audio_stream (GstWpeSrc* src, guint32 id);
G_END_DECLS

View file

@ -75,7 +75,6 @@
/*
* TODO:
* - Audio support (requires an AudioSession implementation in WebKit and a WPEBackend-fdo API for it)
* - DMABuf support (requires changes in WPEBackend-fdo to expose DMABuf planes and fds)
* - Custom EGLMemory allocator
* - Better navigation events handling (would require a new GstNavigation API)
@ -86,7 +85,6 @@
#endif
#include "gstwpevideosrc.h"
#include "gstwpe-private.h"
#include <gst/gl/gl.h>
#include <gst/gl/egl/gstglmemoryegl.h>
#include <gst/gl/wayland/gstgldisplay_wayland.h>
@ -129,8 +127,6 @@ struct _GstWpeVideoSrc
gint64 n_frames; /* total frames sent */
WPEView *view;
const struct wpe_audio_receiver *audio_receiver;
gpointer audio_receiver_data;
GMutex lock;
};
@ -299,11 +295,6 @@ gst_wpe_video_src_start (GstWpeVideoSrc * src)
if (created_view) {
src->n_frames = 0;
if (src->audio_receiver) {
src->view->registerAudioReceiver(src->audio_receiver, src->audio_receiver_data);
src->audio_receiver = NULL,
src->audio_receiver_data = NULL;
}
}
WPE_UNLOCK (src);
return TRUE;
@ -743,17 +734,4 @@ gst_wpe_video_src_class_init (GstWpeVideoSrcClass * klass)
static_cast < GSignalFlags > (G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
G_CALLBACK (gst_wpe_video_src_load_bytes), NULL, NULL, NULL,
G_TYPE_NONE, 1, G_TYPE_BYTES);
}
void
gst_wpe_video_src_register_audio_receiver(GstElement* video_src, const struct wpe_audio_receiver* receiver, gpointer user_data)
{
GstWpeVideoSrc* src = GST_WPE_VIDEO_SOURCE(video_src);
if (!src->view) {
src->audio_receiver = receiver;
src->audio_receiver_data = user_data;
return;
}
src->view->registerAudioReceiver(receiver, user_data);
}

View file

@ -16,12 +16,17 @@ if not wpe_dep.found() or not wpe_fdo_dep.found() or not egl_dep.found() or not
subdir_done()
endif
giounix_dep = dependency('gio-unix-2.0', required: false)
gstwpe = library('gstwpe',
['WPEThreadedView.cpp', 'gstwpe.cpp', 'gstwpevideosrc.cpp', 'gstwpesrcbin.cpp'],
dependencies : [egl_dep, wpe_dep, wpe_fdo_dep, gstallocators_dep, gstaudio_dep, gstvideo_dep, gstbase_dep, gstgl_dep, xkbcommon_dep, wl_server_dep],
dependencies : [egl_dep, wpe_dep, wpe_fdo_dep, gstallocators_dep, gstaudio_dep, gstvideo_dep, gstbase_dep, gstgl_dep, xkbcommon_dep, wl_server_dep, giounix_dep],
cpp_args : gst_plugins_bad_args + ['-DHAVE_CONFIG_H=1'],
include_directories : [configinc],
install : true,
install_dir : plugins_install_dir)
if giounix_dep.found()
subdir('wpe-extension')
endif
pkgconfig.generate(gstwpe, install_dir : plugins_pkgconfig_install_dir)
plugins += [gstwpe]

View file

@ -0,0 +1,278 @@
/* Copyright (C) <2020> Philippe Normand <philn@igalia.com>
* Copyright (C) <2021> Thibault Saunier <tsaunier@igalia.com>
*
* This library is free software; you can redistribute it and/or modify it under the terms of the
* GNU Library General Public License as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public License along with this
* library; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
#include "gstwpeextension.h"
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#define gst_wpe_audio_sink_parent_class parent_class
GST_DEBUG_CATEGORY (wpe_audio_sink_debug);
#define GST_CAT_DEFAULT wpe_audio_sink_debug
struct _GstWpeAudioSink
{
GstBaseSink parent;
guint32 id;
GCancellable *cancellable;;
gchar *caps;
GMutex buf_lock;
GCond buf_cond;
GUnixFDList *fdlist;
};
static guint id = -1; /* atomic */
G_DEFINE_TYPE_WITH_CODE (GstWpeAudioSink, gst_wpe_audio_sink,
GST_TYPE_BASE_SINK, GST_DEBUG_CATEGORY_INIT (wpe_audio_sink_debug,
"wpeaudio_sink", 0, "WPE Sink"););
static GstStaticPadTemplate audio_sink_factory =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("audio/x-raw"));
static void
message_consumed_cb (GObject * source_object, GAsyncResult * res,
GstWpeAudioSink * self)
{
g_mutex_lock (&self->buf_lock);
g_cond_broadcast (&self->buf_cond);
g_mutex_unlock (&self->buf_lock);
}
static GstFlowReturn
render (GstBaseSink * sink, GstBuffer * buf)
{
gsize written_bytes;
static int init = 0;
char filename[1024];
const gint *fds;
WebKitUserMessage *msg;
GstMapInfo info;
GstWpeAudioSink *self = GST_WPE_AUDIO_SINK (sink);
if (!self->caps) {
GST_ELEMENT_ERROR (self, CORE, NEGOTIATION,
("Trying to render buffer before caps were set"), (NULL));
return GST_FLOW_ERROR;
}
if (!gst_buffer_map (buf, &info, GST_MAP_READ)) {
GST_ELEMENT_ERROR (self, RESOURCE, READ, ("Failed to map input buffer"),
(NULL));
return GST_FLOW_ERROR;
}
#ifdef HAVE_MEMFD_CREATE
if (!self->fdlist) {
gint fds[1] = { -1 };
fds[0] = memfd_create ("gstwpe-shm", MFD_CLOEXEC);
#endif
if (fds[0] < 0) {
/* allocate shm pool */
snprintf (filename, 1024, "%s/%s-%d-%s", g_get_user_runtime_dir (),
"gstwpe-shm", init++, "XXXXXX");
fds[0] = g_mkstemp (filename);
if (fds[0] < 0) {
gst_buffer_unmap (buf, &info);
GST_ELEMENT_ERROR (self, RESOURCE, READ,
("opening temp file %s failed: %s", filename, strerror (errno)),
(NULL));
return GST_FLOW_ERROR;
}
unlink (filename);
}
if (fds[0] <= 0)
goto write_error;
self->fdlist = g_unix_fd_list_new_from_array (fds, 1);
msg = webkit_user_message_new_with_fd_list ("gstwpe.set_shm",
g_variant_new ("(u)", self->id), self->fdlist);
gst_wpe_extension_send_message (msg, self->cancellable, NULL, NULL);
}
fds = g_unix_fd_list_peek_fds (self->fdlist, NULL);
if (ftruncate (fds[0], info.size) == -1)
goto write_error;
written_bytes = write (fds[0], info.data, info.size);
if (written_bytes < 0)
goto write_error;
if (written_bytes != info.size)
goto write_error;
if (lseek (fds[0], 0, SEEK_SET) == -1)
goto write_error;
msg = webkit_user_message_new ("gstwpe.new_buffer",
g_variant_new ("(ut)", self->id, info.size));
g_mutex_lock (&self->buf_lock);
gst_wpe_extension_send_message (msg, self->cancellable,
(GAsyncReadyCallback) message_consumed_cb, self);
g_cond_wait (&self->buf_cond, &self->buf_lock);
g_mutex_unlock (&self->buf_lock);
gst_buffer_unmap (buf, &info);
return GST_FLOW_OK;
write_error:
gst_buffer_unmap (buf, &info);
GST_ELEMENT_ERROR (self, RESOURCE, WRITE, ("Couldn't write memfd: %s",
strerror (errno)), (NULL));
return GST_FLOW_ERROR;
}
static gboolean
set_caps (GstBaseSink * sink, GstCaps * caps)
{
GstWpeAudioSink *self = GST_WPE_AUDIO_SINK (sink);
gchar *stream_id;
if (self->caps) {
GST_ERROR_OBJECT (sink, "Renegotiation is not supported yet");
return FALSE;
}
self->caps = gst_caps_to_string (caps);
g_atomic_int_inc (&id);
self->id = g_atomic_int_get (&id);
stream_id = gst_pad_get_stream_id (GST_BASE_SINK_PAD (sink));
gst_wpe_extension_send_message (webkit_user_message_new ("gstwpe.new_stream",
g_variant_new ("(uss)", self->id, self->caps, stream_id)),
self->cancellable, NULL, NULL);
g_free (stream_id);
return TRUE;
}
static gboolean
unlock (GstBaseSink * sink)
{
GstWpeAudioSink *self = GST_WPE_AUDIO_SINK (sink);
g_cancellable_cancel (self->cancellable);
g_mutex_lock (&self->buf_lock);
g_cond_broadcast (&self->buf_cond);
g_mutex_unlock (&self->buf_lock);
return TRUE;
}
static gboolean
stop (GstBaseSink * sink)
{
GstWpeAudioSink *self = GST_WPE_AUDIO_SINK (sink);
if (!self->caps) {
GST_DEBUG_OBJECT (sink, "Stopped before started");
return TRUE;
}
/* Stop processing and claim buffers back */
unlock (sink);
GST_DEBUG_OBJECT (sink, "Stopping %d", self->id);
gst_wpe_extension_send_message (webkit_user_message_new ("gstwpe.stop",
g_variant_new_uint32 (self->id)), self->cancellable, NULL, NULL);
return TRUE;
}
static GstStateChangeReturn
change_state (GstElement * element, GstStateChange transition)
{
GstWpeAudioSink *self = GST_WPE_AUDIO_SINK (element);
switch (transition) {
case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
gst_wpe_extension_send_message (webkit_user_message_new ("gstwpe.pause",
g_variant_new_uint32 (self->id)), self->cancellable, NULL, NULL);
break;
default:
break;
}
return GST_CALL_PARENT_WITH_DEFAULT (GST_ELEMENT_CLASS,
change_state, (element, transition), GST_STATE_CHANGE_SUCCESS);
}
static void
dispose (GObject * object)
{
GstWpeAudioSink *self = GST_WPE_AUDIO_SINK (object);
g_clear_object (&self->cancellable);
g_clear_pointer (&self->caps, g_free);
}
static void
gst_wpe_audio_sink_init (GstWpeAudioSink * self)
{
GstElementClass *klass = GST_ELEMENT_GET_CLASS (self);
GstPadTemplate *pad_template =
gst_element_class_get_pad_template (GST_ELEMENT_CLASS (klass), "sink");
g_return_if_fail (pad_template != NULL);
self->cancellable = g_cancellable_new ();
}
static void
gst_wpe_audio_sink_class_init (GstWpeAudioSinkClass * klass)
{
GstPadTemplate *tmpl;
GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);
GObjectClass *object_class = G_OBJECT_CLASS (klass);
GstBaseSinkClass *gstbasesink_class = (GstBaseSinkClass *) klass;
object_class->dispose = dispose;
gst_element_class_set_static_metadata (gstelement_class,
"WPE internal audio sink", "Sink/Audio",
"Internal sink to be used in wpe when running inside gstwpe",
"Thibault Saunier <tsaunier@igalia.com>");
tmpl = gst_static_pad_template_get (&audio_sink_factory);
gst_element_class_add_pad_template (gstelement_class, tmpl);
gstelement_class->change_state = GST_DEBUG_FUNCPTR (change_state);
gstbasesink_class->stop = GST_DEBUG_FUNCPTR (stop);
gstbasesink_class->unlock = GST_DEBUG_FUNCPTR (unlock);
gstbasesink_class->render = GST_DEBUG_FUNCPTR (render);
gstbasesink_class->set_caps = GST_DEBUG_FUNCPTR (set_caps);
}

View file

@ -0,0 +1,63 @@
/* Copyright (C) <2021> Thibault Saunier <tsaunier@igalia.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include "gstwpeextension.h"
#include <errno.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <gst/gst.h>
#include <gmodule.h>
#include <gio/gunixfdlist.h>
#include <wpe/webkit-web-extension.h>
G_MODULE_EXPORT void webkit_web_extension_initialize (WebKitWebExtension *
extension);
static WebKitWebExtension *global_extension = NULL;
void
webkit_web_extension_initialize (WebKitWebExtension * extension)
{
g_return_if_fail (!global_extension);
gst_init (NULL, NULL);
/* Register our own audio sink to */
gst_element_register (NULL, "gstwpeaudiosink", GST_RANK_PRIMARY + 500,
gst_wpe_audio_sink_get_type ());
GST_INFO ("Mark processus as WebProcess");
if (!g_setenv ("GST_WPE_ID", "1", TRUE))
g_error ("Could not set GST_WPE_ID envvar\n");
global_extension = extension;
GST_INFO_OBJECT (global_extension, "Setting as global extension.");
}
void
gst_wpe_extension_send_message (WebKitUserMessage * msg,
GCancellable * cancellable, GAsyncReadyCallback cb, gpointer udata)
{
webkit_web_extension_send_message_to_context (global_extension, msg,
cancellable, cb, udata);
}

View file

@ -0,0 +1,25 @@
/* Copyright (C) <2021> Thibault Saunier <tsaunier@igalia.com>
*
* This library is free software; you can redistribute it and/or modify it under the terms of the
* GNU Library General Public License as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public License along with this
* library; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#pragma once
#include <wpe/webkit-web-extension.h>
#include <gio/gunixfdlist.h>
#include <gst/gst.h>
#include <gst/base/gstbasesink.h>
void gst_wpe_extension_send_message (WebKitUserMessage *msg, GCancellable *cancellable, GAsyncReadyCallback cb, gpointer udata);
G_DECLARE_FINAL_TYPE (GstWpeAudioSink, gst_wpe_audio_sink, GST, WPE_AUDIO_SINK, GstBaseSink);

View file

@ -0,0 +1,8 @@
library('gstwpeextension',
['gstwpeextension.c', 'gstwpeaudiosink.c'],
dependencies : [wpe_dep, gst_dep, gstbase_dep, giounix_dep],
c_args : ['-DHAVE_CONFIG_H=1'],
include_directories : [configinc],
install : true,
install_dir : plugins_install_dir / 'wpe-extension')