/* Copyright (C) <2018> Philippe Normand * Copyright (C) <2018> Žan Doberšek * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ /** * SECTION:element-wpesrc * @title: wpesrc * * The wpesrc element is used to produce a video texture representing a web page * rendered off-screen by WPE. * * Starting from WPEBackend-FDO 1.6.x, software rendering support is available. This * features allows wpesrc to be used on machines without GPU, and/or for testing * purpose. To enable it, set the `LIBGL_ALWAYS_SOFTWARE=true` environment * variable and make sure `video/x-raw, format=BGRA` caps are negotiated by the * wpesrc element. * * ## Example launch lines * * |[ * gst-launch-1.0 -v wpesrc location="https://gstreamer.freedesktop.org" ! queue ! glimagesink * ]| * Shows the GStreamer website homepage * * |[ * LIBGL_ALWAYS_SOFTWARE=true gst-launch-1.0 -v wpesrc num-buffers=50 location="https://gstreamer.freedesktop.org" ! videoconvert ! pngenc ! multifilesink location=/tmp/snapshot-%05d.png * ]| * Saves the first 50 video frames generated for the GStreamer website as PNG files in /tmp. * * |[ * gst-play-1.0 --videosink gtkglsink wpe://https://gstreamer.freedesktop.org * ]| * Shows the GStreamer website homepage as played with GstPlayer in a GTK+ window. * * |[ * gst-launch-1.0 glvideomixer name=m sink_1::zorder=0 ! glimagesink wpesrc location="file:///home/phil/Downloads/plunk/index.html" draw-background=0 ! m. videotestsrc ! queue ! glupload ! glcolorconvert ! m. * ]| * Composite WPE with a video stream in a single OpenGL scene. * * |[ * gst-launch-1.0 glvideomixer name=m sink_1::zorder=0 sink_0::height=818 sink_0::width=1920 ! gtkglsink wpesrc location="file:///home/phil/Downloads/plunk/index.html" draw-background=0 ! m. uridecodebin uri="http://192.168.1.44/Sintel.2010.1080p.mkv" name=d d. ! queue ! glupload ! glcolorconvert ! m. * ]| * Composite WPE with a video stream, sink_0 pad properties have to match the video dimensions. */ /* * TODO: * - Audio support (requires an AudioSession implementation in WebKit and a WPEBackend-fdo API for it) * - DMABuf support (requires changes in WPEBackend-fdo to expose DMABuf planes and fds) * - Custom EGLMemory allocator * - Better navigation events handling (would require a new GstNavigation API) */ #ifdef HAVE_CONFIG_H #include #endif #include "gstwpesrc.h" #include #include #include #include #include #include "WPEThreadedView.h" GST_DEBUG_CATEGORY (wpe_src_debug); #define GST_CAT_DEFAULT wpe_src_debug #define DEFAULT_WIDTH 1920 #define DEFAULT_HEIGHT 1080 #define DEFAULT_FPS_N 30 #define DEFAULT_FPS_D 1 enum { PROP_0, PROP_LOCATION, PROP_DRAW_BACKGROUND }; enum { SIGNAL_CONFIGURE_WEB_VIEW, SIGNAL_LOAD_BYTES, LAST_SIGNAL }; static guint gst_wpe_src_signals[LAST_SIGNAL] = { 0 }; struct _GstWpeSrc { GstGLBaseSrc parent; WPEThreadedView *view; /* properties */ gchar *location; gboolean draw_background; GBytes *bytes; gboolean gl_enabled; }; static void gst_wpe_src_uri_handler_init (gpointer iface, gpointer data); #define gst_wpe_src_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstWpeSrc, gst_wpe_src, GST_TYPE_GL_BASE_SRC, G_IMPLEMENT_INTERFACE (GST_TYPE_URI_HANDLER, gst_wpe_src_uri_handler_init)); static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS ("video/x-raw(memory:GLMemory), " "format = (string) RGBA, " "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE ", " "framerate = " GST_VIDEO_FPS_RANGE ", " "pixel-aspect-ratio = (fraction)1/1, texture-target = (string)2D;" #if ENABLE_SHM_BUFFER_SUPPORT "video/x-raw, " "format = (string) BGRA, " "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE ", " "framerate = " GST_VIDEO_FPS_RANGE ", " "pixel-aspect-ratio = (fraction)1/1" #endif ) ); static GstFlowReturn gst_wpe_src_create (GstBaseSrc * bsrc, guint64 offset, guint length, GstBuffer ** buf) { GstWpeSrc *src = GST_WPE_SRC (bsrc); GstFlowReturn ret = GST_FLOW_ERROR; GstBuffer *locked_buffer; GST_OBJECT_LOCK (src); if (src->gl_enabled) { GST_OBJECT_UNLOCK (src); return GST_CALL_PARENT_WITH_DEFAULT (GST_BASE_SRC_CLASS, create, (bsrc, offset, length, buf), ret); } locked_buffer = src->view->buffer (); if (locked_buffer != NULL) { *buf = gst_buffer_ref (locked_buffer); ret = GST_FLOW_OK; } GST_OBJECT_UNLOCK (src); return ret; } static gboolean gst_wpe_src_fill_memory (GstGLBaseSrc * bsrc, GstGLMemory * memory) { GstWpeSrc *src = GST_WPE_SRC (bsrc); const GstGLFuncs *gl; guint tex_id; GstEGLImage *locked_image; if (!gst_gl_context_check_feature (GST_GL_CONTEXT (bsrc->context), "EGL_KHR_image_base")) { GST_ERROR_OBJECT (src, "EGL_KHR_image_base is not supported"); return FALSE; } GST_OBJECT_LOCK (src); gl = bsrc->context->gl_vtable; tex_id = gst_gl_memory_get_texture_id (memory); locked_image = src->view->image (); if (!locked_image) { GST_OBJECT_UNLOCK (src); return TRUE; } gl->ActiveTexture (GL_TEXTURE0 + memory->plane); gl->BindTexture (GL_TEXTURE_2D, tex_id); gl->EGLImageTargetTexture2D (GL_TEXTURE_2D, gst_egl_image_get_image (locked_image)); gl->Flush (); GST_OBJECT_UNLOCK (src); return TRUE; } static gboolean gst_wpe_src_gl_start (GstGLBaseSrc * base_src) { GstWpeSrc *src = GST_WPE_SRC (base_src); gboolean result = TRUE; GstCapsFeatures *caps_features; GstGLContext *context = NULL; GstGLDisplay *display = NULL; GST_INFO_OBJECT (src, "Starting up"); GST_OBJECT_LOCK (src); caps_features = gst_caps_get_features (base_src->out_caps, 0); if (caps_features != NULL && gst_caps_features_contains (caps_features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { src->gl_enabled = TRUE; context = base_src->context; display = base_src->display; } else { src->gl_enabled = FALSE; } GST_DEBUG_OBJECT (src, "Will fill GLMemories: %d\n", src->gl_enabled); src->view = new WPEThreadedView; result = src->view->initialize (src, context, display, GST_VIDEO_INFO_WIDTH (&base_src->out_info), GST_VIDEO_INFO_HEIGHT (&base_src->out_info)); if (src->bytes != NULL) { src->view->loadData (src->bytes); g_bytes_unref (src->bytes); src->bytes = NULL; } GST_OBJECT_UNLOCK (src); if (!result) { GST_ELEMENT_ERROR (src, RESOURCE, FAILED, ("WPEBackend-FDO EGL display initialisation failed"), (NULL)); } return result; } static void gst_wpe_src_gl_stop (GstGLBaseSrc * base_src) { GstWpeSrc *src = GST_WPE_SRC (base_src); if (src->view) { delete src->view; src->view = NULL; } } static GstCaps * gst_wpe_src_fixate (GstBaseSrc * base_src, GstCaps * caps) { GstWpeSrc *src = GST_WPE_SRC (base_src); GstStructure *structure; gint width, height; caps = gst_caps_make_writable (caps); structure = gst_caps_get_structure (caps, 0); gst_structure_fixate_field_nearest_int (structure, "width", DEFAULT_WIDTH); gst_structure_fixate_field_nearest_int (structure, "height", DEFAULT_HEIGHT); if (gst_structure_has_field (structure, "framerate")) gst_structure_fixate_field_nearest_fraction (structure, "framerate", DEFAULT_FPS_N, DEFAULT_FPS_D); else gst_structure_set (structure, "framerate", GST_TYPE_FRACTION, DEFAULT_FPS_N, DEFAULT_FPS_D, NULL); caps = GST_BASE_SRC_CLASS (parent_class)->fixate (base_src, caps); GST_INFO_OBJECT (base_src, "Fixated caps to %" GST_PTR_FORMAT, caps); if (src->view) { gst_structure_get (structure, "width", G_TYPE_INT, &width, "height", G_TYPE_INT, &height, NULL); src->view->resize (width, height); } return caps; } void gst_wpe_src_configure_web_view (GstWpeSrc * src, WebKitWebView * webview) { GValue args[2] = { {0}, {0} }; g_value_init (&args[0], GST_TYPE_ELEMENT); g_value_set_object (&args[0], src); g_value_init (&args[1], G_TYPE_OBJECT); g_value_set_object (&args[1], webview); g_signal_emitv (args, gst_wpe_src_signals[SIGNAL_CONFIGURE_WEB_VIEW], 0, NULL); g_value_unset (&args[0]); g_value_unset (&args[1]); } static void gst_wpe_src_load_bytes (GstWpeSrc * src, GBytes * bytes) { if (src->view && GST_STATE (GST_ELEMENT_CAST (src)) > GST_STATE_NULL) src->view->loadData (bytes); else src->bytes = g_bytes_ref (bytes); } static gboolean gst_wpe_src_set_location (GstWpeSrc * src, const gchar * location, GError ** error) { g_free (src->location); src->location = g_strdup (location); if (src->view) src->view->loadUri (src->location); return TRUE; } static void gst_wpe_src_set_draw_background (GstWpeSrc * src, gboolean draw_background) { if (src->view) src->view->setDrawBackground (draw_background); src->draw_background = draw_background; } static void gst_wpe_src_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { GstWpeSrc *src = GST_WPE_SRC (object); switch (prop_id) { case PROP_LOCATION: { const gchar *location; location = g_value_get_string (value); if (location == NULL) { GST_WARNING_OBJECT (src, "location property cannot be NULL"); return; } if (!gst_wpe_src_set_location (src, location, NULL)) { GST_WARNING_OBJECT (src, "badly formatted location"); return; } break; } case PROP_DRAW_BACKGROUND: gst_wpe_src_set_draw_background (src, g_value_get_boolean (value)); break; default: break; } } static void gst_wpe_src_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstWpeSrc *src = GST_WPE_SRC (object); switch (prop_id) { case PROP_LOCATION: g_value_set_string (value, src->location); break; case PROP_DRAW_BACKGROUND: g_value_set_boolean (value, src->draw_background); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } } static gboolean gst_wpe_src_event (GstPad * pad, GstObject * parent, GstEvent * event) { gboolean ret = FALSE; GstWpeSrc *src = GST_WPE_SRC (parent); if (GST_EVENT_TYPE (event) == GST_EVENT_NAVIGATION) { const gchar *key; gint button; gdouble x, y, delta_x, delta_y; GST_DEBUG_OBJECT (src, "Processing event %" GST_PTR_FORMAT, event); if (!src->view) { return FALSE; } switch (gst_navigation_event_get_type (event)) { case GST_NAVIGATION_EVENT_KEY_PRESS: case GST_NAVIGATION_EVENT_KEY_RELEASE: if (gst_navigation_event_parse_key_event (event, &key)) { /* FIXME: This is wrong... The GstNavigation API should pass hardware-level information, not high-level keysym strings */ uint32_t keysym = (uint32_t) xkb_keysym_from_name (key, XKB_KEYSYM_NO_FLAGS); struct wpe_input_keyboard_event wpe_event; wpe_event.key_code = keysym; wpe_event.pressed = gst_navigation_event_get_type (event) == GST_NAVIGATION_EVENT_KEY_PRESS; wpe_view_backend_dispatch_keyboard_event (src->view->backend (), &wpe_event); ret = TRUE; } break; case GST_NAVIGATION_EVENT_MOUSE_BUTTON_PRESS: case GST_NAVIGATION_EVENT_MOUSE_BUTTON_RELEASE: if (gst_navigation_event_parse_mouse_button_event (event, &button, &x, &y)) { struct wpe_input_pointer_event wpe_event; wpe_event.time = GST_TIME_AS_MSECONDS (GST_EVENT_TIMESTAMP (event)); wpe_event.type = wpe_input_pointer_event_type_button; wpe_event.x = (int) x; wpe_event.y = (int) y; if (button == 1) { wpe_event.modifiers = wpe_input_pointer_modifier_button1; } else if (button == 2) { wpe_event.modifiers = wpe_input_pointer_modifier_button2; } else if (button == 3) { wpe_event.modifiers = wpe_input_pointer_modifier_button3; } else if (button == 4) { wpe_event.modifiers = wpe_input_pointer_modifier_button4; } else if (button == 5) { wpe_event.modifiers = wpe_input_pointer_modifier_button5; } wpe_event.button = button; wpe_event.state = gst_navigation_event_get_type (event) == GST_NAVIGATION_EVENT_MOUSE_BUTTON_PRESS; wpe_view_backend_dispatch_pointer_event (src->view->backend (), &wpe_event); ret = TRUE; } break; case GST_NAVIGATION_EVENT_MOUSE_MOVE: if (gst_navigation_event_parse_mouse_move_event (event, &x, &y)) { struct wpe_input_pointer_event wpe_event; wpe_event.time = GST_TIME_AS_MSECONDS (GST_EVENT_TIMESTAMP (event)); wpe_event.type = wpe_input_pointer_event_type_motion; wpe_event.x = (int) x; wpe_event.y = (int) y; wpe_view_backend_dispatch_pointer_event (src->view->backend (), &wpe_event); ret = TRUE; } break; case GST_NAVIGATION_EVENT_MOUSE_SCROLL: if (gst_navigation_event_parse_mouse_scroll_event (event, &x, &y, &delta_x, &delta_y)) { #if WPE_CHECK_VERSION(1, 6, 0) struct wpe_input_axis_2d_event wpe_event; if (delta_x) { wpe_event.x_axis = delta_x; } else { wpe_event.y_axis = delta_y; } wpe_event.base.time = GST_TIME_AS_MSECONDS (GST_EVENT_TIMESTAMP (event)); wpe_event.base.type = static_cast(wpe_input_axis_event_type_mask_2d | wpe_input_axis_event_type_motion_smooth); wpe_event.base.x = (int) x; wpe_event.base.y = (int) y; wpe_view_backend_dispatch_axis_event (src->view->backend (), &wpe_event.base); #else struct wpe_input_axis_event wpe_event; if (delta_x) { wpe_event.axis = 1; wpe_event.value = delta_x; } else { wpe_event.axis = 0; wpe_event.value = delta_y; } wpe_event.time = GST_TIME_AS_MSECONDS (GST_EVENT_TIMESTAMP (event)); wpe_event.type = wpe_input_axis_event_type_motion; wpe_event.x = (int) x; wpe_event.y = (int) y; wpe_view_backend_dispatch_axis_event (src->view->backend (), &wpe_event); #endif ret = TRUE; } break; default: break; } /* FIXME: No touch events handling support in GstNavigation */ } if (!ret) { ret = gst_pad_event_default (pad, parent, event); } return ret; } static void gst_wpe_src_init (GstWpeSrc * src) { GstPad *pad = gst_element_get_static_pad (GST_ELEMENT_CAST (src), "src"); gst_pad_set_event_function (pad, gst_wpe_src_event); gst_object_unref (pad); src->draw_background = TRUE; gst_base_src_set_live (GST_BASE_SRC_CAST (src), TRUE); } static GstURIType gst_wpe_src_uri_get_type (GType) { return GST_URI_SRC; } static const gchar *const * gst_wpe_src_get_protocols (GType) { static const char *protocols[] = { "wpe", NULL }; return protocols; } static gchar * gst_wpe_src_get_uri (GstURIHandler * handler) { GstWpeSrc *src = GST_WPE_SRC (handler); return g_strdup_printf ("wpe://%s", src->location); } static gboolean gst_wpe_src_set_uri (GstURIHandler * handler, const gchar * uri, GError ** error) { GstWpeSrc *src = GST_WPE_SRC (handler); return gst_wpe_src_set_location (src, uri + 6, error); } static void gst_wpe_src_uri_handler_init (gpointer iface_ptr, gpointer data) { GstURIHandlerInterface *iface = (GstURIHandlerInterface *) iface_ptr; iface->get_type = gst_wpe_src_uri_get_type; iface->get_protocols = gst_wpe_src_get_protocols; iface->get_uri = gst_wpe_src_get_uri; iface->set_uri = gst_wpe_src_set_uri; } static void gst_wpe_src_class_init (GstWpeSrcClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass); GstGLBaseSrcClass *gl_base_src_class = GST_GL_BASE_SRC_CLASS (klass); GstBaseSrcClass *base_src_class = GST_BASE_SRC_CLASS (klass); gobject_class->set_property = gst_wpe_src_set_property; gobject_class->get_property = gst_wpe_src_get_property; g_object_class_install_property (gobject_class, PROP_LOCATION, g_param_spec_string ("location", "location", "The URL to display", "", (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); g_object_class_install_property (gobject_class, PROP_DRAW_BACKGROUND, g_param_spec_boolean ("draw-background", "Draws the background", "Whether to draw the WebView background", TRUE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); gst_element_class_set_static_metadata (gstelement_class, "WPE source", "Source/Video", "Creates a video stream from a WPE browser", "Philippe Normand , Žan Doberšek "); gst_element_class_add_static_pad_template (gstelement_class, &src_factory); base_src_class->fixate = GST_DEBUG_FUNCPTR (gst_wpe_src_fixate); base_src_class->create = GST_DEBUG_FUNCPTR (gst_wpe_src_create); gl_base_src_class->supported_gl_api = static_cast < GstGLAPI > (GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2); gl_base_src_class->gl_start = GST_DEBUG_FUNCPTR (gst_wpe_src_gl_start); gl_base_src_class->gl_stop = GST_DEBUG_FUNCPTR (gst_wpe_src_gl_stop); gl_base_src_class->fill_gl_memory = GST_DEBUG_FUNCPTR (gst_wpe_src_fill_memory); /** * GstWpeSrc::configure-web-view: * @src: the object which received the signal * @webview: the webView * * Allow application to configure the webView settings. */ gst_wpe_src_signals[SIGNAL_CONFIGURE_WEB_VIEW] = g_signal_new ("configure-web-view", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_OBJECT); /** * GstWpeSrc::load-bytes: * @src: the object which received the signal * @bytes: the GBytes data to load * * Load the specified bytes into the internal webView. */ gst_wpe_src_signals[SIGNAL_LOAD_BYTES] = g_signal_new_class_handler ("load-bytes", G_TYPE_FROM_CLASS (klass), static_cast < GSignalFlags > (G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION), G_CALLBACK (gst_wpe_src_load_bytes), NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_BYTES); } static gboolean plugin_init (GstPlugin * plugin) { GST_DEBUG_CATEGORY_INIT (wpe_src_debug, "wpesrc", 0, "WPE Source"); return gst_element_register (plugin, "wpesrc", GST_RANK_NONE, GST_TYPE_WPE_SRC); } GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, wpe, "WPE src plugin", plugin_init, VERSION, GST_LICENSE, PACKAGE, GST_PACKAGE_ORIGIN)