ximagesrc: add navigation support

Add a basic navigation support:
- mouse events (buttons/move)
- keyboard events (keys)

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5273>

subprojects/gst-docs/symbols/symbol_index.json

@@ -19347,6 +19347,7 @@
   "GstXImageSrc:use-damage",
   "GstXImageSrc:xid",
   "GstXImageSrc:xname",
+  "GstXImageSrc:enable-navigation-events",
   "GstXingMux",
   "GstXingMux!sink",
   "GstXingMux!src",
@@ -67465,6 +67466,7 @@
   "ximagesrc:use-damage",
   "ximagesrc:xid",
   "ximagesrc:xname",
+  "ximagesrc:enable-navigation-events",
   "xingmux",
   "xvimagesink",
   "xvimagesink:autopaint-colorkey",

subprojects/gst-plugins-good/docs/gst_plugins_cache.json

@@ -28552,6 +28552,18 @@
                         "type": "gchararray",
                         "writable": true
                     },
+                    "enable-navigation-events": {
+                        "blurb": "When enabled, navigation events are handled",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "false",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gboolean",
+                        "writable": true
+                    },
                     "endx": {
                         "blurb": "X coordinate of bottom right corner of area to be recorded (0 for bottom right of screen)",
                         "conditionally-available": false,

subprojects/gst-plugins-good/meson_options.txt

@@ -104,6 +104,7 @@ option('ximagesrc', type : 'feature', value : 'auto', description : 'X11 ximages
 option('ximagesrc-xshm', type : 'feature', value : 'auto', description : 'X11 ximagesrc plugin (XSHM support)')
 option('ximagesrc-xfixes', type : 'feature', value : 'auto', description : 'X11 ximagesrc plugin (XFixes support)')
 option('ximagesrc-xdamage', type : 'feature', value : 'auto', description : 'X11 ximagesrc plugin (XDamage support)')
+option('ximagesrc-navigation', type : 'feature', value : 'auto', description : 'X11 ximagesrc plugin (Navigation support)')
 
 # v4l2 plugin options
 option('v4l2', type : 'feature', value : 'auto', description : 'Build video4linux2 source/sink plugin')

subprojects/gst-plugins-good/sys/ximage/gst-ximage-navigation.c

@@ -0,0 +1,67 @@
+/* GStreamer
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include "gst-ximage-navigation.h"
+#include <X11/extensions/XTest.h>
+
+// Based on xtestlib: https://www.x.org/releases/X11R7.5/doc/Xext/xtestlib.html
+
+void
+gst_ximage_navigation_mouse_move_pointer (Display * display, int x, int y)
+{
+  // If screen_number is -1, the current screen (that the pointer is on) is used
+  XTestFakeMotionEvent (display, -1, x, y, CurrentTime);
+  XSync (display, FALSE);
+  return;
+}
+
+void
+gst_ximage_navigation_mouse_push_button (Display * display,
+    unsigned int button, Bool is_press)
+{
+  /*
+     button values:
+     1 = left button
+     2 = middle button (pressing the scroll wheel)
+     3 = right button
+     4 = turn scroll wheel up
+     5 = turn scroll wheel down
+     6 = push scroll wheel left
+     7 = push scroll wheel right
+     8 = 4th button (aka browser backward button)
+     9 = 5th button (aka browser forward button)
+   */
+  XTestFakeButtonEvent (display, button, is_press, CurrentTime);
+  XSync (display, FALSE);
+  return;
+}
+
+void
+gst_ximage_navigation_key (Display * display, const char *keysym_name,
+    Bool is_press)
+{
+  // keysym_name: one of X11 keysym names defined in https://www.cl.cam.ac.uk/~mgk25/ucs/keysyms.txt
+  unsigned int keysym, keycode;
+  keysym = (unsigned int) XStringToKeysym (keysym_name);
+  keycode = XKeysymToKeycode (display, keysym);
+  if (keycode == 0)             // undefined KeySym
+    return;
+  XTestFakeKeyEvent (display, keycode, is_press, CurrentTime);
+  XSync (display, FALSE);
+  return;
+}

subprojects/gst-plugins-good/sys/ximage/gst-ximage-navigation.h

@@ -0,0 +1,28 @@
+/* GStreamer
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_XIMAGE_NAVIGATION_H__
+#define __GST_XIMAGE_NAVIGATION_H__
+
+#include <ximageutil.h>
+
+void gst_ximage_navigation_mouse_move_pointer(Display * display, int x, int y);
+void gst_ximage_navigation_mouse_push_button(Display * display, unsigned int button, Bool is_press);
+void gst_ximage_navigation_key(Display * display, const char * keysym_name, Bool is_press);
+
+#endif

subprojects/gst-plugins-good/sys/ximage/gstximagesrc.c

@@ -24,9 +24,9 @@
  *
  * This element captures your X Display and creates raw RGB video.  It uses
  * the XDamage extension if available to only capture areas of the screen that
- * have changed since the last frame.  It uses the XFixes extension if
- * available to also capture your mouse pointer.  By default it will fixate to
- * 25 frames per second.
+ * have changed since the last frame. It uses the XFixes extension if
+ * available to also capture your mouse pointer. It supports handling of
+ * mouse and keyboard events. By default it will fixate to 25 frames per second.
  *
  * ## Example pipelines
  * |[
@@ -39,6 +39,7 @@
 #include "config.h"
 #endif
 #include "gstximagesrc.h"
+#include "gst-ximage-navigation.h"
 
 #include <string.h>
 #include <stdlib.h>
@@ -55,6 +56,14 @@
 GST_DEBUG_CATEGORY_STATIC (gst_debug_ximage_src);
 #define GST_CAT_DEFAULT gst_debug_ximage_src
 
+#define MOUSE_SCROLL_UP_BUTTON 4
+#define MOUSE_SCROLL_DOWN_BUTTON 5
+
+#define KEYCODE_CTRL 0x25
+#define KEYCODE_SHIFT 0x32
+#define KEYCODE_ALT 0x40
+#define KEYCODE_META 0x85
+
 static GstStaticPadTemplate t =
 GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
     GST_STATIC_CAPS ("video/x-raw, "
@@ -75,6 +84,7 @@ enum
   PROP_REMOTE,
   PROP_XID,
   PROP_XNAME,
+  PROP_ENABLE_NAVIGATION_EVENTS
 };
 
 #define gst_ximage_src_parent_class parent_class
@@ -943,7 +953,6 @@ gst_ximage_src_set_property (GObject * object, guint prop_id,
 
   switch (prop_id) {
     case PROP_DISPLAY_NAME:
-
       g_free (src->display_name);
       src->display_name = g_value_dup_string (value);
       break;
@@ -987,6 +996,9 @@ gst_ximage_src_set_property (GObject * object, guint prop_id,
       g_free (src->xname);
       src->xname = g_value_dup_string (value);
       break;
+    case PROP_ENABLE_NAVIGATION_EVENTS:
+      src->enable_navigation_events = g_value_get_boolean (value);
+      break;
     default:
       break;
   }
@@ -1004,7 +1016,6 @@ gst_ximage_src_get_property (GObject * object, guint prop_id,
         g_value_set_string (value, DisplayString (src->xcontext->disp));
       else
         g_value_set_string (value, src->display_name);
-
       break;
     case PROP_SHOW_POINTER:
       g_value_set_boolean (value, src->show_pointer);
@@ -1033,6 +1044,9 @@ gst_ximage_src_get_property (GObject * object, guint prop_id,
     case PROP_XNAME:
       g_value_set_string (value, src->xname);
       break;
+    case PROP_ENABLE_NAVIGATION_EVENTS:
+      g_value_set_boolean (value, src->enable_navigation_events);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -1195,6 +1209,70 @@ gst_ximage_src_fixate (GstBaseSrc * bsrc, GstCaps * caps)
   return caps;
 }
 
+static gboolean
+gst_ximage_src_event (GstBaseSrc * base_src, GstEvent * event)
+{
+  gboolean ret = FALSE;
+  gboolean is_press = FALSE;
+  GstXImageSrc *src = GST_XIMAGE_SRC (base_src);
+  if (src->enable_navigation_events
+      && GST_EVENT_TYPE (event) == GST_EVENT_NAVIGATION) {
+    const gchar *key;
+    gint button;
+    gdouble x, y, delta_x, delta_y;
+
+    GST_DEBUG_OBJECT (src, "Processing event %" GST_PTR_FORMAT, event);
+    switch (gst_navigation_event_get_type (event)) {
+      case GST_NAVIGATION_EVENT_KEY_PRESS:
+        is_press = TRUE;        /* FALLTHROUGH */
+      case GST_NAVIGATION_EVENT_KEY_RELEASE:
+        if (gst_navigation_event_parse_key_event (event, &key)) {
+          gst_ximage_navigation_key (src->xcontext->disp, key, is_press);
+          ret = TRUE;
+        }
+        break;
+      case GST_NAVIGATION_EVENT_MOUSE_BUTTON_PRESS:
+        is_press = TRUE;        /* FALLTHROUGH */
+      case GST_NAVIGATION_EVENT_MOUSE_BUTTON_RELEASE:
+        if (gst_navigation_event_parse_mouse_button_event (event, &button, &x,
+                &y)) {
+          gst_ximage_navigation_mouse_push_button (src->xcontext->disp, button,
+              is_press);
+          ret = TRUE;
+        }
+        break;
+      case GST_NAVIGATION_EVENT_MOUSE_MOVE:
+        if (gst_navigation_event_parse_mouse_move_event (event, &x, &y)) {
+          gst_ximage_navigation_mouse_move_pointer (src->xcontext->disp,
+              (int) x, (int) y);
+          ret = TRUE;
+        }
+        break;
+      case GST_NAVIGATION_EVENT_MOUSE_SCROLL:
+        if (gst_navigation_event_parse_mouse_scroll_event (event, &x, &y,
+                &delta_x, &delta_y)) {
+          int scroll_button =
+              (int) delta_y <
+              0 ? MOUSE_SCROLL_DOWN_BUTTON : MOUSE_SCROLL_UP_BUTTON;
+          gst_ximage_navigation_mouse_push_button (src->xcontext->disp,
+              scroll_button, TRUE);
+          gst_ximage_navigation_mouse_push_button (src->xcontext->disp,
+              scroll_button, FALSE);
+          ret = TRUE;
+        }
+        break;
+      default:
+        break;
+    }
+  }
+  if (!ret) {
+    ret =
+        GST_CALL_PARENT_WITH_DEFAULT (GST_BASE_SRC_CLASS, event, (base_src,
+            event), FALSE);
+  }
+  return ret;
+}
+
 static void
 gst_ximage_src_class_init (GstXImageSrcClass * klass)
 {
@@ -1294,6 +1372,17 @@ gst_ximage_src_class_init (GstXImageSrcClass * klass)
           "Window name to capture from", NULL,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+  /**
+   * GstXImageSrc:enable-navigation-events:
+   *
+   * Enable navigation events
+   */
+  g_object_class_install_property (gc, PROP_ENABLE_NAVIGATION_EVENTS,
+      g_param_spec_boolean ("enable-navigation-events",
+          "Enable navigation events",
+          "When enabled, navigation events are handled", FALSE,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
   gst_element_class_set_static_metadata (ec, "Ximage video source",
       "Source/Video",
       "Creates a screenshot video stream",
@@ -1309,6 +1398,10 @@ gst_ximage_src_class_init (GstXImageSrcClass * klass)
   bc->stop = gst_ximage_src_stop;
   bc->unlock = gst_ximage_src_unlock;
   push_class->create = gst_ximage_src_create;
+#ifdef HAVE_NAVIGATION
+  XInitThreads ();
+  bc->event = gst_ximage_src_event;
+#endif /* HAVE_NAVIGATION */
 }
 
 static void
@@ -1328,6 +1421,7 @@ gst_ximage_src_init (GstXImageSrc * ximagesrc)
   ximagesrc->endx_fit_to_screen = TRUE;
   ximagesrc->endy_fit_to_screen = TRUE;
   ximagesrc->remote = FALSE;
+  ximagesrc->enable_navigation_events = FALSE;
 }
 
 static gboolean

subprojects/gst-plugins-good/sys/ximage/gstximagesrc.h

@@ -97,6 +97,9 @@ struct _GstXImageSrc
   /* whether to use remote friendly calls */
   gboolean remote;
 
+  /* enable navigation events */
+  gboolean enable_navigation_events;
+
 #ifdef HAVE_XFIXES
   int fixes_event_base;
   XFixesCursorImage *cursor_image;

subprojects/gst-plugins-good/sys/ximage/meson.build

@@ -2,6 +2,7 @@ x11_dep = dependency('x11', required : get_option('ximagesrc'))
 
 if x11_dep.found()
   x_args = []
+  x_sources = files(['gstximagesrc.c', 'ximageutil.c'])
   xshm_dep = dependency('xext', required : get_option('ximagesrc-xshm'))
   # FIXME: should add a 'required' arg to cc.has_function() in Meson and use it here
   if xshm_dep.found() and cc.has_function('XShmAttach', dependencies: xshm_dep)
@@ -18,12 +19,19 @@ if x11_dep.found()
     x_args += ['-DHAVE_XDAMAGE']
   endif
 
+  xtst_dep = dependency('xtst', required : get_option('ximagesrc-navigation'))
+  if xtst_dep.found()
+    x_args += ['-DHAVE_NAVIGATION']
+    x_sources += files(['gst-ximage-navigation.c'])
+  endif
+
   gstximagesrc = library('gstximagesrc',
-    'gstximagesrc.c', 'ximageutil.c',
+    x_sources,
     c_args : gst_plugins_good_args + x_args,
     include_directories : [configinc, libsinc],
     dependencies : [gstbase_dep, gstvideo_dep, x11_dep,
-                    xshm_dep, xfixes_dep, xdamage_dep],
+                    xshm_dep, xfixes_dep, xdamage_dep,
+                    xtst_dep],
     install : true,
     install_dir : plugins_install_dir,
   )