xvimagesink: Add touch event support

Send touch events for XI_TouchBegin, XI_TouchEnd, and XI_TouchUpdate
events, grouping events with identical timestamps into one TOUCH_FRAME.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1633>

subprojects/gst-plugins-base/sys/xvimage/meson.build

@@ -19,7 +19,7 @@ if xvideo_dep.found()
     xvimage_sources,
     c_args : gst_plugins_base_args + no_warn_args,
     include_directories: [configinc, libsinc],
-    dependencies : glib_deps + [video_dep, gst_base_dep, gst_dep, x11_dep, xshm_dep, xvideo_dep, libm],
+    dependencies : glib_deps + [video_dep, gst_base_dep, gst_dep, x11_dep, xshm_dep, xvideo_dep, xi_dep, libm],
     install : true,
     install_dir : plugins_install_dir,
   )

subprojects/gst-plugins-base/sys/xvimage/xvcontext.c

@@ -33,6 +33,11 @@
 /* for XkbKeycodeToKeysym */
 #include <X11/XKBlib.h>
 
+/* for touchscreen events */
+#ifdef HAVE_XI2
+#include <X11/extensions/XInput2.h>
+#endif
+
 GST_DEBUG_CATEGORY (gst_debug_xv_context);
 #define GST_CAT_DEFAULT gst_debug_xv_context
 
@@ -642,8 +647,12 @@ gst_xvcontext_new (GstXvContextConfig * config, GError ** error)
     "XV_BRIGHTNESS", "XV_CONTRAST"
   };
   int opcode, event, err;
-  int major = XkbMajorVersion;
+  int xkb_major = XkbMajorVersion;
-  int minor = XkbMinorVersion;
+  int xkb_minor = XkbMinorVersion;
+#ifdef HAVE_XI2
+  int xi2_major = XI_2_Major;
+  int xi2_minor = XI_2_Minor;
+#endif
 
   g_return_val_if_fail (config != NULL, NULL);
 
@@ -726,7 +735,8 @@ gst_xvcontext_new (GstXvContextConfig * config, GError ** error)
     context->use_xshm = FALSE;
     GST_DEBUG ("xvimagesink is not using XShm extension");
   }
-  if (XkbQueryExtension (context->disp, &opcode, &event, &err, &major, &minor)) {
+  if (XkbQueryExtension (context->disp, &opcode, &event, &err, &xkb_major,
+          &xkb_minor)) {
     context->use_xkb = TRUE;
     GST_DEBUG ("xvimagesink is using Xkb extension");
   } else {
@@ -734,6 +744,22 @@ gst_xvcontext_new (GstXvContextConfig * config, GError ** error)
     GST_DEBUG ("xvimagesink is not using Xkb extension");
   }
 
+  /* Search for XInput extension support */
+#ifdef HAVE_XI2
+  if (XQueryExtension (context->disp, "XInputExtension",
+          &(context->xi_opcode), &event, &err)
+      && (XIQueryVersion (context->disp, &xi2_major, &xi2_minor) == Success
+          && (xi2_major > 2 || (xi2_major == 2 && xi2_minor >= 2)))) {
+    context->use_xi2 = TRUE;
+    GST_DEBUG ("xvimagesink is using XInput extension");
+  } else
+#endif /* HAVE_XI2 */
+  {
+    context->use_xi2 = FALSE;
+    GST_DEBUG ("xvimagesink is not using XInput extension: "
+        "XInput is not present");
+  }
+
   xv_attr = XvQueryPortAttributes (context->disp, context->xv_port_id, &N_attr);
 
   /* Generate the channels list */
@@ -943,6 +969,14 @@ gst_xvcontext_set_colorimetry (GstXvContext * context,
   g_mutex_unlock (&context->lock);
 }
 
+#ifdef HAVE_XI2
+static void
+gst_xv_touchdevice_free (GstXvTouchDevice * device)
+{
+  g_free (device->name);
+}
+#endif
+
 GstXWindow *
 gst_xvcontext_create_xwindow (GstXvContext * context, gint width, gint height)
 {
@@ -964,6 +998,13 @@ gst_xvcontext_create_xwindow (GstXvContext * context, gint width, gint height)
   window->height = height;
   window->internal = TRUE;
 
+#ifdef HAVE_XI2
+  window->last_touch = 0;
+  window->touch_devices = g_array_new (FALSE, FALSE, sizeof (GstXvTouchDevice));
+  g_array_set_clear_func (window->touch_devices,
+      (GDestroyNotify) gst_xv_touchdevice_free);
+#endif
+
   g_mutex_lock (&context->lock);
 
   window->win = XCreateSimpleWindow (context->disp,
@@ -1033,6 +1074,13 @@ gst_xvcontext_create_xwindow_from_xid (GstXvContext * context, XID xid)
   window->render_rect.w = attr.width;
   window->render_rect.h = attr.height;
 
+#ifdef HAVE_XI2
+  window->last_touch = 0;
+  window->touch_devices = g_array_new (FALSE, FALSE, sizeof (GstXvTouchDevice));
+  g_array_set_clear_func (window->touch_devices,
+      (GDestroyNotify) gst_xv_touchdevice_free);
+#endif
+
   window->gc = XCreateGC (context->disp, window->win, 0, NULL);
   g_mutex_unlock (&context->lock);
 
@@ -1050,6 +1098,10 @@ gst_xwindow_destroy (GstXWindow * window)
 
   g_mutex_lock (&context->lock);
 
+#ifdef HAVE_XI2
+  g_array_free (window->touch_devices, TRUE);
+#endif
+
   /* If we did not create that window we just free the GC and let it live */
   if (window->internal)
     XDestroyWindow (context->disp, window->win);
@@ -1067,6 +1119,91 @@ gst_xwindow_destroy (GstXWindow * window)
   g_slice_free1 (sizeof (GstXWindow), window);
 }
 
+#ifdef HAVE_XI2
+void
+gst_xwindow_select_touch_events (GstXvContext * context, GstXWindow * window)
+{
+  XIDeviceInfo *devices;
+  int ndevices, i, j, mask_len;
+  unsigned char *mask;
+
+  window->touch_devices = g_array_remove_range (window->touch_devices,
+      0, window->touch_devices->len);
+
+  mask_len = (XI_LASTEVENT + 7) << 3;
+  mask = g_new0 (unsigned char, mask_len);
+  XISetMask (mask, XI_TouchBegin);
+  XISetMask (mask, XI_TouchUpdate);
+  XISetMask (mask, XI_TouchEnd);
+
+  devices = XIQueryDevice (window->context->disp, XIAllDevices, &ndevices);
+
+  /* Find suitable touch screen devices, and select touch events for each */
+  for (i = 0; i < ndevices; i++) {
+    XIEventMask mask_data;
+    GstXvTouchDevice temp, *device;
+    gboolean has_touch = FALSE;
+
+    if (devices[i].use != XISlavePointer)
+      continue;
+
+    temp.pressure_valuator = -1;
+    temp.id = devices[i].deviceid;
+    temp.name = devices[i].name;
+
+    for (j = 0; j < devices[i].num_classes; j++) {
+      XIAnyClassInfo *class = devices[i].classes[j];
+
+      /* only pick devices with direct touch, to avoid touchpads and similar */
+      if (class->type == XITouchClass &&
+          ((XITouchClassInfo *) class)->mode == XIDirectTouch) {
+        has_touch = TRUE;
+      }
+
+      /* Find the valuator representing pressure, if present */
+      if (class->type == XIValuatorClass) {
+        XIValuatorClassInfo *val_info = (XIValuatorClassInfo *) class;
+
+        if (val_info->label == XInternAtom (context->disp,
+                "Abs Pressure", TRUE))
+          temp.abs_pressure = TRUE;
+        else if (val_info->label == XInternAtom (context->disp,
+                "Rel Pressure", TRUE))
+          temp.abs_pressure = FALSE;
+        else
+          continue;
+
+        temp.pressure_valuator = i;
+        temp.pressure_min = val_info->min;
+        temp.pressure_max = val_info->max;
+        temp.current_pressure = temp.pressure_min;
+      }
+    }
+
+    if (has_touch) {
+      GST_DEBUG ("found%s touch screen with id %d: %s",
+          temp.pressure_valuator < 0 ? "" : (temp.abs_pressure ?
+              "pressure-sensitive (abs)" : "pressure-sensitive (rel)"),
+          temp.id, temp.name);
+
+      device = g_new (GstXvTouchDevice, 1);
+      *device = temp;
+      device->name = g_strdup (device->name);
+      window->touch_devices =
+          g_array_append_vals (window->touch_devices, device, 1);
+
+      mask_data.deviceid = temp.id;
+      mask_data.mask_len = mask_len;
+      mask_data.mask = mask;
+      XISelectEvents (context->disp, window->win, &mask_data, 1);
+    }
+  }
+
+  g_free (mask);
+  XIFreeDeviceInfo (devices);
+}
+#endif
+
 void
 gst_xwindow_set_event_handling (GstXWindow * window, gboolean handle_events)
 {
@@ -1087,6 +1224,23 @@ gst_xwindow_set_event_handling (GstXWindow * window, gboolean handle_events)
           ExposureMask | StructureNotifyMask | PointerMotionMask |
           KeyPressMask | KeyReleaseMask);
     }
+
+#ifdef HAVE_XI2
+    if (context->use_xi2) {
+      XIEventMask mask_data;
+      unsigned char mask[2];
+
+      gst_xwindow_select_touch_events (context, window);
+
+      XISetMask (mask, XI_HierarchyChanged);
+      mask_data.deviceid = XIAllDevices;
+      mask_data.mask_len = sizeof (mask);
+      mask_data.mask = mask;
+
+      /* register for HierarchyChanged events to see device changes */
+      XISelectEvents (context->disp, window->win, &mask_data, 1);
+    }
+#endif
   } else {
     XSelectInput (context->disp, window->win, 0);
   }

subprojects/gst-plugins-base/sys/xvimage/xvcontext.h

@@ -47,6 +47,7 @@ G_BEGIN_DECLS
 typedef struct _GstXvContextConfig GstXvContextConfig;
 typedef struct _GstXvImageFormat GstXvImageFormat;
 typedef struct _GstXvContext GstXvContext;
+typedef struct _GstXvTouchDevice GstXvTouchDevice;
 
 /**
  * GstXvContextConfig:
@@ -112,6 +113,9 @@ struct _GstXvImageFormat
  * if the Extension is present
  * @use_xkb: used to known wether of not Xkb extension is usable or not even
  * if the Extension is present
+ * @use_xi2: used to known wether of not XInput extension is usable or not even
+ * if the Extension is present
+ * @xi_opcode: XInput opcode
  * @xv_port_id: the XVideo port ID
  * @im_format: used to store at least a valid format for XShm calls checks
  * @formats_list: list of supported image formats on @xv_port_id
@@ -148,6 +152,8 @@ struct _GstXvContext
 
   gboolean use_xshm;
   gboolean use_xkb;
+  gboolean use_xi2;
+  int xi_opcode;
 
   XvPortID xv_port_id;
   guint nb_adaptors;
@@ -217,6 +223,8 @@ typedef struct _GstXWindow GstXWindow;
  * @internal: used to remember if Window @win was created internally or passed
  * through the #GstVideoOverlay interface
  * @gc: the Graphical Context of Window @win
+ * @last_touch: timestamp of the last received touch event
+ * @touch_devices: list of known touchscreen devices
  *
  * Structure used to store information about a Window.
  */
@@ -230,6 +238,30 @@ struct _GstXWindow
   GstVideoRectangle render_rect;
   gboolean internal;
   GC gc;
+
+#ifdef HAVE_XI2
+  Time last_touch;
+  GArray *touch_devices;
+#endif
+};
+
+/*
+ * GstXvTouchDevice:
+ * @name: the name of this decive
+ * @id: the device ID of this device
+ * @pressure_valuator: index of the valuator that encodes pressure data, if present
+ * @abs_pressure: if pressure is reported in absolute or relative values
+ * @current_pressure: stores the most recent pressure value for this device
+ * @pressure_min: lowest possible pressure value
+ * @pressure_max: highest possible pressure value
+ *
+ * Structure used to store information about a touchscreen device.
+ */
+struct _GstXvTouchDevice {
+  gchar *name;
+  gint id, pressure_valuator;
+  gboolean abs_pressure;
+  gdouble current_pressure, pressure_min, pressure_max;
 };
 
 G_END_DECLS
@@ -240,6 +272,10 @@ GstXWindow *   gst_xvcontext_create_xwindow_from_xid (GstXvContext * context, XI
 
 void           gst_xwindow_destroy              (GstXWindow * window);
 
+#ifdef HAVE_XI2
+void           gst_xwindow_select_touch_events  (GstXvContext * context, GstXWindow * window);
+#endif
+
 void           gst_xwindow_set_event_handling   (GstXWindow * window, gboolean handle_events);
 void           gst_xwindow_set_title            (GstXWindow * window, const gchar * title);
 

subprojects/gst-plugins-base/sys/xvimage/xvimagesink.c

@@ -132,6 +132,11 @@
 /* for XkbKeycodeToKeysym */
 #include <X11/XKBlib.h>
 
+/* for touchscreen events */
+#ifdef HAVE_XI2
+#include <X11/extensions/XInput2.h>
+#endif
+
 GST_DEBUG_CATEGORY_EXTERN (gst_debug_xv_context);
 GST_DEBUG_CATEGORY_EXTERN (gst_debug_xv_image_pool);
 GST_DEBUG_CATEGORY (gst_debug_xv_image_sink);
@@ -418,7 +423,7 @@ gst_xv_image_sink_handle_xevents (GstXvImageSink * xvimagesink)
 {
   XEvent e;
   gint pointer_x = 0, pointer_y = 0;
-  gboolean pointer_moved = FALSE;
+  gboolean pointer_moved = FALSE, touch_frame_open = FALSE;
   gboolean exposed = FALSE, configured = FALSE;
 
   g_return_if_fail (GST_IS_XV_IMAGE_SINK (xvimagesink));
@@ -557,11 +562,11 @@ gst_xv_image_sink_handle_xevents (GstXvImageSink * xvimagesink)
     g_mutex_lock (&xvimagesink->context->lock);
   }
 
-  /* Handle Display events */
   while (XPending (xvimagesink->context->disp)) {
     XNextEvent (xvimagesink->context->disp, &e);
 
     switch (e.type) {
+        /* Handle Display events */
       case ClientMessage:{
         Atom wm_delete;
 
@@ -579,13 +584,139 @@ gst_xv_image_sink_handle_xevents (GstXvImageSink * xvimagesink)
         }
         break;
       }
+      default:{
+#ifdef HAVE_XI2
+        /* Handle XInput2 touch screen events */
+        if (xvimagesink->context->use_xi2
+            && XGetEventData (xvimagesink->context->disp,
+                (XGenericEventCookie *) & e)
+            && e.xcookie.extension == xvimagesink->context->xi_opcode) {
+          XGenericEventCookie *cookie;
+          XIDeviceEvent *touch;
+          GstXvTouchDevice device;
+          GstXWindow *xwindow;
+          GstEvent *nav;
+          gdouble pressure;
+          gboolean device_found;
+          unsigned int ev_id, i;
+
+          cookie = &e.xcookie;
+          xwindow = xvimagesink->xwindow;
+          nav = NULL;
+
+          if (cookie->evtype == XI_HierarchyChanged) {
+            GST_DEBUG ("ximagesink devices changed, searching for "
+                "touch devices again");
+            gst_xwindow_select_touch_events (xvimagesink->context, xwindow);
+            break;
+          }
+
+          if (cookie->evtype != XI_TouchBegin
+              && cookie->evtype != XI_TouchUpdate
+              && cookie->evtype != XI_TouchEnd)
+            break;
+
+          touch = cookie->data;
+          ev_id = ((unsigned int) touch->deviceid) << 16 |
+              (((unsigned int) touch->detail) & 0x00ff);
+
+          /* find device that the event belongs to */
+          device_found = FALSE;
+          for (i = 0; i < xwindow->touch_devices->len; i++) {
+            device = g_array_index (xwindow->touch_devices,
+                GstXvTouchDevice, i);
+            if (device.id == touch->deviceid) {
+              device_found = TRUE;
+              break;
+            }
+          }
+          if (!device_found)
+            break;
+
+          if (device.pressure_valuator >= 0) {
+            pressure = touch->valuators.values[device.pressure_valuator];
+            pressure -= device.pressure_min;
+            pressure /= device.pressure_max - device.pressure_min;
+            if (!device.abs_pressure)
+              pressure += device.current_pressure;
+          } else
+            pressure = NAN;
+
+          device.current_pressure = pressure;
+
+          g_mutex_unlock (&xvimagesink->context->lock);
+          g_mutex_unlock (&xvimagesink->flow_lock);
+
+          /* assume the event queue is ordered chronologically, and end */
+          /* the previous touch event frame when the timestamp increases */
+          if (touch->time != xwindow->last_touch && touch_frame_open) {
+            if (touch->time < xwindow->last_touch) {
+              GST_WARNING ("ximagesink out of order touch event "
+                  "with timestamp %lu, not ending touch frame", touch->time);
+            } else {
+              GST_DEBUG ("ximagesink ending touch frame for %lu",
+                  xwindow->last_touch);
+              gst_navigation_send_event_simple (GST_NAVIGATION (xvimagesink),
+                  gst_navigation_event_new_touch_frame ());
+            }
+          }
+
+          switch (cookie->evtype) {
+            case XI_TouchBegin:{
+              GST_DEBUG ("xvimagesink new touch point %d on device %d "
+                  "at %.0f,%.0f", touch->detail, touch->deviceid,
+                  touch->event_x, touch->event_y);
+              nav = gst_navigation_event_new_touch_down (ev_id,
+                  touch->event_x, touch->event_y, pressure);
+              break;
+            }
+            case XI_TouchEnd:{
+              GST_DEBUG ("xvimagesink removed touch point %d from device %d "
+                  "at %.0f,%.0f", touch->detail, touch->deviceid,
+                  touch->event_x, touch->event_y);
+              nav = gst_navigation_event_new_touch_up (ev_id,
+                  touch->event_x, touch->event_y);
+              break;
+            }
+            case XI_TouchUpdate:{
+              GST_DEBUG ("xvimagesink touch point %d on device %d moved "
+                  "to %.0f,%.0f", touch->detail, touch->deviceid,
+                  touch->event_x, touch->event_y);
+              nav = gst_navigation_event_new_touch_motion (ev_id,
+                  touch->event_x, touch->event_y, pressure);
+              break;
+            }
             default:
               break;
           }
+
+          gst_navigation_send_event_simple (GST_NAVIGATION (xvimagesink), nav);
+          xvimagesink->xwindow->last_touch = touch->time;
+          touch_frame_open = TRUE;
+
+          g_mutex_lock (&xvimagesink->flow_lock);
+          g_mutex_lock (&xvimagesink->context->lock);
+          XFreeEventData (xvimagesink->context->disp, cookie);
+        }
+#endif
+        break;
+      }
+    }
   }
 
   g_mutex_unlock (&xvimagesink->context->lock);
   g_mutex_unlock (&xvimagesink->flow_lock);
+
+#ifdef HAVE_XI2
+  /* send a touch-frame event now to avoid delay from having to wait for the */
+  /* next invocation */
+  if (touch_frame_open) {
+    GST_DEBUG ("xvimagesink ending touch frame for %lu",
+        xvimagesink->xwindow->last_touch);
+    gst_navigation_send_event_simple (GST_NAVIGATION (xvimagesink),
+        gst_navigation_event_new_touch_frame ());
+  }
+#endif
 }
 
 static gpointer