gstreamer/subprojects/gst-plugins-bad/sys/androidmedia/gstahssrc.c

688 lines
20 KiB
C
Raw Normal View History

/* GStreamer android.hardware.Sensor Source
* Copyright (C) 2016 SurroundIO
* Author: Martin Kelly <martin@surround.io>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-ahssrc
* @title: gstahssrc
*
* The ahssrc element reads data from Android device sensors
* (android.hardware.Sensor).
*
* ## Example launch line
* |[
* gst-launch -v ahssrc ! fakesink
* ]|
* Push Android sensor data into a fakesink.
*
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <string.h>
#include <gst/gst.h>
#include <gst/gstclock.h>
#include <gst/base/gstbasesrc.h>
#include <gst/base/gstpushsrc.h>
#include "gstjniutils.h"
#include "gst-android-hardware-sensor.h"
#include "gstahssrc.h"
#include "gstsensors.h"
GST_DEBUG_CATEGORY_STATIC (gst_ahs_src_debug);
#define GST_CAT_DEFAULT gst_ahs_src_debug
#define parent_class gst_ahs_src_parent_class
/* GObject */
static void gst_ahs_src_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_ahs_src_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static void gst_ahs_src_dispose (GObject * object);
/* GstBaseSrc */
static gboolean gst_ahs_src_set_caps (GstBaseSrc * src, GstCaps * caps);
static gboolean gst_ahs_src_start (GstBaseSrc * src);
static gboolean gst_ahs_src_stop (GstBaseSrc * src);
static gboolean gst_ahs_src_get_size (GstBaseSrc * src, guint64 * size);
static gboolean gst_ahs_src_is_seekable (GstBaseSrc * src);
static gboolean gst_ahs_src_unlock (GstBaseSrc * src);
static gboolean gst_ahs_src_unlock_stop (GstBaseSrc * src);
/* GstPushSrc */
static GstFlowReturn gst_ahs_src_create (GstPushSrc * src, GstBuffer ** buf);
/* GstAHSSrc */
static void gst_ahs_src_on_sensor_changed (jobject sensor_event,
gpointer user_data);
static void gst_ahs_src_on_accuracy_changed (jobject sensor, gint accuracy,
gpointer user_data);
static gboolean gst_ahs_src_register_callback (GstAHSSrc * self);
enum
{
PROP_0,
PROP_SENSOR_DELAY,
PROP_ALPHA,
PROP_SAMPLE_INTERVAL,
PROP_LAST
};
static GParamSpec *properties[PROP_LAST];
#define GST_AHS_SRC_CAPS_STR GST_SENSOR_CAPS_MAKE (GST_SENSOR_FORMATS_ALL)
static GstStaticPadTemplate gst_ahs_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_AHS_SRC_CAPS_STR));
G_DEFINE_TYPE_WITH_CODE (GstAHSSrc, gst_ahs_src, GST_TYPE_PUSH_SRC,
GST_DEBUG_CATEGORY_INIT (gst_ahs_src_debug, "ahssrc", 0,
"Android hardware sensors"));
#define GST_TYPE_AHS_SENSOR_DELAY (gst_ahs_src_get_sensor_delay ())
static GType
gst_ahs_src_get_sensor_delay (void)
{
static GType ahs_src_sensor_delay = 0;
if (!ahs_src_sensor_delay) {
static GEnumValue sensor_delay[5];
sensor_delay[0].value = AHS_SENSOR_DELAY_FASTEST;
sensor_delay[0].value_name = "fastest";
sensor_delay[0].value_nick = "fastest";
sensor_delay[1].value = AHS_SENSOR_DELAY_GAME;
sensor_delay[1].value_name = "game";
sensor_delay[1].value_nick = "game";
sensor_delay[2].value = AHS_SENSOR_DELAY_NORMAL;
sensor_delay[2].value_name = "normal";
sensor_delay[2].value_nick = "normal";
sensor_delay[3].value = AHS_SENSOR_DELAY_UI;
sensor_delay[3].value_name = "ui";
sensor_delay[3].value_nick = "ui";
sensor_delay[4].value = 0;
sensor_delay[4].value_name = NULL;
sensor_delay[4].value_nick = NULL;
ahs_src_sensor_delay =
g_enum_register_static ("GstAhsSrcSensorDelay", sensor_delay);
}
return ahs_src_sensor_delay;
}
#define GST_TYPE_AHS_SENSOR_TYPE (gst_ahs_src_get_sensor_type ())
static GType
gst_ahs_src_get_sensor_type (void)
{
static GType ahs_src_sensor_type = 0;
if (!ahs_src_sensor_type) {
static const GEnumValue sensor_types[] = {
{AHS_SENSOR_TYPE_ACCELEROMETER, "accelerometer"},
{AHS_SENSOR_TYPE_AMBIENT_TEMPERATURE, "ambient-temperature"},
{AHS_SENSOR_TYPE_GAME_ROTATION_VECTOR, "game-rotation-vector"},
{AHS_SENSOR_TYPE_GEOMAGNETIC_ROTATION_VECTOR,
"geomagnetic-rotation-vector"},
{AHS_SENSOR_TYPE_GRAVITY, "gravity"},
{AHS_SENSOR_TYPE_GYROSCOPE, "gyroscope"},
{AHS_SENSOR_TYPE_GYROSCOPE_UNCALIBRATED, "gyroscope-uncalibrated"},
{AHS_SENSOR_TYPE_HEART_RATE, "heart-rate"},
{AHS_SENSOR_TYPE_LIGHT, "light"},
{AHS_SENSOR_TYPE_LINEAR_ACCELERATION, "linear-acceleration"},
{AHS_SENSOR_TYPE_MAGNETIC_FIELD, "magnetic-field"},
{AHS_SENSOR_TYPE_MAGNETIC_FIELD_UNCALIBRATED,
"magnetic-field-uncalibrated"},
{AHS_SENSOR_TYPE_ORIENTATION, "orientation"},
{AHS_SENSOR_TYPE_PRESSURE, "pressure"},
{AHS_SENSOR_TYPE_PROXIMITY, "proximity"},
{AHS_SENSOR_TYPE_RELATIVE_HUMIDITY, "relative-humidity"},
{AHS_SENSOR_TYPE_ROTATION_VECTOR, "rotation-vector"},
{AHS_SENSOR_TYPE_STEP_COUNTER, "step-counter"},
{AHS_SENSOR_TYPE_STEP_DETECTOR, "step-detector"},
{0, NULL, NULL}
};
ahs_src_sensor_type =
g_enum_register_static ("GstAhsSrcSensorType", sensor_types);
}
return ahs_src_sensor_type;
}
static void
gst_ahs_src_class_init (GstAHSSrcClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstBaseSrcClass *base_src_class = GST_BASE_SRC_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstPushSrcClass *push_src_class = GST_PUSH_SRC_CLASS (klass);
gobject_class->set_property = gst_ahs_src_set_property;
gobject_class->get_property = gst_ahs_src_get_property;
gobject_class->dispose = gst_ahs_src_dispose;
base_src_class->set_caps = GST_DEBUG_FUNCPTR (gst_ahs_src_set_caps);
base_src_class->start = GST_DEBUG_FUNCPTR (gst_ahs_src_start);
base_src_class->stop = GST_DEBUG_FUNCPTR (gst_ahs_src_stop);
base_src_class->get_size = GST_DEBUG_FUNCPTR (gst_ahs_src_get_size);
base_src_class->is_seekable = GST_DEBUG_FUNCPTR (gst_ahs_src_is_seekable);
base_src_class->unlock = GST_DEBUG_FUNCPTR (gst_ahs_src_unlock);
base_src_class->unlock_stop = GST_DEBUG_FUNCPTR (gst_ahs_src_unlock_stop);
push_src_class->create = GST_DEBUG_FUNCPTR (gst_ahs_src_create);
gst_element_class_add_pad_template (element_class,
gst_static_pad_template_get (&gst_ahs_src_template));
properties[PROP_SENSOR_DELAY] = g_param_spec_enum ("sensor-delay",
"Sensor delay", "Configure the sensor rate", GST_TYPE_AHS_SENSOR_DELAY,
AHS_SENSOR_DELAY_NORMAL,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS);
g_object_class_install_property (gobject_class, PROP_SENSOR_DELAY,
properties[PROP_SENSOR_DELAY]);
properties[PROP_ALPHA] = g_param_spec_double ("alpha", "Alpha",
"Alpha value used for exponential smoothing (between 0.0 and 1.0)", 0.0,
1.0, 0.2, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS);
g_object_class_install_property (gobject_class, PROP_ALPHA,
properties[PROP_ALPHA]);
properties[PROP_SAMPLE_INTERVAL] = g_param_spec_uint ("sample-interval",
"Sample interval",
"Sample interval (for interval n, will output a smoothed average every "
"nth sample)", 1, G_MAXUINT, 1,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS);
g_object_class_install_property (gobject_class, PROP_SAMPLE_INTERVAL,
properties[PROP_SAMPLE_INTERVAL]);
gst_element_class_set_static_metadata (element_class,
"Android hardware sensors", "Source/Sensor/Device",
"Source for Android hardware sensor data",
"Martin Kelly <martin@surround.io>");
}
static gboolean
_data_queue_check_full (GstDataQueue * queue, guint visible,
guint bytes, guint64 time, gpointer checkdata)
{
return FALSE;
}
static void
gst_ahs_src_init (GstAHSSrc * self)
{
gst_base_src_set_live (GST_BASE_SRC (self), TRUE);
gst_base_src_set_format (GST_BASE_SRC (self), GST_FORMAT_TIME);
gst_base_src_set_do_timestamp (GST_BASE_SRC (self), FALSE);
self->sensor_enum_class = g_type_class_ref (GST_TYPE_AHS_SENSOR_TYPE);
self->sensor_type_name = NULL;
self->manager = NULL;
self->sensor = NULL;
self->listener = NULL;
self->callback_registered = FALSE;
self->queue = gst_data_queue_new (_data_queue_check_full, NULL, NULL, NULL);
self->previous_time = GST_CLOCK_TIME_NONE;
self->sample_index = 0;
self->current_sample = NULL;
}
static void
gst_ahs_src_dispose (GObject * object)
{
JNIEnv *env = gst_amc_jni_get_env ();
GstAHSSrc *self = GST_AHS_SRC (object);
if (self->manager) {
gst_amc_jni_object_unref (env, self->manager->object);
g_slice_free (GstAHSensorManager, self->manager);
self->manager = NULL;
}
if (self->sensor) {
gst_amc_jni_object_unref (env, self->sensor->object);
g_slice_free (GstAHSensor, self->sensor);
self->sensor = NULL;
}
if (self->listener) {
gst_amc_jni_object_unref (env, self->listener->object);
g_slice_free (GstAHSensorEventListener, self->listener);
self->listener = NULL;
}
if (self->current_sample) {
g_free (self->current_sample);
self->current_sample = NULL;
}
if (self->sensor_enum_class) {
g_type_class_unref (self->sensor_enum_class);
self->sensor_enum_class = NULL;
}
if (self->sensor_type_name) {
g_free ((gpointer) self->sensor_type_name);
self->sensor_type_name = NULL;
}
if (self->queue) {
g_object_unref (self->queue);
self->queue = NULL;
}
G_OBJECT_CLASS (parent_class)->dispose (object);
}
static void
gst_ahs_src_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstAHSSrc *self = GST_AHS_SRC (object);
/*
* Take the mutex to protect against callbacks or changes to the properties
* that the callback uses (e.g. caps changes).
*/
GST_OBJECT_LOCK (self);
switch (prop_id) {
case PROP_SENSOR_DELAY:
self->sensor_delay = g_value_get_enum (value);
/*
* If we already have a callback running, reregister with the new delay.
* Otherwise, wait for the pipeline to start before we register.
*/
if (self->callback_registered)
gst_ahs_src_register_callback (self);
break;
case PROP_ALPHA:
self->alpha = g_value_get_double (value);
break;
case PROP_SAMPLE_INTERVAL:
self->sample_interval = g_value_get_uint (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
GST_OBJECT_UNLOCK (self);
}
static void
gst_ahs_src_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstAHSSrc *self = GST_AHS_SRC (object);
switch (prop_id) {
case PROP_SENSOR_DELAY:
g_value_set_enum (value, self->sensor_delay);
case PROP_ALPHA:
g_value_set_double (value, self->alpha);
break;
case PROP_SAMPLE_INTERVAL:
g_value_set_uint (value, self->sample_interval);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
}
}
static gboolean
gst_ahs_src_register_callback (GstAHSSrc * self)
{
if (self->callback_registered) {
gst_ah_sensor_unregister_listener (self->manager, self->listener);
self->callback_registered = FALSE;
}
if (!gst_ah_sensor_register_listener (self->manager, self->listener,
self->sensor, self->sensor_delay)) {
return FALSE;
}
self->callback_registered = TRUE;
return TRUE;
}
static gboolean
gst_ahs_src_change_sensor_type (GstAHSSrc * self, const gchar * type_str,
gint type)
{
JNIEnv *env = gst_amc_jni_get_env ();
/* Replace sensor type. */
if (self->sensor_type_name)
g_free ((gpointer) self->sensor_type_name);
self->sensor_type_name = type_str;
self->sensor_type = type;
/* Adjust buffer and buffer size. */
self->buffer_size = gst_ah_sensor_get_sensor_data_size (self->sensor_type);
g_assert (self->buffer_size != 0);
self->sample_length = self->buffer_size / sizeof (*self->current_sample);
self->current_sample = g_realloc (self->current_sample, self->buffer_size);
/* Make sure we have a manager. */
if (!self->manager) {
self->manager = gst_ah_sensor_get_manager ();
if (!self->manager) {
GST_ERROR_OBJECT (self, "Failed to get sensor manager");
goto error_sensor_type_name;
}
}
/* Replace sensor object. */
if (self->sensor) {
gst_amc_jni_object_unref (env, self->sensor->object);
g_slice_free (GstAHSensor, self->sensor);
}
self->sensor = gst_ah_sensor_get_default_sensor (self->manager,
self->sensor_type);
if (!self->sensor) {
GST_ERROR_OBJECT (self, "Failed to get sensor type %s",
self->sensor_type_name);
goto error_manager;
}
/* Register for the callback, unregistering first if necessary. */
if (!gst_ahs_src_register_callback (self))
goto error_sensor;
return TRUE;
error_sensor:
gst_amc_jni_object_unref (env, self->sensor->object);
g_slice_free (GstAHSensor, self->sensor);
self->sensor = NULL;
error_manager:
gst_amc_jni_object_unref (env, self->manager->object);
g_slice_free (GstAHSensorManager, self->manager);
self->manager = NULL;
error_sensor_type_name:
g_free ((gpointer) self->sensor_type_name);
self->sensor_type_name = NULL;
return FALSE;
}
static gboolean
gst_ahs_src_set_caps (GstBaseSrc * src, GstCaps * caps)
{
const GstStructure *caps_struct;
GstAHSSrc *self = GST_AHS_SRC (src);
gboolean success;
gint type;
const gchar *type_str;
GEnumValue *value;
caps_struct = gst_caps_get_structure (caps, 0);
type_str = gst_structure_get_string (caps_struct, "type");
value = g_enum_get_value_by_name (self->sensor_enum_class, type_str);
if (!value) {
GST_ERROR_OBJECT (self, "Failed to lookup sensor type %s", type_str);
return FALSE;
}
type_str = g_strdup (type_str);
type = value->value;
/*
* Take the mutex while changing the sensor type in case there are concurrent
* callbacks being processed.
*/
GST_OBJECT_LOCK (self);
success = gst_ahs_src_change_sensor_type (self, type_str, type);
GST_OBJECT_UNLOCK (self);
if (!success)
return FALSE;
return TRUE;
}
static gboolean
gst_ahs_src_start (GstBaseSrc * src)
{
JNIEnv *env = gst_amc_jni_get_env ();
GstAHSSrc *self = GST_AHS_SRC (src);
g_assert_null (self->manager);
g_assert_null (self->listener);
self->manager = gst_ah_sensor_get_manager ();
if (!self->manager) {
GST_ERROR_OBJECT (self, "Failed to get sensor manager");
goto error;
}
self->previous_time = GST_CLOCK_TIME_NONE;
self->listener = gst_ah_sensor_create_listener (gst_ahs_src_on_sensor_changed,
gst_ahs_src_on_accuracy_changed, self);
if (!self->listener) {
GST_ERROR_OBJECT (self, "Failed to create sensor listener");
goto error_manager;
}
return TRUE;
error_manager:
gst_amc_jni_object_unref (env, self->manager->object);
g_slice_free (GstAHSensorManager, self->manager);
self->manager = NULL;
error:
return FALSE;
}
static gboolean
gst_ahs_src_stop (GstBaseSrc * src)
{
GstAHSSrc *self = GST_AHS_SRC (src);
g_assert_nonnull (self->manager);
g_assert_nonnull (self->sensor);
g_assert_nonnull (self->listener);
gst_ah_sensor_unregister_listener (self->manager, self->listener);
self->previous_time = GST_CLOCK_TIME_NONE;
return TRUE;
}
static gboolean
gst_ahs_src_get_size (GstBaseSrc * src, guint64 * size)
{
GstAHSSrc *self = GST_AHS_SRC (src);
return self->buffer_size;
}
static gboolean
gst_ahs_src_is_seekable (GstBaseSrc * src)
{
return FALSE;
}
static gboolean
gst_ahs_src_unlock (GstBaseSrc * src)
{
GstAHSSrc *self = GST_AHS_SRC (src);
gst_data_queue_set_flushing (self->queue, TRUE);
return TRUE;
}
static gboolean
gst_ahs_src_unlock_stop (GstBaseSrc * src)
{
GstAHSSrc *self = GST_AHS_SRC (src);
gst_data_queue_set_flushing (self->queue, FALSE);
return TRUE;
}
static GstFlowReturn
gst_ahs_src_create (GstPushSrc * src, GstBuffer ** buffer)
{
GstAHSSrc *self = GST_AHS_SRC (src);
GstDataQueueItem *item;
if (!gst_data_queue_pop (self->queue, &item)) {
GST_INFO_OBJECT (self, "data queue is empty");
return GST_FLOW_FLUSHING;
}
GST_DEBUG_OBJECT (self, "creating buffer %p->%p", item, item->object);
*buffer = GST_BUFFER (item->object);
g_slice_free (GstDataQueueItem, item);
return GST_FLOW_OK;
}
static void
gst_ahs_src_free_data_queue_item (GstDataQueueItem * item)
{
gst_buffer_unref (GST_BUFFER (item->object));
g_slice_free (GstDataQueueItem, item);
}
static void
gst_ahs_src_update_smoothing (GstAHSSrc * self, const GstAHSensorEvent * event)
{
gint i;
/*
* Since we're doing exponential smoothing, the first sample needs to be
* special-cased to prevent it from being artificially lowered by the alpha
* smoothing factor.
*/
if (self->sample_index == 0) {
for (i = 0; i < self->sample_length; i++) {
self->current_sample[i] = event->data.values[i];
}
} else {
for (i = 0; i < self->sample_length; i++)
self->current_sample[i] =
(1 - self->alpha) * self->current_sample[i] +
self->alpha * event->data.values[i];
}
}
static void
gst_ahs_src_on_sensor_changed (jobject event_object, gpointer user_data)
{
GstBuffer *buffer;
GstClockTime buffer_time;
gfloat *data;
GstAHSensorEvent event;
GstDataQueueItem *item;
GstClock *pipeline_clock;
GstAHSSrc *self = GST_AHS_SRC (user_data);
gboolean success;
GST_OBJECT_LOCK (self);
pipeline_clock = GST_ELEMENT_CLOCK (self);
/* If the clock is NULL, the pipeline is not yet set to PLAYING. */
if (pipeline_clock == NULL)
goto done;
/*
* Unfortunately, the timestamp reported in the Android SensorEvent timestamp
* is not guaranteed to use any particular clock. On some device models, it
* uses system time, and on other models, it uses monotonic time. In addition,
* in some cases, the units are microseconds, and in other cases they are
* nanoseconds. Thus we cannot slave it to the pipeline clock or use any
* similar strategy that would allow us to correlate the two clocks. So
* instead, we approximate the buffer timestamp using the pipeline clock.
*
* See here for more details on issues with the Android SensorEvent timestamp:
* https://code.google.com/p/android/issues/detail?id=7981
*/
buffer_time =
gst_clock_get_time (pipeline_clock) - GST_ELEMENT_CAST (self)->base_time;
success =
gst_ah_sensor_populate_event (&event, event_object, self->buffer_size);
if (!success) {
GST_ERROR_OBJECT (self, "Failed to populate sensor event");
goto done;
}
gst_ahs_src_update_smoothing (self, &event);
gst_ah_sensor_free_sensor_data (&event.data);
self->sample_index++;
if (self->sample_index < self->sample_interval)
goto done;
self->sample_index = 0;
/*
* We want to send off this sample; copy it into a separate data struct so we
* can continue using current_sample for aggregating future samples.
*/
data = g_malloc (self->buffer_size);
memcpy (data, self->current_sample, self->buffer_size);
/* Wrap the datapoint with a buffer and add it to the queue. */
buffer = gst_buffer_new_wrapped (data, self->buffer_size);
GST_BUFFER_DURATION (buffer) = GST_CLOCK_TIME_NONE;
GST_BUFFER_PTS (buffer) = buffer_time;
item = g_slice_new (GstDataQueueItem);
item->object = GST_MINI_OBJECT (buffer);
item->size = gst_buffer_get_size (buffer);
item->duration = GST_BUFFER_DURATION (buffer);
item->visible = TRUE;
item->destroy = (GDestroyNotify) gst_ahs_src_free_data_queue_item;
success = gst_data_queue_push (self->queue, item);
if (!success) {
GST_ERROR_OBJECT (self, "Could not add buffer to queue");
gst_ahs_src_free_data_queue_item (item);
goto done;
}
done:
GST_OBJECT_UNLOCK (self);
}
static void
gst_ahs_src_on_accuracy_changed (jobject sensor, gint accuracy,
gpointer user_data)
{
GstAHSSrc *self = GST_AHS_SRC (user_data);
/* TODO: Perhaps we should do something more with this information. */
GST_DEBUG_OBJECT (self, "Accuracy changed on sensor %p and is now %d", sensor,
accuracy);
}