amc: Add MLSDK implementation

meson_options.txt

@@ -161,6 +161,7 @@ option('winscreencap', type : 'feature', value : 'auto', description : 'Windows
 option('x265', type : 'feature', value : 'auto', description : 'HEVC/H.265 video encoder plugin')
 option('zbar', type : 'feature', value : 'auto', description : 'Barcode image scanner plugin')
 option('wpe', type : 'feature', value : 'auto', description : 'WPE Web browser plugin')
+option('magicleap', type : 'feature', value : 'auto', description : 'Magic Leap platform support')
 
 # HLS plugin options
 option('hls', type : 'feature', value : 'auto', description : 'HTTP Live Streaming plugin')

sys/androidmedia/gstamc.c

@@ -29,8 +29,11 @@
 #define orc_memcpy memcpy
 #endif
 
+#ifdef HAVE_JNI_H
 #include "gstahcsrc.h"
 #include "gstahssrc.h"
+#include "gstjniutils.h"
+#endif
 
 #include "gstamc.h"
 #include "gstamc-constants.h"
@@ -38,7 +41,6 @@
 #include "gstamcvideodec.h"
 #include "gstamcvideoenc.h"
 #include "gstamcaudiodec.h"
-#include "gstjniutils.h"
 
 #include <gst/gst.h>
 #include <gst/video/video.h>
@@ -196,7 +198,7 @@ scan_codecs (GstPlugin * plugin)
       valid_codec = FALSE;
       goto next_codec;
     }
-
+#ifdef HAVE_JNI_H
     /* FIXME: Non-Google codecs usually just don't work and hang forever
      * or crash when not used from a process that started the Java
      * VM via the non-public AndroidRuntime class. Can we somehow
@@ -208,6 +210,7 @@ scan_codecs (GstPlugin * plugin)
       valid_codec = FALSE;
       goto next_codec;
     }
+#endif
 
     if (g_str_has_prefix (name_str, "OMX.ARICENT.")) {
       GST_INFO ("Skipping possible broken codec '%s'", name_str);
@@ -1848,6 +1851,7 @@ amc_init (GstPlugin * plugin)
   return TRUE;
 }
 
+#ifdef HAVE_JNI_H
 static gboolean
 ahc_init (GstPlugin * plugin)
 {
@@ -1885,6 +1889,7 @@ ahs_init (GstPlugin * plugin)
 
   return TRUE;
 }
+#endif
 
 static gboolean
 plugin_init (GstPlugin * plugin)
@@ -1893,17 +1898,21 @@ plugin_init (GstPlugin * plugin)
 
   GST_DEBUG_CATEGORY_INIT (gst_amc_debug, "amc", 0, "android-media-codec");
 
+#ifdef HAVE_JNI_H
   if (!gst_amc_jni_initialize ())
     return FALSE;
+#endif
 
   if (amc_init (plugin))
     init_ok = TRUE;
 
+#ifdef HAVE_JNI_H
   if (ahc_init (plugin))
     init_ok = TRUE;
 
   if (ahs_init (plugin))
     init_ok = TRUE;
+#endif
 
   return init_ok;
 }

sys/androidmedia/magicleap/gstamc-codec-ml.c

@@ -0,0 +1,357 @@
+/*
+ * Copyright (C) 2018 Collabora Ltd.
+ *   Author: Xavier Claessens <xavier.claessens@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstamc-internal-ml.h"
+#include "gstamc-surfacetexture-ml.h"
+#include "../gstamc-codec.h"
+#include "../gstamc-constants.h"
+
+#include <ml_media_codec.h>
+
+struct _GstAmcCodec
+{
+  MLHandle handle;
+  GstAmcSurfaceTexture *surface_texture;
+};
+
+gboolean
+gst_amc_codec_static_init (void)
+{
+  return TRUE;
+}
+
+void
+gst_amc_buffer_free (GstAmcBuffer * buffer)
+{
+  g_free (buffer);
+}
+
+gboolean
+gst_amc_buffer_set_position_and_limit (GstAmcBuffer * buffer, GError ** err,
+    gint position, gint limit)
+{
+/* FIXME: Do we need to do something?
+  buffer->data = buffer->data + position;
+  buffer->size = limit;
+*/
+  return TRUE;
+}
+
+GstAmcCodec *
+gst_amc_codec_new (const gchar * name, gboolean is_encoder, GError ** err)
+{
+  GstAmcCodec *codec = NULL;
+  MLResult result;
+  MLMediaCodecType type;
+
+  g_return_val_if_fail (name != NULL, NULL);
+
+  codec = g_slice_new0 (GstAmcCodec);
+  codec->handle = ML_INVALID_HANDLE;
+  type = is_encoder ? MLMediaCodecType_Encoder : MLMediaCodecType_Decoder;
+  result =
+      MLMediaCodecCreateCodec (MLMediaCodecCreation_ByName, type, name,
+      &codec->handle);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to create codec by name %s: %d", name, result);
+    gst_amc_codec_free (codec);
+    return NULL;
+  }
+
+  return codec;
+}
+
+void
+gst_amc_codec_free (GstAmcCodec * codec)
+{
+  g_return_if_fail (codec != NULL);
+
+  if (codec->handle != ML_INVALID_HANDLE)
+    MLMediaCodecDestroy (codec->handle);
+  g_clear_object (&codec->surface_texture);
+  g_slice_free (GstAmcCodec, codec);
+}
+
+gboolean
+gst_amc_codec_configure (GstAmcCodec * codec, GstAmcFormat * format,
+    GstAmcSurfaceTexture * surface_texture, GError ** err)
+{
+  MLResult result;
+  MLHandle surface_handle = ML_INVALID_HANDLE;
+
+  g_return_val_if_fail (codec != NULL, FALSE);
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (surface_texture == NULL
+      || GST_IS_AMC_SURFACE_TEXTURE_ML (surface_texture), FALSE);
+
+  g_set_object (&codec->surface_texture, surface_texture);
+  if (surface_texture != NULL)
+    surface_handle =
+        gst_amc_surface_texture_ml_get_handle ((GstAmcSurfaceTextureML *)
+        surface_texture);
+
+  result = MLMediaCodecConfigureWithSurface (codec->handle,
+      gst_amc_format_get_handle (format), surface_handle, 0);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to configure codec %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+GstAmcFormat *
+gst_amc_codec_get_output_format (GstAmcCodec * codec, GError ** err)
+{
+  MLHandle format_handle;
+  MLResult result;
+
+  g_return_val_if_fail (codec != NULL, NULL);
+
+  result = MLMediaCodecGetOutputFormat (codec->handle, &format_handle);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get output format %d", result);
+    return NULL;
+  }
+
+  return gst_amc_format_new_handle (format_handle);
+}
+
+gboolean
+gst_amc_codec_start (GstAmcCodec * codec, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (codec != NULL, FALSE);
+
+  result = MLMediaCodecStart (codec->handle);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to start codec %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_codec_stop (GstAmcCodec * codec, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (codec != NULL, FALSE);
+
+  result = MLMediaCodecStop (codec->handle);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to stop codec %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_codec_flush (GstAmcCodec * codec, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (codec != NULL, FALSE);
+
+  result = MLMediaCodecFlush (codec->handle);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to flush codec %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_codec_release (GstAmcCodec * codec, GError ** err)
+{
+  g_return_val_if_fail (codec != NULL, FALSE);
+  return TRUE;
+}
+
+GstAmcBuffer *
+gst_amc_codec_get_output_buffer (GstAmcCodec * codec, gint index, GError ** err)
+{
+  MLResult result;
+  GstAmcBuffer *ret;
+
+  g_return_val_if_fail (codec != NULL, NULL);
+  g_return_val_if_fail (index >= 0, NULL);
+
+  ret = g_new0 (GstAmcBuffer, 1);
+
+  /* When configured with a surface, getting the buffer pointer makes no sense,
+   * but on Android it's not an error, it just return NULL buffer.
+   * But MLMediaCodecGetInputBufferPointer() will return an error instead. */
+  if (codec->surface_texture != NULL) {
+    return ret;
+  }
+
+  result =
+      MLMediaCodecGetOutputBufferPointer (codec->handle, index,
+      (const uint8_t **) &ret->data, &ret->size);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get output buffer %d", result);
+    g_free (ret);
+    return NULL;
+  }
+
+  return ret;
+}
+
+GstAmcBuffer *
+gst_amc_codec_get_input_buffer (GstAmcCodec * codec, gint index, GError ** err)
+{
+  MLResult result;
+  GstAmcBuffer *ret;
+
+  g_return_val_if_fail (codec != NULL, NULL);
+  g_return_val_if_fail (index >= 0, NULL);
+
+  ret = g_new0 (GstAmcBuffer, 1);
+
+  result =
+      MLMediaCodecGetInputBufferPointer (codec->handle, index, &ret->data,
+      &ret->size);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get input buffer %d", result);
+    g_free (ret);
+    return NULL;
+  }
+
+  return ret;
+}
+
+gint
+gst_amc_codec_dequeue_input_buffer (GstAmcCodec * codec, gint64 timeoutUs,
+    GError ** err)
+{
+  MLResult result;
+  int64_t index;
+
+  g_return_val_if_fail (codec != NULL, G_MININT);
+
+  result = MLMediaCodecDequeueInputBuffer (codec->handle, timeoutUs, &index);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to dequeue input buffer %d", result);
+    return G_MININT;
+  }
+
+  if (index == MLMediaCodec_TryAgainLater)
+    return INFO_TRY_AGAIN_LATER;
+
+  return index;
+}
+
+gint
+gst_amc_codec_dequeue_output_buffer (GstAmcCodec * codec,
+    GstAmcBufferInfo * info, gint64 timeoutUs, GError ** err)
+{
+  MLMediaCodecBufferInfo info_;
+  MLResult result;
+  int64_t index;
+
+  g_return_val_if_fail (codec != NULL, G_MININT);
+
+  result =
+      MLMediaCodecDequeueOutputBuffer (codec->handle, &info_, timeoutUs,
+      &index);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to dequeue output buffer %d", result);
+    return G_MININT;
+  }
+
+  if (index == MLMediaCodec_OutputBuffersChanged) {
+    return gst_amc_codec_dequeue_output_buffer (codec, info, timeoutUs, err);
+  } else if (index == MLMediaCodec_FormatChanged) {
+    return INFO_OUTPUT_FORMAT_CHANGED;
+  } else if (index == MLMediaCodec_TryAgainLater) {
+    return INFO_TRY_AGAIN_LATER;
+  }
+
+  info->flags = info_.flags;
+
+  info->offset = info_.offset;
+  info->presentation_time_us = info_.presentation_time_us;
+  info->size = info_.size;
+
+  return index;
+}
+
+gboolean
+gst_amc_codec_queue_input_buffer (GstAmcCodec * codec, gint index,
+    const GstAmcBufferInfo * info, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (codec != NULL, FALSE);
+  g_return_val_if_fail (info != NULL, FALSE);
+
+  result = MLMediaCodecQueueInputBuffer (codec->handle, index, info->offset,
+      info->size, info->presentation_time_us, info->flags);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to queue input buffer %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_codec_release_output_buffer (GstAmcCodec * codec, gint index,
+    gboolean render, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (codec != NULL, FALSE);
+
+  result = MLMediaCodecReleaseOutputBuffer (codec->handle, index, render);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to release output buffer %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+GstAmcSurfaceTexture *
+gst_amc_codec_new_surface_texture (GError ** err)
+{
+  return (GstAmcSurfaceTexture *) gst_amc_surface_texture_ml_new (err);
+}

sys/androidmedia/magicleap/gstamc-codeclist-ml.c

@@ -0,0 +1,214 @@
+/*
+ * Copyright (C) 2018 Collabora Ltd.
+ *   Author: Xavier Claessens <xavier.claessens@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "../gstamc-codeclist.h"
+
+#include <ml_media_codeclist.h>
+
+struct _GstAmcCodecInfoHandle
+{
+  uint64_t index;
+};
+
+struct _GstAmcCodecCapabilitiesHandle
+{
+  uint64_t index;
+  gchar *type;
+};
+
+gboolean
+gst_amc_codeclist_static_init (void)
+{
+  return TRUE;
+}
+
+gboolean
+gst_amc_codeclist_get_count (gint * count, GError ** err)
+{
+  MLResult result;
+  uint64_t n;
+
+  result = MLMediaCodecListCountCodecs (&n);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get codec list count: %d", result);
+    return FALSE;
+  }
+
+  *count = n;
+
+  return TRUE;
+}
+
+GstAmcCodecInfoHandle *
+gst_amc_codeclist_get_codec_info_at (gint index, GError ** err)
+{
+  GstAmcCodecInfoHandle *ret = g_new0 (GstAmcCodecInfoHandle, 1);
+  ret->index = index;
+  return ret;
+}
+
+void
+gst_amc_codec_info_handle_free (GstAmcCodecInfoHandle * handle)
+{
+  g_free (handle);
+}
+
+gchar *
+gst_amc_codec_info_handle_get_name (GstAmcCodecInfoHandle * handle,
+    GError ** err)
+{
+  MLResult result;
+  gchar *name;
+
+  name = g_new0 (gchar, MAX_CODEC_NAME_LENGTH);
+  result = MLMediaCodecListGetCodecName (handle->index, name);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get codec name: %d", result);
+    g_free (name);
+    return NULL;
+  }
+
+  return name;
+}
+
+gboolean
+gst_amc_codec_info_handle_is_encoder (GstAmcCodecInfoHandle * handle,
+    gboolean * is_encoder, GError ** err)
+{
+  MLResult result;
+  bool out;
+
+  result = MLMediaCodecListIsEncoder (handle->index, &out);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to check if codec is an encoder: %d", result);
+    return FALSE;
+  }
+
+  *is_encoder = out;
+
+  return TRUE;
+}
+
+gchar **
+gst_amc_codec_info_handle_get_supported_types (GstAmcCodecInfoHandle * handle,
+    gsize * length, GError ** err)
+{
+  MLMediaCodecListQueryResults types;
+  MLResult result;
+  gchar **ret;
+  gsize i;
+
+  result = MLMediaCodecListGetSupportedMimes (handle->index, &types);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get codec supported types: %d", result);
+    return NULL;
+  }
+
+  *length = types.count;
+  ret = g_new0 (gchar *, *length + 1);
+  for (i = 0; i < *length; i++)
+    ret[i] = g_strdup (types.data[i]);
+
+  MLMediaCodecListQueryResultsRelease (&types);
+
+  return ret;
+}
+
+GstAmcCodecCapabilitiesHandle *
+gst_amc_codec_info_handle_get_capabilities_for_type (GstAmcCodecInfoHandle *
+    handle, const gchar * type, GError ** err)
+{
+  GstAmcCodecCapabilitiesHandle *ret;
+
+  ret = g_new0 (GstAmcCodecCapabilitiesHandle, 1);
+  ret->index = handle->index;
+  ret->type = g_strdup (type);
+  return ret;
+}
+
+void
+gst_amc_codec_capabilities_handle_free (GstAmcCodecCapabilitiesHandle * handle)
+{
+  g_free (handle->type);
+  g_free (handle);
+}
+
+gint *gst_amc_codec_capabilities_handle_get_color_formats
+    (GstAmcCodecCapabilitiesHandle * handle, gsize * length, GError ** err)
+{
+  uint32_t *colorFormats;
+  MLResult result;
+  gint *ret;
+  gsize i;
+
+  result =
+      MLMediaCodecListGetSupportedColorFormats (handle->index, handle->type,
+      &colorFormats, length);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get codec supported color formats: %d", result);
+    return NULL;
+  }
+
+  ret = g_new0 (gint, *length);
+  for (i = 0; i < *length; i++) {
+    ret[i] = colorFormats[i];
+  }
+
+  MLMediaCodecListColorFormatsRelease (colorFormats);
+
+  return ret;
+}
+
+GstAmcCodecProfileLevel *gst_amc_codec_capabilities_handle_get_profile_levels
+    (GstAmcCodecCapabilitiesHandle * handle, gsize * length, GError ** err)
+{
+  MLMediaCodecListProfileLevel *profileLevels;
+  GstAmcCodecProfileLevel *ret;
+  MLResult result;
+  gsize i;
+
+  result =
+      MLMediaCodecListGetSupportedProfileLevels (handle->index, handle->type,
+      &profileLevels, length);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get codec supported types: %d", result);
+    return NULL;
+  }
+
+  ret = g_new0 (GstAmcCodecProfileLevel, *length);
+  for (i = 0; i < *length; i++) {
+    ret[i].profile = profileLevels[i].profile;
+    ret[i].level = profileLevels[i].level;
+  }
+
+  MLMediaCodecListProfileLevelsRelease (profileLevels);
+
+  return ret;
+}

sys/androidmedia/magicleap/gstamc-format-ml.c

@@ -0,0 +1,280 @@
+/*
+ * Copyright (C) 2018 Collabora Ltd.
+ *   Author: Xavier Claessens <xavier.claessens@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstamc-internal-ml.h"
+#include "../gstamc-format.h"
+
+#include <ml_media_format.h>
+
+struct _GstAmcFormat
+{
+  MLHandle handle;
+};
+
+gboolean
+gst_amc_format_static_init (void)
+{
+  return TRUE;
+}
+
+GstAmcFormat *
+gst_amc_format_new_audio (const gchar * mime, gint sample_rate, gint channels,
+    GError ** err)
+{
+  GstAmcFormat *format = g_slice_new0 (GstAmcFormat);
+  MLResult result;
+
+  result =
+      MLMediaFormatCreateAudio (mime, sample_rate, channels, &format->handle);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to create audio format");
+    g_slice_free (GstAmcFormat, format);
+    return NULL;
+  }
+
+  return format;
+}
+
+GstAmcFormat *
+gst_amc_format_new_video (const gchar * mime, gint width, gint height,
+    GError ** err)
+{
+  GstAmcFormat *format = g_slice_new0 (GstAmcFormat);
+  MLResult result;
+
+  result = MLMediaFormatCreateVideo (mime, width, height, &format->handle);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to create video format");
+    g_slice_free (GstAmcFormat, format);
+    return NULL;
+  }
+
+  return format;
+}
+
+GstAmcFormat *
+gst_amc_format_new_handle (MLHandle handle)
+{
+  GstAmcFormat *format = g_slice_new0 (GstAmcFormat);
+  format->handle = handle;
+  return format;
+}
+
+MLHandle
+gst_amc_format_get_handle (GstAmcFormat * format)
+{
+  return format->handle;
+}
+
+void
+gst_amc_format_free (GstAmcFormat * format)
+{
+  g_return_if_fail (format != NULL);
+  g_slice_free (GstAmcFormat, format);
+}
+
+gchar *
+gst_amc_format_to_string (GstAmcFormat * format, GError ** err)
+{
+  MLResult result;
+  gchar *str;
+
+  str = g_new0 (gchar, MAX_FORMAT_STRING_SIZE);
+  result = MLMediaFormatObjectToString (format->handle, str);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to convert format to string: %d", result);
+    g_free (str);
+    return NULL;
+  }
+
+  return str;
+}
+
+gboolean
+gst_amc_format_get_float (GstAmcFormat * format, const gchar * key,
+    gfloat * value, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (key != NULL, FALSE);
+  g_return_val_if_fail (value != NULL, FALSE);
+
+  result = MLMediaFormatGetKeyValueFloat (format->handle, key, value);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get float");
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_format_set_float (GstAmcFormat * format, const gchar * key,
+    gfloat value, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (key != NULL, FALSE);
+
+  result = MLMediaFormatSetKeyFloat (format->handle, key, value);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to set float");
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_format_get_int (GstAmcFormat * format, const gchar * key, gint * value,
+    GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (key != NULL, FALSE);
+  g_return_val_if_fail (value != NULL, FALSE);
+
+  result = MLMediaFormatGetKeyValueInt32 (format->handle, key, value);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get int");
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_format_set_int (GstAmcFormat * format, const gchar * key, gint value,
+    GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (key != NULL, FALSE);
+
+  result = MLMediaFormatSetKeyInt32 (format->handle, key, value);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to set int");
+    return FALSE;
+  }
+  return TRUE;
+}
+
+gboolean
+gst_amc_format_get_string (GstAmcFormat * format, const gchar * key,
+    gchar ** value, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (key != NULL, FALSE);
+  g_return_val_if_fail (value != NULL, FALSE);
+
+  *value = g_new0 (gchar, MAX_KEY_STRING_SIZE);
+  result = MLMediaFormatGetKeyString (format->handle, key, *value);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get string");
+    g_clear_pointer (value, g_free);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_format_set_string (GstAmcFormat * format, const gchar * key,
+    const gchar * value, GError ** err)
+{
+  MLResult result;
+
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (key != NULL, FALSE);
+
+  result = MLMediaFormatSetKeyString (format->handle, key, value);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to set string");
+    return FALSE;
+  }
+  return TRUE;
+}
+
+gboolean
+gst_amc_format_get_buffer (GstAmcFormat * format, const gchar * key,
+    guint8 ** data, gsize * size, GError ** err)
+{
+  MLResult result;
+  MLMediaFormatByteArray buffer;
+
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (key != NULL, FALSE);
+  g_return_val_if_fail (data != NULL, FALSE);
+  g_return_val_if_fail (size != NULL, FALSE);
+
+  result = MLMediaFormatGetKeyByteBuffer (format->handle, key, &buffer);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get buffer");
+    return FALSE;
+  }
+
+  *size = buffer.length;
+  *data = (guint8 *) g_memdup (buffer.ptr, buffer.length);
+  MLMediaFormatKeyByteBufferRelease (format->handle, &buffer);
+
+  return TRUE;
+}
+
+gboolean
+gst_amc_format_set_buffer (GstAmcFormat * format, const gchar * key,
+    guint8 * data, gsize size, GError ** err)
+{
+  MLResult result;
+  MLMediaFormatByteArray buffer;
+
+  g_return_val_if_fail (format != NULL, FALSE);
+  g_return_val_if_fail (key != NULL, FALSE);
+  g_return_val_if_fail (data != NULL, FALSE);
+
+  buffer.ptr = data;
+  buffer.length = size;
+  result = MLMediaFormatSetKeyByteBuffer (format->handle, key, &buffer);
+  if (result != MLResult_Ok) {
+    g_set_error_literal (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to set buffer");
+    return FALSE;
+  }
+  return TRUE;
+}

sys/androidmedia/magicleap/gstamc-internal-ml.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2018 Collabora Ltd.
+ *   Author: Xavier Claessens <xavier.claessens@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifndef __GST_AMC_INTERNAL_ML_H__
+#define __GST_AMC_INTERNAL_ML_H__
+
+#include "../gstamc-format.h"
+#include <ml_api.h>
+
+G_BEGIN_DECLS
+
+GstAmcFormat *gst_amc_format_new_handle (MLHandle handle);
+MLHandle gst_amc_format_get_handle (GstAmcFormat * format);
+
+G_END_DECLS
+
+#endif /* __GST_AMC_INTERNAL_ML_H__ */

sys/androidmedia/magicleap/gstamc-surfacetexture-ml.c

@@ -0,0 +1,232 @@
+/*
+ * Copyright (C) 2018 Collabora Ltd.
+ *   Author: Xavier Claessens <xavier.claessens@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstamc-surfacetexture-ml.h"
+
+#include <gst/gst.h>
+#include <ml_media_surface_texture.h>
+
+struct _GstAmcSurfaceTextureML
+{
+  GObject parent;
+
+  MLHandle handle;
+  GstAmcSurfaceTextureOnFrameAvailableCallback callback;
+  gpointer user_data;
+};
+
+G_DEFINE_TYPE (GstAmcSurfaceTextureML, gst_amc_surface_texture_ml,
+    GST_TYPE_AMC_SURFACE_TEXTURE);
+
+gboolean
+gst_amc_surface_texture_static_init (void)
+{
+  return TRUE;
+}
+
+static gboolean
+gst_amc_surface_texture_ml_update_tex_image (GstAmcSurfaceTexture * base,
+    GError ** err)
+{
+  GstAmcSurfaceTextureML *self = GST_AMC_SURFACE_TEXTURE_ML (base);
+  MLResult result;
+
+  result = MLMediaSurfaceTextureUpdateTexImage (self->handle);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to update tex image: %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_amc_surface_texture_ml_detach_from_gl_context (GstAmcSurfaceTexture * base,
+    GError ** err)
+{
+  GstAmcSurfaceTextureML *self = GST_AMC_SURFACE_TEXTURE_ML (base);
+  MLResult result;
+
+  result = MLMediaSurfaceTextureDetachFromGLContext (self->handle);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to detach from gl context: %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_amc_surface_texture_ml_attach_to_gl_context (GstAmcSurfaceTexture * base,
+    gint texture_id, GError ** err)
+{
+  GstAmcSurfaceTextureML *self = GST_AMC_SURFACE_TEXTURE_ML (base);
+  MLResult result;
+
+  result = MLMediaSurfaceTextureAttachToGLContext (self->handle, texture_id);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to attach to gl context: %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_amc_surface_texture_ml_get_transform_matrix (GstAmcSurfaceTexture * base,
+    gfloat * matrix, GError ** err)
+{
+  GstAmcSurfaceTextureML *self = GST_AMC_SURFACE_TEXTURE_ML (base);
+  MLResult result;
+
+  result = MLMediaSurfaceTextureGetTransformationMatrix (self->handle, matrix);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get transformation matrix: %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_amc_surface_texture_ml_get_timestamp (GstAmcSurfaceTexture * base,
+    gint64 * timestamp, GError ** err)
+{
+  GstAmcSurfaceTextureML *self = GST_AMC_SURFACE_TEXTURE_ML (base);
+  MLResult result;
+
+  result = MLMediaSurfaceTextureGetTimestamp (self->handle, timestamp);
+  if (result != MLResult_Ok) {
+    g_set_error (err, GST_LIBRARY_ERROR, GST_LIBRARY_ERROR_FAILED,
+        "Failed to get timestamp: %d", result);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_amc_surface_texture_ml_release (GstAmcSurfaceTexture * base, GError ** err)
+{
+  /* Nothing to do here, resources will be released when this object gets
+   * destroyed. */
+  return TRUE;
+}
+
+static gboolean
+    gst_amc_surface_texture_ml_set_on_frame_available_callback
+    (GstAmcSurfaceTexture * base,
+    GstAmcSurfaceTextureOnFrameAvailableCallback callback, gpointer user_data,
+    GError ** err)
+{
+  GstAmcSurfaceTextureML *self = GST_AMC_SURFACE_TEXTURE_ML (base);
+
+  self->callback = callback;
+  self->user_data = user_data;
+  return TRUE;
+}
+
+static void
+gst_amc_surface_texture_ml_dispose (GObject * object)
+{
+  GstAmcSurfaceTextureML *self = GST_AMC_SURFACE_TEXTURE_ML (object);
+
+  MLMediaSurfaceTextureSetOnFrameAvailableCallback (self->handle, NULL, NULL);
+  MLMediaSurfaceTextureDestroy (&self->handle);
+
+  G_OBJECT_CLASS (gst_amc_surface_texture_ml_parent_class)->dispose (object);
+}
+
+static void
+gst_amc_surface_texture_ml_class_init (GstAmcSurfaceTextureMLClass * klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+  GstAmcSurfaceTextureClass *surface_texture_class =
+      GST_AMC_SURFACE_TEXTURE_CLASS (klass);
+
+  gobject_class->dispose = gst_amc_surface_texture_ml_dispose;
+
+  surface_texture_class->update_tex_image =
+      gst_amc_surface_texture_ml_update_tex_image;
+  surface_texture_class->detach_from_gl_context =
+      gst_amc_surface_texture_ml_detach_from_gl_context;
+  surface_texture_class->attach_to_gl_context =
+      gst_amc_surface_texture_ml_attach_to_gl_context;
+  surface_texture_class->get_transform_matrix =
+      gst_amc_surface_texture_ml_get_transform_matrix;
+  surface_texture_class->get_timestamp =
+      gst_amc_surface_texture_ml_get_timestamp;
+  surface_texture_class->release = gst_amc_surface_texture_ml_release;
+  surface_texture_class->set_on_frame_available_callback =
+      gst_amc_surface_texture_ml_set_on_frame_available_callback;
+}
+
+static void
+on_frame_available_cb (MLHandle handle, gpointer user_data)
+{
+  GstAmcSurfaceTextureML *self = user_data;
+
+  if (self->callback != NULL)
+    self->callback (GST_AMC_SURFACE_TEXTURE (self), self->user_data);
+}
+
+static void
+gst_amc_surface_texture_ml_init (GstAmcSurfaceTextureML * self)
+{
+  MLResult result;
+
+  result =
+      MLMediaSurfaceTextureCreate (MLMediaSurfaceTextureBackend_OpenGL,
+      &self->handle);
+  if (result != MLResult_Ok) {
+    GST_ERROR ("MLMediaSurfaceTextureCreate returned error: %d", result);
+    return;
+  }
+
+  result =
+      MLMediaSurfaceTextureSetOnFrameAvailableCallback (self->handle,
+      on_frame_available_cb, self);
+  if (result != MLResult_Ok) {
+    GST_ERROR
+        ("MLMediaSurfaceTextureSetOnFrameAvailableCallback returned error: %d",
+        result);
+    return;
+  }
+}
+
+GstAmcSurfaceTextureML *
+gst_amc_surface_texture_ml_new (GError ** err)
+{
+  return g_object_new (GST_TYPE_AMC_SURFACE_TEXTURE_ML, NULL);
+}
+
+MLHandle
+gst_amc_surface_texture_ml_get_handle (GstAmcSurfaceTextureML * self)
+{
+  return self->handle;
+}

sys/androidmedia/magicleap/gstamc-surfacetexture-ml.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2018 Collabora Ltd.
+ *   Author: Xavier Claessens <xavier.claessens@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+#ifndef __GST_AMC_SURFACE_TEXTURE_ML_H__
+#define __GST_AMC_SURFACE_TEXTURE_ML_H__
+
+#include "../gstamcsurfacetexture.h"
+#include <ml_api.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_AMC_SURFACE_TEXTURE_ML gst_amc_surface_texture_ml_get_type ()
+G_DECLARE_FINAL_TYPE (GstAmcSurfaceTextureML, gst_amc_surface_texture_ml, GST, AMC_SURFACE_TEXTURE_ML, GstAmcSurfaceTexture)
+
+GstAmcSurfaceTextureML * gst_amc_surface_texture_ml_new (GError ** err);
+MLHandle gst_amc_surface_texture_ml_get_handle (GstAmcSurfaceTextureML * self);
+
+G_END_DECLS
+
+#endif

sys/androidmedia/meson.build

@@ -1,20 +1,9 @@
 androidmedia_sources = [
-  'gstahcsrc.c',
-  'gstahssrc.c',
   'gstamcaudiodec.c',
   'gstamc.c',
   'gstamcsurfacetexture.c',
   'gstamcvideodec.c',
   'gstamcvideoenc.c',
-  'gst-android-graphics-imageformat.c',
-  'gst-android-hardware-camera.c',
-  'gst-android-hardware-sensor.c',
-  'gstjniutils.c',
-  'jni/gstamc-codec-jni.c',
-  'jni/gstamc-codeclist-jni.c',
-  'jni/gstamc-format-jni.c',
-  'jni/gstamcsurface.c',
-  'jni/gstamcsurfacetexture-jni.c',
 ]
 
 androidmedia_java_sources = [
@@ -23,28 +12,65 @@ androidmedia_java_sources = [
   'org/freedesktop/gstreamer/androidmedia/GstAmcOnFrameAvailableListener.java',
 ]
 
-if host_system != 'android' or get_option('androidmedia').disabled()
+amc_opt = get_option('androidmedia')
+mgl_opt = get_option('magicleap')
+
+if host_system != 'android' or (amc_opt.disabled() and mgl_opt.disabled())
   subdir_done()
 endif
 
 if not gstgl_dep.found()
-  if get_option('androidmedia').enabled()
+  if amc_opt.enabled() or mgl_opt.enabled()
     error('androidmedia plugin enabled but GL support was not detected')
   endif
   subdir_done()
 endif
 
-have_jni_h = cc.has_header('jni.h', required : false)
+# Check if we have MLSDK
-if not have_jni_h and get_option('androidmedia').enabled()
+ml_deps = []
-  error('androidmedia plugin enabled but jni.h not found')
+have_mlsdk = true
+foreach lib : ['ml_mediacodec', 'ml_mediacodeclist', 'ml_mediaformat']
+  dep = cc.find_library(lib, required : mgl_opt)
+  have_mlsdk = have_mlsdk and dep.found()
+  ml_deps += dep
+endforeach
+
+extra_deps = []
+extra_cargs = []
+if have_mlsdk
+  androidmedia_sources += [
+    'magicleap/gstamc-codec-ml.c',
+    'magicleap/gstamc-codeclist-ml.c',
+    'magicleap/gstamc-format-ml.c',
+    'magicleap/gstamc-surfacetexture-ml.c',
+  ]
+  extra_deps = ml_deps
+  have_jni_h = false
+else
+  have_jni_h = cc.has_header('jni.h', required : amc_opt)
+  extra_cargs += '-DHAVE_JNI_H'
+  androidmedia_sources += [
+    'gstahcsrc.c',
+    'gstahssrc.c',
+    'gst-android-graphics-imageformat.c',
+    'gst-android-hardware-camera.c',
+    'gst-android-hardware-sensor.c',
+    'gstjniutils.c',
+    'jni/gstamc-codec-jni.c',
+    'jni/gstamc-codeclist-jni.c',
+    'jni/gstamc-format-jni.c',
+    'jni/gstamcsurface.c',
+    'jni/gstamcsurfacetexture-jni.c',
+  ]
 endif
 
-if have_jni_h
+if have_jni_h or have_mlsdk
   gstandroidmedia = library('gstandroidmedia',
     androidmedia_sources,
-    c_args : gst_plugins_bad_args,
+    c_args : [gst_plugins_bad_args, extra_cargs],
     include_directories : [configinc],
-    dependencies : [gstgl_dep, gstpbutils_dep, gstaudio_dep, gstvideo_dep, gstphotography_dep, gmodule_dep, orc_dep],
+    dependencies : [gstgl_dep, gstpbutils_dep, gstaudio_dep, gstvideo_dep,
+                    gstphotography_dep, gmodule_dep, orc_dep, extra_deps],
     install : true,
     install_dir : plugins_install_dir)
   pkgconfig.generate(gstandroidmedia, install_dir : plugins_pkgconfig_install_dir)