From 92bcdd9c4be315bb791c965c9748075667711c07 Mon Sep 17 00:00:00 2001 From: Alessandro Decina Date: Tue, 23 Jul 2013 10:30:39 +0200 Subject: [PATCH] applemedia: add AudioToolbox based audio decoder --- sys/applemedia/Makefile.am | 8 +- sys/applemedia/atdec.c | 405 +++++++++++++++++++++++++++++++++++++ sys/applemedia/atdec.h | 52 +++++ sys/applemedia/plugin.m | 2 + 4 files changed, 465 insertions(+), 2 deletions(-) create mode 100644 sys/applemedia/atdec.c create mode 100644 sys/applemedia/atdec.h diff --git a/sys/applemedia/Makefile.am b/sys/applemedia/Makefile.am index ccf675cf89..13f79942bd 100644 --- a/sys/applemedia/Makefile.am +++ b/sys/applemedia/Makefile.am @@ -7,6 +7,7 @@ libgstapplemedia_la_SOURCES = \ coremediabuffer.c \ coremediactx.c \ vtapi.c \ + atdec.c \ dynapi.c libgstapplemedia_la_CPPFLAGS = \ @@ -38,7 +39,8 @@ endif libgstapplemedia_la_LIBADD = \ $(GST_BASE_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \ - -lgstvideo-$(GST_API_VERSION) + -lgstvideo-$(GST_API_VERSION) \ + -lgstaudio-$(GST_API_VERSION) libgstapplemedia_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) @@ -48,7 +50,8 @@ libgstapplemedia_la_LDFLAGS = \ $(GST_PLUGINS_BASE_LDFLAGS) \ -Wl,-framework -Wl,CoreFoundation \ -Wl,-framework -Wl,CoreMedia \ - -Wl,-framework -Wl,CoreVideo + -Wl,-framework -Wl,CoreVideo \ + -Wl,-framework -Wl,AudioToolbox noinst_HEADERS = \ qtkitvideosrc.h \ @@ -62,6 +65,7 @@ noinst_HEADERS = \ coremediabuffer.h \ coremediactx.h \ vtapi.h \ + atdec.h \ dynapi.h \ dynapi-internal.h diff --git a/sys/applemedia/atdec.c b/sys/applemedia/atdec.c new file mode 100644 index 0000000000..6228c7bb86 --- /dev/null +++ b/sys/applemedia/atdec.c @@ -0,0 +1,405 @@ +/* GStreamer + * Copyright (C) 2013 Alessandro Decina + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Suite 500, + * Boston, MA 02110-1335, USA. + */ +/** + * SECTION:element-gstatdec + * + * AudioToolbox based decoder. + * + * + * Example launch line + * |[ + * gst-launch -v filesrc location=file.mov ! qtdemux ! queue ! aacparse ! atdec ! autoaudiosink + * ]| + * Decode aac audio from a mov file + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include "atdec.h" + +GST_DEBUG_CATEGORY_STATIC (gst_atdec_debug_category); +#define GST_CAT_DEFAULT gst_atdec_debug_category + +static void gst_atdec_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_atdec_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static void gst_atdec_finalize (GObject * object); + +static gboolean gst_atdec_start (GstAudioDecoder * decoder); +static gboolean gst_atdec_stop (GstAudioDecoder * decoder); +static gboolean gst_atdec_set_format (GstAudioDecoder * decoder, + GstCaps * caps); +static GstFlowReturn gst_atdec_handle_frame (GstAudioDecoder * decoder, + GstBuffer * buffer); +static void gst_atdec_flush (GstAudioDecoder * decoder, gboolean hard); +static void gst_atdec_buffer_emptied (void *user_data, + AudioQueueRef queue, AudioQueueBufferRef buffer); + +enum +{ + PROP_0 +}; + +static GstStaticPadTemplate gst_atdec_src_template = + GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE ("S16LE") ";" + GST_AUDIO_CAPS_MAKE ("F32LE") + ) + ); + +static GstStaticPadTemplate gst_atdec_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("audio/mpeg, mpegversion=4, framed=true," + "channels=[1,max]") + ); + +G_DEFINE_TYPE_WITH_CODE (GstATDec, gst_atdec, GST_TYPE_AUDIO_DECODER, + GST_DEBUG_CATEGORY_INIT (gst_atdec_debug_category, "atdec", 0, + "debug category for atdec element")); + +static GstStaticCaps aac_caps = GST_STATIC_CAPS ("audio/mpeg, mpegversion=4"); +static GstStaticCaps mp3_caps = +GST_STATIC_CAPS ("audio/mpeg, mpegversion=1, layer=3"); +static GstStaticCaps raw_caps = GST_STATIC_CAPS ("audio/x-raw"); + +static void +gst_atdec_class_init (GstATDecClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstAudioDecoderClass *audio_decoder_class = GST_AUDIO_DECODER_CLASS (klass); + + gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), + gst_static_pad_template_get (&gst_atdec_src_template)); + gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), + gst_static_pad_template_get (&gst_atdec_sink_template)); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "AudioToolbox based audio decoder", + "Codec/Decoder/Audio", + "AudioToolbox based audio decoder", + "Alessandro Decina "); + + gobject_class->set_property = gst_atdec_set_property; + gobject_class->get_property = gst_atdec_get_property; + gobject_class->finalize = gst_atdec_finalize; + audio_decoder_class->start = GST_DEBUG_FUNCPTR (gst_atdec_start); + audio_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_atdec_stop); + audio_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_atdec_set_format); + audio_decoder_class->handle_frame = + GST_DEBUG_FUNCPTR (gst_atdec_handle_frame); + audio_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_atdec_flush); +} + +static void +gst_atdec_init (GstATDec * atdec) +{ + gst_audio_decoder_set_needs_format (GST_AUDIO_DECODER (atdec), TRUE); + atdec->queue = NULL; +} + +void +gst_atdec_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstATDec *atdec = GST_ATDEC (object); + + GST_DEBUG_OBJECT (atdec, "set_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_atdec_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstATDec *atdec = GST_ATDEC (object); + + GST_DEBUG_OBJECT (atdec, "get_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +static void +gst_atdec_destroy_queue (GstATDec * atdec, gboolean drain) +{ + AudioQueueStop (atdec->queue, drain); + AudioQueueDispose (atdec->queue, true); + atdec->queue = NULL; +} + +void +gst_atdec_finalize (GObject * object) +{ + GstATDec *atdec = GST_ATDEC (object); + + GST_DEBUG_OBJECT (atdec, "finalize"); + + if (atdec->queue) + gst_atdec_destroy_queue (atdec, FALSE); + + G_OBJECT_CLASS (gst_atdec_parent_class)->finalize (object); +} + +static gboolean +gst_atdec_start (GstAudioDecoder * decoder) +{ + GstATDec *atdec = GST_ATDEC (decoder); + + GST_DEBUG_OBJECT (atdec, "start"); + + return TRUE; +} + +static gboolean +gst_atdec_stop (GstAudioDecoder * decoder) +{ + GstATDec *atdec = GST_ATDEC (decoder); + + gst_atdec_destroy_queue (atdec, FALSE); + + return TRUE; +} + +static gboolean +can_intersect_static_caps (GstCaps * caps, GstStaticCaps * caps1) +{ + GstCaps *tmp; + gboolean ret; + + tmp = gst_static_caps_get (caps1); + ret = gst_caps_can_intersect (caps, tmp); + gst_caps_unref (tmp); + + return ret; +} + +static gboolean +gst_caps_to_at_format (GstCaps * caps, AudioStreamBasicDescription * format) +{ + int channels = 0; + int rate = 0; + GstStructure *structure; + + structure = gst_caps_get_structure (caps, 0); + gst_structure_get_int (structure, "rate", &rate); + gst_structure_get_int (structure, "channels", &channels); + format->mSampleRate = rate; + format->mChannelsPerFrame = channels; + + if (can_intersect_static_caps (caps, &aac_caps)) + format->mFormatID = kAudioFormatMPEG4AAC; + else if (can_intersect_static_caps (caps, &mp3_caps)) + format->mFormatID = kAudioFormatMPEGLayer3; + else if (can_intersect_static_caps (caps, &raw_caps)) { + GstAudioFormat audio_format; + const char *audio_format_str; + + format->mFormatID = kAudioFormatLinearPCM; + format->mFramesPerPacket = 1; + + audio_format_str = gst_structure_get_string (structure, "format"); + if (!audio_format_str) + audio_format_str = "S16LE"; + + audio_format = gst_audio_format_from_string (audio_format_str); + switch (audio_format) { + case GST_AUDIO_FORMAT_S16LE: + format->mFormatFlags = + kLinearPCMFormatFlagIsPacked | kLinearPCMFormatFlagIsSignedInteger; + format->mBitsPerChannel = 16; + format->mBytesPerPacket = format->mBytesPerFrame = 2 * channels; + break; + case GST_AUDIO_FORMAT_F32LE: + format->mFormatFlags = + kLinearPCMFormatFlagIsPacked | kLinearPCMFormatFlagIsFloat; + format->mBitsPerChannel = 32; + format->mBytesPerPacket = format->mBytesPerFrame = 4 * channels; + break; + default: + g_warn_if_reached (); + break; + } + } + + return TRUE; +} + +static gboolean +gst_atdec_set_format (GstAudioDecoder * decoder, GstCaps * caps) +{ + OSStatus status; + AudioStreamBasicDescription input_format = { 0 }; + AudioStreamBasicDescription output_format = { 0 }; + GstAudioInfo output_info = { 0 }; + AudioChannelLayout output_layout = { 0 }; + GstCaps *output_caps; + GstATDec *atdec = GST_ATDEC (decoder); + + GST_DEBUG_OBJECT (atdec, "set_format"); + + if (atdec->queue) + gst_atdec_destroy_queue (atdec, TRUE); + + // configure input_format from caps + gst_caps_to_at_format (caps, &input_format); + + // negotiate output caps + output_caps = gst_pad_get_allowed_caps (GST_AUDIO_DECODER_SRC_PAD (atdec)); + output_caps = gst_caps_fixate (output_caps); + if (!output_caps) + goto negotiation_error; + + gst_caps_set_simple (output_caps, + "rate", G_TYPE_INT, (int) input_format.mSampleRate, NULL); + + // configure output_format from caps + gst_caps_to_at_format (output_caps, &output_format); + + // set the format we want to negotiate downstream + gst_audio_info_from_caps (&output_info, output_caps); + gst_audio_info_set_format (&output_info, + output_format.mFormatFlags & kLinearPCMFormatFlagIsSignedInteger ? + GST_AUDIO_FORMAT_S16LE : GST_AUDIO_FORMAT_F32LE, + output_format.mSampleRate, output_format.mChannelsPerFrame, NULL); + gst_audio_decoder_set_output_format (decoder, &output_info); + gst_caps_unref (output_caps); + + status = AudioQueueNewOutput (&input_format, gst_atdec_buffer_emptied, + atdec, NULL, NULL, 0, &atdec->queue); + if (status) + goto create_queue_error; + + // FIXME: figure out how to map this properly + if (output_format.mChannelsPerFrame == 1) + output_layout.mChannelLayoutTag = kAudioChannelLayoutTag_Mono; + else + output_layout.mChannelLayoutTag = kAudioChannelLayoutTag_Stereo; + + status = AudioQueueSetOfflineRenderFormat (atdec->queue, + &output_format, &output_layout); + if (status) + goto set_format_error; + + status = AudioQueueStart (atdec->queue, NULL); + if (status) + goto start_error; + + return TRUE; + +negotiation_error: + GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL), + ("no compatible downstream caps")); + return FALSE; + +create_queue_error: + GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL), + ("AudioQueueNewOutput returned error: %d", status)); + return FALSE; + +set_format_error: + GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL), + ("AudioQueueSetOfflineRenderFormat returned error: %d", status)); + return FALSE; + +start_error: + GST_ELEMENT_ERROR (atdec, STREAM, FORMAT, (NULL), + ("AudioQueueStart returned error: %d", status)); + return FALSE; +} + +static void +gst_atdec_buffer_emptied (void *user_data, AudioQueueRef queue, + AudioQueueBufferRef buffer) +{ + AudioQueueFreeBuffer (queue, buffer); +} + +static GstFlowReturn +gst_atdec_handle_frame (GstAudioDecoder * decoder, GstBuffer * buffer) +{ + AudioTimeStamp timestamp = { 0 }; + AudioStreamPacketDescription packet; + AudioQueueBufferRef input_buffer, output_buffer; + GstBuffer *out; + GstMapInfo info; + GstAudioInfo *audio_info; + int size, out_frames; + GstATDec *atdec = GST_ATDEC (decoder); + + // copy the input buffer into an AudioQueueBuffer + size = gst_buffer_get_size (buffer); + AudioQueueAllocateBuffer (atdec->queue, size, &input_buffer); + gst_buffer_extract (buffer, 0, input_buffer->mAudioData, size); + input_buffer->mAudioDataByteSize = size; + + // assume framed input + packet.mStartOffset = 0; + packet.mVariableFramesInPacket = 1; + packet.mDataByteSize = size; + + // enqueue the buffer. It will get free'd once the gst_atdec_buffer_emptied + // callback is called + AudioQueueEnqueueBuffer (atdec->queue, input_buffer, 1, &packet); + + // figure out how many frames we need to pull out of the queue + audio_info = gst_audio_decoder_get_audio_info (decoder); + out_frames = + GST_CLOCK_TIME_TO_FRAMES (GST_BUFFER_DURATION (buffer), audio_info->rate); + size = out_frames * audio_info->bpf; + AudioQueueAllocateBuffer (atdec->queue, size, &output_buffer); + + // pull the frames + AudioQueueOfflineRender (atdec->queue, ×tamp, output_buffer, out_frames); + out = + gst_audio_decoder_allocate_output_buffer (decoder, + output_buffer->mAudioDataByteSize); + gst_buffer_map (out, &info, GST_MAP_WRITE); + memcpy (info.data, output_buffer->mAudioData, + output_buffer->mAudioDataByteSize); + gst_buffer_unmap (out, &info); + AudioQueueFreeBuffer (atdec->queue, output_buffer); + + return gst_audio_decoder_finish_frame (decoder, out, 1); +} + +static void +gst_atdec_flush (GstAudioDecoder * decoder, gboolean hard) +{ + GstATDec *atdec = GST_ATDEC (decoder); + + AudioQueueFlush (atdec->queue); +} diff --git a/sys/applemedia/atdec.h b/sys/applemedia/atdec.h new file mode 100644 index 0000000000..8f9bbded79 --- /dev/null +++ b/sys/applemedia/atdec.h @@ -0,0 +1,52 @@ +/* GStreamer + * Copyright (C) 2013 FIXME + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _GST_ATDEC_H_ +#define _GST_ATDEC_H_ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_ATDEC (gst_atdec_get_type()) +#define GST_ATDEC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_ATDEC,GstATDec)) +#define GST_ATDEC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_ATDEC,GstATDecClass)) +#define GST_IS_ATDEC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_ATDEC)) +#define GST_IS_ATDEC_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_ATDEC)) + +typedef struct _GstATDec GstATDec; +typedef struct _GstATDecClass GstATDecClass; + +struct _GstATDec +{ + GstAudioDecoder decoder; + AudioQueueRef queue; +}; + +struct _GstATDecClass +{ + GstAudioDecoderClass decoder_class; +}; + +GType gst_atdec_get_type (void); + +G_END_DECLS + +#endif diff --git a/sys/applemedia/plugin.m b/sys/applemedia/plugin.m index da98244ef9..4548b143e3 100644 --- a/sys/applemedia/plugin.m +++ b/sys/applemedia/plugin.m @@ -31,6 +31,7 @@ #include "vth264encbin.h" #include "vtenc.h" #include "vtdec.h" +#include "atdec.h" #ifndef HAVE_IOS static void @@ -64,6 +65,7 @@ plugin_init (GstPlugin * plugin) res &= gst_element_register (plugin, "vth264encbin", GST_RANK_NONE, GST_TYPE_VT_H264_ENC_BIN); #endif + res &= gst_element_register (plugin, "atdec", GST_RANK_PRIMARY, GST_TYPE_ATDEC); gst_vtenc_register_elements (plugin); gst_vtdec_register_elements (plugin);