From 46cc64e09f16b8fecffc99969c9f4591df1bd61d Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Thu, 18 Jun 2020 04:03:59 +1000 Subject: [PATCH] mpegtsmux: Fix handling of MPEG-2 AAC The audio/mpeg,mpegversion=2 caps in GStreamer refer to MPEG-2 AAC (ISO 13818-7), not to the extended MP3 (ISO 13818-3), which is audio/mpeg,mpegversion=1,mpegaudioversion=2/3 Fix the caps, and add handling for MPEG-2 AAC in both ADTS and raw form, adding ADTS headers for the latter. --- docs/plugins/gst_plugins_cache.json | 2 +- gst/mpegtsmux/gstbasetsmux.c | 54 ++++++++--- gst/mpegtsmux/gstbasetsmuxaac.c | 141 ++++++++++++++++++++++++---- gst/mpegtsmux/gstbasetsmuxaac.h | 8 +- gst/mpegtsmux/gstmpegtsmux.c | 6 +- 5 files changed, 176 insertions(+), 35 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 8db5c5fd93..a5b09c1851 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -208846,7 +208846,7 @@ "long-name": "MPEG Transport Stream Muxer", "pad-templates": { "sink_%%d": { - "caps": "video/mpeg:\n parsed: true\n mpegversion: { (int)1, (int)2, (int)4 }\n systemstream: false\nvideo/x-dirac:\nimage/x-jpc:\nvideo/x-h264:\n stream-format: byte-stream\n alignment: { (string)au, (string)nal }\nvideo/x-h265:\n stream-format: byte-stream\n alignment: { (string)au, (string)nal }\naudio/mpeg:\n parsed: true\n mpegversion: { (int)1, (int)2 }\naudio/mpeg:\n framed: true\n mpegversion: 4\n stream-format: adts\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\naudio/x-lpcm:\n width: { (int)16, (int)20, (int)24 }\n rate: { (int)48000, (int)96000 }\n channels: [ 1, 8 ]\n dynamic_range: [ 0, 255 ]\n emphasis: { (boolean)false, (boolean)true }\n mute: { (boolean)false, (boolean)true }\naudio/x-ac3:\n framed: true\naudio/x-dts:\n framed: true\naudio/x-opus:\n channels: [ 1, 8 ]\nchannel-mapping-family: { (int)0, (int)1 }\nsubpicture/x-dvb:\napplication/x-teletext:\nmeta/x-klv:\n parsed: true\nimage/x-jpc:\n profile: [ 0, 49151 ]\n", + "caps": "video/mpeg:\n parsed: true\n mpegversion: { (int)1, (int)2, (int)4 }\n systemstream: false\nvideo/x-dirac:\nimage/x-jpc:\nvideo/x-h264:\n stream-format: byte-stream\n alignment: { (string)au, (string)nal }\nvideo/x-h265:\n stream-format: byte-stream\n alignment: { (string)au, (string)nal }\naudio/mpeg:\n parsed: true\n mpegversion: 1\naudio/mpeg:\n framed: true\n mpegversion: { (int)2, (int)4 }\n stream-format: { (string)adts, (string)raw }\naudio/x-lpcm:\n width: { (int)16, (int)20, (int)24 }\n rate: { (int)48000, (int)96000 }\n channels: [ 1, 8 ]\n dynamic_range: [ 0, 255 ]\n emphasis: { (boolean)false, (boolean)true }\n mute: { (boolean)false, (boolean)true }\naudio/x-ac3:\n framed: true\naudio/x-dts:\n framed: true\naudio/x-opus:\n channels: [ 1, 8 ]\nchannel-mapping-family: { (int)0, (int)1 }\nsubpicture/x-dvb:\napplication/x-teletext:\nmeta/x-klv:\n parsed: true\nimage/x-jpc:\n profile: [ 0, 49151 ]\n", "direction": "sink", "presence": "request", "type": "GstBaseTsMuxPad" diff --git a/gst/mpegtsmux/gstbasetsmux.c b/gst/mpegtsmux/gstbasetsmux.c index 462cf0ef06..fe7d061eda 100644 --- a/gst/mpegtsmux/gstbasetsmux.c +++ b/gst/mpegtsmux/gstbasetsmux.c @@ -389,6 +389,7 @@ gst_base_ts_mux_create_stream (GstBaseTsMux * mux, GstBaseTsMuxPad * ts_pad) guint32 max_rate = 0; guint8 color_spec = 0; j2k_private_data *private_data = NULL; + const gchar *stream_format = NULL; pad = GST_PAD (ts_pad); caps = gst_pad_get_current_caps (pad); @@ -405,6 +406,8 @@ gst_base_ts_mux_create_stream (GstBaseTsMux * mux, GstBaseTsMuxPad * ts_pad) if (value != NULL) codec_data = gst_value_get_buffer (value); + stream_format = gst_structure_get_string (s, "stream-format"); + if (strcmp (mt, "video/x-dirac") == 0) { st = TSMUX_ST_VIDEO_DIRAC; } else if (strcmp (mt, "audio/x-ac3") == 0) { @@ -426,23 +429,50 @@ gst_base_ts_mux_create_stream (GstBaseTsMux * mux, GstBaseTsMuxPad * ts_pad) } switch (mpegversion) { - case 1: - st = TSMUX_ST_AUDIO_MPEG1; + case 1:{ + int mpegaudioversion = 1; /* Assume mpegaudioversion=1 for backwards compatibility */ + (void) gst_structure_get_int (s, "mpegaudioversion", &mpegaudioversion); + + if (mpegaudioversion == 1) + st = TSMUX_ST_AUDIO_MPEG1; + else + st = TSMUX_ST_AUDIO_MPEG2; break; - case 2: - st = TSMUX_ST_AUDIO_MPEG2; + } + case 2:{ + /* mpegversion=2 in GStreamer refers to MPEG-2 Part 7 audio, */ + + st = TSMUX_ST_AUDIO_AAC; + + /* Check the stream format. If raw, make dummy internal codec data from the caps */ + if (g_strcmp0 (stream_format, "raw") == 0) { + ts_pad->codec_data = + gst_base_ts_mux_aac_mpeg2_make_codec_data (mux, caps); + ts_pad->prepare_func = gst_base_ts_mux_prepare_aac_mpeg2; + if (ts_pad->codec_data == NULL) { + GST_ERROR_OBJECT (mux, "Invalid or incomplete caps for MPEG-2 AAC"); + goto not_negotiated; + } + } break; + } case 4: { st = TSMUX_ST_AUDIO_AAC; - if (codec_data) { /* TODO - Check stream format - codec data should only come with RAW stream */ - GST_DEBUG_OBJECT (pad, - "we have additional codec data (%" G_GSIZE_FORMAT " bytes)", - gst_buffer_get_size (codec_data)); - ts_pad->codec_data = gst_buffer_ref (codec_data); - ts_pad->prepare_func = gst_base_ts_mux_prepare_aac; - } else { - ts_pad->codec_data = NULL; + + /* Check the stream format. We need codec_data with RAW streams and mpegversion=4 */ + if (g_strcmp0 (stream_format, "raw") == 0) { + if (codec_data) { + GST_DEBUG_OBJECT (pad, + "we have additional codec data (%" G_GSIZE_FORMAT " bytes)", + gst_buffer_get_size (codec_data)); + ts_pad->codec_data = gst_buffer_ref (codec_data); + ts_pad->prepare_func = gst_base_ts_mux_prepare_aac_mpeg4; + } else { + ts_pad->codec_data = NULL; + GST_ERROR_OBJECT (mux, "Need codec_data for raw MPEG-4 AAC"); + goto not_negotiated; + } } break; } diff --git a/gst/mpegtsmux/gstbasetsmuxaac.c b/gst/mpegtsmux/gstbasetsmuxaac.c index 455cd2fd7f..1080165d05 100644 --- a/gst/mpegtsmux/gstbasetsmuxaac.c +++ b/gst/mpegtsmux/gstbasetsmuxaac.c @@ -84,44 +84,40 @@ #include "config.h" #endif +#include + #include "gstbasetsmuxaac.h" #include #define GST_CAT_DEFAULT gst_base_ts_mux_debug -GstBuffer * -gst_base_ts_mux_prepare_aac (GstBuffer * buf, GstBaseTsMuxPad * pad, - GstBaseTsMux * mux) +static GstBuffer * +gst_base_ts_mux_prepare_aac_adts (GstBuffer * buf, + GstBaseTsMux * mux, gboolean is_mpeg2, guint8 obj_type_profile, + guint8 rate_idx, guint8 channels) { guint8 adts_header[7] = { 0, }; gsize out_size = gst_buffer_get_size (buf) + 7; GstBuffer *out_buf = gst_buffer_new_and_alloc (out_size); gsize out_offset = 0; - guint8 rate_idx = 0, channels = 0, obj_type = 0; - GstMapInfo codec_data_map; GstMapInfo buf_map; + /* Generate ADTS header */ GST_DEBUG_OBJECT (mux, "Preparing AAC buffer for output"); gst_buffer_copy_into (out_buf, buf, GST_BUFFER_COPY_METADATA | GST_BUFFER_COPY_TIMESTAMPS, 0, 0); - gst_buffer_map (pad->codec_data, &codec_data_map, GST_MAP_READ); + GST_DEBUG_OBJECT (mux, "Rate index %u, channels %u, object type/profile %u", + rate_idx, channels, obj_type_profile); - /* Generate ADTS header */ - obj_type = GST_READ_UINT8 (codec_data_map.data) >> 3; - rate_idx = (GST_READ_UINT8 (codec_data_map.data) & 0x7) << 1; - rate_idx |= (GST_READ_UINT8 (codec_data_map.data + 1) & 0x80) >> 7; - channels = (GST_READ_UINT8 (codec_data_map.data + 1) & 0x78) >> 3; - GST_DEBUG_OBJECT (mux, "Rate index %u, channels %u, object type %u", rate_idx, - channels, obj_type); /* Sync point over a full byte */ adts_header[0] = 0xFF; /* Sync point continued over first 4 bits + static 4 bits * (ID, layer, protection)*/ - adts_header[1] = 0xF1; - /* Object type over first 2 bits */ - adts_header[2] = (obj_type - 1) << 6; + adts_header[1] = 0xF1 | (is_mpeg2 ? 0x8 : 0x0); + /* Object type (MPEG4) / Profile (MPEG2) over first 2 bits */ + adts_header[2] = (obj_type_profile - 1) << 6; /* rate index over next 4 bits */ adts_header[2] |= (rate_idx << 2); /* channels over last 2 bits */ @@ -149,8 +145,119 @@ gst_base_ts_mux_prepare_aac (GstBuffer * buf, GstBaseTsMuxPad * pad, /* Now copy complete frame */ gst_buffer_fill (out_buf, out_offset, buf_map.data, buf_map.size); - gst_buffer_unmap (pad->codec_data, &codec_data_map); gst_buffer_unmap (buf, &buf_map); return out_buf; } + +/* Constructs a dummy codec_data buffer for generating ADTS headers + * from raw MPEG-2 AAC input, where we don't expect codec_data in the caps, + * and need to get the info from the profile/channels/rate fields */ +GstBuffer * +gst_base_ts_mux_aac_mpeg2_make_codec_data (GstBaseTsMux * mux, + const GstCaps * caps) +{ + const GstStructure *s; + const gchar *profile_str; + gint channels, rate; + guint8 profile_idx, channel_idx; + gint rate_idx; + GstMapInfo map; + GstBuffer *ret; + + s = gst_caps_get_structure (caps, 0); + profile_str = gst_structure_get_string (s, "profile"); + if (G_UNLIKELY (profile_str == NULL)) { + GST_ERROR_OBJECT (mux, "AAC caps do not contain profile"); + return NULL; + } + + if (G_UNLIKELY (!gst_structure_get_int (s, "rate", &rate))) { + GST_ERROR_OBJECT (mux, "AAC caps do not contain a sample rate"); + return NULL; + } + if (G_UNLIKELY (!gst_structure_get_int (s, "channels", &channels))) { + GST_ERROR_OBJECT (mux, "AAC caps do not contain channel count"); + return NULL; + } + + if (g_strcmp0 (profile_str, "main") == 0) { + profile_idx = (guint8) 0U; + } else if (g_strcmp0 (profile_str, "lc") == 0) { + profile_idx = (guint8) 1U; + } else if (g_strcmp0 (profile_str, "ssr") == 0) { + profile_idx = (guint8) 2U; + } else { + GST_ERROR_OBJECT (mux, "Invalid profile %s for MPEG-2 AAC caps", + profile_str); + return NULL; + } + + if (channels >= 1 && channels <= 6) /* Mono up to & including 5.1 */ + channel_idx = (guint8) channels; + else if (channels == 8) /* 7.1 */ + channel_idx = (guint8) 7U; + else { + GST_ERROR_OBJECT (mux, "Invalid channel count %d for MPEG-2 AAC caps", + channels); + return NULL; + } + + rate_idx = gst_codec_utils_aac_get_index_from_sample_rate (rate); + if (rate_idx < 0) { + GST_ERROR_OBJECT (mux, "Invalid samplerate %d for MPEG-2 AAC caps", rate); + return NULL; + } + + ret = gst_buffer_new_and_alloc (3); + gst_buffer_map (ret, &map, GST_MAP_READ); + map.data[0] = profile_idx; + map.data[1] = (guint8) rate_idx; + map.data[2] = channel_idx; + gst_buffer_unmap (ret, &map); + + return ret; +} + +GstBuffer * +gst_base_ts_mux_prepare_aac_mpeg4 (GstBuffer * buf, GstBaseTsMuxPad * pad, + GstBaseTsMux * mux) +{ + GstMapInfo codec_data_map; + guint8 rate_idx = 0, channels = 0, obj_type = 0; + + g_return_val_if_fail (pad->codec_data != NULL, NULL); + + gst_buffer_map (pad->codec_data, &codec_data_map, GST_MAP_READ); + + obj_type = GST_READ_UINT8 (codec_data_map.data) >> 3; + rate_idx = (GST_READ_UINT8 (codec_data_map.data) & 0x7) << 1; + rate_idx |= (GST_READ_UINT8 (codec_data_map.data + 1) & 0x80) >> 7; + channels = (GST_READ_UINT8 (codec_data_map.data + 1) & 0x78) >> 3; + gst_buffer_unmap (pad->codec_data, &codec_data_map); + + return gst_base_ts_mux_prepare_aac_adts (buf, mux, FALSE, obj_type, rate_idx, + channels); +} + +GstBuffer * +gst_base_ts_mux_prepare_aac_mpeg2 (GstBuffer * buf, GstBaseTsMuxPad * pad, + GstBaseTsMux * mux) +{ + GstMapInfo codec_data_map; + guint8 rate_idx = 0, channels = 0, profile_obj_type = 0; + + g_return_val_if_fail (pad->codec_data != NULL, NULL); + + /* Dummy codec data with 3 bytes of profile_idx, rate_idx, channel_idx */ + gst_buffer_map (pad->codec_data, &codec_data_map, GST_MAP_READ); + + profile_obj_type = GST_READ_UINT8 (codec_data_map.data); + rate_idx = GST_READ_UINT8 (codec_data_map.data + 1); + channels = GST_READ_UINT8 (codec_data_map.data + 2); + + gst_buffer_unmap (pad->codec_data, &codec_data_map); + + return gst_base_ts_mux_prepare_aac_adts (buf, mux, TRUE, profile_obj_type, + rate_idx, channels); +} diff --git a/gst/mpegtsmux/gstbasetsmuxaac.h b/gst/mpegtsmux/gstbasetsmuxaac.h index 58eb46c3f7..6e9f0b034c 100644 --- a/gst/mpegtsmux/gstbasetsmuxaac.h +++ b/gst/mpegtsmux/gstbasetsmuxaac.h @@ -85,7 +85,13 @@ #include "gstbasetsmux.h" -GstBuffer * gst_base_ts_mux_prepare_aac (GstBuffer * buf, GstBaseTsMuxPad * pad, +GstBuffer * gst_base_ts_mux_prepare_aac_mpeg4 (GstBuffer * buf, GstBaseTsMuxPad * pad, GstBaseTsMux * mux); + +GstBuffer * gst_base_ts_mux_aac_mpeg2_make_codec_data (GstBaseTsMux * mux, const GstCaps *caps); +GstBuffer * gst_base_ts_mux_prepare_aac_mpeg2 (GstBuffer * buf, GstBaseTsMuxPad * pad, + GstBaseTsMux * mux); + + #endif /* __BASETSMUX_AAC_H__ */ diff --git a/gst/mpegtsmux/gstmpegtsmux.c b/gst/mpegtsmux/gstmpegtsmux.c index a2ca00ad71..1b3638e52f 100644 --- a/gst/mpegtsmux/gstmpegtsmux.c +++ b/gst/mpegtsmux/gstmpegtsmux.c @@ -122,12 +122,10 @@ static GstStaticPadTemplate gst_mpeg_ts_mux_sink_factory = "alignment=(string){au, nal}; " "audio/mpeg, " "parsed = (boolean) TRUE, " - "mpegversion = (int) { 1, 2 };" + "mpegversion = (int) 1;" "audio/mpeg, " "framed = (boolean) TRUE, " - "mpegversion = (int) 4, stream-format = (string) adts;" - "audio/mpeg, " - "mpegversion = (int) 4, stream-format = (string) raw;" + "mpegversion = (int) {2, 4}, stream-format = (string) { adts, raw };" "audio/x-lpcm, " "width = (int) { 16, 20, 24 }, " "rate = (int) { 48000, 96000 }, "