From 19cea52dbd8c49cf273a23b5c7a2647e56278787 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Tue, 18 Apr 2023 11:37:25 +0200 Subject: [PATCH] mpegts: Add provisional AV1 mapping The main difference with the WIP av1-in-mpegts mapping is that the payload data is not startcode-escaped. Most of the rest is sensible usage of it: * Custom AV1G (AV1 Gstreamer) registration descriptor instead of AV01 * AV1CodecConfigurationRecord is stored in the same 0x80 custom descriptor and conforms fully to the isobmff spec (i.e. does not the HDR fields from the provisional mpegts specification which conflict with that one). * Data is stored as OBU * Access Unit is the frame level (same as provisional mpegts mapping) Part-of: --- .../docs/plugins/gst_plugins_cache.json | 4 ++-- .../gst/mpegtsdemux/gstmpegdesc.h | 1 + .../gst-plugins-bad/gst/mpegtsdemux/tsdemux.c | 19 +++++++++++++++++++ .../gst/mpegtsmux/gstbasetsmux.c | 17 +++++++++++++++++ .../gst/mpegtsmux/gstmpegtsmux.c | 1 + .../gst/mpegtsmux/tsmux/tsmuxstream.c | 14 ++++++++++++++ .../gst/mpegtsmux/tsmux/tsmuxstream.h | 1 + 7 files changed, 55 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json index b88276f0af..eea53fa05c 100644 --- a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json +++ b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json @@ -223744,7 +223744,7 @@ "presence": "sometimes" }, "video_%%01x_%%05x": { - "caps": "video/mpeg:\n mpegversion: { (int)1, (int)2, (int)4 }\n systemstream: false\nvideo/x-h264:\n stream-format: byte-stream\nvideo/x-h265:\n stream-format: byte-stream\nvideo/x-vp9:\nvideo/x-dirac:\nvideo/x-cavs:\nvideo/x-wmv:\n wmvversion: 3\n format: WVC1\nimage/x-jpc:\nimage/x-jxsc:\n", + "caps": "video/mpeg:\n mpegversion: { (int)1, (int)2, (int)4 }\n systemstream: false\nvideo/x-h264:\n stream-format: byte-stream\nvideo/x-h265:\n stream-format: byte-stream\nvideo/x-vp9:\nvideo/x-av1:\n stream-format: obu-stream\n alignment: frame\nvideo/x-dirac:\nvideo/x-cavs:\nvideo/x-wmv:\n wmvversion: 3\n format: WVC1\nimage/x-jpc:\nimage/x-jxsc:\n", "direction": "src", "presence": "sometimes" } @@ -224002,7 +224002,7 @@ "long-name": "MPEG Transport Stream Muxer", "pad-templates": { "sink_%%d": { - "caps": "video/mpeg:\n parsed: true\n mpegversion: { (int)1, (int)2, (int)4 }\n systemstream: false\nvideo/x-dirac:\nimage/x-jpc:\n alignment: frame\nvideo/x-h264:\n stream-format: byte-stream\n alignment: { (string)au, (string)nal }\nvideo/x-h265:\n stream-format: byte-stream\n alignment: { (string)au, (string)nal }\naudio/mpeg:\n parsed: true\n mpegversion: 1\naudio/mpeg:\n framed: true\n mpegversion: { (int)2, (int)4 }\n stream-format: { (string)adts, (string)raw }\naudio/x-lpcm:\n width: { (int)16, (int)20, (int)24 }\n rate: { (int)48000, (int)96000 }\n channels: [ 1, 8 ]\n dynamic_range: [ 0, 255 ]\n emphasis: { (boolean)false, (boolean)true }\n mute: { (boolean)false, (boolean)true }\naudio/x-ac3:\n framed: true\naudio/x-dts:\n framed: true\naudio/x-opus:\n channels: [ 1, 255 ]\naudio/x-smpte-302m:\nsubpicture/x-dvb:\napplication/x-teletext:\nmeta/x-klv:\n parsed: true\nmeta/x-id3:\n parsed: true\nmeta/x-st-2038:\n alignment: line\nvideo/x-vp9:\nimage/x-jpc:\n alignment: frame\n profile: [ 0, 49151 ]\nimage/x-jxsc:\n alignment: frame\n sampling: { (string)YCbCr-4:2:2, (string)YCbCr-4:4:4 }\n", + "caps": "video/mpeg:\n parsed: true\n mpegversion: { (int)1, (int)2, (int)4 }\n systemstream: false\nvideo/x-dirac:\nimage/x-jpc:\n alignment: frame\nvideo/x-av1:\n stream-format: obu-stream\n alignment: frame\nvideo/x-h264:\n stream-format: byte-stream\n alignment: { (string)au, (string)nal }\nvideo/x-h265:\n stream-format: byte-stream\n alignment: { (string)au, (string)nal }\naudio/mpeg:\n parsed: true\n mpegversion: 1\naudio/mpeg:\n framed: true\n mpegversion: { (int)2, (int)4 }\n stream-format: { (string)adts, (string)raw }\naudio/x-lpcm:\n width: { (int)16, (int)20, (int)24 }\n rate: { (int)48000, (int)96000 }\n channels: [ 1, 8 ]\n dynamic_range: [ 0, 255 ]\n emphasis: { (boolean)false, (boolean)true }\n mute: { (boolean)false, (boolean)true }\naudio/x-ac3:\n framed: true\naudio/x-dts:\n framed: true\naudio/x-opus:\n channels: [ 1, 255 ]\naudio/x-smpte-302m:\nsubpicture/x-dvb:\napplication/x-teletext:\nmeta/x-klv:\n parsed: true\nmeta/x-id3:\n parsed: true\nmeta/x-st-2038:\n alignment: line\nvideo/x-vp9:\nimage/x-jpc:\n alignment: frame\n profile: [ 0, 49151 ]\nimage/x-jxsc:\n alignment: frame\n sampling: { (string)YCbCr-4:2:2, (string)YCbCr-4:4:4 }\n", "direction": "sink", "presence": "request", "type": "GstBaseTsMuxPad" diff --git a/subprojects/gst-plugins-bad/gst/mpegtsdemux/gstmpegdesc.h b/subprojects/gst-plugins-bad/gst/mpegtsdemux/gstmpegdesc.h index 31bc97c754..376da543db 100644 --- a/subprojects/gst-plugins-bad/gst/mpegtsdemux/gstmpegdesc.h +++ b/subprojects/gst-plugins-bad/gst/mpegtsdemux/gstmpegdesc.h @@ -242,5 +242,6 @@ #define DRF_ID_ID3 0x49443320 /* defined in SMPTE registration authority */ #define DRF_ID_VANC 0x56414e43 /* defined in SMPTE ST-2038 */ #define DRF_ID_VP09 0x56503039 /* Custom GStreamer vp9 */ +#define DRF_ID_AV1G 0x41563147 /* Custom AV1 GStreamer mapping */ #endif /* __GST_MPEG_DESC_H__ */ diff --git a/subprojects/gst-plugins-bad/gst/mpegtsdemux/tsdemux.c b/subprojects/gst-plugins-bad/gst/mpegtsdemux/tsdemux.c index 9bce7c884e..595510118c 100644 --- a/subprojects/gst-plugins-bad/gst/mpegtsdemux/tsdemux.c +++ b/subprojects/gst-plugins-bad/gst/mpegtsdemux/tsdemux.c @@ -230,6 +230,7 @@ struct _TSDemuxStream "video/x-h264,stream-format=(string)byte-stream;" \ "video/x-h265,stream-format=(string)byte-stream;" \ "video/x-vp9;" \ + "video/x-av1,stream-format=(string)obu-stream,alignment=(string)frame;" \ "video/x-dirac;" \ "video/x-cavs;" \ "video/x-wmv," \ @@ -1498,6 +1499,24 @@ create_pad_for_stream (MpegTSBase * base, MpegTSBaseStream * bstream, is_audio = TRUE; caps = gst_caps_new_empty_simple ("audio/x-smpte-302m"); break; + case DRF_ID_AV1G: + GST_DEBUG ("AV1"); + is_video = TRUE; + caps = + gst_caps_new_simple ("video/x-av1", "stream-format", + G_TYPE_STRING, "obu-stream", "alignment", G_TYPE_STRING, "frame", + NULL); + desc = mpegts_get_descriptor_from_stream (bstream, 0x80); + if (desc != NULL) { + GstCaps *av1c_caps; + GstBuffer *buf = gst_buffer_new_wrapped (g_memdup2 (desc->data + 2, + desc->length), desc->length); + av1c_caps = gst_codec_utils_av1_create_caps_from_av1c (buf); + gst_caps_set_simple (av1c_caps, "codec_data", GST_TYPE_BUFFER, buf, + NULL); + caps = gst_caps_intersect (caps, av1c_caps); + } + break; case DRF_ID_OPUS: desc = mpegts_get_descriptor_from_stream (bstream, GST_MTS_DESC_DVB_EXTENSION); diff --git a/subprojects/gst-plugins-bad/gst/mpegtsmux/gstbasetsmux.c b/subprojects/gst-plugins-bad/gst/mpegtsmux/gstbasetsmux.c index c73a4335cb..53e2e88d68 100644 --- a/subprojects/gst-plugins-bad/gst/mpegtsmux/gstbasetsmux.c +++ b/subprojects/gst-plugins-bad/gst/mpegtsmux/gstbasetsmux.c @@ -633,6 +633,23 @@ gst_base_ts_mux_create_or_update_stream (GstBaseTsMux * mux, GST_ERROR_OBJECT (mux, "VP9 requires enabling custom mapping which does not have a public specification"); } + } else if (strcmp (mt, "video/x-av1") == 0) { + if (mux->enable_custom_mappings) { + st = TSMUX_ST_PS_VIDEO_AV1; + if (!codec_data) + codec_data = gst_codec_utils_av1_create_av1c_from_caps (caps); + if (codec_data) { + GstMapInfo map; + if (gst_buffer_map (codec_data, &map, GST_MAP_READ)) { + pmt_descriptor = + gst_mpegts_descriptor_from_custom (0x80, map.data, map.size); + gst_buffer_unmap (codec_data, &map); + } + } + } else { + GST_ERROR_OBJECT (mux, + "AV1 requires enabling custom mapping which does not have a public specification yet"); + } } else if (strcmp (mt, "audio/mpeg") == 0) { gint mpegversion; diff --git a/subprojects/gst-plugins-bad/gst/mpegtsmux/gstmpegtsmux.c b/subprojects/gst-plugins-bad/gst/mpegtsmux/gstmpegtsmux.c index 019da51aa7..df4f12a451 100644 --- a/subprojects/gst-plugins-bad/gst/mpegtsmux/gstmpegtsmux.c +++ b/subprojects/gst-plugins-bad/gst/mpegtsmux/gstmpegtsmux.c @@ -103,6 +103,7 @@ static GstStaticPadTemplate gst_mpeg_ts_mux_sink_factory = "systemstream = (boolean) false; " "video/x-dirac;" "image/x-jpc, alignment = (string) frame;" + "video/x-av1,stream-format=(string)obu-stream,alignment=(string)frame;" "video/x-h264,stream-format=(string)byte-stream," "alignment=(string){au, nal}; " "video/x-h265,stream-format=(string)byte-stream," diff --git a/subprojects/gst-plugins-bad/gst/mpegtsmux/tsmux/tsmuxstream.c b/subprojects/gst-plugins-bad/gst/mpegtsmux/tsmux/tsmuxstream.c index 9acb436b8d..68502daba8 100644 --- a/subprojects/gst-plugins-bad/gst/mpegtsmux/tsmux/tsmuxstream.c +++ b/subprojects/gst-plugins-bad/gst/mpegtsmux/tsmux/tsmuxstream.c @@ -258,6 +258,13 @@ tsmux_stream_new (guint16 pid, guint stream_type, guint stream_number) stream->stream_type = TSMUX_ST_PRIVATE_DATA; stream->pi.flags |= TSMUX_PACKET_FLAG_PES_FULL_HEADER; break; + case TSMUX_ST_PS_VIDEO_AV1: + /* FIXME: assign sequential extended IDs? */ + stream->id = 0xBD; + stream->gst_stream_type = GST_STREAM_TYPE_VIDEO; + stream->stream_type = TSMUX_ST_PRIVATE_DATA; + stream->pi.flags |= TSMUX_PACKET_FLAG_PES_FULL_HEADER; + break; default: /* Might be a custom stream type implemented by a subclass */ break; @@ -1045,6 +1052,13 @@ tsmux_stream_default_get_es_descrs (TsMuxStream * stream, GST_DEBUG ("adding VP09 registration descriptor"); g_ptr_array_add (pmt_stream->descriptors, descriptor); } + if (stream->internal_stream_type == TSMUX_ST_PS_VIDEO_AV1) { + /* AV1G is our custom non-bytestream format ! */ + descriptor = gst_mpegts_descriptor_from_registration ("AV1G", NULL, 0); + g_ptr_array_add (pmt_stream->descriptors, descriptor); + if (stream->pmt_descriptor) + g_ptr_array_add (pmt_stream->descriptors, stream->pmt_descriptor); + } default: break; } diff --git a/subprojects/gst-plugins-bad/gst/mpegtsmux/tsmux/tsmuxstream.h b/subprojects/gst-plugins-bad/gst/mpegtsmux/tsmux/tsmuxstream.h index 24c7bb1ae1..fb1123a9df 100644 --- a/subprojects/gst-plugins-bad/gst/mpegtsmux/tsmux/tsmuxstream.h +++ b/subprojects/gst-plugins-bad/gst/mpegtsmux/tsmux/tsmuxstream.h @@ -136,6 +136,7 @@ enum TsMuxStreamType { TSMUX_ST_PS_ST_2038 = 0x91, /* only used internally */ TSMUX_ST_PS_S302M = 0x92, /* only used internally */ TSMUX_ST_PS_VP9 = 0x93, /* only used internally */ + TSMUX_ST_PS_VIDEO_AV1 = 0x94, /* only used internally */ TSMUX_ST_PS_DVD_SUBPICTURE = 0xff, /* Non-standard definitions */