From c1ff2b37305c8c6e7013e68320ce8d561688614b Mon Sep 17 00:00:00 2001 From: Rafael Caricio Date: Tue, 12 Oct 2021 23:06:47 +0200 Subject: [PATCH 1/3] Support parsing of unknown tags on segments --- .../media-playlist-with-cues-1.m3u8 | 25 ++++++++++ .../media-playlist-with-cues.m3u8 | 20 ++++++++ src/lib.rs | 2 +- src/parser.rs | 27 +++++----- src/playlist.rs | 20 ++++---- tests/lib.rs | 50 +++++++++++++++++-- 6 files changed, 113 insertions(+), 31 deletions(-) create mode 100644 sample-playlists/media-playlist-with-cues-1.m3u8 create mode 100644 sample-playlists/media-playlist-with-cues.m3u8 diff --git a/sample-playlists/media-playlist-with-cues-1.m3u8 b/sample-playlists/media-playlist-with-cues-1.m3u8 new file mode 100644 index 0000000..04c78c8 --- /dev/null +++ b/sample-playlists/media-playlist-with-cues-1.m3u8 @@ -0,0 +1,25 @@ +#EXTM3U +# Borrowed from https://github.com/grafov/m3u8/pull/83 +#EXT-X-VERSION:3 +#EXT-X-MEDIA-SEQUENCE:0 +#EXT-X-TARGETDURATION:6 +#EXTINF:6.000, +1.ts +#EXT-X-DATERANGE:ID="20",START-DATE="2020-06-03T14:56:00Z",PLANNED-DURATION=19,SCTE35-OUT=0xFC302000000000000000FFF00F05000000147FFFFE001A17B0C0000000000061DFD67D +#EXT-X-CUE-OUT:19.0 +#EXT-X-PROGRAM-DATE-TIME:2020-06-03T14:56:00Z +#EXTINF:6.000, +2.ts +#EXTINF:6.000, +3.ts +#EXTINF:6.000, +4.ts +#EXT-X-CUE-IN +#EXTINF:6.000, +5.ts +#EXTINF:6.000, +6.ts +#EXTINF:6.000, +7.ts +#EXTINF:6.000, +8.ts diff --git a/sample-playlists/media-playlist-with-cues.m3u8 b/sample-playlists/media-playlist-with-cues.m3u8 new file mode 100644 index 0000000..b20902b --- /dev/null +++ b/sample-playlists/media-playlist-with-cues.m3u8 @@ -0,0 +1,20 @@ +#EXTINF:10, +http://media.example.com/fileSequence7796.ts +#EXTINF:6, +http://media.example.com/fileSequence7797.ts +#EXT-X-CUE-OUT:DURATION=30 +#EXTINF:4, +http://media.example.com/fileSequence7798.ts +#EXTINF:10, +http://media.example.com/fileSequence7799.ts +#EXTINF:10, +http://media.example.com/fileSequence7800.ts +#EXTINF:6, +http://media.example.com/fileSequence7801.ts +#EXT-X-CUE-IN +#EXTINF:4, +http://media.example.com/fileSequence7802.ts +#EXTINF:10, +http://media.example.com/fileSequence7803.ts +#EXTINF:3, +http://media.example.com/fileSequence7804.ts diff --git a/src/lib.rs b/src/lib.rs index dcbe36e..8bad94a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,4 +6,4 @@ pub mod playlist; mod parser; #[cfg(feature = "parser")] -pub use self::parser::*; \ No newline at end of file +pub use self::parser::*; diff --git a/src/parser.rs b/src/parser.rs index 6fe3fb5..15d0a87 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -104,7 +104,7 @@ use playlist::*; /// ``` /// use std::io::Read; /// use m3u8_rs::playlist::{Playlist}; -/// +/// /// let mut file = std::fs::File::open("playlist.m3u8").unwrap(); /// let mut bytes: Vec = Vec::new(); /// file.read_to_end(&mut bytes).unwrap(); @@ -129,8 +129,8 @@ pub fn parse_playlist(input: &[u8]) -> IResult<&[u8], Playlist> { } /// Parses an m3u8 playlist just like `parse_playlist`, except that this returns an [std::result::Result](std::result::Result) instead of a [nom::IResult](https://docs.rs/nom/1.2.3/nom/enum.IResult.html). -/// However, since [nom::IResult](nom::IResult) is now an [alias to Result](https://github.com/Geal/nom/blob/master/doc/upgrading_to_nom_5.md), this is no longer needed. -/// +/// However, since [nom::IResult](nom::IResult) is now an [alias to Result](https://github.com/Geal/nom/blob/master/doc/upgrading_to_nom_5.md), this is no longer needed. +/// /// # Examples /// /// ``` @@ -274,7 +274,6 @@ pub enum MasterPlaylistTag { SessionKey(SessionKey), Start(Start), IndependentSegments, - Unknown(ExtTag), Comment(String), Uri(String), } @@ -292,7 +291,6 @@ pub fn master_playlist_tag(input: &[u8]) -> IResult<&[u8], MasterPlaylistTag> { | map!(start_tag, MasterPlaylistTag::Start) | map!(tag!("#EXT-X-INDEPENDENT-SEGMENTS"), |_| MasterPlaylistTag::IndependentSegments) - | map!(ext_tag, MasterPlaylistTag::Unknown) | map!(comment_tag, MasterPlaylistTag::Comment) | map!(consume_line, MasterPlaylistTag::Uri) @@ -330,9 +328,6 @@ pub fn master_playlist_from_tags(mut tags: Vec) -> MasterPlay MasterPlaylistTag::IndependentSegments => { master_playlist.independent_segments = true; } - MasterPlaylistTag::Unknown(unknown) => { - master_playlist.unknown_tags.push(unknown); - } _ => (), } } @@ -392,6 +387,7 @@ pub enum MediaPlaylistTag { PlaylistType(MediaPlaylistType), IFramesOnly, Start(Start), + Unknown(ExtTag), IndependentSegments, } @@ -474,7 +470,7 @@ pub fn media_playlist_from_tags(mut tags: Vec) -> MediaPlaylis next_segment.daterange = Some(d); } SegmentTag::Unknown(t) => { - media_playlist.unknown_tags.push(t); + next_segment.unknown_tags.push(t); } SegmentTag::Uri(u) => { next_segment.key = encryption_key.clone(); @@ -588,13 +584,14 @@ named!(pub start_tag, named!(pub ext_tag, do_parse!( - tag!("#EXT-") - >> tag: map_res!(take_until!(":"), from_utf8_slice) + tag!("#EXT-") + >> tag: map_res!(is_not!("\r\n:"), from_utf8_slice) + >> opt!(tag!(":")) + >> rest: opt!(map_res!(is_not!("\r\n"), from_utf8_slice)) >> take!(1) - >> rest: map_res!(is_not!("\r\n"), from_utf8_slice) - >> take!(1) - >> - (ExtTag { tag: tag, rest: rest }) + >> ( + ExtTag { tag: tag, rest: rest } + ) ) ); diff --git a/src/playlist.rs b/src/playlist.rs index 91384b1..aec9fd6 100644 --- a/src/playlist.rs +++ b/src/playlist.rs @@ -69,7 +69,6 @@ pub struct MasterPlaylist { pub start: Option, pub independent_segments: bool, pub alternatives: Vec, // EXT-X-MEDIA tags - pub unknown_tags: Vec } impl MasterPlaylist { @@ -101,9 +100,6 @@ impl MasterPlaylist { if self.independent_segments { writeln!(w, "#EXT-X-INDEPENDENT-SEGMENTS")?; } - for unknown_tag in &self.unknown_tags { - write!(w, "#EXT-{}:{}\n", unknown_tag.tag, unknown_tag.rest)?; - } Ok(()) } @@ -398,8 +394,6 @@ pub struct MediaPlaylist { pub start: Option, /// `#EXT-X-INDEPENDENT-SEGMENTS` pub independent_segments: bool, - /// `#EXT-X-` - pub unknown_tags: Vec } impl MediaPlaylist { @@ -433,9 +427,6 @@ impl MediaPlaylist { for segment in &self.segments { segment.write_to(w)?; } - for unknown_tag in &self.unknown_tags { - write!(w, "#EXT-{}:{}\n", unknown_tag.tag, unknown_tag.rest)?; - } Ok(()) } @@ -501,6 +492,8 @@ pub struct MediaSegment { pub program_date_time: Option, /// `#EXT-X-DATERANGE:` pub daterange: Option, + /// `#EXT-` + pub unknown_tags: Vec, } impl MediaSegment { @@ -534,6 +527,13 @@ impl MediaSegment { if let Some(ref v) = self.daterange { writeln!(w, "#EXT-X-DATERANGE:{}", v)?; } + for unknown_tag in &self.unknown_tags { + write!(w, "#EXT-{}", unknown_tag.tag)?; + if let Some(v) = &unknown_tag.rest { + writeln!(w, ":{}", v)?; + } + write!(w, "\n")?; + } write!(w, "#EXTINF:{},", self.duration)?; @@ -689,6 +689,6 @@ impl Start { #[derive(Debug, Default, PartialEq, Clone)] pub struct ExtTag { pub tag: String, - pub rest: String, + pub rest: Option, } diff --git a/tests/lib.rs b/tests/lib.rs index 76a21dc..5b58224 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -85,6 +85,26 @@ fn playlist_media_without_segments() { assert!(print_parse_playlist_test("media-playlist-without-segments.m3u8")); } +#[test] +fn playlist_media_with_cues() { + assert!(print_parse_playlist_test("media-playlist-with-cues.m3u8")); +} + +#[test] +fn playlist_media_with_cues1() { + assert!(print_parse_playlist_test("media-playlist-with-cues-1.m3u8")); +} + +#[test] +fn playlist_media_with_scte35() { + assert!(print_parse_playlist_test("media-playlist-with-scte35.m3u8")); +} + +#[test] +fn playlist_media_with_scte35_1() { + assert!(print_parse_playlist_test("media-playlist-with-scte35-1.m3u8")); +} + // ----------------------------------------------------------------------------------------------- // Playlist with no newline end @@ -114,10 +134,10 @@ fn playlist_type_is_master() { } // #[test] -// fn playlist_type_with_unkown_tag() { +// fn playlist_type_with_unknown_tag() { // let input = get_sample_playlist("!!"); // let result = is_master_playlist(input.as_bytes()); -// println!("Playlist_type_with_unkown_tag is master playlist: {:?}", result); +// println!("Playlist_type_with_unknown_tag is master playlist: {:?}", result); // assert_eq!(true, result); // } @@ -195,6 +215,22 @@ fn test_key_value_pair() { ); } +#[test] +fn ext_with_value() { + assert_eq!( + ext_tag(b"#EXT-X-CUE-OUT:DURATION=30\nxxx"), + Result::Ok((b"xxx".as_bytes(), ExtTag { tag: "X-CUE-OUT".into(), rest: Some("DURATION=30".into()) })) + ); +} + +#[test] +fn ext_without_value() { + assert_eq!( + ext_tag(b"#EXT-X-CUE-IN\nxxx"), + Result::Ok((b"xxx".as_bytes(), ExtTag { tag: "X-CUE-IN".into(), rest: None })) + ); +} + #[test] fn comment() { assert_eq!( @@ -213,7 +249,7 @@ fn quotes() { #[test] fn consume_line_empty() { - let expected = Result::Ok(("rest".as_bytes(), "".to_string())); + let expected = Result::Ok(("rest".as_bytes(), "".to_string())); let actual = consume_line(b"\r\nrest"); assert_eq!(expected, actual); } @@ -354,7 +390,6 @@ fn create_and_parse_master_playlist_full() { precise: Some("YES".into()), }), independent_segments: true, - unknown_tags: vec![], }); let playlist_parsed = print_create_and_parse_playlist(&mut playlist_original); assert_eq!(playlist_original, playlist_parsed); @@ -420,9 +455,14 @@ fn create_and_parse_media_playlist_full() { }), program_date_time: Some("broodlordinfestorgg".into()), daterange: None, + unknown_tags: vec![ + ExtTag { + tag: "X-CUE-OUT".into(), + rest: Some("DURATION=2.002".into()) + } + ] }, ], - unknown_tags: vec![], }); let playlist_parsed = print_create_and_parse_playlist(&mut playlist_original); assert_eq!(playlist_original, playlist_parsed); From dc352301a37482ad1db9b507603386f3f573a1c3 Mon Sep 17 00:00:00 2001 From: Rafael Caricio Date: Thu, 14 Oct 2021 21:21:03 +0200 Subject: [PATCH 2/3] Allow unknown tags at the master playlist level --- src/parser.rs | 6 ++++++ src/playlist.rs | 51 ++++++++++++++++++++++++++++++++++++++++++++----- tests/lib.rs | 1 + 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 15d0a87..064a85b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -276,6 +276,7 @@ pub enum MasterPlaylistTag { IndependentSegments, Comment(String), Uri(String), + Unknown(ExtTag), } pub fn master_playlist_tag(input: &[u8]) -> IResult<&[u8], MasterPlaylistTag> { @@ -291,6 +292,8 @@ pub fn master_playlist_tag(input: &[u8]) -> IResult<&[u8], MasterPlaylistTag> { | map!(start_tag, MasterPlaylistTag::Start) | map!(tag!("#EXT-X-INDEPENDENT-SEGMENTS"), |_| MasterPlaylistTag::IndependentSegments) + | map!(ext_tag, MasterPlaylistTag::Unknown) + | map!(comment_tag, MasterPlaylistTag::Comment) | map!(consume_line, MasterPlaylistTag::Uri) @@ -328,6 +331,9 @@ pub fn master_playlist_from_tags(mut tags: Vec) -> MasterPlay MasterPlaylistTag::IndependentSegments => { master_playlist.independent_segments = true; } + MasterPlaylistTag::Unknown(unknown) => { + master_playlist.unknown_tags.push(unknown); + } _ => (), } } diff --git a/src/playlist.rs b/src/playlist.rs index aec9fd6..a1c71d0 100644 --- a/src/playlist.rs +++ b/src/playlist.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::str::FromStr; use std::fmt; use std::f32; +use std::fmt::Display; macro_rules! write_some_attribute_quoted { ($w:expr, $tag:expr, $o:expr) => ( @@ -69,6 +70,7 @@ pub struct MasterPlaylist { pub start: Option, pub independent_segments: bool, pub alternatives: Vec, // EXT-X-MEDIA tags + pub unknown_tags: Vec, } impl MasterPlaylist { @@ -100,6 +102,9 @@ impl MasterPlaylist { if self.independent_segments { writeln!(w, "#EXT-X-INDEPENDENT-SEGMENTS")?; } + for unknown_tag in &self.unknown_tags { + writeln!(w, "{}", unknown_tag)?; + } Ok(()) } @@ -528,11 +533,7 @@ impl MediaSegment { writeln!(w, "#EXT-X-DATERANGE:{}", v)?; } for unknown_tag in &self.unknown_tags { - write!(w, "#EXT-{}", unknown_tag.tag)?; - if let Some(v) = &unknown_tag.rest { - writeln!(w, ":{}", v)?; - } - write!(w, "\n")?; + writeln!(w, "{}", unknown_tag)?; } write!(w, "#EXTINF:{},", self.duration)?; @@ -692,3 +693,43 @@ pub struct ExtTag { pub rest: Option, } +impl Display for ExtTag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "#EXT-{}", self.tag)?; + if let Some(v) = &self.rest { + write!(f, ":{}", v)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn ext_tag_with_value_is_printable() { + let cue_out_tag = ExtTag { + tag: "X-CUE-OUT".into(), + rest: Some("DURATION=30".into()), + }; + + let mut output = Vec::new(); + write!(output, "{}", cue_out_tag); + + assert_eq!(std::str::from_utf8(output.as_slice()).unwrap(), "#EXT-X-CUE-OUT:DURATION=30") + } + + #[test] + fn ext_tag_without_value_is_printable() { + let cue_in_tag = ExtTag { + tag: "X-CUE-IN".into(), + rest: None, + }; + + let mut output = Vec::new(); + write!(output, "{}", cue_in_tag); + + assert_eq!(std::str::from_utf8(output.as_slice()).unwrap(), "#EXT-X-CUE-IN") + } +} diff --git a/tests/lib.rs b/tests/lib.rs index 5b58224..5fecc42 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -390,6 +390,7 @@ fn create_and_parse_master_playlist_full() { precise: Some("YES".into()), }), independent_segments: true, + unknown_tags: vec![], }); let playlist_parsed = print_create_and_parse_playlist(&mut playlist_original); assert_eq!(playlist_original, playlist_parsed); From 677027e22c7d94ee12d4a5a0ac0c9185e4dd1497 Mon Sep 17 00:00:00 2001 From: Rafael Caricio Date: Thu, 14 Oct 2021 21:35:35 +0200 Subject: [PATCH 3/3] Update readme with new attribute --- README.md | 7 +++++-- src/playlist.rs | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2c62075..a4ff161 100644 --- a/README.md +++ b/README.md @@ -67,12 +67,12 @@ match parsed { } ``` -Currently the parser will succeed even if REQUIRED attributes/tags are missing from a playlist (such as the `#EXT-X-VERSION` tag). +Currently, the parser will succeed even if REQUIRED attributes/tags are missing from a playlist (such as the `#EXT-X-VERSION` tag). The option to abort parsing when attributes/tags are missing may be something to add later on. # Structure Summary -All of the details about the structs are taken from https://tools.ietf.org/html/draft-pantos-http-live-streaming-19. +All the details about the structs are taken from https://tools.ietf.org/html/draft-pantos-http-live-streaming-19. ```rust @@ -91,6 +91,8 @@ pub struct MasterPlaylist { pub session_key: Option, pub start: Option, pub independent_segments: bool, + pub alternatives: Vec, + pub unknown_tags: Vec, } pub struct MediaPlaylist { @@ -131,6 +133,7 @@ pub struct MediaSegment { pub map: Option, pub program_date_time: Option, pub daterange: Option, + pub unknown_tags: Vec, } ``` diff --git a/src/playlist.rs b/src/playlist.rs index a1c71d0..9bd2f17 100644 --- a/src/playlist.rs +++ b/src/playlist.rs @@ -715,7 +715,7 @@ mod test { }; let mut output = Vec::new(); - write!(output, "{}", cue_out_tag); + write!(output, "{}", cue_out_tag).unwrap(); assert_eq!(std::str::from_utf8(output.as_slice()).unwrap(), "#EXT-X-CUE-OUT:DURATION=30") } @@ -728,7 +728,7 @@ mod test { }; let mut output = Vec::new(); - write!(output, "{}", cue_in_tag); + write!(output, "{}", cue_in_tag).unwrap(); assert_eq!(std::str::from_utf8(output.as_slice()).unwrap(), "#EXT-X-CUE-IN") }