Merge pull request #30 from rafaelcaricio/support-segment-unknown-tags

Support parsing of unknown tags on segments

README.md

@@ -67,12 +67,12 @@ match parsed {
 }
 ```
 
-Currently the parser will succeed even if REQUIRED attributes/tags are missing from a playlist (such as the `#EXT-X-VERSION` tag).
+Currently, the parser will succeed even if REQUIRED attributes/tags are missing from a playlist (such as the `#EXT-X-VERSION` tag).
 The option to abort parsing when attributes/tags are missing may be something to add later on.
 
 # Structure Summary
 
-All of the details about the structs are taken from https://tools.ietf.org/html/draft-pantos-http-live-streaming-19.
+All the details about the structs are taken from https://tools.ietf.org/html/draft-pantos-http-live-streaming-19.
 
 
 ```rust
@@ -91,6 +91,8 @@ pub struct MasterPlaylist {
     pub session_key: Option<SessionKey>,
     pub start: Option<Start>,
     pub independent_segments: bool,
+    pub alternatives: Vec<AlternativeMedia>,
+    pub unknown_tags: Vec<ExtTag>,
 }
 
 pub struct MediaPlaylist {
@@ -131,6 +133,7 @@ pub struct MediaSegment {
     pub map: Option<Map>,
     pub program_date_time: Option<String>,
     pub daterange: Option<String>,
+    pub unknown_tags: Vec<ExtTag>,
 }
 
 ```

sample-playlists/media-playlist-with-cues-1.m3u8

@@ -0,0 +1,25 @@
+#EXTM3U
+# Borrowed from https://github.com/grafov/m3u8/pull/83
+#EXT-X-VERSION:3
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-TARGETDURATION:6
+#EXTINF:6.000,
+1.ts
+#EXT-X-DATERANGE:ID="20",START-DATE="2020-06-03T14:56:00Z",PLANNED-DURATION=19,SCTE35-OUT=0xFC302000000000000000FFF00F05000000147FFFFE001A17B0C0000000000061DFD67D
+#EXT-X-CUE-OUT:19.0
+#EXT-X-PROGRAM-DATE-TIME:2020-06-03T14:56:00Z
+#EXTINF:6.000,
+2.ts
+#EXTINF:6.000,
+3.ts
+#EXTINF:6.000,
+4.ts
+#EXT-X-CUE-IN
+#EXTINF:6.000,
+5.ts
+#EXTINF:6.000,
+6.ts
+#EXTINF:6.000,
+7.ts
+#EXTINF:6.000,
+8.ts

sample-playlists/media-playlist-with-cues.m3u8

@@ -0,0 +1,20 @@
+#EXTINF:10,
+http://media.example.com/fileSequence7796.ts
+#EXTINF:6,
+http://media.example.com/fileSequence7797.ts
+#EXT-X-CUE-OUT:DURATION=30
+#EXTINF:4,
+http://media.example.com/fileSequence7798.ts
+#EXTINF:10,
+http://media.example.com/fileSequence7799.ts
+#EXTINF:10,
+http://media.example.com/fileSequence7800.ts
+#EXTINF:6,
+http://media.example.com/fileSequence7801.ts
+#EXT-X-CUE-IN
+#EXTINF:4,
+http://media.example.com/fileSequence7802.ts
+#EXTINF:10,
+http://media.example.com/fileSequence7803.ts
+#EXTINF:3,
+http://media.example.com/fileSequence7804.ts

src/lib.rs

@@ -6,4 +6,4 @@ pub mod playlist;
 mod parser;
 
 #[cfg(feature = "parser")]
-pub use self::parser::*;
+pub use self::parser::*;

src/parser.rs

@@ -104,7 +104,7 @@ use playlist::*;
 /// ```
 /// use std::io::Read;
 /// use m3u8_rs::playlist::{Playlist};
-/// 
+///
 /// let mut file = std::fs::File::open("playlist.m3u8").unwrap();
 /// let mut bytes: Vec<u8> = Vec::new();
 /// file.read_to_end(&mut bytes).unwrap();
@@ -129,8 +129,8 @@ pub fn parse_playlist(input: &[u8]) -> IResult<&[u8], Playlist> {
 }
 
 /// Parses an m3u8 playlist just like `parse_playlist`, except that this returns an [std::result::Result](std::result::Result) instead of a [nom::IResult](https://docs.rs/nom/1.2.3/nom/enum.IResult.html).
-/// However, since [nom::IResult](nom::IResult) is now an [alias to Result](https://github.com/Geal/nom/blob/master/doc/upgrading_to_nom_5.md), this is no longer needed. 
-/// 
+/// However, since [nom::IResult](nom::IResult) is now an [alias to Result](https://github.com/Geal/nom/blob/master/doc/upgrading_to_nom_5.md), this is no longer needed.
+///
 /// # Examples
 ///
 /// ```
@@ -274,9 +274,9 @@ pub enum MasterPlaylistTag {
     SessionKey(SessionKey),
     Start(Start),
     IndependentSegments,
-    Unknown(ExtTag),
     Comment(String),
     Uri(String),
+    Unknown(ExtTag),
 }
 
 pub fn master_playlist_tag(input: &[u8]) -> IResult<&[u8], MasterPlaylistTag> {
@@ -293,6 +293,7 @@ pub fn master_playlist_tag(input: &[u8]) -> IResult<&[u8], MasterPlaylistTag> {
         | map!(tag!("#EXT-X-INDEPENDENT-SEGMENTS"), |_| MasterPlaylistTag::IndependentSegments)
 
         | map!(ext_tag, MasterPlaylistTag::Unknown)
+
         | map!(comment_tag, MasterPlaylistTag::Comment)
 
         | map!(consume_line, MasterPlaylistTag::Uri)
@@ -392,6 +393,7 @@ pub enum MediaPlaylistTag {
     PlaylistType(MediaPlaylistType),
     IFramesOnly,
     Start(Start),
+    Unknown(ExtTag),
     IndependentSegments,
 }
 
@@ -474,7 +476,7 @@ pub fn media_playlist_from_tags(mut tags: Vec<MediaPlaylistTag>) -> MediaPlaylis
                         next_segment.daterange = Some(d);
                     }
                     SegmentTag::Unknown(t) => {
-                        media_playlist.unknown_tags.push(t);
+                        next_segment.unknown_tags.push(t);
                     }
                     SegmentTag::Uri(u) => {
                         next_segment.key = encryption_key.clone();
@@ -588,13 +590,14 @@ named!(pub start_tag<Start>,
 
 named!(pub ext_tag<ExtTag>,
     do_parse!(
-          tag!("#EXT-")
-        >> tag: map_res!(take_until!(":"), from_utf8_slice)
+        tag!("#EXT-")
+        >> tag: map_res!(is_not!("\r\n:"), from_utf8_slice)
+        >> opt!(tag!(":"))
+        >> rest: opt!(map_res!(is_not!("\r\n"), from_utf8_slice))
         >> take!(1)
-        >> rest: map_res!(is_not!("\r\n"), from_utf8_slice)
-        >> take!(1)
-        >>
-        (ExtTag { tag: tag, rest: rest })
+        >> (
+            ExtTag { tag: tag, rest: rest }
+        )
     )
 );
 

src/playlist.rs

@@ -8,6 +8,7 @@ use std::collections::HashMap;
 use std::str::FromStr;
 use std::fmt;
 use std::f32;
+use std::fmt::Display;
 
 macro_rules! write_some_attribute_quoted {
     ($w:expr, $tag:expr, $o:expr) => (
@@ -69,7 +70,7 @@ pub struct MasterPlaylist {
     pub start: Option<Start>,
     pub independent_segments: bool,
     pub alternatives: Vec<AlternativeMedia>, // EXT-X-MEDIA tags
-    pub unknown_tags: Vec<ExtTag>
+    pub unknown_tags: Vec<ExtTag>,
 }
 
 impl MasterPlaylist {
@@ -102,7 +103,7 @@ impl MasterPlaylist {
             writeln!(w, "#EXT-X-INDEPENDENT-SEGMENTS")?;
         }
         for unknown_tag in &self.unknown_tags {
-            write!(w, "#EXT-{}:{}\n", unknown_tag.tag, unknown_tag.rest)?;
+            writeln!(w, "{}", unknown_tag)?;
         }
 
         Ok(())
@@ -398,8 +399,6 @@ pub struct MediaPlaylist {
     pub start: Option<Start>,
     /// `#EXT-X-INDEPENDENT-SEGMENTS`
     pub independent_segments: bool,
-    /// `#EXT-X-` 
-    pub unknown_tags: Vec<ExtTag>
 }
 
 impl MediaPlaylist {
@@ -433,9 +432,6 @@ impl MediaPlaylist {
         for segment in &self.segments {
             segment.write_to(w)?;
         }
-        for unknown_tag in &self.unknown_tags {
-            write!(w, "#EXT-{}:{}\n", unknown_tag.tag, unknown_tag.rest)?;
-        }
 
         Ok(())
     }
@@ -501,6 +497,8 @@ pub struct MediaSegment {
     pub program_date_time: Option<String>,
     /// `#EXT-X-DATERANGE:<attribute-list>`
     pub daterange: Option<String>,
+    /// `#EXT-`
+    pub unknown_tags: Vec<ExtTag>,
 }
 
 impl MediaSegment {
@@ -534,6 +532,9 @@ impl MediaSegment {
         if let Some(ref v) = self.daterange {
             writeln!(w, "#EXT-X-DATERANGE:{}", v)?;
         }
+        for unknown_tag in &self.unknown_tags {
+            writeln!(w, "{}", unknown_tag)?;
+        }
 
         write!(w, "#EXTINF:{},", self.duration)?;
 
@@ -689,6 +690,46 @@ impl Start {
 #[derive(Debug, Default, PartialEq, Clone)]
 pub struct ExtTag {
     pub tag: String,
-    pub rest: String,
+    pub rest: Option<String>,
 }
 
+impl Display for ExtTag {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "#EXT-{}", self.tag)?;
+        if let Some(v) = &self.rest {
+            write!(f, ":{}", v)?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn ext_tag_with_value_is_printable() {
+        let cue_out_tag = ExtTag {
+            tag: "X-CUE-OUT".into(),
+            rest: Some("DURATION=30".into()),
+        };
+
+        let mut output = Vec::new();
+        write!(output, "{}", cue_out_tag).unwrap();
+
+        assert_eq!(std::str::from_utf8(output.as_slice()).unwrap(), "#EXT-X-CUE-OUT:DURATION=30")
+    }
+
+    #[test]
+    fn ext_tag_without_value_is_printable() {
+        let cue_in_tag = ExtTag {
+            tag: "X-CUE-IN".into(),
+            rest: None,
+        };
+
+        let mut output = Vec::new();
+        write!(output, "{}", cue_in_tag).unwrap();
+
+        assert_eq!(std::str::from_utf8(output.as_slice()).unwrap(), "#EXT-X-CUE-IN")
+    }
+}

tests/lib.rs

@@ -85,6 +85,26 @@ fn playlist_media_without_segments() {
     assert!(print_parse_playlist_test("media-playlist-without-segments.m3u8"));
 }
 
+#[test]
+fn playlist_media_with_cues() {
+    assert!(print_parse_playlist_test("media-playlist-with-cues.m3u8"));
+}
+
+#[test]
+fn playlist_media_with_cues1() {
+    assert!(print_parse_playlist_test("media-playlist-with-cues-1.m3u8"));
+}
+
+#[test]
+fn playlist_media_with_scte35() {
+    assert!(print_parse_playlist_test("media-playlist-with-scte35.m3u8"));
+}
+
+#[test]
+fn playlist_media_with_scte35_1() {
+    assert!(print_parse_playlist_test("media-playlist-with-scte35-1.m3u8"));
+}
+
 // -----------------------------------------------------------------------------------------------
 // Playlist with no newline end
 
@@ -114,10 +134,10 @@ fn playlist_type_is_master() {
 }
 
 // #[test]
-// fn playlist_type_with_unkown_tag() {
+// fn playlist_type_with_unknown_tag() {
 //     let input = get_sample_playlist("!!");
 //     let result = is_master_playlist(input.as_bytes());
-//     println!("Playlist_type_with_unkown_tag is master playlist: {:?}", result);
+//     println!("Playlist_type_with_unknown_tag is master playlist: {:?}", result);
 //     assert_eq!(true, result);
 // }
 
@@ -195,6 +215,22 @@ fn test_key_value_pair() {
     );
 }
 
+#[test]
+fn ext_with_value() {
+    assert_eq!(
+        ext_tag(b"#EXT-X-CUE-OUT:DURATION=30\nxxx"),
+        Result::Ok((b"xxx".as_bytes(), ExtTag { tag: "X-CUE-OUT".into(), rest: Some("DURATION=30".into()) }))
+    );
+}
+
+#[test]
+fn ext_without_value() {
+    assert_eq!(
+        ext_tag(b"#EXT-X-CUE-IN\nxxx"),
+        Result::Ok((b"xxx".as_bytes(), ExtTag { tag: "X-CUE-IN".into(), rest: None }))
+    );
+}
+
 #[test]
 fn comment() {
     assert_eq!(
@@ -213,7 +249,7 @@ fn quotes() {
 
 #[test]
 fn consume_line_empty() {
-    let expected = Result::Ok(("rest".as_bytes(), "".to_string())); 
+    let expected = Result::Ok(("rest".as_bytes(), "".to_string()));
     let actual = consume_line(b"\r\nrest");
     assert_eq!(expected, actual);
 }
@@ -420,9 +456,14 @@ fn create_and_parse_media_playlist_full() {
                 }),
                 program_date_time: Some("broodlordinfestorgg".into()),
                 daterange: None,
+                unknown_tags: vec![
+                    ExtTag {
+                        tag: "X-CUE-OUT".into(),
+                        rest: Some("DURATION=2.002".into())
+                    }
+                ]
             },
         ],
-        unknown_tags: vec![],
     });
     let playlist_parsed = print_create_and_parse_playlist(&mut playlist_original);
     assert_eq!(playlist_original, playlist_parsed);