From 3537614c2b04606939e09595fe01700b3e65acb1 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Fri, 12 Aug 2022 04:56:47 +1000 Subject: [PATCH] hlsdemux2: Add and use gst_hls_media_playlist_find_position() Add a function for synchronising current position with the contents of a playlist that is specifically for that and can handle synchronising to a partial segment. gst_hls_media_playlist_seek() will be used only when performing external seek requests, to find the best segment or partial segment at which to resume playback. Part-of: --- .../ext/adaptivedemux2/hls/gsthlsdemux-util.c | 30 ++-- .../ext/adaptivedemux2/hls/gsthlsdemux.c | 55 +++--- .../ext/adaptivedemux2/hls/m3u8.c | 168 ++++++++++++++++++ .../ext/adaptivedemux2/hls/m3u8.h | 17 ++ 4 files changed, 236 insertions(+), 34 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux-util.c b/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux-util.c index 89a949df88..0b1e8c5e0f 100644 --- a/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux-util.c +++ b/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux-util.c @@ -962,7 +962,6 @@ out: g_free (original_content); if (out_of_bounds) { - GstM3U8MediaSegment *candidate_segment; /* The computed stream time falls outside of the guesstimated stream time, * reassess which segment we really are in */ @@ -974,22 +973,31 @@ out: GST_STIME_ARGS (current_segment->stream_time + current_segment->duration)); - /* FIXME: Could seek to an INDEPENDENT partial segment in LL-HLS */ - candidate_segment = - gst_hls_media_playlist_seek (hls_stream->playlist, TRUE, - GST_SEEK_FLAG_SNAP_NEAREST, low_stream_time); - if (candidate_segment) { - g_assert (candidate_segment != current_segment); + GstM3U8SeekResult seek_result; + + if (gst_hls_media_playlist_find_position (hls_stream->playlist, + low_stream_time, hls_stream->in_partial_segments, &seek_result)) { + g_assert (seek_result.segment != current_segment); GST_DEBUG_OBJECT (hls_stream, "Stream time corresponds to segment %" GST_STIME_FORMAT " duration %" GST_TIME_FORMAT, - GST_STIME_ARGS (candidate_segment->stream_time), - GST_TIME_ARGS (candidate_segment->duration)); + GST_STIME_ARGS (seek_result.segment->stream_time), + GST_TIME_ARGS (seek_result.segment->duration)); + + /* When we land in the middle of a partial segment, actually + * use the full segment position to resync the playlist */ + if (seek_result.found_partial_segment) { + hls_stream->current_segment->stream_time = + seek_result.segment->stream_time; + } else { + hls_stream->current_segment->stream_time = seek_result.stream_time; + } + /* Recalculate everything and ask parent class to restart */ - hls_stream->current_segment->stream_time = candidate_segment->stream_time; gst_hls_media_playlist_recalculate_stream_time (hls_stream->playlist, hls_stream->current_segment); - gst_m3u8_media_segment_unref (candidate_segment); + gst_m3u8_media_segment_unref (seek_result.segment); + ret = GST_HLS_PARSER_RESULT_RESYNC; } } diff --git a/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux.c b/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux.c index ea7e3035c5..d2185be241 100644 --- a/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux.c +++ b/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux.c @@ -1359,25 +1359,27 @@ gst_hlsdemux_handle_internal_time (GstHLSDemux * demux, gst_hls_media_playlist_dump (hls_stream->playlist); } + /* FIXME: When playing partial segments, the threshold should be + * half the part duration */ if (ABS (difference) > (hls_stream->current_segment->duration / 2)) { GstAdaptiveDemux2Stream *stream = (GstAdaptiveDemux2Stream *) hls_stream; - GstM3U8MediaSegment *actual_segment; + GstM3U8SeekResult seek_result; /* We are at the wrong segment, try to figure out the *actual* segment */ GST_DEBUG_OBJECT (hls_stream, - "Trying to seek to the correct segment for %" GST_STIME_FORMAT, - GST_STIME_ARGS (current_stream_time)); - /* FIXME: Allow jumping to partial segments in LL-HLS */ - actual_segment = - gst_hls_media_playlist_seek (hls_stream->playlist, TRUE, - GST_SEEK_FLAG_SNAP_NEAREST, current_stream_time); + "Trying to find the correct segment in the playlist for %" + GST_STIME_FORMAT, GST_STIME_ARGS (current_stream_time)); + if (gst_hls_media_playlist_find_position (hls_stream->playlist, + current_stream_time, hls_stream->in_partial_segments, + &seek_result)) { - if (actual_segment) { GST_DEBUG_OBJECT (hls_stream, "Synced to position %" GST_STIME_FORMAT, - GST_STIME_ARGS (actual_segment->stream_time)); + GST_STIME_ARGS (seek_result.stream_time)); + gst_m3u8_media_segment_unref (hls_stream->current_segment); - hls_stream->current_segment = actual_segment; - hls_stream->in_partial_segments = FALSE; + hls_stream->current_segment = seek_result.segment; + hls_stream->in_partial_segments = seek_result.found_partial_segment; + hls_stream->part_idx = seek_result.part_idx; /* Ask parent class to restart this fragment */ return GST_HLS_PARSER_RESULT_RESYNC; @@ -2499,11 +2501,11 @@ gst_hls_demux_stream_update_fragment_info (GstAdaptiveDemux2Stream * stream) if (hlsdemux_stream->current_segment->partial_only) { /* FIXME: We might find an independent partial segment * that's still old enough (beyond the part_hold_back threshold) - * but closer to the live edge than the start of the segment */ + * but closer to the live edge than the start of the segment. This + * check should be done inside get_starting_segment() */ hlsdemux_stream->in_partial_segments = TRUE; hlsdemux_stream->part_idx = 0; } - } else { if (gst_hls_media_playlist_has_lost_sync (hlsdemux_stream->playlist, stream->current_position)) { @@ -2513,21 +2515,28 @@ gst_hls_demux_stream_update_fragment_info (GstAdaptiveDemux2Stream * stream) GST_DEBUG_OBJECT (stream, "Looking up segment for position %" GST_TIME_FORMAT, GST_TIME_ARGS (stream->current_position)); - /* FIXME: Look up partial segments in LL-HLS */ - hlsdemux_stream->current_segment = - gst_hls_media_playlist_seek (hlsdemux_stream->playlist, TRUE, - GST_SEEK_FLAG_SNAP_NEAREST, stream->current_position); - if (hlsdemux_stream->current_segment == NULL) { + GstM3U8SeekResult seek_result; + if (!gst_hls_media_playlist_find_position (hlsdemux_stream->playlist, + stream->current_position, hlsdemux_stream->in_partial_segments, + &seek_result)) { GST_INFO_OBJECT (stream, "At the end of the current media playlist"); return GST_FLOW_EOS; } - /* Update time mapping. If it already exists it will be ignored */ - gst_hls_demux_add_time_mapping (hlsdemux, - hlsdemux_stream->current_segment->discont_sequence, - hlsdemux_stream->current_segment->stream_time, - hlsdemux_stream->current_segment->datetime); + hlsdemux_stream->current_segment = seek_result.segment; + hlsdemux_stream->in_partial_segments = seek_result.found_partial_segment; + hlsdemux_stream->part_idx = seek_result.part_idx; + + /* If on a full segment, update time mapping. If it already exists it will be ignored. + * Don't add time mappings for partial segments, wait for a full segment boundary */ + if (!hlsdemux_stream->in_partial_segments + || hlsdemux_stream->part_idx == 0) { + gst_hls_demux_add_time_mapping (hlsdemux, + hlsdemux_stream->current_segment->discont_sequence, + hlsdemux_stream->current_segment->stream_time, + hlsdemux_stream->current_segment->datetime); + } } } diff --git a/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/m3u8.c b/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/m3u8.c index d332e952c8..8b983600e8 100644 --- a/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/m3u8.c +++ b/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/m3u8.c @@ -1207,6 +1207,14 @@ gst_hls_media_playlist_has_same_data (GstHLSMediaPlaylist * self, return ret; } +/* gst_hls_media_playlist_seek() is used when performing + * an actual seek. It finds a suitable segment (or partial segment + * for LL-HLS) at which to resume playback. Only partial segments + * in the last 2 target durations of the live edge are considered + * when playing live, otherwise we might start playing a partial + * segment group that disappears before we're done with it. + * We want a segment or partial that contains a keyframe if possible + */ GstM3U8MediaSegment * gst_hls_media_playlist_seek (GstHLSMediaPlaylist * playlist, gboolean forward, GstSeekFlags flags, GstClockTimeDiff ts) @@ -1262,6 +1270,166 @@ out: return res; } +static gboolean +gst_hls_media_playlist_find_partial_position (GstHLSMediaPlaylist * playlist, + GstM3U8MediaSegment * seg, GstClockTimeDiff ts, + GstM3U8SeekResult * seek_result) +{ + guint i; + + /* As with full segment search below, we more often want to find our position + * near the end of a live playlist, so iterate segments backward */ + for (i = seg->partial_segments->len; i > 0; i--) { + guint part_idx = i - 1; + GstM3U8PartialSegment *cand = + g_ptr_array_index (seg->partial_segments, part_idx); + + GST_DEBUG ("partial segment %d ts:%" GST_STIME_FORMAT " end:%" + GST_STIME_FORMAT, part_idx, GST_STIME_ARGS (cand->stream_time), + GST_STIME_ARGS (cand->stream_time + cand->duration)); + + /* If the target timestamp is before this partial segment, or in the first half, this + * is the partial segment to land in */ + if (cand->stream_time + (cand->duration / 2) >= ts && + cand->stream_time <= ts + (cand->duration / 2)) { + GST_DEBUG ("choosing partial segment %d", part_idx); + seek_result->segment = gst_m3u8_media_segment_ref (seg); + seek_result->found_partial_segment = TRUE; + seek_result->part_idx = part_idx; + seek_result->stream_time = cand->stream_time; + return TRUE; + } + } + + return FALSE; +} + +/* gst_hls_media_playlist_find_position() is used + * when finding the segment or partial segment that corresponds + * to our current playback position. + * If we're "playing partial segments", we want to find the partial segment + whose stream_time matches the target position most closely (or fail + if there's no partial segment, since the target partial segment was + removed from the playlist and we lost sync. + * If not currently playing partial segment, find the segment with a + * stream_time that matches, or the partial segment exactly at the start + * of the 'partial_only' segment. + */ +gboolean +gst_hls_media_playlist_find_position (GstHLSMediaPlaylist * playlist, + GstClockTimeDiff ts, gboolean in_partial_segments, + GstM3U8SeekResult * seek_result) +{ + guint i; + GstM3U8MediaSegment *seg = NULL; + + GST_DEBUG ("ts:%" GST_STIME_FORMAT + " in_partial_segments %d (live %d) playlist uri: %s", GST_STIME_ARGS (ts), + in_partial_segments, GST_HLS_MEDIA_PLAYLIST_IS_LIVE (playlist), + playlist->uri); + + /* The *common* case is that we want to find our position in a live playback + * scenario, when we're playing close to the live edge, so start at the end + * of the segments and go backward */ + for (i = playlist->segments->len; i != 0; i--) { + guint seg_idx = i - 1; + GstM3U8MediaSegment *cand = g_ptr_array_index (playlist->segments, seg_idx); + + GST_DEBUG ("segment %d ts:%" GST_STIME_FORMAT " end:%" GST_STIME_FORMAT + " partial only: %d", + seg_idx, GST_STIME_ARGS (cand->stream_time), + GST_STIME_ARGS (cand->stream_time + cand->duration), + cand->partial_only); + + /* Ignore any (disallowed by the spec) partial_only segment if + * the playlist is no longer live */ + if (cand->partial_only && !GST_HLS_MEDIA_PLAYLIST_IS_LIVE (playlist)) + continue; + + /* If the target stream time is definitely past the end + * of this segment, no earlier segment (with lower stream time) + * could match, so we fail */ + if (ts >= cand->stream_time + (3 * cand->duration / 2)) { + break; + } + + if (in_partial_segments || cand->partial_only) { + if (cand->partial_segments == NULL) { + GstClockTime partial_targetduration = playlist->partial_targetduration; + + /* Default, if the playlist fails to give us a part duration (REQUIRED attribute, but + * maybe it got removed) */ + if (!GST_CLOCK_TIME_IS_VALID (partial_targetduration)) { + partial_targetduration = 200 * GST_MSECOND; + } + + /* If we want to match a partial segment but this segment doesn't have + * any, then the partial segment we want got removed from the playlist, + * so we need to fail, except in the specific case that our target + * timestamp is within half a part duration of the segment start + * itself (ie, we wanted the *first* partial segment + */ + if (cand->stream_time + (partial_targetduration / 2) >= ts && + cand->stream_time <= ts + (partial_targetduration / 2)) { + GST_DEBUG ("choosing full segment %d", seg_idx); + seek_result->stream_time = seg->stream_time; + seek_result->segment = gst_m3u8_media_segment_ref (seg); + seek_result->found_partial_segment = FALSE; + return TRUE; + } + + GST_DEBUG ("Couldn't find a matching partial segment"); + return FALSE; + } + + /* If our partial segment target ts is within half a partial duration + * of this segment start/finish, check the partial segments for a match */ + if (gst_hls_media_playlist_find_partial_position (playlist, cand, ts, + seek_result)) { + GST_DEBUG ("Returning partial segment sn:%" G_GINT64_FORMAT + " part %u stream_time:%" GST_STIME_FORMAT, cand->sequence, + seek_result->part_idx, GST_STIME_ARGS (seek_result->stream_time)); + return TRUE; + } + } + + /* Otherwise, we're doing a full segment match so check that the timestamp is + * within half a segment duration of this segment stream_time */ + if (cand->stream_time + (cand->duration / 2) >= ts && + cand->stream_time <= ts + (cand->duration / 2)) { + GST_DEBUG ("choosing segment %d", seg_idx); + seg = cand; + break; + } + } + + if (seg == NULL) { + GST_DEBUG ("Couldn't find a matching segment"); + return FALSE; + } + + /* The partial_only segment case should have been handled above + * by gst_hls_media_playlist_find_partial_position(). If it + * wasn't, it implies the segment we're looking for was not + * present in the available partial segments at all, + * so we need to return FALSE */ + if (seg->partial_only) { + GST_DEBUG + ("Couldn't find a matching partial segment in the partial_only segment"); + return FALSE; + } + + seek_result->stream_time = seg->stream_time; + seek_result->segment = gst_m3u8_media_segment_ref (seg); + seek_result->found_partial_segment = FALSE; + + GST_DEBUG ("Returning segment sn:%" G_GINT64_FORMAT " stream_time:%" + GST_STIME_FORMAT " duration:%" GST_TIME_FORMAT, seg->sequence, + GST_STIME_ARGS (seg->stream_time), GST_TIME_ARGS (seg->duration)); + + return TRUE; +} + /* Recalculate all segment DSN based on the DSN of the provided anchor segment * (which must belong to the playlist). */ static void diff --git a/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/m3u8.h b/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/m3u8.h index dc497ac2d0..cee63e34bd 100644 --- a/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/m3u8.h +++ b/subprojects/gst-plugins-good/ext/adaptivedemux2/hls/m3u8.h @@ -34,6 +34,7 @@ G_BEGIN_DECLS typedef struct _GstHLSMediaPlaylist GstHLSMediaPlaylist; typedef struct _GstHLSTimeMap GstHLSTimeMap; +typedef struct _GstM3U8SeekResult GstM3U8SeekResult; typedef struct _GstM3U8MediaSegment GstM3U8MediaSegment; typedef struct _GstM3U8PartialSegment GstM3U8PartialSegment; typedef struct _GstM3U8InitFile GstM3U8InitFile; @@ -70,6 +71,16 @@ typedef enum { * flags */ #define GST_HLS_M3U8_SEEK_FLAG_ALLOW_PARTIAL (1 << 16) /* Allow seeking to a partial segment */ +struct _GstM3U8SeekResult { + /* stream time of the segment or partial segment */ + GstClockTimeDiff stream_time; + + GstM3U8MediaSegment *segment; + + gboolean found_partial_segment; + guint part_idx; +}; + /** * GstHLSMediaPlaylist: * @@ -305,6 +316,12 @@ gst_hls_media_playlist_seek (GstHLSMediaPlaylist *playlist, gboolean forward, GstSeekFlags flags, GstClockTimeDiff ts); + +gboolean +gst_hls_media_playlist_find_position (GstHLSMediaPlaylist *playlist, + GstClockTimeDiff ts, gboolean in_partial_segments, + GstM3U8SeekResult *seek_result); + void gst_hls_media_playlist_dump (GstHLSMediaPlaylist* self);