adaptivedemux: Add 'backoff' logic for HTTP request

So that the user can configure waits between retries

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8128>

subprojects/gst-plugins-good/docs/gst_plugins_cache.json

@@ -1431,6 +1431,34 @@
                         "readable": true,
                         "type": "guint",
                         "writable": true
+                    },
+                    "retry-backoff-factor": {
+                        "blurb": "Exponential retry backoff factor in seconds",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "0",
+                        "max": "1.79769e+308",
+                        "min": "0",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gdouble",
+                        "writable": true
+                    },
+                    "retry-backoff-max": {
+                        "blurb": "Maximum backoff delay in seconds",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "60",
+                        "max": "1.79769e+308",
+                        "min": "0",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gdouble",
+                        "writable": true
                     }
                 }
             }

subprojects/gst-plugins-good/ext/adaptivedemux2/gstadaptivedemux-private.h

@@ -167,6 +167,9 @@ struct _GstAdaptiveDemuxPrivate
   /* The maximum number of times HTTP request can be required before considering
    * failed */
   gint max_retries;
+  /* The backoff factor and max for the HTTP request retries */
+  gdouble retry_backoff_factor;
+  gdouble retry_backoff_max;
 
 };
 

subprojects/gst-plugins-good/ext/adaptivedemux2/gstadaptivedemux-stream.c

@@ -1241,8 +1241,8 @@ on_download_error (DownloadRequest * request, DownloadRequestState state,
             GST_LOG_OBJECT (stream,
                 "Scheduling delayed load_a_fragment() call");
             stream->pending_cb_id =
-                gst_adaptive_demux_loop_call_delayed (demux->
+                gst_adaptive_demux_loop_call_delayed (demux->priv->
-                priv->scheduler_task, wait_time,
+                scheduler_task, wait_time,
                 (GSourceFunc) gst_adaptive_demux2_stream_load_a_fragment,
                 gst_object_ref (stream), (GDestroyNotify) gst_object_unref);
             return;
@@ -1281,10 +1281,16 @@ on_download_error (DownloadRequest * request, DownloadRequestState state,
 
 again:
   /* wait a short time in case the server needs a bit to recover */
-  GST_LOG_OBJECT (stream,
-      "Scheduling delayed load_a_fragment() call to retry in 10 milliseconds");
   g_assert (stream->pending_cb_id == 0);
-  stream->pending_cb_id = gst_adaptive_demux_loop_call_delayed (demux->priv->scheduler_task, 10 * GST_MSECOND,  /* Retry in 10 ms */
+
+  GstClockTime delay =
+      gst_adaptive_demux_retry_delay (demux, stream->download_error_count,
+      10 * GST_MSECOND);
+  GST_DEBUG_OBJECT (stream,
+      "Scheduling delayed reload_manifest_cb() %d call in %" GST_TIMEP_FORMAT,
+      stream->download_error_count, &delay);
+  stream->pending_cb_id =
+      gst_adaptive_demux_loop_call_delayed (demux->priv->scheduler_task, delay,
       (GSourceFunc) gst_adaptive_demux2_stream_load_a_fragment,
       gst_object_ref (stream), (GDestroyNotify) gst_object_unref);
 }
@@ -2030,12 +2036,15 @@ gst_adaptive_demux2_stream_load_a_fragment (GstAdaptiveDemux2Stream * stream)
           }
         }
 
-        /* Wait half the fragment duration before retrying */
+        GstClockTime delay = gst_adaptive_demux_retry_delay (stream->demux,
-        GST_LOG_OBJECT (stream, "Scheduling delayed reload_manifest_cb() call");
+            stream->download_error_count, stream->fragment.duration / 2);
+        GST_DEBUG_OBJECT (stream,
+            "Scheduling delayed reload_manifest_cb() call in %"
+            GST_TIMEP_FORMAT, &delay);
         g_assert (stream->pending_cb_id == 0);
         stream->pending_cb_id =
             gst_adaptive_demux_loop_call_delayed (demux->priv->scheduler_task,
-            stream->fragment.duration / 2,
+            delay,
             (GSourceFunc) gst_adaptive_demux2_stream_reload_manifest_cb,
             gst_object_ref (stream), (GDestroyNotify) gst_object_unref);
         return FALSE;

subprojects/gst-plugins-good/ext/adaptivedemux2/gstadaptivedemux.c

@@ -114,6 +114,8 @@ GST_DEBUG_CATEGORY_EXTERN (adaptivedemux2_debug);
 
 #define DEFAULT_FAILED_COUNT 3
 #define DEFAULT_MAX_RETRIES 3
+#define DEFAULT_RETRY_BACKOFF_FACTOR 0.0
+#define DEFAULT_RETRY_BACKOFF_MAX    60.0
 #define DEFAULT_CONNECTION_BITRATE 0
 #define DEFAULT_BANDWIDTH_TARGET_RATIO 0.8f
 
@@ -135,6 +137,8 @@ enum
   PROP_0,
   PROP_CONNECTION_SPEED,
   PROP_MAX_RETRIES,
+  PROP_RETRY_BACKOFF_FACTOR,
+  PROP_RETRY_BACKOFF_MAX,
   PROP_BANDWIDTH_TARGET_RATIO,
   PROP_CONNECTION_BITRATE,
   PROP_MIN_BITRATE,
@@ -329,6 +333,12 @@ gst_adaptive_demux_set_property (GObject * object, guint prop_id,
     case PROP_BUFFERING_LOW_WATERMARK_FRAGMENTS:
       demux->buffering_low_watermark_fragments = g_value_get_double (value);
       break;
+    case PROP_RETRY_BACKOFF_FACTOR:
+      demux->priv->retry_backoff_factor = g_value_get_double (value);
+      break;
+    case PROP_RETRY_BACKOFF_MAX:
+      demux->priv->retry_backoff_max = g_value_get_double (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -388,6 +398,12 @@ gst_adaptive_demux_get_property (GObject * object, guint prop_id,
     case PROP_CURRENT_LEVEL_TIME_AUDIO:
       g_value_set_uint64 (value, demux->current_level_time_audio);
       break;
+    case PROP_RETRY_BACKOFF_FACTOR:
+      g_value_set_double (value, demux->priv->retry_backoff_factor);
+      break;
+    case PROP_RETRY_BACKOFF_MAX:
+      g_value_set_double (value, demux->priv->retry_backoff_max);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -527,7 +543,41 @@ gst_adaptive_demux_class_init (GstAdaptiveDemuxClass * klass)
   g_object_class_install_property (gobject_class, PROP_MAX_RETRIES,
       g_param_spec_int ("max-retries", "Maximum Retries",
           "Maximum number of retries for HTTP requests (-1=infinite)",
-          -1, G_MAXUINT, DEFAULT_MAX_RETRIES,
+          -1, G_MAXINT, DEFAULT_MAX_RETRIES,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+ /**
+   * GstAdaptiveDemux2:retry-backoff-factor:
+   *
+   * A backoff factor to apply between attempts after the second try
+   * (most errors are resolved immediately by a second try without a delay).
+   * souphttpsrc will sleep for:
+   *
+   * ```
+   * {backoff factor} * (2 ** ({number of previous retries}))
+   * ``
+   *
+   * seconds
+   *
+   * Since: 1.26
+   */
+  g_object_class_install_property (gobject_class, PROP_RETRY_BACKOFF_FACTOR,
+      g_param_spec_double ("retry-backoff-factor", "Backoff Factor",
+          "Exponential retry backoff factor in seconds", 0.0, G_MAXDOUBLE,
+          DEFAULT_RETRY_BACKOFF_FACTOR,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  /**
+   * GstAdaptiveDemux2:retry-backoff-max:
+   *
+   * Maximum retry backoff delay in seconds
+   *
+   * Since: 1.26
+   */
+  g_object_class_install_property (gobject_class, PROP_RETRY_BACKOFF_MAX,
+      g_param_spec_double ("retry-backoff-max", "Maximum retry Backoff delay",
+          "Maximum backoff delay in seconds", 0.0, G_MAXDOUBLE,
+          DEFAULT_RETRY_BACKOFF_MAX,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   gst_element_class_add_static_pad_template (gstelement_class,
@@ -613,6 +663,8 @@ gst_adaptive_demux_init (GstAdaptiveDemux * demux,
   demux->current_level_time_video = DEFAULT_CURRENT_LEVEL_TIME_VIDEO;
   demux->current_level_time_audio = DEFAULT_CURRENT_LEVEL_TIME_AUDIO;
   demux->priv->max_retries = DEFAULT_MAX_RETRIES;
+  demux->priv->retry_backoff_factor = DEFAULT_RETRY_BACKOFF_FACTOR;
+  demux->priv->retry_backoff_max = DEFAULT_RETRY_BACKOFF_MAX;
 
   gst_element_add_pad (GST_ELEMENT (demux), demux->sinkpad);
 
@@ -3991,3 +4043,21 @@ gst_adaptive_demux_max_retries (GstAdaptiveDemux * self)
 
   return res;
 }
+
+GstClockTime
+gst_adaptive_demux_retry_delay (GstAdaptiveDemux * self, gint retry,
+    GstClockTime default_delay)
+{
+  GST_OBJECT_LOCK (self);
+  gdouble backoff_factor = self->priv->retry_backoff_factor;
+  gdouble backoff_max = self->priv->retry_backoff_max;
+  GST_OBJECT_UNLOCK (self);
+
+  GstClockTime delay = default_delay;
+  if (backoff_factor > 0) {
+    GstClockTime backoff_delay = (backoff_factor * (1 << retry)) * GST_SECOND;
+    delay = MIN (backoff_delay, (backoff_max * GST_SECOND));
+  }
+
+  return delay;
+}

subprojects/gst-plugins-good/ext/adaptivedemux2/gstadaptivedemux.h

@@ -209,20 +209,20 @@ struct _GstAdaptiveDemuxPeriod
   gint ref_count;
 
   GstAdaptiveDemux *demux;
-  
+
   /* TRUE if the streams of this period were prepared and can be started */
   gboolean prepared;
 
 
   /* TRUE if there is another period after this one */
   gboolean has_next_period;
-  
+
   /* An increasing unique identifier for the period.
    *
    * Note: unrelated to dash period id (which can be identical across
    * periods) */
   guint period_num;
-  
+
   /* The list of GstAdaptiveDemux2Stream (ref hold) */
   GList *streams;
 
@@ -264,7 +264,7 @@ struct _GstAdaptiveDemux
 
   /* Period used for input */
   GstAdaptiveDemuxPeriod *input_period;
-  
+
   GstSegment segment;
   gdouble instant_rate_multiplier; /* 1.0 by default, or from instant-rate seek */
 
@@ -482,6 +482,7 @@ gdouble gst_adaptive_demux_play_rate (GstAdaptiveDemux *demux);
 void gst_adaptive_demux2_manual_manifest_update (GstAdaptiveDemux * demux);
 GstAdaptiveDemuxLoop *gst_adaptive_demux_get_loop (GstAdaptiveDemux *demux);
 gint gst_adaptive_demux_max_retries (GstAdaptiveDemux *self);
+GstClockTime gst_adaptive_demux_retry_delay (GstAdaptiveDemux * self, gint retry, GstClockTime default_delay);
 
 G_END_DECLS
 

subprojects/gst-plugins-good/ext/adaptivedemux2/hls/gsthlsdemux-playlist-loader.c

@@ -482,8 +482,15 @@ handle_download_error (GstHLSDemuxPlaylistLoader * pl,
   /* The error callback may have provided a new playlist to load, which
    * will have scheduled a state update immediately. In that case,
    * don't trigger our own delayed retry */
-  if (priv->pending_cb_id == 0)
+  if (priv->pending_cb_id == 0) {
-    schedule_next_playlist_load (pl, priv, 100 * GST_MSECOND);
+    GstClockTime delay =
+        gst_adaptive_demux_retry_delay (priv->demux, priv->download_error_count,
+        100 * GST_MSECOND);
+    GST_DEBUG_OBJECT (pl,
+        "Scheduling delayed next playlist download in %" GST_TIMEP_FORMAT,
+        &delay);
+    schedule_next_playlist_load (pl, priv, delay);
+  }
 }
 
 static void