Fix retry infinite loops. Fixes #1964 (#1967)

* Fix retry infinite loops. Fixes #1964

* Moving retry_limit to settings
This commit is contained in:
Dessalines 2021-12-05 10:03:13 -05:00 committed by GitHub
parent 8bf0f31b0f
commit 040770d7ba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 16 additions and 10 deletions

View file

@ -111,4 +111,6 @@
slur_filter: "(\bThis\b)|(\bis\b)|(\bsample\b)" slur_filter: "(\bThis\b)|(\bis\b)|(\bsample\b)"
# Maximum length of local community and user names # Maximum length of local community and user names
actor_name_max_length: 20 actor_name_max_length: 20
# Maximum number of HTTP requests allowed to handle a single incoming activity (or a single object fetch through the search).
http_fetch_retry_limit: 25
} }

View file

@ -81,6 +81,11 @@ where
); );
debug!("Fetching webfinger url: {}", &fetch_url); debug!("Fetching webfinger url: {}", &fetch_url);
*request_counter += 1;
if *request_counter > context.settings().http_fetch_retry_limit {
return Err(LemmyError::from(anyhow!("Request retry limit reached")));
}
let response = retry(|| context.client().get(&fetch_url).send()).await?; let response = retry(|| context.client().get(&fetch_url).send()).await?;
let res: WebfingerResponse = response let res: WebfingerResponse = response

View file

@ -18,10 +18,6 @@ use std::{
use tracing::info; use tracing::info;
use url::Url; use url::Url;
/// Maximum number of HTTP requests allowed to handle a single incoming activity (or a single object
/// fetch through the search). This should be configurable.
static REQUEST_LIMIT: i32 = 25;
static CLIENT: Lazy<Client> = Lazy::new(|| { static CLIENT: Lazy<Client> = Lazy::new(|| {
Client::builder() Client::builder()
.user_agent(build_user_agent(&Settings::get())) .user_agent(build_user_agent(&Settings::get()))
@ -119,8 +115,8 @@ where
info!("Fetching remote object {}", self.to_string()); info!("Fetching remote object {}", self.to_string());
*request_counter += 1; *request_counter += 1;
if *request_counter > REQUEST_LIMIT { if *request_counter > Settings::get().http_fetch_retry_limit {
return Err(LemmyError::from(anyhow!("Request limit reached"))); return Err(LemmyError::from(anyhow!("Request retry limit reached")));
} }
let res = retry(|| { let res = retry(|| {

View file

@ -58,7 +58,7 @@ pub struct SiteMetadata {
/// Fetches the post link html tags (like title, description, image, etc) /// Fetches the post link html tags (like title, description, image, etc)
pub async fn fetch_site_metadata(client: &Client, url: &Url) -> Result<SiteMetadata, LemmyError> { pub async fn fetch_site_metadata(client: &Client, url: &Url) -> Result<SiteMetadata, LemmyError> {
let response = retry(|| client.get(url.as_str()).send()).await?; let response = client.get(url.as_str()).send().await?;
let html = response let html = response
.text() .text()
@ -132,7 +132,7 @@ pub(crate) async fn fetch_pictrs(
utf8_percent_encode(image_url.as_str(), NON_ALPHANUMERIC) // TODO this might not be needed utf8_percent_encode(image_url.as_str(), NON_ALPHANUMERIC) // TODO this might not be needed
); );
let response = retry(|| client.get(&fetch_url).send()).await?; let response = client.get(&fetch_url).send().await?;
let response: PictrsResponse = response let response: PictrsResponse = response
.json() .json()
@ -201,8 +201,8 @@ pub async fn fetch_site_data(
} }
} }
async fn is_image_content_type(client: &Client, test: &Url) -> Result<(), LemmyError> { async fn is_image_content_type(client: &Client, url: &Url) -> Result<(), LemmyError> {
let response = retry(|| client.get(test.to_owned()).send()).await?; let response = client.get(url.as_str()).send().await?;
if response if response
.headers() .headers()
.get("Content-Type") .get("Content-Type")

View file

@ -46,6 +46,9 @@ pub struct Settings {
/// Maximum length of local community and user names /// Maximum length of local community and user names
#[default(20)] #[default(20)]
pub actor_name_max_length: usize, pub actor_name_max_length: usize,
/// Maximum number of HTTP requests allowed to handle a single incoming activity (or a single object fetch through the search).
#[default(25)]
pub http_fetch_retry_limit: i32,
} }
#[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)] #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)]