Set limit on a number of requests in import_post

This commit is contained in:
silverpill 2022-12-29 17:23:15 +00:00
parent 7ccd29abf2
commit 471442a22a
2 changed files with 7 additions and 1 deletions

View file

@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed ### Changed
- Updated installation instructions, default mitra config and recommended nginx config. - Updated installation instructions, default mitra config and recommended nginx config.
- Limited the number of requests made during the processing of a thread.
### Fixed ### Fixed

View file

@ -149,6 +149,7 @@ pub async fn import_post(
}; };
let mut queue = vec![object_id]; // LIFO queue let mut queue = vec![object_id]; // LIFO queue
let mut fetch_count = 0;
let mut maybe_object = object_received; let mut maybe_object = object_received;
let mut objects: Vec<Object> = vec![]; let mut objects: Vec<Object> = vec![];
let mut redirects: HashMap<String, String> = HashMap::new(); let mut redirects: HashMap<String, String> = HashMap::new();
@ -157,7 +158,6 @@ pub async fn import_post(
// Fetch ancestors by going through inReplyTo references // Fetch ancestors by going through inReplyTo references
// TODO: fetch replies too // TODO: fetch replies too
#[allow(clippy::while_let_loop)] #[allow(clippy::while_let_loop)]
#[allow(clippy::manual_map)]
loop { loop {
let object_id = match queue.pop() { let object_id = match queue.pop() {
Some(object_id) => { Some(object_id) => {
@ -196,12 +196,17 @@ pub async fn import_post(
let object = match maybe_object { let object = match maybe_object {
Some(object) => object, Some(object) => object,
None => { None => {
if fetch_count >= 50 {
// TODO: create tombstone
return Err(ValidationError("too many objects").into());
};
let object = fetch_object(&instance, &object_id).await let object = fetch_object(&instance, &object_id).await
.map_err(|err| { .map_err(|err| {
log::warn!("{}", err); log::warn!("{}", err);
ValidationError("failed to fetch object") ValidationError("failed to fetch object")
})?; })?;
log::info!("fetched object {}", object.id); log::info!("fetched object {}", object.id);
fetch_count += 1;
object object
}, },
}; };