Set limit on a number of requests in import_post
This commit is contained in:
parent
7ccd29abf2
commit
471442a22a
2 changed files with 7 additions and 1 deletions
|
@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- Updated installation instructions, default mitra config and recommended nginx config.
|
- Updated installation instructions, default mitra config and recommended nginx config.
|
||||||
|
- Limited the number of requests made during the processing of a thread.
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
|
|
@ -149,6 +149,7 @@ pub async fn import_post(
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut queue = vec![object_id]; // LIFO queue
|
let mut queue = vec![object_id]; // LIFO queue
|
||||||
|
let mut fetch_count = 0;
|
||||||
let mut maybe_object = object_received;
|
let mut maybe_object = object_received;
|
||||||
let mut objects: Vec<Object> = vec![];
|
let mut objects: Vec<Object> = vec![];
|
||||||
let mut redirects: HashMap<String, String> = HashMap::new();
|
let mut redirects: HashMap<String, String> = HashMap::new();
|
||||||
|
@ -157,7 +158,6 @@ pub async fn import_post(
|
||||||
// Fetch ancestors by going through inReplyTo references
|
// Fetch ancestors by going through inReplyTo references
|
||||||
// TODO: fetch replies too
|
// TODO: fetch replies too
|
||||||
#[allow(clippy::while_let_loop)]
|
#[allow(clippy::while_let_loop)]
|
||||||
#[allow(clippy::manual_map)]
|
|
||||||
loop {
|
loop {
|
||||||
let object_id = match queue.pop() {
|
let object_id = match queue.pop() {
|
||||||
Some(object_id) => {
|
Some(object_id) => {
|
||||||
|
@ -196,12 +196,17 @@ pub async fn import_post(
|
||||||
let object = match maybe_object {
|
let object = match maybe_object {
|
||||||
Some(object) => object,
|
Some(object) => object,
|
||||||
None => {
|
None => {
|
||||||
|
if fetch_count >= 50 {
|
||||||
|
// TODO: create tombstone
|
||||||
|
return Err(ValidationError("too many objects").into());
|
||||||
|
};
|
||||||
let object = fetch_object(&instance, &object_id).await
|
let object = fetch_object(&instance, &object_id).await
|
||||||
.map_err(|err| {
|
.map_err(|err| {
|
||||||
log::warn!("{}", err);
|
log::warn!("{}", err);
|
||||||
ValidationError("failed to fetch object")
|
ValidationError("failed to fetch object")
|
||||||
})?;
|
})?;
|
||||||
log::info!("fetched object {}", object.id);
|
log::info!("fetched object {}", object.id);
|
||||||
|
fetch_count += 1;
|
||||||
object
|
object
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
Loading…
Reference in a new issue