mirror of
https://github.com/LemmyNet/activitypub-federation-rust.git
synced 2024-09-22 11:30:00 +00:00
Compare commits
52 commits
0.5.0-beta
...
main
Author | SHA1 | Date | |
---|---|---|---|
|
6dfd30a8ab | ||
|
df8876c096 | ||
|
a35c8cbea5 | ||
|
1126603b61 | ||
|
027b386514 | ||
|
2079b82de7 | ||
|
487c988377 | ||
|
83a156394e | ||
|
d45ce32e88 | ||
|
a0e0c54b57 | ||
|
4920d1a2de | ||
|
472f6ffac5 | ||
|
8f47daa2e2 | ||
|
08af457453 | ||
|
930c928878 | ||
|
6edbc06a78 | ||
|
175b22006b | ||
|
e118e4f240 | ||
|
a251140952 | ||
|
32da1b747c | ||
|
16844f048a | ||
|
cf1f84993b | ||
|
24afad7abc | ||
|
c48de9e944 | ||
|
be69efdee3 | ||
|
ddc455510b | ||
|
ee268405f7 | ||
|
54e8a1145f | ||
|
779313ac22 | ||
|
7def01a19a | ||
|
a2ac97db98 | ||
|
5402bc9c19 | ||
|
1b46dd6f80 | ||
|
da28c9c890 | ||
|
9b31a7b44b | ||
|
147f144769 | ||
|
3b5e5f66ba | ||
|
636b47c8b2 | ||
|
a859db05bb | ||
|
f907b6efa7 | ||
|
ec97b44de4 | ||
|
3efa99514c | ||
|
9c3c756890 | ||
|
9e8d466b40 | ||
|
709f29b7f8 | ||
|
fec0af2406 | ||
|
71ece55641 | ||
|
50db596ce0 | ||
|
12aad8bf3c | ||
|
24830070f6 | ||
|
1f7de85a53 | ||
|
69b80aa6e1 |
34 changed files with 1615 additions and 442 deletions
1
.github/CODEOWNERS
vendored
Normal file
1
.github/CODEOWNERS
vendored
Normal file
|
@ -0,0 +1 @@
|
||||||
|
* @Nutomic @dessalines
|
10
.gitignore
vendored
10
.gitignore
vendored
|
@ -2,4 +2,12 @@
|
||||||
/.idea
|
/.idea
|
||||||
/Cargo.lock
|
/Cargo.lock
|
||||||
perf.data*
|
perf.data*
|
||||||
flamegraph.svg
|
flamegraph.svg
|
||||||
|
|
||||||
|
# direnv
|
||||||
|
/.direnv
|
||||||
|
/.envrc
|
||||||
|
|
||||||
|
# nix flake
|
||||||
|
/flake.nix
|
||||||
|
/flake.lock
|
||||||
|
|
|
@ -1,54 +1,56 @@
|
||||||
pipeline:
|
variables:
|
||||||
|
- &rust_image "rust:1.78-bullseye"
|
||||||
|
|
||||||
|
steps:
|
||||||
cargo_fmt:
|
cargo_fmt:
|
||||||
image: rustdocker/rust:nightly
|
image: rustdocker/rust:nightly
|
||||||
commands:
|
commands:
|
||||||
- /root/.cargo/bin/cargo fmt -- --check
|
- /root/.cargo/bin/cargo fmt -- --check
|
||||||
|
when:
|
||||||
cargo_check:
|
- event: pull_request
|
||||||
image: rust:1.70-bullseye
|
|
||||||
environment:
|
|
||||||
CARGO_HOME: .cargo
|
|
||||||
commands:
|
|
||||||
- cargo check --all-features --all-targets
|
|
||||||
|
|
||||||
cargo_clippy:
|
cargo_clippy:
|
||||||
image: rust:1.70-bullseye
|
image: *rust_image
|
||||||
environment:
|
environment:
|
||||||
CARGO_HOME: .cargo
|
CARGO_HOME: .cargo
|
||||||
commands:
|
commands:
|
||||||
- rustup component add clippy
|
- rustup component add clippy
|
||||||
- cargo clippy --all-targets --all-features --
|
- cargo clippy --all-targets --all-features
|
||||||
-D warnings -D deprecated -D clippy::perf -D clippy::complexity
|
when:
|
||||||
-D clippy::dbg_macro -D clippy::inefficient_to_string
|
- event: pull_request
|
||||||
-D clippy::items-after-statements -D clippy::implicit_clone
|
|
||||||
-D clippy::wildcard_imports -D clippy::cast_lossless
|
|
||||||
-D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls
|
|
||||||
- cargo clippy --all-features -- -D clippy::unwrap_used
|
|
||||||
|
|
||||||
cargo_test:
|
cargo_test:
|
||||||
image: rust:1.70-bullseye
|
image: *rust_image
|
||||||
environment:
|
environment:
|
||||||
CARGO_HOME: .cargo
|
CARGO_HOME: .cargo
|
||||||
commands:
|
commands:
|
||||||
- cargo test --all-features --no-fail-fast
|
- cargo test --all-features --no-fail-fast
|
||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
cargo_doc:
|
cargo_doc:
|
||||||
image: rust:1.70-bullseye
|
image: *rust_image
|
||||||
environment:
|
environment:
|
||||||
CARGO_HOME: .cargo
|
CARGO_HOME: .cargo
|
||||||
commands:
|
commands:
|
||||||
- cargo doc --all-features
|
- cargo doc --all-features
|
||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
cargo_run_actix_example:
|
cargo_run_actix_example:
|
||||||
image: rust:1.70-bullseye
|
image: *rust_image
|
||||||
environment:
|
environment:
|
||||||
CARGO_HOME: .cargo
|
CARGO_HOME: .cargo
|
||||||
commands:
|
commands:
|
||||||
- cargo run --example local_federation actix-web
|
- cargo run --example local_federation actix-web
|
||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
||||||
cargo_run_axum_example:
|
cargo_run_axum_example:
|
||||||
image: rust:1.70-bullseye
|
image: *rust_image
|
||||||
environment:
|
environment:
|
||||||
CARGO_HOME: .cargo
|
CARGO_HOME: .cargo
|
||||||
commands:
|
commands:
|
||||||
- cargo run --example local_federation axum
|
- cargo run --example local_federation axum
|
||||||
|
when:
|
||||||
|
- event: pull_request
|
||||||
|
|
115
Cargo.toml
115
Cargo.toml
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "activitypub_federation"
|
name = "activitypub_federation"
|
||||||
version = "0.5.0-beta.5"
|
version = "0.6.0-alpha2"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
description = "High-level Activitypub framework"
|
description = "High-level Activitypub framework"
|
||||||
keywords = ["activitypub", "activitystreams", "federation", "fediverse"]
|
keywords = ["activitypub", "activitystreams", "federation", "fediverse"]
|
||||||
|
@ -8,73 +8,94 @@ license = "AGPL-3.0"
|
||||||
repository = "https://github.com/LemmyNet/activitypub-federation-rust"
|
repository = "https://github.com/LemmyNet/activitypub-federation-rust"
|
||||||
documentation = "https://docs.rs/activitypub_federation/"
|
documentation = "https://docs.rs/activitypub_federation/"
|
||||||
|
|
||||||
|
[features]
|
||||||
|
default = ["actix-web", "axum"]
|
||||||
|
actix-web = ["dep:actix-web", "dep:http02"]
|
||||||
|
axum = ["dep:axum", "dep:tower"]
|
||||||
|
diesel = ["dep:diesel"]
|
||||||
|
|
||||||
|
[lints.rust]
|
||||||
|
warnings = "deny"
|
||||||
|
deprecated = "deny"
|
||||||
|
|
||||||
|
[lints.clippy]
|
||||||
|
perf = { level = "deny", priority = -1 }
|
||||||
|
complexity = { level = "deny", priority = -1 }
|
||||||
|
dbg_macro = "deny"
|
||||||
|
inefficient_to_string = "deny"
|
||||||
|
items-after-statements = "deny"
|
||||||
|
implicit_clone = "deny"
|
||||||
|
wildcard_imports = "deny"
|
||||||
|
cast_lossless = "deny"
|
||||||
|
manual_string_new = "deny"
|
||||||
|
redundant_closure_for_method_calls = "deny"
|
||||||
|
unwrap_used = "deny"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
chrono = { version = "0.4.31", features = ["clock"], default-features = false }
|
chrono = { version = "0.4.38", features = ["clock"], default-features = false }
|
||||||
serde = { version = "1.0.189", features = ["derive"] }
|
serde = { version = "1.0.204", features = ["derive"] }
|
||||||
async-trait = "0.1.74"
|
async-trait = "0.1.81"
|
||||||
url = { version = "2.4.1", features = ["serde"] }
|
url = { version = "2.5.2", features = ["serde"] }
|
||||||
serde_json = { version = "1.0.107", features = ["preserve_order"] }
|
serde_json = { version = "1.0.120", features = ["preserve_order"] }
|
||||||
reqwest = { version = "0.11.22", features = ["json", "stream"] }
|
reqwest = { version = "0.12.5", default-features = false, features = [
|
||||||
reqwest-middleware = "0.2.3"
|
"json",
|
||||||
|
"stream",
|
||||||
|
"rustls-tls",
|
||||||
|
] }
|
||||||
|
reqwest-middleware = "0.3.2"
|
||||||
tracing = "0.1.40"
|
tracing = "0.1.40"
|
||||||
base64 = "0.21.5"
|
base64 = "0.22.1"
|
||||||
openssl = "0.10.57"
|
rand = "0.8.5"
|
||||||
once_cell = "1.18.0"
|
rsa = "0.9.6"
|
||||||
http = "0.2.9"
|
once_cell = "1.19.0"
|
||||||
sha2 = "0.10.8"
|
http = "1.1.0"
|
||||||
thiserror = "1.0.50"
|
sha2 = { version = "0.10.8", features = ["oid"] }
|
||||||
derive_builder = "0.12.0"
|
thiserror = "1.0.62"
|
||||||
itertools = "0.10.5"
|
derive_builder = "0.20.0"
|
||||||
dyn-clone = "1.0.14"
|
itertools = "0.13.0"
|
||||||
|
dyn-clone = "1.0.17"
|
||||||
enum_delegate = "0.2.0"
|
enum_delegate = "0.2.0"
|
||||||
httpdate = "1.0.3"
|
httpdate = "1.0.3"
|
||||||
http-signature-normalization-reqwest = { version = "0.8.0", default-features = false, features = [
|
http-signature-normalization-reqwest = { version = "0.12.0", default-features = false, features = [
|
||||||
"sha-2",
|
"sha-2",
|
||||||
"middleware",
|
"middleware",
|
||||||
|
"default-spawner",
|
||||||
] }
|
] }
|
||||||
http-signature-normalization = "0.7.0"
|
http-signature-normalization = "0.7.0"
|
||||||
bytes = "1.5.0"
|
bytes = "1.6.1"
|
||||||
futures-core = { version = "0.3.28", default-features = false }
|
futures-core = { version = "0.3.30", default-features = false }
|
||||||
pin-project-lite = "0.2.13"
|
pin-project-lite = "0.2.14"
|
||||||
activitystreams-kinds = "0.3.0"
|
activitystreams-kinds = "0.3.0"
|
||||||
regex = { version = "1.10.2", default-features = false, features = ["std", "unicode-case"] }
|
regex = { version = "1.10.5", default-features = false, features = [
|
||||||
tokio = { version = "1.33.0", features = [
|
"std",
|
||||||
|
"unicode",
|
||||||
|
] }
|
||||||
|
tokio = { version = "1.38.0", features = [
|
||||||
"sync",
|
"sync",
|
||||||
"rt",
|
"rt",
|
||||||
"rt-multi-thread",
|
"rt-multi-thread",
|
||||||
"time",
|
"time",
|
||||||
] }
|
] }
|
||||||
|
diesel = { version = "2.2.1", features = [
|
||||||
|
"postgres",
|
||||||
|
], default-features = false, optional = true }
|
||||||
|
futures = "0.3.30"
|
||||||
|
moka = { version = "0.12.8", features = ["future"] }
|
||||||
|
|
||||||
# Actix-web
|
# Actix-web
|
||||||
actix-web = { version = "4.4.0", default-features = false, optional = true }
|
actix-web = { version = "4.8.0", default-features = false, optional = true }
|
||||||
|
http02 = { package = "http", version = "0.2.12", optional = true }
|
||||||
|
|
||||||
# Axum
|
# Axum
|
||||||
axum = { version = "0.6.20", features = [
|
axum = { version = "0.7.5", features = ["json"], default-features = false, optional = true }
|
||||||
"json",
|
|
||||||
"headers",
|
|
||||||
], default-features = false, optional = true }
|
|
||||||
tower = { version = "0.4.13", optional = true }
|
tower = { version = "0.4.13", optional = true }
|
||||||
hyper = { version = "0.14", optional = true }
|
|
||||||
futures = "0.3.28"
|
|
||||||
moka = { version = "0.11.3", features = ["future"] }
|
|
||||||
|
|
||||||
[features]
|
|
||||||
default = ["actix-web", "axum"]
|
|
||||||
actix-web = ["dep:actix-web"]
|
|
||||||
axum = ["dep:axum", "dep:tower", "dep:hyper"]
|
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
anyhow = "1.0.75"
|
anyhow = "1.0.86"
|
||||||
rand = "0.8.5"
|
axum = { version = "0.7.5", features = ["macros"] }
|
||||||
env_logger = "0.10.0"
|
axum-extra = { version = "0.9.3", features = ["typed-header"] }
|
||||||
tower-http = { version = "0.4.4", features = ["map-request-body", "util"] }
|
env_logger = "0.11.3"
|
||||||
axum = { version = "0.6.20", features = [
|
tokio = { version = "1.38.0", features = ["full"] }
|
||||||
"http1",
|
|
||||||
"tokio",
|
|
||||||
"query",
|
|
||||||
], default-features = false }
|
|
||||||
axum-macros = "0.3.8"
|
|
||||||
tokio = { version = "1.33.0", features = ["full"] }
|
|
||||||
|
|
||||||
[profile.dev]
|
[profile.dev]
|
||||||
strip = "symbols"
|
strip = "symbols"
|
||||||
|
|
|
@ -15,9 +15,9 @@ The next step is to allow other servers to fetch our actors and objects. For thi
|
||||||
# use activitypub_federation::config::FederationMiddleware;
|
# use activitypub_federation::config::FederationMiddleware;
|
||||||
# use axum::routing::get;
|
# use axum::routing::get;
|
||||||
# use crate::activitypub_federation::traits::Object;
|
# use crate::activitypub_federation::traits::Object;
|
||||||
# use axum::headers::ContentType;
|
# use axum_extra::headers::ContentType;
|
||||||
# use activitypub_federation::FEDERATION_CONTENT_TYPE;
|
# use activitypub_federation::FEDERATION_CONTENT_TYPE;
|
||||||
# use axum::TypedHeader;
|
# use axum_extra::TypedHeader;
|
||||||
# use axum::response::IntoResponse;
|
# use axum::response::IntoResponse;
|
||||||
# use http::HeaderMap;
|
# use http::HeaderMap;
|
||||||
# async fn generate_user_html(_: String, _: Data<DbConnection>) -> axum::response::Response { todo!() }
|
# async fn generate_user_html(_: String, _: Data<DbConnection>) -> axum::response::Response { todo!() }
|
||||||
|
@ -34,10 +34,9 @@ async fn main() -> Result<(), Error> {
|
||||||
.layer(FederationMiddleware::new(data));
|
.layer(FederationMiddleware::new(data));
|
||||||
|
|
||||||
let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
|
let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
|
||||||
|
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||||
tracing::debug!("listening on {}", addr);
|
tracing::debug!("listening on {}", addr);
|
||||||
axum::Server::bind(&addr)
|
axum::serve(listener, app.into_make_service()).await?;
|
||||||
.serve(app.into_make_service())
|
|
||||||
.await?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -48,7 +47,7 @@ async fn http_get_user(
|
||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
let accept = header_map.get("accept").map(|v| v.to_str().unwrap());
|
let accept = header_map.get("accept").map(|v| v.to_str().unwrap());
|
||||||
if accept == Some(FEDERATION_CONTENT_TYPE) {
|
if accept == Some(FEDERATION_CONTENT_TYPE) {
|
||||||
let db_user = data.read_local_user(name).await.unwrap();
|
let db_user = data.read_local_user(&name).await.unwrap();
|
||||||
let json_user = db_user.into_json(&data).await.unwrap();
|
let json_user = db_user.into_json(&data).await.unwrap();
|
||||||
FederationJson(WithContext::new_default(json_user)).into_response()
|
FederationJson(WithContext::new_default(json_user)).into_response()
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,46 @@ To send an activity we need to initialize our previously defined struct, and pic
|
||||||
|
|
||||||
```
|
```
|
||||||
# use activitypub_federation::config::FederationConfig;
|
# use activitypub_federation::config::FederationConfig;
|
||||||
|
# use activitypub_federation::activity_queue::queue_activity;
|
||||||
|
# use activitypub_federation::http_signatures::generate_actor_keypair;
|
||||||
|
# use activitypub_federation::traits::Actor;
|
||||||
|
# use activitypub_federation::fetch::object_id::ObjectId;
|
||||||
|
# use activitypub_federation::traits::tests::{DB_USER, DbConnection, Follow};
|
||||||
|
# tokio::runtime::Runtime::new().unwrap().block_on(async {
|
||||||
|
# let db_connection = DbConnection;
|
||||||
|
# let config = FederationConfig::builder()
|
||||||
|
# .domain("example.com")
|
||||||
|
# .app_data(db_connection)
|
||||||
|
# .build().await?;
|
||||||
|
# let data = config.to_request_data();
|
||||||
|
# let sender = DB_USER.clone();
|
||||||
|
# let recipient = DB_USER.clone();
|
||||||
|
let activity = Follow {
|
||||||
|
actor: ObjectId::parse("https://lemmy.ml/u/nutomic")?,
|
||||||
|
object: recipient.federation_id.clone().into(),
|
||||||
|
kind: Default::default(),
|
||||||
|
id: "https://lemmy.ml/activities/321".try_into()?
|
||||||
|
};
|
||||||
|
let inboxes = vec![recipient.shared_inbox_or_inbox()];
|
||||||
|
|
||||||
|
queue_activity(&activity, &sender, inboxes, &data).await?;
|
||||||
|
# Ok::<(), anyhow::Error>(())
|
||||||
|
# }).unwrap()
|
||||||
|
```
|
||||||
|
|
||||||
|
The list of inboxes gets deduplicated (important for shared inbox). All inboxes on the local domain and those which fail the [crate::config::UrlVerifier] check are excluded from delivery. For each remaining inbox a background tasks is created. It signs the HTTP header with the given private key. Finally the activity is delivered to the inbox.
|
||||||
|
|
||||||
|
It is possible that delivery fails because the target instance is temporarily unreachable. In this case the task is scheduled for retry after a certain waiting time. For each task delivery is retried up to 3 times after the initial attempt. The retry intervals are as follows:
|
||||||
|
|
||||||
|
- one minute, in case of service restart
|
||||||
|
- one hour, in case of instance maintenance
|
||||||
|
- 2.5 days, in case of major incident with rebuild from backup
|
||||||
|
|
||||||
|
In case [crate::config::FederationConfigBuilder::debug] is enabled, no background thread is used but activities are sent directly on the foreground. This makes it easier to catch delivery errors and avoids complicated steps to await delivery in tests.
|
||||||
|
|
||||||
|
In some cases you may want to bypass the builtin activity queue, and implement your own. For example to specify different retry intervals, or to persist retries across application restarts. You can do it with the following code:
|
||||||
|
```rust
|
||||||
|
# use activitypub_federation::config::FederationConfig;
|
||||||
# use activitypub_federation::activity_sending::SendActivityTask;
|
# use activitypub_federation::activity_sending::SendActivityTask;
|
||||||
# use activitypub_federation::http_signatures::generate_actor_keypair;
|
# use activitypub_federation::http_signatures::generate_actor_keypair;
|
||||||
# use activitypub_federation::traits::Actor;
|
# use activitypub_federation::traits::Actor;
|
||||||
|
@ -28,23 +68,8 @@ let inboxes = vec![recipient.shared_inbox_or_inbox()];
|
||||||
|
|
||||||
let sends = SendActivityTask::prepare(&activity, &sender, inboxes, &data).await?;
|
let sends = SendActivityTask::prepare(&activity, &sender, inboxes, &data).await?;
|
||||||
for send in sends {
|
for send in sends {
|
||||||
send.sign_and_send(&data).await?;
|
send.sign_and_send(&data).await?;
|
||||||
}
|
}
|
||||||
# Ok::<(), anyhow::Error>(())
|
# Ok::<(), anyhow::Error>(())
|
||||||
# }).unwrap()
|
# }).unwrap()
|
||||||
```
|
```
|
||||||
|
|
||||||
The list of inboxes gets deduplicated (important for shared inbox). All inboxes on the local
|
|
||||||
domain and those which fail the [crate::config::UrlVerifier] check are excluded from delivery.
|
|
||||||
For each remaining inbox a background tasks is created. It signs the HTTP header with the given
|
|
||||||
private key. Finally the activity is delivered to the inbox.
|
|
||||||
|
|
||||||
It is possible that delivery fails because the target instance is temporarily unreachable. In
|
|
||||||
this case the task is scheduled for retry after a certain waiting time. For each task delivery
|
|
||||||
is retried up to 3 times after the initial attempt. The retry intervals are as follows:
|
|
||||||
|
|
||||||
- one minute, in case of service restart
|
|
||||||
- one hour, in case of instance maintenance
|
|
||||||
- 2.5 days, in case of major incident with rebuild from backup
|
|
||||||
|
|
||||||
In case [crate::config::FederationConfigBuilder::debug] is enabled, no background thread is used but activities are sent directly on the foreground. This makes it easier to catch delivery errors and avoids complicated steps to await delivery in tests.
|
|
|
@ -14,11 +14,11 @@ use activitypub_federation::{
|
||||||
traits::Object,
|
traits::Object,
|
||||||
};
|
};
|
||||||
use axum::{
|
use axum::{
|
||||||
|
debug_handler,
|
||||||
extract::{Path, Query},
|
extract::{Path, Query},
|
||||||
response::{IntoResponse, Response},
|
response::{IntoResponse, Response},
|
||||||
Json,
|
Json,
|
||||||
};
|
};
|
||||||
use axum_macros::debug_handler;
|
|
||||||
use http::StatusCode;
|
use http::StatusCode;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
@ -61,7 +61,7 @@ pub async fn webfinger(
|
||||||
data: Data<DatabaseHandle>,
|
data: Data<DatabaseHandle>,
|
||||||
) -> Result<Json<Webfinger>, Error> {
|
) -> Result<Json<Webfinger>, Error> {
|
||||||
let name = extract_webfinger_name(&query.resource, &data)?;
|
let name = extract_webfinger_name(&query.resource, &data)?;
|
||||||
let db_user = data.read_user(&name)?;
|
let db_user = data.read_user(name)?;
|
||||||
Ok(Json(build_webfinger_response(
|
Ok(Json(build_webfinger_response(
|
||||||
query.resource,
|
query.resource,
|
||||||
db_user.ap_id.into_inner(),
|
db_user.ap_id.into_inner(),
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
#![allow(clippy::unwrap_used)]
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
database::Database,
|
database::Database,
|
||||||
http::{http_get_user, http_post_user_inbox, webfinger},
|
http::{http_get_user, http_post_user_inbox, webfinger},
|
||||||
|
@ -62,9 +64,8 @@ async fn main() -> Result<(), Error> {
|
||||||
.to_socket_addrs()?
|
.to_socket_addrs()?
|
||||||
.next()
|
.next()
|
||||||
.expect("Failed to lookup domain name");
|
.expect("Failed to lookup domain name");
|
||||||
axum::Server::bind(&addr)
|
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||||
.serve(app.into_make_service())
|
axum::serve(listener, app.into_make_service()).await?;
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,7 +21,6 @@ pub struct DbPost {
|
||||||
pub text: String,
|
pub text: String,
|
||||||
pub ap_id: ObjectId<DbPost>,
|
pub ap_id: ObjectId<DbPost>,
|
||||||
pub creator: ObjectId<DbUser>,
|
pub creator: ObjectId<DbUser>,
|
||||||
pub local: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Serialize, Debug)]
|
#[derive(Deserialize, Serialize, Debug)]
|
||||||
|
@ -59,7 +58,15 @@ impl Object for DbPost {
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
|
async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
|
||||||
unimplemented!()
|
Ok(Note {
|
||||||
|
kind: NoteType::Note,
|
||||||
|
id: self.ap_id,
|
||||||
|
content: self.text,
|
||||||
|
attributed_to: self.creator,
|
||||||
|
to: vec![public()],
|
||||||
|
tag: vec![],
|
||||||
|
in_reply_to: None,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn verify(
|
async fn verify(
|
||||||
|
@ -81,7 +88,6 @@ impl Object for DbPost {
|
||||||
text: json.content,
|
text: json.content,
|
||||||
ap_id: json.id.clone(),
|
ap_id: json.id.clone(),
|
||||||
creator: json.attributed_to.clone(),
|
creator: json.attributed_to.clone(),
|
||||||
local: false,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let mention = Mention {
|
let mention = Mention {
|
||||||
|
|
|
@ -67,7 +67,7 @@ impl ActivityHandler for Follow {
|
||||||
let id = generate_object_id(data.domain())?;
|
let id = generate_object_id(data.domain())?;
|
||||||
let accept = Accept::new(local_user.ap_id.clone(), self, id.clone());
|
let accept = Accept::new(local_user.ap_id.clone(), self, id.clone());
|
||||||
local_user
|
local_user
|
||||||
.send(accept, vec![follower.shared_inbox_or_inbox()], data)
|
.send(accept, vec![follower.shared_inbox_or_inbox()], false, data)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
@ -89,7 +89,7 @@ pub async fn webfinger(
|
||||||
data: Data<DatabaseHandle>,
|
data: Data<DatabaseHandle>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error> {
|
||||||
let name = extract_webfinger_name(&query.resource, &data)?;
|
let name = extract_webfinger_name(&query.resource, &data)?;
|
||||||
let db_user = data.read_user(&name)?;
|
let db_user = data.read_user(name)?;
|
||||||
Ok(HttpResponse::Ok().json(build_webfinger_response(
|
Ok(HttpResponse::Ok().json(build_webfinger_response(
|
||||||
query.resource.clone(),
|
query.resource.clone(),
|
||||||
db_user.ap_id.into_inner(),
|
db_user.ap_id.into_inner(),
|
||||||
|
|
|
@ -14,13 +14,13 @@ use activitypub_federation::{
|
||||||
traits::Object,
|
traits::Object,
|
||||||
};
|
};
|
||||||
use axum::{
|
use axum::{
|
||||||
|
debug_handler,
|
||||||
extract::{Path, Query},
|
extract::{Path, Query},
|
||||||
response::IntoResponse,
|
response::IntoResponse,
|
||||||
routing::{get, post},
|
routing::{get, post},
|
||||||
Json,
|
Json,
|
||||||
Router,
|
Router,
|
||||||
};
|
};
|
||||||
use axum_macros::debug_handler;
|
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use std::net::ToSocketAddrs;
|
use std::net::ToSocketAddrs;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
@ -39,9 +39,14 @@ pub fn listen(config: &FederationConfig<DatabaseHandle>) -> Result<(), Error> {
|
||||||
.to_socket_addrs()?
|
.to_socket_addrs()?
|
||||||
.next()
|
.next()
|
||||||
.expect("Failed to lookup domain name");
|
.expect("Failed to lookup domain name");
|
||||||
let server = axum::Server::bind(&addr).serve(app.into_make_service());
|
let fut = async move {
|
||||||
|
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
|
||||||
|
axum::serve(listener, app.into_make_service())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
};
|
||||||
|
|
||||||
tokio::spawn(server);
|
tokio::spawn(fut);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -78,7 +83,7 @@ async fn webfinger(
|
||||||
data: Data<DatabaseHandle>,
|
data: Data<DatabaseHandle>,
|
||||||
) -> Result<Json<Webfinger>, Error> {
|
) -> Result<Json<Webfinger>, Error> {
|
||||||
let name = extract_webfinger_name(&query.resource, &data)?;
|
let name = extract_webfinger_name(&query.resource, &data)?;
|
||||||
let db_user = data.read_user(&name)?;
|
let db_user = data.read_user(name)?;
|
||||||
Ok(Json(build_webfinger_response(
|
Ok(Json(build_webfinger_response(
|
||||||
query.resource,
|
query.resource,
|
||||||
db_user.ap_id.into_inner(),
|
db_user.ap_id.into_inner(),
|
||||||
|
|
|
@ -28,6 +28,7 @@ pub async fn new_instance(
|
||||||
.domain(hostname)
|
.domain(hostname)
|
||||||
.signed_fetch_actor(&system_user)
|
.signed_fetch_actor(&system_user)
|
||||||
.app_data(database)
|
.app_data(database)
|
||||||
|
.url_verifier(Box::new(MyUrlVerifier()))
|
||||||
.debug(true)
|
.debug(true)
|
||||||
.build()
|
.build()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
#![allow(clippy::unwrap_used)]
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
instance::{listen, new_instance, Webserver},
|
instance::{listen, new_instance, Webserver},
|
||||||
objects::post::DbPost,
|
objects::post::DbPost,
|
||||||
|
|
|
@ -6,6 +6,7 @@ use crate::{
|
||||||
utils::generate_object_id,
|
utils::generate_object_id,
|
||||||
};
|
};
|
||||||
use activitypub_federation::{
|
use activitypub_federation::{
|
||||||
|
activity_queue::queue_activity,
|
||||||
activity_sending::SendActivityTask,
|
activity_sending::SendActivityTask,
|
||||||
config::Data,
|
config::Data,
|
||||||
fetch::{object_id::ObjectId, webfinger::webfinger_resolve_actor},
|
fetch::{object_id::ObjectId, webfinger::webfinger_resolve_actor},
|
||||||
|
@ -85,7 +86,7 @@ impl DbUser {
|
||||||
let other: DbUser = webfinger_resolve_actor(other, data).await?;
|
let other: DbUser = webfinger_resolve_actor(other, data).await?;
|
||||||
let id = generate_object_id(data.domain())?;
|
let id = generate_object_id(data.domain())?;
|
||||||
let follow = Follow::new(self.ap_id.clone(), other.ap_id.clone(), id.clone());
|
let follow = Follow::new(self.ap_id.clone(), other.ap_id.clone(), id.clone());
|
||||||
self.send(follow, vec![other.shared_inbox_or_inbox()], data)
|
self.send(follow, vec![other.shared_inbox_or_inbox()], false, data)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -98,7 +99,7 @@ impl DbUser {
|
||||||
let user: DbUser = ObjectId::from(f).dereference(data).await?;
|
let user: DbUser = ObjectId::from(f).dereference(data).await?;
|
||||||
inboxes.push(user.shared_inbox_or_inbox());
|
inboxes.push(user.shared_inbox_or_inbox());
|
||||||
}
|
}
|
||||||
self.send(create, inboxes, data).await?;
|
self.send(create, inboxes, true, data).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,6 +107,7 @@ impl DbUser {
|
||||||
&self,
|
&self,
|
||||||
activity: Activity,
|
activity: Activity,
|
||||||
recipients: Vec<Url>,
|
recipients: Vec<Url>,
|
||||||
|
use_queue: bool,
|
||||||
data: &Data<DatabaseHandle>,
|
data: &Data<DatabaseHandle>,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
|
@ -113,9 +115,14 @@ impl DbUser {
|
||||||
<Activity as ActivityHandler>::Error: From<anyhow::Error> + From<serde_json::Error>,
|
<Activity as ActivityHandler>::Error: From<anyhow::Error> + From<serde_json::Error>,
|
||||||
{
|
{
|
||||||
let activity = WithContext::new_default(activity);
|
let activity = WithContext::new_default(activity);
|
||||||
let sends = SendActivityTask::prepare(&activity, self, recipients, data).await?;
|
// Send through queue in some cases and bypass it in others to test both code paths
|
||||||
for send in sends {
|
if use_queue {
|
||||||
send.sign_and_send(data).await?;
|
queue_activity(&activity, self, recipients, data).await?;
|
||||||
|
} else {
|
||||||
|
let sends = SendActivityTask::prepare(&activity, self, recipients, data).await?;
|
||||||
|
for send in sends {
|
||||||
|
send.sign_and_send(data).await?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
521
src/activity_queue.rs
Normal file
521
src/activity_queue.rs
Normal file
|
@ -0,0 +1,521 @@
|
||||||
|
//! Queue for signing and sending outgoing activities with retry
|
||||||
|
//!
|
||||||
|
#![doc = include_str!("../docs/09_sending_activities.md")]
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
activity_sending::{build_tasks, SendActivityTask},
|
||||||
|
config::Data,
|
||||||
|
error::Error,
|
||||||
|
traits::{ActivityHandler, Actor},
|
||||||
|
};
|
||||||
|
|
||||||
|
use futures_core::Future;
|
||||||
|
|
||||||
|
use reqwest_middleware::ClientWithMiddleware;
|
||||||
|
use serde::Serialize;
|
||||||
|
use std::{
|
||||||
|
fmt::{Debug, Display},
|
||||||
|
sync::{
|
||||||
|
atomic::{AtomicUsize, Ordering},
|
||||||
|
Arc,
|
||||||
|
},
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
use tokio::{
|
||||||
|
sync::mpsc::{unbounded_channel, UnboundedSender},
|
||||||
|
task::{JoinHandle, JoinSet},
|
||||||
|
};
|
||||||
|
use tracing::{info, warn};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
|
/// Send a new activity to the given inboxes with automatic retry on failure. Alternatively you
|
||||||
|
/// can implement your own queue and then send activities using [[crate::activity_sending::SendActivityTask]].
|
||||||
|
///
|
||||||
|
/// - `activity`: The activity to be sent, gets converted to json
|
||||||
|
/// - `private_key`: Private key belonging to the actor who sends the activity, for signing HTTP
|
||||||
|
/// signature. Generated with [crate::http_signatures::generate_actor_keypair].
|
||||||
|
/// - `inboxes`: List of remote actor inboxes that should receive the activity. Ignores local actor
|
||||||
|
/// inboxes. Should be built by calling [crate::traits::Actor::shared_inbox_or_inbox]
|
||||||
|
/// for each target actor.
|
||||||
|
pub async fn queue_activity<Activity, Datatype, ActorType>(
|
||||||
|
activity: &Activity,
|
||||||
|
actor: &ActorType,
|
||||||
|
inboxes: Vec<Url>,
|
||||||
|
data: &Data<Datatype>,
|
||||||
|
) -> Result<(), Error>
|
||||||
|
where
|
||||||
|
Activity: ActivityHandler + Serialize + Debug,
|
||||||
|
Datatype: Clone,
|
||||||
|
ActorType: Actor,
|
||||||
|
{
|
||||||
|
let config = &data.config;
|
||||||
|
let tasks = build_tasks(activity, actor, inboxes, data).await?;
|
||||||
|
|
||||||
|
for task in tasks {
|
||||||
|
// Don't use the activity queue if this is in debug mode, send and wait directly
|
||||||
|
if config.debug {
|
||||||
|
if let Err(err) = sign_and_send(
|
||||||
|
&task,
|
||||||
|
&config.client,
|
||||||
|
config.request_timeout,
|
||||||
|
Default::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
warn!("{err}");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// This field is only optional to make builder work, its always present at this point
|
||||||
|
let activity_queue = config
|
||||||
|
.activity_queue
|
||||||
|
.as_ref()
|
||||||
|
.expect("Config has activity queue");
|
||||||
|
activity_queue.queue(task).await?;
|
||||||
|
let stats = activity_queue.get_stats();
|
||||||
|
let running = stats.running.load(Ordering::Relaxed);
|
||||||
|
if running == config.queue_worker_count && config.queue_worker_count != 0 {
|
||||||
|
warn!("Reached max number of send activity workers ({}). Consider increasing worker count to avoid federation delays", config.queue_worker_count);
|
||||||
|
warn!("{:?}", stats);
|
||||||
|
} else {
|
||||||
|
info!("{:?}", stats);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sign_and_send(
|
||||||
|
task: &SendActivityTask,
|
||||||
|
client: &ClientWithMiddleware,
|
||||||
|
timeout: Duration,
|
||||||
|
retry_strategy: RetryStrategy,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
retry(
|
||||||
|
|| task.sign_and_send_internal(client, timeout),
|
||||||
|
retry_strategy,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A simple activity queue which spawns tokio workers to send out requests
|
||||||
|
/// When creating a queue, it will spawn a task per worker thread
|
||||||
|
/// Uses an unbounded mpsc queue for communication (i.e, all messages are in memory)
|
||||||
|
pub(crate) struct ActivityQueue {
|
||||||
|
// Stats shared between the queue and workers
|
||||||
|
stats: Arc<Stats>,
|
||||||
|
sender: UnboundedSender<SendActivityTask>,
|
||||||
|
sender_task: JoinHandle<()>,
|
||||||
|
retry_sender_task: JoinHandle<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Simple stat counter to show where we're up to with sending messages
|
||||||
|
/// This is a lock-free way to share things between tasks
|
||||||
|
/// When reading these values it's possible (but extremely unlikely) to get stale data if a worker task is in the middle of transitioning
|
||||||
|
#[derive(Default)]
|
||||||
|
pub(crate) struct Stats {
|
||||||
|
pending: AtomicUsize,
|
||||||
|
running: AtomicUsize,
|
||||||
|
retries: AtomicUsize,
|
||||||
|
dead_last_hour: AtomicUsize,
|
||||||
|
completed_last_hour: AtomicUsize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Debug for Stats {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"Activity queue stats: pending: {}, running: {}, retries: {}, dead: {}, complete: {}",
|
||||||
|
self.pending.load(Ordering::Relaxed),
|
||||||
|
self.running.load(Ordering::Relaxed),
|
||||||
|
self.retries.load(Ordering::Relaxed),
|
||||||
|
self.dead_last_hour.load(Ordering::Relaxed),
|
||||||
|
self.completed_last_hour.load(Ordering::Relaxed)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Default)]
|
||||||
|
struct RetryStrategy {
|
||||||
|
/// Amount of time in seconds to back off
|
||||||
|
backoff: usize,
|
||||||
|
/// Amount of times to retry
|
||||||
|
retries: usize,
|
||||||
|
/// If this particular request has already been retried, you can add an offset here to increment the count to start
|
||||||
|
offset: usize,
|
||||||
|
/// Number of seconds to sleep before trying
|
||||||
|
initial_sleep: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A tokio spawned worker which is responsible for submitting requests to federated servers
|
||||||
|
/// This will retry up to one time with the same signature, and if it fails, will move it to the retry queue.
|
||||||
|
/// We need to retry activity sending in case the target instances is temporarily unreachable.
|
||||||
|
/// In this case, the task is stored and resent when the instance is hopefully back up. This
|
||||||
|
/// list shows the retry intervals, and which events of the target instance can be covered:
|
||||||
|
/// - 60s (one minute, service restart) -- happens in the worker w/ same signature
|
||||||
|
/// - 60min (one hour, instance maintenance) --- happens in the retry worker
|
||||||
|
/// - 60h (2.5 days, major incident with rebuild from backup) --- happens in the retry worker
|
||||||
|
async fn worker(
|
||||||
|
client: ClientWithMiddleware,
|
||||||
|
timeout: Duration,
|
||||||
|
message: SendActivityTask,
|
||||||
|
retry_queue: UnboundedSender<SendActivityTask>,
|
||||||
|
stats: Arc<Stats>,
|
||||||
|
strategy: RetryStrategy,
|
||||||
|
) {
|
||||||
|
stats.pending.fetch_sub(1, Ordering::Relaxed);
|
||||||
|
stats.running.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
||||||
|
let outcome = sign_and_send(&message, &client, timeout, strategy).await;
|
||||||
|
|
||||||
|
// "Running" has finished, check the outcome
|
||||||
|
stats.running.fetch_sub(1, Ordering::Relaxed);
|
||||||
|
|
||||||
|
match outcome {
|
||||||
|
Ok(_) => {
|
||||||
|
stats.completed_last_hour.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
Err(_err) => {
|
||||||
|
stats.retries.fetch_add(1, Ordering::Relaxed);
|
||||||
|
warn!(
|
||||||
|
"Sending activity {} to {} to the retry queue to be tried again later",
|
||||||
|
message.activity_id, message.inbox
|
||||||
|
);
|
||||||
|
// Send to the retry queue. Ignoring whether it succeeds or not
|
||||||
|
retry_queue.send(message).ok();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn retry_worker(
|
||||||
|
client: ClientWithMiddleware,
|
||||||
|
timeout: Duration,
|
||||||
|
message: SendActivityTask,
|
||||||
|
stats: Arc<Stats>,
|
||||||
|
strategy: RetryStrategy,
|
||||||
|
) {
|
||||||
|
// Because the times are pretty extravagant between retries, we have to re-sign each time
|
||||||
|
let outcome = retry(
|
||||||
|
|| {
|
||||||
|
sign_and_send(
|
||||||
|
&message,
|
||||||
|
&client,
|
||||||
|
timeout,
|
||||||
|
RetryStrategy {
|
||||||
|
backoff: 0,
|
||||||
|
retries: 0,
|
||||||
|
offset: 0,
|
||||||
|
initial_sleep: 0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
},
|
||||||
|
strategy,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
stats.retries.fetch_sub(1, Ordering::Relaxed);
|
||||||
|
|
||||||
|
match outcome {
|
||||||
|
Ok(_) => {
|
||||||
|
stats.completed_last_hour.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
Err(_err) => {
|
||||||
|
stats.dead_last_hour.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActivityQueue {
|
||||||
|
fn new(
|
||||||
|
client: ClientWithMiddleware,
|
||||||
|
worker_count: usize,
|
||||||
|
retry_count: usize,
|
||||||
|
timeout: Duration,
|
||||||
|
backoff: usize, // This should be 60 seconds by default or 1 second in tests
|
||||||
|
) -> Self {
|
||||||
|
let stats: Arc<Stats> = Default::default();
|
||||||
|
|
||||||
|
// This task clears the dead/completed stats every hour
|
||||||
|
let hour_stats = stats.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let duration = Duration::from_secs(3600);
|
||||||
|
loop {
|
||||||
|
tokio::time::sleep(duration).await;
|
||||||
|
hour_stats.completed_last_hour.store(0, Ordering::Relaxed);
|
||||||
|
hour_stats.dead_last_hour.store(0, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let (retry_sender, mut retry_receiver) = unbounded_channel();
|
||||||
|
let retry_stats = stats.clone();
|
||||||
|
let retry_client = client.clone();
|
||||||
|
|
||||||
|
// The "fast path" retry
|
||||||
|
// The backoff should be < 5 mins for this to work otherwise signatures may expire
|
||||||
|
// This strategy is the one that is used with the *same* signature
|
||||||
|
let strategy = RetryStrategy {
|
||||||
|
backoff,
|
||||||
|
retries: 1,
|
||||||
|
offset: 0,
|
||||||
|
initial_sleep: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
// The "retry path" strategy
|
||||||
|
// After the fast path fails, a task will sleep up to backoff ^ 2 and then retry again
|
||||||
|
let retry_strategy = RetryStrategy {
|
||||||
|
backoff,
|
||||||
|
retries: 3,
|
||||||
|
offset: 2,
|
||||||
|
initial_sleep: backoff.pow(2), // wait 60 mins before even trying
|
||||||
|
};
|
||||||
|
|
||||||
|
let retry_sender_task = tokio::spawn(async move {
|
||||||
|
let mut join_set = JoinSet::new();
|
||||||
|
|
||||||
|
while let Some(message) = retry_receiver.recv().await {
|
||||||
|
let retry_task = retry_worker(
|
||||||
|
retry_client.clone(),
|
||||||
|
timeout,
|
||||||
|
message,
|
||||||
|
retry_stats.clone(),
|
||||||
|
retry_strategy,
|
||||||
|
);
|
||||||
|
|
||||||
|
if retry_count > 0 {
|
||||||
|
// If we're over the limit of retries, wait for them to finish before spawning
|
||||||
|
while join_set.len() >= retry_count {
|
||||||
|
join_set.join_next().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
join_set.spawn(retry_task);
|
||||||
|
} else {
|
||||||
|
// If the retry worker count is `0` then just spawn and don't use the join_set
|
||||||
|
tokio::spawn(retry_task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
while !join_set.is_empty() {
|
||||||
|
join_set.join_next().await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let (sender, mut receiver) = unbounded_channel();
|
||||||
|
|
||||||
|
let sender_stats = stats.clone();
|
||||||
|
|
||||||
|
let sender_task = tokio::spawn(async move {
|
||||||
|
let mut join_set = JoinSet::new();
|
||||||
|
|
||||||
|
while let Some(message) = receiver.recv().await {
|
||||||
|
let task = worker(
|
||||||
|
client.clone(),
|
||||||
|
timeout,
|
||||||
|
message,
|
||||||
|
retry_sender.clone(),
|
||||||
|
sender_stats.clone(),
|
||||||
|
strategy,
|
||||||
|
);
|
||||||
|
|
||||||
|
if worker_count > 0 {
|
||||||
|
// If we're over the limit of workers, wait for them to finish before spawning
|
||||||
|
while join_set.len() >= worker_count {
|
||||||
|
join_set.join_next().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
join_set.spawn(task);
|
||||||
|
} else {
|
||||||
|
// If the worker count is `0` then just spawn and don't use the join_set
|
||||||
|
tokio::spawn(task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(retry_sender);
|
||||||
|
|
||||||
|
while !join_set.is_empty() {
|
||||||
|
join_set.join_next().await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Self {
|
||||||
|
stats,
|
||||||
|
sender,
|
||||||
|
sender_task,
|
||||||
|
retry_sender_task,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn queue(&self, message: SendActivityTask) -> Result<(), Error> {
|
||||||
|
self.stats.pending.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.sender
|
||||||
|
.send(message)
|
||||||
|
.map_err(|e| Error::ActivityQueueError(e.0.activity_id))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_stats(&self) -> &Stats {
|
||||||
|
&self.stats
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(unused)]
|
||||||
|
// Drops all the senders and shuts down the workers
|
||||||
|
pub(crate) async fn shutdown(self, wait_for_retries: bool) -> Result<Arc<Stats>, Error> {
|
||||||
|
drop(self.sender);
|
||||||
|
|
||||||
|
self.sender_task.await?;
|
||||||
|
|
||||||
|
if wait_for_retries {
|
||||||
|
self.retry_sender_task.await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(self.stats)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates an activity queue using tokio spawned tasks
|
||||||
|
/// Note: requires a tokio runtime
|
||||||
|
pub(crate) fn create_activity_queue(
|
||||||
|
client: ClientWithMiddleware,
|
||||||
|
worker_count: usize,
|
||||||
|
retry_count: usize,
|
||||||
|
request_timeout: Duration,
|
||||||
|
) -> ActivityQueue {
|
||||||
|
ActivityQueue::new(client, worker_count, retry_count, request_timeout, 60)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retries a future action factory function up to `amount` times with an exponential backoff timer between tries
|
||||||
|
async fn retry<T, E: Display + Debug, F: Future<Output = Result<T, E>>, A: FnMut() -> F>(
|
||||||
|
mut action: A,
|
||||||
|
strategy: RetryStrategy,
|
||||||
|
) -> Result<T, E> {
|
||||||
|
let mut count = strategy.offset;
|
||||||
|
|
||||||
|
// Do an initial sleep if it's called for
|
||||||
|
if strategy.initial_sleep > 0 {
|
||||||
|
let sleep_dur = Duration::from_secs(strategy.initial_sleep as u64);
|
||||||
|
tokio::time::sleep(sleep_dur).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match action().await {
|
||||||
|
Ok(val) => return Ok(val),
|
||||||
|
Err(err) => {
|
||||||
|
if count < strategy.retries {
|
||||||
|
count += 1;
|
||||||
|
|
||||||
|
let sleep_amt = strategy.backoff.pow(count as u32) as u64;
|
||||||
|
let sleep_dur = Duration::from_secs(sleep_amt);
|
||||||
|
warn!("{err:?}. Sleeping for {sleep_dur:?} and trying again");
|
||||||
|
tokio::time::sleep(sleep_dur).await;
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::http_signatures::generate_actor_keypair;
|
||||||
|
use axum::extract::State;
|
||||||
|
use bytes::Bytes;
|
||||||
|
use http::{HeaderMap, StatusCode};
|
||||||
|
use std::time::Instant;
|
||||||
|
use tracing::debug;
|
||||||
|
|
||||||
|
// This will periodically send back internal errors to test the retry
|
||||||
|
async fn dodgy_handler(
|
||||||
|
State(state): State<Arc<AtomicUsize>>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
body: Bytes,
|
||||||
|
) -> Result<(), StatusCode> {
|
||||||
|
debug!("Headers:{:?}", headers);
|
||||||
|
debug!("Body len:{}", body.len());
|
||||||
|
|
||||||
|
if state.fetch_add(1, Ordering::Relaxed) % 20 == 0 {
|
||||||
|
return Err(StatusCode::INTERNAL_SERVER_ERROR);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn test_server() {
|
||||||
|
use axum::{routing::post, Router};
|
||||||
|
|
||||||
|
// We should break every now and then ;)
|
||||||
|
let state = Arc::new(AtomicUsize::new(0));
|
||||||
|
|
||||||
|
let app = Router::new()
|
||||||
|
.route("/", post(dodgy_handler))
|
||||||
|
.with_state(state);
|
||||||
|
|
||||||
|
let listener = tokio::net::TcpListener::bind("0.0.0.0:8002").await.unwrap();
|
||||||
|
axum::serve(listener, app.into_make_service())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
// Queues 100 messages and then asserts that the worker runs them
|
||||||
|
async fn test_activity_queue_workers() {
|
||||||
|
let num_workers = 64;
|
||||||
|
let num_messages: usize = 100;
|
||||||
|
|
||||||
|
tokio::spawn(test_server());
|
||||||
|
|
||||||
|
/*
|
||||||
|
// uncomment for debug logs & stats
|
||||||
|
use tracing::log::LevelFilter;
|
||||||
|
|
||||||
|
env_logger::builder()
|
||||||
|
.filter_level(LevelFilter::Warn)
|
||||||
|
.filter_module("activitypub_federation", LevelFilter::Info)
|
||||||
|
.format_timestamp(None)
|
||||||
|
.init();
|
||||||
|
|
||||||
|
*/
|
||||||
|
|
||||||
|
let activity_queue = ActivityQueue::new(
|
||||||
|
reqwest::Client::default().into(),
|
||||||
|
num_workers,
|
||||||
|
num_workers,
|
||||||
|
Duration::from_secs(10),
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
|
||||||
|
let keypair = generate_actor_keypair().unwrap();
|
||||||
|
|
||||||
|
let message = SendActivityTask {
|
||||||
|
actor_id: "http://localhost:8002".parse().unwrap(),
|
||||||
|
activity_id: "http://localhost:8002/activity".parse().unwrap(),
|
||||||
|
activity: "{}".into(),
|
||||||
|
inbox: "http://localhost:8002".parse().unwrap(),
|
||||||
|
private_key: keypair.private_key().unwrap(),
|
||||||
|
http_signature_compat: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
|
||||||
|
for _ in 0..num_messages {
|
||||||
|
activity_queue.queue(message.clone()).await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Queue Sent: {:?}", start.elapsed());
|
||||||
|
|
||||||
|
let stats = activity_queue.shutdown(true).await.unwrap();
|
||||||
|
|
||||||
|
info!(
|
||||||
|
"Queue Finished. Num msgs: {}, Time {:?}, msg/s: {:0.0}",
|
||||||
|
num_messages,
|
||||||
|
start.elapsed(),
|
||||||
|
num_messages as f64 / start.elapsed().as_secs_f64()
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
stats.completed_last_hour.load(Ordering::Relaxed),
|
||||||
|
num_messages
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
|
@ -10,132 +10,123 @@ use crate::{
|
||||||
traits::{ActivityHandler, Actor},
|
traits::{ActivityHandler, Actor},
|
||||||
FEDERATION_CONTENT_TYPE,
|
FEDERATION_CONTENT_TYPE,
|
||||||
};
|
};
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
use http::{header::HeaderName, HeaderMap, HeaderValue};
|
use http::StatusCode;
|
||||||
use httpdate::fmt_http_date;
|
use httpdate::fmt_http_date;
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
use openssl::pkey::{PKey, Private};
|
use reqwest::{
|
||||||
use reqwest::Request;
|
header::{HeaderMap, HeaderName, HeaderValue},
|
||||||
|
Response,
|
||||||
|
};
|
||||||
use reqwest_middleware::ClientWithMiddleware;
|
use reqwest_middleware::ClientWithMiddleware;
|
||||||
|
use rsa::{pkcs8::DecodePrivateKey, RsaPrivateKey};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use std::{
|
use std::{
|
||||||
self,
|
|
||||||
fmt::{Debug, Display},
|
fmt::{Debug, Display},
|
||||||
time::{Duration, SystemTime},
|
time::{Duration, Instant, SystemTime},
|
||||||
};
|
};
|
||||||
use tracing::debug;
|
use tracing::{debug, warn};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
/// all info needed to send one activity to one inbox
|
/// All info needed to sign and send one activity to one inbox. You should generally use
|
||||||
pub struct SendActivityTask<'a> {
|
/// [[crate::activity_queue::queue_activity]] unless you want implement your own queue.
|
||||||
actor_id: &'a Url,
|
pub struct SendActivityTask {
|
||||||
activity_id: &'a Url,
|
pub(crate) actor_id: Url,
|
||||||
activity: Bytes,
|
pub(crate) activity_id: Url,
|
||||||
inbox: Url,
|
pub(crate) activity: Bytes,
|
||||||
private_key: PKey<Private>,
|
pub(crate) inbox: Url,
|
||||||
http_signature_compat: bool,
|
pub(crate) private_key: RsaPrivateKey,
|
||||||
|
pub(crate) http_signature_compat: bool,
|
||||||
}
|
}
|
||||||
impl Display for SendActivityTask<'_> {
|
|
||||||
|
impl Display for SendActivityTask {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
write!(f, "{} to {}", self.activity_id, self.inbox)
|
write!(f, "{} to {}", self.activity_id, self.inbox)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SendActivityTask<'_> {
|
impl SendActivityTask {
|
||||||
/// prepare an activity for sending
|
/// Prepare an activity for sending
|
||||||
///
|
///
|
||||||
/// - `activity`: The activity to be sent, gets converted to json
|
/// - `activity`: The activity to be sent, gets converted to json
|
||||||
/// - `inboxes`: List of remote actor inboxes that should receive the activity. Ignores local actor
|
/// - `inboxes`: List of remote actor inboxes that should receive the activity. Ignores local actor
|
||||||
/// inboxes. Should be built by calling [crate::traits::Actor::shared_inbox_or_inbox]
|
/// inboxes. Should be built by calling [crate::traits::Actor::shared_inbox_or_inbox]
|
||||||
/// for each target actor.
|
/// for each target actor.
|
||||||
pub async fn prepare<'a, Activity, Datatype, ActorType>(
|
pub async fn prepare<Activity, Datatype, ActorType>(
|
||||||
activity: &'a Activity,
|
activity: &Activity,
|
||||||
actor: &ActorType,
|
actor: &ActorType,
|
||||||
inboxes: Vec<Url>,
|
inboxes: Vec<Url>,
|
||||||
data: &Data<Datatype>,
|
data: &Data<Datatype>,
|
||||||
) -> Result<Vec<SendActivityTask<'a>>, Error>
|
) -> Result<Vec<SendActivityTask>, Error>
|
||||||
where
|
where
|
||||||
Activity: ActivityHandler + Serialize,
|
Activity: ActivityHandler + Serialize + Debug,
|
||||||
Datatype: Clone,
|
Datatype: Clone,
|
||||||
ActorType: Actor,
|
ActorType: Actor,
|
||||||
{
|
{
|
||||||
let config = &data.config;
|
build_tasks(activity, actor, inboxes, data).await
|
||||||
let actor_id = activity.actor();
|
|
||||||
let activity_id = activity.id();
|
|
||||||
let activity_serialized: Bytes = serde_json::to_vec(&activity).map_err(Error::Json)?.into();
|
|
||||||
let private_key = get_pkey_cached(data, actor).await?;
|
|
||||||
|
|
||||||
Ok(futures::stream::iter(
|
|
||||||
inboxes
|
|
||||||
.into_iter()
|
|
||||||
.unique()
|
|
||||||
.filter(|i| !config.is_local_url(i)),
|
|
||||||
)
|
|
||||||
.filter_map(|inbox| async {
|
|
||||||
if let Err(err) = config.verify_url_valid(&inbox).await {
|
|
||||||
debug!("inbox url invalid, skipping: {inbox}: {err}");
|
|
||||||
return None;
|
|
||||||
};
|
|
||||||
Some(SendActivityTask {
|
|
||||||
actor_id,
|
|
||||||
activity_id,
|
|
||||||
inbox,
|
|
||||||
activity: activity_serialized.clone(),
|
|
||||||
private_key: private_key.clone(),
|
|
||||||
http_signature_compat: config.http_signature_compat,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
.await)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// convert a sendactivitydata to a request, signing and sending it
|
/// convert a sendactivitydata to a request, signing and sending it
|
||||||
pub async fn sign_and_send<Datatype: Clone>(&self, data: &Data<Datatype>) -> Result<(), Error> {
|
pub async fn sign_and_send<Datatype: Clone>(&self, data: &Data<Datatype>) -> Result<(), Error> {
|
||||||
let req = self
|
self.sign_and_send_internal(&data.config.client, data.config.request_timeout)
|
||||||
.sign(&data.config.client, data.config.request_timeout)
|
.await
|
||||||
.await?;
|
|
||||||
self.send(&data.config.client, req).await
|
|
||||||
}
|
}
|
||||||
async fn sign(
|
|
||||||
|
pub(crate) async fn sign_and_send_internal(
|
||||||
&self,
|
&self,
|
||||||
client: &ClientWithMiddleware,
|
client: &ClientWithMiddleware,
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
) -> Result<Request, Error> {
|
) -> Result<(), Error> {
|
||||||
let task = self;
|
debug!("Sending {} to {}", self.activity_id, self.inbox,);
|
||||||
let request_builder = client
|
let request_builder = client
|
||||||
.post(task.inbox.to_string())
|
.post(self.inbox.to_string())
|
||||||
.timeout(timeout)
|
.timeout(timeout)
|
||||||
.headers(generate_request_headers(&task.inbox));
|
.headers(generate_request_headers(&self.inbox));
|
||||||
let request = sign_request(
|
let request = sign_request(
|
||||||
request_builder,
|
request_builder,
|
||||||
task.actor_id,
|
&self.actor_id,
|
||||||
task.activity.clone(),
|
self.activity.clone(),
|
||||||
task.private_key.clone(),
|
self.private_key.clone(),
|
||||||
task.http_signature_compat,
|
self.http_signature_compat,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(request)
|
|
||||||
|
// Send the activity, and log a warning if its too slow.
|
||||||
|
let now = Instant::now();
|
||||||
|
let response = client.execute(request).await?;
|
||||||
|
let elapsed = now.elapsed().as_secs();
|
||||||
|
if elapsed > 10 {
|
||||||
|
warn!(
|
||||||
|
"Sending activity {} to {} took {}s",
|
||||||
|
self.activity_id, self.inbox, elapsed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
self.handle_response(response).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn send(&self, client: &ClientWithMiddleware, request: Request) -> Result<(), Error> {
|
/// Based on the HTTP status code determines if an activity was delivered successfully. In that case
|
||||||
let response = client.execute(request).await?;
|
/// Ok is returned. Otherwise it returns Err and the activity send should be retried later.
|
||||||
|
///
|
||||||
match response {
|
/// Equivalent code in mastodon: https://github.com/mastodon/mastodon/blob/v4.2.8/app/helpers/jsonld_helper.rb#L215-L217
|
||||||
o if o.status().is_success() => {
|
async fn handle_response(&self, response: Response) -> Result<(), Error> {
|
||||||
|
match response.status() {
|
||||||
|
status if status.is_success() => {
|
||||||
debug!("Activity {self} delivered successfully");
|
debug!("Activity {self} delivered successfully");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
o if o.status().is_client_error() => {
|
status
|
||||||
let text = o.text_limited().await?;
|
if status.is_client_error()
|
||||||
|
&& status != StatusCode::REQUEST_TIMEOUT
|
||||||
|
&& status != StatusCode::TOO_MANY_REQUESTS =>
|
||||||
|
{
|
||||||
|
let text = response.text_limited().await?;
|
||||||
debug!("Activity {self} was rejected, aborting: {text}");
|
debug!("Activity {self} was rejected, aborting: {text}");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
o => {
|
status => {
|
||||||
let status = o.status();
|
let text = response.text_limited().await?;
|
||||||
let text = o.text_limited().await?;
|
|
||||||
|
|
||||||
Err(Error::Other(format!(
|
Err(Error::Other(format!(
|
||||||
"Activity {self} failure with status {status}: {text}",
|
"Activity {self} failure with status {status}: {text}",
|
||||||
|
@ -145,10 +136,53 @@ impl SendActivityTask<'_> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_pkey_cached<ActorType>(
|
pub(crate) async fn build_tasks<'a, Activity, Datatype, ActorType>(
|
||||||
|
activity: &'a Activity,
|
||||||
|
actor: &ActorType,
|
||||||
|
inboxes: Vec<Url>,
|
||||||
|
data: &Data<Datatype>,
|
||||||
|
) -> Result<Vec<SendActivityTask>, Error>
|
||||||
|
where
|
||||||
|
Activity: ActivityHandler + Serialize + Debug,
|
||||||
|
Datatype: Clone,
|
||||||
|
ActorType: Actor,
|
||||||
|
{
|
||||||
|
let config = &data.config;
|
||||||
|
let actor_id = activity.actor();
|
||||||
|
let activity_id = activity.id();
|
||||||
|
let activity_serialized: Bytes = serde_json::to_vec(activity)
|
||||||
|
.map_err(|e| Error::SerializeOutgoingActivity(e, format!("{:?}", activity)))?
|
||||||
|
.into();
|
||||||
|
let private_key = get_pkey_cached(data, actor).await?;
|
||||||
|
|
||||||
|
Ok(futures::stream::iter(
|
||||||
|
inboxes
|
||||||
|
.into_iter()
|
||||||
|
.unique()
|
||||||
|
.filter(|i| !config.is_local_url(i)),
|
||||||
|
)
|
||||||
|
.filter_map(|inbox| async {
|
||||||
|
if let Err(err) = config.verify_url_valid(&inbox).await {
|
||||||
|
debug!("inbox url invalid, skipping: {inbox}: {err}");
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
Some(SendActivityTask {
|
||||||
|
actor_id: actor_id.clone(),
|
||||||
|
activity_id: activity_id.clone(),
|
||||||
|
inbox,
|
||||||
|
activity: activity_serialized.clone(),
|
||||||
|
private_key: private_key.clone(),
|
||||||
|
http_signature_compat: config.http_signature_compat,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
.await)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn get_pkey_cached<ActorType>(
|
||||||
data: &Data<impl Clone>,
|
data: &Data<impl Clone>,
|
||||||
actor: &ActorType,
|
actor: &ActorType,
|
||||||
) -> Result<PKey<Private>, Error>
|
) -> Result<RsaPrivateKey, Error>
|
||||||
where
|
where
|
||||||
ActorType: Actor,
|
ActorType: Actor,
|
||||||
{
|
{
|
||||||
|
@ -165,13 +199,13 @@ where
|
||||||
|
|
||||||
// This is a mostly expensive blocking call, we don't want to tie up other tasks while this is happening
|
// This is a mostly expensive blocking call, we don't want to tie up other tasks while this is happening
|
||||||
let pkey = tokio::task::spawn_blocking(move || {
|
let pkey = tokio::task::spawn_blocking(move || {
|
||||||
PKey::private_key_from_pem(private_key_pem.as_bytes()).map_err(|err| {
|
RsaPrivateKey::from_pkcs8_pem(&private_key_pem).map_err(|err| {
|
||||||
Error::Other(format!("Could not create private key from PEM data:{err}"))
|
Error::Other(format!("Could not create private key from PEM data:{err}"))
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|err| Error::Other(format!("Error joining: {err}")))??;
|
.map_err(|err| Error::Other(format!("Error joining: {err}")))??;
|
||||||
std::result::Result::<PKey<Private>, Error>::Ok(pkey)
|
std::result::Result::<RsaPrivateKey, Error>::Ok(pkey)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|e| Error::Other(format!("cloned error: {e}")))
|
.map_err(|e| Error::Other(format!("cloned error: {e}")))
|
||||||
|
@ -200,33 +234,20 @@ pub(crate) fn generate_request_headers(inbox_url: &Url) -> HeaderMap {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use axum::extract::State;
|
use super::*;
|
||||||
use bytes::Bytes;
|
use crate::{config::FederationConfig, http_signatures::generate_actor_keypair};
|
||||||
use http::StatusCode;
|
|
||||||
use std::{
|
use std::{
|
||||||
sync::{atomic::AtomicUsize, Arc},
|
sync::{atomic::AtomicUsize, Arc},
|
||||||
time::Instant,
|
time::Instant,
|
||||||
};
|
};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use crate::{config::FederationConfig, http_signatures::generate_actor_keypair};
|
|
||||||
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[allow(unused)]
|
|
||||||
// This will periodically send back internal errors to test the retry
|
// This will periodically send back internal errors to test the retry
|
||||||
async fn dodgy_handler(
|
async fn dodgy_handler(headers: HeaderMap, body: Bytes) -> Result<(), StatusCode> {
|
||||||
State(state): State<Arc<AtomicUsize>>,
|
|
||||||
headers: HeaderMap,
|
|
||||||
body: Bytes,
|
|
||||||
) -> Result<(), StatusCode> {
|
|
||||||
debug!("Headers:{:?}", headers);
|
debug!("Headers:{:?}", headers);
|
||||||
debug!("Body len:{}", body.len());
|
debug!("Body len:{}", body.len());
|
||||||
|
|
||||||
/*if state.fetch_add(1, Ordering::Relaxed) % 20 == 0 {
|
|
||||||
return Err(StatusCode::INTERNAL_SERVER_ERROR);
|
|
||||||
}*/
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -240,8 +261,8 @@ mod tests {
|
||||||
.route("/", post(dodgy_handler))
|
.route("/", post(dodgy_handler))
|
||||||
.with_state(state);
|
.with_state(state);
|
||||||
|
|
||||||
axum::Server::bind(&"0.0.0.0:8001".parse().unwrap())
|
let listener = tokio::net::TcpListener::bind("0.0.0.0:8001").await.unwrap();
|
||||||
.serve(app.into_make_service())
|
axum::serve(listener, app.into_make_service())
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
@ -267,8 +288,8 @@ mod tests {
|
||||||
let keypair = generate_actor_keypair().unwrap();
|
let keypair = generate_actor_keypair().unwrap();
|
||||||
|
|
||||||
let message = SendActivityTask {
|
let message = SendActivityTask {
|
||||||
actor_id: &"http://localhost:8001".parse().unwrap(),
|
actor_id: "http://localhost:8001".parse().unwrap(),
|
||||||
activity_id: &"http://localhost:8001/activity".parse().unwrap(),
|
activity_id: "http://localhost:8001/activity".parse().unwrap(),
|
||||||
activity: "{}".into(),
|
activity: "{}".into(),
|
||||||
inbox: "http://localhost:8001".parse().unwrap(),
|
inbox: "http://localhost:8001".parse().unwrap(),
|
||||||
private_key: keypair.private_key().unwrap(),
|
private_key: keypair.private_key().unwrap(),
|
||||||
|
@ -284,10 +305,54 @@ mod tests {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
|
|
||||||
for _ in 0..num_messages {
|
for _ in 0..num_messages {
|
||||||
message.sign_and_send(&data).await?;
|
message.clone().sign_and_send(&data).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Queue Sent: {:?}", start.elapsed());
|
info!("Queue Sent: {:?}", start.elapsed());
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_handle_response() {
|
||||||
|
let keypair = generate_actor_keypair().unwrap();
|
||||||
|
let message = SendActivityTask {
|
||||||
|
actor_id: "http://localhost:8001".parse().unwrap(),
|
||||||
|
activity_id: "http://localhost:8001/activity".parse().unwrap(),
|
||||||
|
activity: "{}".into(),
|
||||||
|
inbox: "http://localhost:8001".parse().unwrap(),
|
||||||
|
private_key: keypair.private_key().unwrap(),
|
||||||
|
http_signature_compat: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
let res = |status| {
|
||||||
|
http::Response::builder()
|
||||||
|
.status(status)
|
||||||
|
.body(vec![])
|
||||||
|
.unwrap()
|
||||||
|
.into()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(message.handle_response(res(StatusCode::OK)).await.is_ok());
|
||||||
|
assert!(message
|
||||||
|
.handle_response(res(StatusCode::BAD_REQUEST))
|
||||||
|
.await
|
||||||
|
.is_ok());
|
||||||
|
|
||||||
|
assert!(message
|
||||||
|
.handle_response(res(StatusCode::MOVED_PERMANENTLY))
|
||||||
|
.await
|
||||||
|
.is_err());
|
||||||
|
assert!(message
|
||||||
|
.handle_response(res(StatusCode::REQUEST_TIMEOUT))
|
||||||
|
.await
|
||||||
|
.is_err());
|
||||||
|
assert!(message
|
||||||
|
.handle_response(res(StatusCode::TOO_MANY_REQUESTS))
|
||||||
|
.await
|
||||||
|
.is_err());
|
||||||
|
assert!(message
|
||||||
|
.handle_response(res(StatusCode::INTERNAL_SERVER_ERROR))
|
||||||
|
.await
|
||||||
|
.is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
30
src/actix_web/http_compat.rs
Normal file
30
src/actix_web/http_compat.rs
Normal file
|
@ -0,0 +1,30 @@
|
||||||
|
//! Remove these conversion helpers after actix-web upgrades to http 1.0
|
||||||
|
|
||||||
|
use std::str::FromStr;
|
||||||
|
|
||||||
|
pub fn header_value(v: &http02::HeaderValue) -> http::HeaderValue {
|
||||||
|
http::HeaderValue::from_bytes(v.as_bytes()).expect("can convert http types")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn header_map<'a, H>(m: H) -> http::HeaderMap
|
||||||
|
where
|
||||||
|
H: IntoIterator<Item = (&'a http02::HeaderName, &'a http02::HeaderValue)>,
|
||||||
|
{
|
||||||
|
let mut new_map = http::HeaderMap::new();
|
||||||
|
for (n, v) in m {
|
||||||
|
new_map.insert(
|
||||||
|
http::HeaderName::from_lowercase(n.as_str().as_bytes())
|
||||||
|
.expect("can convert http types"),
|
||||||
|
header_value(v),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
new_map
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn method(m: &http02::Method) -> http::Method {
|
||||||
|
http::Method::from_bytes(m.as_str().as_bytes()).expect("can convert http types")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn uri(m: &http02::Uri) -> http::Uri {
|
||||||
|
http::Uri::from_str(&m.to_string()).expect("can convert http types")
|
||||||
|
}
|
|
@ -1,10 +1,11 @@
|
||||||
//! Handles incoming activities, verifying HTTP signatures and other checks
|
//! Handles incoming activities, verifying HTTP signatures and other checks
|
||||||
|
|
||||||
|
use super::http_compat;
|
||||||
use crate::{
|
use crate::{
|
||||||
config::Data,
|
config::Data,
|
||||||
error::Error,
|
error::Error,
|
||||||
fetch::object_id::ObjectId,
|
|
||||||
http_signatures::{verify_body_hash, verify_signature},
|
http_signatures::{verify_body_hash, verify_signature},
|
||||||
|
parse_received_activity,
|
||||||
traits::{ActivityHandler, Actor, Object},
|
traits::{ActivityHandler, Actor, Object},
|
||||||
};
|
};
|
||||||
use actix_web::{web::Bytes, HttpRequest, HttpResponse};
|
use actix_web::{web::Bytes, HttpRequest, HttpResponse};
|
||||||
|
@ -27,20 +28,18 @@ where
|
||||||
<ActorT as Object>::Error: From<Error>,
|
<ActorT as Object>::Error: From<Error>,
|
||||||
Datatype: Clone,
|
Datatype: Clone,
|
||||||
{
|
{
|
||||||
verify_body_hash(request.headers().get("Digest"), &body)?;
|
let digest_header = request
|
||||||
|
.headers()
|
||||||
|
.get("Digest")
|
||||||
|
.map(http_compat::header_value);
|
||||||
|
verify_body_hash(digest_header.as_ref(), &body)?;
|
||||||
|
|
||||||
let activity: Activity = serde_json::from_slice(&body).map_err(Error::Json)?;
|
let (activity, actor) = parse_received_activity::<Activity, ActorT, _>(&body, data).await?;
|
||||||
data.config.verify_url_and_domain(&activity).await?;
|
|
||||||
let actor = ObjectId::<ActorT>::from(activity.actor().clone())
|
|
||||||
.dereference(data)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
verify_signature(
|
let headers = http_compat::header_map(request.headers());
|
||||||
request.headers(),
|
let method = http_compat::method(request.method());
|
||||||
request.method(),
|
let uri = http_compat::uri(request.uri());
|
||||||
request.uri(),
|
verify_signature(&headers, &method, &uri, actor.public_key_pem())?;
|
||||||
actor.public_key_pem(),
|
|
||||||
)?;
|
|
||||||
|
|
||||||
debug!("Receiving activity {}", activity.id().to_string());
|
debug!("Receiving activity {}", activity.id().to_string());
|
||||||
activity.verify(data).await?;
|
activity.verify(data).await?;
|
||||||
|
@ -49,19 +48,32 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
mod test {
|
mod test {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::{
|
use crate::{
|
||||||
activity_sending::generate_request_headers,
|
activity_sending::generate_request_headers,
|
||||||
config::FederationConfig,
|
config::FederationConfig,
|
||||||
|
fetch::object_id::ObjectId,
|
||||||
http_signatures::sign_request,
|
http_signatures::sign_request,
|
||||||
traits::tests::{DbConnection, DbUser, Follow, DB_USER_KEYPAIR},
|
traits::tests::{DbConnection, DbUser, Follow, DB_USER_KEYPAIR},
|
||||||
};
|
};
|
||||||
use actix_web::test::TestRequest;
|
use actix_web::test::TestRequest;
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
use reqwest_middleware::ClientWithMiddleware;
|
use reqwest_middleware::ClientWithMiddleware;
|
||||||
|
use serde_json::json;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
|
/// Remove this conversion helper after actix-web upgrades to http 1.0
|
||||||
|
fn header_pair(
|
||||||
|
p: (&http::HeaderName, &http::HeaderValue),
|
||||||
|
) -> (http02::HeaderName, http02::HeaderValue) {
|
||||||
|
(
|
||||||
|
http02::HeaderName::from_lowercase(p.0.as_str().as_bytes()).unwrap(),
|
||||||
|
http02::HeaderValue::from_bytes(p.1.as_bytes()).unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_receive_activity() {
|
async fn test_receive_activity() {
|
||||||
let (body, incoming_request, config) = setup_receive_test().await;
|
let (body, incoming_request, config) = setup_receive_test().await;
|
||||||
|
@ -105,22 +117,49 @@ mod test {
|
||||||
assert_eq!(&err, &Error::ActivitySignatureInvalid)
|
assert_eq!(&err, &Error::ActivitySignatureInvalid)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn setup_receive_test() -> (Bytes, TestRequest, FederationConfig<DbConnection>) {
|
#[tokio::test]
|
||||||
|
async fn test_receive_unparseable_activity() {
|
||||||
|
let (_, _, config) = setup_receive_test().await;
|
||||||
|
|
||||||
|
let actor = Url::parse("http://ds9.lemmy.ml/u/lemmy_alpha").unwrap();
|
||||||
|
let id = "http://localhost:123/1";
|
||||||
|
let activity = json!({
|
||||||
|
"actor": actor.as_str(),
|
||||||
|
"to": ["https://www.w3.org/ns/activitystreams#Public"],
|
||||||
|
"object": "http://ds9.lemmy.ml/post/1",
|
||||||
|
"cc": ["http://enterprise.lemmy.ml/c/main"],
|
||||||
|
"type": "Delete",
|
||||||
|
"id": id
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let body: Bytes = serde_json::to_vec(&activity).unwrap().into();
|
||||||
|
let incoming_request = construct_request(&body, &actor).await;
|
||||||
|
|
||||||
|
// intentionally cause a parse error by using wrong type for deser
|
||||||
|
let res = receive_activity::<Follow, DbUser, DbConnection>(
|
||||||
|
incoming_request.to_http_request(),
|
||||||
|
body,
|
||||||
|
&config.to_request_data(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
match res {
|
||||||
|
Err(Error::ParseReceivedActivity(_, url)) => {
|
||||||
|
assert_eq!(id, url.expect("has url").as_str());
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn construct_request(body: &Bytes, actor: &Url) -> TestRequest {
|
||||||
let inbox = "https://example.com/inbox";
|
let inbox = "https://example.com/inbox";
|
||||||
let headers = generate_request_headers(&Url::parse(inbox).unwrap());
|
let headers = generate_request_headers(&Url::parse(inbox).unwrap());
|
||||||
let request_builder = ClientWithMiddleware::from(Client::default())
|
let request_builder = ClientWithMiddleware::from(Client::default())
|
||||||
.post(inbox)
|
.post(inbox)
|
||||||
.headers(headers);
|
.headers(headers);
|
||||||
let activity = Follow {
|
|
||||||
actor: ObjectId::parse("http://localhost:123").unwrap(),
|
|
||||||
object: ObjectId::parse("http://localhost:124").unwrap(),
|
|
||||||
kind: Default::default(),
|
|
||||||
id: "http://localhost:123/1".try_into().unwrap(),
|
|
||||||
};
|
|
||||||
let body: Bytes = serde_json::to_vec(&activity).unwrap().into();
|
|
||||||
let outgoing_request = sign_request(
|
let outgoing_request = sign_request(
|
||||||
request_builder,
|
request_builder,
|
||||||
&activity.actor.into_inner(),
|
actor,
|
||||||
body.clone(),
|
body.clone(),
|
||||||
DB_USER_KEYPAIR.private_key().unwrap(),
|
DB_USER_KEYPAIR.private_key().unwrap(),
|
||||||
false,
|
false,
|
||||||
|
@ -129,8 +168,20 @@ mod test {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mut incoming_request = TestRequest::post().uri(outgoing_request.url().path());
|
let mut incoming_request = TestRequest::post().uri(outgoing_request.url().path());
|
||||||
for h in outgoing_request.headers() {
|
for h in outgoing_request.headers() {
|
||||||
incoming_request = incoming_request.append_header(h);
|
incoming_request = incoming_request.append_header(header_pair(h));
|
||||||
}
|
}
|
||||||
|
incoming_request
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn setup_receive_test() -> (Bytes, TestRequest, FederationConfig<DbConnection>) {
|
||||||
|
let activity = Follow {
|
||||||
|
actor: ObjectId::parse("http://localhost:123").unwrap(),
|
||||||
|
object: ObjectId::parse("http://localhost:124").unwrap(),
|
||||||
|
kind: Default::default(),
|
||||||
|
id: "http://localhost:123/1".try_into().unwrap(),
|
||||||
|
};
|
||||||
|
let body: Bytes = serde_json::to_vec(&activity).unwrap().into();
|
||||||
|
let incoming_request = construct_request(&body, activity.actor.inner()).await;
|
||||||
|
|
||||||
let config = FederationConfig::builder()
|
let config = FederationConfig::builder()
|
||||||
.domain("localhost:8002")
|
.domain("localhost:8002")
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
//! Utilities for using this library with actix-web framework
|
//! Utilities for using this library with actix-web framework
|
||||||
|
|
||||||
|
mod http_compat;
|
||||||
pub mod inbox;
|
pub mod inbox;
|
||||||
#[doc(hidden)]
|
#[doc(hidden)]
|
||||||
pub mod middleware;
|
pub mod middleware;
|
||||||
|
@ -25,7 +26,14 @@ where
|
||||||
<A as Object>::Error: From<Error>,
|
<A as Object>::Error: From<Error>,
|
||||||
for<'de2> <A as Object>::Kind: Deserialize<'de2>,
|
for<'de2> <A as Object>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
verify_body_hash(request.headers().get("Digest"), &body.unwrap_or_default())?;
|
let digest_header = request
|
||||||
|
.headers()
|
||||||
|
.get("Digest")
|
||||||
|
.map(http_compat::header_value);
|
||||||
|
verify_body_hash(digest_header.as_ref(), &body.unwrap_or_default())?;
|
||||||
|
|
||||||
http_signatures::signing_actor(request.headers(), request.method(), request.uri(), data).await
|
let headers = http_compat::header_map(request.headers());
|
||||||
|
let method = http_compat::method(request.method());
|
||||||
|
let uri = http_compat::uri(request.uri());
|
||||||
|
http_signatures::signing_actor(&headers, &method, &uri, data).await
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,13 +5,13 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
config::Data,
|
config::Data,
|
||||||
error::Error,
|
error::Error,
|
||||||
fetch::object_id::ObjectId,
|
http_signatures::verify_signature,
|
||||||
http_signatures::{verify_body_hash, verify_signature},
|
parse_received_activity,
|
||||||
traits::{ActivityHandler, Actor, Object},
|
traits::{ActivityHandler, Actor, Object},
|
||||||
};
|
};
|
||||||
use axum::{
|
use axum::{
|
||||||
async_trait,
|
async_trait,
|
||||||
body::{Bytes, HttpBody},
|
body::Body,
|
||||||
extract::FromRequest,
|
extract::FromRequest,
|
||||||
http::{Request, StatusCode},
|
http::{Request, StatusCode},
|
||||||
response::{IntoResponse, Response},
|
response::{IntoResponse, Response},
|
||||||
|
@ -33,13 +33,8 @@ where
|
||||||
<ActorT as Object>::Error: From<Error>,
|
<ActorT as Object>::Error: From<Error>,
|
||||||
Datatype: Clone,
|
Datatype: Clone,
|
||||||
{
|
{
|
||||||
verify_body_hash(activity_data.headers.get("Digest"), &activity_data.body)?;
|
let (activity, actor) =
|
||||||
|
parse_received_activity::<Activity, ActorT, _>(&activity_data.body, data).await?;
|
||||||
let activity: Activity = serde_json::from_slice(&activity_data.body).map_err(Error::Json)?;
|
|
||||||
data.config.verify_url_and_domain(&activity).await?;
|
|
||||||
let actor = ObjectId::<ActorT>::from(activity.actor().clone())
|
|
||||||
.dereference(data)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
verify_signature(
|
verify_signature(
|
||||||
&activity_data.headers,
|
&activity_data.headers,
|
||||||
|
@ -64,21 +59,17 @@ pub struct ActivityData {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<S, B> FromRequest<S, B> for ActivityData
|
impl<S> FromRequest<S> for ActivityData
|
||||||
where
|
where
|
||||||
Bytes: FromRequest<S, B>,
|
|
||||||
B: HttpBody + Send + 'static,
|
|
||||||
S: Send + Sync,
|
S: Send + Sync,
|
||||||
<B as HttpBody>::Error: std::fmt::Display,
|
|
||||||
<B as HttpBody>::Data: Send,
|
|
||||||
{
|
{
|
||||||
type Rejection = Response;
|
type Rejection = Response;
|
||||||
|
|
||||||
async fn from_request(req: Request<B>, _state: &S) -> Result<Self, Self::Rejection> {
|
async fn from_request(req: Request<Body>, _state: &S) -> Result<Self, Self::Rejection> {
|
||||||
let (parts, body) = req.into_parts();
|
let (parts, body) = req.into_parts();
|
||||||
|
|
||||||
// this wont work if the body is an long running stream
|
// this wont work if the body is an long running stream
|
||||||
let bytes = hyper::body::to_bytes(body)
|
let bytes = axum::body::to_bytes(body, usize::MAX)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response())?;
|
.map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response())?;
|
||||||
|
|
||||||
|
|
|
@ -9,7 +9,7 @@
|
||||||
//! # use activitypub_federation::traits::Object;
|
//! # use activitypub_federation::traits::Object;
|
||||||
//! # use activitypub_federation::traits::tests::{DbConnection, DbUser, Person};
|
//! # use activitypub_federation::traits::tests::{DbConnection, DbUser, Person};
|
||||||
//! async fn http_get_user(Path(name): Path<String>, data: Data<DbConnection>) -> Result<FederationJson<WithContext<Person>>, Error> {
|
//! async fn http_get_user(Path(name): Path<String>, data: Data<DbConnection>) -> Result<FederationJson<WithContext<Person>>, Error> {
|
||||||
//! let user: DbUser = data.read_local_user(name).await?;
|
//! let user: DbUser = data.read_local_user(&name).await?;
|
||||||
//! let person = user.into_json(&data).await?;
|
//! let person = user.into_json(&data).await?;
|
||||||
//!
|
//!
|
||||||
//! Ok(FederationJson(WithContext::new_default(person)))
|
//! Ok(FederationJson(WithContext::new_default(person)))
|
||||||
|
|
|
@ -15,6 +15,7 @@
|
||||||
//! ```
|
//! ```
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
|
activity_queue::{create_activity_queue, ActivityQueue},
|
||||||
error::Error,
|
error::Error,
|
||||||
protocol::verification::verify_domains_match,
|
protocol::verification::verify_domains_match,
|
||||||
traits::{ActivityHandler, Actor},
|
traits::{ActivityHandler, Actor},
|
||||||
|
@ -23,8 +24,8 @@ use async_trait::async_trait;
|
||||||
use derive_builder::Builder;
|
use derive_builder::Builder;
|
||||||
use dyn_clone::{clone_trait_object, DynClone};
|
use dyn_clone::{clone_trait_object, DynClone};
|
||||||
use moka::future::Cache;
|
use moka::future::Cache;
|
||||||
use openssl::pkey::{PKey, Private};
|
|
||||||
use reqwest_middleware::ClientWithMiddleware;
|
use reqwest_middleware::ClientWithMiddleware;
|
||||||
|
use rsa::{pkcs8::DecodePrivateKey, RsaPrivateKey};
|
||||||
use serde::de::DeserializeOwned;
|
use serde::de::DeserializeOwned;
|
||||||
use std::{
|
use std::{
|
||||||
ops::Deref,
|
ops::Deref,
|
||||||
|
@ -79,12 +80,26 @@ pub struct FederationConfig<T: Clone> {
|
||||||
/// This can be used to implement secure mode federation.
|
/// This can be used to implement secure mode federation.
|
||||||
/// <https://docs.joinmastodon.org/spec/activitypub/#secure-mode>
|
/// <https://docs.joinmastodon.org/spec/activitypub/#secure-mode>
|
||||||
#[builder(default = "None", setter(custom))]
|
#[builder(default = "None", setter(custom))]
|
||||||
pub(crate) signed_fetch_actor: Option<Arc<(Url, PKey<Private>)>>,
|
pub(crate) signed_fetch_actor: Option<Arc<(Url, RsaPrivateKey)>>,
|
||||||
#[builder(
|
#[builder(
|
||||||
default = "Cache::builder().max_capacity(10000).build()",
|
default = "Cache::builder().max_capacity(10000).build()",
|
||||||
setter(custom)
|
setter(custom)
|
||||||
)]
|
)]
|
||||||
pub(crate) actor_pkey_cache: Cache<Url, PKey<Private>>,
|
pub(crate) actor_pkey_cache: Cache<Url, RsaPrivateKey>,
|
||||||
|
/// Queue for sending outgoing activities. Only optional to make builder work, its always
|
||||||
|
/// present once constructed.
|
||||||
|
#[builder(setter(skip))]
|
||||||
|
pub(crate) activity_queue: Option<Arc<ActivityQueue>>,
|
||||||
|
/// When sending with activity queue: Number of tasks that can be in-flight concurrently.
|
||||||
|
/// Tasks are retried once after a minute, then put into the retry queue.
|
||||||
|
/// Setting this count to `0` means that there is no limit to concurrency
|
||||||
|
#[builder(default = "0")]
|
||||||
|
pub(crate) queue_worker_count: usize,
|
||||||
|
/// When sending with activity queue: Number of concurrent tasks that are being retried
|
||||||
|
/// in-flight concurrently. Tasks are retried after an hour, then again in 60 hours.
|
||||||
|
/// Setting this count to `0` means that there is no limit to concurrency
|
||||||
|
#[builder(default = "0")]
|
||||||
|
pub(crate) queue_retry_count: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<T: Clone> FederationConfig<T> {
|
impl<T: Clone> FederationConfig<T> {
|
||||||
|
@ -159,11 +174,17 @@ impl<T: Clone> FederationConfig<T> {
|
||||||
/// Returns true if the url refers to this instance. Handles hostnames like `localhost:8540` for
|
/// Returns true if the url refers to this instance. Handles hostnames like `localhost:8540` for
|
||||||
/// local debugging.
|
/// local debugging.
|
||||||
pub(crate) fn is_local_url(&self, url: &Url) -> bool {
|
pub(crate) fn is_local_url(&self, url: &Url) -> bool {
|
||||||
let mut domain = url.host_str().expect("id has domain").to_string();
|
match url.host_str() {
|
||||||
if let Some(port) = url.port() {
|
Some(domain) => {
|
||||||
domain = format!("{}:{}", domain, port);
|
let domain = if let Some(port) = url.port() {
|
||||||
|
format!("{}:{}", domain, port)
|
||||||
|
} else {
|
||||||
|
domain.to_string()
|
||||||
|
};
|
||||||
|
domain == self.domain
|
||||||
|
}
|
||||||
|
None => false,
|
||||||
}
|
}
|
||||||
domain == self.domain
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the local domain
|
/// Returns the local domain
|
||||||
|
@ -179,8 +200,8 @@ impl<T: Clone> FederationConfigBuilder<T> {
|
||||||
.private_key_pem()
|
.private_key_pem()
|
||||||
.expect("actor does not have a private key to sign with");
|
.expect("actor does not have a private key to sign with");
|
||||||
|
|
||||||
let private_key = PKey::private_key_from_pem(private_key_pem.as_bytes())
|
let private_key =
|
||||||
.expect("Could not decode PEM data");
|
RsaPrivateKey::from_pkcs8_pem(&private_key_pem).expect("Could not decode PEM data");
|
||||||
self.signed_fetch_actor = Some(Some(Arc::new((actor.id(), private_key))));
|
self.signed_fetch_actor = Some(Some(Arc::new((actor.id(), private_key))));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
@ -197,7 +218,14 @@ impl<T: Clone> FederationConfigBuilder<T> {
|
||||||
/// queue for outgoing activities, which is stored internally in the config struct.
|
/// queue for outgoing activities, which is stored internally in the config struct.
|
||||||
/// Requires a tokio runtime for the background queue.
|
/// Requires a tokio runtime for the background queue.
|
||||||
pub async fn build(&mut self) -> Result<FederationConfig<T>, FederationConfigBuilderError> {
|
pub async fn build(&mut self) -> Result<FederationConfig<T>, FederationConfigBuilderError> {
|
||||||
let config = self.partial_build()?;
|
let mut config = self.partial_build()?;
|
||||||
|
let queue = create_activity_queue(
|
||||||
|
config.client.clone(),
|
||||||
|
config.queue_worker_count,
|
||||||
|
config.queue_retry_count,
|
||||||
|
config.request_timeout,
|
||||||
|
);
|
||||||
|
config.activity_queue = Some(Arc::new(queue));
|
||||||
Ok(config)
|
Ok(config)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -319,3 +347,34 @@ impl<T: Clone> FederationMiddleware<T> {
|
||||||
FederationMiddleware(config)
|
FederationMiddleware(config)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
|
mod test {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
async fn config() -> FederationConfig<i32> {
|
||||||
|
FederationConfig::builder()
|
||||||
|
.domain("example.com")
|
||||||
|
.app_data(1)
|
||||||
|
.build()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_url_is_local() -> Result<(), Error> {
|
||||||
|
let config = config().await;
|
||||||
|
assert!(config.is_local_url(&Url::parse("http://example.com")?));
|
||||||
|
assert!(!config.is_local_url(&Url::parse("http://other.com")?));
|
||||||
|
// ensure that missing domain doesnt cause crash
|
||||||
|
assert!(!config.is_local_url(&Url::parse("http://127.0.0.1")?));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_get_domain() {
|
||||||
|
let config = config().await;
|
||||||
|
assert_eq!("example.com", config.domain());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
60
src/error.rs
60
src/error.rs
|
@ -1,9 +1,13 @@
|
||||||
//! Error messages returned by this library
|
//! Error messages returned by this library
|
||||||
|
|
||||||
use std::string::FromUtf8Error;
|
use crate::fetch::webfinger::WebFingerError;
|
||||||
|
|
||||||
use http_signature_normalization_reqwest::SignError;
|
use http_signature_normalization_reqwest::SignError;
|
||||||
use openssl::error::ErrorStack;
|
use rsa::{
|
||||||
|
errors::Error as RsaError,
|
||||||
|
pkcs8::{spki::Error as SpkiError, Error as Pkcs8Error},
|
||||||
|
};
|
||||||
|
use std::string::FromUtf8Error;
|
||||||
|
use tokio::task::JoinError;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
/// Error messages returned by this library
|
/// Error messages returned by this library
|
||||||
|
@ -32,13 +36,19 @@ pub enum Error {
|
||||||
ActivitySignatureInvalid,
|
ActivitySignatureInvalid,
|
||||||
/// Failed to resolve actor via webfinger
|
/// Failed to resolve actor via webfinger
|
||||||
#[error("Failed to resolve actor via webfinger")]
|
#[error("Failed to resolve actor via webfinger")]
|
||||||
WebfingerResolveFailed,
|
WebfingerResolveFailed(#[from] WebFingerError),
|
||||||
/// Failed to resolve actor via webfinger
|
/// Failed to serialize outgoing activity
|
||||||
#[error("Webfinger regex failed to match")]
|
#[error("Failed to serialize outgoing activity {1}: {0}")]
|
||||||
WebfingerRegexFailed,
|
SerializeOutgoingActivity(serde_json::Error, String),
|
||||||
/// JSON Error
|
/// Failed to parse an object fetched from url
|
||||||
#[error(transparent)]
|
#[error("Failed to parse object {1} with content {2}: {0}")]
|
||||||
Json(#[from] serde_json::Error),
|
ParseFetchedObject(serde_json::Error, Url, String),
|
||||||
|
/// Failed to parse an activity received from another instance
|
||||||
|
#[error("Failed to parse incoming activity {}: {0}", match .1 {
|
||||||
|
Some(t) => format!("with id {t}"),
|
||||||
|
None => String::new(),
|
||||||
|
})]
|
||||||
|
ParseReceivedActivity(serde_json::Error, Option<Url>),
|
||||||
/// Reqwest Middleware Error
|
/// Reqwest Middleware Error
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
ReqwestMiddleware(#[from] reqwest_middleware::Error),
|
ReqwestMiddleware(#[from] reqwest_middleware::Error),
|
||||||
|
@ -54,13 +64,39 @@ pub enum Error {
|
||||||
/// Signing errors
|
/// Signing errors
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
SignError(#[from] SignError),
|
SignError(#[from] SignError),
|
||||||
|
/// Failed to queue activity for sending
|
||||||
|
#[error("Failed to queue activity {0} for sending")]
|
||||||
|
ActivityQueueError(Url),
|
||||||
|
/// Stop activity queue
|
||||||
|
#[error(transparent)]
|
||||||
|
StopActivityQueue(#[from] JoinError),
|
||||||
|
/// Attempted to fetch object which doesn't have valid ActivityPub Content-Type
|
||||||
|
#[error(
|
||||||
|
"Attempted to fetch object from {0} which doesn't have valid ActivityPub Content-Type"
|
||||||
|
)]
|
||||||
|
FetchInvalidContentType(Url),
|
||||||
|
/// Attempted to fetch object but the response's id field doesn't match
|
||||||
|
#[error("Attempted to fetch object from {0} but the response's id field doesn't match")]
|
||||||
|
FetchWrongId(Url),
|
||||||
/// Other generic errors
|
/// Other generic errors
|
||||||
#[error("{0}")]
|
#[error("{0}")]
|
||||||
Other(String),
|
Other(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<ErrorStack> for Error {
|
impl From<RsaError> for Error {
|
||||||
fn from(value: ErrorStack) -> Self {
|
fn from(value: RsaError) -> Self {
|
||||||
|
Error::Other(value.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Pkcs8Error> for Error {
|
||||||
|
fn from(value: Pkcs8Error) -> Self {
|
||||||
|
Error::Other(value.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<SpkiError> for Error {
|
||||||
|
fn from(value: SpkiError) -> Self {
|
||||||
Error::Other(value.to_string())
|
Error::Other(value.to_string())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,12 +20,8 @@ where
|
||||||
for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
|
for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
/// Construct a new CollectionId instance
|
/// Construct a new CollectionId instance
|
||||||
pub fn parse<T>(url: T) -> Result<Self, url::ParseError>
|
pub fn parse(url: &str) -> Result<Self, url::ParseError> {
|
||||||
where
|
Ok(Self(Box::new(Url::parse(url)?), PhantomData::<Kind>))
|
||||||
T: TryInto<Url>,
|
|
||||||
url::ParseError: From<<T as TryInto<Url>>::Error>,
|
|
||||||
{
|
|
||||||
Ok(Self(Box::new(url.try_into()?), PhantomData::<Kind>))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetches collection over HTTP
|
/// Fetches collection over HTTP
|
||||||
|
@ -106,3 +102,92 @@ where
|
||||||
self.0.eq(&other.0) && self.1 == other.1
|
self.0.eq(&other.0) && self.1 == other.1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "diesel")]
|
||||||
|
const _IMPL_DIESEL_NEW_TYPE_FOR_COLLECTION_ID: () = {
|
||||||
|
use diesel::{
|
||||||
|
backend::Backend,
|
||||||
|
deserialize::{FromSql, FromStaticSqlRow},
|
||||||
|
expression::AsExpression,
|
||||||
|
internal::derives::as_expression::Bound,
|
||||||
|
pg::Pg,
|
||||||
|
query_builder::QueryId,
|
||||||
|
serialize,
|
||||||
|
serialize::{Output, ToSql},
|
||||||
|
sql_types::{HasSqlType, SingleValue, Text},
|
||||||
|
Expression,
|
||||||
|
Queryable,
|
||||||
|
};
|
||||||
|
|
||||||
|
// TODO: this impl only works for Postgres db because of to_string() call which requires reborrow
|
||||||
|
impl<Kind, ST> ToSql<ST, Pg> for CollectionId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Collection,
|
||||||
|
for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
|
||||||
|
String: ToSql<ST, Pg>,
|
||||||
|
{
|
||||||
|
fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> serialize::Result {
|
||||||
|
let v = self.0.to_string();
|
||||||
|
<String as ToSql<Text, Pg>>::to_sql(&v, &mut out.reborrow())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<'expr, Kind, ST> AsExpression<ST> for &'expr CollectionId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Collection,
|
||||||
|
for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
|
||||||
|
Bound<ST, String>: Expression<SqlType = ST>,
|
||||||
|
ST: SingleValue,
|
||||||
|
{
|
||||||
|
type Expression = Bound<ST, &'expr str>;
|
||||||
|
fn as_expression(self) -> Self::Expression {
|
||||||
|
Bound::new(self.0.as_str())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<Kind, ST> AsExpression<ST> for CollectionId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Collection,
|
||||||
|
for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
|
||||||
|
Bound<ST, String>: Expression<SqlType = ST>,
|
||||||
|
ST: SingleValue,
|
||||||
|
{
|
||||||
|
type Expression = Bound<ST, String>;
|
||||||
|
fn as_expression(self) -> Self::Expression {
|
||||||
|
Bound::new(self.0.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<Kind, ST, DB> FromSql<ST, DB> for CollectionId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Collection + Send + 'static,
|
||||||
|
for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
|
||||||
|
String: FromSql<ST, DB>,
|
||||||
|
DB: Backend,
|
||||||
|
DB: HasSqlType<ST>,
|
||||||
|
{
|
||||||
|
fn from_sql(
|
||||||
|
raw: DB::RawValue<'_>,
|
||||||
|
) -> Result<Self, Box<dyn ::std::error::Error + Send + Sync>> {
|
||||||
|
let string: String = FromSql::<ST, DB>::from_sql(raw)?;
|
||||||
|
Ok(CollectionId::parse(&string)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<Kind, ST, DB> Queryable<ST, DB> for CollectionId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Collection + Send + 'static,
|
||||||
|
for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
|
||||||
|
String: FromStaticSqlRow<ST, DB>,
|
||||||
|
DB: Backend,
|
||||||
|
DB: HasSqlType<ST>,
|
||||||
|
{
|
||||||
|
type Row = String;
|
||||||
|
fn build(row: Self::Row) -> diesel::deserialize::Result<Self> {
|
||||||
|
Ok(CollectionId::parse(&row)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<Kind> QueryId for CollectionId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Collection + 'static,
|
||||||
|
for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
|
||||||
|
{
|
||||||
|
type QueryId = Self;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
111
src/fetch/mod.rs
111
src/fetch/mod.rs
|
@ -4,13 +4,14 @@
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
config::Data,
|
config::Data,
|
||||||
error::Error,
|
error::{Error, Error::ParseFetchedObject},
|
||||||
|
extract_id,
|
||||||
http_signatures::sign_request,
|
http_signatures::sign_request,
|
||||||
reqwest_shim::ResponseExt,
|
reqwest_shim::ResponseExt,
|
||||||
FEDERATION_CONTENT_TYPE,
|
FEDERATION_CONTENT_TYPE,
|
||||||
};
|
};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http::StatusCode;
|
use http::{HeaderValue, StatusCode};
|
||||||
use serde::de::DeserializeOwned;
|
use serde::de::DeserializeOwned;
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
@ -29,6 +30,8 @@ pub struct FetchObjectResponse<Kind> {
|
||||||
pub object: Kind,
|
pub object: Kind,
|
||||||
/// Contains the final URL (different from request URL in case of redirect)
|
/// Contains the final URL (different from request URL in case of redirect)
|
||||||
pub url: Url,
|
pub url: Url,
|
||||||
|
content_type: Option<HeaderValue>,
|
||||||
|
object_id: Option<Url>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch a remote object over HTTP and convert to `Kind`.
|
/// Fetch a remote object over HTTP and convert to `Kind`.
|
||||||
|
@ -42,12 +45,52 @@ pub struct FetchObjectResponse<Kind> {
|
||||||
/// [Error::RequestLimit]. This prevents denial of service attacks where an attack triggers
|
/// [Error::RequestLimit]. This prevents denial of service attacks where an attack triggers
|
||||||
/// infinite, recursive fetching of data.
|
/// infinite, recursive fetching of data.
|
||||||
///
|
///
|
||||||
/// The `Accept` header will be set to the content of [`FEDERATION_CONTENT_TYPE`].
|
/// The `Accept` header will be set to the content of [`FEDERATION_CONTENT_TYPE`]. When parsing the
|
||||||
|
/// response it ensures that it has a valid `Content-Type` header as defined by ActivityPub, to
|
||||||
|
/// prevent security vulnerabilities like [this one](https://github.com/mastodon/mastodon/security/advisories/GHSA-jhrq-qvrm-qr36).
|
||||||
|
/// Additionally it checks that the `id` field is identical to the fetch URL (after redirects).
|
||||||
pub async fn fetch_object_http<T: Clone, Kind: DeserializeOwned>(
|
pub async fn fetch_object_http<T: Clone, Kind: DeserializeOwned>(
|
||||||
url: &Url,
|
url: &Url,
|
||||||
data: &Data<T>,
|
data: &Data<T>,
|
||||||
) -> Result<FetchObjectResponse<Kind>, Error> {
|
) -> Result<FetchObjectResponse<Kind>, Error> {
|
||||||
fetch_object_http_with_accept(url, data, FEDERATION_CONTENT_TYPE).await
|
static FETCH_CONTENT_TYPE: HeaderValue = HeaderValue::from_static(FEDERATION_CONTENT_TYPE);
|
||||||
|
const VALID_RESPONSE_CONTENT_TYPES: [&str; 3] = [
|
||||||
|
FEDERATION_CONTENT_TYPE, // lemmy
|
||||||
|
r#"application/ld+json; profile="https://www.w3.org/ns/activitystreams""#, // activitypub standard
|
||||||
|
r#"application/activity+json; charset=utf-8"#, // mastodon
|
||||||
|
];
|
||||||
|
let res = fetch_object_http_with_accept(url, data, &FETCH_CONTENT_TYPE).await?;
|
||||||
|
|
||||||
|
// Ensure correct content-type to prevent vulnerabilities, with case insensitive comparison.
|
||||||
|
let content_type = res
|
||||||
|
.content_type
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|c| Some(c.to_str().ok()?.to_lowercase()))
|
||||||
|
.ok_or(Error::FetchInvalidContentType(res.url.clone()))?;
|
||||||
|
if !VALID_RESPONSE_CONTENT_TYPES.contains(&content_type.as_str()) {
|
||||||
|
return Err(Error::FetchInvalidContentType(res.url));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure id field matches final url after redirect
|
||||||
|
if res.object_id.as_ref() != Some(&res.url) {
|
||||||
|
if let Some(res_object_id) = res.object_id {
|
||||||
|
// If id is different but still on the same domain, attempt to request object
|
||||||
|
// again from url in id field.
|
||||||
|
if res_object_id.domain() == res.url.domain() {
|
||||||
|
return Box::pin(fetch_object_http(&res_object_id, data)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Failed to fetch the object from its specified id
|
||||||
|
return Err(Error::FetchWrongId(res.url));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dont allow fetching local object. Only check this after the request as a local url
|
||||||
|
// may redirect to a remote object.
|
||||||
|
if data.config.is_local_url(&res.url) {
|
||||||
|
return Err(Error::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(res)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch a remote object over HTTP and convert to `Kind`. This function works exactly as
|
/// Fetch a remote object over HTTP and convert to `Kind`. This function works exactly as
|
||||||
|
@ -55,15 +98,15 @@ pub async fn fetch_object_http<T: Clone, Kind: DeserializeOwned>(
|
||||||
async fn fetch_object_http_with_accept<T: Clone, Kind: DeserializeOwned>(
|
async fn fetch_object_http_with_accept<T: Clone, Kind: DeserializeOwned>(
|
||||||
url: &Url,
|
url: &Url,
|
||||||
data: &Data<T>,
|
data: &Data<T>,
|
||||||
content_type: &str,
|
content_type: &HeaderValue,
|
||||||
) -> Result<FetchObjectResponse<Kind>, Error> {
|
) -> Result<FetchObjectResponse<Kind>, Error> {
|
||||||
let config = &data.config;
|
let config = &data.config;
|
||||||
// dont fetch local objects this way
|
|
||||||
debug_assert!(url.domain() != Some(&config.domain));
|
|
||||||
config.verify_url_valid(url).await?;
|
config.verify_url_valid(url).await?;
|
||||||
info!("Fetching remote object {}", url.to_string());
|
info!("Fetching remote object {}", url.to_string());
|
||||||
|
|
||||||
let counter = data.request_counter.fetch_add(1, Ordering::SeqCst);
|
let mut counter = data.request_counter.fetch_add(1, Ordering::SeqCst);
|
||||||
|
// fetch_add returns old value so we need to increment manually here
|
||||||
|
counter += 1;
|
||||||
if counter > config.http_fetch_limit {
|
if counter > config.http_fetch_limit {
|
||||||
return Err(Error::RequestLimit);
|
return Err(Error::RequestLimit);
|
||||||
}
|
}
|
||||||
|
@ -93,8 +136,52 @@ async fn fetch_object_http_with_accept<T: Clone, Kind: DeserializeOwned>(
|
||||||
}
|
}
|
||||||
|
|
||||||
let url = res.url().clone();
|
let url = res.url().clone();
|
||||||
Ok(FetchObjectResponse {
|
let content_type = res.headers().get("Content-Type").cloned();
|
||||||
object: res.json_limited().await?,
|
let text = res.bytes_limited().await?;
|
||||||
url,
|
let object_id = extract_id(&text).ok();
|
||||||
})
|
|
||||||
|
match serde_json::from_slice(&text) {
|
||||||
|
Ok(object) => Ok(FetchObjectResponse {
|
||||||
|
object,
|
||||||
|
url,
|
||||||
|
content_type,
|
||||||
|
object_id,
|
||||||
|
}),
|
||||||
|
Err(e) => Err(ParseFetchedObject(
|
||||||
|
e,
|
||||||
|
url,
|
||||||
|
String::from_utf8(Vec::from(text))?,
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::{
|
||||||
|
config::FederationConfig,
|
||||||
|
traits::tests::{DbConnection, Person},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_request_limit() -> Result<(), Error> {
|
||||||
|
let config = FederationConfig::builder()
|
||||||
|
.domain("example.com")
|
||||||
|
.app_data(DbConnection)
|
||||||
|
.http_fetch_limit(0)
|
||||||
|
.build()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let data = config.to_request_data();
|
||||||
|
|
||||||
|
let fetch_url = "https://example.net/".to_string();
|
||||||
|
|
||||||
|
let res: Result<FetchObjectResponse<Person>, Error> =
|
||||||
|
fetch_object_http(&Url::parse(&fetch_url).map_err(Error::UrlParse)?, &data).await;
|
||||||
|
|
||||||
|
assert_eq!(res.err(), Some(Error::RequestLimit));
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -57,20 +57,16 @@ where
|
||||||
pub struct ObjectId<Kind>(Box<Url>, PhantomData<Kind>)
|
pub struct ObjectId<Kind>(Box<Url>, PhantomData<Kind>)
|
||||||
where
|
where
|
||||||
Kind: Object,
|
Kind: Object,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>;
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>;
|
||||||
|
|
||||||
impl<Kind> ObjectId<Kind>
|
impl<Kind> ObjectId<Kind>
|
||||||
where
|
where
|
||||||
Kind: Object + Send + Debug + 'static,
|
Kind: Object + Send + Debug + 'static,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
/// Construct a new objectid instance
|
/// Construct a new objectid instance
|
||||||
pub fn parse<T>(url: T) -> Result<Self, url::ParseError>
|
pub fn parse(url: &str) -> Result<Self, url::ParseError> {
|
||||||
where
|
Ok(Self(Box::new(Url::parse(url)?), PhantomData::<Kind>))
|
||||||
T: TryInto<Url>,
|
|
||||||
url::ParseError: From<<T as TryInto<Url>>::Error>,
|
|
||||||
{
|
|
||||||
Ok(ObjectId(Box::new(url.try_into()?), PhantomData::<Kind>))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns a reference to the wrapped URL value
|
/// Returns a reference to the wrapped URL value
|
||||||
|
@ -92,19 +88,13 @@ where
|
||||||
<Kind as Object>::Error: From<Error>,
|
<Kind as Object>::Error: From<Error>,
|
||||||
{
|
{
|
||||||
let db_object = self.dereference_from_db(data).await?;
|
let db_object = self.dereference_from_db(data).await?;
|
||||||
// if its a local object, only fetch it from the database and not over http
|
|
||||||
if data.config.is_local_url(&self.0) {
|
|
||||||
return match db_object {
|
|
||||||
None => Err(Error::NotFound.into()),
|
|
||||||
Some(o) => Ok(o),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// object found in database
|
// object found in database
|
||||||
if let Some(object) = db_object {
|
if let Some(object) = db_object {
|
||||||
// object is old and should be refetched
|
|
||||||
if let Some(last_refreshed_at) = object.last_refreshed_at() {
|
if let Some(last_refreshed_at) = object.last_refreshed_at() {
|
||||||
if should_refetch_object(last_refreshed_at) {
|
let is_local = data.config.is_local_url(&self.0);
|
||||||
|
if !is_local && should_refetch_object(last_refreshed_at) {
|
||||||
|
// object is outdated and should be refetched
|
||||||
return self.dereference_from_http(data, Some(object)).await;
|
return self.dereference_from_http(data, Some(object)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -116,6 +106,24 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// If this is a remote object, fetch it from origin instance unconditionally to get the
|
||||||
|
/// latest version, regardless of refresh interval.
|
||||||
|
pub async fn dereference_forced(
|
||||||
|
&self,
|
||||||
|
data: &Data<<Kind as Object>::DataType>,
|
||||||
|
) -> Result<Kind, <Kind as Object>::Error>
|
||||||
|
where
|
||||||
|
<Kind as Object>::Error: From<Error>,
|
||||||
|
{
|
||||||
|
if data.config.is_local_url(&self.0) {
|
||||||
|
self.dereference_from_db(data)
|
||||||
|
.await
|
||||||
|
.map(|o| o.ok_or(Error::NotFound.into()))?
|
||||||
|
} else {
|
||||||
|
self.dereference_from_http(data, None).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Fetch an object from the local db. Instead of falling back to http, this throws an error if
|
/// Fetch an object from the local db. Instead of falling back to http, this throws an error if
|
||||||
/// the object is not found in the database.
|
/// the object is not found in the database.
|
||||||
pub async fn dereference_local(
|
pub async fn dereference_local(
|
||||||
|
@ -138,6 +146,10 @@ where
|
||||||
Object::read_from_id(*id, data).await
|
Object::read_from_id(*id, data).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fetch object from origin instance over HTTP, then verify and parse it.
|
||||||
|
///
|
||||||
|
/// Uses Box::pin to wrap futures to reduce stack size and avoid stack overflow when
|
||||||
|
/// when fetching objects recursively.
|
||||||
async fn dereference_from_http(
|
async fn dereference_from_http(
|
||||||
&self,
|
&self,
|
||||||
data: &Data<<Kind as Object>::DataType>,
|
data: &Data<<Kind as Object>::DataType>,
|
||||||
|
@ -146,7 +158,7 @@ where
|
||||||
where
|
where
|
||||||
<Kind as Object>::Error: From<Error>,
|
<Kind as Object>::Error: From<Error>,
|
||||||
{
|
{
|
||||||
let res = fetch_object_http(&self.0, data).await;
|
let res = Box::pin(fetch_object_http(&self.0, data)).await;
|
||||||
|
|
||||||
if let Err(Error::ObjectDeleted(url)) = res {
|
if let Err(Error::ObjectDeleted(url)) = res {
|
||||||
if let Some(db_object) = db_object {
|
if let Some(db_object) = db_object {
|
||||||
|
@ -158,8 +170,13 @@ where
|
||||||
let res = res?;
|
let res = res?;
|
||||||
let redirect_url = &res.url;
|
let redirect_url = &res.url;
|
||||||
|
|
||||||
Kind::verify(&res.object, redirect_url, data).await?;
|
Box::pin(Kind::verify(&res.object, redirect_url, data)).await?;
|
||||||
Kind::from_json(res.object, data).await
|
Box::pin(Kind::from_json(res.object, data)).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns true if the object's domain matches the one defined in [[FederationConfig.domain]].
|
||||||
|
pub fn is_local(&self, data: &Data<<Kind as Object>::DataType>) -> bool {
|
||||||
|
data.config.is_local_url(&self.0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -167,7 +184,7 @@ where
|
||||||
impl<Kind> Clone for ObjectId<Kind>
|
impl<Kind> Clone for ObjectId<Kind>
|
||||||
where
|
where
|
||||||
Kind: Object,
|
Kind: Object,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
fn clone(&self) -> Self {
|
fn clone(&self) -> Self {
|
||||||
ObjectId(self.0.clone(), self.1)
|
ObjectId(self.0.clone(), self.1)
|
||||||
|
@ -183,9 +200,9 @@ static ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG: i64 = 20;
|
||||||
fn should_refetch_object(last_refreshed: DateTime<Utc>) -> bool {
|
fn should_refetch_object(last_refreshed: DateTime<Utc>) -> bool {
|
||||||
let update_interval = if cfg!(debug_assertions) {
|
let update_interval = if cfg!(debug_assertions) {
|
||||||
// avoid infinite loop when fetching community outbox
|
// avoid infinite loop when fetching community outbox
|
||||||
ChronoDuration::seconds(ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG)
|
ChronoDuration::try_seconds(ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG).expect("valid duration")
|
||||||
} else {
|
} else {
|
||||||
ChronoDuration::seconds(ACTOR_REFETCH_INTERVAL_SECONDS)
|
ChronoDuration::try_seconds(ACTOR_REFETCH_INTERVAL_SECONDS).expect("valid duration")
|
||||||
};
|
};
|
||||||
let refresh_limit = Utc::now() - update_interval;
|
let refresh_limit = Utc::now() - update_interval;
|
||||||
last_refreshed.lt(&refresh_limit)
|
last_refreshed.lt(&refresh_limit)
|
||||||
|
@ -194,7 +211,7 @@ fn should_refetch_object(last_refreshed: DateTime<Utc>) -> bool {
|
||||||
impl<Kind> Display for ObjectId<Kind>
|
impl<Kind> Display for ObjectId<Kind>
|
||||||
where
|
where
|
||||||
Kind: Object,
|
Kind: Object,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||||
write!(f, "{}", self.0.as_str())
|
write!(f, "{}", self.0.as_str())
|
||||||
|
@ -204,7 +221,7 @@ where
|
||||||
impl<Kind> Debug for ObjectId<Kind>
|
impl<Kind> Debug for ObjectId<Kind>
|
||||||
where
|
where
|
||||||
Kind: Object,
|
Kind: Object,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||||
write!(f, "{}", self.0.as_str())
|
write!(f, "{}", self.0.as_str())
|
||||||
|
@ -214,7 +231,7 @@ where
|
||||||
impl<Kind> From<ObjectId<Kind>> for Url
|
impl<Kind> From<ObjectId<Kind>> for Url
|
||||||
where
|
where
|
||||||
Kind: Object,
|
Kind: Object,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
fn from(id: ObjectId<Kind>) -> Self {
|
fn from(id: ObjectId<Kind>) -> Self {
|
||||||
*id.0
|
*id.0
|
||||||
|
@ -224,7 +241,7 @@ where
|
||||||
impl<Kind> From<Url> for ObjectId<Kind>
|
impl<Kind> From<Url> for ObjectId<Kind>
|
||||||
where
|
where
|
||||||
Kind: Object + Send + 'static,
|
Kind: Object + Send + 'static,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
fn from(url: Url) -> Self {
|
fn from(url: Url) -> Self {
|
||||||
ObjectId(Box::new(url), PhantomData::<Kind>)
|
ObjectId(Box::new(url), PhantomData::<Kind>)
|
||||||
|
@ -234,17 +251,107 @@ where
|
||||||
impl<Kind> PartialEq for ObjectId<Kind>
|
impl<Kind> PartialEq for ObjectId<Kind>
|
||||||
where
|
where
|
||||||
Kind: Object,
|
Kind: Object,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
{
|
{
|
||||||
fn eq(&self, other: &Self) -> bool {
|
fn eq(&self, other: &Self) -> bool {
|
||||||
self.0.eq(&other.0) && self.1 == other.1
|
self.0.eq(&other.0) && self.1 == other.1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "diesel")]
|
||||||
|
const _IMPL_DIESEL_NEW_TYPE_FOR_OBJECT_ID: () = {
|
||||||
|
use diesel::{
|
||||||
|
backend::Backend,
|
||||||
|
deserialize::{FromSql, FromStaticSqlRow},
|
||||||
|
expression::AsExpression,
|
||||||
|
internal::derives::as_expression::Bound,
|
||||||
|
pg::Pg,
|
||||||
|
query_builder::QueryId,
|
||||||
|
serialize,
|
||||||
|
serialize::{Output, ToSql},
|
||||||
|
sql_types::{HasSqlType, SingleValue, Text},
|
||||||
|
Expression,
|
||||||
|
Queryable,
|
||||||
|
};
|
||||||
|
|
||||||
|
// TODO: this impl only works for Postgres db because of to_string() call which requires reborrow
|
||||||
|
impl<Kind, ST> ToSql<ST, Pg> for ObjectId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Object,
|
||||||
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
|
String: ToSql<ST, Pg>,
|
||||||
|
{
|
||||||
|
fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> serialize::Result {
|
||||||
|
let v = self.0.to_string();
|
||||||
|
<String as ToSql<Text, Pg>>::to_sql(&v, &mut out.reborrow())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<'expr, Kind, ST> AsExpression<ST> for &'expr ObjectId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Object,
|
||||||
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
|
Bound<ST, String>: Expression<SqlType = ST>,
|
||||||
|
ST: SingleValue,
|
||||||
|
{
|
||||||
|
type Expression = Bound<ST, &'expr str>;
|
||||||
|
fn as_expression(self) -> Self::Expression {
|
||||||
|
Bound::new(self.0.as_str())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<Kind, ST> AsExpression<ST> for ObjectId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Object,
|
||||||
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
|
Bound<ST, String>: Expression<SqlType = ST>,
|
||||||
|
ST: SingleValue,
|
||||||
|
{
|
||||||
|
type Expression = Bound<ST, String>;
|
||||||
|
fn as_expression(self) -> Self::Expression {
|
||||||
|
Bound::new(self.0.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<Kind, ST, DB> FromSql<ST, DB> for ObjectId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Object + Send + 'static,
|
||||||
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
|
String: FromSql<ST, DB>,
|
||||||
|
DB: Backend,
|
||||||
|
DB: HasSqlType<ST>,
|
||||||
|
{
|
||||||
|
fn from_sql(
|
||||||
|
raw: DB::RawValue<'_>,
|
||||||
|
) -> Result<Self, Box<dyn ::std::error::Error + Send + Sync>> {
|
||||||
|
let string: String = FromSql::<ST, DB>::from_sql(raw)?;
|
||||||
|
Ok(ObjectId::parse(&string)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<Kind, ST, DB> Queryable<ST, DB> for ObjectId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Object + Send + 'static,
|
||||||
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
|
String: FromStaticSqlRow<ST, DB>,
|
||||||
|
DB: Backend,
|
||||||
|
DB: HasSqlType<ST>,
|
||||||
|
{
|
||||||
|
type Row = String;
|
||||||
|
fn build(row: Self::Row) -> diesel::deserialize::Result<Self> {
|
||||||
|
Ok(ObjectId::parse(&row)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<Kind> QueryId for ObjectId<Kind>
|
||||||
|
where
|
||||||
|
Kind: Object + 'static,
|
||||||
|
for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
|
||||||
|
{
|
||||||
|
type QueryId = Self;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
pub mod tests {
|
pub mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::{fetch::object_id::should_refetch_object, traits::tests::DbUser};
|
use crate::traits::tests::DbUser;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_deserialize() {
|
fn test_deserialize() {
|
||||||
|
@ -259,10 +366,10 @@ pub mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_should_refetch_object() {
|
fn test_should_refetch_object() {
|
||||||
let one_second_ago = Utc::now() - ChronoDuration::seconds(1);
|
let one_second_ago = Utc::now() - ChronoDuration::try_seconds(1).unwrap();
|
||||||
assert!(!should_refetch_object(one_second_ago));
|
assert!(!should_refetch_object(one_second_ago));
|
||||||
|
|
||||||
let two_days_ago = Utc::now() - ChronoDuration::days(2);
|
let two_days_ago = Utc::now() - ChronoDuration::try_days(2).unwrap();
|
||||||
assert!(should_refetch_object(two_days_ago));
|
assert!(should_refetch_object(two_days_ago));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,17 +1,42 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
config::Data,
|
config::Data,
|
||||||
error::{Error, Error::WebfingerResolveFailed},
|
error::Error,
|
||||||
fetch::{fetch_object_http_with_accept, object_id::ObjectId},
|
fetch::{fetch_object_http_with_accept, object_id::ObjectId},
|
||||||
traits::{Actor, Object},
|
traits::{Actor, Object},
|
||||||
FEDERATION_CONTENT_TYPE,
|
FEDERATION_CONTENT_TYPE,
|
||||||
};
|
};
|
||||||
|
use http::HeaderValue;
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
|
use once_cell::sync::Lazy;
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::HashMap;
|
use std::{collections::HashMap, fmt::Display};
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
|
/// Errors relative to webfinger handling
|
||||||
|
#[derive(thiserror::Error, Debug)]
|
||||||
|
pub enum WebFingerError {
|
||||||
|
/// The webfinger identifier is invalid
|
||||||
|
#[error("The webfinger identifier is invalid")]
|
||||||
|
WrongFormat,
|
||||||
|
/// The webfinger identifier doesn't match the expected instance domain name
|
||||||
|
#[error("The webfinger identifier doesn't match the expected instance domain name")]
|
||||||
|
WrongDomain,
|
||||||
|
/// The wefinger object did not contain any link to an activitypub item
|
||||||
|
#[error("The webfinger object did not contain any link to an activitypub item")]
|
||||||
|
NoValidLink,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebFingerError {
|
||||||
|
fn into_crate_error(self) -> Error {
|
||||||
|
self.into()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The content-type for webfinger responses.
|
||||||
|
pub static WEBFINGER_CONTENT_TYPE: HeaderValue = HeaderValue::from_static("application/jrd+json");
|
||||||
|
|
||||||
/// Takes an identifier of the form `name@example.com`, and returns an object of `Kind`.
|
/// Takes an identifier of the form `name@example.com`, and returns an object of `Kind`.
|
||||||
///
|
///
|
||||||
/// For this the identifier is first resolved via webfinger protocol to an Activitypub ID. This ID
|
/// For this the identifier is first resolved via webfinger protocol to an Activitypub ID. This ID
|
||||||
|
@ -23,12 +48,12 @@ pub async fn webfinger_resolve_actor<T: Clone, Kind>(
|
||||||
where
|
where
|
||||||
Kind: Object + Actor + Send + 'static + Object<DataType = T>,
|
Kind: Object + Actor + Send + 'static + Object<DataType = T>,
|
||||||
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
|
||||||
<Kind as Object>::Error: From<crate::error::Error> + Send + Sync,
|
<Kind as Object>::Error: From<crate::error::Error> + Send + Sync + Display,
|
||||||
{
|
{
|
||||||
let (_, domain) = identifier
|
let (_, domain) = identifier
|
||||||
.splitn(2, '@')
|
.splitn(2, '@')
|
||||||
.collect_tuple()
|
.collect_tuple()
|
||||||
.ok_or(WebfingerResolveFailed)?;
|
.ok_or(WebFingerError::WrongFormat.into_crate_error())?;
|
||||||
let protocol = if data.config.debug { "http" } else { "https" };
|
let protocol = if data.config.debug { "http" } else { "https" };
|
||||||
let fetch_url =
|
let fetch_url =
|
||||||
format!("{protocol}://{domain}/.well-known/webfinger?resource=acct:{identifier}");
|
format!("{protocol}://{domain}/.well-known/webfinger?resource=acct:{identifier}");
|
||||||
|
@ -37,7 +62,7 @@ where
|
||||||
let res: Webfinger = fetch_object_http_with_accept(
|
let res: Webfinger = fetch_object_http_with_accept(
|
||||||
&Url::parse(&fetch_url).map_err(Error::UrlParse)?,
|
&Url::parse(&fetch_url).map_err(Error::UrlParse)?,
|
||||||
data,
|
data,
|
||||||
"application/jrd+json",
|
&WEBFINGER_CONTENT_TYPE,
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
.object;
|
.object;
|
||||||
|
@ -55,13 +80,15 @@ where
|
||||||
})
|
})
|
||||||
.filter_map(|l| l.href.clone())
|
.filter_map(|l| l.href.clone())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
for l in links {
|
for l in links {
|
||||||
let object = ObjectId::<Kind>::from(l).dereference(data).await;
|
let object = ObjectId::<Kind>::from(l).dereference(data).await;
|
||||||
if object.is_ok() {
|
match object {
|
||||||
return object;
|
Ok(obj) => return Ok(obj),
|
||||||
|
Err(error) => debug!(%error, "Failed to dereference link"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(WebfingerResolveFailed.into())
|
Err(WebFingerError::NoValidLink.into_crate_error().into())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extracts username from a webfinger resource parameter.
|
/// Extracts username from a webfinger resource parameter.
|
||||||
|
@ -89,22 +116,24 @@ where
|
||||||
/// # Ok::<(), anyhow::Error>(())
|
/// # Ok::<(), anyhow::Error>(())
|
||||||
/// }).unwrap();
|
/// }).unwrap();
|
||||||
///```
|
///```
|
||||||
pub fn extract_webfinger_name<T>(query: &str, data: &Data<T>) -> Result<String, Error>
|
pub fn extract_webfinger_name<'i, T>(query: &'i str, data: &Data<T>) -> Result<&'i str, Error>
|
||||||
where
|
where
|
||||||
T: Clone,
|
T: Clone,
|
||||||
{
|
{
|
||||||
|
static WEBFINGER_REGEX: Lazy<Regex> =
|
||||||
|
Lazy::new(|| Regex::new(r"^acct:([\p{L}0-9_\.\-]+)@(.*)$").expect("compile regex"));
|
||||||
// Regex to extract usernames from webfinger query. Supports different alphabets using `\p{L}`.
|
// Regex to extract usernames from webfinger query. Supports different alphabets using `\p{L}`.
|
||||||
// TODO: would be nice if we could implement this without regex and remove the dependency
|
// TODO: This should use a URL parser
|
||||||
let result = Regex::new(&format!(r"^acct:([\p{{L}}0-9_]+)@{}$", data.domain()))
|
let captures = WEBFINGER_REGEX
|
||||||
.map_err(|_| Error::WebfingerRegexFailed)
|
.captures(query)
|
||||||
.and_then(|regex| {
|
.ok_or(WebFingerError::WrongFormat)?;
|
||||||
regex
|
|
||||||
.captures(query)
|
|
||||||
.and_then(|c| c.get(1))
|
|
||||||
.ok_or_else(|| Error::WebfingerRegexFailed)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
return Ok(result.as_str().to_string());
|
let account_name = captures.get(1).ok_or(WebFingerError::WrongFormat)?;
|
||||||
|
|
||||||
|
if captures.get(2).map(|m| m.as_str()) != Some(data.domain()) {
|
||||||
|
return Err(WebFingerError::WrongDomain.into());
|
||||||
|
}
|
||||||
|
Ok(account_name.as_str())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Builds a basic webfinger response for the actor.
|
/// Builds a basic webfinger response for the actor.
|
||||||
|
@ -216,6 +245,7 @@ pub struct WebfingerLink {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::{
|
use crate::{
|
||||||
|
@ -234,8 +264,6 @@ mod tests {
|
||||||
let data = config.to_request_data();
|
let data = config.to_request_data();
|
||||||
|
|
||||||
webfinger_resolve_actor::<DbConnection, DbUser>("LemmyDev@mastodon.social", &data).await?;
|
webfinger_resolve_actor::<DbConnection, DbUser>("LemmyDev@mastodon.social", &data).await?;
|
||||||
// poa.st is as of 2023-07-14 the largest Pleroma instance
|
|
||||||
webfinger_resolve_actor::<DbConnection, DbUser>("graf@poa.st", &data).await?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -252,15 +280,23 @@ mod tests {
|
||||||
request_counter: Default::default(),
|
request_counter: Default::default(),
|
||||||
};
|
};
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
Ok("test123".to_string()),
|
Ok("test123"),
|
||||||
extract_webfinger_name("acct:test123@example.com", &data)
|
extract_webfinger_name("acct:test123@example.com", &data)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
Ok("Владимир".to_string()),
|
Ok("Владимир"),
|
||||||
extract_webfinger_name("acct:Владимир@example.com", &data)
|
extract_webfinger_name("acct:Владимир@example.com", &data)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
Ok("تجريب".to_string()),
|
Ok("example.com"),
|
||||||
|
extract_webfinger_name("acct:example.com@example.com", &data)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
Ok("da-sh"),
|
||||||
|
extract_webfinger_name("acct:da-sh@example.com", &data)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
Ok("تجريب"),
|
||||||
extract_webfinger_name("acct:تجريب@example.com", &data)
|
extract_webfinger_name("acct:تجريب@example.com", &data)
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
//! Generating keypairs, creating and verifying signatures
|
//! Generating keypairs, creating and verifying signatures
|
||||||
//!
|
//!
|
||||||
//! Signature creation and verification is handled internally in the library. See
|
//! Signature creation and verification is handled internally in the library. See
|
||||||
//! [send_activity](crate::activity_sending::send_activity) and
|
//! [send_activity](crate::activity_sending::SendActivityTask::sign_and_send) and
|
||||||
//! [receive_activity (actix-web)](crate::actix_web::inbox::receive_activity) /
|
//! [receive_activity (actix-web)](crate::actix_web::inbox::receive_activity) /
|
||||||
//! [receive_activity (axum)](crate::axum::inbox::receive_activity).
|
//! [receive_activity (axum)](crate::axum::inbox::receive_activity).
|
||||||
|
|
||||||
|
@ -15,19 +15,22 @@ use crate::{
|
||||||
use base64::{engine::general_purpose::STANDARD as Base64, Engine};
|
use base64::{engine::general_purpose::STANDARD as Base64, Engine};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http::{header::HeaderName, uri::PathAndQuery, HeaderValue, Method, Uri};
|
use http::{header::HeaderName, uri::PathAndQuery, HeaderValue, Method, Uri};
|
||||||
use http_signature_normalization_reqwest::prelude::{Config, SignExt};
|
use http_signature_normalization_reqwest::{
|
||||||
use once_cell::sync::Lazy;
|
prelude::{Config, SignExt},
|
||||||
use openssl::{
|
DefaultSpawner,
|
||||||
hash::MessageDigest,
|
|
||||||
pkey::{PKey, Private},
|
|
||||||
rsa::Rsa,
|
|
||||||
sign::{Signer, Verifier},
|
|
||||||
};
|
};
|
||||||
|
use once_cell::sync::Lazy;
|
||||||
use reqwest::Request;
|
use reqwest::Request;
|
||||||
use reqwest_middleware::RequestBuilder;
|
use reqwest_middleware::RequestBuilder;
|
||||||
|
use rsa::{
|
||||||
|
pkcs8::{DecodePublicKey, EncodePrivateKey, EncodePublicKey, LineEnding},
|
||||||
|
Pkcs1v15Sign,
|
||||||
|
RsaPrivateKey,
|
||||||
|
RsaPublicKey,
|
||||||
|
};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::{collections::BTreeMap, fmt::Debug, io::ErrorKind, time::Duration};
|
use std::{collections::BTreeMap, fmt::Debug, time::Duration};
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
|
@ -43,27 +46,23 @@ pub struct Keypair {
|
||||||
impl Keypair {
|
impl Keypair {
|
||||||
/// Helper method to turn this into an openssl private key
|
/// Helper method to turn this into an openssl private key
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) fn private_key(&self) -> Result<PKey<Private>, anyhow::Error> {
|
pub(crate) fn private_key(&self) -> Result<RsaPrivateKey, anyhow::Error> {
|
||||||
Ok(PKey::private_key_from_pem(self.private_key.as_bytes())?)
|
use rsa::pkcs8::DecodePrivateKey;
|
||||||
|
|
||||||
|
Ok(RsaPrivateKey::from_pkcs8_pem(&self.private_key)?)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Generate a random asymmetric keypair for ActivityPub HTTP signatures.
|
/// Generate a random asymmetric keypair for ActivityPub HTTP signatures.
|
||||||
pub fn generate_actor_keypair() -> Result<Keypair, std::io::Error> {
|
pub fn generate_actor_keypair() -> Result<Keypair, Error> {
|
||||||
let rsa = Rsa::generate(2048)?;
|
let mut rng = rand::thread_rng();
|
||||||
let pkey = PKey::from_rsa(rsa)?;
|
let rsa = RsaPrivateKey::new(&mut rng, 2048)?;
|
||||||
let public_key = pkey.public_key_to_pem()?;
|
let pkey = RsaPublicKey::from(&rsa);
|
||||||
let private_key = pkey.private_key_to_pem_pkcs8()?;
|
let public_key = pkey.to_public_key_pem(LineEnding::default())?;
|
||||||
let key_to_string = |key| match String::from_utf8(key) {
|
let private_key = rsa.to_pkcs8_pem(LineEnding::default())?.to_string();
|
||||||
Ok(s) => Ok(s),
|
|
||||||
Err(e) => Err(std::io::Error::new(
|
|
||||||
ErrorKind::Other,
|
|
||||||
format!("Failed converting key to string: {}", e),
|
|
||||||
)),
|
|
||||||
};
|
|
||||||
Ok(Keypair {
|
Ok(Keypair {
|
||||||
private_key: key_to_string(private_key)?,
|
private_key,
|
||||||
public_key: key_to_string(public_key)?,
|
public_key,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -80,10 +79,11 @@ pub(crate) async fn sign_request(
|
||||||
request_builder: RequestBuilder,
|
request_builder: RequestBuilder,
|
||||||
actor_id: &Url,
|
actor_id: &Url,
|
||||||
activity: Bytes,
|
activity: Bytes,
|
||||||
private_key: PKey<Private>,
|
private_key: RsaPrivateKey,
|
||||||
http_signature_compat: bool,
|
http_signature_compat: bool,
|
||||||
) -> Result<Request, Error> {
|
) -> Result<Request, Error> {
|
||||||
static CONFIG: Lazy<Config> = Lazy::new(|| Config::new().set_expiration(EXPIRES_AFTER));
|
static CONFIG: Lazy<Config<DefaultSpawner>> =
|
||||||
|
Lazy::new(|| Config::new().set_expiration(EXPIRES_AFTER));
|
||||||
static CONFIG_COMPAT: Lazy<Config> = Lazy::new(|| {
|
static CONFIG_COMPAT: Lazy<Config> = Lazy::new(|| {
|
||||||
Config::new()
|
Config::new()
|
||||||
.mastodon_compat()
|
.mastodon_compat()
|
||||||
|
@ -102,10 +102,10 @@ pub(crate) async fn sign_request(
|
||||||
Sha256::new(),
|
Sha256::new(),
|
||||||
activity,
|
activity,
|
||||||
move |signing_string| {
|
move |signing_string| {
|
||||||
let mut signer = Signer::new(MessageDigest::sha256(), &private_key)?;
|
Ok(Base64.encode(private_key.sign(
|
||||||
signer.update(signing_string.as_bytes())?;
|
Pkcs1v15Sign::new::<Sha256>(),
|
||||||
|
&Sha256::digest(signing_string.as_bytes()),
|
||||||
Ok(Base64.encode(signer.sign_to_vec()?)) as Result<_, Error>
|
)?)) as Result<_, Error>
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
@ -185,8 +185,11 @@ fn verify_signature_inner(
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
public_key: &str,
|
public_key: &str,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
static CONFIG: Lazy<http_signature_normalization::Config> =
|
static CONFIG: Lazy<http_signature_normalization::Config> = Lazy::new(|| {
|
||||||
Lazy::new(|| http_signature_normalization::Config::new().set_expiration(EXPIRES_AFTER));
|
http_signature_normalization::Config::new()
|
||||||
|
.set_expiration(EXPIRES_AFTER)
|
||||||
|
.require_digest()
|
||||||
|
});
|
||||||
|
|
||||||
let path_and_query = uri.path_and_query().map(PathAndQuery::as_str).unwrap_or("");
|
let path_and_query = uri.path_and_query().map(PathAndQuery::as_str).unwrap_or("");
|
||||||
|
|
||||||
|
@ -198,15 +201,19 @@ fn verify_signature_inner(
|
||||||
"Verifying with key {}, message {}",
|
"Verifying with key {}, message {}",
|
||||||
&public_key, &signing_string
|
&public_key, &signing_string
|
||||||
);
|
);
|
||||||
let public_key = PKey::public_key_from_pem(public_key.as_bytes())?;
|
let public_key = RsaPublicKey::from_public_key_pem(public_key)?;
|
||||||
let mut verifier = Verifier::new(MessageDigest::sha256(), &public_key)?;
|
|
||||||
verifier.update(signing_string.as_bytes())?;
|
|
||||||
|
|
||||||
let base64_decoded = Base64
|
let base64_decoded = Base64
|
||||||
.decode(signature)
|
.decode(signature)
|
||||||
.map_err(|err| Error::Other(err.to_string()))?;
|
.map_err(|err| Error::Other(err.to_string()))?;
|
||||||
|
|
||||||
Ok(verifier.verify(&base64_decoded)?)
|
Ok(public_key
|
||||||
|
.verify(
|
||||||
|
Pkcs1v15Sign::new::<Sha256>(),
|
||||||
|
&Sha256::digest(signing_string.as_bytes()),
|
||||||
|
&base64_decoded,
|
||||||
|
)
|
||||||
|
.is_ok())
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if verified {
|
if verified {
|
||||||
|
@ -271,11 +278,13 @@ pub(crate) fn verify_body_hash(
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(clippy::unwrap_used)]
|
||||||
pub mod test {
|
pub mod test {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::activity_sending::generate_request_headers;
|
use crate::activity_sending::generate_request_headers;
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
use reqwest_middleware::ClientWithMiddleware;
|
use reqwest_middleware::ClientWithMiddleware;
|
||||||
|
use rsa::{pkcs1::DecodeRsaPrivateKey, pkcs8::DecodePrivateKey};
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
|
|
||||||
static ACTOR_ID: Lazy<Url> = Lazy::new(|| Url::parse("https://example.com/u/alice").unwrap());
|
static ACTOR_ID: Lazy<Url> = Lazy::new(|| Url::parse("https://example.com/u/alice").unwrap());
|
||||||
|
@ -298,7 +307,7 @@ pub mod test {
|
||||||
request_builder,
|
request_builder,
|
||||||
&ACTOR_ID,
|
&ACTOR_ID,
|
||||||
"my activity".into(),
|
"my activity".into(),
|
||||||
PKey::private_key_from_pem(test_keypair().private_key.as_bytes()).unwrap(),
|
RsaPrivateKey::from_pkcs8_pem(&test_keypair().private_key).unwrap(),
|
||||||
// set this to prevent created/expires headers to be generated and inserted
|
// set this to prevent created/expires headers to be generated and inserted
|
||||||
// automatically from current time
|
// automatically from current time
|
||||||
true,
|
true,
|
||||||
|
@ -334,7 +343,7 @@ pub mod test {
|
||||||
request_builder,
|
request_builder,
|
||||||
&ACTOR_ID,
|
&ACTOR_ID,
|
||||||
"my activity".to_string().into(),
|
"my activity".to_string().into(),
|
||||||
PKey::private_key_from_pem(test_keypair().private_key.as_bytes()).unwrap(),
|
RsaPrivateKey::from_pkcs8_pem(&test_keypair().private_key).unwrap(),
|
||||||
false,
|
false,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
@ -370,13 +379,13 @@ pub mod test {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn test_keypair() -> Keypair {
|
pub fn test_keypair() -> Keypair {
|
||||||
let rsa = Rsa::private_key_from_pem(PRIVATE_KEY.as_bytes()).unwrap();
|
let rsa = RsaPrivateKey::from_pkcs1_pem(PRIVATE_KEY).unwrap();
|
||||||
let pkey = PKey::from_rsa(rsa).unwrap();
|
let pkey = RsaPublicKey::from(&rsa);
|
||||||
let private_key = pkey.private_key_to_pem_pkcs8().unwrap();
|
let public_key = pkey.to_public_key_pem(LineEnding::default()).unwrap();
|
||||||
let public_key = pkey.public_key_to_pem().unwrap();
|
let private_key = rsa.to_pkcs8_pem(LineEnding::default()).unwrap().to_string();
|
||||||
Keypair {
|
Keypair {
|
||||||
private_key: String::from_utf8(private_key).unwrap(),
|
private_key,
|
||||||
public_key: String::from_utf8(public_key).unwrap(),
|
public_key,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
47
src/lib.rs
47
src/lib.rs
|
@ -10,6 +10,7 @@
|
||||||
#![doc = include_str!("../docs/10_fetching_objects_with_unknown_type.md")]
|
#![doc = include_str!("../docs/10_fetching_objects_with_unknown_type.md")]
|
||||||
#![deny(missing_docs)]
|
#![deny(missing_docs)]
|
||||||
|
|
||||||
|
pub mod activity_queue;
|
||||||
pub mod activity_sending;
|
pub mod activity_sending;
|
||||||
#[cfg(feature = "actix-web")]
|
#[cfg(feature = "actix-web")]
|
||||||
pub mod actix_web;
|
pub mod actix_web;
|
||||||
|
@ -23,7 +24,51 @@ pub mod protocol;
|
||||||
pub(crate) mod reqwest_shim;
|
pub(crate) mod reqwest_shim;
|
||||||
pub mod traits;
|
pub mod traits;
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
config::Data,
|
||||||
|
error::Error,
|
||||||
|
fetch::object_id::ObjectId,
|
||||||
|
traits::{ActivityHandler, Actor, Object},
|
||||||
|
};
|
||||||
pub use activitystreams_kinds as kinds;
|
pub use activitystreams_kinds as kinds;
|
||||||
|
|
||||||
|
use serde::{de::DeserializeOwned, Deserialize};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
/// Mime type for Activitypub data, used for `Accept` and `Content-Type` HTTP headers
|
/// Mime type for Activitypub data, used for `Accept` and `Content-Type` HTTP headers
|
||||||
pub static FEDERATION_CONTENT_TYPE: &str = "application/activity+json";
|
pub const FEDERATION_CONTENT_TYPE: &str = "application/activity+json";
|
||||||
|
|
||||||
|
/// Deserialize incoming inbox activity to the given type, perform basic
|
||||||
|
/// validation and extract the actor.
|
||||||
|
async fn parse_received_activity<Activity, ActorT, Datatype>(
|
||||||
|
body: &[u8],
|
||||||
|
data: &Data<Datatype>,
|
||||||
|
) -> Result<(Activity, ActorT), <Activity as ActivityHandler>::Error>
|
||||||
|
where
|
||||||
|
Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
|
||||||
|
ActorT: Object<DataType = Datatype> + Actor + Send + 'static,
|
||||||
|
for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
|
||||||
|
<Activity as ActivityHandler>::Error: From<Error> + From<<ActorT as Object>::Error>,
|
||||||
|
<ActorT as Object>::Error: From<Error>,
|
||||||
|
Datatype: Clone,
|
||||||
|
{
|
||||||
|
let activity: Activity = serde_json::from_slice(body).map_err(|e| {
|
||||||
|
// Attempt to include activity id in error message
|
||||||
|
let id = extract_id(body).ok();
|
||||||
|
Error::ParseReceivedActivity(e, id)
|
||||||
|
})?;
|
||||||
|
data.config.verify_url_and_domain(&activity).await?;
|
||||||
|
let actor = ObjectId::<ActorT>::from(activity.actor().clone())
|
||||||
|
.dereference(data)
|
||||||
|
.await?;
|
||||||
|
Ok((activity, actor))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Attempt to parse id field from serialized json
|
||||||
|
fn extract_id(data: &[u8]) -> serde_json::Result<Url> {
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct Id {
|
||||||
|
id: Url,
|
||||||
|
}
|
||||||
|
Ok(serde_json::from_slice::<Id>(data)?.id)
|
||||||
|
}
|
||||||
|
|
|
@ -15,11 +15,11 @@
|
||||||
//! };
|
//! };
|
||||||
//! let note_with_context = WithContext::new_default(note);
|
//! let note_with_context = WithContext::new_default(note);
|
||||||
//! let serialized = serde_json::to_string(¬e_with_context)?;
|
//! let serialized = serde_json::to_string(¬e_with_context)?;
|
||||||
//! assert_eq!(serialized, r#"{"@context":["https://www.w3.org/ns/activitystreams"],"content":"Hello world"}"#);
|
//! assert_eq!(serialized, r#"{"@context":"https://www.w3.org/ns/activitystreams","content":"Hello world"}"#);
|
||||||
//! Ok::<(), serde_json::error::Error>(())
|
//! Ok::<(), serde_json::error::Error>(())
|
||||||
//! ```
|
//! ```
|
||||||
|
|
||||||
use crate::{config::Data, protocol::helpers::deserialize_one_or_many, traits::ActivityHandler};
|
use crate::{config::Data, traits::ActivityHandler};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
@ -31,8 +31,7 @@ const DEFAULT_CONTEXT: &str = "https://www.w3.org/ns/activitystreams";
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
#[derive(Serialize, Deserialize, Debug)]
|
||||||
pub struct WithContext<T> {
|
pub struct WithContext<T> {
|
||||||
#[serde(rename = "@context")]
|
#[serde(rename = "@context")]
|
||||||
#[serde(deserialize_with = "deserialize_one_or_many")]
|
context: Value,
|
||||||
context: Vec<Value>,
|
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
inner: T,
|
inner: T,
|
||||||
}
|
}
|
||||||
|
@ -40,12 +39,12 @@ pub struct WithContext<T> {
|
||||||
impl<T> WithContext<T> {
|
impl<T> WithContext<T> {
|
||||||
/// Create a new wrapper with the default Activitypub context.
|
/// Create a new wrapper with the default Activitypub context.
|
||||||
pub fn new_default(inner: T) -> WithContext<T> {
|
pub fn new_default(inner: T) -> WithContext<T> {
|
||||||
let context = vec![Value::String(DEFAULT_CONTEXT.to_string())];
|
let context = Value::String(DEFAULT_CONTEXT.to_string());
|
||||||
WithContext::new(inner, context)
|
WithContext::new(inner, context)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create new wrapper with custom context. Use this in case you are implementing extensions.
|
/// Create new wrapper with custom context. Use this in case you are implementing extensions.
|
||||||
pub fn new(inner: T, context: Vec<Value>) -> WithContext<T> {
|
pub fn new(inner: T, context: Value) -> WithContext<T> {
|
||||||
WithContext { context, inner }
|
WithContext { context, inner }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -56,12 +56,12 @@ where
|
||||||
/// #[derive(serde::Deserialize)]
|
/// #[derive(serde::Deserialize)]
|
||||||
/// struct Note {
|
/// struct Note {
|
||||||
/// #[serde(deserialize_with = "deserialize_one")]
|
/// #[serde(deserialize_with = "deserialize_one")]
|
||||||
/// to: Url
|
/// to: [Url; 1]
|
||||||
/// }
|
/// }
|
||||||
///
|
///
|
||||||
/// let note = serde_json::from_str::<Note>(r#"{"to": ["https://example.com/u/alice"] }"#);
|
/// let note = serde_json::from_str::<Note>(r#"{"to": ["https://example.com/u/alice"] }"#);
|
||||||
/// assert!(note.is_ok());
|
/// assert!(note.is_ok());
|
||||||
pub fn deserialize_one<'de, T, D>(deserializer: D) -> Result<T, D::Error>
|
pub fn deserialize_one<'de, T, D>(deserializer: D) -> Result<[T; 1], D::Error>
|
||||||
where
|
where
|
||||||
T: Deserialize<'de>,
|
T: Deserialize<'de>,
|
||||||
D: Deserializer<'de>,
|
D: Deserializer<'de>,
|
||||||
|
@ -75,8 +75,8 @@ where
|
||||||
|
|
||||||
let result: MaybeArray<T> = Deserialize::deserialize(deserializer)?;
|
let result: MaybeArray<T> = Deserialize::deserialize(deserializer)?;
|
||||||
Ok(match result {
|
Ok(match result {
|
||||||
MaybeArray::Simple(value) => value,
|
MaybeArray::Simple(value) => [value],
|
||||||
MaybeArray::Array([value]) => value,
|
MaybeArray::Array([value]) => [value],
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -125,7 +125,7 @@ mod tests {
|
||||||
#[derive(serde::Deserialize)]
|
#[derive(serde::Deserialize)]
|
||||||
struct Note {
|
struct Note {
|
||||||
#[serde(deserialize_with = "deserialize_one")]
|
#[serde(deserialize_with = "deserialize_one")]
|
||||||
_to: Url,
|
_to: [Url; 1],
|
||||||
}
|
}
|
||||||
|
|
||||||
let note = serde_json::from_str::<Note>(
|
let note = serde_json::from_str::<Note>(
|
||||||
|
|
|
@ -3,10 +3,8 @@ use bytes::{BufMut, Bytes, BytesMut};
|
||||||
use futures_core::{ready, stream::BoxStream, Stream};
|
use futures_core::{ready, stream::BoxStream, Stream};
|
||||||
use pin_project_lite::pin_project;
|
use pin_project_lite::pin_project;
|
||||||
use reqwest::Response;
|
use reqwest::Response;
|
||||||
use serde::de::DeserializeOwned;
|
|
||||||
use std::{
|
use std::{
|
||||||
future::Future,
|
future::Future,
|
||||||
marker::PhantomData,
|
|
||||||
mem,
|
mem,
|
||||||
pin::Pin,
|
pin::Pin,
|
||||||
task::{Context, Poll},
|
task::{Context, Poll},
|
||||||
|
@ -46,27 +44,6 @@ impl Future for BytesFuture {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pin_project! {
|
|
||||||
pub struct JsonFuture<T> {
|
|
||||||
_t: PhantomData<T>,
|
|
||||||
#[pin]
|
|
||||||
future: BytesFuture,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T> Future for JsonFuture<T>
|
|
||||||
where
|
|
||||||
T: DeserializeOwned,
|
|
||||||
{
|
|
||||||
type Output = Result<T, Error>;
|
|
||||||
|
|
||||||
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
|
||||||
let this = self.project();
|
|
||||||
let bytes = ready!(this.future.poll(cx))?;
|
|
||||||
Poll::Ready(serde_json::from_slice(&bytes).map_err(Error::Json))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pin_project! {
|
pin_project! {
|
||||||
pub struct TextFuture {
|
pub struct TextFuture {
|
||||||
#[pin]
|
#[pin]
|
||||||
|
@ -94,20 +71,16 @@ impl Future for TextFuture {
|
||||||
/// TODO: Remove this shim as soon as reqwest gets support for size-limited bodies.
|
/// TODO: Remove this shim as soon as reqwest gets support for size-limited bodies.
|
||||||
pub trait ResponseExt {
|
pub trait ResponseExt {
|
||||||
type BytesFuture;
|
type BytesFuture;
|
||||||
type JsonFuture<T>;
|
|
||||||
type TextFuture;
|
type TextFuture;
|
||||||
|
|
||||||
/// Size limited version of `bytes` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
|
/// Size limited version of `bytes` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
|
||||||
fn bytes_limited(self) -> Self::BytesFuture;
|
fn bytes_limited(self) -> Self::BytesFuture;
|
||||||
/// Size limited version of `json` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
|
|
||||||
fn json_limited<T>(self) -> Self::JsonFuture<T>;
|
|
||||||
/// Size limited version of `text` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
|
/// Size limited version of `text` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
|
||||||
fn text_limited(self) -> Self::TextFuture;
|
fn text_limited(self) -> Self::TextFuture;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ResponseExt for Response {
|
impl ResponseExt for Response {
|
||||||
type BytesFuture = BytesFuture;
|
type BytesFuture = BytesFuture;
|
||||||
type JsonFuture<T> = JsonFuture<T>;
|
|
||||||
type TextFuture = TextFuture;
|
type TextFuture = TextFuture;
|
||||||
|
|
||||||
fn bytes_limited(self) -> Self::BytesFuture {
|
fn bytes_limited(self) -> Self::BytesFuture {
|
||||||
|
@ -118,13 +91,6 @@ impl ResponseExt for Response {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn json_limited<T>(self) -> Self::JsonFuture<T> {
|
|
||||||
JsonFuture {
|
|
||||||
_t: PhantomData,
|
|
||||||
future: self.bytes_limited(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn text_limited(self) -> Self::TextFuture {
|
fn text_limited(self) -> Self::TextFuture {
|
||||||
TextFuture {
|
TextFuture {
|
||||||
future: self.bytes_limited(),
|
future: self.bytes_limited(),
|
||||||
|
|
|
@ -338,12 +338,12 @@ pub trait Collection: Sized {
|
||||||
#[doc(hidden)]
|
#[doc(hidden)]
|
||||||
#[allow(clippy::unwrap_used)]
|
#[allow(clippy::unwrap_used)]
|
||||||
pub mod tests {
|
pub mod tests {
|
||||||
use super::*;
|
use super::{async_trait, ActivityHandler, Actor, Data, Debug, Object, PublicKey, Url};
|
||||||
use crate::{
|
use crate::{
|
||||||
error::Error,
|
error::Error,
|
||||||
fetch::object_id::ObjectId,
|
fetch::object_id::ObjectId,
|
||||||
http_signatures::{generate_actor_keypair, Keypair},
|
http_signatures::{generate_actor_keypair, Keypair},
|
||||||
protocol::{public_key::PublicKey, verification::verify_domains_match},
|
protocol::verification::verify_domains_match,
|
||||||
};
|
};
|
||||||
use activitystreams_kinds::{activity::FollowType, actor::PersonType};
|
use activitystreams_kinds::{activity::FollowType, actor::PersonType};
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::Lazy;
|
||||||
|
@ -356,7 +356,7 @@ pub mod tests {
|
||||||
pub async fn read_post_from_json_id<T>(&self, _: Url) -> Result<Option<T>, Error> {
|
pub async fn read_post_from_json_id<T>(&self, _: Url) -> Result<Option<T>, Error> {
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
pub async fn read_local_user(&self, _: String) -> Result<DbUser, Error> {
|
pub async fn read_local_user(&self, _: &str) -> Result<DbUser, Error> {
|
||||||
todo!()
|
todo!()
|
||||||
}
|
}
|
||||||
pub async fn upsert<T>(&self, _: &T) -> Result<(), Error> {
|
pub async fn upsert<T>(&self, _: &T) -> Result<(), Error> {
|
||||||
|
|
Loading…
Reference in a new issue