Version 0.5.6

* Make response content-type check case insensitive

For wordpress compat

* cleaner

* clippy

* fmt

* fmt

.drone.yml

@@ -1,40 +0,0 @@
----
-kind: pipeline
-name: amd64
-
-platform:
-  os: linux
-  arch: amd64
-
-steps:
-
-  - name: cargo fmt
-    image: rustdocker/rust:nightly
-    commands:
-      - /root/.cargo/bin/cargo fmt -- --check
-
-  - name: cargo check
-    image: rust:1.61-bullseye
-    commands:
-      - cargo check --all --all-targets
-
-  - name: cargo clippy
-    image: rust:1.61-bullseye
-    commands:
-      - rustup component add clippy
-      - cargo clippy --workspace --tests --all-targets --all-features -- -D warnings -D deprecated -D clippy::perf -D clippy::complexity -D clippy::dbg_macro
-      - cargo clippy --workspace -- -D clippy::unwrap_used
-
-  - name: cargo test
-    image: rust:1.61-bullseye
-    environment:
-      RUST_BACKTRACE: 1
-    commands:
-      - cargo test --workspace --no-fail-fast
-
-  - name: cargo run
-    image: rust:1.61-bullseye
-    environment:
-      RUST_BACKTRACE: 1
-    commands:
-      - cargo run -p activitypub_federation --example federation

.gitignore

@@ -1 +1,5 @@
 /target
+/.idea
+/Cargo.lock
+perf.data*
+flamegraph.svg

.idea/.gitignore

@@ -1,3 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml

.idea/activitypub-federation-rust.iml

@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="JAVA_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://$MODULE_DIR$">
-      <sourceFolder url="file://$MODULE_DIR$/derive/src" isTestSource="false" />
-      <sourceFolder url="file://$MODULE_DIR$/examples" isTestSource="false" />
-      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
-      <excludeFolder url="file://$MODULE_DIR$/target" />
-    </content>
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/activitypub-federation-rust.iml" filepath="$PROJECT_DIR$/.idea/activitypub-federation-rust.iml" />
-    </modules>
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

.woodpecker.yml

@@ -0,0 +1,56 @@
+variables:
+  - &rust_image "rust:1.77-bullseye"
+  
+steps:
+  cargo_fmt:
+    image: rustdocker/rust:nightly
+    commands:
+      - /root/.cargo/bin/cargo fmt -- --check
+    when:
+      - event: pull_request
+
+  cargo_clippy:
+    image: *rust_image
+    environment:
+      CARGO_HOME: .cargo
+    commands:
+      - rustup component add clippy
+      - cargo clippy --all-targets --all-features
+    when:
+      - event: pull_request
+
+  cargo_test:
+    image: *rust_image
+    environment:
+      CARGO_HOME: .cargo
+    commands:
+      - cargo test --all-features --no-fail-fast
+    when:
+      - event: pull_request
+
+  cargo_doc:
+    image: *rust_image
+    environment:
+      CARGO_HOME: .cargo
+    commands:
+      - cargo doc --all-features
+    when:
+      - event: pull_request
+
+  cargo_run_actix_example:
+    image: *rust_image
+    environment:
+      CARGO_HOME: .cargo
+    commands:
+      - cargo run --example local_federation actix-web
+    when:
+      - event: pull_request
+
+  cargo_run_axum_example:
+    image: *rust_image
+    environment:
+      CARGO_HOME: .cargo
+    commands:
+      - cargo run --example local_federation axum
+    when:
+      - event: pull_request

Cargo.toml

@@ -1,41 +1,110 @@
 [package]
 name = "activitypub_federation"
-version = "0.2.0"
+version = "0.5.6"
 edition = "2021"
 description = "High-level Activitypub framework"
+keywords = ["activitypub", "activitystreams", "federation", "fediverse"]
 license = "AGPL-3.0"
 repository = "https://github.com/LemmyNet/activitypub-federation-rust"
+documentation = "https://docs.rs/activitypub_federation/"
 
-[workspace]
-members = ["derive"]
+[features]
+default = ["actix-web", "axum"]
+actix-web = ["dep:actix-web"]
+axum = ["dep:axum", "dep:tower", "dep:hyper", "dep:http-body-util"]
+diesel = ["dep:diesel"]
+
+[lints.rust]
+warnings = "deny"
+deprecated = "deny"
+
+[lints.clippy]
+perf = "deny"
+complexity = "deny"
+dbg_macro  = "deny"
+inefficient_to_string = "deny"
+items-after-statements  = "deny" 
+implicit_clone = "deny"
+wildcard_imports = "deny"
+cast_lossless = "deny"
+manual_string_new = "deny"
+redundant_closure_for_method_calls = "deny"
+unwrap_used = "deny"
 
 [dependencies]
-activitypub_federation_derive = { version = "0.2.0", path = "derive" }
-chrono = { version = "0.4.19", features = ["clock"], default-features = false }
-serde = { version = "1.0.137", features = ["derive"] }
-async-trait = "0.1.56"
-url = { version = "2.2.2", features = ["serde"] }
-serde_json = { version = "1.0.81", features = ["preserve_order"] }
-anyhow = "1.0.57"
-reqwest = { version = "0.11.10", features = ["json"] }
-reqwest-middleware = "0.1.6"
-tracing = "0.1.34"
-base64 = "0.13.0"
-openssl = "0.10.40"
-once_cell = "1.12.0"
-http = "0.2.8"
-sha2 = "0.10.2"
-actix-web = { version = "4.0.1", default-features = false }
-http-signature-normalization-actix = { version = "0.6.1", default-features = false, features = ["server", "sha-2"] }
-http-signature-normalization-reqwest = { version = "0.6.0", default-features = false, features = ["sha-2", "middleware"] }
-background-jobs = "0.12.0"
-thiserror = "1.0.31"
-derive_builder = "0.11.2"
-itertools = "0.10.3"
+chrono = { version = "0.4.38", features = ["clock"], default-features = false }
+serde = { version = "1.0.200", features = ["derive"] }
+async-trait = "0.1.80"
+url = { version = "2.5.0", features = ["serde"] }
+serde_json = { version = "1.0.116", features = ["preserve_order"] }
+reqwest = { version = "0.11.27", features = ["json", "stream"] }
+reqwest-middleware = "0.2.5"
+tracing = "0.1.40"
+base64 = "0.22.1"
+openssl = "0.10.64"
+once_cell = "1.19.0"
+http = "0.2.12"
+sha2 = "0.10.8"
+thiserror = "1.0.59"
+derive_builder = "0.20.0"
+itertools = "0.12.1"
+dyn-clone = "1.0.17"
+enum_delegate = "0.2.0"
+httpdate = "1.0.3"
+http-signature-normalization-reqwest = { version = "0.10.0", default-features = false, features = [
+  "sha-2",
+  "middleware",
+  "default-spawner",
+] }
+http-signature-normalization = "0.7.0"
+bytes = "1.6.0"
+futures-core = { version = "0.3.30", default-features = false }
+pin-project-lite = "0.2.14"
+activitystreams-kinds = "0.3.0"
+regex = { version = "1.10.4", default-features = false, features = ["std", "unicode-case"] }
+tokio = { version = "1.37.0", features = [
+  "sync",
+  "rt",
+  "rt-multi-thread",
+  "time",
+] }
+diesel = { version = "2.1.6", features = ["postgres"], default-features = false, optional = true }
+futures = "0.3.30"
+moka = { version = "0.12.7", features = ["future"] }
+
+# Actix-web
+actix-web = { version = "4.5.1", default-features = false, optional = true }
+
+# Axum
+axum = { version = "0.6.20", features = [
+  "json",
+  "headers",
+], default-features = false, optional = true }
+tower = { version = "0.4.13", optional = true }
+hyper = { version = "0.14", optional = true }
+http-body-util = {version = "0.1.1", optional = true }
 
 [dev-dependencies]
-activitystreams-kinds = "0.2.1"
+anyhow = "1.0.82"
 rand = "0.8.5"
-actix-rt = "2.7.0"
-tokio = "1.19.2"
-env_logger = { version = "0.9.0", default-features = false }
+env_logger = "0.11.3"
+tower-http = { version = "0.5.2", features = ["map-request-body", "util"] }
+axum = { version = "0.6.20", features = [
+  "http1",
+  "tokio",
+  "query",
+], default-features = false }
+axum-macros = "0.3.8"
+tokio = { version = "1.37.0", features = ["full"] }
+
+[profile.dev]
+strip = "symbols"
+debug = 0
+
+[[example]]
+name = "local_federation"
+path = "examples/local_federation/main.rs"
+
+[[example]]
+name = "live_federation"
+path = "examples/live_federation/main.rs"

README.md

@@ -1,36 +1,27 @@
 Activitypub-Federation
 ===
-[![Build Status](https://cloud.drone.io/api/badges/LemmyNet/activitypub-federation-rust/status.svg)](https://cloud.drone.io/LemmyNet/activitypub-federation-rust)
 [![Crates.io](https://img.shields.io/crates/v/activitypub-federation.svg)](https://crates.io/crates/activitypub-federation)
+[![Documentation](https://shields.io/docsrs/activitypub_federation)](https://docs.rs/activitypub-federation/)
+[![Build Status](https://woodpecker.join-lemmy.org/api/badges/LemmyNet/activitypub-federation-rust/status.svg)](https://drone.join-lemmy.org/LemmyNet/activitypub-federation-rust)
 
-A high-level framework for [ActivityPub](https://www.w3.org/TR/activitypub/) federation in Rust, extracted from [Lemmy](https://join-lemmy.org/). The goal is that this library can take care of almost everything related to federation for different projects. For now it is still far away from that goal, and has many rough edges that need to be smoothed.
+<!-- be sure to keep this file in sync with docs/01_intro.md -->
 
-## Features
+A high-level framework for [ActivityPub](https://www.w3.org/TR/activitypub/) federation in Rust. The goal is to encapsulate all basic functionality, so that developers can easily use the protocol without any prior knowledge.
 
-- ObjectId type, wraps the `id` url and allows for type safe fetching of objects, both from database and HTTP
-- Queue for activity sending, handles HTTP signatures, retry with exponential backoff, all in background workers
-- Inbox for receiving activities, verifies HTTP signatures, performs other basic checks and helps with routing
-- Data structures for federation are defined by the user, not the library. This gives you maximal flexibility, and lets you accept only messages which your code can handle. Others are rejected automatically during deserialization.
-- Generic error type (unfortunately this was necessary)
-- various helpers for verification, (de)serialization, context etc
+The ActivityPub protocol is a decentralized social networking protocol. It allows web servers to exchange data using JSON over HTTP. Data can be fetched on demand, and also delivered directly to inboxes for live updates.
 
-## How to use
+While Activitypub is not in widespread use yet, is has the potential to form the basis of the next generation of social media. This is because it has a number of major advantages compared to existing platforms and alternative technologies:
 
-To get started, have a look at the example. You can also find some [ActivityPub resources in the Lemmy documentation](https://join-lemmy.org/docs/en/contributing/resources.html#activitypub-resources). If anything is unclear, please open an issue for clarification. For a more advanced implementation, take a look at the [Lemmy federation code](https://github.com/LemmyNet/lemmy/tree/main/crates/apub).
+- **Interoperability**: Imagine being able to comment under a Youtube video directly from twitter.com, and having the comment shown under the video on youtube.com. Or following a Subreddit from Facebook. Such functionality is already available on the equivalent Fediverse platforms, thanks to common usage of Activitypub.
+- **Ease of use**: From a user perspective, decentralized social media works almost identically to existing websites: a website with email and password based login. Unlike pure peer-to-peer networks, it is not necessary to handle private keys or install any local software.
+- **Open ecosystem**: All existing Fediverse software is open source, and there are no legal or bureaucratic requirements to start federating. That means anyone can create or fork federated software. In this way different software platforms can exist in the same network according to the preferences of different user groups. It is not necessary to target the lowest common denominator as with corporate social media.
+- **Censorship resistance**: Current social media platforms are under the control of a few corporations and are actively being censored as revealed by the [Twitter Files](https://jordansather.substack.com/p/running-list-of-all-twitter-files). This would be much more difficult on a federated network, as it would require the cooperation of every single instance administrator. Additionally, users who are affected by censorship can create their own websites and stay connected with the network.
+- **Low barrier to entry**: All it takes to host a federated website are a small server, a domain and a TLS certificate. All of this is easily in the reach of individual hobbyists. There is also some technical knowledge needed, but this can be avoided with managed hosting platforms.
 
-## Roadmap
+[Visit the documentation](https://docs.rs/activitypub_federation) for a full guide that explains how to create a federated project from scratch.
+
+Feel free to open an issue if you have any questions regarding this crate. You can also join the Matrix channel [#activitystreams](https://matrix.to/#/%23activitystreams:matrix.asonix.dog) for discussion about Activitypub in Rust. Additionally check out [Socialhub forum](https://socialhub.activitypub.rocks/) for general ActivityPub development.
 
-Things to work on in the future:
-- **Improve documentation and example**: Some things could probably be documented better. The example code should be simplified. where possible.
-- **Simplify generics**: The library uses a lot of generic parameters, where clauses and associated types. It should be possible to simplify them.
-- **Improve macro**: The macro is implemented very badly and doesn't have any error handling.
-- **Generate HTTP endpoints**: It would be possible to generate HTTP endpoints automatically for each actor.
-- **Support for other web frameworks**: Can be implemented using feature flags if other projects require it.
-- **Signed fetch**: JSON can only be fetched by authenticated actors, which means that fetches from blocked instances can also be blocked. In combination with the previous point, this could be handled entirely in the library.
-- **Helpers for testing**: Lemmy has a pretty useful test suite which (de)serializes json from other projects, to ensure that federation remains compatible. Helpers for this could be added to the library.
-- **[Webfinger](https://datatracker.ietf.org/doc/html/rfc7033) support**: Not part of the Activitypub standard, but often used together for user discovery.
-- **Remove request_counter from API**: It should be handled internally and not exposed. Maybe as part of `Data` struct.
-- 
 ## License
 
-Licensed under [AGPLv3](LICENSE).
+Licensed under [AGPLv3](/LICENSE).

derive/Cargo.toml

@@ -1,18 +0,0 @@
-[package]
-name = "activitypub_federation_derive"
-version = "0.2.0"
-edition = "2021"
-description = "High-level Activitypub framework"
-license = "AGPL-3.0"
-repository = "https://github.com/LemmyNet/activitypub-federation-rust"
-
-[lib]
-proc-macro = true
-
-[dev-dependencies]
-trybuild = { version = "1.0.63", features = ["diff"] }
-
-[dependencies]
-proc-macro2 = "1.0.39"
-syn = "1.0.96"
-quote = "1.0.18"

derive/src/lib.rs

@@ -1,137 +0,0 @@
-use proc_macro2::TokenStream;
-use quote::quote;
-use syn::{parse_macro_input, Data, DeriveInput, Fields::Unnamed, Ident, Variant};
-
-/// Generates implementation ActivityHandler for an enum, which looks like the following (handling
-/// all enum variants).
-///
-/// Based on this code:
-/// ```ignore
-/// #[derive(serde::Deserialize, serde::Serialize)]
-/// #[serde(untagged)]
-/// #[activity_handler(LemmyContext, LemmyError)]
-/// pub enum PersonInboxActivities {
-///  CreateNote(CreateNote),
-///  UpdateNote(UpdateNote),
-/// }
-/// ```
-/// It will generate this:
-/// ```ignore
-/// impl ActivityHandler for PersonInboxActivities {
-///     type DataType = LemmyContext;
-///     type Error = LemmyError;
-///
-///     async fn verify(
-///     &self,
-///     data: &Self::DataType,
-///     request_counter: &mut i32,
-///   ) -> Result<(), Self::Error> {
-///     match self {
-///       PersonInboxActivities::CreateNote(a) => a.verify(data, request_counter).await,
-///       PersonInboxActivities::UpdateNote(a) => a.verify(context, request_counter).await,
-///     }
-///   }
-///
-///   async fn receive(
-///   &self,
-///   data: &Self::DataType,
-///   request_counter: &mut i32,
-/// ) -> Result<(), Self::Error> {
-///     match self {
-///       PersonInboxActivities::CreateNote(a) => a.receive(data, request_counter).await,
-///       PersonInboxActivities::UpdateNote(a) => a.receive(data, request_counter).await,
-///     }
-///   }
-/// ```
-#[proc_macro_attribute]
-pub fn activity_handler(
-    attr: proc_macro::TokenStream,
-    input: proc_macro::TokenStream,
-) -> proc_macro::TokenStream {
-    let derive_input = parse_macro_input!(input as DeriveInput);
-    let derive_input2 = derive_input.clone();
-    let attr = proc_macro2::TokenStream::from(attr);
-    let mut attr = attr.into_iter();
-    let data_type = attr.next().expect("data type input");
-    let _delimiter = attr.next();
-    let error = attr.next().expect("error type input");
-
-    let enum_name = derive_input2.ident;
-
-    let (impl_generics, ty_generics, where_clause) = derive_input2.generics.split_for_impl();
-
-    let enum_variants = if let Data::Enum(d) = derive_input2.data {
-        d.variants
-    } else {
-        unimplemented!()
-    };
-
-    let impl_id = enum_variants
-        .iter()
-        .map(|v| generate_match_arm(&enum_name, v, &quote! {a.id()}));
-    let impl_actor = enum_variants
-        .iter()
-        .map(|v| generate_match_arm(&enum_name, v, &quote! {a.actor()}));
-    let body_verify = quote! {a.verify(context, request_counter).await};
-    let impl_verify = enum_variants
-        .iter()
-        .map(|v| generate_match_arm(&enum_name, v, &body_verify));
-    let body_receive = quote! {a.receive(context, request_counter).await};
-    let impl_receive = enum_variants
-        .iter()
-        .map(|v| generate_match_arm(&enum_name, v, &body_receive));
-
-    let expanded = quote! {
-        #derive_input
-        #[async_trait::async_trait(?Send)]
-        impl #impl_generics activitypub_federation::traits::ActivityHandler for #enum_name #ty_generics #where_clause {
-          type DataType = #data_type;
-          type Error = #error;
-            fn id(
-                &self,
-              ) -> &Url {
-              match self {
-                #(#impl_id)*
-              }
-            }
-            fn actor(
-              &self,
-            ) -> &Url {
-              match self {
-                #(#impl_actor)*
-              }
-            }
-            async fn verify(
-                &self,
-                context: &activitypub_federation::data::Data<Self::DataType>,
-                request_counter: &mut i32,
-              ) -> Result<(), Self::Error> {
-              match self {
-                #(#impl_verify)*
-              }
-            }
-            async fn receive(
-              self,
-              context: &activitypub_federation::data::Data<Self::DataType>,
-              request_counter: &mut i32,
-            ) -> Result<(), Self::Error> {
-              match self {
-                #(#impl_receive)*
-              }
-            }
-        }
-    };
-    expanded.into()
-}
-
-fn generate_match_arm(enum_name: &Ident, variant: &Variant, body: &TokenStream) -> TokenStream {
-    let id = &variant.ident;
-    match &variant.fields {
-        Unnamed(_) => {
-            quote! {
-              #enum_name::#id(a) => #body,
-            }
-        }
-        _ => unimplemented!(),
-    }
-}

docs/01_intro.md

@@ -0,0 +1,17 @@
+<!-- be sure to keep this file in sync with README.md -->
+
+A high-level framework for [ActivityPub](https://www.w3.org/TR/activitypub/) federation in Rust. The goal is to encapsulate all basic functionality, so that developers can easily use the protocol without any prior knowledge.
+
+The ActivityPub protocol is a decentralized social networking protocol. It allows web servers to exchange data using JSON over HTTP. Data can be fetched on demand, and also delivered directly to inboxes for live updates.
+
+While Activitypub is not in widespread use yet, is has the potential to form the basis of the next generation of social media. This is because it has a number of major advantages compared to existing platforms and alternative technologies:
+
+- **Interoperability**: Imagine being able to comment under a Youtube video directly from twitter.com, and having the comment shown under the video on youtube.com. Or following a Subreddit from Facebook. Such functionality is already available on the equivalent Fediverse platforms, thanks to common usage of Activitypub.
+- **Ease of use**: From a user perspective, decentralized social media works almost identically to existing websites: a website with email and password based login. Unlike pure peer-to-peer networks, it is not necessary to handle private keys or install any local software.
+- **Open ecosystem**: All existing Fediverse software is open source, and there are no legal or bureaucratic requirements to start federating. That means anyone can create or fork federated software. In this way different software platforms can exist in the same network according to the preferences of different user groups. It is not necessary to target the lowest common denominator as with corporate social media.
+- **Censorship resistance**: Current social media platforms are under the control of a few corporations and are actively being censored as revealed by the [Twitter Files](https://jordansather.substack.com/p/running-list-of-all-twitter-files). This would be much more difficult on a federated network, as it would require the cooperation of every single instance administrator. Additionally, users who are affected by censorship can create their own websites and stay connected with the network.
+- **Low barrier to entry**: All it takes to host a federated website are a small server, a domain and a TLS certificate. All of this is easily in the reach of individual hobbyists. There is also some technical knowledge needed, but this can be avoided with managed hosting platforms.
+
+Below you can find a complete guide that explains how to create a federated project from scratch.
+
+Feel free to open an issue if you have any questions regarding this crate. You can also join the Matrix channel [#activitystreams](https://matrix.to/#/%23activitystreams:matrix.asonix.dog) for discussion about Activitypub in Rust. Additionally check out [Socialhub forum](https://socialhub.activitypub.rocks/) for general ActivityPub development.

docs/02_overview.md

@@ -0,0 +1,21 @@
+## Overview
+
+It is recommended to read the [W3C Activitypub standard document](https://www.w3.org/TR/activitypub/) which explains in detail how the protocol works. Note that it includes a section about client to server interactions, this functionality is not implemented by any major Fediverse project. Other relevant standard documents are [Activitystreams](https://www.w3.org/ns/activitystreams) and [Activity Vocabulary](https://www.w3.org/TR/activitystreams-vocabulary/). Its a good idea to keep these around as references during development.
+
+This crate provides high level abstractions for the core functionality of Activitypub: fetching, sending and receiving data, as well as handling HTTP signatures. It was built from the experience of developing [Lemmy](https://join-lemmy.org/) which is the biggest Fediverse project written in Rust. Nevertheless it very generic and appropriate for any type of application wishing to implement the Activitypub protocol.
+
+There are two examples included to see how the library altogether:
+
+- `local_federation`: Creates two instances which run on localhost and federate with each other. This setup is ideal for quick development and well as automated tests.
+- `live_federation`: A minimal application which can be deployed on a server and federate with other platforms such as Mastodon. For this it needs run at the root of a (sub)domain which is available over HTTPS. Edit `main.rs` to configure the server domain and your Fediverse handle. Once started, it will automatically send a message to you and log any incoming messages.
+
+To see how this library is used in production, have a look at the [Lemmy federation code](https://github.com/LemmyNet/lemmy/tree/main/crates/apub).
+
+### Security
+This framework does not inherently perform data sanitization upon receiving federated activity data. 
+
+Please, never place implicit trust in the security of data received from the Fediverse. Always keep in mind that malicious entities can be easily created through anonymous fediverse handles.
+
+When implementing our crate in your application, ensure to incorporate data sanitization and validation measures before storing the received data in your database and using it in your user interface. This would significantly reduce the risk of malicious data or actions affecting your application's security and performance.
+
+This framework is designed to simplify your development process, but it's your responsibility to ensure the security of your application. Always follow best practices for data handling, sanitization, and security. 

docs/03_federating_users.md

@@ -0,0 +1,90 @@
+## Federating users
+
+This library intentionally doesn't include any predefined data structures for federated data. The reason is that each federated application is different, and needs different data formats. Activitypub also doesn't define any specific data structures, but provides a few mandatory fields and many which are optional. For this reason it works best to let each application define its own data structures, and take advantage of serde for (de)serialization. This means we don't use `json-ld` which Activitypub is based on, but that doesn't cause any problems in practice.
+
+The first thing we need to federate are users. Its easiest to get started by looking at the data sent by other platforms. Here we fetch an account from Mastodon, ignoring the many optional fields. This curl command is generally very helpful to inspect and debug federated services.
+
+```text
+$ curl -H 'Accept: application/activity+json' https://mastodon.social/@LemmyDev | jq
+{
+    "id": "https://mastodon.social/users/LemmyDev",
+    "type": "Person",
+    "preferredUsername": "LemmyDev",
+    "name": "Lemmy",
+    "inbox": "https://mastodon.social/users/LemmyDev/inbox",
+    "outbox": "https://mastodon.social/users/LemmyDev/outbox",
+    "publicKey": {
+        "id": "https://mastodon.social/users/LemmyDev#main-key",
+        "owner": "https://mastodon.social/users/LemmyDev",
+        "publicKeyPem": "..."
+    },
+    ...
+}
+```
+
+The most important fields are:
+- `id`: Unique identifier for this object. At the same time it is the URL where we can fetch the object from
+- `type`: The type of this object
+- `preferredUsername`: Immutable username which was chosen at signup and is used in URLs as well as in mentions like `@LemmyDev@mastodon.social`
+- `name`: Displayname which can be freely changed at any time
+- `inbox`: URL where incoming activities are delivered to, treated in a later section
+  see xx document for a definition of each field
+- `publicKey`: Key which is used for [HTTP Signatures](https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-message-signatures)
+
+Refer to [Activity Vocabulary](https://www.w3.org/TR/activitystreams-vocabulary/) for further details and description of other fields. You can also inspect many other URLs on federated platforms with the given curl command.
+
+Based on this we can define the following minimal struct to (de)serialize a `Person` with serde.
+
+```rust
+# use activitypub_federation::protocol::public_key::PublicKey;
+# use activitypub_federation::fetch::object_id::ObjectId;
+# use serde::{Deserialize, Serialize};
+# use activitystreams_kinds::actor::PersonType;
+# use url::Url;
+# use activitypub_federation::traits::tests::DbUser;
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Person {
+    id: ObjectId<DbUser>,
+    #[serde(rename = "type")]
+    kind: PersonType,
+    preferred_username: String,
+    name: String,
+    inbox: Url,
+    outbox: Url,
+    public_key: PublicKey,
+}
+```
+
+`ObjectId` is a wrapper for `Url` which helps to fetch data from a remote server, and convert it to `DbUser` which is the type that's stored in our local database. It also helps with caching data so that it doesn't have to be refetched every time.
+
+`PersonType` is an enum with a single variant `Person`. It is used to deserialize objects in a typesafe way: If the JSON type value does not match the string `Person`, deserialization fails. This helps in places where we don't know the exact data type that is being deserialized, as you will see later.
+
+Besides we also need a second struct to represent the data which gets stored in our local database (for example PostgreSQL). This is necessary because the data format used by SQL is very different from that used by that from Activitypub. It is organized by an integer primary key instead of a link id. Nested structs are complicated to represent and easier if flattened. Some fields like `type` don't need to be stored at all. On the other hand, the database contains fields which can't be federated, such as the private key and a boolean indicating if the item is local or remote.
+
+```rust
+# use url::Url;
+# use chrono::{DateTime, Utc};
+
+pub struct DbUser {
+    pub id: i32,
+    pub name: String,
+    pub display_name: String,
+    pub password_hash: Option<String>,
+    pub email: Option<String>,
+    pub federation_id: Url,
+    pub inbox: Url,
+    pub outbox: Url,
+    pub local: bool,
+    pub public_key: String,
+    pub private_key: Option<String>,
+    pub last_refreshed_at: DateTime<Utc>,
+}
+```
+
+Field names and other details of this type can be chosen freely according to your requirements. It only matters that the required data is being stored. Its important that this struct doesn't represent only local users who registered directly on our website, but also remote users that are registered on other instances and federated to us. The `local` column helps to easily distinguish both. It can also be distinguished from the domain of the `federation_id` URL, but that would be a much more expensive operation. All users have a `public_key`, but only local users have a `private_key`. On the other hand, `password_hash` and `email` are only present for local users. inbox` and `outbox` URLs need to be stored because each implementation is free to choose its own format for them, so they can't be regenerated on the fly.
+
+In larger projects it makes sense to split this data in two. One for data relevant to local users (`password_hash`, `email` etc.) and one for data that is shared by both local and federated users (`federation_id`, `public_key` etc).
+
+Finally we need to implement the traits [Object](crate::traits::Object) and [Actor](crate::traits::Actor) for `DbUser`. These traits are used to convert between `Person` and `DbUser` types. [Object::from_json](crate::traits::Object::from_json) must store the received object in database, so that it can later be retrieved without network calls using [Object::read_from_id](crate::traits::Object::read_from_id). Refer to the documentation for more details.

docs/04_federating_posts.md

@@ -0,0 +1,28 @@
+## Federating posts
+
+We repeat the same steps taken above for users in order to federate our posts.
+
+```text
+$ curl -H 'Accept: application/activity+json' https://mastodon.social/@LemmyDev/109790106847504642 | jq
+{
+    "id": "https://mastodon.social/users/LemmyDev/statuses/109790106847504642",
+    "type": "Note",
+    "content": "<p><a href=\"https://mastodon.social/tags/lemmy\" ...",
+    "attributedTo": "https://mastodon.social/users/LemmyDev",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+        "https://mastodon.social/users/LemmyDev/followers"
+    ],
+}
+```
+
+The most important fields are:
+- `id`: Unique identifier for this object. At the same time it is the URL where we can fetch the object from
+- `type`: The type of this object
+- `content`: Post text in HTML format
+- `attributedTo`: ID of the user who created this post
+- `to`, `cc`: Who the object is for. The special "public" URL indicates that everyone can view it.  It also gets delivered to followers of the LemmyDev account.
+
+Just like for `Person` before, we need to implement a protocol type and a database type, then implement trait `Object`. See the example for details.

docs/05_configuration.md

@@ -0,0 +1,17 @@
+## Configuration
+
+Next we need to do some configuration. Most importantly we need to specify the domain where the federated instance is running. It should be at the domain root and available over HTTPS for production. See the documentation for a list of config options. The parameter `user_data` is for anything that your application requires in handler functions, such as database connection handle, configuration etc.
+
+```
+# use activitypub_federation::config::FederationConfig;
+# let db_connection = ();
+# tokio::runtime::Runtime::new().unwrap().block_on(async {
+let config = FederationConfig::builder()
+    .domain("example.com")
+    .app_data(db_connection)
+    .build().await?;
+# Ok::<(), anyhow::Error>(())
+# }).unwrap()
+```
+
+`debug` is necessary to test federation with http and localhost URLs, but it should never be used in production. `url_verifier` can be used to implement a domain blacklist.

docs/06_http_endpoints_axum.md

@@ -0,0 +1,94 @@
+## HTTP endpoints
+
+The next step is to allow other servers to fetch our actors and objects. For this we need to create an HTTP route, most commonly at the same path where the actor or object can be viewed in a web browser. On this path there should be another route which responds to requests with header `Accept: application/activity+json` and serves the JSON data. This needs to be done for all actors and objects. Note that only local items should be served in this way.
+
+```no_run
+# use std::net::SocketAddr;
+# use activitypub_federation::config::FederationConfig;
+# use activitypub_federation::protocol::context::WithContext;
+# use activitypub_federation::axum::json::FederationJson;
+# use anyhow::Error;
+# use activitypub_federation::traits::tests::Person;
+# use activitypub_federation::config::Data;
+# use activitypub_federation::traits::tests::DbConnection;
+# use axum::extract::Path;
+# use activitypub_federation::config::FederationMiddleware;
+# use axum::routing::get;
+# use crate::activitypub_federation::traits::Object;
+# use axum::headers::ContentType;
+# use activitypub_federation::FEDERATION_CONTENT_TYPE;
+# use axum::TypedHeader;
+# use axum::response::IntoResponse;
+# use http::HeaderMap;
+# async fn generate_user_html(_: String, _: Data<DbConnection>) -> axum::response::Response { todo!() }
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    let data = FederationConfig::builder()
+        .domain("example.com")
+        .app_data(DbConnection)
+        .build().await?;
+        
+    let app = axum::Router::new()
+        .route("/user/:name", get(http_get_user))
+        .layer(FederationMiddleware::new(data));
+
+    let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
+    tracing::debug!("listening on {}", addr);
+    axum::Server::bind(&addr)
+        .serve(app.into_make_service())
+        .await?;
+    Ok(())
+}
+
+async fn http_get_user(
+    header_map: HeaderMap,
+    Path(name): Path<String>,
+    data: Data<DbConnection>,
+) -> impl IntoResponse {
+    let accept = header_map.get("accept").map(|v| v.to_str().unwrap());
+    if accept == Some(FEDERATION_CONTENT_TYPE) {
+        let db_user = data.read_local_user(&name).await.unwrap();
+        let json_user = db_user.into_json(&data).await.unwrap();
+        FederationJson(WithContext::new_default(json_user)).into_response()
+    }
+    else {
+        generate_user_html(name, data).await
+    }
+}
+```
+
+There are a couple of things going on here. Like before we are constructing the federation config with our domain and application data. We pass this to a middleware to make it available in request handlers, then listening on a port with the axum webserver.
+
+The `http_get_user` method allows retrieving a user profile from `/user/:name`. It checks the `accept` header, and compares it to the one used by Activitypub (`application/activity+json`). If it matches, the user is read from database and converted to Activitypub json format. The `context` field is added (`WithContext` for `json-ld` compliance), and it is converted to a JSON response with header `content-type: application/activity+json` using `FederationJson`. It can now be retrieved with the command `curl -H 'Accept: application/activity+json' ...` introduced earlier, or with `ObjectId`.
+
+If the `accept` header doesn't match, it renders the user profile as HTML for viewing in a web browser.
+
+We also need to implement a webfinger endpoint, which can resolve a handle like `@nutomic@lemmy.ml` into an ID like `https://lemmy.ml/u/nutomic` that can be used by Activitypub. Webfinger is not part of the ActivityPub standard, but the fact that Mastodon requires it makes it de-facto mandatory. It is defined in [RFC 7033](https://www.rfc-editor.org/rfc/rfc7033). Implementing it basically means handling requests of the form`https://mastodon.social/.well-known/webfinger?resource=acct:LemmyDev@mastodon.social`.
+
+To do this we can implement the following HTTP handler which must be bound to path `.well-known/webfinger`.
+
+```rust
+# use serde::Deserialize;
+# use axum::{extract::Query, Json};
+# use activitypub_federation::config::Data;
+# use activitypub_federation::fetch::webfinger::Webfinger;
+# use anyhow::Error;
+# use activitypub_federation::traits::tests::DbConnection;
+# use activitypub_federation::fetch::webfinger::extract_webfinger_name;
+# use activitypub_federation::fetch::webfinger::build_webfinger_response;
+
+#[derive(Deserialize)]
+struct WebfingerQuery {
+    resource: String,
+}
+
+async fn webfinger(
+    Query(query): Query<WebfingerQuery>,
+    data: Data<DbConnection>,
+) -> Result<Json<Webfinger>, Error> {
+    let name = extract_webfinger_name(&query.resource, &data)?;
+    let db_user = data.read_local_user(name).await?;
+    Ok(Json(build_webfinger_response(query.resource, db_user.federation_id)))
+}
+```

docs/07_fetching_data.md

@@ -0,0 +1,42 @@
+## Fetching data
+
+After setting up our structs, implementing traits and initializing configuration, we can easily fetch data from remote servers:
+
+```no_run
+# use activitypub_federation::fetch::object_id::ObjectId;
+# use activitypub_federation::traits::tests::DbUser;
+# use activitypub_federation::config::FederationConfig;
+# let db_connection = activitypub_federation::traits::tests::DbConnection;
+# tokio::runtime::Runtime::new().unwrap().block_on(async {
+let config = FederationConfig::builder()
+    .domain("example.com")
+    .app_data(db_connection)
+    .build().await?;
+let user_id = ObjectId::<DbUser>::parse("https://mastodon.social/@LemmyDev")?;
+let data = config.to_request_data();
+let user = user_id.dereference(&data).await;
+assert!(user.is_ok());
+# Ok::<(), anyhow::Error>(())
+# }).unwrap()
+```
+
+`dereference` retrieves the object JSON at the given URL, and uses serde to convert it to `Person`. It then calls your method `Object::from_json` which inserts it in the database and returns a `DbUser` struct. `request_data` contains the federation config as well as a counter of outgoing HTTP requests. If this counter exceeds the configured maximum, further requests are aborted in order to avoid recursive fetching which could allow for a denial of service attack.
+
+After dereferencing a remote object, it is stored in the local database and can be retrieved using [ObjectId::dereference_local](crate::fetch::object_id::ObjectId::dereference_local) without any network requests. This is important for performance reasons and for searching.
+
+We can similarly dereference a user over webfinger with the following method. It fetches the webfinger response from `.well-known/webfinger` and then fetches the actor using [ObjectId::dereference](crate::fetch::object_id::ObjectId::dereference) as above.
+```rust
+# use activitypub_federation::traits::tests::DbConnection;
+# use activitypub_federation::config::FederationConfig;
+# use activitypub_federation::fetch::webfinger::webfinger_resolve_actor;
+# use activitypub_federation::traits::tests::DbUser;
+# let db_connection = DbConnection;
+# tokio::runtime::Runtime::new().unwrap().block_on(async {
+# let config = FederationConfig::builder().domain("example.com").app_data(db_connection).build().await?;
+# let data = config.to_request_data();
+let user: DbUser = webfinger_resolve_actor("ruud@lemmy.world", &data).await?;
+# Ok::<(), anyhow::Error>(())
+# }).unwrap();
+```
+
+Note that webfinger queries don't contain a leading `@`. It is possible tha there are multiple Activitypub IDs returned for a single webfinger query in case of multiple actors with the same name (for example Lemmy permits group and person with the same name). In this case `webfinger_resolve_actor` automatically loops and returns the first item which can be dereferenced successfully to the given type.

docs/08_receiving_activities.md

@@ -0,0 +1,88 @@
+## Sending and receiving activities
+
+Activitypub propagates actions across servers using `Activities`. For this each actor has an inbox and a public/private key pair. We already defined a `Person` actor with keypair. Whats left is to define an activity. This is similar to the way we defined `Person` and `Note` structs before. In this case we need to implement the [ActivityHandler](trait@crate::traits::ActivityHandler) trait.
+
+```
+# use serde::{Deserialize, Serialize};
+# use url::Url;
+# use anyhow::Error;
+# use async_trait::async_trait;
+# use activitypub_federation::fetch::object_id::ObjectId;
+# use activitypub_federation::traits::tests::{DbConnection, DbUser};
+# use activitystreams_kinds::activity::FollowType;
+# use activitypub_federation::traits::ActivityHandler;
+# use activitypub_federation::config::Data;
+# async fn send_accept() -> Result<(), Error> { Ok(()) }
+
+#[derive(Deserialize, Serialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Follow {
+    pub actor: ObjectId<DbUser>,
+    pub object: ObjectId<DbUser>,
+    #[serde(rename = "type")]
+    pub kind: FollowType,
+    pub id: Url,
+}
+
+#[async_trait]
+impl ActivityHandler for Follow {
+    type DataType = DbConnection;
+    type Error = Error;
+
+    fn id(&self) -> &Url {
+        &self.id
+    }
+
+    fn actor(&self) -> &Url {
+        self.actor.inner()
+    }
+    
+    async fn verify(&self,  _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        Ok(())
+    }
+
+    async fn receive(self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        let actor = self.actor.dereference(data).await?;
+        let followed = self.object.dereference(data).await?;
+        data.add_follower(followed, actor).await?;
+        Ok(())
+    }
+}
+```
+
+In this case there is no need to convert to a database type, because activities don't need to be stored in the database in full. Instead we dereference the involved user accounts, and create a follow relation in the database.
+
+Next its time to setup the actual HTTP handler for the inbox. For this we first define an enum of all activities which are accepted by the actor. Then we just need to define an HTTP endpoint at the path of our choice (identical to `Person.inbox` defined earlier). This endpoint needs to hand received data over to [receive_activity](crate::axum::inbox::receive_activity). This method verifies the HTTP signature, checks the blocklist with [FederationConfigBuilder::url_verifier](crate::config::FederationConfigBuilder::url_verifier) and more. If everything is valid, the activity is passed to the `receive` method we defined above.
+
+```
+# use axum::response::IntoResponse;
+# use activitypub_federation::axum::inbox::{ActivityData, receive_activity};
+# use activitypub_federation::config::Data;
+# use activitypub_federation::protocol::context::WithContext;
+# use activitypub_federation::traits::ActivityHandler;
+# use activitypub_federation::traits::tests::{DbConnection, DbUser, Follow};
+# use serde::{Deserialize, Serialize};
+# use url::Url;
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(untagged)]
+#[enum_delegate::implement(ActivityHandler)]
+pub enum PersonAcceptedActivities {
+    Follow(Follow),
+}
+
+async fn http_post_user_inbox(
+    data: Data<DbConnection>,
+    activity_data: ActivityData,
+) -> impl IntoResponse {
+    receive_activity::<WithContext<PersonAcceptedActivities>, DbUser, DbConnection>(
+        activity_data,
+        &data,
+    )
+        .await.unwrap()
+}
+```
+
+The `PersonAcceptedActivities` works by attempting to parse the received JSON data with each variant in order. The first variant which parses without errors is used for receiving. This means you should avoid defining multiple activities in a way that they might conflict and parse the same data.
+
+Activity enums can also be nested. 

docs/09_sending_activities.md

@@ -0,0 +1,75 @@
+## Sending activities
+
+To send an activity we need to initialize our previously defined struct, and pick an actor for sending. We also need a list of all actors that should receive the activity.
+
+```
+# use activitypub_federation::config::FederationConfig;
+# use activitypub_federation::activity_queue::queue_activity;
+# use activitypub_federation::http_signatures::generate_actor_keypair;
+# use activitypub_federation::traits::Actor;
+# use activitypub_federation::fetch::object_id::ObjectId;
+# use activitypub_federation::traits::tests::{DB_USER, DbConnection, Follow};
+# tokio::runtime::Runtime::new().unwrap().block_on(async {
+# let db_connection = DbConnection;
+# let config = FederationConfig::builder()
+#     .domain("example.com")
+#     .app_data(db_connection)
+#     .build().await?;
+# let data = config.to_request_data();
+# let sender = DB_USER.clone();
+# let recipient = DB_USER.clone();
+let activity = Follow {
+    actor: ObjectId::parse("https://lemmy.ml/u/nutomic")?,
+    object: recipient.federation_id.clone().into(),
+    kind: Default::default(),
+    id: "https://lemmy.ml/activities/321".try_into()?
+};
+let inboxes = vec![recipient.shared_inbox_or_inbox()];
+
+queue_activity(&activity, &sender, inboxes, &data).await?;
+# Ok::<(), anyhow::Error>(())
+# }).unwrap()
+```
+
+The list of inboxes gets deduplicated (important for shared inbox). All inboxes on the local domain and those which fail the [crate::config::UrlVerifier] check are excluded from delivery. For each remaining inbox a background tasks is created. It signs the HTTP header with the given private key. Finally the activity is delivered to the inbox.
+
+It is possible that delivery fails because the target instance is temporarily unreachable. In this case the task is scheduled for retry after a certain waiting time. For each task delivery is retried up to 3 times after the initial attempt. The retry intervals are as follows:
+
+- one minute, in case of service restart
+- one hour, in case of instance maintenance
+- 2.5 days, in case of major incident with rebuild from backup
+
+In case [crate::config::FederationConfigBuilder::debug] is enabled, no background thread is used but activities are sent directly on the foreground. This makes it easier to catch delivery errors and avoids complicated steps to await delivery in tests.
+
+In some cases you may want to bypass the builtin activity queue, and implement your own. For example to specify different retry intervals, or to persist retries across application restarts. You can do it with the following code:
+```rust
+# use activitypub_federation::config::FederationConfig;
+# use activitypub_federation::activity_sending::SendActivityTask;
+# use activitypub_federation::http_signatures::generate_actor_keypair;
+# use activitypub_federation::traits::Actor;
+# use activitypub_federation::fetch::object_id::ObjectId;
+# use activitypub_federation::traits::tests::{DB_USER, DbConnection, Follow};
+# tokio::runtime::Runtime::new().unwrap().block_on(async {
+# let db_connection = DbConnection;
+# let config = FederationConfig::builder()
+#     .domain("example.com")
+#     .app_data(db_connection)
+#     .build().await?;
+# let data = config.to_request_data();
+# let sender = DB_USER.clone();
+# let recipient = DB_USER.clone();
+let activity = Follow {
+    actor: ObjectId::parse("https://lemmy.ml/u/nutomic")?,
+    object: recipient.federation_id.clone().into(),
+    kind: Default::default(),
+    id: "https://lemmy.ml/activities/321".try_into()?
+};
+let inboxes = vec![recipient.shared_inbox_or_inbox()];
+
+let sends = SendActivityTask::prepare(&activity, &sender, inboxes, &data).await?;
+for send in sends {
+send.sign_and_send(&data).await?;
+}
+# Ok::<(), anyhow::Error>(())
+# }).unwrap()
+```

docs/10_fetching_objects_with_unknown_type.md

@@ -0,0 +1,81 @@
+## Fetching remote object with unknown type
+
+It is sometimes necessary to fetch from a URL, but we don't know the exact type of object it will return. An example is the search field in most federated platforms, which allows pasting and `id` URL and fetches it from the origin server. It can be implemented in the following way:
+
+```no_run
+# use activitypub_federation::traits::tests::{DbUser, DbPost};
+# use activitypub_federation::fetch::object_id::ObjectId;
+# use activitypub_federation::traits::Object;
+# use activitypub_federation::config::FederationConfig;
+# use serde::{Deserialize, Serialize};
+# use activitypub_federation::traits::tests::DbConnection;
+# use activitypub_federation::config::Data;
+# use url::Url;
+# use activitypub_federation::traits::tests::{Person, Note};
+
+#[derive(Debug)]
+pub enum SearchableDbObjects {
+    User(DbUser),
+    Post(DbPost)
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(untagged)]
+pub enum SearchableObjects {
+    Person(Person),
+    Note(Note)
+}
+
+#[async_trait::async_trait]
+impl Object for SearchableDbObjects {
+    type DataType = DbConnection;
+    type Kind = SearchableObjects;
+    type Error = anyhow::Error;
+
+    async fn read_from_id(
+        object_id: Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<Option<Self>, Self::Error> {
+        Ok(None)
+    }
+
+    async fn into_json(
+        self,
+        data: &Data<Self::DataType>,
+    ) -> Result<Self::Kind, Self::Error> {
+        unimplemented!();
+    }
+    
+    async fn verify(json: &Self::Kind, expected_domain: &Url, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        Ok(())
+    }
+
+    async fn from_json(
+        json: Self::Kind,
+        data: &Data<Self::DataType>,
+    ) -> Result<Self, Self::Error> {
+        use SearchableDbObjects::*;
+        match json {
+            SearchableObjects::Person(p) => Ok(User(DbUser::from_json(p, data).await?)),
+            SearchableObjects::Note(n) => Ok(Post(DbPost::from_json(n, data).await?)),
+        }
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), anyhow::Error> {
+    # let config = FederationConfig::builder().domain("example.com").app_data(DbConnection).build().await.unwrap();
+    # let data = config.to_request_data();
+    let query = "https://example.com/id/413";
+    let query_result = ObjectId::<SearchableDbObjects>::parse(query)?
+        .dereference(&data)
+        .await?;
+    match query_result {
+        SearchableDbObjects::Post(post) => {} // retrieved object is a post
+        SearchableDbObjects::User(user) => {} // object is a user
+    };
+    Ok(())
+}
+```
+
+This is similar to the way receiving activities are handled in the previous section. The remote JSON is fetched, and received using the first enum variant which can successfully deserialize the data.

examples/README.md

@@ -0,0 +1,38 @@
+# Examples
+
+## Local Federation
+
+Creates two instances which run on localhost and federate with each other. This setup is ideal for quick development and well as automated tests. In this case both instances run in the same process and are controlled from the main function. 
+
+In case of Lemmy we are using the same setup for continuous integration tests, only that multiple instances are started with a bash script as different threads, and controlled over the API.
+
+Use one of the following commands to run the example with the specified web framework:
+
+`cargo run --example local_federation axum`
+
+`cargo run --example local_federation actix-web`
+
+## Live Federation
+
+A minimal application which can be deployed on a server and federate with other platforms such as Mastodon. For this it needs run at the root of a (sub)domain which is available over HTTPS. Edit `main.rs` to configure the server domain and your Fediverse handle.
+
+Setup instructions:
+
+- Deploy the project to a server. For this you can clone the git repository on the server and execute `cargo run --example live_federation`. Alternatively run `cargo build --example live_federation` and copy the binary at `target/debug/examples/live_federation` to the server.
+- Create a TLS certificate. With Let's Encrypt certbot you can use a command like `certbot certonly --nginx -d 'example.com' -m '*your-email@domain.com*'` (replace with your actual domain and email).
+- Setup a reverse proxy which handles TLS and passes requests to the example project. With nginx you can use the following basic config, again using your actual domain:
+```
+server {
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+    server_name example.com;
+    ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
+    location / {
+        proxy_pass "http://localhost:8003";
+        proxy_set_header Host $host;
+    }
+}
+```
+- Test with `curl -H 'Accept: application/activity+json' https://example.com/alison | jq` and `curl -H 'Accept: application/activity+json' "https://example.com/.well-known/webfinger?resource=acct:alison@example.com" | jq` that the server is setup correctly and serving correct responses.
+- Login to a Fediverse platform like Mastodon, and search for `@alison@example.com`, with the actual domain and username from your `main.rs`. If you send a message, it will automatically send a response.

examples/federation/activities/accept.rs

@@ -1,56 +0,0 @@
-use crate::{activities::follow::Follow, instance::InstanceHandle, objects::person::MyUser};
-use activitypub_federation::{core::object_id::ObjectId, data::Data, traits::ActivityHandler};
-use activitystreams_kinds::activity::AcceptType;
-use serde::{Deserialize, Serialize};
-use url::Url;
-
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct Accept {
-    actor: ObjectId<MyUser>,
-    object: Follow,
-    #[serde(rename = "type")]
-    kind: AcceptType,
-    id: Url,
-}
-
-impl Accept {
-    pub fn new(actor: ObjectId<MyUser>, object: Follow, id: Url) -> Accept {
-        Accept {
-            actor,
-            object,
-            kind: Default::default(),
-            id,
-        }
-    }
-}
-
-#[async_trait::async_trait(?Send)]
-impl ActivityHandler for Accept {
-    type DataType = InstanceHandle;
-    type Error = crate::error::Error;
-
-    fn id(&self) -> &Url {
-        &self.id
-    }
-
-    fn actor(&self) -> &Url {
-        self.actor.inner()
-    }
-
-    async fn verify(
-        &self,
-        _data: &Data<Self::DataType>,
-        _request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        Ok(())
-    }
-
-    async fn receive(
-        self,
-        _data: &Data<Self::DataType>,
-        _request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        Ok(())
-    }
-}

examples/federation/activities/create_note.rs

@@ -1,70 +0,0 @@
-use crate::{
-    instance::InstanceHandle,
-    objects::{note::Note, person::MyUser},
-    MyPost,
-};
-use activitypub_federation::{
-    core::object_id::ObjectId,
-    data::Data,
-    deser::helpers::deserialize_one_or_many,
-    traits::{ActivityHandler, ApubObject},
-};
-use activitystreams_kinds::activity::CreateType;
-use serde::{Deserialize, Serialize};
-use url::Url;
-
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct CreateNote {
-    pub(crate) actor: ObjectId<MyUser>,
-    #[serde(deserialize_with = "deserialize_one_or_many")]
-    pub(crate) to: Vec<Url>,
-    pub(crate) object: Note,
-    #[serde(rename = "type")]
-    pub(crate) kind: CreateType,
-    pub(crate) id: Url,
-}
-
-impl CreateNote {
-    pub fn new(note: Note, id: Url) -> CreateNote {
-        CreateNote {
-            actor: note.attributed_to.clone(),
-            to: note.to.clone(),
-            object: note,
-            kind: CreateType::Create,
-            id,
-        }
-    }
-}
-
-#[async_trait::async_trait(?Send)]
-impl ActivityHandler for CreateNote {
-    type DataType = InstanceHandle;
-    type Error = crate::error::Error;
-
-    fn id(&self) -> &Url {
-        &self.id
-    }
-
-    fn actor(&self) -> &Url {
-        self.actor.inner()
-    }
-
-    async fn verify(
-        &self,
-        data: &Data<Self::DataType>,
-        request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        MyPost::verify(&self.object, self.id(), data, request_counter).await?;
-        Ok(())
-    }
-
-    async fn receive(
-        self,
-        data: &Data<Self::DataType>,
-        request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        MyPost::from_apub(self.object, data, request_counter).await?;
-        Ok(())
-    }
-}

examples/federation/activities/follow.rs

@@ -1,88 +0,0 @@
-use crate::{
-    activities::accept::Accept,
-    generate_object_id,
-    instance::InstanceHandle,
-    objects::person::MyUser,
-};
-use activitypub_federation::{
-    core::object_id::ObjectId,
-    data::Data,
-    traits::{ActivityHandler, Actor},
-};
-use activitystreams_kinds::activity::FollowType;
-use serde::{Deserialize, Serialize};
-use url::Url;
-
-#[derive(Deserialize, Serialize, Clone, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct Follow {
-    pub(crate) actor: ObjectId<MyUser>,
-    pub(crate) object: ObjectId<MyUser>,
-    #[serde(rename = "type")]
-    kind: FollowType,
-    id: Url,
-}
-
-impl Follow {
-    pub fn new(actor: ObjectId<MyUser>, object: ObjectId<MyUser>, id: Url) -> Follow {
-        Follow {
-            actor,
-            object,
-            kind: Default::default(),
-            id,
-        }
-    }
-}
-
-#[async_trait::async_trait(?Send)]
-impl ActivityHandler for Follow {
-    type DataType = InstanceHandle;
-    type Error = crate::error::Error;
-
-    fn id(&self) -> &Url {
-        &self.id
-    }
-
-    fn actor(&self) -> &Url {
-        self.actor.inner()
-    }
-
-    async fn verify(
-        &self,
-        _data: &Data<Self::DataType>,
-        _request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        Ok(())
-    }
-
-    // Ignore clippy false positive: https://github.com/rust-lang/rust-clippy/issues/6446
-    #[allow(clippy::await_holding_lock)]
-    async fn receive(
-        self,
-        data: &Data<Self::DataType>,
-        request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        // add to followers
-        let mut users = data.users.lock().unwrap();
-        let local_user = users.first_mut().unwrap();
-        local_user.followers.push(self.actor.inner().clone());
-        let local_user = local_user.clone();
-        drop(users);
-
-        // send back an accept
-        let follower = self
-            .actor
-            .dereference(data, data.local_instance(), request_counter)
-            .await?;
-        let id = generate_object_id(data.local_instance().hostname())?;
-        let accept = Accept::new(local_user.ap_id.clone(), self, id.clone());
-        local_user
-            .send(
-                accept,
-                vec![follower.shared_inbox_or_inbox()],
-                data.local_instance(),
-            )
-            .await?;
-        Ok(())
-    }
-}

examples/federation/instance.rs

@@ -1,121 +0,0 @@
-use crate::{
-    error::Error,
-    generate_object_id,
-    objects::{
-        note::MyPost,
-        person::{MyUser, PersonAcceptedActivities},
-    },
-};
-use activitypub_federation::{
-    core::{inbox::receive_activity, object_id::ObjectId, signatures::generate_actor_keypair},
-    data::Data,
-    deser::context::WithContext,
-    traits::ApubObject,
-    InstanceSettingsBuilder,
-    LocalInstance,
-    APUB_JSON_CONTENT_TYPE,
-};
-use actix_web::{web, App, HttpRequest, HttpResponse, HttpServer};
-use http_signature_normalization_actix::prelude::VerifyDigest;
-use reqwest::Client;
-use sha2::{Digest, Sha256};
-use std::{
-    ops::Deref,
-    sync::{Arc, Mutex},
-};
-use tokio::task;
-use url::Url;
-
-pub type InstanceHandle = Arc<Instance>;
-
-pub struct Instance {
-    /// This holds all library data
-    local_instance: LocalInstance,
-    /// Our "database" which contains all known users (local and federated)
-    pub users: Mutex<Vec<MyUser>>,
-    /// Same, but for posts
-    pub posts: Mutex<Vec<MyPost>>,
-}
-
-impl Instance {
-    pub fn new(hostname: String) -> Result<InstanceHandle, Error> {
-        let settings = InstanceSettingsBuilder::default().debug(true).build()?;
-        let local_instance =
-            LocalInstance::new(hostname.clone(), Client::default().into(), settings);
-        let local_user = MyUser::new(generate_object_id(&hostname)?, generate_actor_keypair()?);
-        let instance = Arc::new(Instance {
-            local_instance,
-            users: Mutex::new(vec![local_user]),
-            posts: Mutex::new(vec![]),
-        });
-        Ok(instance)
-    }
-
-    pub fn local_user(&self) -> MyUser {
-        self.users.lock().unwrap().first().cloned().unwrap()
-    }
-
-    pub fn local_instance(&self) -> &LocalInstance {
-        &self.local_instance
-    }
-
-    pub fn listen(instance: &InstanceHandle) -> Result<(), Error> {
-        let hostname = instance.local_instance.hostname();
-        let instance = instance.clone();
-        let server = HttpServer::new(move || {
-            App::new()
-                .app_data(web::Data::new(instance.clone()))
-                .route("/objects/{user_name}", web::get().to(http_get_user))
-                .service(
-                    web::scope("")
-                        // Important: this ensures that the activity json matches the hashsum in signed
-                        // HTTP header
-                        // TODO: it would be possible to get rid of this by verifying hash in
-                        //       receive_activity()
-                        .wrap(VerifyDigest::new(Sha256::new()))
-                        // Just a single, global inbox for simplicity
-                        .route("/inbox", web::post().to(http_post_user_inbox)),
-                )
-        })
-        .bind(hostname)?
-        .run();
-        task::spawn(server);
-        Ok(())
-    }
-}
-
-/// Handles requests to fetch user json over HTTP
-async fn http_get_user(
-    request: HttpRequest,
-    data: web::Data<InstanceHandle>,
-) -> Result<HttpResponse, Error> {
-    let data: InstanceHandle = data.into_inner().deref().clone();
-    let hostname: String = data.local_instance.hostname().to_string();
-    let request_url = format!("http://{}{}", hostname, &request.uri().to_string());
-    let url = Url::parse(&request_url)?;
-    let user = ObjectId::<MyUser>::new(url)
-        .dereference_local(&data)
-        .await?
-        .into_apub(&data)
-        .await?;
-    Ok(HttpResponse::Ok()
-        .content_type(APUB_JSON_CONTENT_TYPE)
-        .json(WithContext::new_default(user)))
-}
-
-/// Handles messages received in user inbox
-async fn http_post_user_inbox(
-    request: HttpRequest,
-    payload: String,
-    data: web::Data<InstanceHandle>,
-) -> Result<HttpResponse, Error> {
-    let data: InstanceHandle = data.into_inner().deref().clone();
-    let activity = serde_json::from_str(&payload)?;
-    receive_activity::<WithContext<PersonAcceptedActivities>, MyUser, InstanceHandle>(
-        request,
-        activity,
-        &data.clone().local_instance,
-        &Data::new(data),
-    )
-    .await
-}

examples/federation/main.rs

@@ -1,42 +0,0 @@
-use crate::{error::Error, instance::Instance, lib::generate_object_id, objects::note::MyPost};
-use tracing::log::LevelFilter;
-
-mod activities;
-mod error;
-mod instance;
-mod lib;
-mod objects;
-
-#[actix_rt::main]
-async fn main() -> Result<(), Error> {
-    env_logger::builder()
-        .filter_level(LevelFilter::Debug)
-        .init();
-
-    let alpha = Instance::new("localhost:8001".to_string())?;
-    let beta = Instance::new("localhost:8002".to_string())?;
-    Instance::listen(&alpha)?;
-    Instance::listen(&beta)?;
-
-    // alpha user follows beta user
-    alpha
-        .local_user()
-        .follow(&beta.local_user(), &alpha)
-        .await?;
-    // assert that follow worked correctly
-    assert_eq!(
-        beta.local_user().followers(),
-        &vec![alpha.local_user().ap_id.inner().clone()]
-    );
-
-    // beta sends a post to its followers
-    let sent_post = MyPost::new("hello world!".to_string(), beta.local_user().ap_id);
-    beta.local_user().post(sent_post.clone(), &beta).await?;
-    let received_post = alpha.posts.lock().unwrap().first().cloned().unwrap();
-
-    // assert that alpha received the post
-    assert_eq!(received_post.text, sent_post.text);
-    assert_eq!(received_post.ap_id.inner(), sent_post.ap_id.inner());
-    assert_eq!(received_post.creator.inner(), sent_post.creator.inner());
-    Ok(())
-}

examples/federation/objects/note.rs

@@ -1,92 +0,0 @@
-use crate::{generate_object_id, instance::InstanceHandle, objects::person::MyUser};
-use activitypub_federation::{
-    core::object_id::ObjectId,
-    deser::helpers::deserialize_one_or_many,
-    traits::ApubObject,
-};
-use activitystreams_kinds::{object::NoteType, public};
-use serde::{Deserialize, Serialize};
-use url::Url;
-
-#[derive(Clone, Debug)]
-pub struct MyPost {
-    pub text: String,
-    pub ap_id: ObjectId<MyPost>,
-    pub creator: ObjectId<MyUser>,
-    pub local: bool,
-}
-
-impl MyPost {
-    pub fn new(text: String, creator: ObjectId<MyUser>) -> MyPost {
-        MyPost {
-            text,
-            ap_id: ObjectId::new(generate_object_id(creator.inner().domain().unwrap()).unwrap()),
-            creator,
-            local: true,
-        }
-    }
-}
-
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct Note {
-    #[serde(rename = "type")]
-    kind: NoteType,
-    id: ObjectId<MyPost>,
-    pub(crate) attributed_to: ObjectId<MyUser>,
-    #[serde(deserialize_with = "deserialize_one_or_many")]
-    pub(crate) to: Vec<Url>,
-    content: String,
-}
-
-#[async_trait::async_trait(?Send)]
-impl ApubObject for MyPost {
-    type DataType = InstanceHandle;
-    type ApubType = Note;
-    type DbType = ();
-    type Error = crate::error::Error;
-
-    async fn read_from_apub_id(
-        _object_id: Url,
-        _data: &Self::DataType,
-    ) -> Result<Option<Self>, Self::Error> {
-        todo!()
-    }
-
-    async fn into_apub(self, data: &Self::DataType) -> Result<Self::ApubType, Self::Error> {
-        let creator = self.creator.dereference_local(data).await?;
-        Ok(Note {
-            kind: Default::default(),
-            id: self.ap_id,
-            attributed_to: self.creator,
-            to: vec![public(), creator.followers_url()?],
-            content: self.text,
-        })
-    }
-
-    async fn verify(
-        _apub: &Self::ApubType,
-        _expected_domain: &Url,
-        _data: &Self::DataType,
-        _request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        Ok(())
-    }
-
-    async fn from_apub(
-        apub: Self::ApubType,
-        data: &Self::DataType,
-        _request_counter: &mut i32,
-    ) -> Result<Self, Self::Error> {
-        let post = MyPost {
-            text: apub.content,
-            ap_id: apub.id,
-            creator: apub.attributed_to,
-            local: false,
-        };
-
-        let mut lock = data.posts.lock().unwrap();
-        lock.push(post.clone());
-        Ok(post)
-    }
-}

examples/federation/objects/person.rs

@@ -1,195 +0,0 @@
-use crate::{
-    activities::{accept::Accept, create_note::CreateNote, follow::Follow},
-    error::Error,
-    instance::InstanceHandle,
-    lib::generate_object_id,
-    objects::note::MyPost,
-};
-use activitypub_federation::{
-    core::{
-        activity_queue::send_activity,
-        object_id::ObjectId,
-        signatures::{Keypair, PublicKey},
-    },
-    deser::context::WithContext,
-    traits::{ActivityHandler, Actor, ApubObject},
-    LocalInstance,
-};
-use activitypub_federation_derive::activity_handler;
-use activitystreams_kinds::actor::PersonType;
-use serde::{Deserialize, Serialize};
-use url::Url;
-
-#[derive(Debug, Clone)]
-pub struct MyUser {
-    pub ap_id: ObjectId<MyUser>,
-    pub inbox: Url,
-    // exists for all users (necessary to verify http signatures)
-    public_key: String,
-    // exists only for local users
-    private_key: Option<String>,
-    pub followers: Vec<Url>,
-    pub local: bool,
-}
-
-/// List of all activities which this actor can receive.
-#[activity_handler(InstanceHandle, Error)]
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(untagged)]
-pub enum PersonAcceptedActivities {
-    Follow(Follow),
-    Accept(Accept),
-    CreateNote(CreateNote),
-}
-
-impl MyUser {
-    pub fn new(ap_id: Url, keypair: Keypair) -> MyUser {
-        let mut inbox = ap_id.clone();
-        inbox.set_path("/inbox");
-        let ap_id = ObjectId::new(ap_id);
-        MyUser {
-            ap_id,
-            inbox,
-            public_key: keypair.public_key,
-            private_key: Some(keypair.private_key),
-            followers: vec![],
-            local: true,
-        }
-    }
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct Person {
-    #[serde(rename = "type")]
-    kind: PersonType,
-    id: ObjectId<MyUser>,
-    inbox: Url,
-    public_key: PublicKey,
-}
-
-impl MyUser {
-    pub fn followers(&self) -> &Vec<Url> {
-        &self.followers
-    }
-
-    pub fn followers_url(&self) -> Result<Url, Error> {
-        Ok(Url::parse(&format!("{}/followers", self.ap_id.inner()))?)
-    }
-
-    fn public_key(&self) -> PublicKey {
-        PublicKey::new_main_key(self.ap_id.clone().into_inner(), self.public_key.clone())
-    }
-
-    pub async fn follow(&self, other: &MyUser, instance: &InstanceHandle) -> Result<(), Error> {
-        let id = generate_object_id(instance.local_instance().hostname())?;
-        let follow = Follow::new(self.ap_id.clone(), other.ap_id.clone(), id.clone());
-        self.send(
-            follow,
-            vec![other.shared_inbox_or_inbox()],
-            instance.local_instance(),
-        )
-        .await?;
-        Ok(())
-    }
-
-    pub async fn post(&self, post: MyPost, instance: &InstanceHandle) -> Result<(), Error> {
-        let id = generate_object_id(instance.local_instance().hostname())?;
-        let create = CreateNote::new(post.into_apub(instance).await?, id.clone());
-        let mut inboxes = vec![];
-        for f in self.followers.clone() {
-            let user: MyUser = ObjectId::new(f)
-                .dereference(instance, instance.local_instance(), &mut 0)
-                .await?;
-            inboxes.push(user.shared_inbox_or_inbox());
-        }
-        self.send(create, inboxes, instance.local_instance())
-            .await?;
-        Ok(())
-    }
-
-    pub(crate) async fn send<Activity>(
-        &self,
-        activity: Activity,
-        recipients: Vec<Url>,
-        local_instance: &LocalInstance,
-    ) -> Result<(), <Activity as ActivityHandler>::Error>
-    where
-        Activity: ActivityHandler + Serialize,
-        <Activity as ActivityHandler>::Error: From<anyhow::Error> + From<serde_json::Error>,
-    {
-        let activity = WithContext::new_default(activity);
-        send_activity(
-            activity,
-            self.public_key(),
-            self.private_key.clone().expect("has private key"),
-            recipients,
-            local_instance,
-        )
-        .await?;
-        Ok(())
-    }
-}
-
-#[async_trait::async_trait(?Send)]
-impl ApubObject for MyUser {
-    type DataType = InstanceHandle;
-    type ApubType = Person;
-    type DbType = MyUser;
-    type Error = crate::error::Error;
-
-    async fn read_from_apub_id(
-        object_id: Url,
-        data: &Self::DataType,
-    ) -> Result<Option<Self>, Self::Error> {
-        let users = data.users.lock().unwrap();
-        let res = users
-            .clone()
-            .into_iter()
-            .find(|u| u.ap_id.inner() == &object_id);
-        Ok(res)
-    }
-
-    async fn into_apub(self, _data: &Self::DataType) -> Result<Self::ApubType, Self::Error> {
-        Ok(Person {
-            kind: Default::default(),
-            id: self.ap_id.clone(),
-            inbox: self.inbox.clone(),
-            public_key: self.public_key(),
-        })
-    }
-
-    async fn verify(
-        _apub: &Self::ApubType,
-        _expected_domain: &Url,
-        _data: &Self::DataType,
-        _request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        Ok(())
-    }
-
-    async fn from_apub(
-        apub: Self::ApubType,
-        _data: &Self::DataType,
-        _request_counter: &mut i32,
-    ) -> Result<Self, Self::Error> {
-        Ok(MyUser {
-            ap_id: apub.id,
-            inbox: apub.inbox,
-            public_key: apub.public_key.public_key_pem,
-            private_key: None,
-            followers: vec![],
-            local: false,
-        })
-    }
-}
-
-impl Actor for MyUser {
-    fn public_key(&self) -> &str {
-        &self.public_key
-    }
-
-    fn inbox(&self) -> Url {
-        self.inbox.clone()
-    }
-}

examples/live_federation/activities/create_post.rs

@@ -0,0 +1,74 @@
+use crate::{
+    database::DatabaseHandle,
+    error::Error,
+    objects::{person::DbUser, post::Note},
+    utils::generate_object_id,
+    DbPost,
+};
+use activitypub_federation::{
+    activity_sending::SendActivityTask,
+    config::Data,
+    fetch::object_id::ObjectId,
+    kinds::activity::CreateType,
+    protocol::{context::WithContext, helpers::deserialize_one_or_many},
+    traits::{ActivityHandler, Object},
+};
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct CreatePost {
+    pub(crate) actor: ObjectId<DbUser>,
+    #[serde(deserialize_with = "deserialize_one_or_many")]
+    pub(crate) to: Vec<Url>,
+    pub(crate) object: Note,
+    #[serde(rename = "type")]
+    pub(crate) kind: CreateType,
+    pub(crate) id: Url,
+}
+
+impl CreatePost {
+    pub async fn send(note: Note, inbox: Url, data: &Data<DatabaseHandle>) -> Result<(), Error> {
+        print!("Sending reply to {}", &note.attributed_to);
+        let create = CreatePost {
+            actor: note.attributed_to.clone(),
+            to: note.to.clone(),
+            object: note,
+            kind: CreateType::Create,
+            id: generate_object_id(data.domain())?,
+        };
+        let create_with_context = WithContext::new_default(create);
+        let sends =
+            SendActivityTask::prepare(&create_with_context, &data.local_user(), vec![inbox], data)
+                .await?;
+        for send in sends {
+            send.sign_and_send(data).await?;
+        }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl ActivityHandler for CreatePost {
+    type DataType = DatabaseHandle;
+    type Error = crate::error::Error;
+
+    fn id(&self) -> &Url {
+        &self.id
+    }
+
+    fn actor(&self) -> &Url {
+        self.actor.inner()
+    }
+
+    async fn verify(&self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        DbPost::verify(&self.object, &self.id, data).await?;
+        Ok(())
+    }
+
+    async fn receive(self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        DbPost::from_json(self.object, data).await?;
+        Ok(())
+    }
+}

examples/live_federation/activities/mod.rs

@@ -0,0 +1 @@
+pub mod create_post;

examples/live_federation/database.rs

@@ -0,0 +1,26 @@
+use crate::{objects::person::DbUser, Error};
+use anyhow::anyhow;
+use std::sync::{Arc, Mutex};
+
+pub type DatabaseHandle = Arc<Database>;
+
+/// Our "database" which contains all known users (local and federated)
+pub struct Database {
+    pub users: Mutex<Vec<DbUser>>,
+}
+
+impl Database {
+    pub fn local_user(&self) -> DbUser {
+        let lock = self.users.lock().unwrap();
+        lock.first().unwrap().clone()
+    }
+
+    pub fn read_user(&self, name: &str) -> Result<DbUser, Error> {
+        let db_user = self.local_user();
+        if name == db_user.name {
+            Ok(db_user)
+        } else {
+            Err(anyhow!("Invalid user {name}").into())
+        }
+    }
+}

examples/live_federation/error.rs

@@ -1,11 +1,8 @@
-use actix_web::ResponseError;
 use std::fmt::{Display, Formatter};
 
 /// Necessary because of this issue: https://github.com/actix/actix-web/issues/1711
 #[derive(Debug)]
-pub struct Error(anyhow::Error);
-
-impl ResponseError for Error {}
+pub struct Error(pub(crate) anyhow::Error);
 
 impl Display for Error {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {

examples/live_federation/http.rs

@@ -0,0 +1,69 @@
+use crate::{
+    database::DatabaseHandle,
+    error::Error,
+    objects::person::{DbUser, Person, PersonAcceptedActivities},
+};
+use activitypub_federation::{
+    axum::{
+        inbox::{receive_activity, ActivityData},
+        json::FederationJson,
+    },
+    config::Data,
+    fetch::webfinger::{build_webfinger_response, extract_webfinger_name, Webfinger},
+    protocol::context::WithContext,
+    traits::Object,
+};
+use axum::{
+    extract::{Path, Query},
+    response::{IntoResponse, Response},
+    Json,
+};
+use axum_macros::debug_handler;
+use http::StatusCode;
+use serde::Deserialize;
+
+impl IntoResponse for Error {
+    fn into_response(self) -> Response {
+        (StatusCode::INTERNAL_SERVER_ERROR, format!("{}", self.0)).into_response()
+    }
+}
+
+#[debug_handler]
+pub async fn http_get_user(
+    Path(name): Path<String>,
+    data: Data<DatabaseHandle>,
+) -> Result<FederationJson<WithContext<Person>>, Error> {
+    let db_user = data.read_user(&name)?;
+    let json_user = db_user.into_json(&data).await?;
+    Ok(FederationJson(WithContext::new_default(json_user)))
+}
+
+#[debug_handler]
+pub async fn http_post_user_inbox(
+    data: Data<DatabaseHandle>,
+    activity_data: ActivityData,
+) -> impl IntoResponse {
+    receive_activity::<WithContext<PersonAcceptedActivities>, DbUser, DatabaseHandle>(
+        activity_data,
+        &data,
+    )
+    .await
+}
+
+#[derive(Deserialize)]
+pub struct WebfingerQuery {
+    resource: String,
+}
+
+#[debug_handler]
+pub async fn webfinger(
+    Query(query): Query<WebfingerQuery>,
+    data: Data<DatabaseHandle>,
+) -> Result<Json<Webfinger>, Error> {
+    let name = extract_webfinger_name(&query.resource, &data)?;
+    let db_user = data.read_user(name)?;
+    Ok(Json(build_webfinger_response(
+        query.resource,
+        db_user.ap_id.into_inner(),
+    )))
+}

examples/live_federation/main.rs

@@ -0,0 +1,72 @@
+#![allow(clippy::unwrap_used)]
+
+use crate::{
+    database::Database,
+    http::{http_get_user, http_post_user_inbox, webfinger},
+    objects::{person::DbUser, post::DbPost},
+    utils::generate_object_id,
+};
+use activitypub_federation::config::{FederationConfig, FederationMiddleware};
+use axum::{
+    routing::{get, post},
+    Router,
+};
+use error::Error;
+use std::{
+    net::ToSocketAddrs,
+    sync::{Arc, Mutex},
+};
+use tracing::log::{info, LevelFilter};
+
+mod activities;
+mod database;
+mod error;
+#[allow(clippy::diverging_sub_expression, clippy::items_after_statements)]
+mod http;
+mod objects;
+mod utils;
+
+const DOMAIN: &str = "example.com";
+const LOCAL_USER_NAME: &str = "alison";
+const BIND_ADDRESS: &str = "localhost:8003";
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    env_logger::builder()
+        .filter_level(LevelFilter::Warn)
+        .filter_module("activitypub_federation", LevelFilter::Info)
+        .filter_module("live_federation", LevelFilter::Info)
+        .format_timestamp(None)
+        .init();
+
+    info!("Setup local user and database");
+    let local_user = DbUser::new(DOMAIN, LOCAL_USER_NAME)?;
+    let database = Arc::new(Database {
+        users: Mutex::new(vec![local_user]),
+    });
+
+    info!("Setup configuration");
+    let config = FederationConfig::builder()
+        .domain(DOMAIN)
+        .app_data(database)
+        .build()
+        .await?;
+
+    info!("Listen with HTTP server on {BIND_ADDRESS}");
+    let config = config.clone();
+    let app = Router::new()
+        .route("/:user", get(http_get_user))
+        .route("/:user/inbox", post(http_post_user_inbox))
+        .route("/.well-known/webfinger", get(webfinger))
+        .layer(FederationMiddleware::new(config));
+
+    let addr = BIND_ADDRESS
+        .to_socket_addrs()?
+        .next()
+        .expect("Failed to lookup domain name");
+    axum::Server::bind(&addr)
+        .serve(app.into_make_service())
+        .await?;
+
+    Ok(())
+}

examples/live_federation/objects/mod.rs

@@ -1,2 +1,2 @@
-pub mod note;
 pub mod person;
+pub mod post;

examples/live_federation/objects/person.rs

@@ -0,0 +1,140 @@
+use crate::{activities::create_post::CreatePost, database::DatabaseHandle, error::Error};
+use activitypub_federation::{
+    config::Data,
+    fetch::object_id::ObjectId,
+    http_signatures::generate_actor_keypair,
+    kinds::actor::PersonType,
+    protocol::{public_key::PublicKey, verification::verify_domains_match},
+    traits::{ActivityHandler, Actor, Object},
+};
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use std::fmt::Debug;
+use url::Url;
+
+#[derive(Debug, Clone)]
+pub struct DbUser {
+    pub name: String,
+    pub ap_id: ObjectId<DbUser>,
+    pub inbox: Url,
+    // exists for all users (necessary to verify http signatures)
+    pub public_key: String,
+    // exists only for local users
+    pub private_key: Option<String>,
+    last_refreshed_at: DateTime<Utc>,
+    pub followers: Vec<Url>,
+    pub local: bool,
+}
+
+/// List of all activities which this actor can receive.
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(untagged)]
+#[enum_delegate::implement(ActivityHandler)]
+pub enum PersonAcceptedActivities {
+    CreateNote(CreatePost),
+}
+
+impl DbUser {
+    pub fn new(hostname: &str, name: &str) -> Result<DbUser, Error> {
+        let ap_id = Url::parse(&format!("https://{}/{}", hostname, &name))?.into();
+        let inbox = Url::parse(&format!("https://{}/{}/inbox", hostname, &name))?;
+        let keypair = generate_actor_keypair()?;
+        Ok(DbUser {
+            name: name.to_string(),
+            ap_id,
+            inbox,
+            public_key: keypair.public_key,
+            private_key: Some(keypair.private_key),
+            last_refreshed_at: Utc::now(),
+            followers: vec![],
+            local: true,
+        })
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Person {
+    #[serde(rename = "type")]
+    kind: PersonType,
+    preferred_username: String,
+    id: ObjectId<DbUser>,
+    inbox: Url,
+    public_key: PublicKey,
+}
+
+#[async_trait::async_trait]
+impl Object for DbUser {
+    type DataType = DatabaseHandle;
+    type Kind = Person;
+    type Error = Error;
+
+    fn last_refreshed_at(&self) -> Option<DateTime<Utc>> {
+        Some(self.last_refreshed_at)
+    }
+
+    async fn read_from_id(
+        object_id: Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<Option<Self>, Self::Error> {
+        let users = data.users.lock().unwrap();
+        let res = users
+            .clone()
+            .into_iter()
+            .find(|u| u.ap_id.inner() == &object_id);
+        Ok(res)
+    }
+
+    async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+        Ok(Person {
+            preferred_username: self.name.clone(),
+            kind: Default::default(),
+            id: self.ap_id.clone(),
+            inbox: self.inbox.clone(),
+            public_key: self.public_key(),
+        })
+    }
+
+    async fn verify(
+        json: &Self::Kind,
+        expected_domain: &Url,
+        _data: &Data<Self::DataType>,
+    ) -> Result<(), Self::Error> {
+        verify_domains_match(json.id.inner(), expected_domain)?;
+        Ok(())
+    }
+
+    async fn from_json(
+        json: Self::Kind,
+        _data: &Data<Self::DataType>,
+    ) -> Result<Self, Self::Error> {
+        Ok(DbUser {
+            name: json.preferred_username,
+            ap_id: json.id,
+            inbox: json.inbox,
+            public_key: json.public_key.public_key_pem,
+            private_key: None,
+            last_refreshed_at: Utc::now(),
+            followers: vec![],
+            local: false,
+        })
+    }
+}
+
+impl Actor for DbUser {
+    fn id(&self) -> Url {
+        self.ap_id.inner().clone()
+    }
+
+    fn public_key_pem(&self) -> &str {
+        &self.public_key
+    }
+
+    fn private_key_pem(&self) -> Option<String> {
+        self.private_key.clone()
+    }
+
+    fn inbox(&self) -> Url {
+        self.inbox.clone()
+    }
+}

examples/live_federation/objects/post.rs

@@ -0,0 +1,110 @@
+use crate::{
+    activities::create_post::CreatePost,
+    database::DatabaseHandle,
+    error::Error,
+    generate_object_id,
+    objects::person::DbUser,
+};
+use activitypub_federation::{
+    config::Data,
+    fetch::object_id::ObjectId,
+    kinds::{object::NoteType, public},
+    protocol::{helpers::deserialize_one_or_many, verification::verify_domains_match},
+    traits::{Actor, Object},
+};
+use activitystreams_kinds::link::MentionType;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+#[derive(Clone, Debug)]
+pub struct DbPost {
+    pub text: String,
+    pub ap_id: ObjectId<DbPost>,
+    pub creator: ObjectId<DbUser>,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Note {
+    #[serde(rename = "type")]
+    kind: NoteType,
+    id: ObjectId<DbPost>,
+    pub(crate) attributed_to: ObjectId<DbUser>,
+    #[serde(deserialize_with = "deserialize_one_or_many")]
+    pub(crate) to: Vec<Url>,
+    content: String,
+    in_reply_to: Option<ObjectId<DbPost>>,
+    tag: Vec<Mention>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Mention {
+    pub href: Url,
+    #[serde(rename = "type")]
+    pub kind: MentionType,
+}
+
+#[async_trait::async_trait]
+impl Object for DbPost {
+    type DataType = DatabaseHandle;
+    type Kind = Note;
+    type Error = Error;
+
+    async fn read_from_id(
+        _object_id: Url,
+        _data: &Data<Self::DataType>,
+    ) -> Result<Option<Self>, Self::Error> {
+        Ok(None)
+    }
+
+    async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+        Ok(Note {
+            kind: NoteType::Note,
+            id: self.ap_id,
+            content: self.text,
+            attributed_to: self.creator,
+            to: vec![public()],
+            tag: vec![],
+            in_reply_to: None,
+        })
+    }
+
+    async fn verify(
+        json: &Self::Kind,
+        expected_domain: &Url,
+        _data: &Data<Self::DataType>,
+    ) -> Result<(), Self::Error> {
+        verify_domains_match(json.id.inner(), expected_domain)?;
+        Ok(())
+    }
+
+    async fn from_json(json: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, Self::Error> {
+        println!(
+            "Received post with content {} and id {}",
+            &json.content, &json.id
+        );
+        let creator = json.attributed_to.dereference(data).await?;
+        let post = DbPost {
+            text: json.content,
+            ap_id: json.id.clone(),
+            creator: json.attributed_to.clone(),
+        };
+
+        let mention = Mention {
+            href: creator.ap_id.clone().into_inner(),
+            kind: Default::default(),
+        };
+        let note = Note {
+            kind: Default::default(),
+            id: generate_object_id(data.domain())?.into(),
+            attributed_to: data.local_user().ap_id,
+            to: vec![public()],
+            content: format!("Hello {}", creator.name),
+            in_reply_to: Some(json.id.clone()),
+            tag: vec![mention],
+        };
+        CreatePost::send(note, creator.shared_inbox_or_inbox(), data).await?;
+
+        Ok(post)
+    }
+}

examples/live_federation/utils.rs

@@ -3,11 +3,11 @@ use url::{ParseError, Url};
 
 /// Just generate random url as object id. In a real project, you probably want to use
 /// an url which contains the database id for easy retrieval (or store the random id in db).
-pub fn generate_object_id(hostname: &str) -> Result<Url, ParseError> {
+pub fn generate_object_id(domain: &str) -> Result<Url, ParseError> {
     let id: String = thread_rng()
         .sample_iter(&Alphanumeric)
         .take(7)
         .map(char::from)
         .collect();
-    Url::parse(&format!("http://{}/objects/{}", hostname, id))
+    Url::parse(&format!("https://{}/objects/{}", domain, id))
 }

examples/local_federation/activities/accept.rs

@@ -0,0 +1,52 @@
+use crate::{activities::follow::Follow, instance::DatabaseHandle, objects::person::DbUser};
+use activitypub_federation::{
+    config::Data,
+    fetch::object_id::ObjectId,
+    kinds::activity::AcceptType,
+    traits::ActivityHandler,
+};
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Accept {
+    actor: ObjectId<DbUser>,
+    object: Follow,
+    #[serde(rename = "type")]
+    kind: AcceptType,
+    id: Url,
+}
+
+impl Accept {
+    pub fn new(actor: ObjectId<DbUser>, object: Follow, id: Url) -> Accept {
+        Accept {
+            actor,
+            object,
+            kind: Default::default(),
+            id,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl ActivityHandler for Accept {
+    type DataType = DatabaseHandle;
+    type Error = crate::error::Error;
+
+    fn id(&self) -> &Url {
+        &self.id
+    }
+
+    fn actor(&self) -> &Url {
+        self.actor.inner()
+    }
+
+    async fn verify(&self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        Ok(())
+    }
+
+    async fn receive(self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}

examples/local_federation/activities/create_post.rs

@@ -0,0 +1,62 @@
+use crate::{
+    instance::DatabaseHandle,
+    objects::{person::DbUser, post::Note},
+    DbPost,
+};
+use activitypub_federation::{
+    config::Data,
+    fetch::object_id::ObjectId,
+    kinds::activity::CreateType,
+    protocol::helpers::deserialize_one_or_many,
+    traits::{ActivityHandler, Object},
+};
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct CreatePost {
+    pub(crate) actor: ObjectId<DbUser>,
+    #[serde(deserialize_with = "deserialize_one_or_many")]
+    pub(crate) to: Vec<Url>,
+    pub(crate) object: Note,
+    #[serde(rename = "type")]
+    pub(crate) kind: CreateType,
+    pub(crate) id: Url,
+}
+
+impl CreatePost {
+    pub fn new(note: Note, id: Url) -> CreatePost {
+        CreatePost {
+            actor: note.attributed_to.clone(),
+            to: note.to.clone(),
+            object: note,
+            kind: CreateType::Create,
+            id,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl ActivityHandler for CreatePost {
+    type DataType = DatabaseHandle;
+    type Error = crate::error::Error;
+
+    fn id(&self) -> &Url {
+        &self.id
+    }
+
+    fn actor(&self) -> &Url {
+        self.actor.inner()
+    }
+
+    async fn verify(&self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        DbPost::verify(&self.object, &self.id, data).await?;
+        Ok(())
+    }
+
+    async fn receive(self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        DbPost::from_json(self.object, data).await?;
+        Ok(())
+    }
+}

examples/local_federation/activities/follow.rs

@@ -0,0 +1,74 @@
+use crate::{
+    activities::accept::Accept,
+    generate_object_id,
+    instance::DatabaseHandle,
+    objects::person::DbUser,
+};
+use activitypub_federation::{
+    config::Data,
+    fetch::object_id::ObjectId,
+    kinds::activity::FollowType,
+    traits::{ActivityHandler, Actor},
+};
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+#[derive(Deserialize, Serialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Follow {
+    pub(crate) actor: ObjectId<DbUser>,
+    pub(crate) object: ObjectId<DbUser>,
+    #[serde(rename = "type")]
+    kind: FollowType,
+    id: Url,
+}
+
+impl Follow {
+    pub fn new(actor: ObjectId<DbUser>, object: ObjectId<DbUser>, id: Url) -> Follow {
+        Follow {
+            actor,
+            object,
+            kind: Default::default(),
+            id,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl ActivityHandler for Follow {
+    type DataType = DatabaseHandle;
+    type Error = crate::error::Error;
+
+    fn id(&self) -> &Url {
+        &self.id
+    }
+
+    fn actor(&self) -> &Url {
+        self.actor.inner()
+    }
+
+    async fn verify(&self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        Ok(())
+    }
+
+    // Ignore clippy false positive: https://github.com/rust-lang/rust-clippy/issues/6446
+    #[allow(clippy::await_holding_lock)]
+    async fn receive(self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        // add to followers
+        let local_user = {
+            let mut users = data.users.lock().unwrap();
+            let local_user = users.first_mut().unwrap();
+            local_user.followers.push(self.actor.inner().clone());
+            local_user.clone()
+        };
+
+        // send back an accept
+        let follower = self.actor.dereference(data).await?;
+        let id = generate_object_id(data.domain())?;
+        let accept = Accept::new(local_user.ap_id.clone(), self, id.clone());
+        local_user
+            .send(accept, vec![follower.shared_inbox_or_inbox()], false, data)
+            .await?;
+        Ok(())
+    }
+}

examples/local_federation/activities/mod.rs

@@ -1,3 +1,3 @@
 pub mod accept;
-pub mod create_note;
+pub mod create_post;
 pub mod follow;

examples/local_federation/actix_web/http.rs

@@ -0,0 +1,97 @@
+use crate::{
+    error::Error,
+    instance::DatabaseHandle,
+    objects::person::{DbUser, PersonAcceptedActivities},
+};
+use activitypub_federation::{
+    actix_web::{inbox::receive_activity, signing_actor},
+    config::{Data, FederationConfig, FederationMiddleware},
+    fetch::webfinger::{build_webfinger_response, extract_webfinger_name},
+    protocol::context::WithContext,
+    traits::{Actor, Object},
+    FEDERATION_CONTENT_TYPE,
+};
+use actix_web::{web, web::Bytes, App, HttpRequest, HttpResponse, HttpServer};
+use anyhow::anyhow;
+use serde::Deserialize;
+use tracing::info;
+
+pub fn listen(config: &FederationConfig<DatabaseHandle>) -> Result<(), Error> {
+    let hostname = config.domain();
+    info!("Listening with actix-web on {hostname}");
+    let config = config.clone();
+    let server = HttpServer::new(move || {
+        App::new()
+            .wrap(FederationMiddleware::new(config.clone()))
+            .route("/", web::get().to(http_get_system_user))
+            .route("/{user}", web::get().to(http_get_user))
+            .route("/{user}/inbox", web::post().to(http_post_user_inbox))
+            .route("/.well-known/webfinger", web::get().to(webfinger))
+    })
+    .bind(hostname)?
+    .run();
+    tokio::spawn(server);
+    Ok(())
+}
+
+/// Handles requests to fetch system user json over HTTP
+pub async fn http_get_system_user(data: Data<DatabaseHandle>) -> Result<HttpResponse, Error> {
+    let json_user = data.system_user.clone().into_json(&data).await?;
+    Ok(HttpResponse::Ok()
+        .content_type(FEDERATION_CONTENT_TYPE)
+        .json(WithContext::new_default(json_user)))
+}
+
+/// Handles requests to fetch user json over HTTP
+pub async fn http_get_user(
+    request: HttpRequest,
+    user_name: web::Path<String>,
+    data: Data<DatabaseHandle>,
+) -> Result<HttpResponse, Error> {
+    let signed_by = signing_actor::<DbUser>(&request, None, &data).await?;
+    // here, checks can be made on the actor or the domain to which
+    // it belongs, to verify whether it is allowed to access this resource
+    info!(
+        "Fetch user request is signed by system account {}",
+        signed_by.id()
+    );
+
+    let db_user = data.local_user();
+    if user_name.into_inner() == db_user.name {
+        let json_user = db_user.into_json(&data).await?;
+        Ok(HttpResponse::Ok()
+            .content_type(FEDERATION_CONTENT_TYPE)
+            .json(WithContext::new_default(json_user)))
+    } else {
+        Err(anyhow!("Invalid user").into())
+    }
+}
+
+/// Handles messages received in user inbox
+pub async fn http_post_user_inbox(
+    request: HttpRequest,
+    body: Bytes,
+    data: Data<DatabaseHandle>,
+) -> Result<HttpResponse, Error> {
+    receive_activity::<WithContext<PersonAcceptedActivities>, DbUser, DatabaseHandle>(
+        request, body, &data,
+    )
+    .await
+}
+
+#[derive(Deserialize)]
+pub struct WebfingerQuery {
+    resource: String,
+}
+
+pub async fn webfinger(
+    query: web::Query<WebfingerQuery>,
+    data: Data<DatabaseHandle>,
+) -> Result<HttpResponse, Error> {
+    let name = extract_webfinger_name(&query.resource, &data)?;
+    let db_user = data.read_user(name)?;
+    Ok(HttpResponse::Ok().json(build_webfinger_response(
+        query.resource.clone(),
+        db_user.ap_id.into_inner(),
+    )))
+}

examples/local_federation/actix_web/mod.rs

@@ -0,0 +1,6 @@
+use crate::error::Error;
+use actix_web::ResponseError;
+
+pub(crate) mod http;
+
+impl ResponseError for Error {}

examples/local_federation/axum/http.rs

@@ -0,0 +1,86 @@
+use crate::{
+    error::Error,
+    instance::DatabaseHandle,
+    objects::person::{DbUser, Person, PersonAcceptedActivities},
+};
+use activitypub_federation::{
+    axum::{
+        inbox::{receive_activity, ActivityData},
+        json::FederationJson,
+    },
+    config::{Data, FederationConfig, FederationMiddleware},
+    fetch::webfinger::{build_webfinger_response, extract_webfinger_name, Webfinger},
+    protocol::context::WithContext,
+    traits::Object,
+};
+use axum::{
+    extract::{Path, Query},
+    response::IntoResponse,
+    routing::{get, post},
+    Json,
+    Router,
+};
+use axum_macros::debug_handler;
+use serde::Deserialize;
+use std::net::ToSocketAddrs;
+use tracing::info;
+
+pub fn listen(config: &FederationConfig<DatabaseHandle>) -> Result<(), Error> {
+    let hostname = config.domain();
+    info!("Listening with axum on {hostname}");
+    let config = config.clone();
+    let app = Router::new()
+        .route("/:user/inbox", post(http_post_user_inbox))
+        .route("/:user", get(http_get_user))
+        .route("/.well-known/webfinger", get(webfinger))
+        .layer(FederationMiddleware::new(config));
+
+    let addr = hostname
+        .to_socket_addrs()?
+        .next()
+        .expect("Failed to lookup domain name");
+    let server = axum::Server::bind(&addr).serve(app.into_make_service());
+
+    tokio::spawn(server);
+    Ok(())
+}
+
+#[debug_handler]
+async fn http_get_user(
+    Path(name): Path<String>,
+    data: Data<DatabaseHandle>,
+) -> Result<FederationJson<WithContext<Person>>, Error> {
+    let db_user = data.read_user(&name)?;
+    let json_user = db_user.into_json(&data).await?;
+    Ok(FederationJson(WithContext::new_default(json_user)))
+}
+
+#[debug_handler]
+async fn http_post_user_inbox(
+    data: Data<DatabaseHandle>,
+    activity_data: ActivityData,
+) -> impl IntoResponse {
+    receive_activity::<WithContext<PersonAcceptedActivities>, DbUser, DatabaseHandle>(
+        activity_data,
+        &data,
+    )
+    .await
+}
+
+#[derive(Deserialize)]
+struct WebfingerQuery {
+    resource: String,
+}
+
+#[debug_handler]
+async fn webfinger(
+    Query(query): Query<WebfingerQuery>,
+    data: Data<DatabaseHandle>,
+) -> Result<Json<Webfinger>, Error> {
+    let name = extract_webfinger_name(&query.resource, &data)?;
+    let db_user = data.read_user(name)?;
+    Ok(Json(build_webfinger_response(
+        query.resource,
+        db_user.ap_id.into_inner(),
+    )))
+}

examples/local_federation/axum/mod.rs

@@ -0,0 +1,12 @@
+use crate::error::Error;
+use ::http::StatusCode;
+use axum::response::{IntoResponse, Response};
+
+#[allow(clippy::diverging_sub_expression, clippy::items_after_statements)]
+pub mod http;
+
+impl IntoResponse for Error {
+    fn into_response(self) -> Response {
+        (StatusCode::INTERNAL_SERVER_ERROR, format!("{}", self.0)).into_response()
+    }
+}

examples/local_federation/error.rs

@@ -0,0 +1,20 @@
+use std::fmt::{Display, Formatter};
+
+/// Necessary because of this issue: https://github.com/actix/actix-web/issues/1711
+#[derive(Debug)]
+pub struct Error(pub(crate) anyhow::Error);
+
+impl Display for Error {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.0, f)
+    }
+}
+
+impl<T> From<T> for Error
+where
+    T: Into<anyhow::Error>,
+{
+    fn from(t: T) -> Self {
+        Error(t.into())
+    }
+}

examples/local_federation/instance.rs

@@ -0,0 +1,106 @@
+use crate::{
+    objects::{person::DbUser, post::DbPost},
+    Error,
+};
+use activitypub_federation::config::{FederationConfig, UrlVerifier};
+use anyhow::anyhow;
+use async_trait::async_trait;
+use std::{
+    str::FromStr,
+    sync::{Arc, Mutex},
+};
+use url::Url;
+
+pub async fn new_instance(
+    hostname: &str,
+    name: String,
+) -> Result<FederationConfig<DatabaseHandle>, Error> {
+    let mut system_user = DbUser::new(hostname, "system".into())?;
+    system_user.ap_id = Url::parse(&format!("http://{}/", hostname))?.into();
+
+    let local_user = DbUser::new(hostname, name)?;
+    let database = Arc::new(Database {
+        system_user: system_user.clone(),
+        users: Mutex::new(vec![local_user]),
+        posts: Mutex::new(vec![]),
+    });
+    let config = FederationConfig::builder()
+        .domain(hostname)
+        .signed_fetch_actor(&system_user)
+        .app_data(database)
+        .url_verifier(Box::new(MyUrlVerifier()))
+        .debug(true)
+        .build()
+        .await?;
+    Ok(config)
+}
+
+pub type DatabaseHandle = Arc<Database>;
+
+/// Our "database" which contains all known posts and users (local and federated)
+pub struct Database {
+    pub system_user: DbUser,
+    pub users: Mutex<Vec<DbUser>>,
+    pub posts: Mutex<Vec<DbPost>>,
+}
+
+/// Use this to store your federation blocklist, or a database connection needed to retrieve it.
+#[derive(Clone)]
+struct MyUrlVerifier();
+
+#[async_trait]
+impl UrlVerifier for MyUrlVerifier {
+    async fn verify(&self, url: &Url) -> Result<(), activitypub_federation::error::Error> {
+        if url.domain() == Some("malicious.com") {
+            Err(activitypub_federation::error::Error::Other(
+                "malicious domain".into(),
+            ))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+pub enum Webserver {
+    Axum,
+    ActixWeb,
+}
+
+impl FromStr for Webserver {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "axum" => Webserver::Axum,
+            "actix-web" => Webserver::ActixWeb,
+            _ => panic!("Invalid webserver parameter, must be either `axum` or `actix-web`"),
+        })
+    }
+}
+
+pub fn listen(
+    config: &FederationConfig<DatabaseHandle>,
+    webserver: &Webserver,
+) -> Result<(), Error> {
+    match webserver {
+        Webserver::Axum => crate::axum::http::listen(config)?,
+        Webserver::ActixWeb => crate::actix_web::http::listen(config)?,
+    }
+    Ok(())
+}
+
+impl Database {
+    pub fn local_user(&self) -> DbUser {
+        let lock = self.users.lock().unwrap();
+        lock.first().unwrap().clone()
+    }
+
+    pub fn read_user(&self, name: &str) -> Result<DbUser, Error> {
+        let db_user = self.local_user();
+        if name == db_user.name {
+            Ok(db_user)
+        } else {
+            Err(anyhow!("Invalid user {name}").into())
+        }
+    }
+}

examples/local_federation/main.rs

@@ -0,0 +1,68 @@
+#![allow(clippy::unwrap_used)]
+
+use crate::{
+    instance::{listen, new_instance, Webserver},
+    objects::post::DbPost,
+    utils::generate_object_id,
+};
+use error::Error;
+use std::{env::args, str::FromStr};
+use tracing::log::{info, LevelFilter};
+
+mod activities;
+#[cfg(feature = "actix-web")]
+mod actix_web;
+#[cfg(feature = "axum")]
+mod axum;
+mod error;
+mod instance;
+mod objects;
+mod utils;
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    env_logger::builder()
+        .filter_level(LevelFilter::Warn)
+        .filter_module("activitypub_federation", LevelFilter::Info)
+        .filter_module("local_federation", LevelFilter::Info)
+        .format_timestamp(None)
+        .init();
+
+    info!("Start with parameter `axum` or `actix-web` to select the webserver");
+    let webserver = args()
+        .nth(1)
+        .map(|arg| Webserver::from_str(&arg).unwrap())
+        .unwrap_or(Webserver::Axum);
+
+    let alpha = new_instance("localhost:8001", "alpha".to_string()).await?;
+    let beta = new_instance("localhost:8002", "beta".to_string()).await?;
+    listen(&alpha, &webserver)?;
+    listen(&beta, &webserver)?;
+    info!("Local instances started");
+
+    info!("Alpha user follows beta user via webfinger");
+    alpha
+        .local_user()
+        .follow("beta@localhost:8002", &alpha.to_request_data())
+        .await?;
+    assert_eq!(
+        beta.local_user().followers(),
+        &vec![alpha.local_user().ap_id.inner().clone()]
+    );
+    info!("Follow was successful");
+
+    info!("Beta sends a post to its followers");
+    let sent_post = DbPost::new("Hello world!".to_string(), beta.local_user().ap_id)?;
+    beta.local_user()
+        .post(sent_post.clone(), &beta.to_request_data())
+        .await?;
+    let received_post = alpha.posts.lock().unwrap().first().cloned().unwrap();
+    info!("Alpha received post: {}", received_post.text);
+
+    // assert that alpha received the post
+    assert_eq!(received_post.text, sent_post.text);
+    assert_eq!(received_post.ap_id.inner(), sent_post.ap_id.inner());
+    assert_eq!(received_post.creator.inner(), sent_post.creator.inner());
+    info!("Test completed");
+    Ok(())
+}

examples/local_federation/objects/mod.rs

@@ -0,0 +1,2 @@
+pub mod person;
+pub mod post;

examples/local_federation/objects/person.rs

@@ -0,0 +1,205 @@
+use crate::{
+    activities::{accept::Accept, create_post::CreatePost, follow::Follow},
+    error::Error,
+    instance::DatabaseHandle,
+    objects::post::DbPost,
+    utils::generate_object_id,
+};
+use activitypub_federation::{
+    activity_queue::queue_activity,
+    activity_sending::SendActivityTask,
+    config::Data,
+    fetch::{object_id::ObjectId, webfinger::webfinger_resolve_actor},
+    http_signatures::generate_actor_keypair,
+    kinds::actor::PersonType,
+    protocol::{context::WithContext, public_key::PublicKey, verification::verify_domains_match},
+    traits::{ActivityHandler, Actor, Object},
+};
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use std::fmt::Debug;
+use url::Url;
+
+#[derive(Debug, Clone)]
+pub struct DbUser {
+    pub name: String,
+    pub ap_id: ObjectId<DbUser>,
+    pub inbox: Url,
+    // exists for all users (necessary to verify http signatures)
+    public_key: String,
+    // exists only for local users
+    private_key: Option<String>,
+    last_refreshed_at: DateTime<Utc>,
+    pub followers: Vec<Url>,
+    pub local: bool,
+}
+
+/// List of all activities which this actor can receive.
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(untagged)]
+#[enum_delegate::implement(ActivityHandler)]
+pub enum PersonAcceptedActivities {
+    Follow(Follow),
+    Accept(Accept),
+    CreateNote(CreatePost),
+}
+
+impl DbUser {
+    pub fn new(hostname: &str, name: String) -> Result<DbUser, Error> {
+        let ap_id = Url::parse(&format!("http://{}/{}", hostname, &name))?.into();
+        let inbox = Url::parse(&format!("http://{}/{}/inbox", hostname, &name))?;
+        let keypair = generate_actor_keypair()?;
+        Ok(DbUser {
+            name,
+            ap_id,
+            inbox,
+            public_key: keypair.public_key,
+            private_key: Some(keypair.private_key),
+            last_refreshed_at: Utc::now(),
+            followers: vec![],
+            local: true,
+        })
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Person {
+    #[serde(rename = "type")]
+    kind: PersonType,
+    preferred_username: String,
+    id: ObjectId<DbUser>,
+    inbox: Url,
+    public_key: PublicKey,
+}
+
+impl DbUser {
+    pub fn followers(&self) -> &Vec<Url> {
+        &self.followers
+    }
+
+    pub fn followers_url(&self) -> Result<Url, Error> {
+        Ok(Url::parse(&format!("{}/followers", self.ap_id.inner()))?)
+    }
+
+    pub async fn follow(&self, other: &str, data: &Data<DatabaseHandle>) -> Result<(), Error> {
+        let other: DbUser = webfinger_resolve_actor(other, data).await?;
+        let id = generate_object_id(data.domain())?;
+        let follow = Follow::new(self.ap_id.clone(), other.ap_id.clone(), id.clone());
+        self.send(follow, vec![other.shared_inbox_or_inbox()], false, data)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn post(&self, post: DbPost, data: &Data<DatabaseHandle>) -> Result<(), Error> {
+        let id = generate_object_id(data.domain())?;
+        let create = CreatePost::new(post.into_json(data).await?, id.clone());
+        let mut inboxes = vec![];
+        for f in self.followers.clone() {
+            let user: DbUser = ObjectId::from(f).dereference(data).await?;
+            inboxes.push(user.shared_inbox_or_inbox());
+        }
+        self.send(create, inboxes, true, data).await?;
+        Ok(())
+    }
+
+    pub(crate) async fn send<Activity>(
+        &self,
+        activity: Activity,
+        recipients: Vec<Url>,
+        use_queue: bool,
+        data: &Data<DatabaseHandle>,
+    ) -> Result<(), Error>
+    where
+        Activity: ActivityHandler + Serialize + Debug + Send + Sync,
+        <Activity as ActivityHandler>::Error: From<anyhow::Error> + From<serde_json::Error>,
+    {
+        let activity = WithContext::new_default(activity);
+        // Send through queue in some cases and bypass it in others to test both code paths
+        if use_queue {
+            queue_activity(&activity, self, recipients, data).await?;
+        } else {
+            let sends = SendActivityTask::prepare(&activity, self, recipients, data).await?;
+            for send in sends {
+                send.sign_and_send(data).await?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl Object for DbUser {
+    type DataType = DatabaseHandle;
+    type Kind = Person;
+    type Error = Error;
+
+    fn last_refreshed_at(&self) -> Option<DateTime<Utc>> {
+        Some(self.last_refreshed_at)
+    }
+
+    async fn read_from_id(
+        object_id: Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<Option<Self>, Self::Error> {
+        let users = data.users.lock().unwrap();
+        let res = users
+            .clone()
+            .into_iter()
+            .find(|u| u.ap_id.inner() == &object_id);
+        Ok(res)
+    }
+
+    async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+        Ok(Person {
+            preferred_username: self.name.clone(),
+            kind: Default::default(),
+            id: self.ap_id.clone(),
+            inbox: self.inbox.clone(),
+            public_key: self.public_key(),
+        })
+    }
+
+    async fn verify(
+        json: &Self::Kind,
+        expected_domain: &Url,
+        _data: &Data<Self::DataType>,
+    ) -> Result<(), Self::Error> {
+        verify_domains_match(json.id.inner(), expected_domain)?;
+        Ok(())
+    }
+
+    async fn from_json(json: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, Self::Error> {
+        let user = DbUser {
+            name: json.preferred_username,
+            ap_id: json.id,
+            inbox: json.inbox,
+            public_key: json.public_key.public_key_pem,
+            private_key: None,
+            last_refreshed_at: Utc::now(),
+            followers: vec![],
+            local: false,
+        };
+        let mut mutex = data.users.lock().unwrap();
+        mutex.push(user.clone());
+        Ok(user)
+    }
+}
+
+impl Actor for DbUser {
+    fn id(&self) -> Url {
+        self.ap_id.inner().clone()
+    }
+
+    fn public_key_pem(&self) -> &str {
+        &self.public_key
+    }
+
+    fn private_key_pem(&self) -> Option<String> {
+        self.private_key.clone()
+    }
+
+    fn inbox(&self) -> Url {
+        self.inbox.clone()
+    }
+}

examples/local_federation/objects/post.rs

@@ -0,0 +1,94 @@
+use crate::{error::Error, generate_object_id, instance::DatabaseHandle, objects::person::DbUser};
+use activitypub_federation::{
+    config::Data,
+    fetch::object_id::ObjectId,
+    kinds::{object::NoteType, public},
+    protocol::{helpers::deserialize_one_or_many, verification::verify_domains_match},
+    traits::Object,
+};
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+#[derive(Clone, Debug)]
+pub struct DbPost {
+    pub text: String,
+    pub ap_id: ObjectId<DbPost>,
+    pub creator: ObjectId<DbUser>,
+    pub local: bool,
+}
+
+impl DbPost {
+    pub fn new(text: String, creator: ObjectId<DbUser>) -> Result<DbPost, Error> {
+        let ap_id = generate_object_id(creator.inner().domain().unwrap())?.into();
+        Ok(DbPost {
+            text,
+            ap_id,
+            creator,
+            local: true,
+        })
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Note {
+    #[serde(rename = "type")]
+    kind: NoteType,
+    id: ObjectId<DbPost>,
+    pub(crate) attributed_to: ObjectId<DbUser>,
+    #[serde(deserialize_with = "deserialize_one_or_many")]
+    pub(crate) to: Vec<Url>,
+    content: String,
+}
+
+#[async_trait::async_trait]
+impl Object for DbPost {
+    type DataType = DatabaseHandle;
+    type Kind = Note;
+    type Error = Error;
+
+    async fn read_from_id(
+        object_id: Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<Option<Self>, Self::Error> {
+        let posts = data.posts.lock().unwrap();
+        let res = posts
+            .clone()
+            .into_iter()
+            .find(|u| u.ap_id.inner() == &object_id);
+        Ok(res)
+    }
+
+    async fn into_json(self, data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+        let creator = self.creator.dereference_local(data).await?;
+        Ok(Note {
+            kind: Default::default(),
+            id: self.ap_id,
+            attributed_to: self.creator,
+            to: vec![public(), creator.followers_url()?],
+            content: self.text,
+        })
+    }
+
+    async fn verify(
+        json: &Self::Kind,
+        expected_domain: &Url,
+        _data: &Data<Self::DataType>,
+    ) -> Result<(), Self::Error> {
+        verify_domains_match(json.id.inner(), expected_domain)?;
+        Ok(())
+    }
+
+    async fn from_json(json: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, Self::Error> {
+        let post = DbPost {
+            text: json.content,
+            ap_id: json.id,
+            creator: json.attributed_to,
+            local: false,
+        };
+
+        let mut lock = data.posts.lock().unwrap();
+        lock.push(post.clone());
+        Ok(post)
+    }
+}

examples/local_federation/utils.rs

@@ -0,0 +1,13 @@
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use url::{ParseError, Url};
+
+/// Just generate random url as object id. In a real project, you probably want to use
+/// an url which contains the database id for easy retrieval (or store the random id in db).
+pub fn generate_object_id(domain: &str) -> Result<Url, ParseError> {
+    let id: String = thread_rng()
+        .sample_iter(&Alphanumeric)
+        .take(7)
+        .map(char::from)
+        .collect();
+    Url::parse(&format!("http://{}/objects/{}", domain, id))
+}

src/activity_queue.rs

@@ -0,0 +1,521 @@
+//! Queue for signing and sending outgoing activities with retry
+//!
+#![doc = include_str!("../docs/09_sending_activities.md")]
+
+use crate::{
+    activity_sending::{build_tasks, SendActivityTask},
+    config::Data,
+    error::Error,
+    traits::{ActivityHandler, Actor},
+};
+
+use futures_core::Future;
+
+use reqwest_middleware::ClientWithMiddleware;
+use serde::Serialize;
+use std::{
+    fmt::{Debug, Display},
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+use tokio::{
+    sync::mpsc::{unbounded_channel, UnboundedSender},
+    task::{JoinHandle, JoinSet},
+};
+use tracing::{info, warn};
+use url::Url;
+
+/// Send a new activity to the given inboxes with automatic retry on failure. Alternatively you
+/// can implement your own queue and then send activities using [[crate::activity_sending::SendActivityTask]].
+///
+/// - `activity`: The activity to be sent, gets converted to json
+/// - `private_key`: Private key belonging to the actor who sends the activity, for signing HTTP
+///                  signature. Generated with [crate::http_signatures::generate_actor_keypair].
+/// - `inboxes`: List of remote actor inboxes that should receive the activity. Ignores local actor
+///              inboxes. Should be built by calling [crate::traits::Actor::shared_inbox_or_inbox]
+///              for each target actor.
+pub async fn queue_activity<Activity, Datatype, ActorType>(
+    activity: &Activity,
+    actor: &ActorType,
+    inboxes: Vec<Url>,
+    data: &Data<Datatype>,
+) -> Result<(), Error>
+where
+    Activity: ActivityHandler + Serialize + Debug,
+    Datatype: Clone,
+    ActorType: Actor,
+{
+    let config = &data.config;
+    let tasks = build_tasks(activity, actor, inboxes, data).await?;
+
+    for task in tasks {
+        // Don't use the activity queue if this is in debug mode, send and wait directly
+        if config.debug {
+            if let Err(err) = sign_and_send(
+                &task,
+                &config.client,
+                config.request_timeout,
+                Default::default(),
+            )
+            .await
+            {
+                warn!("{err}");
+            }
+        } else {
+            // This field is only optional to make builder work, its always present at this point
+            let activity_queue = config
+                .activity_queue
+                .as_ref()
+                .expect("Config has activity queue");
+            activity_queue.queue(task).await?;
+            let stats = activity_queue.get_stats();
+            let running = stats.running.load(Ordering::Relaxed);
+            if running == config.queue_worker_count && config.queue_worker_count != 0 {
+                warn!("Reached max number of send activity workers ({}). Consider increasing worker count to avoid federation delays", config.queue_worker_count);
+                warn!("{:?}", stats);
+            } else {
+                info!("{:?}", stats);
+            }
+        }
+    }
+    Ok(())
+}
+
+async fn sign_and_send(
+    task: &SendActivityTask,
+    client: &ClientWithMiddleware,
+    timeout: Duration,
+    retry_strategy: RetryStrategy,
+) -> Result<(), Error> {
+    retry(
+        || task.sign_and_send_internal(client, timeout),
+        retry_strategy,
+    )
+    .await
+}
+
+/// A simple activity queue which spawns tokio workers to send out requests
+/// When creating a queue, it will spawn a task per worker thread
+/// Uses an unbounded mpsc queue for communication (i.e, all messages are in memory)
+pub(crate) struct ActivityQueue {
+    // Stats shared between the queue and workers
+    stats: Arc<Stats>,
+    sender: UnboundedSender<SendActivityTask>,
+    sender_task: JoinHandle<()>,
+    retry_sender_task: JoinHandle<()>,
+}
+
+/// Simple stat counter to show where we're up to with sending messages
+/// This is a lock-free way to share things between tasks
+/// When reading these values it's possible (but extremely unlikely) to get stale data if a worker task is in the middle of transitioning
+#[derive(Default)]
+pub(crate) struct Stats {
+    pending: AtomicUsize,
+    running: AtomicUsize,
+    retries: AtomicUsize,
+    dead_last_hour: AtomicUsize,
+    completed_last_hour: AtomicUsize,
+}
+
+impl Debug for Stats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Activity queue stats: pending: {}, running: {}, retries: {}, dead: {}, complete: {}",
+            self.pending.load(Ordering::Relaxed),
+            self.running.load(Ordering::Relaxed),
+            self.retries.load(Ordering::Relaxed),
+            self.dead_last_hour.load(Ordering::Relaxed),
+            self.completed_last_hour.load(Ordering::Relaxed)
+        )
+    }
+}
+
+#[derive(Clone, Copy, Default)]
+struct RetryStrategy {
+    /// Amount of time in seconds to back off
+    backoff: usize,
+    /// Amount of times to retry
+    retries: usize,
+    /// If this particular request has already been retried, you can add an offset here to increment the count to start
+    offset: usize,
+    /// Number of seconds to sleep before trying
+    initial_sleep: usize,
+}
+
+/// A tokio spawned worker which is responsible for submitting requests to federated servers
+/// This will retry up to one time with the same signature, and if it fails, will move it to the retry queue.
+/// We need to retry activity sending in case the target instances is temporarily unreachable.
+/// In this case, the task is stored and resent when the instance is hopefully back up. This
+/// list shows the retry intervals, and which events of the target instance can be covered:
+/// - 60s (one minute, service restart) -- happens in the worker w/ same signature
+/// - 60min (one hour, instance maintenance) --- happens in the retry worker
+/// - 60h (2.5 days, major incident with rebuild from backup) --- happens in the retry worker
+async fn worker(
+    client: ClientWithMiddleware,
+    timeout: Duration,
+    message: SendActivityTask,
+    retry_queue: UnboundedSender<SendActivityTask>,
+    stats: Arc<Stats>,
+    strategy: RetryStrategy,
+) {
+    stats.pending.fetch_sub(1, Ordering::Relaxed);
+    stats.running.fetch_add(1, Ordering::Relaxed);
+
+    let outcome = sign_and_send(&message, &client, timeout, strategy).await;
+
+    // "Running" has finished, check the outcome
+    stats.running.fetch_sub(1, Ordering::Relaxed);
+
+    match outcome {
+        Ok(_) => {
+            stats.completed_last_hour.fetch_add(1, Ordering::Relaxed);
+        }
+        Err(_err) => {
+            stats.retries.fetch_add(1, Ordering::Relaxed);
+            warn!(
+                "Sending activity {} to {} to the retry queue to be tried again later",
+                message.activity_id, message.inbox
+            );
+            // Send to the retry queue.  Ignoring whether it succeeds or not
+            retry_queue.send(message).ok();
+        }
+    }
+}
+
+async fn retry_worker(
+    client: ClientWithMiddleware,
+    timeout: Duration,
+    message: SendActivityTask,
+    stats: Arc<Stats>,
+    strategy: RetryStrategy,
+) {
+    // Because the times are pretty extravagant between retries, we have to re-sign each time
+    let outcome = retry(
+        || {
+            sign_and_send(
+                &message,
+                &client,
+                timeout,
+                RetryStrategy {
+                    backoff: 0,
+                    retries: 0,
+                    offset: 0,
+                    initial_sleep: 0,
+                },
+            )
+        },
+        strategy,
+    )
+    .await;
+
+    stats.retries.fetch_sub(1, Ordering::Relaxed);
+
+    match outcome {
+        Ok(_) => {
+            stats.completed_last_hour.fetch_add(1, Ordering::Relaxed);
+        }
+        Err(_err) => {
+            stats.dead_last_hour.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+}
+
+impl ActivityQueue {
+    fn new(
+        client: ClientWithMiddleware,
+        worker_count: usize,
+        retry_count: usize,
+        timeout: Duration,
+        backoff: usize, // This should be 60 seconds by default or 1 second in tests
+    ) -> Self {
+        let stats: Arc<Stats> = Default::default();
+
+        // This task clears the dead/completed stats every hour
+        let hour_stats = stats.clone();
+        tokio::spawn(async move {
+            let duration = Duration::from_secs(3600);
+            loop {
+                tokio::time::sleep(duration).await;
+                hour_stats.completed_last_hour.store(0, Ordering::Relaxed);
+                hour_stats.dead_last_hour.store(0, Ordering::Relaxed);
+            }
+        });
+
+        let (retry_sender, mut retry_receiver) = unbounded_channel();
+        let retry_stats = stats.clone();
+        let retry_client = client.clone();
+
+        // The "fast path" retry
+        // The backoff should be < 5 mins for this to work otherwise signatures may expire
+        // This strategy is the one that is used with the *same* signature
+        let strategy = RetryStrategy {
+            backoff,
+            retries: 1,
+            offset: 0,
+            initial_sleep: 0,
+        };
+
+        // The "retry path" strategy
+        // After the fast path fails, a task will sleep up to backoff ^ 2 and then retry again
+        let retry_strategy = RetryStrategy {
+            backoff,
+            retries: 3,
+            offset: 2,
+            initial_sleep: backoff.pow(2), // wait 60 mins before even trying
+        };
+
+        let retry_sender_task = tokio::spawn(async move {
+            let mut join_set = JoinSet::new();
+
+            while let Some(message) = retry_receiver.recv().await {
+                let retry_task = retry_worker(
+                    retry_client.clone(),
+                    timeout,
+                    message,
+                    retry_stats.clone(),
+                    retry_strategy,
+                );
+
+                if retry_count > 0 {
+                    // If we're over the limit of retries, wait for them to finish before spawning
+                    while join_set.len() >= retry_count {
+                        join_set.join_next().await;
+                    }
+
+                    join_set.spawn(retry_task);
+                } else {
+                    // If the retry worker count is `0` then just spawn and don't use the join_set
+                    tokio::spawn(retry_task);
+                }
+            }
+
+            while !join_set.is_empty() {
+                join_set.join_next().await;
+            }
+        });
+
+        let (sender, mut receiver) = unbounded_channel();
+
+        let sender_stats = stats.clone();
+
+        let sender_task = tokio::spawn(async move {
+            let mut join_set = JoinSet::new();
+
+            while let Some(message) = receiver.recv().await {
+                let task = worker(
+                    client.clone(),
+                    timeout,
+                    message,
+                    retry_sender.clone(),
+                    sender_stats.clone(),
+                    strategy,
+                );
+
+                if worker_count > 0 {
+                    // If we're over the limit of workers, wait for them to finish before spawning
+                    while join_set.len() >= worker_count {
+                        join_set.join_next().await;
+                    }
+
+                    join_set.spawn(task);
+                } else {
+                    // If the worker count is `0` then just spawn and don't use the join_set
+                    tokio::spawn(task);
+                }
+            }
+
+            drop(retry_sender);
+
+            while !join_set.is_empty() {
+                join_set.join_next().await;
+            }
+        });
+
+        Self {
+            stats,
+            sender,
+            sender_task,
+            retry_sender_task,
+        }
+    }
+
+    async fn queue(&self, message: SendActivityTask) -> Result<(), Error> {
+        self.stats.pending.fetch_add(1, Ordering::Relaxed);
+        self.sender
+            .send(message)
+            .map_err(|e| Error::ActivityQueueError(e.0.activity_id))?;
+
+        Ok(())
+    }
+
+    fn get_stats(&self) -> &Stats {
+        &self.stats
+    }
+
+    #[allow(unused)]
+    // Drops all the senders and shuts down the workers
+    pub(crate) async fn shutdown(self, wait_for_retries: bool) -> Result<Arc<Stats>, Error> {
+        drop(self.sender);
+
+        self.sender_task.await?;
+
+        if wait_for_retries {
+            self.retry_sender_task.await?;
+        }
+
+        Ok(self.stats)
+    }
+}
+
+/// Creates an activity queue using tokio spawned tasks
+/// Note: requires a tokio runtime
+pub(crate) fn create_activity_queue(
+    client: ClientWithMiddleware,
+    worker_count: usize,
+    retry_count: usize,
+    request_timeout: Duration,
+) -> ActivityQueue {
+    ActivityQueue::new(client, worker_count, retry_count, request_timeout, 60)
+}
+
+/// Retries a future action factory function up to `amount` times with an exponential backoff timer between tries
+async fn retry<T, E: Display + Debug, F: Future<Output = Result<T, E>>, A: FnMut() -> F>(
+    mut action: A,
+    strategy: RetryStrategy,
+) -> Result<T, E> {
+    let mut count = strategy.offset;
+
+    // Do an initial sleep if it's called for
+    if strategy.initial_sleep > 0 {
+        let sleep_dur = Duration::from_secs(strategy.initial_sleep as u64);
+        tokio::time::sleep(sleep_dur).await;
+    }
+
+    loop {
+        match action().await {
+            Ok(val) => return Ok(val),
+            Err(err) => {
+                if count < strategy.retries {
+                    count += 1;
+
+                    let sleep_amt = strategy.backoff.pow(count as u32) as u64;
+                    let sleep_dur = Duration::from_secs(sleep_amt);
+                    warn!("{err:?}.  Sleeping for {sleep_dur:?} and trying again");
+                    tokio::time::sleep(sleep_dur).await;
+                    continue;
+                } else {
+                    return Err(err);
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::http_signatures::generate_actor_keypair;
+    use axum::extract::State;
+    use bytes::Bytes;
+    use http::{HeaderMap, StatusCode};
+    use std::time::Instant;
+    use tracing::debug;
+
+    // This will periodically send back internal errors to test the retry
+    async fn dodgy_handler(
+        State(state): State<Arc<AtomicUsize>>,
+        headers: HeaderMap,
+        body: Bytes,
+    ) -> Result<(), StatusCode> {
+        debug!("Headers:{:?}", headers);
+        debug!("Body len:{}", body.len());
+
+        if state.fetch_add(1, Ordering::Relaxed) % 20 == 0 {
+            return Err(StatusCode::INTERNAL_SERVER_ERROR);
+        }
+        Ok(())
+    }
+
+    async fn test_server() {
+        use axum::{routing::post, Router};
+
+        // We should break every now and then ;)
+        let state = Arc::new(AtomicUsize::new(0));
+
+        let app = Router::new()
+            .route("/", post(dodgy_handler))
+            .with_state(state);
+
+        axum::Server::bind(&"0.0.0.0:8002".parse().unwrap())
+            .serve(app.into_make_service())
+            .await
+            .unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    // Queues 100 messages and then asserts that the worker runs them
+    async fn test_activity_queue_workers() {
+        let num_workers = 64;
+        let num_messages: usize = 100;
+
+        tokio::spawn(test_server());
+
+        /*
+        // uncomment for debug logs & stats
+        use tracing::log::LevelFilter;
+
+        env_logger::builder()
+            .filter_level(LevelFilter::Warn)
+            .filter_module("activitypub_federation", LevelFilter::Info)
+            .format_timestamp(None)
+            .init();
+
+        */
+
+        let activity_queue = ActivityQueue::new(
+            reqwest::Client::default().into(),
+            num_workers,
+            num_workers,
+            Duration::from_secs(10),
+            1,
+        );
+
+        let keypair = generate_actor_keypair().unwrap();
+
+        let message = SendActivityTask {
+            actor_id: "http://localhost:8002".parse().unwrap(),
+            activity_id: "http://localhost:8002/activity".parse().unwrap(),
+            activity: "{}".into(),
+            inbox: "http://localhost:8002".parse().unwrap(),
+            private_key: keypair.private_key().unwrap(),
+            http_signature_compat: true,
+        };
+
+        let start = Instant::now();
+
+        for _ in 0..num_messages {
+            activity_queue.queue(message.clone()).await.unwrap();
+        }
+
+        info!("Queue Sent: {:?}", start.elapsed());
+
+        let stats = activity_queue.shutdown(true).await.unwrap();
+
+        info!(
+            "Queue Finished.  Num msgs: {}, Time {:?}, msg/s: {:0.0}",
+            num_messages,
+            start.elapsed(),
+            num_messages as f64 / start.elapsed().as_secs_f64()
+        );
+
+        assert_eq!(
+            stats.completed_last_hour.load(Ordering::Relaxed),
+            num_messages
+        );
+    }
+}

src/activity_sending.rs

@@ -0,0 +1,348 @@
+//! Queue for signing and sending outgoing activities with retry
+//!
+#![doc = include_str!("../docs/09_sending_activities.md")]
+
+use crate::{
+    config::Data,
+    error::Error,
+    http_signatures::sign_request,
+    reqwest_shim::ResponseExt,
+    traits::{ActivityHandler, Actor},
+    FEDERATION_CONTENT_TYPE,
+};
+use bytes::Bytes;
+use futures::StreamExt;
+use http::StatusCode;
+use httpdate::fmt_http_date;
+use itertools::Itertools;
+use openssl::pkey::{PKey, Private};
+use reqwest::{
+    header::{HeaderMap, HeaderName, HeaderValue},
+    Response,
+};
+use reqwest_middleware::ClientWithMiddleware;
+use serde::Serialize;
+use std::{
+    fmt::{Debug, Display},
+    time::{Duration, SystemTime},
+};
+use tracing::debug;
+use url::Url;
+
+#[derive(Clone, Debug)]
+/// All info needed to sign and send one activity to one inbox. You should generally use
+/// [[crate::activity_queue::queue_activity]] unless you want implement your own queue.
+pub struct SendActivityTask {
+    pub(crate) actor_id: Url,
+    pub(crate) activity_id: Url,
+    pub(crate) activity: Bytes,
+    pub(crate) inbox: Url,
+    pub(crate) private_key: PKey<Private>,
+    pub(crate) http_signature_compat: bool,
+}
+
+impl Display for SendActivityTask {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} to {}", self.activity_id, self.inbox)
+    }
+}
+
+impl SendActivityTask {
+    /// Prepare an activity for sending
+    ///
+    /// - `activity`: The activity to be sent, gets converted to json
+    /// - `inboxes`: List of remote actor inboxes that should receive the activity. Ignores local actor
+    ///              inboxes. Should be built by calling [crate::traits::Actor::shared_inbox_or_inbox]
+    ///              for each target actor.
+    pub async fn prepare<Activity, Datatype, ActorType>(
+        activity: &Activity,
+        actor: &ActorType,
+        inboxes: Vec<Url>,
+        data: &Data<Datatype>,
+    ) -> Result<Vec<SendActivityTask>, Error>
+    where
+        Activity: ActivityHandler + Serialize + Debug,
+        Datatype: Clone,
+        ActorType: Actor,
+    {
+        build_tasks(activity, actor, inboxes, data).await
+    }
+
+    /// convert a sendactivitydata to a request, signing and sending it
+    pub async fn sign_and_send<Datatype: Clone>(&self, data: &Data<Datatype>) -> Result<(), Error> {
+        self.sign_and_send_internal(&data.config.client, data.config.request_timeout)
+            .await
+    }
+
+    pub(crate) async fn sign_and_send_internal(
+        &self,
+        client: &ClientWithMiddleware,
+        timeout: Duration,
+    ) -> Result<(), Error> {
+        debug!("Sending {} to {}", self.activity_id, self.inbox,);
+        let request_builder = client
+            .post(self.inbox.to_string())
+            .timeout(timeout)
+            .headers(generate_request_headers(&self.inbox));
+        let request = sign_request(
+            request_builder,
+            &self.actor_id,
+            self.activity.clone(),
+            self.private_key.clone(),
+            self.http_signature_compat,
+        )
+        .await?;
+        let response = client.execute(request).await?;
+        self.handle_response(response).await
+    }
+
+    /// Based on the HTTP status code determines if an activity was delivered successfully. In that case
+    /// Ok is returned. Otherwise it returns Err and the activity send should be retried later.
+    ///
+    /// Equivalent code in mastodon: https://github.com/mastodon/mastodon/blob/v4.2.8/app/helpers/jsonld_helper.rb#L215-L217
+    async fn handle_response(&self, response: Response) -> Result<(), Error> {
+        match response.status() {
+            status if status.is_success() => {
+                debug!("Activity {self} delivered successfully");
+                Ok(())
+            }
+            status
+                if status.is_client_error()
+                    && status != StatusCode::REQUEST_TIMEOUT
+                    && status != StatusCode::TOO_MANY_REQUESTS =>
+            {
+                let text = response.text_limited().await?;
+                debug!("Activity {self} was rejected, aborting: {text}");
+                Ok(())
+            }
+            status => {
+                let text = response.text_limited().await?;
+
+                Err(Error::Other(format!(
+                    "Activity {self} failure with status {status}: {text}",
+                )))
+            }
+        }
+    }
+}
+
+pub(crate) async fn build_tasks<'a, Activity, Datatype, ActorType>(
+    activity: &'a Activity,
+    actor: &ActorType,
+    inboxes: Vec<Url>,
+    data: &Data<Datatype>,
+) -> Result<Vec<SendActivityTask>, Error>
+where
+    Activity: ActivityHandler + Serialize + Debug,
+    Datatype: Clone,
+    ActorType: Actor,
+{
+    let config = &data.config;
+    let actor_id = activity.actor();
+    let activity_id = activity.id();
+    let activity_serialized: Bytes = serde_json::to_vec(activity)
+        .map_err(|e| Error::SerializeOutgoingActivity(e, format!("{:?}", activity)))?
+        .into();
+    let private_key = get_pkey_cached(data, actor).await?;
+
+    Ok(futures::stream::iter(
+        inboxes
+            .into_iter()
+            .unique()
+            .filter(|i| !config.is_local_url(i)),
+    )
+    .filter_map(|inbox| async {
+        if let Err(err) = config.verify_url_valid(&inbox).await {
+            debug!("inbox url invalid, skipping: {inbox}: {err}");
+            return None;
+        };
+        Some(SendActivityTask {
+            actor_id: actor_id.clone(),
+            activity_id: activity_id.clone(),
+            inbox,
+            activity: activity_serialized.clone(),
+            private_key: private_key.clone(),
+            http_signature_compat: config.http_signature_compat,
+        })
+    })
+    .collect()
+    .await)
+}
+
+pub(crate) async fn get_pkey_cached<ActorType>(
+    data: &Data<impl Clone>,
+    actor: &ActorType,
+) -> Result<PKey<Private>, Error>
+where
+    ActorType: Actor,
+{
+    let actor_id = actor.id();
+    // PKey is internally like an Arc<>, so cloning is ok
+    data.config
+        .actor_pkey_cache
+        .try_get_with_by_ref(&actor_id, async {
+            let private_key_pem = actor.private_key_pem().ok_or_else(|| {
+                Error::Other(format!(
+                    "Actor {actor_id} does not contain a private key for signing"
+                ))
+            })?;
+
+            // This is a mostly expensive blocking call, we don't want to tie up other tasks while this is happening
+            let pkey = tokio::task::spawn_blocking(move || {
+                PKey::private_key_from_pem(private_key_pem.as_bytes()).map_err(|err| {
+                    Error::Other(format!("Could not create private key from PEM data:{err}"))
+                })
+            })
+            .await
+            .map_err(|err| Error::Other(format!("Error joining: {err}")))??;
+            std::result::Result::<PKey<Private>, Error>::Ok(pkey)
+        })
+        .await
+        .map_err(|e| Error::Other(format!("cloned error: {e}")))
+}
+
+pub(crate) fn generate_request_headers(inbox_url: &Url) -> HeaderMap {
+    let mut host = inbox_url.domain().expect("read inbox domain").to_string();
+    if let Some(port) = inbox_url.port() {
+        host = format!("{}:{}", host, port);
+    }
+
+    let mut headers = HeaderMap::new();
+    headers.insert(
+        HeaderName::from_static("content-type"),
+        HeaderValue::from_static(FEDERATION_CONTENT_TYPE),
+    );
+    headers.insert(
+        HeaderName::from_static("host"),
+        HeaderValue::from_str(&host).expect("Hostname is valid"),
+    );
+    headers.insert(
+        "date",
+        HeaderValue::from_str(&fmt_http_date(SystemTime::now())).expect("Date is valid"),
+    );
+    headers
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::{config::FederationConfig, http_signatures::generate_actor_keypair};
+    use std::{
+        sync::{atomic::AtomicUsize, Arc},
+        time::Instant,
+    };
+    use tracing::info;
+
+    // This will periodically send back internal errors to test the retry
+    async fn dodgy_handler(headers: HeaderMap, body: Bytes) -> Result<(), StatusCode> {
+        debug!("Headers:{:?}", headers);
+        debug!("Body len:{}", body.len());
+        Ok(())
+    }
+
+    async fn test_server() {
+        use axum::{routing::post, Router};
+
+        // We should break every now and then ;)
+        let state = Arc::new(AtomicUsize::new(0));
+
+        let app = Router::new()
+            .route("/", post(dodgy_handler))
+            .with_state(state);
+
+        axum::Server::bind(&"0.0.0.0:8001".parse().unwrap())
+            .serve(app.into_make_service())
+            .await
+            .unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    // Sends 100 messages
+    async fn test_activity_sending() -> anyhow::Result<()> {
+        let num_messages: usize = 100;
+
+        tokio::spawn(test_server());
+
+        /*
+        // uncomment for debug logs & stats
+        use tracing::log::LevelFilter;
+
+        env_logger::builder()
+            .filter_level(LevelFilter::Warn)
+            .filter_module("activitypub_federation", LevelFilter::Info)
+            .format_timestamp(None)
+            .init();
+
+        */
+        let keypair = generate_actor_keypair().unwrap();
+
+        let message = SendActivityTask {
+            actor_id: "http://localhost:8001".parse().unwrap(),
+            activity_id: "http://localhost:8001/activity".parse().unwrap(),
+            activity: "{}".into(),
+            inbox: "http://localhost:8001".parse().unwrap(),
+            private_key: keypair.private_key().unwrap(),
+            http_signature_compat: true,
+        };
+        let data = FederationConfig::builder()
+            .app_data(())
+            .domain("localhost")
+            .build()
+            .await?
+            .to_request_data();
+
+        let start = Instant::now();
+
+        for _ in 0..num_messages {
+            message.clone().sign_and_send(&data).await?;
+        }
+
+        info!("Queue Sent: {:?}", start.elapsed());
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_handle_response() {
+        let keypair = generate_actor_keypair().unwrap();
+        let message = SendActivityTask {
+            actor_id: "http://localhost:8001".parse().unwrap(),
+            activity_id: "http://localhost:8001/activity".parse().unwrap(),
+            activity: "{}".into(),
+            inbox: "http://localhost:8001".parse().unwrap(),
+            private_key: keypair.private_key().unwrap(),
+            http_signature_compat: true,
+        };
+
+        let res = |status| {
+            http::Response::builder()
+                .status(status)
+                .body(vec![])
+                .unwrap()
+                .into()
+        };
+
+        assert!(message.handle_response(res(StatusCode::OK)).await.is_ok());
+        assert!(message
+            .handle_response(res(StatusCode::BAD_REQUEST))
+            .await
+            .is_ok());
+
+        assert!(message
+            .handle_response(res(StatusCode::MOVED_PERMANENTLY))
+            .await
+            .is_err());
+        assert!(message
+            .handle_response(res(StatusCode::REQUEST_TIMEOUT))
+            .await
+            .is_err());
+        assert!(message
+            .handle_response(res(StatusCode::TOO_MANY_REQUESTS))
+            .await
+            .is_err());
+        assert!(message
+            .handle_response(res(StatusCode::INTERNAL_SERVER_ERROR))
+            .await
+            .is_err());
+    }
+}

src/actix_web/inbox.rs

@@ -0,0 +1,182 @@
+//! Handles incoming activities, verifying HTTP signatures and other checks
+
+use crate::{
+    config::Data,
+    error::Error,
+    http_signatures::{verify_body_hash, verify_signature},
+    parse_received_activity,
+    traits::{ActivityHandler, Actor, Object},
+};
+use actix_web::{web::Bytes, HttpRequest, HttpResponse};
+use serde::de::DeserializeOwned;
+use tracing::debug;
+
+/// Handles incoming activities, verifying HTTP signatures and other checks
+///
+/// After successful validation, activities are passed to respective [trait@ActivityHandler].
+pub async fn receive_activity<Activity, ActorT, Datatype>(
+    request: HttpRequest,
+    body: Bytes,
+    data: &Data<Datatype>,
+) -> Result<HttpResponse, <Activity as ActivityHandler>::Error>
+where
+    Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + 'static,
+    for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
+    <Activity as ActivityHandler>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
+    Datatype: Clone,
+{
+    verify_body_hash(request.headers().get("Digest"), &body)?;
+
+    let (activity, actor) = parse_received_activity::<Activity, ActorT, _>(&body, data).await?;
+
+    verify_signature(
+        request.headers(),
+        request.method(),
+        request.uri(),
+        actor.public_key_pem(),
+    )?;
+
+    debug!("Receiving activity {}", activity.id().to_string());
+    activity.verify(data).await?;
+    activity.receive(data).await?;
+    Ok(HttpResponse::Ok().finish())
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod test {
+    use super::*;
+    use crate::{
+        activity_sending::generate_request_headers,
+        config::FederationConfig,
+        fetch::object_id::ObjectId,
+        http_signatures::sign_request,
+        traits::tests::{DbConnection, DbUser, Follow, DB_USER_KEYPAIR},
+    };
+    use actix_web::test::TestRequest;
+    use reqwest::Client;
+    use reqwest_middleware::ClientWithMiddleware;
+    use serde_json::json;
+    use url::Url;
+
+    #[tokio::test]
+    async fn test_receive_activity() {
+        let (body, incoming_request, config) = setup_receive_test().await;
+        receive_activity::<Follow, DbUser, DbConnection>(
+            incoming_request.to_http_request(),
+            body,
+            &config.to_request_data(),
+        )
+        .await
+        .unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_receive_activity_invalid_body_signature() {
+        let (_, incoming_request, config) = setup_receive_test().await;
+        let err = receive_activity::<Follow, DbUser, DbConnection>(
+            incoming_request.to_http_request(),
+            "invalid".into(),
+            &config.to_request_data(),
+        )
+        .await
+        .err()
+        .unwrap();
+
+        assert_eq!(&err, &Error::ActivityBodyDigestInvalid)
+    }
+
+    #[tokio::test]
+    async fn test_receive_activity_invalid_path() {
+        let (body, incoming_request, config) = setup_receive_test().await;
+        let incoming_request = incoming_request.uri("/wrong");
+        let err = receive_activity::<Follow, DbUser, DbConnection>(
+            incoming_request.to_http_request(),
+            body,
+            &config.to_request_data(),
+        )
+        .await
+        .err()
+        .unwrap();
+
+        assert_eq!(&err, &Error::ActivitySignatureInvalid)
+    }
+
+    #[tokio::test]
+    async fn test_receive_unparseable_activity() {
+        let (_, _, config) = setup_receive_test().await;
+
+        let actor = Url::parse("http://ds9.lemmy.ml/u/lemmy_alpha").unwrap();
+        let id = "http://localhost:123/1";
+        let activity = json!({
+          "actor": actor.as_str(),
+          "to": ["https://www.w3.org/ns/activitystreams#Public"],
+          "object": "http://ds9.lemmy.ml/post/1",
+          "cc": ["http://enterprise.lemmy.ml/c/main"],
+          "type": "Delete",
+          "id": id
+        }
+        );
+        let body: Bytes = serde_json::to_vec(&activity).unwrap().into();
+        let incoming_request = construct_request(&body, &actor).await;
+
+        // intentionally cause a parse error by using wrong type for deser
+        let res = receive_activity::<Follow, DbUser, DbConnection>(
+            incoming_request.to_http_request(),
+            body,
+            &config.to_request_data(),
+        )
+        .await;
+
+        match res {
+            Err(Error::ParseReceivedActivity(_, url)) => {
+                assert_eq!(id, url.expect("has url").as_str());
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    async fn construct_request(body: &Bytes, actor: &Url) -> TestRequest {
+        let inbox = "https://example.com/inbox";
+        let headers = generate_request_headers(&Url::parse(inbox).unwrap());
+        let request_builder = ClientWithMiddleware::from(Client::default())
+            .post(inbox)
+            .headers(headers);
+        let outgoing_request = sign_request(
+            request_builder,
+            actor,
+            body.clone(),
+            DB_USER_KEYPAIR.private_key().unwrap(),
+            false,
+        )
+        .await
+        .unwrap();
+        let mut incoming_request = TestRequest::post().uri(outgoing_request.url().path());
+        for h in outgoing_request.headers() {
+            incoming_request = incoming_request.append_header(h);
+        }
+        incoming_request
+    }
+
+    async fn setup_receive_test() -> (Bytes, TestRequest, FederationConfig<DbConnection>) {
+        let activity = Follow {
+            actor: ObjectId::parse("http://localhost:123").unwrap(),
+            object: ObjectId::parse("http://localhost:124").unwrap(),
+            kind: Default::default(),
+            id: "http://localhost:123/1".try_into().unwrap(),
+        };
+        let body: Bytes = serde_json::to_vec(&activity).unwrap().into();
+        let incoming_request = construct_request(&body, activity.actor.inner()).await;
+
+        let config = FederationConfig::builder()
+            .domain("localhost:8002")
+            .app_data(DbConnection)
+            .debug(true)
+            .build()
+            .await
+            .unwrap();
+        (body, incoming_request, config)
+    }
+}

src/actix_web/middleware.rs

@@ -0,0 +1,76 @@
+use crate::config::{Data, FederationConfig, FederationMiddleware};
+use actix_web::{
+    dev::{forward_ready, Payload, Service, ServiceRequest, ServiceResponse, Transform},
+    Error,
+    FromRequest,
+    HttpMessage,
+    HttpRequest,
+};
+use std::future::{ready, Ready};
+
+impl<S, B, T> Transform<S, ServiceRequest> for FederationMiddleware<T>
+where
+    S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = Error>,
+    S::Future: 'static,
+    B: 'static,
+    T: Clone + Sync + 'static,
+{
+    type Response = ServiceResponse<B>;
+    type Error = Error;
+    type Transform = FederationService<S, T>;
+    type InitError = ();
+    type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+    fn new_transform(&self, service: S) -> Self::Future {
+        ready(Ok(FederationService {
+            service,
+            config: self.0.clone(),
+        }))
+    }
+}
+
+/// Passes [FederationConfig] to HTTP handlers, converting it to [Data] in the process
+#[doc(hidden)]
+pub struct FederationService<S, T: Clone>
+where
+    S: Service<ServiceRequest, Error = Error>,
+    S::Future: 'static,
+    T: Sync,
+{
+    service: S,
+    config: FederationConfig<T>,
+}
+
+impl<S, B, T> Service<ServiceRequest> for FederationService<S, T>
+where
+    S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = Error>,
+    S::Future: 'static,
+    B: 'static,
+    T: Clone + Sync + 'static,
+{
+    type Response = ServiceResponse<B>;
+    type Error = Error;
+    type Future = S::Future;
+
+    forward_ready!(service);
+
+    fn call(&self, req: ServiceRequest) -> Self::Future {
+        req.extensions_mut().insert(self.config.clone());
+
+        self.service.call(req)
+    }
+}
+
+impl<T: Clone + 'static> FromRequest for Data<T> {
+    type Error = Error;
+    type Future = Ready<Result<Self, Self::Error>>;
+
+    fn from_request(req: &HttpRequest, _payload: &mut Payload) -> Self::Future {
+        ready(match req.extensions().get::<FederationConfig<T>>() {
+            Some(c) => Ok(c.to_request_data()),
+            None => Err(actix_web::error::ErrorBadRequest(
+                "Missing extension, did you register FederationMiddleware?",
+            )),
+        })
+    }
+}

src/actix_web/mod.rs

@@ -0,0 +1,31 @@
+//! Utilities for using this library with actix-web framework
+
+pub mod inbox;
+#[doc(hidden)]
+pub mod middleware;
+
+use crate::{
+    config::Data,
+    error::Error,
+    http_signatures::{self, verify_body_hash},
+    traits::{Actor, Object},
+};
+use actix_web::{web::Bytes, HttpRequest};
+use serde::Deserialize;
+
+/// Checks whether the request is signed by an actor of type A, and returns
+/// the actor in question if a valid signature is found.
+pub async fn signing_actor<A>(
+    request: &HttpRequest,
+    body: Option<Bytes>,
+    data: &Data<<A as Object>::DataType>,
+) -> Result<A, <A as Object>::Error>
+where
+    A: Object + Actor,
+    <A as Object>::Error: From<Error>,
+    for<'de2> <A as Object>::Kind: Deserialize<'de2>,
+{
+    verify_body_hash(request.headers().get("Digest"), &body.unwrap_or_default())?;
+
+    http_signatures::signing_actor(request.headers(), request.method(), request.uri(), data).await
+}

src/axum/inbox.rs

@@ -0,0 +1,89 @@
+//! Handles incoming activities, verifying HTTP signatures and other checks
+//!
+#![doc = include_str!("../../docs/08_receiving_activities.md")]
+
+use crate::{
+    config::Data,
+    error::Error,
+    http_signatures::verify_signature,
+    parse_received_activity,
+    traits::{ActivityHandler, Actor, Object},
+};
+use axum::{
+    async_trait,
+    body::{Bytes, HttpBody},
+    extract::FromRequest,
+    http::{Request, StatusCode},
+    response::{IntoResponse, Response},
+};
+use http::{HeaderMap, Method, Uri};
+use serde::de::DeserializeOwned;
+use tracing::debug;
+
+/// Handles incoming activities, verifying HTTP signatures and other checks
+pub async fn receive_activity<Activity, ActorT, Datatype>(
+    activity_data: ActivityData,
+    data: &Data<Datatype>,
+) -> Result<(), <Activity as ActivityHandler>::Error>
+where
+    Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + 'static,
+    for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
+    <Activity as ActivityHandler>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
+    Datatype: Clone,
+{
+    let (activity, actor) =
+        parse_received_activity::<Activity, ActorT, _>(&activity_data.body, data).await?;
+
+    verify_signature(
+        &activity_data.headers,
+        &activity_data.method,
+        &activity_data.uri,
+        actor.public_key_pem(),
+    )?;
+
+    debug!("Receiving activity {}", activity.id().to_string());
+    activity.verify(data).await?;
+    activity.receive(data).await?;
+    Ok(())
+}
+
+/// Contains all data that is necessary to receive an activity from an HTTP request
+#[derive(Debug)]
+pub struct ActivityData {
+    headers: HeaderMap,
+    method: Method,
+    uri: Uri,
+    body: Vec<u8>,
+}
+
+#[async_trait]
+impl<S, B> FromRequest<S, B> for ActivityData
+where
+    Bytes: FromRequest<S, B>,
+    B: HttpBody + Send + 'static,
+    S: Send + Sync,
+    <B as HttpBody>::Error: std::fmt::Display,
+    <B as HttpBody>::Data: Send,
+{
+    type Rejection = Response;
+
+    async fn from_request(req: Request<B>, _state: &S) -> Result<Self, Self::Rejection> {
+        let (parts, body) = req.into_parts();
+
+        // this wont work if the body is an long running stream
+        let bytes = hyper::body::to_bytes(body)
+            .await
+            .map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response())?;
+
+        Ok(Self {
+            headers: parts.headers,
+            method: parts.method,
+            uri: parts.uri,
+            body: bytes.to_vec(),
+        })
+    }
+}
+
+// TODO: copy tests from actix-web inbox and implement for axum as well

src/axum/json.rs

@@ -0,0 +1,39 @@
+//! Wrapper struct to respond with `application/activity+json` in axum handlers
+//!
+//! ```
+//! # use anyhow::Error;
+//! # use axum::extract::Path;
+//! # use activitypub_federation::axum::json::FederationJson;
+//! # use activitypub_federation::protocol::context::WithContext;
+//! # use activitypub_federation::config::Data;
+//! # use activitypub_federation::traits::Object;
+//! # use activitypub_federation::traits::tests::{DbConnection, DbUser, Person};
+//! async fn http_get_user(Path(name): Path<String>, data: Data<DbConnection>) -> Result<FederationJson<WithContext<Person>>, Error> {
+//!     let user: DbUser = data.read_local_user(&name).await?;
+//!     let person = user.into_json(&data).await?;
+//!
+//!     Ok(FederationJson(WithContext::new_default(person)))
+//! }
+//! ```
+
+use crate::FEDERATION_CONTENT_TYPE;
+use axum::response::IntoResponse;
+use http::header;
+use serde::Serialize;
+
+/// Wrapper struct to respond with `application/activity+json` in axum handlers
+#[derive(Debug, Clone, Copy, Default)]
+pub struct FederationJson<Json: Serialize>(pub Json);
+
+impl<Json: Serialize> IntoResponse for FederationJson<Json> {
+    fn into_response(self) -> axum::response::Response {
+        let mut response = axum::response::Json(self.0).into_response();
+        response.headers_mut().insert(
+            header::CONTENT_TYPE,
+            FEDERATION_CONTENT_TYPE
+                .parse()
+                .expect("Parsing 'application/activity+json' should never fail"),
+        );
+        response
+    }
+}

src/axum/middleware.rs

@@ -0,0 +1,63 @@
+use crate::config::{Data, FederationConfig, FederationMiddleware};
+use axum::{async_trait, body::Body, extract::FromRequestParts, http::Request, response::Response};
+use http::{request::Parts, StatusCode};
+use std::task::{Context, Poll};
+use tower::{Layer, Service};
+
+impl<S, T: Clone> Layer<S> for FederationMiddleware<T> {
+    type Service = FederationService<S, T>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        FederationService {
+            inner,
+            config: self.0.clone(),
+        }
+    }
+}
+
+/// Passes [FederationConfig] to HTTP handlers, converting it to [Data] in the process
+#[doc(hidden)]
+#[derive(Clone)]
+pub struct FederationService<S, T: Clone> {
+    inner: S,
+    config: FederationConfig<T>,
+}
+
+impl<S, T> Service<Request<Body>> for FederationService<S, T>
+where
+    S: Service<Request<Body>, Response = Response> + Send + 'static,
+    S::Future: Send + 'static,
+    T: Clone + Send + Sync + 'static,
+{
+    type Response = S::Response;
+    type Error = S::Error;
+    type Future = S::Future;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut request: Request<Body>) -> Self::Future {
+        request.extensions_mut().insert(self.config.clone());
+        self.inner.call(request)
+    }
+}
+
+#[async_trait]
+impl<S, T: Clone + 'static> FromRequestParts<S> for Data<T>
+where
+    S: Send + Sync,
+    T: Send + Sync,
+{
+    type Rejection = (StatusCode, &'static str);
+
+    async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
+        match parts.extensions.get::<FederationConfig<T>>() {
+            Some(c) => Ok(c.to_request_data()),
+            None => Err((
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "Missing extension, did you register FederationMiddleware?",
+            )),
+        }
+    }
+}

src/axum/mod.rs

@@ -0,0 +1,8 @@
+//! Utilities for using this library with axum web framework
+//!
+#![doc = include_str!("../../docs/06_http_endpoints_axum.md")]
+
+pub mod inbox;
+pub mod json;
+#[doc(hidden)]
+pub mod middleware;

src/config.rs

@@ -0,0 +1,380 @@
+//! Configuration for this library, with various federation settings
+//!
+//! Use [FederationConfig::builder](crate::config::FederationConfig::builder) to initialize it.
+//!
+//! ```
+//! # use activitypub_federation::config::FederationConfig;
+//! # tokio::runtime::Runtime::new().unwrap().block_on(async {
+//! let settings = FederationConfig::builder()
+//!     .domain("example.com")
+//!     .app_data(())
+//!     .http_fetch_limit(50)
+//!     .build().await?;
+//! # Ok::<(), anyhow::Error>(())
+//! # }).unwrap()
+//! ```
+
+use crate::{
+    activity_queue::{create_activity_queue, ActivityQueue},
+    error::Error,
+    protocol::verification::verify_domains_match,
+    traits::{ActivityHandler, Actor},
+};
+use async_trait::async_trait;
+use derive_builder::Builder;
+use dyn_clone::{clone_trait_object, DynClone};
+use moka::future::Cache;
+use openssl::pkey::{PKey, Private};
+use reqwest_middleware::ClientWithMiddleware;
+use serde::de::DeserializeOwned;
+use std::{
+    ops::Deref,
+    sync::{
+        atomic::{AtomicU32, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+use url::Url;
+
+/// Configuration for this library, with various federation related settings
+#[derive(Builder, Clone)]
+#[builder(build_fn(private, name = "partial_build"))]
+pub struct FederationConfig<T: Clone> {
+    /// The domain where this federated instance is running
+    #[builder(setter(into))]
+    pub(crate) domain: String,
+    /// Data which the application requires in handlers, such as database connection
+    /// or configuration.
+    pub(crate) app_data: T,
+    /// Maximum number of outgoing HTTP requests per incoming HTTP request. See
+    /// [crate::fetch::object_id::ObjectId] for more details.
+    #[builder(default = "20")]
+    pub(crate) http_fetch_limit: u32,
+    #[builder(default = "reqwest::Client::default().into()")]
+    /// HTTP client used for all outgoing requests. Middleware can be used to add functionality
+    /// like log tracing or retry of failed requests.
+    pub(crate) client: ClientWithMiddleware,
+    /// Run library in debug mode. This allows usage of http and localhost urls. It also sends
+    /// outgoing activities synchronously, not in background thread. This helps to make tests
+    /// more consistent. Do not use for production.
+    #[builder(default = "false")]
+    pub(crate) debug: bool,
+    /// Allow HTTP urls even in production mode
+    #[builder(default = "self.debug.unwrap_or(false)")]
+    pub(crate) allow_http_urls: bool,
+    /// Timeout for all HTTP requests. HTTP signatures are valid for 10s, so it makes sense to
+    /// use the same as timeout when sending
+    #[builder(default = "Duration::from_secs(10)")]
+    pub(crate) request_timeout: Duration,
+    /// Function used to verify that urls are valid, See [UrlVerifier] for details.
+    #[builder(default = "Box::new(DefaultUrlVerifier())")]
+    pub(crate) url_verifier: Box<dyn UrlVerifier + Sync>,
+    /// Enable to sign HTTP signatures according to draft 10, which does not include (created) and
+    /// (expires) fields. This is required for compatibility with some software like Pleroma.
+    /// <https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-10>
+    /// <https://git.pleroma.social/pleroma/pleroma/-/issues/2939>
+    #[builder(default = "false")]
+    pub(crate) http_signature_compat: bool,
+    /// Actor Id and private key to use to sign all federated fetch requests.
+    /// This can be used to implement secure mode federation.
+    /// <https://docs.joinmastodon.org/spec/activitypub/#secure-mode>
+    #[builder(default = "None", setter(custom))]
+    pub(crate) signed_fetch_actor: Option<Arc<(Url, PKey<Private>)>>,
+    #[builder(
+        default = "Cache::builder().max_capacity(10000).build()",
+        setter(custom)
+    )]
+    pub(crate) actor_pkey_cache: Cache<Url, PKey<Private>>,
+    /// Queue for sending outgoing activities. Only optional to make builder work, its always
+    /// present once constructed.
+    #[builder(setter(skip))]
+    pub(crate) activity_queue: Option<Arc<ActivityQueue>>,
+    /// When sending with activity queue: Number of tasks that can be in-flight concurrently.
+    /// Tasks are retried once after a minute, then put into the retry queue.
+    /// Setting this count to `0` means that there is no limit to concurrency
+    #[builder(default = "0")]
+    pub(crate) queue_worker_count: usize,
+    /// When sending with activity queue: Number of concurrent tasks that are being retried
+    /// in-flight concurrently. Tasks are retried after an hour, then again in 60 hours.
+    /// Setting this count to `0` means that there is no limit to concurrency
+    #[builder(default = "0")]
+    pub(crate) queue_retry_count: usize,
+}
+
+impl<T: Clone> FederationConfig<T> {
+    /// Returns a new config builder with default values.
+    pub fn builder() -> FederationConfigBuilder<T> {
+        FederationConfigBuilder::default()
+    }
+
+    pub(crate) async fn verify_url_and_domain<Activity, Datatype>(
+        &self,
+        activity: &Activity,
+    ) -> Result<(), Error>
+    where
+        Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    {
+        verify_domains_match(activity.id(), activity.actor())?;
+        self.verify_url_valid(activity.id()).await?;
+        if self.is_local_url(activity.id()) {
+            return Err(Error::UrlVerificationError(
+                "Activity was sent from local instance",
+            ));
+        }
+
+        Ok(())
+    }
+
+    /// Create new [Data] from this. You should prefer to use a middleware if possible.
+    pub fn to_request_data(&self) -> Data<T> {
+        Data {
+            config: self.clone(),
+            request_counter: Default::default(),
+        }
+    }
+
+    /// Perform some security checks on URLs as mentioned in activitypub spec, and call user-supplied
+    /// [`InstanceSettings.verify_url_function`].
+    ///
+    /// https://www.w3.org/TR/activitypub/#security-considerations
+    pub(crate) async fn verify_url_valid(&self, url: &Url) -> Result<(), Error> {
+        match url.scheme() {
+            "https" => {}
+            "http" => {
+                if !self.allow_http_urls {
+                    return Err(Error::UrlVerificationError(
+                        "Http urls are only allowed in debug mode",
+                    ));
+                }
+            }
+            _ => return Err(Error::UrlVerificationError("Invalid url scheme")),
+        };
+
+        // Urls which use our local domain are not a security risk, no further verification needed
+        if self.is_local_url(url) {
+            return Ok(());
+        }
+
+        if url.domain().is_none() {
+            return Err(Error::UrlVerificationError("Url must have a domain"));
+        }
+
+        if url.domain() == Some("localhost") && !self.debug {
+            return Err(Error::UrlVerificationError(
+                "Localhost is only allowed in debug mode",
+            ));
+        }
+
+        self.url_verifier.verify(url).await?;
+
+        Ok(())
+    }
+
+    /// Returns true if the url refers to this instance. Handles hostnames like `localhost:8540` for
+    /// local debugging.
+    pub(crate) fn is_local_url(&self, url: &Url) -> bool {
+        match url.host_str() {
+            Some(domain) => {
+                let domain = if let Some(port) = url.port() {
+                    format!("{}:{}", domain, port)
+                } else {
+                    domain.to_string()
+                };
+                domain == self.domain
+            }
+            None => false,
+        }
+    }
+
+    /// Returns the local domain
+    pub fn domain(&self) -> &str {
+        &self.domain
+    }
+}
+
+impl<T: Clone> FederationConfigBuilder<T> {
+    /// Sets an actor to use to sign all federated fetch requests
+    pub fn signed_fetch_actor<A: Actor>(&mut self, actor: &A) -> &mut Self {
+        let private_key_pem = actor
+            .private_key_pem()
+            .expect("actor does not have a private key to sign with");
+
+        let private_key = PKey::private_key_from_pem(private_key_pem.as_bytes())
+            .expect("Could not decode PEM data");
+        self.signed_fetch_actor = Some(Some(Arc::new((actor.id(), private_key))));
+        self
+    }
+
+    /// sets the number of parsed actor private keys to keep in memory
+    pub fn actor_pkey_cache(&mut self, cache_size: u64) -> &mut Self {
+        self.actor_pkey_cache = Some(Cache::builder().max_capacity(cache_size).build());
+        self
+    }
+
+    /// Constructs a new config instance with the values supplied to builder.
+    ///
+    /// Values which are not explicitly specified use the defaults. Also initializes the
+    /// queue for outgoing activities, which is stored internally in the config struct.
+    /// Requires a tokio runtime for the background queue.
+    pub async fn build(&mut self) -> Result<FederationConfig<T>, FederationConfigBuilderError> {
+        let mut config = self.partial_build()?;
+        let queue = create_activity_queue(
+            config.client.clone(),
+            config.queue_worker_count,
+            config.queue_retry_count,
+            config.request_timeout,
+        );
+        config.activity_queue = Some(Arc::new(queue));
+        Ok(config)
+    }
+}
+
+impl<T: Clone> Deref for FederationConfig<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.app_data
+    }
+}
+
+/// Handler for validating URLs.
+///
+/// This is used for implementing domain blocklists and similar functionality. It is called
+/// with the ID of newly received activities, when fetching remote data from a given URL
+/// and before sending an activity to a given inbox URL. If processing for this domain/URL should
+/// be aborted, return an error. In case of `Ok(())`, processing continues.
+///
+/// ```
+/// # use async_trait::async_trait;
+/// # use url::Url;
+/// # use activitypub_federation::config::UrlVerifier;
+/// # use activitypub_federation::error::Error;
+/// # #[derive(Clone)]
+/// # struct DatabaseConnection();
+/// # async fn get_blocklist(_: &DatabaseConnection) -> Vec<String> {
+/// #     vec![]
+/// # }
+/// #[derive(Clone)]
+/// struct Verifier {
+///     db_connection: DatabaseConnection,
+/// }
+///
+/// #[async_trait]
+/// impl UrlVerifier for Verifier {
+///     async fn verify(&self, url: &Url) -> Result<(), Error> {
+///         let blocklist = get_blocklist(&self.db_connection).await;
+///         let domain = url.domain().unwrap().to_string();
+///         if blocklist.contains(&domain) {
+///             Err(Error::Other("Domain is blocked".into()))
+///         } else {
+///             Ok(())
+///         }
+///     }
+/// }
+/// ```
+#[async_trait]
+pub trait UrlVerifier: DynClone + Send {
+    /// Should return Ok iff the given url is valid for processing.
+    async fn verify(&self, url: &Url) -> Result<(), Error>;
+}
+
+/// Default URL verifier which does nothing.
+#[derive(Clone)]
+struct DefaultUrlVerifier();
+
+#[async_trait]
+impl UrlVerifier for DefaultUrlVerifier {
+    async fn verify(&self, _url: &Url) -> Result<(), Error> {
+        Ok(())
+    }
+}
+
+clone_trait_object!(UrlVerifier);
+
+/// Stores data for handling one specific HTTP request.
+///
+/// It gives acess to the `app_data` which was passed to [FederationConfig::builder].
+///
+/// Additionally it contains a counter for outgoing HTTP requests. This is necessary to
+/// prevent denial of service attacks, where an attacker triggers fetching of recursive objects.
+///
+/// <https://www.w3.org/TR/activitypub/#security-recursive-objects>
+pub struct Data<T: Clone> {
+    pub(crate) config: FederationConfig<T>,
+    pub(crate) request_counter: AtomicU32,
+}
+
+impl<T: Clone> Data<T> {
+    /// Returns the data which was stored in [FederationConfigBuilder::app_data]
+    pub fn app_data(&self) -> &T {
+        &self.config.app_data
+    }
+
+    /// The domain that was configured in [FederationConfig].
+    pub fn domain(&self) -> &str {
+        &self.config.domain
+    }
+
+    /// Returns a new instance of `Data` with request counter set to 0.
+    pub fn reset_request_count(&self) -> Self {
+        Data {
+            config: self.config.clone(),
+            request_counter: Default::default(),
+        }
+    }
+    /// Total number of outgoing HTTP requests made with this data.
+    pub fn request_count(&self) -> u32 {
+        self.request_counter.load(Ordering::Relaxed)
+    }
+}
+
+impl<T: Clone> Deref for Data<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.config.app_data
+    }
+}
+
+/// Middleware for HTTP handlers which provides access to [Data]
+#[derive(Clone)]
+pub struct FederationMiddleware<T: Clone>(pub(crate) FederationConfig<T>);
+
+impl<T: Clone> FederationMiddleware<T> {
+    /// Construct a new middleware instance
+    pub fn new(config: FederationConfig<T>) -> Self {
+        FederationMiddleware(config)
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod test {
+    use super::*;
+
+    async fn config() -> FederationConfig<i32> {
+        FederationConfig::builder()
+            .domain("example.com")
+            .app_data(1)
+            .build()
+            .await
+            .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_url_is_local() -> Result<(), Error> {
+        let config = config().await;
+        assert!(config.is_local_url(&Url::parse("http://example.com")?));
+        assert!(!config.is_local_url(&Url::parse("http://other.com")?));
+        // ensure that missing domain doesnt cause crash
+        assert!(!config.is_local_url(&Url::parse("http://127.0.0.1")?));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_get_domain() {
+        let config = config().await;
+        assert_eq!("example.com", config.domain());
+    }
+}

src/core/activity_queue.rs

@@ -1,216 +0,0 @@
-use crate::{
-    core::signatures::{sign_request, PublicKey},
-    traits::ActivityHandler,
-    utils::verify_url_valid,
-    Error,
-    InstanceSettings,
-    LocalInstance,
-    APUB_JSON_CONTENT_TYPE,
-};
-use anyhow::anyhow;
-use background_jobs::{
-    memory_storage::Storage,
-    ActixJob,
-    Backoff,
-    Manager,
-    MaxRetries,
-    WorkerConfig,
-};
-use http::{header::HeaderName, HeaderMap, HeaderValue};
-use itertools::Itertools;
-use reqwest_middleware::ClientWithMiddleware;
-use serde::{Deserialize, Serialize};
-use std::{fmt::Debug, future::Future, pin::Pin, time::Duration};
-use tracing::{info, warn};
-use url::Url;
-
-/// Send out the given activity to all inboxes, automatically generating the HTTP signatures. By
-/// default, sending is done on a background thread, and automatically retried on failure with
-/// exponential backoff.
-///
-/// - `activity`: The activity to be sent, gets converted to json
-/// - `public_key`: The sending actor's public key. In fact, we only need the key id for signing
-/// - `private_key`: The sending actor's private key for signing HTTP signature
-/// - `recipients`: List of actors who should receive the activity. This gets deduplicated, and
-///                 local/invalid inbox urls removed
-pub async fn send_activity<Activity>(
-    activity: Activity,
-    public_key: PublicKey,
-    private_key: String,
-    recipients: Vec<Url>,
-    instance: &LocalInstance,
-) -> Result<(), <Activity as ActivityHandler>::Error>
-where
-    Activity: ActivityHandler + Serialize,
-    <Activity as ActivityHandler>::Error: From<anyhow::Error> + From<serde_json::Error>,
-{
-    let activity_id = activity.id();
-    let activity_serialized = serde_json::to_string_pretty(&activity)?;
-    let inboxes: Vec<Url> = recipients
-        .into_iter()
-        .unique()
-        .filter(|i| !instance.is_local_url(i))
-        .filter(|i| verify_url_valid(i, &instance.settings).is_ok())
-        .collect();
-
-    let activity_queue = &instance.activity_queue;
-    for inbox in inboxes {
-        if verify_url_valid(&inbox, &instance.settings).is_err() {
-            continue;
-        }
-        let message = SendActivityTask {
-            activity_id: activity_id.clone(),
-            inbox,
-            activity: activity_serialized.clone(),
-            public_key: public_key.clone(),
-            private_key: private_key.clone(),
-        };
-        if instance.settings.debug {
-            let res = do_send(message, &instance.client, instance.settings.request_timeout).await;
-            // Don't fail on error, as we intentionally do some invalid actions in tests, to verify that
-            // they are rejected on the receiving side. These errors shouldn't bubble up to make the API
-            // call fail. This matches the behaviour in production.
-            if let Err(e) = res {
-                warn!("{}", e);
-            }
-        } else {
-            activity_queue.queue::<SendActivityTask>(message).await?;
-            let stats = activity_queue.get_stats().await?;
-            info!(
-        "Activity queue stats: pending: {}, running: {}, dead (this hour): {}, complete (this hour): {}",
-        stats.pending,
-        stats.running,
-        stats.dead.this_hour(),
-        stats.complete.this_hour()
-      );
-        }
-    }
-
-    Ok(())
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-struct SendActivityTask {
-    activity_id: Url,
-    inbox: Url,
-    activity: String,
-    public_key: PublicKey,
-    private_key: String,
-}
-
-/// Signs the activity with the sending actor's key, and delivers to the given inbox. Also retries
-/// if the delivery failed.
-impl ActixJob for SendActivityTask {
-    type State = MyState;
-    type Future = Pin<Box<dyn Future<Output = Result<(), anyhow::Error>>>>;
-    const NAME: &'static str = "SendActivityTask";
-
-    /// With these params, retries are made at the following intervals:
-    ///          3s
-    ///          9s
-    ///         27s
-    ///      1m 21s
-    ///      4m  3s
-    ///     12m  9s
-    ///     36m 27s
-    ///  1h 49m 21s
-    ///  5h 28m  3s
-    /// 16h 24m  9s
-    const MAX_RETRIES: MaxRetries = MaxRetries::Count(10);
-    const BACKOFF: Backoff = Backoff::Exponential(3);
-
-    fn run(self, state: Self::State) -> Self::Future {
-        Box::pin(async move { do_send(self, &state.client, state.timeout).await })
-    }
-}
-
-async fn do_send(
-    task: SendActivityTask,
-    client: &ClientWithMiddleware,
-    timeout: Duration,
-) -> Result<(), anyhow::Error> {
-    info!("Sending {} to {}", task.activity_id, task.inbox);
-    let request_builder = client
-        .post(&task.inbox.to_string())
-        .timeout(timeout)
-        .headers(generate_request_headers(&task.inbox));
-    let request = sign_request(
-        request_builder,
-        task.activity.clone(),
-        task.public_key.clone(),
-        task.private_key.to_owned(),
-    )
-    .await?;
-    let response = client.execute(request).await;
-
-    match response {
-        Ok(o) => {
-            if o.status().is_success() {
-                Ok(())
-            } else {
-                let status = o.status();
-                let text = o.text().await.map_err(Error::conv)?;
-
-                Err(anyhow!(
-                    "Send {} to {} failed with status {}: {}",
-                    task.activity_id,
-                    task.inbox,
-                    status,
-                    text,
-                ))
-            }
-        }
-        Err(e) => Err(anyhow!(
-            "Failed to send activity {} to {}: {}",
-            &task.activity_id,
-            task.inbox,
-            e
-        )),
-    }
-}
-
-fn generate_request_headers(inbox_url: &Url) -> HeaderMap {
-    let mut host = inbox_url.domain().expect("read inbox domain").to_string();
-    if let Some(port) = inbox_url.port() {
-        host = format!("{}:{}", host, port);
-    }
-
-    let mut headers = HeaderMap::new();
-    headers.insert(
-        HeaderName::from_static("content-type"),
-        HeaderValue::from_static(APUB_JSON_CONTENT_TYPE),
-    );
-    headers.insert(
-        HeaderName::from_static("host"),
-        HeaderValue::from_str(&host).expect("Hostname is valid"),
-    );
-    headers
-}
-
-pub(crate) fn create_activity_queue(
-    client: ClientWithMiddleware,
-    settings: &InstanceSettings,
-) -> Manager {
-    // queue is not used in debug mod, so dont create any workers to avoid log spam
-    let worker_count = if settings.debug {
-        0
-    } else {
-        settings.worker_count
-    };
-    let timeout = settings.request_timeout;
-
-    // Configure and start our workers
-    WorkerConfig::new_managed(Storage::new(), move |_| MyState {
-        client: client.clone(),
-        timeout,
-    })
-    .register::<SendActivityTask>()
-    .set_worker_count("default", worker_count)
-    .start()
-}
-
-#[derive(Clone)]
-struct MyState {
-    client: ClientWithMiddleware,
-    timeout: Duration,
-}

src/core/inbox.rs

@@ -1,46 +0,0 @@
-use crate::{
-    core::{object_id::ObjectId, signatures::verify_signature},
-    data::Data,
-    traits::{ActivityHandler, Actor, ApubObject},
-    utils::{verify_domains_match, verify_url_valid},
-    Error,
-    LocalInstance,
-};
-use actix_web::{HttpRequest, HttpResponse};
-use serde::de::DeserializeOwned;
-use tracing::log::debug;
-
-/// Receive an activity and perform some basic checks, including HTTP signature verification.
-pub async fn receive_activity<Activity, ActorT, Datatype>(
-    request: HttpRequest,
-    activity: Activity,
-    local_instance: &LocalInstance,
-    data: &Data<Datatype>,
-) -> Result<HttpResponse, <Activity as ActivityHandler>::Error>
-where
-    Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
-    ActorT: ApubObject<DataType = Datatype> + Actor + Send + 'static,
-    for<'de2> <ActorT as ApubObject>::ApubType: serde::Deserialize<'de2>,
-    <Activity as ActivityHandler>::Error:
-        From<anyhow::Error> + From<Error> + From<<ActorT as ApubObject>::Error>,
-    <ActorT as ApubObject>::Error: From<Error> + From<anyhow::Error>,
-{
-    verify_domains_match(activity.id(), activity.actor())?;
-    verify_url_valid(activity.id(), &local_instance.settings)?;
-    if local_instance.is_local_url(activity.id()) {
-        return Err(Error::UrlVerificationError("Activity was sent from local instance").into());
-    }
-
-    let request_counter = &mut 0;
-    let actor = ObjectId::<ActorT>::new(activity.actor().clone())
-        .dereference(data, local_instance, request_counter)
-        .await?;
-    verify_signature(&request, actor.public_key())?;
-
-    debug!("Verifying activity {}", activity.id().to_string());
-    activity.verify(data, request_counter).await?;
-
-    debug!("Receiving activity {}", activity.id().to_string());
-    activity.receive(data, request_counter).await?;
-    Ok(HttpResponse::Ok().finish())
-}

src/core/mod.rs

@@ -1,4 +0,0 @@
-pub mod activity_queue;
-pub mod inbox;
-pub mod object_id;
-pub mod signatures;

src/core/object_id.rs

@@ -1,260 +0,0 @@
-use crate::{traits::ApubObject, utils::fetch_object_http, Error, LocalInstance};
-use anyhow::anyhow;
-use chrono::{Duration as ChronoDuration, NaiveDateTime, Utc};
-use serde::{Deserialize, Serialize};
-use std::{
-    fmt::{Debug, Display, Formatter},
-    marker::PhantomData,
-};
-use url::Url;
-
-/// We store Url on the heap because it is quite large (88 bytes).
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(transparent)]
-pub struct ObjectId<Kind>(Box<Url>, PhantomData<Kind>)
-where
-    Kind: ApubObject + Send + 'static,
-    for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>;
-
-impl<Kind> ObjectId<Kind>
-where
-    Kind: ApubObject + Send + 'static,
-    for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
-{
-    pub fn new<T>(url: T) -> Self
-    where
-        T: Into<Url>,
-    {
-        ObjectId(Box::new(url.into()), PhantomData::<Kind>)
-    }
-
-    pub fn inner(&self) -> &Url {
-        &self.0
-    }
-
-    pub fn into_inner(self) -> Url {
-        *self.0
-    }
-
-    /// Fetches an activitypub object, either from local database (if possible), or over http.
-    pub async fn dereference(
-        &self,
-        data: &<Kind as ApubObject>::DataType,
-        instance: &LocalInstance,
-        request_counter: &mut i32,
-    ) -> Result<Kind, <Kind as ApubObject>::Error>
-    where
-        <Kind as ApubObject>::Error: From<Error> + From<anyhow::Error>,
-    {
-        let db_object = self.dereference_from_db(data).await?;
-
-        // if its a local object, only fetch it from the database and not over http
-        if instance.is_local_url(&self.0) {
-            return match db_object {
-                None => Err(Error::NotFound.into()),
-                Some(o) => Ok(o),
-            };
-        }
-
-        // object found in database
-        if let Some(object) = db_object {
-            // object is old and should be refetched
-            if let Some(last_refreshed_at) = object.last_refreshed_at() {
-                if should_refetch_object(last_refreshed_at) {
-                    return self
-                        .dereference_from_http(data, instance, request_counter, Some(object))
-                        .await;
-                }
-            }
-            Ok(object)
-        }
-        // object not found, need to fetch over http
-        else {
-            self.dereference_from_http(data, instance, request_counter, None)
-                .await
-        }
-    }
-
-    /// Fetch an object from the local db. Instead of falling back to http, this throws an error if
-    /// the object is not found in the database.
-    pub async fn dereference_local(
-        &self,
-        data: &<Kind as ApubObject>::DataType,
-    ) -> Result<Kind, <Kind as ApubObject>::Error>
-    where
-        <Kind as ApubObject>::Error: From<Error>,
-    {
-        let object = self.dereference_from_db(data).await?;
-        object.ok_or_else(|| Error::NotFound.into())
-    }
-
-    /// returning none means the object was not found in local db
-    async fn dereference_from_db(
-        &self,
-        data: &<Kind as ApubObject>::DataType,
-    ) -> Result<Option<Kind>, <Kind as ApubObject>::Error> {
-        let id = self.0.clone();
-        ApubObject::read_from_apub_id(*id, data).await
-    }
-
-    async fn dereference_from_http(
-        &self,
-        data: &<Kind as ApubObject>::DataType,
-        instance: &LocalInstance,
-        request_counter: &mut i32,
-        db_object: Option<Kind>,
-    ) -> Result<Kind, <Kind as ApubObject>::Error>
-    where
-        <Kind as ApubObject>::Error: From<Error> + From<anyhow::Error>,
-    {
-        let res = fetch_object_http(&self.0, instance, request_counter).await;
-
-        if let Err(Error::ObjectDeleted) = &res {
-            if let Some(db_object) = db_object {
-                db_object.delete(data).await?;
-            }
-            return Err(anyhow!("Fetched remote object {} which was deleted", self).into());
-        }
-
-        let res2 = res?;
-
-        Kind::verify(&res2, self.inner(), data, request_counter).await?;
-        Kind::from_apub(res2, data, request_counter).await
-    }
-}
-
-/// Need to implement clone manually, to avoid requiring Kind to be Clone
-impl<Kind> Clone for ObjectId<Kind>
-where
-    Kind: ApubObject + Send + 'static,
-    for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
-{
-    fn clone(&self) -> Self {
-        ObjectId(self.0.clone(), self.1)
-    }
-}
-
-static ACTOR_REFETCH_INTERVAL_SECONDS: i64 = 24 * 60 * 60;
-static ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG: i64 = 20;
-
-/// Determines when a remote actor should be refetched from its instance. In release builds, this is
-/// `ACTOR_REFETCH_INTERVAL_SECONDS` after the last refetch, in debug builds
-/// `ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG`.
-///
-/// TODO it won't pick up new avatars, summaries etc until a day after.
-/// Actors need an "update" activity pushed to other servers to fix this.
-fn should_refetch_object(last_refreshed: NaiveDateTime) -> bool {
-    let update_interval = if cfg!(debug_assertions) {
-        // avoid infinite loop when fetching community outbox
-        ChronoDuration::seconds(ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG)
-    } else {
-        ChronoDuration::seconds(ACTOR_REFETCH_INTERVAL_SECONDS)
-    };
-    let refresh_limit = Utc::now().naive_utc() - update_interval;
-    last_refreshed.lt(&refresh_limit)
-}
-
-impl<Kind> Display for ObjectId<Kind>
-where
-    Kind: ApubObject + Send + 'static,
-    for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
-{
-    #[allow(clippy::recursive_format_impl)]
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        // Use to_string here because Url.display is not useful for us
-        write!(f, "{}", self.0)
-    }
-}
-
-impl<Kind> From<ObjectId<Kind>> for Url
-where
-    Kind: ApubObject + Send + 'static,
-    for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
-{
-    fn from(id: ObjectId<Kind>) -> Self {
-        *id.0
-    }
-}
-
-impl<Kind> PartialEq for ObjectId<Kind>
-where
-    Kind: ApubObject + Send + 'static,
-    for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
-{
-    fn eq(&self, other: &Self) -> bool {
-        self.0.eq(&other.0) && self.1 == other.1
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::core::object_id::should_refetch_object;
-    use anyhow::Error;
-
-    #[derive(Debug)]
-    struct TestObject {}
-
-    #[async_trait::async_trait(?Send)]
-    impl ApubObject for TestObject {
-        type DataType = TestObject;
-        type ApubType = ();
-        type DbType = ();
-        type Error = Error;
-
-        async fn read_from_apub_id(
-            _object_id: Url,
-            _data: &Self::DataType,
-        ) -> Result<Option<Self>, Self::Error>
-        where
-            Self: Sized,
-        {
-            todo!()
-        }
-
-        async fn into_apub(self, _data: &Self::DataType) -> Result<Self::ApubType, Self::Error> {
-            todo!()
-        }
-
-        async fn verify(
-            _apub: &Self::ApubType,
-            _expected_domain: &Url,
-            _data: &Self::DataType,
-            _request_counter: &mut i32,
-        ) -> Result<(), Self::Error> {
-            todo!()
-        }
-
-        async fn from_apub(
-            _apub: Self::ApubType,
-            _data: &Self::DataType,
-            _request_counter: &mut i32,
-        ) -> Result<Self, Self::Error>
-        where
-            Self: Sized,
-        {
-            todo!()
-        }
-    }
-
-    #[test]
-    fn test_deserialize() {
-        let url = Url::parse("http://test.com/").unwrap();
-        let id = ObjectId::<TestObject>::new(url);
-
-        let string = serde_json::to_string(&id).unwrap();
-        assert_eq!("\"http://test.com/\"", string);
-
-        let parsed: ObjectId<TestObject> = serde_json::from_str(&string).unwrap();
-        assert_eq!(parsed, id);
-    }
-
-    #[test]
-    fn test_should_refetch_object() {
-        let one_second_ago = Utc::now().naive_utc() - ChronoDuration::seconds(1);
-        assert!(!should_refetch_object(one_second_ago));
-
-        let two_days_ago = Utc::now().naive_utc() - ChronoDuration::days(2);
-        assert!(should_refetch_object(two_days_ago));
-    }
-}

src/core/signatures.rs

@@ -1,124 +0,0 @@
-use actix_web::HttpRequest;
-use anyhow::anyhow;
-use http_signature_normalization_actix::Config as ConfigActix;
-use http_signature_normalization_reqwest::prelude::{Config, SignExt};
-use once_cell::sync::Lazy;
-use openssl::{
-    hash::MessageDigest,
-    pkey::PKey,
-    rsa::Rsa,
-    sign::{Signer, Verifier},
-};
-use reqwest::Request;
-use reqwest_middleware::RequestBuilder;
-use serde::{Deserialize, Serialize};
-use sha2::{Digest, Sha256};
-use std::io::{Error, ErrorKind};
-use tracing::debug;
-use url::Url;
-
-static CONFIG2: Lazy<ConfigActix> = Lazy::new(ConfigActix::new);
-static HTTP_SIG_CONFIG: Lazy<Config> = Lazy::new(Config::new);
-
-/// A private/public key pair used for HTTP signatures
-#[derive(Debug, Clone)]
-pub struct Keypair {
-    pub private_key: String,
-    pub public_key: String,
-}
-
-/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
-pub fn generate_actor_keypair() -> Result<Keypair, Error> {
-    let rsa = Rsa::generate(2048)?;
-    let pkey = PKey::from_rsa(rsa)?;
-    let public_key = pkey.public_key_to_pem()?;
-    let private_key = pkey.private_key_to_pem_pkcs8()?;
-    let key_to_string = |key| match String::from_utf8(key) {
-        Ok(s) => Ok(s),
-        Err(e) => Err(Error::new(
-            ErrorKind::Other,
-            format!("Failed converting key to string: {}", e),
-        )),
-    };
-    Ok(Keypair {
-        private_key: key_to_string(private_key)?,
-        public_key: key_to_string(public_key)?,
-    })
-}
-
-/// Creates an HTTP post request to `inbox_url`, with the given `client` and `headers`, and
-/// `activity` as request body. The request is signed with `private_key` and then sent.
-pub(crate) async fn sign_request(
-    request_builder: RequestBuilder,
-    activity: String,
-    public_key: PublicKey,
-    private_key: String,
-) -> Result<Request, anyhow::Error> {
-    request_builder
-        .signature_with_digest(
-            HTTP_SIG_CONFIG.clone(),
-            public_key.id,
-            Sha256::new(),
-            activity,
-            move |signing_string| {
-                let private_key = PKey::private_key_from_pem(private_key.as_bytes())?;
-                let mut signer = Signer::new(MessageDigest::sha256(), &private_key)?;
-                signer.update(signing_string.as_bytes())?;
-
-                Ok(base64::encode(signer.sign_to_vec()?)) as Result<_, anyhow::Error>
-            },
-        )
-        .await
-}
-
-/// Verifies the HTTP signature on an incoming inbox request.
-pub fn verify_signature(request: &HttpRequest, public_key: &str) -> Result<(), anyhow::Error> {
-    let verified = CONFIG2
-        .begin_verify(
-            request.method(),
-            request.uri().path_and_query(),
-            request.headers().clone(),
-        )?
-        .verify(|signature, signing_string| -> Result<bool, anyhow::Error> {
-            debug!(
-                "Verifying with key {}, message {}",
-                &public_key, &signing_string
-            );
-            let public_key = PKey::public_key_from_pem(public_key.as_bytes())?;
-            let mut verifier = Verifier::new(MessageDigest::sha256(), &public_key)?;
-            verifier.update(signing_string.as_bytes())?;
-            Ok(verifier.verify(&base64::decode(signature)?)?)
-        })?;
-
-    if verified {
-        debug!("verified signature for {}", &request.uri());
-        Ok(())
-    } else {
-        Err(anyhow!("Invalid signature on request: {}", &request.uri()))
-    }
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct PublicKey {
-    pub(crate) id: String,
-    pub(crate) owner: Url,
-    pub public_key_pem: String,
-}
-
-impl PublicKey {
-    /// Create public key with default id, for actors that only have a single keypair
-    pub fn new_main_key(owner: Url, public_key_pem: String) -> Self {
-        let key_id = format!("{}#main-key", &owner);
-        PublicKey::new(key_id, owner, public_key_pem)
-    }
-
-    /// Create public key with custom key id. Use this method if there are multiple keypairs per actor
-    pub fn new(id: String, owner: Url, public_key_pem: String) -> Self {
-        PublicKey {
-            id,
-            owner,
-            public_key_pem,
-        }
-    }
-}

src/data.rs

@@ -1,37 +0,0 @@
-use std::{ops::Deref, sync::Arc};
-
-/// This type can be used to pass your own data into library functions and traits. It can be useful
-/// to pass around database connections or other context.
-#[derive(Debug)]
-pub struct Data<T: ?Sized>(Arc<T>);
-
-impl<T> Data<T> {
-    /// Create new `Data` instance.
-    pub fn new(state: T) -> Data<T> {
-        Data(Arc::new(state))
-    }
-
-    /// Get reference to inner app data.
-    pub fn get_ref(&self) -> &T {
-        self.0.as_ref()
-    }
-
-    /// Convert to the internal Arc<T>
-    pub fn into_inner(self) -> Arc<T> {
-        self.0
-    }
-}
-
-impl<T: ?Sized> Deref for Data<T> {
-    type Target = Arc<T>;
-
-    fn deref(&self) -> &Arc<T> {
-        &self.0
-    }
-}
-
-impl<T: ?Sized> Clone for Data<T> {
-    fn clone(&self) -> Data<T> {
-        Data(self.0.clone())
-    }
-}

src/deser/context.rs

@@ -1,62 +0,0 @@
-use crate::{data::Data, deser::helpers::deserialize_one_or_many, traits::ActivityHandler};
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use std::str::FromStr;
-use url::Url;
-
-const DEFAULT_CONTEXT: &str = "[\"https://www.w3.org/ns/activitystreams\"]";
-
-/// Simple wrapper which adds json-ld context to an object or activity. Doing it this way ensures
-/// that nested objects dont have any context, but only the outermost one.
-#[derive(Serialize, Deserialize, Debug)]
-pub struct WithContext<T> {
-    #[serde(rename = "@context")]
-    #[serde(deserialize_with = "deserialize_one_or_many")]
-    context: Vec<Value>,
-    #[serde(flatten)]
-    inner: T,
-}
-
-impl<T> WithContext<T> {
-    pub fn new_default(inner: T) -> WithContext<T> {
-        let context = vec![Value::from_str(DEFAULT_CONTEXT).expect("valid context")];
-        WithContext::new(inner, context)
-    }
-
-    pub fn new(inner: T, context: Vec<Value>) -> WithContext<T> {
-        WithContext { context, inner }
-    }
-}
-
-#[async_trait::async_trait(?Send)]
-impl<T> ActivityHandler for WithContext<T>
-where
-    T: ActivityHandler,
-{
-    type DataType = <T as ActivityHandler>::DataType;
-    type Error = <T as ActivityHandler>::Error;
-
-    fn id(&self) -> &Url {
-        self.inner.id()
-    }
-
-    fn actor(&self) -> &Url {
-        self.inner.actor()
-    }
-
-    async fn verify(
-        &self,
-        data: &Data<Self::DataType>,
-        request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        self.inner.verify(data, request_counter).await
-    }
-
-    async fn receive(
-        self,
-        data: &Data<Self::DataType>,
-        request_counter: &mut i32,
-    ) -> Result<(), Self::Error> {
-        self.inner.receive(data, request_counter).await
-    }
-}

src/deser/helpers.rs

@@ -1,66 +0,0 @@
-use serde::{Deserialize, Deserializer};
-
-/// Deserialize either a single json value, or a json array. In either case, the items are returned
-/// as an array.
-///
-/// Usage:
-/// `#[serde(deserialize_with = "deserialize_one_or_many")]`
-pub fn deserialize_one_or_many<'de, T, D>(deserializer: D) -> Result<Vec<T>, D::Error>
-where
-    T: Deserialize<'de>,
-    D: Deserializer<'de>,
-{
-    #[derive(Deserialize)]
-    #[serde(untagged)]
-    enum OneOrMany<T> {
-        One(T),
-        Many(Vec<T>),
-    }
-
-    let result: OneOrMany<T> = Deserialize::deserialize(deserializer)?;
-    Ok(match result {
-        OneOrMany::Many(list) => list,
-        OneOrMany::One(value) => vec![value],
-    })
-}
-
-/// Deserialize either a single json value, or a json array with one element. In both cases it
-/// returns an array with a single element.
-///
-/// Usage:
-/// `#[serde(deserialize_with = "deserialize_one")]`
-pub fn deserialize_one<'de, T, D>(deserializer: D) -> Result<[T; 1], D::Error>
-where
-    T: Deserialize<'de>,
-    D: Deserializer<'de>,
-{
-    #[derive(Deserialize)]
-    #[serde(untagged)]
-    enum MaybeArray<T> {
-        Simple(T),
-        Array([T; 1]),
-    }
-
-    let result: MaybeArray<T> = Deserialize::deserialize(deserializer)?;
-    Ok(match result {
-        MaybeArray::Simple(value) => [value],
-        MaybeArray::Array(value) => value,
-    })
-}
-
-/// Attempts to deserialize the item. If any error happens, its ignored and the type's default
-/// value is returned.
-///
-/// Usage:
-/// `#[serde(deserialize_with = "deserialize_skip_error")]`
-pub fn deserialize_skip_error<'de, T, D>(deserializer: D) -> Result<T, D::Error>
-where
-    T: Deserialize<'de> + Default,
-    D: Deserializer<'de>,
-{
-    let result = Deserialize::deserialize(deserializer);
-    Ok(match result {
-        Ok(o) => o,
-        Err(_) => Default::default(),
-    })
-}

src/deser/mod.rs

@@ -1,3 +0,0 @@
-pub mod context;
-pub mod helpers;
-pub mod values;

src/error.rs

@@ -0,0 +1,93 @@
+//! Error messages returned by this library
+
+use crate::fetch::webfinger::WebFingerError;
+use http_signature_normalization_reqwest::SignError;
+use openssl::error::ErrorStack;
+use std::string::FromUtf8Error;
+use tokio::task::JoinError;
+use url::Url;
+
+/// Error messages returned by this library
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    /// Object was not found in local database
+    #[error("Object was not found in local database")]
+    NotFound,
+    /// Request limit was reached during fetch
+    #[error("Request limit was reached during fetch")]
+    RequestLimit,
+    /// Response body limit was reached during fetch
+    #[error("Response body limit was reached during fetch")]
+    ResponseBodyLimit,
+    /// Object to be fetched was deleted
+    #[error("Fetched remote object {0} which was deleted")]
+    ObjectDeleted(Url),
+    /// url verification error
+    #[error("URL failed verification: {0}")]
+    UrlVerificationError(&'static str),
+    /// Incoming activity has invalid digest for body
+    #[error("Incoming activity has invalid digest for body")]
+    ActivityBodyDigestInvalid,
+    /// Incoming activity has invalid signature
+    #[error("Incoming activity has invalid signature")]
+    ActivitySignatureInvalid,
+    /// Failed to resolve actor via webfinger
+    #[error("Failed to resolve actor via webfinger")]
+    WebfingerResolveFailed(#[from] WebFingerError),
+    /// Failed to serialize outgoing activity
+    #[error("Failed to serialize outgoing activity {1}: {0}")]
+    SerializeOutgoingActivity(serde_json::Error, String),
+    /// Failed to parse an object fetched from url
+    #[error("Failed to parse object {1} with content {2}: {0}")]
+    ParseFetchedObject(serde_json::Error, Url, String),
+    /// Failed to parse an activity received from another instance
+    #[error("Failed to parse incoming activity {}: {0}", match .1 {
+        Some(t) => format!("with id {t}"),
+        None => String::new(),
+    })]
+    ParseReceivedActivity(serde_json::Error, Option<Url>),
+    /// Reqwest Middleware Error
+    #[error(transparent)]
+    ReqwestMiddleware(#[from] reqwest_middleware::Error),
+    /// Reqwest Error
+    #[error(transparent)]
+    Reqwest(#[from] reqwest::Error),
+    /// UTF-8 error
+    #[error(transparent)]
+    Utf8(#[from] FromUtf8Error),
+    /// Url Parse
+    #[error(transparent)]
+    UrlParse(#[from] url::ParseError),
+    /// Signing errors
+    #[error(transparent)]
+    SignError(#[from] SignError),
+    /// Failed to queue activity for sending
+    #[error("Failed to queue activity {0} for sending")]
+    ActivityQueueError(Url),
+    /// Stop activity queue
+    #[error(transparent)]
+    StopActivityQueue(#[from] JoinError),
+    /// Attempted to fetch object which doesn't have valid ActivityPub Content-Type
+    #[error(
+        "Attempted to fetch object from {0} which doesn't have valid ActivityPub Content-Type"
+    )]
+    FetchInvalidContentType(Url),
+    /// Attempted to fetch object but the response's id field doesn't match
+    #[error("Attempted to fetch object from {0} but the response's id field doesn't match")]
+    FetchWrongId(Url),
+    /// Other generic errors
+    #[error("{0}")]
+    Other(String),
+}
+
+impl From<ErrorStack> for Error {
+    fn from(value: ErrorStack) -> Self {
+        Error::Other(value.to_string())
+    }
+}
+
+impl PartialEq for Error {
+    fn eq(&self, other: &Self) -> bool {
+        std::mem::discriminant(self) == std::mem::discriminant(other)
+    }
+}

src/fetch/collection_id.rs

@@ -0,0 +1,193 @@
+use crate::{config::Data, error::Error, fetch::fetch_object_http, traits::Collection};
+use serde::{Deserialize, Serialize};
+use std::{
+    fmt::{Debug, Display, Formatter},
+    marker::PhantomData,
+};
+use url::Url;
+
+/// Typed wrapper for Activitypub Collection ID which helps with dereferencing.
+#[derive(Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct CollectionId<Kind>(Box<Url>, PhantomData<Kind>)
+where
+    Kind: Collection,
+    for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>;
+
+impl<Kind> CollectionId<Kind>
+where
+    Kind: Collection,
+    for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
+{
+    /// Construct a new CollectionId instance
+    pub fn parse(url: &str) -> Result<Self, url::ParseError> {
+        Ok(Self(Box::new(Url::parse(url)?), PhantomData::<Kind>))
+    }
+
+    /// Fetches collection over HTTP
+    ///
+    /// Unlike [ObjectId::dereference](crate::fetch::object_id::ObjectId::dereference) this method doesn't do
+    /// any caching.
+    pub async fn dereference(
+        &self,
+        owner: &<Kind as Collection>::Owner,
+        data: &Data<<Kind as Collection>::DataType>,
+    ) -> Result<Kind, <Kind as Collection>::Error>
+    where
+        <Kind as Collection>::Error: From<Error>,
+    {
+        let res = fetch_object_http(&self.0, data).await?;
+        let redirect_url = &res.url;
+        Kind::verify(&res.object, redirect_url, data).await?;
+        Kind::from_json(res.object, owner, data).await
+    }
+}
+
+/// Need to implement clone manually, to avoid requiring Kind to be Clone
+impl<Kind> Clone for CollectionId<Kind>
+where
+    Kind: Collection,
+    for<'de2> <Kind as Collection>::Kind: serde::Deserialize<'de2>,
+{
+    fn clone(&self) -> Self {
+        CollectionId(self.0.clone(), self.1)
+    }
+}
+
+impl<Kind> Display for CollectionId<Kind>
+where
+    Kind: Collection,
+    for<'de2> <Kind as Collection>::Kind: serde::Deserialize<'de2>,
+{
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0.as_str())
+    }
+}
+
+impl<Kind> Debug for CollectionId<Kind>
+where
+    Kind: Collection,
+    for<'de2> <Kind as Collection>::Kind: serde::Deserialize<'de2>,
+{
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0.as_str())
+    }
+}
+impl<Kind> From<CollectionId<Kind>> for Url
+where
+    Kind: Collection,
+    for<'de2> <Kind as Collection>::Kind: serde::Deserialize<'de2>,
+{
+    fn from(id: CollectionId<Kind>) -> Self {
+        *id.0
+    }
+}
+
+impl<Kind> From<Url> for CollectionId<Kind>
+where
+    Kind: Collection + Send + 'static,
+    for<'de2> <Kind as Collection>::Kind: serde::Deserialize<'de2>,
+{
+    fn from(url: Url) -> Self {
+        CollectionId(Box::new(url), PhantomData::<Kind>)
+    }
+}
+
+impl<Kind> PartialEq for CollectionId<Kind>
+where
+    Kind: Collection,
+    for<'de2> <Kind as Collection>::Kind: serde::Deserialize<'de2>,
+{
+    fn eq(&self, other: &Self) -> bool {
+        self.0.eq(&other.0) && self.1 == other.1
+    }
+}
+
+#[cfg(feature = "diesel")]
+const _IMPL_DIESEL_NEW_TYPE_FOR_COLLECTION_ID: () = {
+    use diesel::{
+        backend::Backend,
+        deserialize::{FromSql, FromStaticSqlRow},
+        expression::AsExpression,
+        internal::derives::as_expression::Bound,
+        pg::Pg,
+        query_builder::QueryId,
+        serialize,
+        serialize::{Output, ToSql},
+        sql_types::{HasSqlType, SingleValue, Text},
+        Expression,
+        Queryable,
+    };
+
+    // TODO: this impl only works for Postgres db because of to_string() call which requires reborrow
+    impl<Kind, ST> ToSql<ST, Pg> for CollectionId<Kind>
+    where
+        Kind: Collection,
+        for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
+        String: ToSql<ST, Pg>,
+    {
+        fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> serialize::Result {
+            let v = self.0.to_string();
+            <String as ToSql<Text, Pg>>::to_sql(&v, &mut out.reborrow())
+        }
+    }
+    impl<'expr, Kind, ST> AsExpression<ST> for &'expr CollectionId<Kind>
+    where
+        Kind: Collection,
+        for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
+        Bound<ST, String>: Expression<SqlType = ST>,
+        ST: SingleValue,
+    {
+        type Expression = Bound<ST, &'expr str>;
+        fn as_expression(self) -> Self::Expression {
+            Bound::new(self.0.as_str())
+        }
+    }
+    impl<Kind, ST> AsExpression<ST> for CollectionId<Kind>
+    where
+        Kind: Collection,
+        for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
+        Bound<ST, String>: Expression<SqlType = ST>,
+        ST: SingleValue,
+    {
+        type Expression = Bound<ST, String>;
+        fn as_expression(self) -> Self::Expression {
+            Bound::new(self.0.to_string())
+        }
+    }
+    impl<Kind, ST, DB> FromSql<ST, DB> for CollectionId<Kind>
+    where
+        Kind: Collection + Send + 'static,
+        for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
+        String: FromSql<ST, DB>,
+        DB: Backend,
+        DB: HasSqlType<ST>,
+    {
+        fn from_sql(
+            raw: DB::RawValue<'_>,
+        ) -> Result<Self, Box<dyn ::std::error::Error + Send + Sync>> {
+            let string: String = FromSql::<ST, DB>::from_sql(raw)?;
+            Ok(CollectionId::parse(&string)?)
+        }
+    }
+    impl<Kind, ST, DB> Queryable<ST, DB> for CollectionId<Kind>
+    where
+        Kind: Collection + Send + 'static,
+        for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
+        String: FromStaticSqlRow<ST, DB>,
+        DB: Backend,
+        DB: HasSqlType<ST>,
+    {
+        type Row = String;
+        fn build(row: Self::Row) -> diesel::deserialize::Result<Self> {
+            Ok(CollectionId::parse(&row)?)
+        }
+    }
+    impl<Kind> QueryId for CollectionId<Kind>
+    where
+        Kind: Collection + 'static,
+        for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
+    {
+        type QueryId = Self;
+    }
+};

src/fetch/mod.rs

@@ -0,0 +1,148 @@
+//! Utilities for fetching data from other servers
+//!
+#![doc = include_str!("../../docs/07_fetching_data.md")]
+
+use crate::{
+    config::Data,
+    error::{Error, Error::ParseFetchedObject},
+    extract_id,
+    http_signatures::sign_request,
+    reqwest_shim::ResponseExt,
+    FEDERATION_CONTENT_TYPE,
+};
+use bytes::Bytes;
+use http::{HeaderValue, StatusCode};
+use serde::de::DeserializeOwned;
+use std::sync::atomic::Ordering;
+use tracing::info;
+use url::Url;
+
+/// Typed wrapper for collection IDs
+pub mod collection_id;
+/// Typed wrapper for Activitypub Object ID which helps with dereferencing and caching
+pub mod object_id;
+/// Resolves identifiers of the form `name@example.com`
+pub mod webfinger;
+
+/// Response from fetching a remote object
+pub struct FetchObjectResponse<Kind> {
+    /// The resolved object
+    pub object: Kind,
+    /// Contains the final URL (different from request URL in case of redirect)
+    pub url: Url,
+    content_type: Option<HeaderValue>,
+    object_id: Option<Url>,
+}
+
+/// Fetch a remote object over HTTP and convert to `Kind`.
+///
+/// [crate::fetch::object_id::ObjectId::dereference] wraps this function to add caching and
+/// conversion to database type. Only use this function directly in exceptional cases where that
+/// behaviour is undesired.
+///
+/// Every time an object is fetched via HTTP, [RequestData.request_counter] is incremented by one.
+/// If the value exceeds [FederationSettings.http_fetch_limit], the request is aborted with
+/// [Error::RequestLimit]. This prevents denial of service attacks where an attack triggers
+/// infinite, recursive fetching of data.
+///
+/// The `Accept` header will be set to the content of [`FEDERATION_CONTENT_TYPE`]. When parsing the
+/// response it ensures that it has a valid `Content-Type` header as defined by ActivityPub, to
+/// prevent security vulnerabilities like [this one](https://github.com/mastodon/mastodon/security/advisories/GHSA-jhrq-qvrm-qr36).
+/// Additionally it checks that the `id` field is identical to the fetch URL (after redirects).
+pub async fn fetch_object_http<T: Clone, Kind: DeserializeOwned>(
+    url: &Url,
+    data: &Data<T>,
+) -> Result<FetchObjectResponse<Kind>, Error> {
+    static FETCH_CONTENT_TYPE: HeaderValue = HeaderValue::from_static(FEDERATION_CONTENT_TYPE);
+    const VALID_RESPONSE_CONTENT_TYPES: [&str; 3] = [
+        FEDERATION_CONTENT_TYPE, // lemmy
+        r#"application/ld+json; profile="https://www.w3.org/ns/activitystreams""#, // activitypub standard
+        r#"application/activity+json; charset=utf-8"#,                             // mastodon
+    ];
+    let res = fetch_object_http_with_accept(url, data, &FETCH_CONTENT_TYPE).await?;
+
+    // Ensure correct content-type to prevent vulnerabilities, with case insensitive comparison.
+    let content_type = res
+        .content_type
+        .as_ref()
+        .and_then(|c| c.to_str().ok())
+        .ok_or(Error::FetchInvalidContentType(res.url.clone()))?;
+    if !VALID_RESPONSE_CONTENT_TYPES.contains(&content_type) {
+        return Err(Error::FetchInvalidContentType(res.url));
+    }
+
+    // Ensure id field matches final url after redirect
+    if res.object_id.as_ref() != Some(&res.url) {
+        return Err(Error::FetchWrongId(res.url));
+    }
+
+    // Dont allow fetching local object. Only check this after the request as a local url
+    // may redirect to a remote object.
+    if data.config.is_local_url(&res.url) {
+        return Err(Error::NotFound);
+    }
+
+    Ok(res)
+}
+
+/// Fetch a remote object over HTTP and convert to `Kind`. This function works exactly as
+/// [`fetch_object_http`] except that the `Accept` header is specified in `content_type`.
+async fn fetch_object_http_with_accept<T: Clone, Kind: DeserializeOwned>(
+    url: &Url,
+    data: &Data<T>,
+    content_type: &HeaderValue,
+) -> Result<FetchObjectResponse<Kind>, Error> {
+    let config = &data.config;
+    config.verify_url_valid(url).await?;
+    info!("Fetching remote object {}", url.to_string());
+
+    let mut counter = data.request_counter.fetch_add(1, Ordering::SeqCst);
+    // fetch_add returns old value so we need to increment manually here
+    counter += 1;
+    if counter > config.http_fetch_limit {
+        return Err(Error::RequestLimit);
+    }
+
+    let req = config
+        .client
+        .get(url.as_str())
+        .header("Accept", content_type)
+        .timeout(config.request_timeout);
+
+    let res = if let Some((actor_id, private_key_pem)) = config.signed_fetch_actor.as_deref() {
+        let req = sign_request(
+            req,
+            actor_id,
+            Bytes::new(),
+            private_key_pem.clone(),
+            data.config.http_signature_compat,
+        )
+        .await?;
+        config.client.execute(req).await?
+    } else {
+        req.send().await?
+    };
+
+    if res.status() == StatusCode::GONE {
+        return Err(Error::ObjectDeleted(url.clone()));
+    }
+
+    let url = res.url().clone();
+    let content_type = res.headers().get("Content-Type").cloned();
+    let text = res.bytes_limited().await?;
+    let object_id = extract_id(&text).ok();
+
+    match serde_json::from_slice(&text) {
+        Ok(object) => Ok(FetchObjectResponse {
+            object,
+            url,
+            content_type,
+            object_id,
+        }),
+        Err(e) => Err(ParseFetchedObject(
+            e,
+            url,
+            String::from_utf8(Vec::from(text))?,
+        )),
+    }
+}

src/fetch/object_id.rs

@@ -0,0 +1,371 @@
+use crate::{config::Data, error::Error, fetch::fetch_object_http, traits::Object};
+use chrono::{DateTime, Duration as ChronoDuration, Utc};
+use serde::{Deserialize, Serialize};
+use std::{
+    fmt::{Debug, Display, Formatter},
+    marker::PhantomData,
+    str::FromStr,
+};
+use url::Url;
+
+impl<T> FromStr for ObjectId<T>
+where
+    T: Object + Send + Debug + 'static,
+    for<'de2> <T as Object>::Kind: Deserialize<'de2>,
+{
+    type Err = url::ParseError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        ObjectId::parse(s)
+    }
+}
+/// Typed wrapper for Activitypub Object ID which helps with dereferencing and caching.
+///
+/// It provides convenient methods for fetching the object from remote server or local database.
+/// Objects are automatically cached locally, so they don't have to be fetched every time. Much of
+/// the crate functionality relies on this wrapper.
+///
+/// Every time an object is fetched via HTTP, [RequestData.request_counter] is incremented by one.
+/// If the value exceeds [FederationSettings.http_fetch_limit], the request is aborted with
+/// [Error::RequestLimit]. This prevents denial of service attacks where an attack triggers
+/// infinite, recursive fetching of data.
+///
+/// ```
+/// # use activitypub_federation::fetch::object_id::ObjectId;
+/// # use activitypub_federation::config::FederationConfig;
+/// # use activitypub_federation::error::Error::NotFound;
+/// # use activitypub_federation::traits::tests::{DbConnection, DbUser};
+/// # tokio::runtime::Runtime::new().unwrap().block_on(async {
+/// # let db_connection = DbConnection;
+/// let config = FederationConfig::builder()
+///     .domain("example.com")
+///     .app_data(db_connection)
+///     .build().await?;
+/// let request_data = config.to_request_data();
+/// let object_id = ObjectId::<DbUser>::parse("https://lemmy.ml/u/nutomic")?;
+/// // Attempt to fetch object from local database or fall back to remote server
+/// let user = object_id.dereference(&request_data).await;
+/// assert!(user.is_ok());
+/// // Now you can also read the object from local database without network requests
+/// let user = object_id.dereference_local(&request_data).await;
+/// assert!(user.is_ok());
+/// # Ok::<(), anyhow::Error>(())
+/// # }).unwrap();
+/// ```
+#[derive(Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct ObjectId<Kind>(Box<Url>, PhantomData<Kind>)
+where
+    Kind: Object,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>;
+
+impl<Kind> ObjectId<Kind>
+where
+    Kind: Object + Send + Debug + 'static,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+{
+    /// Construct a new objectid instance
+    pub fn parse(url: &str) -> Result<Self, url::ParseError> {
+        Ok(Self(Box::new(Url::parse(url)?), PhantomData::<Kind>))
+    }
+
+    /// Returns a reference to the wrapped URL value
+    pub fn inner(&self) -> &Url {
+        &self.0
+    }
+
+    /// Returns the wrapped URL value
+    pub fn into_inner(self) -> Url {
+        *self.0
+    }
+
+    /// Fetches an activitypub object, either from local database (if possible), or over http.
+    pub async fn dereference(
+        &self,
+        data: &Data<<Kind as Object>::DataType>,
+    ) -> Result<Kind, <Kind as Object>::Error>
+    where
+        <Kind as Object>::Error: From<Error>,
+    {
+        let db_object = self.dereference_from_db(data).await?;
+
+        // object found in database
+        if let Some(object) = db_object {
+            if let Some(last_refreshed_at) = object.last_refreshed_at() {
+                let is_local = data.config.is_local_url(&self.0);
+                if !is_local && should_refetch_object(last_refreshed_at) {
+                    // object is outdated and should be refetched
+                    return self.dereference_from_http(data, Some(object)).await;
+                }
+            }
+            Ok(object)
+        }
+        // object not found, need to fetch over http
+        else {
+            self.dereference_from_http(data, None).await
+        }
+    }
+
+    /// If this is a remote object, fetch it from origin instance unconditionally to get the
+    /// latest version, regardless of refresh interval.
+    pub async fn dereference_forced(
+        &self,
+        data: &Data<<Kind as Object>::DataType>,
+    ) -> Result<Kind, <Kind as Object>::Error>
+    where
+        <Kind as Object>::Error: From<Error>,
+    {
+        if data.config.is_local_url(&self.0) {
+            self.dereference_from_db(data)
+                .await
+                .map(|o| o.ok_or(Error::NotFound.into()))?
+        } else {
+            self.dereference_from_http(data, None).await
+        }
+    }
+
+    /// Fetch an object from the local db. Instead of falling back to http, this throws an error if
+    /// the object is not found in the database.
+    pub async fn dereference_local(
+        &self,
+        data: &Data<<Kind as Object>::DataType>,
+    ) -> Result<Kind, <Kind as Object>::Error>
+    where
+        <Kind as Object>::Error: From<Error>,
+    {
+        let object = self.dereference_from_db(data).await?;
+        object.ok_or_else(|| Error::NotFound.into())
+    }
+
+    /// returning none means the object was not found in local db
+    async fn dereference_from_db(
+        &self,
+        data: &Data<<Kind as Object>::DataType>,
+    ) -> Result<Option<Kind>, <Kind as Object>::Error> {
+        let id = self.0.clone();
+        Object::read_from_id(*id, data).await
+    }
+
+    async fn dereference_from_http(
+        &self,
+        data: &Data<<Kind as Object>::DataType>,
+        db_object: Option<Kind>,
+    ) -> Result<Kind, <Kind as Object>::Error>
+    where
+        <Kind as Object>::Error: From<Error>,
+    {
+        let res = fetch_object_http(&self.0, data).await;
+
+        if let Err(Error::ObjectDeleted(url)) = res {
+            if let Some(db_object) = db_object {
+                db_object.delete(data).await?;
+            }
+            return Err(Error::ObjectDeleted(url).into());
+        }
+
+        let res = res?;
+        let redirect_url = &res.url;
+
+        Kind::verify(&res.object, redirect_url, data).await?;
+        Kind::from_json(res.object, data).await
+    }
+
+    /// Returns true if the object's domain matches the one defined in [[FederationConfig.domain]].
+    pub fn is_local(&self, data: &Data<<Kind as Object>::DataType>) -> bool {
+        data.config.is_local_url(&self.0)
+    }
+}
+
+/// Need to implement clone manually, to avoid requiring Kind to be Clone
+impl<Kind> Clone for ObjectId<Kind>
+where
+    Kind: Object,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+{
+    fn clone(&self) -> Self {
+        ObjectId(self.0.clone(), self.1)
+    }
+}
+
+static ACTOR_REFETCH_INTERVAL_SECONDS: i64 = 24 * 60 * 60;
+static ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG: i64 = 20;
+
+/// Determines when a remote actor should be refetched from its instance. In release builds, this is
+/// `ACTOR_REFETCH_INTERVAL_SECONDS` after the last refetch, in debug builds
+/// `ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG`.
+fn should_refetch_object(last_refreshed: DateTime<Utc>) -> bool {
+    let update_interval = if cfg!(debug_assertions) {
+        // avoid infinite loop when fetching community outbox
+        ChronoDuration::try_seconds(ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG).expect("valid duration")
+    } else {
+        ChronoDuration::try_seconds(ACTOR_REFETCH_INTERVAL_SECONDS).expect("valid duration")
+    };
+    let refresh_limit = Utc::now() - update_interval;
+    last_refreshed.lt(&refresh_limit)
+}
+
+impl<Kind> Display for ObjectId<Kind>
+where
+    Kind: Object,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+{
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0.as_str())
+    }
+}
+
+impl<Kind> Debug for ObjectId<Kind>
+where
+    Kind: Object,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+{
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0.as_str())
+    }
+}
+
+impl<Kind> From<ObjectId<Kind>> for Url
+where
+    Kind: Object,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+{
+    fn from(id: ObjectId<Kind>) -> Self {
+        *id.0
+    }
+}
+
+impl<Kind> From<Url> for ObjectId<Kind>
+where
+    Kind: Object + Send + 'static,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+{
+    fn from(url: Url) -> Self {
+        ObjectId(Box::new(url), PhantomData::<Kind>)
+    }
+}
+
+impl<Kind> PartialEq for ObjectId<Kind>
+where
+    Kind: Object,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+{
+    fn eq(&self, other: &Self) -> bool {
+        self.0.eq(&other.0) && self.1 == other.1
+    }
+}
+
+#[cfg(feature = "diesel")]
+const _IMPL_DIESEL_NEW_TYPE_FOR_OBJECT_ID: () = {
+    use diesel::{
+        backend::Backend,
+        deserialize::{FromSql, FromStaticSqlRow},
+        expression::AsExpression,
+        internal::derives::as_expression::Bound,
+        pg::Pg,
+        query_builder::QueryId,
+        serialize,
+        serialize::{Output, ToSql},
+        sql_types::{HasSqlType, SingleValue, Text},
+        Expression,
+        Queryable,
+    };
+
+    // TODO: this impl only works for Postgres db because of to_string() call which requires reborrow
+    impl<Kind, ST> ToSql<ST, Pg> for ObjectId<Kind>
+    where
+        Kind: Object,
+        for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+        String: ToSql<ST, Pg>,
+    {
+        fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> serialize::Result {
+            let v = self.0.to_string();
+            <String as ToSql<Text, Pg>>::to_sql(&v, &mut out.reborrow())
+        }
+    }
+    impl<'expr, Kind, ST> AsExpression<ST> for &'expr ObjectId<Kind>
+    where
+        Kind: Object,
+        for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+        Bound<ST, String>: Expression<SqlType = ST>,
+        ST: SingleValue,
+    {
+        type Expression = Bound<ST, &'expr str>;
+        fn as_expression(self) -> Self::Expression {
+            Bound::new(self.0.as_str())
+        }
+    }
+    impl<Kind, ST> AsExpression<ST> for ObjectId<Kind>
+    where
+        Kind: Object,
+        for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+        Bound<ST, String>: Expression<SqlType = ST>,
+        ST: SingleValue,
+    {
+        type Expression = Bound<ST, String>;
+        fn as_expression(self) -> Self::Expression {
+            Bound::new(self.0.to_string())
+        }
+    }
+    impl<Kind, ST, DB> FromSql<ST, DB> for ObjectId<Kind>
+    where
+        Kind: Object + Send + 'static,
+        for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+        String: FromSql<ST, DB>,
+        DB: Backend,
+        DB: HasSqlType<ST>,
+    {
+        fn from_sql(
+            raw: DB::RawValue<'_>,
+        ) -> Result<Self, Box<dyn ::std::error::Error + Send + Sync>> {
+            let string: String = FromSql::<ST, DB>::from_sql(raw)?;
+            Ok(ObjectId::parse(&string)?)
+        }
+    }
+    impl<Kind, ST, DB> Queryable<ST, DB> for ObjectId<Kind>
+    where
+        Kind: Object + Send + 'static,
+        for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+        String: FromStaticSqlRow<ST, DB>,
+        DB: Backend,
+        DB: HasSqlType<ST>,
+    {
+        type Row = String;
+        fn build(row: Self::Row) -> diesel::deserialize::Result<Self> {
+            Ok(ObjectId::parse(&row)?)
+        }
+    }
+    impl<Kind> QueryId for ObjectId<Kind>
+    where
+        Kind: Object + 'static,
+        for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+    {
+        type QueryId = Self;
+    }
+};
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+pub mod tests {
+    use super::*;
+    use crate::traits::tests::DbUser;
+
+    #[test]
+    fn test_deserialize() {
+        let id = ObjectId::<DbUser>::parse("http://test.com/").unwrap();
+
+        let string = serde_json::to_string(&id).unwrap();
+        assert_eq!("\"http://test.com/\"", string);
+
+        let parsed: ObjectId<DbUser> = serde_json::from_str(&string).unwrap();
+        assert_eq!(parsed, id);
+    }
+
+    #[test]
+    fn test_should_refetch_object() {
+        let one_second_ago = Utc::now() - ChronoDuration::try_seconds(1).unwrap();
+        assert!(!should_refetch_object(one_second_ago));
+
+        let two_days_ago = Utc::now() - ChronoDuration::try_days(2).unwrap();
+        assert!(should_refetch_object(two_days_ago));
+    }
+}

src/fetch/webfinger.rs

@@ -0,0 +1,306 @@
+use crate::{
+    config::Data,
+    error::Error,
+    fetch::{fetch_object_http_with_accept, object_id::ObjectId},
+    traits::{Actor, Object},
+    FEDERATION_CONTENT_TYPE,
+};
+use http::HeaderValue;
+use itertools::Itertools;
+use once_cell::sync::Lazy;
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+use std::{collections::HashMap, fmt::Display};
+use tracing::debug;
+use url::Url;
+
+/// Errors relative to webfinger handling
+#[derive(thiserror::Error, Debug)]
+pub enum WebFingerError {
+    /// The webfinger identifier is invalid
+    #[error("The webfinger identifier is invalid")]
+    WrongFormat,
+    /// The webfinger identifier doesn't match the expected instance domain name
+    #[error("The webfinger identifier doesn't match the expected instance domain name")]
+    WrongDomain,
+    /// The wefinger object did not contain any link to an activitypub item
+    #[error("The webfinger object did not contain any link to an activitypub item")]
+    NoValidLink,
+}
+
+impl WebFingerError {
+    fn into_crate_error(self) -> Error {
+        self.into()
+    }
+}
+
+/// The content-type for webfinger responses.
+pub static WEBFINGER_CONTENT_TYPE: HeaderValue = HeaderValue::from_static("application/jrd+json");
+
+/// Takes an identifier of the form `name@example.com`, and returns an object of `Kind`.
+///
+/// For this the identifier is first resolved via webfinger protocol to an Activitypub ID. This ID
+/// is then fetched using [ObjectId::dereference], and the result returned.
+pub async fn webfinger_resolve_actor<T: Clone, Kind>(
+    identifier: &str,
+    data: &Data<T>,
+) -> Result<Kind, <Kind as Object>::Error>
+where
+    Kind: Object + Actor + Send + 'static + Object<DataType = T>,
+    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
+    <Kind as Object>::Error: From<crate::error::Error> + Send + Sync + Display,
+{
+    let (_, domain) = identifier
+        .splitn(2, '@')
+        .collect_tuple()
+        .ok_or(WebFingerError::WrongFormat.into_crate_error())?;
+    let protocol = if data.config.debug { "http" } else { "https" };
+    let fetch_url =
+        format!("{protocol}://{domain}/.well-known/webfinger?resource=acct:{identifier}");
+    debug!("Fetching webfinger url: {}", &fetch_url);
+
+    let res: Webfinger = fetch_object_http_with_accept(
+        &Url::parse(&fetch_url).map_err(Error::UrlParse)?,
+        data,
+        &WEBFINGER_CONTENT_TYPE,
+    )
+    .await?
+    .object;
+
+    debug_assert_eq!(res.subject, format!("acct:{identifier}"));
+    let links: Vec<Url> = res
+        .links
+        .iter()
+        .filter(|link| {
+            if let Some(type_) = &link.kind {
+                type_.starts_with("application/")
+            } else {
+                false
+            }
+        })
+        .filter_map(|l| l.href.clone())
+        .collect();
+
+    for l in links {
+        let object = ObjectId::<Kind>::from(l).dereference(data).await;
+        match object {
+            Ok(obj) => return Ok(obj),
+            Err(error) => debug!(%error, "Failed to dereference link"),
+        }
+    }
+    Err(WebFingerError::NoValidLink.into_crate_error().into())
+}
+
+/// Extracts username from a webfinger resource parameter.
+///
+/// Use this method for your HTTP handler at `.well-known/webfinger` to handle incoming webfinger
+/// request. For a parameter of the form `acct:gargron@mastodon.social` it returns `gargron`.
+///
+/// Returns an error if query doesn't match local domain.
+///
+///```
+/// # use activitypub_federation::config::FederationConfig;
+/// # use activitypub_federation::traits::tests::DbConnection;
+/// # use activitypub_federation::fetch::webfinger::extract_webfinger_name;
+/// # tokio::runtime::Runtime::new().unwrap().block_on(async {
+/// # let db_connection = DbConnection;
+/// let config = FederationConfig::builder()
+///     .domain("example.com")
+///     .app_data(db_connection)
+///     .build()
+///     .await
+///     .unwrap();
+/// let data = config.to_request_data();
+/// let res = extract_webfinger_name("acct:test_user@example.com", &data).unwrap();
+/// assert_eq!(res, "test_user");
+/// # Ok::<(), anyhow::Error>(())
+/// }).unwrap();
+///```
+pub fn extract_webfinger_name<'i, T>(query: &'i str, data: &Data<T>) -> Result<&'i str, Error>
+where
+    T: Clone,
+{
+    static WEBFINGER_REGEX: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"^acct:([\p{L}0-9_\.\-]+)@(.*)$").expect("compile regex"));
+    // Regex to extract usernames from webfinger query. Supports different alphabets using `\p{L}`.
+    // TODO: This should use a URL parser
+    let captures = WEBFINGER_REGEX
+        .captures(query)
+        .ok_or(WebFingerError::WrongFormat)?;
+
+    let account_name = captures.get(1).ok_or(WebFingerError::WrongFormat)?;
+
+    if captures.get(2).map(|m| m.as_str()) != Some(data.domain()) {
+        return Err(WebFingerError::WrongDomain.into());
+    }
+    Ok(account_name.as_str())
+}
+
+/// Builds a basic webfinger response for the actor.
+///
+/// It assumes that the given URL is valid both to the view the actor in a browser as HTML, and
+/// for fetching it over Activitypub with `activity+json`. This setup is commonly used for ease
+/// of discovery.
+///
+/// ```
+/// # use url::Url;
+/// # use activitypub_federation::fetch::webfinger::build_webfinger_response;
+/// let subject = "acct:nutomic@lemmy.ml".to_string();
+/// let url = Url::parse("https://lemmy.ml/u/nutomic")?;
+/// build_webfinger_response(subject, url);
+/// # Ok::<(), anyhow::Error>(())
+/// ```
+pub fn build_webfinger_response(subject: String, url: Url) -> Webfinger {
+    build_webfinger_response_with_type(subject, vec![(url, None)])
+}
+
+/// Builds a webfinger response similar to `build_webfinger_response`. Use this when you want to
+/// return multiple actors who share the same namespace and to specify the type of the actor.
+///
+/// `urls` takes a vector of tuples. The first item of the tuple is the URL while the second
+/// item is the type, such as `"Person"` or `"Group"`. If `None` is passed for the type, the field
+/// will be empty.
+///
+/// ```
+/// # use url::Url;
+/// # use activitypub_federation::fetch::webfinger::build_webfinger_response_with_type;
+/// let subject = "acct:nutomic@lemmy.ml".to_string();
+/// let user = Url::parse("https://lemmy.ml/u/nutomic")?;
+/// let group = Url::parse("https://lemmy.ml/c/asklemmy")?;
+/// build_webfinger_response_with_type(subject, vec![
+///     (user, Some("Person")),
+///     (group, Some("Group"))]);
+/// # Ok::<(), anyhow::Error>(())
+/// ```
+pub fn build_webfinger_response_with_type(
+    subject: String,
+    urls: Vec<(Url, Option<&str>)>,
+) -> Webfinger {
+    Webfinger {
+        subject,
+        links: urls.iter().fold(vec![], |mut acc, (url, kind)| {
+            let properties: HashMap<Url, String> = kind
+                .map(|kind| {
+                    HashMap::from([(
+                        "https://www.w3.org/ns/activitystreams#type"
+                            .parse()
+                            .expect("parse url"),
+                        kind.to_string(),
+                    )])
+                })
+                .unwrap_or_default();
+            let mut links = vec![
+                WebfingerLink {
+                    rel: Some("http://webfinger.net/rel/profile-page".to_string()),
+                    kind: Some("text/html".to_string()),
+                    href: Some(url.clone()),
+                    ..Default::default()
+                },
+                WebfingerLink {
+                    rel: Some("self".to_string()),
+                    kind: Some(FEDERATION_CONTENT_TYPE.to_string()),
+                    href: Some(url.clone()),
+                    properties,
+                    ..Default::default()
+                },
+            ];
+            acc.append(&mut links);
+            acc
+        }),
+        aliases: vec![],
+        properties: Default::default(),
+    }
+}
+
+/// A webfinger response with information about a `Person` or other type of actor.
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct Webfinger {
+    /// The actor which is described here, for example `acct:LemmyDev@mastodon.social`
+    pub subject: String,
+    /// Links where further data about `subject` can be retrieved
+    pub links: Vec<WebfingerLink>,
+    /// Other Urls which identify the same actor as the `subject`
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub aliases: Vec<Url>,
+    /// Additional data about the subject
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub properties: HashMap<Url, String>,
+}
+
+/// A single link included as part of a [Webfinger] response.
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct WebfingerLink {
+    /// Relationship of the link, such as `self` or `http://webfinger.net/rel/profile-page`
+    pub rel: Option<String>,
+    /// Media type of the target resource
+    #[serde(rename = "type")]
+    pub kind: Option<String>,
+    /// Url pointing to the target resource
+    pub href: Option<Url>,
+    /// Used for remote follow external interaction url
+    pub template: Option<String>,
+    /// Additional data about the link
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub properties: HashMap<Url, String>,
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::{
+        config::FederationConfig,
+        traits::tests::{DbConnection, DbUser},
+    };
+
+    #[tokio::test]
+    async fn test_webfinger() -> Result<(), Error> {
+        let config = FederationConfig::builder()
+            .domain("example.com")
+            .app_data(DbConnection)
+            .build()
+            .await
+            .unwrap();
+        let data = config.to_request_data();
+
+        webfinger_resolve_actor::<DbConnection, DbUser>("LemmyDev@mastodon.social", &data).await?;
+        // poa.st is as of 2023-07-14 the largest Pleroma instance
+        webfinger_resolve_actor::<DbConnection, DbUser>("graf@poa.st", &data).await?;
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_webfinger_extract_name() -> Result<(), Error> {
+        use crate::traits::tests::DbConnection;
+        let data = Data {
+            config: FederationConfig::builder()
+                .domain("example.com")
+                .app_data(DbConnection)
+                .build()
+                .await
+                .unwrap(),
+            request_counter: Default::default(),
+        };
+        assert_eq!(
+            Ok("test123"),
+            extract_webfinger_name("acct:test123@example.com", &data)
+        );
+        assert_eq!(
+            Ok("Владимир"),
+            extract_webfinger_name("acct:Владимир@example.com", &data)
+        );
+        assert_eq!(
+            Ok("example.com"),
+            extract_webfinger_name("acct:example.com@example.com", &data)
+        );
+        assert_eq!(
+            Ok("da-sh"),
+            extract_webfinger_name("acct:da-sh@example.com", &data)
+        );
+        assert_eq!(
+            Ok("تجريب"),
+            extract_webfinger_name("acct:تجريب@example.com", &data)
+        );
+        Ok(())
+    }
+}

src/http_signatures.rs

@@ -0,0 +1,421 @@
+//! Generating keypairs, creating and verifying signatures
+//!
+//! Signature creation and verification is handled internally in the library. See
+//! [send_activity](crate::activity_sending::SendActivityTask::sign_and_send) and
+//! [receive_activity (actix-web)](crate::actix_web::inbox::receive_activity) /
+//! [receive_activity (axum)](crate::axum::inbox::receive_activity).
+
+use crate::{
+    config::Data,
+    error::{Error, Error::ActivitySignatureInvalid},
+    fetch::object_id::ObjectId,
+    protocol::public_key::main_key_id,
+    traits::{Actor, Object},
+};
+use base64::{engine::general_purpose::STANDARD as Base64, Engine};
+use bytes::Bytes;
+use http::{header::HeaderName, uri::PathAndQuery, HeaderValue, Method, Uri};
+use http_signature_normalization_reqwest::{
+    prelude::{Config, SignExt},
+    DefaultSpawner,
+};
+use once_cell::sync::Lazy;
+use openssl::{
+    hash::MessageDigest,
+    pkey::{PKey, Private},
+    rsa::Rsa,
+    sign::{Signer, Verifier},
+};
+use reqwest::Request;
+use reqwest_middleware::RequestBuilder;
+use serde::Deserialize;
+use sha2::{Digest, Sha256};
+use std::{collections::BTreeMap, fmt::Debug, io::ErrorKind, time::Duration};
+use tracing::debug;
+use url::Url;
+
+/// A private/public key pair used for HTTP signatures
+#[derive(Debug, Clone)]
+pub struct Keypair {
+    /// Private key in PEM format
+    pub private_key: String,
+    /// Public key in PEM format
+    pub public_key: String,
+}
+
+impl Keypair {
+    /// Helper method to turn this into an openssl private key
+    #[cfg(test)]
+    pub(crate) fn private_key(&self) -> Result<PKey<Private>, anyhow::Error> {
+        Ok(PKey::private_key_from_pem(self.private_key.as_bytes())?)
+    }
+}
+
+/// Generate a random asymmetric keypair for ActivityPub HTTP signatures.
+pub fn generate_actor_keypair() -> Result<Keypair, std::io::Error> {
+    let rsa = Rsa::generate(2048)?;
+    let pkey = PKey::from_rsa(rsa)?;
+    let public_key = pkey.public_key_to_pem()?;
+    let private_key = pkey.private_key_to_pem_pkcs8()?;
+    let key_to_string = |key| match String::from_utf8(key) {
+        Ok(s) => Ok(s),
+        Err(e) => Err(std::io::Error::new(
+            ErrorKind::Other,
+            format!("Failed converting key to string: {}", e),
+        )),
+    };
+    Ok(Keypair {
+        private_key: key_to_string(private_key)?,
+        public_key: key_to_string(public_key)?,
+    })
+}
+
+/// Time for which HTTP signatures are valid.
+///
+/// This field is optional in the standard, but required by the Rust library. It is not clear
+/// what security concerns this expiration solves (if any), so we set a very high value of one hour
+/// to avoid any potential problems due to wrong clocks, overloaded servers or delayed delivery.
+pub(crate) const EXPIRES_AFTER: Duration = Duration::from_secs(60 * 60);
+
+/// Creates an HTTP post request to `inbox_url`, with the given `client` and `headers`, and
+/// `activity` as request body. The request is signed with `private_key` and then sent.
+pub(crate) async fn sign_request(
+    request_builder: RequestBuilder,
+    actor_id: &Url,
+    activity: Bytes,
+    private_key: PKey<Private>,
+    http_signature_compat: bool,
+) -> Result<Request, Error> {
+    static CONFIG: Lazy<Config<DefaultSpawner>> =
+        Lazy::new(|| Config::new().set_expiration(EXPIRES_AFTER));
+    static CONFIG_COMPAT: Lazy<Config> = Lazy::new(|| {
+        Config::new()
+            .mastodon_compat()
+            .set_expiration(EXPIRES_AFTER)
+    });
+
+    let key_id = main_key_id(actor_id);
+    let sig_conf = match http_signature_compat {
+        false => CONFIG.clone(),
+        true => CONFIG_COMPAT.clone(),
+    };
+    request_builder
+        .signature_with_digest(
+            sig_conf.clone(),
+            key_id,
+            Sha256::new(),
+            activity,
+            move |signing_string| {
+                let mut signer = Signer::new(MessageDigest::sha256(), &private_key)?;
+                signer.update(signing_string.as_bytes())?;
+
+                Ok(Base64.encode(signer.sign_to_vec()?)) as Result<_, Error>
+            },
+        )
+        .await
+}
+
+/// Verifies the HTTP signature on an incoming federation request
+/// for a given actor's public key.
+///
+/// Internally, this just converts the headers to a BTreeMap and passes to
+/// `verify_signature_inner` for actual signature verification.
+pub(crate) fn verify_signature<'a, H>(
+    headers: H,
+    method: &Method,
+    uri: &Uri,
+    public_key: &str,
+) -> Result<(), Error>
+where
+    H: IntoIterator<Item = (&'a HeaderName, &'a HeaderValue)>,
+{
+    let mut header_map = BTreeMap::<String, String>::new();
+    for (name, value) in headers {
+        if let Ok(value) = value.to_str() {
+            header_map.insert(name.to_string(), value.to_string());
+        }
+    }
+
+    verify_signature_inner(header_map, method, uri, public_key)
+}
+
+/// Checks whether the given federation request has a valid signature,
+/// from any actor of type A, and returns that actor if a valid signature is found.
+/// This function will return an `Err` variant when no signature is found
+/// or if the signature could not be verified.
+pub(crate) async fn signing_actor<'a, A, H>(
+    headers: H,
+    method: &Method,
+    uri: &Uri,
+    data: &Data<<A as Object>::DataType>,
+) -> Result<A, <A as Object>::Error>
+where
+    A: Object + Actor,
+    <A as Object>::Error: From<Error>,
+    for<'de2> <A as Object>::Kind: Deserialize<'de2>,
+    H: IntoIterator<Item = (&'a HeaderName, &'a HeaderValue)>,
+{
+    let mut header_map = BTreeMap::<String, String>::new();
+    for (name, value) in headers {
+        if let Ok(value) = value.to_str() {
+            header_map.insert(name.to_string(), value.to_string());
+        }
+    }
+    let signature = header_map
+        .get("signature")
+        .ok_or(Error::ActivitySignatureInvalid)?;
+
+    let actor_id_re = regex::Regex::new("keyId=\"([^\"]+)#([^\"]+)\"").expect("regex error");
+    let actor_id = match actor_id_re.captures(signature) {
+        None => return Err(Error::ActivitySignatureInvalid.into()),
+        Some(caps) => caps.get(1).expect("regex error").as_str(),
+    };
+    let actor_url = Url::parse(actor_id).map_err(|_| Error::ActivitySignatureInvalid)?;
+    let actor_id: ObjectId<A> = actor_url.into();
+
+    let actor = actor_id.dereference(data).await?;
+    let public_key = actor.public_key_pem();
+
+    verify_signature_inner(header_map, method, uri, public_key)?;
+
+    Ok(actor)
+}
+
+/// Verifies that the signature present in the request is valid for
+/// the specified actor's public key.
+fn verify_signature_inner(
+    header_map: BTreeMap<String, String>,
+    method: &Method,
+    uri: &Uri,
+    public_key: &str,
+) -> Result<(), Error> {
+    static CONFIG: Lazy<http_signature_normalization::Config> = Lazy::new(|| {
+        http_signature_normalization::Config::new()
+            .set_expiration(EXPIRES_AFTER)
+            .require_digest()
+    });
+
+    let path_and_query = uri.path_and_query().map(PathAndQuery::as_str).unwrap_or("");
+
+    let verified = CONFIG
+        .begin_verify(method.as_str(), path_and_query, header_map)
+        .map_err(|val| Error::Other(val.to_string()))?
+        .verify(|signature, signing_string| -> Result<bool, Error> {
+            debug!(
+                "Verifying with key {}, message {}",
+                &public_key, &signing_string
+            );
+            let public_key = PKey::public_key_from_pem(public_key.as_bytes())?;
+            let mut verifier = Verifier::new(MessageDigest::sha256(), &public_key)?;
+            verifier.update(signing_string.as_bytes())?;
+
+            let base64_decoded = Base64
+                .decode(signature)
+                .map_err(|err| Error::Other(err.to_string()))?;
+
+            Ok(verifier.verify(&base64_decoded)?)
+        })?;
+
+    if verified {
+        debug!("verified signature for {}", uri);
+        Ok(())
+    } else {
+        Err(ActivitySignatureInvalid)
+    }
+}
+
+#[derive(Clone, Debug)]
+struct DigestPart {
+    /// We assume that SHA256 is used which is the case with all major fediverse platforms
+    #[allow(dead_code)]
+    pub algorithm: String,
+    /// The hashsum
+    pub digest: String,
+}
+
+impl DigestPart {
+    fn try_from_header(h: &HeaderValue) -> Option<Vec<DigestPart>> {
+        let h = h.to_str().ok()?.split(';').next()?;
+        let v: Vec<_> = h
+            .split(',')
+            .filter_map(|p| {
+                let mut iter = p.splitn(2, '=');
+                iter.next()
+                    .and_then(|alg| iter.next().map(|value| (alg, value)))
+            })
+            .map(|(alg, value)| DigestPart {
+                algorithm: alg.to_owned(),
+                digest: value.to_owned(),
+            })
+            .collect();
+
+        if v.is_empty() {
+            None
+        } else {
+            Some(v)
+        }
+    }
+}
+
+/// Verify body of an inbox request against the hash provided in `Digest` header.
+pub(crate) fn verify_body_hash(
+    digest_header: Option<&HeaderValue>,
+    body: &[u8],
+) -> Result<(), Error> {
+    let digest = digest_header
+        .and_then(DigestPart::try_from_header)
+        .ok_or(Error::ActivityBodyDigestInvalid)?;
+    let mut hasher = Sha256::new();
+
+    for part in digest {
+        hasher.update(body);
+        if Base64.encode(hasher.finalize_reset()) != part.digest {
+            return Err(Error::ActivityBodyDigestInvalid);
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+pub mod test {
+    use super::*;
+    use crate::activity_sending::generate_request_headers;
+    use reqwest::Client;
+    use reqwest_middleware::ClientWithMiddleware;
+    use std::str::FromStr;
+
+    static ACTOR_ID: Lazy<Url> = Lazy::new(|| Url::parse("https://example.com/u/alice").unwrap());
+    static INBOX_URL: Lazy<Url> =
+        Lazy::new(|| Url::parse("https://example.com/u/alice/inbox").unwrap());
+
+    #[tokio::test]
+    async fn test_sign() {
+        let mut headers = generate_request_headers(&INBOX_URL);
+        // use hardcoded date in order to test against hardcoded signature
+        headers.insert(
+            "date",
+            HeaderValue::from_str("Tue, 28 Mar 2023 21:03:44 GMT").unwrap(),
+        );
+
+        let request_builder = ClientWithMiddleware::from(Client::new())
+            .post(INBOX_URL.to_string())
+            .headers(headers);
+        let request = sign_request(
+            request_builder,
+            &ACTOR_ID,
+            "my activity".into(),
+            PKey::private_key_from_pem(test_keypair().private_key.as_bytes()).unwrap(),
+            // set this to prevent created/expires headers to be generated and inserted
+            // automatically from current time
+            true,
+        )
+        .await
+        .unwrap();
+        let signature = request
+            .headers()
+            .get("signature")
+            .unwrap()
+            .to_str()
+            .unwrap();
+        let expected_signature = concat!(
+            "keyId=\"https://example.com/u/alice#main-key\",",
+            "algorithm=\"hs2019\",",
+            "headers=\"(request-target) content-type date digest host\",",
+            "signature=\"BpZhHNqzd6d6jhWOxyJ0jXwWWxiKMNK7i3mrr/5mVFnH7fUpicwqw8cSYVr",
+            "cwWjt0I07HW7rZFUfIdSgCoOEdvxtrccF/hTrwYgm8O6SQRHl1UfFtDR6e9EpfPieVmTjo0",
+            "QVfyzLLa41rmnz/yFqqer/v0kcdED51/dGe8NCGPBbhgK6C4oh7r+XHsQZMIhh38BcfZVWN",
+            "YaMqgyhFxu2f34IKnOEk6NjSaNtO+PzQUhbksTvH0Vvi6R0dtQINJFdONVBl4AwDC1INeF5",
+            "uhQo/SaKHfP3UitUHdM5Pbn+LhZYDB9AaQAW5ZGD43Aw15ecwsnKi4HcjV8nBw4zehlvaQ==\""
+        );
+        assert_eq!(signature, expected_signature);
+    }
+
+    #[tokio::test]
+    async fn test_verify() {
+        let headers = generate_request_headers(&INBOX_URL);
+        let request_builder = ClientWithMiddleware::from(Client::new())
+            .post(INBOX_URL.to_string())
+            .headers(headers);
+        let request = sign_request(
+            request_builder,
+            &ACTOR_ID,
+            "my activity".to_string().into(),
+            PKey::private_key_from_pem(test_keypair().private_key.as_bytes()).unwrap(),
+            false,
+        )
+        .await
+        .unwrap();
+
+        let valid = verify_signature(
+            request.headers(),
+            request.method(),
+            &Uri::from_str(request.url().as_str()).unwrap(),
+            &test_keypair().public_key,
+        );
+        println!("{:?}", &valid);
+        assert!(valid.is_ok());
+    }
+
+    #[test]
+    fn test_verify_body_hash_valid() {
+        let digest_header =
+            HeaderValue::from_static("SHA-256=lzFT+G7C2hdI5j8M+FuJg1tC+O6AGMVJhooTCKGfbKM=");
+        let body = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.";
+        let valid = verify_body_hash(Some(&digest_header), body.as_bytes());
+        println!("{:?}", &valid);
+        assert!(valid.is_ok());
+    }
+
+    #[test]
+    fn test_verify_body_hash_not_valid() {
+        let digest_header =
+            HeaderValue::from_static("SHA-256=Z9h7DJfYWjffXw2XftmWCnpEaK/yqOHKvzCIzIaqgbU=");
+        let body = "lorem ipsum";
+        let invalid = verify_body_hash(Some(&digest_header), body.as_bytes());
+        assert_eq!(invalid, Err(Error::ActivityBodyDigestInvalid));
+    }
+
+    pub fn test_keypair() -> Keypair {
+        let rsa = Rsa::private_key_from_pem(PRIVATE_KEY.as_bytes()).unwrap();
+        let pkey = PKey::from_rsa(rsa).unwrap();
+        let private_key = pkey.private_key_to_pem_pkcs8().unwrap();
+        let public_key = pkey.public_key_to_pem().unwrap();
+        Keypair {
+            private_key: String::from_utf8(private_key).unwrap(),
+            public_key: String::from_utf8(public_key).unwrap(),
+        }
+    }
+
+    /// Hardcoded private key so that signature doesn't change across runs
+    const PRIVATE_KEY: &str = concat!(
+        "-----BEGIN RSA PRIVATE KEY-----\n",
+        "MIIEogIBAAKCAQEA2kZpsvWYrwM9zMQiDwo4k6/VfpK2aDTeVe9ZkcvDrrWfqt72\n",
+        "QSjjtXLa8sxJlEn+/zbnZ1lG3AO/WsKs2jiOycNQHBS1ITnSZKEpdKnAoLUn4k16\n",
+        "YivRmALyLedOfIrvMtQzH8a+kOQ71u2Wa3H9jpkCT5W9OneEBa3VjQp49kcrF3tm\n",
+        "mrEUhfai5GJM4xrdr587y7exkBF4wObepta9opSeuBkPV4QXZPfgmjwW+oOTheVH\n",
+        "6L7yjzvjW92j4/T6XKAcu0kn/aQhR8SiGtPBMyOlcW4S2eDHWf1RlqbNGb5L9Qam\n",
+        "fb0WAymx0ANLUDQyXAu5zViMrd4g8mgdkg7C1wIDAQABAoIBAAHAT0Uvsguz0Frq\n",
+        "0Li8+A4I4U/RQeqW6f9XtHWpl3NSYuqOPJZY2DxypHRB1Iex13x/gBHH/8jwgShR\n",
+        "2x/3ev9kmsLu6f+CcdniCFQdFiRaVh/IFI0Ve7cz5tkcoiuSB2NDNcaYFwIdYqfr\n",
+        "Ytz2OCn2hLQHKB9M9pLMSnDsPmMAOveY11XfhkECrWlh1bx9YPyJScnNKTblB3M+\n",
+        "GhYL3xzuCxPCC9nUfqz7Y8FnZTCmePOwcRflJDTLFs6Bqkv1PZOZWzI+7akaJxfI\n",
+        "SOSw3VkGegsdoGVgHobqT2tqL8vuKM1bs47PFwWjVCGEoOvcC/Ha1+INemWbh7VA\n",
+        "Xa/jvxkCgYEA/+AxeMCLCmH/F696W3RpPdFL25wSYQr1auV2xRfmsT+hhpSp3yz/\n",
+        "ypkazS9TbnSCm18up+jE9rJ1c9VIZrgcTeKzPURzE68RR8uOsa9o9kaUzfyvRAzb\n",
+        "fmQXMvv2rmm9U7srhjpvKo1BcHpQIQYToKt0TOv7soSEY2jGNvaK6i0CgYEA2mGL\n",
+        "sL36WoHF3x2DZNvknLJGjxPSMmdjjfflFRqxKeP+Sf54C4QH/1hxHe/yl/KMBTfa\n",
+        "woBl05SrwTnQ7bOeR8VTmzP53JfkECT5I9h/g8vT8dkz5WQXWNDgy61Imq/UmWwm\n",
+        "DHElGrkF31oy5w6+aZ58Sa5bXhBDYpkUP9+pV5MCgYAW5BCo89i8gg3XKZyxp9Vu\n",
+        "cVXu/KRsSBWyjXq1oTDDNKUXrB8SVy0/C7lpF83H+OZiTf6XiOxuAYMebLtAbUIi\n",
+        "+Z/9YC1HWocaPCy02rNyLNhNIUjwtpHAWeX1arMj4VPNtNXs+TdOwDpVfKvEeI2y\n",
+        "9wO9ifMHgnFxj0MEUcQVtQKBgHg2Mhs8uM+RmEbVjDq9AP9w835XPuIYH6lKyIPx\n",
+        "iYyxwI0i0xojt/NL0BjWuQgDsCg/MuDWpTbvJAzdsrDmqz5+1SMeXXCc/CIW+D5P\n",
+        "MwJt9WGwWuzvSBrQAK6d2NWt7K335on6zp4DM8RbdqHSb+bcIza8D/ebpDxmX8s5\n",
+        "Z5KZAoGAX8u+63w1uy1FLhf48SqmjOqkAjdUZCWEmaim69koAOdTIBSSDOnAqzGu\n",
+        "wIVdLLzI6xTgbYmfErCwpU2v8MfUWr0BDzjQ9G6c5rhcS1BkfxbeAsC42XaVIgCk\n",
+        "2sMNMqi6f96jbp4IQI70BpecsnBAUa+VoT57bZRvy0lW26w9tYI=\n",
+        "-----END RSA PRIVATE KEY-----\n"
+    );
+}

src/lib.rs

@@ -1,99 +1,74 @@
-use crate::core::activity_queue::create_activity_queue;
-use background_jobs::Manager;
-use derive_builder::Builder;
-use reqwest_middleware::ClientWithMiddleware;
-use std::time::Duration;
+#![doc = include_str!("../docs/01_intro.md")]
+#![doc = include_str!("../docs/02_overview.md")]
+#![doc = include_str!("../docs/03_federating_users.md")]
+#![doc = include_str!("../docs/04_federating_posts.md")]
+#![doc = include_str!("../docs/05_configuration.md")]
+#![doc = include_str!("../docs/06_http_endpoints_axum.md")]
+#![doc = include_str!("../docs/07_fetching_data.md")]
+#![doc = include_str!("../docs/08_receiving_activities.md")]
+#![doc = include_str!("../docs/09_sending_activities.md")]
+#![doc = include_str!("../docs/10_fetching_objects_with_unknown_type.md")]
+#![deny(missing_docs)]
+
+pub mod activity_queue;
+pub mod activity_sending;
+#[cfg(feature = "actix-web")]
+pub mod actix_web;
+#[cfg(feature = "axum")]
+pub mod axum;
+pub mod config;
+pub mod error;
+pub mod fetch;
+pub mod http_signatures;
+pub mod protocol;
+pub(crate) mod reqwest_shim;
+pub mod traits;
+
+use crate::{
+    config::Data,
+    error::Error,
+    fetch::object_id::ObjectId,
+    traits::{ActivityHandler, Actor, Object},
+};
+pub use activitystreams_kinds as kinds;
+
+use serde::{de::DeserializeOwned, Deserialize};
 use url::Url;
 
-pub mod core;
-pub mod data;
-pub mod deser;
-pub mod traits;
-pub mod utils;
+/// Mime type for Activitypub data, used for `Accept` and `Content-Type` HTTP headers
+pub const FEDERATION_CONTENT_TYPE: &str = "application/activity+json";
 
-/// Mime type for Activitypub, used for `Accept` and `Content-Type` HTTP headers
-pub static APUB_JSON_CONTENT_TYPE: &str = "application/activity+json";
-
-/// Represents a single, federated instance (for example lemmy.ml). There should only be one of
-/// this in your application (except for testing).
-pub struct LocalInstance {
-    hostname: String,
-    client: ClientWithMiddleware,
-    activity_queue: Manager,
-    settings: InstanceSettings,
+/// Deserialize incoming inbox activity to the given type, perform basic
+/// validation and extract the actor.
+async fn parse_received_activity<Activity, ActorT, Datatype>(
+    body: &[u8],
+    data: &Data<Datatype>,
+) -> Result<(Activity, ActorT), <Activity as ActivityHandler>::Error>
+where
+    Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + 'static,
+    for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
+    <Activity as ActivityHandler>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
+    Datatype: Clone,
+{
+    let activity: Activity = serde_json::from_slice(body).map_err(|e| {
+        // Attempt to include activity id in error message
+        let id = extract_id(body).ok();
+        Error::ParseReceivedActivity(e, id)
+    })?;
+    data.config.verify_url_and_domain(&activity).await?;
+    let actor = ObjectId::<ActorT>::from(activity.actor().clone())
+        .dereference(data)
+        .await?;
+    Ok((activity, actor))
 }
 
-// Use InstanceSettingsBuilder to initialize this
-#[derive(Builder)]
-pub struct InstanceSettings {
-    /// Maximum number of outgoing HTTP requests per incoming activity
-    #[builder(default = "20")]
-    http_fetch_retry_limit: i32,
-    /// Number of worker threads for sending outgoing activities
-    #[builder(default = "64")]
-    worker_count: u64,
-    /// Run library in debug mode. This allows usage of http and localhost urls. It also sends
-    /// outgoing activities synchronously, not in background thread. This helps to make tests
-    /// more consistent.
-    /// Do not use for production.
-    #[builder(default = "false")]
-    debug: bool,
-    /// Timeout for all HTTP requests. HTTP signatures are valid for 10s, so it makes sense to
-    /// use the same as timeout when sending
-    #[builder(default = "Duration::from_secs(10)")]
-    request_timeout: Duration,
-    /// Function used to verify that urls are valid, used when receiving activities or fetching remote
-    /// objects. Use this to implement functionality like federation blocklists. In case verification
-    /// fails, it should return an error message.
-    #[builder(default = "|_| { Ok(()) }")]
-    verify_url_function: fn(&Url) -> Result<(), &'static str>,
-}
-
-impl LocalInstance {
-    pub fn new(domain: String, client: ClientWithMiddleware, settings: InstanceSettings) -> Self {
-        let activity_queue = create_activity_queue(client.clone(), &settings);
-        LocalInstance {
-            hostname: domain,
-            client,
-            activity_queue,
-            settings,
-        }
-    }
-    /// Returns true if the url refers to this instance. Handles hostnames like `localhost:8540` for
-    /// local debugging.
-    fn is_local_url(&self, url: &Url) -> bool {
-        let mut domain = url.domain().expect("id has domain").to_string();
-        if let Some(port) = url.port() {
-            domain = format!("{}:{}", domain, port);
-        }
-        domain == self.hostname
-    }
-
-    /// Returns the local hostname
-    pub fn hostname(&self) -> &str {
-        &self.hostname
-    }
-}
-
-#[derive(thiserror::Error, Debug)]
-pub enum Error {
-    #[error("Object was not found in database")]
-    NotFound,
-    #[error("Request limit was reached during fetch")]
-    RequestLimit,
-    #[error("Object to be fetched was deleted")]
-    ObjectDeleted,
-    #[error("{0}")]
-    UrlVerificationError(&'static str),
-    #[error(transparent)]
-    Other(#[from] anyhow::Error),
-}
-
-impl Error {
-    pub fn conv<T>(error: T) -> Self
-    where
-        T: Into<anyhow::Error>,
-    {
-        Error::Other(error.into())
+/// Attempt to parse id field from serialized json
+fn extract_id(data: &[u8]) -> serde_json::Result<Url> {
+    #[derive(Deserialize)]
+    struct Id {
+        id: Url,
     }
+    Ok(serde_json::from_slice::<Id>(data)?.id)
 }

src/main.rs

@@ -1,3 +0,0 @@
-fn main() {
-    println!("Hello, world!");
-}

src/protocol/context.rs

@@ -0,0 +1,92 @@
+//! Wrapper for federated structs which handles `@context` field.
+//!
+//! This wrapper can be used when sending Activitypub data, to automatically add `@context`. It
+//! avoids having to repeat the `@context` property on every struct, and getting multiple contexts
+//! in nested structs.
+//!
+//! ```
+//! # use activitypub_federation::protocol::context::WithContext;
+//! #[derive(serde::Serialize)]
+//! struct Note {
+//!     content: String
+//! }
+//! let note = Note {
+//!     content: "Hello world".to_string()
+//! };
+//! let note_with_context = WithContext::new_default(note);
+//! let serialized = serde_json::to_string(&note_with_context)?;
+//! assert_eq!(serialized, r#"{"@context":"https://www.w3.org/ns/activitystreams","content":"Hello world"}"#);
+//! Ok::<(), serde_json::error::Error>(())
+//! ```
+
+use crate::{config::Data, traits::ActivityHandler};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use url::Url;
+
+/// Default context used in Activitypub
+const DEFAULT_CONTEXT: &str = "https://www.w3.org/ns/activitystreams";
+
+/// Wrapper for federated structs which handles `@context` field.
+#[derive(Serialize, Deserialize, Debug)]
+pub struct WithContext<T> {
+    #[serde(rename = "@context")]
+    context: Value,
+    #[serde(flatten)]
+    inner: T,
+}
+
+impl<T> WithContext<T> {
+    /// Create a new wrapper with the default Activitypub context.
+    pub fn new_default(inner: T) -> WithContext<T> {
+        let context = Value::String(DEFAULT_CONTEXT.to_string());
+        WithContext::new(inner, context)
+    }
+
+    /// Create new wrapper with custom context. Use this in case you are implementing extensions.
+    pub fn new(inner: T, context: Value) -> WithContext<T> {
+        WithContext { context, inner }
+    }
+
+    /// Returns the inner `T` object which this `WithContext` object is wrapping
+    pub fn inner(&self) -> &T {
+        &self.inner
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> ActivityHandler for WithContext<T>
+where
+    T: ActivityHandler + Send + Sync,
+{
+    type DataType = <T as ActivityHandler>::DataType;
+    type Error = <T as ActivityHandler>::Error;
+
+    fn id(&self) -> &Url {
+        self.inner.id()
+    }
+
+    fn actor(&self) -> &Url {
+        self.inner.actor()
+    }
+
+    async fn verify(&self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        self.inner.verify(data).await
+    }
+
+    async fn receive(self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        self.inner.receive(data).await
+    }
+}
+
+impl<T> Clone for WithContext<T>
+where
+    T: Clone,
+{
+    fn clone(&self) -> Self {
+        Self {
+            context: self.context.clone(),
+            inner: self.inner.clone(),
+        }
+    }
+}

src/protocol/helpers.rs

@@ -0,0 +1,136 @@
+//! Serde deserialization functions which help to receive differently shaped data
+
+use serde::{Deserialize, Deserializer};
+
+/// Deserialize JSON single value or array into Vec.
+///
+/// Useful if your application can handle multiple values for a field, but another federated
+/// platform only sends a single one.
+///
+/// ```
+/// # use activitypub_federation::protocol::helpers::deserialize_one_or_many;
+/// # use url::Url;
+/// #[derive(serde::Deserialize)]
+/// struct Note {
+///     #[serde(deserialize_with = "deserialize_one_or_many")]
+///     to: Vec<Url>
+/// }
+///
+/// let single: Note = serde_json::from_str(r#"{"to": "https://example.com/u/alice" }"#)?;
+/// assert_eq!(single.to.len(), 1);
+///
+/// let multiple: Note = serde_json::from_str(
+/// r#"{"to": [
+///      "https://example.com/u/alice",
+///      "https://lemmy.ml/u/bob"
+/// ]}"#)?;
+/// assert_eq!(multiple.to.len(), 2);
+/// Ok::<(), anyhow::Error>(())
+pub fn deserialize_one_or_many<'de, T, D>(deserializer: D) -> Result<Vec<T>, D::Error>
+where
+    T: Deserialize<'de>,
+    D: Deserializer<'de>,
+{
+    #[derive(Deserialize)]
+    #[serde(untagged)]
+    enum OneOrMany<T> {
+        One(T),
+        Many(Vec<T>),
+    }
+
+    let result: OneOrMany<T> = Deserialize::deserialize(deserializer)?;
+    Ok(match result {
+        OneOrMany::Many(list) => list,
+        OneOrMany::One(value) => vec![value],
+    })
+}
+
+/// Deserialize JSON single value or single element array into single value.
+///
+/// Useful if your application can only handle a single value for a field, but another federated
+/// platform sends single value wrapped in array. Fails if array contains multiple items.
+///
+/// ```
+/// # use activitypub_federation::protocol::helpers::deserialize_one;
+/// # use url::Url;
+/// #[derive(serde::Deserialize)]
+/// struct Note {
+///     #[serde(deserialize_with = "deserialize_one")]
+///     to: [Url; 1]
+/// }
+///
+/// let note = serde_json::from_str::<Note>(r#"{"to": ["https://example.com/u/alice"] }"#);
+/// assert!(note.is_ok());
+pub fn deserialize_one<'de, T, D>(deserializer: D) -> Result<[T; 1], D::Error>
+where
+    T: Deserialize<'de>,
+    D: Deserializer<'de>,
+{
+    #[derive(Deserialize)]
+    #[serde(untagged)]
+    enum MaybeArray<T> {
+        Simple(T),
+        Array([T; 1]),
+    }
+
+    let result: MaybeArray<T> = Deserialize::deserialize(deserializer)?;
+    Ok(match result {
+        MaybeArray::Simple(value) => [value],
+        MaybeArray::Array([value]) => [value],
+    })
+}
+
+/// Attempts to deserialize item, in case of error falls back to the type's default value.
+///
+/// Useful for optional fields which are sent with a different type from another platform,
+/// eg object instead of array. Should always be used together with `#[serde(default)]`, so
+/// that a mssing value doesn't cause an error.
+///
+/// ```
+/// # use activitypub_federation::protocol::helpers::deserialize_skip_error;
+/// # use url::Url;
+/// #[derive(serde::Deserialize)]
+/// struct Note {
+///     content: String,
+///     #[serde(deserialize_with = "deserialize_skip_error", default)]
+///     source: Option<String>
+/// }
+///
+/// let note = serde_json::from_str::<Note>(
+/// r#"{
+///     "content": "How are you?",
+///     "source": {
+///         "content": "How are you?",
+///         "mediaType": "text/markdown"
+///     }
+/// }"#);
+/// assert_eq!(note.unwrap().source, None);
+/// # Ok::<(), anyhow::Error>(())
+pub fn deserialize_skip_error<'de, T, D>(deserializer: D) -> Result<T, D::Error>
+where
+    T: Deserialize<'de> + Default,
+    D: Deserializer<'de>,
+{
+    let value = serde_json::Value::deserialize(deserializer)?;
+    let inner = T::deserialize(value).unwrap_or_default();
+    Ok(inner)
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn deserialize_one_multiple_values() {
+        use crate::protocol::helpers::deserialize_one;
+        use url::Url;
+        #[derive(serde::Deserialize)]
+        struct Note {
+            #[serde(deserialize_with = "deserialize_one")]
+            _to: [Url; 1],
+        }
+
+        let note = serde_json::from_str::<Note>(
+            r#"{"_to": ["https://example.com/u/alice", "https://example.com/u/bob"] }"#,
+        );
+        assert!(note.is_err());
+    }
+}

src/protocol/mod.rs

@@ -0,0 +1,7 @@
+//! Data structures which help to define federated messages
+
+pub mod context;
+pub mod helpers;
+pub mod public_key;
+pub mod values;
+pub mod verification;

src/protocol/public_key.rs

@@ -0,0 +1,36 @@
+//! Struct which is used to federate actor key for HTTP signatures
+
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+/// Public key of actors which is used for HTTP signatures.
+///
+/// This needs to be federated in the `public_key` field of all actors.
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct PublicKey {
+    /// Id of this private key.
+    pub id: String,
+    /// ID of the actor that this public key belongs to
+    pub owner: Url,
+    /// The actual public key in PEM format
+    pub public_key_pem: String,
+}
+
+impl PublicKey {
+    /// Create a new [PublicKey] struct for the `owner` with `public_key_pem`.
+    ///
+    /// It uses an standard key id of `{actor_id}#main-key`
+    pub(crate) fn new(owner: Url, public_key_pem: String) -> Self {
+        let id = main_key_id(&owner);
+        PublicKey {
+            id,
+            owner,
+            public_key_pem,
+        }
+    }
+}
+
+pub(crate) fn main_key_id(owner: &Url) -> String {
+    format!("{}#main-key", &owner)
+}

src/protocol/values.rs

@@ -1,20 +1,16 @@
+//! Single value enums used to receive JSON with specific expected string values
+//!
 //! The enums here serve to limit a json string value to a single, hardcoded value which can be
 //! verified at compilation time. When using it as the type of a struct field, the struct can only
 //! be constructed or deserialized if the field has the exact same value.
 //!
-//! If we used String as the field type, any value would be accepted, and we would have to check
-//! manually at runtime that it contains the expected value.
-//!
-//! The enums in `activitystreams::activity::kind` work in the same way, and can be used to
-//! distinguish different activity types.
-//!
 //! In the example below, `MyObject` can only be constructed or
 //! deserialized if `media_type` is `text/markdown`, but not if it is `text/html`.
 //!
 //! ```
 //! use serde_json::from_str;
 //! use serde::{Deserialize, Serialize};
-//! use activitypub_federation::deser::values::MediaTypeMarkdown;
+//! use activitypub_federation::protocol::values::MediaTypeMarkdown;
 //!
 //! #[derive(Deserialize, Serialize)]
 //! struct MyObject {
@@ -30,14 +26,18 @@
 //! let from_html = from_str::<MyObject>(markdown_html);
 //! assert!(from_html.is_err());
 //! ```
+//!
+//! The enums in [activitystreams_kinds] work in the same way, and can be used to
+//! distinguish different activity types.
 
 use serde::{Deserialize, Serialize};
 
 /// Media type for markdown text.
 ///
 /// <https://www.iana.org/assignments/media-types/media-types.xhtml>
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
 pub enum MediaTypeMarkdown {
+    /// `text/markdown`
     #[serde(rename = "text/markdown")]
     Markdown,
 }
@@ -45,17 +45,20 @@ pub enum MediaTypeMarkdown {
 /// Media type for HTML text.
 ///
 /// <https://www.iana.org/assignments/media-types/media-types.xhtml>
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
 pub enum MediaTypeHtml {
+    /// `text/html`
     #[serde(rename = "text/html")]
     Html,
 }
 
 /// Media type which allows both markdown and HTML.
-#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
 pub enum MediaTypeMarkdownOrHtml {
+    /// `text/markdown`
     #[serde(rename = "text/markdown")]
     Markdown,
+    /// `text/html`
     #[serde(rename = "text/html")]
     Html,
 }

src/protocol/verification.rs

@@ -0,0 +1,38 @@
+//! Verify that received data is valid
+
+use crate::error::Error;
+use url::Url;
+
+/// Check that both urls have the same domain. If not, return UrlVerificationError.
+///
+/// ```
+/// # use url::Url;
+/// # use activitypub_federation::protocol::verification::verify_domains_match;
+/// let a = Url::parse("https://example.com/abc")?;
+/// let b = Url::parse("https://sample.net/abc")?;
+/// assert!(verify_domains_match(&a, &b).is_err());
+/// # Ok::<(), url::ParseError>(())
+/// ```
+pub fn verify_domains_match(a: &Url, b: &Url) -> Result<(), Error> {
+    if a.domain() != b.domain() {
+        return Err(Error::UrlVerificationError("Domains do not match"));
+    }
+    Ok(())
+}
+
+/// Check that both urls are identical. If not, return UrlVerificationError.
+///
+/// ```
+/// # use url::Url;
+/// # use activitypub_federation::protocol::verification::verify_urls_match;
+/// let a = Url::parse("https://example.com/abc")?;
+/// let b = Url::parse("https://example.com/123")?;
+/// assert!(verify_urls_match(&a, &b).is_err());
+/// # Ok::<(), url::ParseError>(())
+/// ```
+pub fn verify_urls_match(a: &Url, b: &Url) -> Result<(), Error> {
+    if a != b {
+        return Err(Error::UrlVerificationError("Urls do not match"));
+    }
+    Ok(())
+}

src/reqwest_shim.rs

@@ -0,0 +1,99 @@
+use crate::error::Error;
+use bytes::{BufMut, Bytes, BytesMut};
+use futures_core::{ready, stream::BoxStream, Stream};
+use pin_project_lite::pin_project;
+use reqwest::Response;
+use std::{
+    future::Future,
+    mem,
+    pin::Pin,
+    task::{Context, Poll},
+};
+
+/// 200KB
+const MAX_BODY_SIZE: usize = 204800;
+
+pin_project! {
+    pub struct BytesFuture {
+        #[pin]
+        stream: BoxStream<'static, reqwest::Result<Bytes>>,
+        limit: usize,
+        aggregator: BytesMut,
+    }
+}
+
+impl Future for BytesFuture {
+    type Output = Result<Bytes, Error>;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        loop {
+            let this = self.as_mut().project();
+            if let Some(chunk) = ready!(this.stream.poll_next(cx)).transpose()? {
+                this.aggregator.put(chunk);
+                if this.aggregator.len() > *this.limit {
+                    return Poll::Ready(Err(Error::ResponseBodyLimit));
+                }
+
+                continue;
+            }
+
+            break;
+        }
+
+        Poll::Ready(Ok(mem::take(&mut self.aggregator).freeze()))
+    }
+}
+
+pin_project! {
+    pub struct TextFuture {
+        #[pin]
+        future: BytesFuture,
+    }
+}
+
+impl Future for TextFuture {
+    type Output = Result<String, Error>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let this = self.project();
+        let bytes = ready!(this.future.poll(cx))?;
+        Poll::Ready(String::from_utf8(bytes.to_vec()).map_err(Error::Utf8))
+    }
+}
+
+/// Response shim to work around [an issue in reqwest](https://github.com/seanmonstar/reqwest/issues/1234) (there is an [open pull request](https://github.com/seanmonstar/reqwest/pull/1532) fixing this).
+///
+/// Reqwest doesn't limit the response body size by default nor does it offer an option to configure one.
+/// Since we have to fetch data from untrusted sources, not restricting the maximum size is a DoS hazard for us.
+///
+/// This shim reimplements the `bytes`, `json`, and `text` functions and restricts the bodies to 100KB.
+///
+/// TODO: Remove this shim as soon as reqwest gets support for size-limited bodies.
+pub trait ResponseExt {
+    type BytesFuture;
+    type TextFuture;
+
+    /// Size limited version of `bytes` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
+    fn bytes_limited(self) -> Self::BytesFuture;
+    /// Size limited version of `text` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
+    fn text_limited(self) -> Self::TextFuture;
+}
+
+impl ResponseExt for Response {
+    type BytesFuture = BytesFuture;
+    type TextFuture = TextFuture;
+
+    fn bytes_limited(self) -> Self::BytesFuture {
+        BytesFuture {
+            stream: Box::pin(self.bytes_stream()),
+            limit: MAX_BODY_SIZE,
+            aggregator: BytesMut::new(),
+        }
+    }
+
+    fn text_limited(self) -> Self::TextFuture {
+        TextFuture {
+            future: self.bytes_limited(),
+        }
+    }
+}

src/traits.rs

@@ -1,12 +1,213 @@
-use crate::data::Data;
-pub use activitypub_federation_derive::*;
-use chrono::NaiveDateTime;
+//! Traits which need to be implemented for federated data types
+
+use crate::{config::Data, protocol::public_key::PublicKey};
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use serde::Deserialize;
+use std::{fmt::Debug, ops::Deref};
 use url::Url;
 
-/// Trait which allows verification and reception of incoming activities.
-#[async_trait::async_trait(?Send)]
+/// Helper for converting between database structs and federated protocol structs.
+///
+/// ```
+/// # use activitystreams_kinds::{object::NoteType, public};
+/// # use chrono::{Local, DateTime, Utc};
+/// # use serde::{Deserialize, Serialize};
+/// # use url::Url;
+/// # use activitypub_federation::protocol::{public_key::PublicKey, helpers::deserialize_one_or_many};
+/// # use activitypub_federation::config::Data;
+/// # use activitypub_federation::fetch::object_id::ObjectId;
+/// # use activitypub_federation::protocol::verification::verify_domains_match;
+/// # use activitypub_federation::traits::{Actor, Object};
+/// # use activitypub_federation::traits::tests::{DbConnection, DbUser};
+/// #
+/// /// How the post is read/written in the local database
+/// #[derive(Debug)]
+/// pub struct DbPost {
+///     pub text: String,
+///     pub ap_id: ObjectId<DbPost>,
+///     pub creator: ObjectId<DbUser>,
+///     pub local: bool,
+/// }
+///
+/// /// How the post is serialized and represented as Activitypub JSON
+/// #[derive(Deserialize, Serialize, Debug)]
+/// #[serde(rename_all = "camelCase")]
+/// pub struct Note {
+///     #[serde(rename = "type")]
+///     kind: NoteType,
+///     id: ObjectId<DbPost>,
+///     pub(crate) attributed_to: ObjectId<DbUser>,
+///     #[serde(deserialize_with = "deserialize_one_or_many")]
+///     pub(crate) to: Vec<Url>,
+///     content: String,
+/// }
+///
+/// #[async_trait::async_trait]
+/// impl Object for DbPost {
+///     type DataType = DbConnection;
+///     type Kind = Note;
+///     type Error = anyhow::Error;
+///
+/// async fn read_from_id(object_id: Url, data: &Data<Self::DataType>) -> Result<Option<Self>, Self::Error> {
+///         // Attempt to read object from local database. Return Ok(None) if not found.
+///         let post: Option<DbPost> = data.read_post_from_json_id(object_id).await?;
+///         Ok(post)
+///     }
+///
+/// async fn into_json(self, data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+///         // Called when a local object gets sent out over Activitypub. Simply convert it to the
+///         // protocol struct
+///         Ok(Note {
+///             kind: Default::default(),
+///             id: self.ap_id.clone().into(),
+///             attributed_to: self.creator,
+///             to: vec![public()],
+///             content: self.text,
+///         })
+///     }
+///
+///     async fn verify(json: &Self::Kind, expected_domain: &Url, data: &Data<Self::DataType>,) -> Result<(), Self::Error> {
+///         verify_domains_match(json.id.inner(), expected_domain)?;
+///         // additional application specific checks
+///         Ok(())
+///     }
+///
+///     async fn from_json(json: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, Self::Error> {
+///         // Called when a remote object gets received over Activitypub. Validate and insert it
+///         // into the database.
+///
+///         let post = DbPost {
+///             text: json.content,
+///             ap_id: json.id,
+///             creator: json.attributed_to,
+///             local: false,
+///         };
+///
+///         // Here we need to persist the object in the local database. Note that Activitypub
+///         // doesnt distinguish between creating and updating an object. Thats why we need to
+///         // use upsert functionality.
+///         data.upsert(&post).await?;
+///
+///         Ok(post)
+///     }
+///
+/// }
+#[async_trait]
+pub trait Object: Sized + Debug {
+    /// App data type passed to handlers. Must be identical to
+    /// [crate::config::FederationConfigBuilder::app_data] type.
+    type DataType: Clone + Send + Sync;
+    /// The type of protocol struct which gets sent over network to federate this database struct.
+    type Kind;
+    /// Error type returned by handler methods
+    type Error;
+
+    /// Returns the last time this object was updated.
+    ///
+    /// If this returns `Some` and the value is too long ago, the object is refetched from the
+    /// original instance. This should always be implemented for actors, because there is no active
+    /// update mechanism prescribed. It is possible to send `Update/Person` activities for profile
+    /// changes, but not all implementations do this, so `last_refreshed_at` is still necessary.
+    ///
+    /// The object is refetched if `last_refreshed_at` value is more than 24 hours ago. In debug
+    /// mode this is reduced to 20 seconds.
+    fn last_refreshed_at(&self) -> Option<DateTime<Utc>> {
+        None
+    }
+
+    /// Try to read the object with given `id` from local database.
+    ///
+    /// Should return `Ok(None)` if not found.
+    async fn read_from_id(
+        object_id: Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<Option<Self>, Self::Error>;
+
+    /// Mark remote object as deleted in local database.
+    ///
+    /// Called when a `Delete` activity is received, or if fetch returns a `Tombstone` object.
+    async fn delete(self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        Ok(())
+    }
+
+    /// Convert database type to Activitypub type.
+    ///
+    /// Called when a local object gets fetched by another instance over HTTP, or when an object
+    /// gets sent in an activity.
+    async fn into_json(self, data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error>;
+
+    /// Verifies that the received object is valid.
+    ///
+    /// You should check here that the domain of id matches `expected_domain`. Additionally you
+    /// should perform any application specific checks.
+    ///
+    /// It is necessary to use a separate method for this, because it might be used for activities
+    /// like `Delete/Note`, which shouldn't perform any database write for the inner `Note`.
+    async fn verify(
+        json: &Self::Kind,
+        expected_domain: &Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<(), Self::Error>;
+
+    /// Convert object from ActivityPub type to database type.
+    ///
+    /// Called when an object is received from HTTP fetch or as part of an activity. This method
+    /// should write the received object to database. Note that there is no distinction between
+    /// create and update, so an `upsert` operation should be used.
+    async fn from_json(json: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, Self::Error>;
+}
+
+/// Handler for receiving incoming activities.
+///
+/// ```
+/// # use activitystreams_kinds::activity::FollowType;
+/// # use url::Url;
+/// # use activitypub_federation::fetch::object_id::ObjectId;
+/// # use activitypub_federation::config::Data;
+/// # use activitypub_federation::traits::ActivityHandler;
+/// # use activitypub_federation::traits::tests::{DbConnection, DbUser};
+/// #[derive(serde::Deserialize)]
+/// struct Follow {
+///     actor: ObjectId<DbUser>,
+///     object: ObjectId<DbUser>,
+///     #[serde(rename = "type")]
+///     kind: FollowType,
+///     id: Url,
+/// }
+///
+/// #[async_trait::async_trait]
+/// impl ActivityHandler for Follow {
+///     type DataType = DbConnection;
+///     type Error = anyhow::Error;
+///
+///     fn id(&self) -> &Url {
+///         &self.id
+///     }
+///
+///     fn actor(&self) -> &Url {
+///         self.actor.inner()
+///     }
+///
+///     async fn verify(&self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+///         Ok(())
+///     }
+///
+///     async fn receive(self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+///         let local_user = self.object.dereference(data).await?;
+///         let follower = self.actor.dereference(data).await?;
+///         data.add_follower(local_user, follower).await?;
+///         Ok(())
+///     }
+/// }
+/// ```
+#[async_trait]
+#[enum_delegate::register]
 pub trait ActivityHandler {
-    type DataType;
+    /// App data type passed to handlers. Must be identical to
+    /// [crate::config::FederationConfigBuilder::app_data] type.
+    type DataType: Clone + Send + Sync;
+    /// Error type returned by handler methods
     type Error;
 
     /// `id` field of the activity
@@ -15,94 +216,322 @@ pub trait ActivityHandler {
     /// `actor` field of activity
     fn actor(&self) -> &Url;
 
-    /// Verify that the activity is valid. If this method returns an error, the activity will be
-    /// discarded. This is separate from receive(), so that it can be called recursively on nested
-    /// objects, without storing something in the database by accident.
-    async fn verify(
-        &self,
-        data: &Data<Self::DataType>,
-        request_counter: &mut i32,
-    ) -> Result<(), Self::Error>;
-
-    /// Receives the activity and stores its action in database.
-    async fn receive(
-        self,
-        data: &Data<Self::DataType>,
-        request_counter: &mut i32,
-    ) -> Result<(), Self::Error>;
-}
-
-#[async_trait::async_trait(?Send)]
-pub trait ApubObject {
-    type DataType;
-    type ApubType;
-    type DbType;
-    type Error;
-
-    /// If the object is stored in the database, this method should return the fetch time. Used to
-    /// update actors after certain interval.
-    fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
-        None
-    }
-
-    /// Try to read the object with given ID from local database. Returns Ok(None) if it doesn't exist.
-    async fn read_from_apub_id(
-        object_id: Url,
-        data: &Self::DataType,
-    ) -> Result<Option<Self>, Self::Error>
-    where
-        Self: Sized;
-
-    /// Marks the object as deleted in local db. Called when a delete activity is received, or if
-    /// fetch returns a tombstone.
-    async fn delete(self, _data: &Self::DataType) -> Result<(), Self::Error>
-    where
-        Self: Sized,
-    {
-        Ok(())
-    }
-
-    /// Trait for converting an object or actor into the respective ActivityPub type.
-    async fn into_apub(self, data: &Self::DataType) -> Result<Self::ApubType, Self::Error>;
-
-    /// Verify that the object is valid. If this method returns an error, it will be
-    /// discarded. This is separate from from_apub(), so that it can be called recursively on nested
-    /// objects, without storing something in the database by accident.
-    async fn verify(
-        apub: &Self::ApubType,
-        expected_domain: &Url,
-        data: &Self::DataType,
-        request_counter: &mut i32,
-    ) -> Result<(), Self::Error>;
-
-    /// Converts an object from ActivityPub type to Lemmy internal type.
+    /// Verifies that the received activity is valid.
     ///
-    /// * `apub` The object to read from
-    /// * `context` LemmyContext which holds DB pool, HTTP client etc
-    /// * `expected_domain` Domain where the object was received from. None in case of mod action.
-    /// * `mod_action_allowed` True if the object can be a mod activity, ignore `expected_domain` in this case
-    async fn from_apub(
-        apub: Self::ApubType,
-        data: &Self::DataType,
-        request_counter: &mut i32,
-    ) -> Result<Self, Self::Error>
-    where
-        Self: Sized;
+    /// This needs to be a separate method, because it might be used for activities
+    /// like `Undo/Follow`, which shouldn't perform any database write for the inner `Follow`.
+    async fn verify(&self, data: &Data<Self::DataType>) -> Result<(), Self::Error>;
+
+    /// Called when an activity is received.
+    ///
+    /// Should perform validation and possibly write action to the database. In case the activity
+    /// has a nested `object` field, must call `object.from_json` handler.
+    async fn receive(self, data: &Data<Self::DataType>) -> Result<(), Self::Error>;
 }
 
-pub trait Actor: ApubObject {
-    /// Returns the actor's public key for verification of HTTP signatures
-    fn public_key(&self) -> &str;
+/// Trait to allow retrieving common Actor data.
+pub trait Actor: Object + Send + 'static {
+    /// `id` field of the actor
+    fn id(&self) -> Url;
+
+    /// The actor's public key for verifying signatures of incoming activities.
+    ///
+    /// Use [generate_actor_keypair](crate::http_signatures::generate_actor_keypair) to create the
+    /// actor keypair.
+    fn public_key_pem(&self) -> &str;
+
+    /// The actor's private key for signing outgoing activities.
+    ///
+    /// Use [generate_actor_keypair](crate::http_signatures::generate_actor_keypair) to create the
+    /// actor keypair.
+    fn private_key_pem(&self) -> Option<String>;
 
     /// The inbox where activities for this user should be sent to
     fn inbox(&self) -> Url;
 
+    /// Generates a public key struct for use in the actor json representation
+    fn public_key(&self) -> PublicKey {
+        PublicKey::new(self.id(), self.public_key_pem().to_string())
+    }
+
     /// The actor's shared inbox, if any
     fn shared_inbox(&self) -> Option<Url> {
         None
     }
 
+    /// Returns shared inbox if it exists, normal inbox otherwise.
     fn shared_inbox_or_inbox(&self) -> Url {
         self.shared_inbox().unwrap_or_else(|| self.inbox())
     }
 }
+
+/// Allow for boxing of enum variants
+#[async_trait]
+impl<T> ActivityHandler for Box<T>
+where
+    T: ActivityHandler + Send + Sync,
+{
+    type DataType = T::DataType;
+    type Error = T::Error;
+
+    fn id(&self) -> &Url {
+        self.deref().id()
+    }
+
+    fn actor(&self) -> &Url {
+        self.deref().actor()
+    }
+
+    async fn verify(&self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        self.deref().verify(data).await
+    }
+
+    async fn receive(self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        (*self).receive(data).await
+    }
+}
+
+/// Trait for federating collections
+#[async_trait]
+pub trait Collection: Sized {
+    /// Actor or object that this collection belongs to
+    type Owner;
+    /// App data type passed to handlers. Must be identical to
+    /// [crate::config::FederationConfigBuilder::app_data] type.
+    type DataType: Clone + Send + Sync;
+    /// The type of protocol struct which gets sent over network to federate this database struct.
+    type Kind: for<'de2> Deserialize<'de2>;
+    /// Error type returned by handler methods
+    type Error;
+
+    /// Reads local collection from database and returns it as Activitypub JSON.
+    async fn read_local(
+        owner: &Self::Owner,
+        data: &Data<Self::DataType>,
+    ) -> Result<Self::Kind, Self::Error>;
+
+    /// Verifies that the received object is valid.
+    ///
+    /// You should check here that the domain of id matches `expected_domain`. Additionally you
+    /// should perform any application specific checks.
+    async fn verify(
+        json: &Self::Kind,
+        expected_domain: &Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<(), Self::Error>;
+
+    /// Convert object from ActivityPub type to database type.
+    ///
+    /// Called when an object is received from HTTP fetch or as part of an activity. This method
+    /// should also write the received object to database. Note that there is no distinction
+    /// between create and update, so an `upsert` operation should be used.
+    async fn from_json(
+        json: Self::Kind,
+        owner: &Self::Owner,
+        data: &Data<Self::DataType>,
+    ) -> Result<Self, Self::Error>;
+}
+
+/// Some impls of these traits for use in tests. Dont use this from external crates.
+///
+/// TODO: Should be using `cfg[doctest]` but blocked by <https://github.com/rust-lang/rust/issues/67295>
+#[doc(hidden)]
+#[allow(clippy::unwrap_used)]
+pub mod tests {
+    use super::*;
+    use crate::{
+        error::Error,
+        fetch::object_id::ObjectId,
+        http_signatures::{generate_actor_keypair, Keypair},
+        protocol::verification::verify_domains_match,
+    };
+    use activitystreams_kinds::{activity::FollowType, actor::PersonType};
+    use once_cell::sync::Lazy;
+    use serde::{Deserialize, Serialize};
+
+    #[derive(Clone)]
+    pub struct DbConnection;
+
+    impl DbConnection {
+        pub async fn read_post_from_json_id<T>(&self, _: Url) -> Result<Option<T>, Error> {
+            Ok(None)
+        }
+        pub async fn read_local_user(&self, _: &str) -> Result<DbUser, Error> {
+            todo!()
+        }
+        pub async fn upsert<T>(&self, _: &T) -> Result<(), Error> {
+            Ok(())
+        }
+        pub async fn add_follower(&self, _: DbUser, _: DbUser) -> Result<(), Error> {
+            Ok(())
+        }
+    }
+
+    #[derive(Clone, Debug, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Person {
+        #[serde(rename = "type")]
+        pub kind: PersonType,
+        pub preferred_username: String,
+        pub id: ObjectId<DbUser>,
+        pub inbox: Url,
+        pub public_key: PublicKey,
+    }
+    #[derive(Debug, Clone)]
+    pub struct DbUser {
+        pub name: String,
+        pub federation_id: Url,
+        pub inbox: Url,
+        pub public_key: String,
+        #[allow(dead_code)]
+        private_key: Option<String>,
+        pub followers: Vec<Url>,
+        pub local: bool,
+    }
+
+    pub static DB_USER_KEYPAIR: Lazy<Keypair> = Lazy::new(|| generate_actor_keypair().unwrap());
+
+    pub static DB_USER: Lazy<DbUser> = Lazy::new(|| DbUser {
+        name: String::new(),
+        federation_id: "https://localhost/123".parse().unwrap(),
+        inbox: "https://localhost/123/inbox".parse().unwrap(),
+        public_key: DB_USER_KEYPAIR.public_key.clone(),
+        private_key: Some(DB_USER_KEYPAIR.private_key.clone()),
+        followers: vec![],
+        local: false,
+    });
+
+    #[async_trait]
+    impl Object for DbUser {
+        type DataType = DbConnection;
+        type Kind = Person;
+        type Error = Error;
+
+        async fn read_from_id(
+            _object_id: Url,
+            _data: &Data<Self::DataType>,
+        ) -> Result<Option<Self>, Self::Error> {
+            Ok(Some(DB_USER.clone()))
+        }
+
+        async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+            Ok(Person {
+                preferred_username: self.name.clone(),
+                kind: Default::default(),
+                id: self.federation_id.clone().into(),
+                inbox: self.inbox.clone(),
+                public_key: self.public_key(),
+            })
+        }
+
+        async fn verify(
+            json: &Self::Kind,
+            expected_domain: &Url,
+            _data: &Data<Self::DataType>,
+        ) -> Result<(), Self::Error> {
+            verify_domains_match(json.id.inner(), expected_domain)?;
+            Ok(())
+        }
+
+        async fn from_json(
+            json: Self::Kind,
+            _data: &Data<Self::DataType>,
+        ) -> Result<Self, Self::Error> {
+            Ok(DbUser {
+                name: json.preferred_username,
+                federation_id: json.id.into(),
+                inbox: json.inbox,
+                public_key: json.public_key.public_key_pem,
+                private_key: None,
+                followers: vec![],
+                local: false,
+            })
+        }
+    }
+
+    impl Actor for DbUser {
+        fn id(&self) -> Url {
+            self.federation_id.clone()
+        }
+
+        fn public_key_pem(&self) -> &str {
+            &self.public_key
+        }
+
+        fn private_key_pem(&self) -> Option<String> {
+            self.private_key.clone()
+        }
+
+        fn inbox(&self) -> Url {
+            self.inbox.clone()
+        }
+    }
+
+    #[derive(Deserialize, Serialize, Clone, Debug)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Follow {
+        pub actor: ObjectId<DbUser>,
+        pub object: ObjectId<DbUser>,
+        #[serde(rename = "type")]
+        pub kind: FollowType,
+        pub id: Url,
+    }
+
+    #[async_trait]
+    impl ActivityHandler for Follow {
+        type DataType = DbConnection;
+        type Error = Error;
+
+        fn id(&self) -> &Url {
+            &self.id
+        }
+
+        fn actor(&self) -> &Url {
+            self.actor.inner()
+        }
+
+        async fn verify(&self, _: &Data<Self::DataType>) -> Result<(), Self::Error> {
+            Ok(())
+        }
+
+        async fn receive(self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+            Ok(())
+        }
+    }
+
+    #[derive(Clone, Debug, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Note {}
+    #[derive(Debug, Clone)]
+    pub struct DbPost {}
+
+    #[async_trait]
+    impl Object for DbPost {
+        type DataType = DbConnection;
+        type Kind = Note;
+        type Error = Error;
+
+        async fn read_from_id(
+            _: Url,
+            _: &Data<Self::DataType>,
+        ) -> Result<Option<Self>, Self::Error> {
+            todo!()
+        }
+
+        async fn into_json(self, _: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+            todo!()
+        }
+
+        async fn verify(
+            _: &Self::Kind,
+            _: &Url,
+            _: &Data<Self::DataType>,
+        ) -> Result<(), Self::Error> {
+            todo!()
+        }
+
+        async fn from_json(_: Self::Kind, _: &Data<Self::DataType>) -> Result<Self, Self::Error> {
+            todo!()
+        }
+    }
+}

src/utils.rs

@@ -1,84 +0,0 @@
-use crate::{Error, InstanceSettings, LocalInstance, APUB_JSON_CONTENT_TYPE};
-use http::StatusCode;
-use serde::de::DeserializeOwned;
-use tracing::log::info;
-use url::Url;
-
-pub async fn fetch_object_http<Kind: DeserializeOwned>(
-    url: &Url,
-    instance: &LocalInstance,
-    request_counter: &mut i32,
-) -> Result<Kind, Error> {
-    // dont fetch local objects this way
-    debug_assert!(url.domain() != Some(&instance.hostname));
-    verify_url_valid(url, &instance.settings)?;
-    info!("Fetching remote object {}", url.to_string());
-
-    *request_counter += 1;
-    if *request_counter > instance.settings.http_fetch_retry_limit {
-        return Err(Error::RequestLimit);
-    }
-
-    let res = instance
-        .client
-        .get(url.as_str())
-        .header("Accept", APUB_JSON_CONTENT_TYPE)
-        .timeout(instance.settings.request_timeout)
-        .send()
-        .await
-        .map_err(Error::conv)?;
-
-    if res.status() == StatusCode::GONE {
-        return Err(Error::ObjectDeleted);
-    }
-
-    res.json().await.map_err(Error::conv)
-}
-
-/// Check that both urls have the same domain. If not, return UrlVerificationError.
-pub fn verify_domains_match(a: &Url, b: &Url) -> Result<(), Error> {
-    if a.domain() != b.domain() {
-        return Err(Error::UrlVerificationError("Domains do not match"));
-    }
-    Ok(())
-}
-
-/// Check that both urls are identical. If not, return UrlVerificationError.
-pub fn verify_urls_match(a: &Url, b: &Url) -> Result<(), Error> {
-    if a != b {
-        return Err(Error::UrlVerificationError("Urls do not match"));
-    }
-    Ok(())
-}
-
-/// Perform some security checks on URLs as mentioned in activitypub spec, and call user-supplied
-/// [`InstanceSettings.verify_url_function`].
-///
-/// https://www.w3.org/TR/activitypub/#security-considerations
-pub fn verify_url_valid(url: &Url, settings: &InstanceSettings) -> Result<(), Error> {
-    match url.scheme() {
-        "https" => {}
-        "http" => {
-            if !settings.debug {
-                return Err(Error::UrlVerificationError(
-                    "Http urls are only allowed in debug mode",
-                ));
-            }
-        }
-        _ => return Err(Error::UrlVerificationError("Invalid url scheme")),
-    };
-
-    if url.domain().is_none() {
-        return Err(Error::UrlVerificationError("Url must have a domain"));
-    }
-
-    if url.domain() == Some("localhost") && !settings.debug {
-        return Err(Error::UrlVerificationError(
-            "Localhost is only allowed in debug mode",
-        ));
-    }
-
-    (settings.verify_url_function)(url).map_err(Error::UrlVerificationError)?;
-
-    Ok(())
-}