add support for multiple streams

This commit is contained in:
Astro 2023-01-11 18:45:45 +01:00
parent 1c0bf9be2d
commit 610e3732c5
5 changed files with 42 additions and 27 deletions

View file

@ -1,5 +1,14 @@
# Sources
streams:
# The fedi.buzz firehose stream
- "https://fedi.buzz/api/v1/streaming/public"
# You may list the streaming API of other instances here
# external https hostname
hostname: relay.fedi.buzz hostname: relay.fedi.buzz
# where your reverse proxy will connect to
listen_port: 3000 listen_port: 3000
# ActivityPub signing keypair
priv_key_file: private-key.pem priv_key_file: private-key.pem
pub_key_file: public-key.pem pub_key_file: public-key.pem
# PostgreSQL
db: "host=localhost user=relay password=xyz dbname=buzzrelay" db: "host=localhost user=relay password=xyz dbname=buzzrelay"

View file

@ -2,6 +2,12 @@
{ config, lib, pkgs, ... }: { { config, lib, pkgs, ... }: {
options.services.buzzrelay = with lib; { options.services.buzzrelay = with lib; {
enable = mkEnableOption "Enable Fedi.buzz relay"; enable = mkEnableOption "Enable Fedi.buzz relay";
streams = mkOption {
type = types.listOf str;
default = [
"https://fedi.buzz/api/v1/streaming/public"
];
};
listenPort = mkOption { listenPort = mkOption {
type = types.int; type = types.int;
default = 8000; default = 8000;
@ -34,6 +40,7 @@
cfg = config.services.buzzrelay; cfg = config.services.buzzrelay;
configFile = builtins.toFile "buzzrelay.toml" ( configFile = builtins.toFile "buzzrelay.toml" (
lib.generators.toYAML {} { lib.generators.toYAML {} {
streams = cfg.streams;
hostname = cfg.hostName; hostname = cfg.hostName;
listen_port = cfg.listenPort; listen_port = cfg.listenPort;
priv_key_file = cfg.privKeyFile; priv_key_file = cfg.privKeyFile;

View file

@ -1,14 +1,9 @@
use serde::Deserialize; use serde::Deserialize;
use sigh::{PrivateKey, PublicKey, Key}; use sigh::{PrivateKey, PublicKey, Key};
fn default_upstream() -> String {
"fedi.buzz".to_string()
}
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct Config { pub struct Config {
#[serde(default = "default_upstream")] pub streams: Vec<String>,
pub upstream: String,
pub db: String, pub db: String,
pub hostname: String, pub hostname: String,
pub listen_port: u16, pub listen_port: u16,

View file

@ -248,6 +248,8 @@ async fn main() {
&std::env::args().nth(1) &std::env::args().nth(1)
.expect("Call with config.yaml") .expect("Call with config.yaml")
); );
let priv_key = config.priv_key();
let pub_key = config.pub_key();
let recorder = PrometheusBuilder::new() let recorder = PrometheusBuilder::new()
.add_global_label("application", env!("CARGO_PKG_NAME")) .add_global_label("application", env!("CARGO_PKG_NAME"))
@ -257,7 +259,7 @@ async fn main() {
let database = db::Database::connect(&config.db).await; let database = db::Database::connect(&config.db).await;
let stream_rx = stream::spawn(config.upstream.clone()); let stream_rx = stream::spawn(config.streams.into_iter());
let client = Arc::new( let client = Arc::new(
reqwest::Client::builder() reqwest::Client::builder()
.timeout(Duration::from_secs(5)) .timeout(Duration::from_secs(5))
@ -272,7 +274,7 @@ async fn main() {
.unwrap() .unwrap()
); );
let hostname = Arc::new(config.hostname.clone()); let hostname = Arc::new(config.hostname.clone());
relay::spawn(client.clone(), hostname.clone(), database.clone(), config.priv_key(), stream_rx); relay::spawn(client.clone(), hostname.clone(), database.clone(), priv_key.clone(), stream_rx);
let app = Router::new() let app = Router::new()
.route("/tag/:tag", get(get_tag_actor).post(post_tag_relay)) .route("/tag/:tag", get(get_tag_actor).post(post_tag_relay))
@ -285,8 +287,8 @@ async fn main() {
database, database,
client, client,
hostname, hostname,
priv_key: config.priv_key(), priv_key,
pub_key: config.pub_key(), pub_key,
}) })
.merge(SpaRouter::new("/", "static")); .merge(SpaRouter::new("/", "static"));

View file

@ -13,8 +13,7 @@ pub enum StreamError {
InvalidContentType, InvalidContentType,
} }
async fn run(host: &str) -> Result<impl Stream<Item = String>, StreamError> { async fn run(url: &str) -> Result<impl Stream<Item = String>, StreamError> {
let url = format!("https://{}/api/v1/streaming/public", host);
let client = reqwest::Client::new(); let client = reqwest::Client::new();
let res = client.get(url) let res = client.get(url)
.timeout(Duration::MAX) .timeout(Duration::MAX)
@ -44,9 +43,11 @@ async fn run(host: &str) -> Result<impl Stream<Item = String>, StreamError> {
Ok(src) Ok(src)
} }
pub fn spawn<H: Into<String>>(host: H) -> Receiver<String> { pub fn spawn(hosts: impl Iterator<Item = impl Into<String>>) -> Receiver<String> {
let host = host.into();
let (tx, rx) = channel(1024); let (tx, rx) = channel(1024);
for host in hosts {
let host = host.into();
let tx = tx.clone();
tokio::spawn(async move { tokio::spawn(async move {
loop { loop {
match run(&host).await { match run(&host).await {
@ -61,5 +62,6 @@ pub fn spawn<H: Into<String>>(host: H) -> Receiver<String> {
sleep(Duration::from_secs(1)).await; sleep(Duration::from_secs(1)).await;
} }
}); });
}
rx rx
} }