update metrics dependencies

This commit is contained in:
Astro 2024-05-26 22:06:50 +02:00
parent 2c9a28784b
commit 473d001b16
6 changed files with 84 additions and 173 deletions

211
Cargo.lock generated
View file

@ -104,10 +104,10 @@ dependencies = [
"axum-core", "axum-core",
"bytes", "bytes",
"futures-util", "futures-util",
"http 1.1.0", "http",
"http-body 1.0.0", "http-body",
"http-body-util", "http-body-util",
"hyper 1.3.1", "hyper",
"hyper-util", "hyper-util",
"itoa", "itoa",
"matchit", "matchit",
@ -137,8 +137,8 @@ dependencies = [
"async-trait", "async-trait",
"bytes", "bytes",
"futures-util", "futures-util",
"http 1.1.0", "http",
"http-body 1.0.0", "http-body",
"http-body-util", "http-body-util",
"mime", "mime",
"pin-project-lite", "pin-project-lite",
@ -224,7 +224,7 @@ dependencies = [
"deunicode", "deunicode",
"eventsource-stream", "eventsource-stream",
"futures", "futures",
"http 1.1.0", "http",
"http_digest_headers", "http_digest_headers",
"httpdate", "httpdate",
"lru", "lru",
@ -632,29 +632,14 @@ dependencies = [
"fnv", "fnv",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"http 1.1.0", "http",
"indexmap 2.2.6", "indexmap",
"slab", "slab",
"tokio", "tokio",
"tokio-util", "tokio-util",
"tracing", "tracing",
] ]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038"
dependencies = [
"ahash",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.5" version = "0.14.5"
@ -742,17 +727,6 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "http"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]] [[package]]
name = "http" name = "http"
version = "1.1.0" version = "1.1.0"
@ -764,17 +738,6 @@ dependencies = [
"itoa", "itoa",
] ]
[[package]]
name = "http-body"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
dependencies = [
"bytes",
"http 0.2.12",
"pin-project-lite",
]
[[package]] [[package]]
name = "http-body" name = "http-body"
version = "1.0.0" version = "1.0.0"
@ -782,7 +745,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
dependencies = [ dependencies = [
"bytes", "bytes",
"http 1.1.0", "http",
] ]
[[package]] [[package]]
@ -793,8 +756,8 @@ checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d"
dependencies = [ dependencies = [
"bytes", "bytes",
"futures-core", "futures-core",
"http 1.1.0", "http",
"http-body 1.0.0", "http-body",
"pin-project-lite", "pin-project-lite",
] ]
@ -828,29 +791,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "hyper"
version = "0.14.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80"
dependencies = [
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"http 0.2.12",
"http-body 0.4.6",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"socket2 0.5.7",
"tokio",
"tower-service",
"tracing",
"want",
]
[[package]] [[package]]
name = "hyper" name = "hyper"
version = "1.3.1" version = "1.3.1"
@ -861,8 +801,8 @@ dependencies = [
"futures-channel", "futures-channel",
"futures-util", "futures-util",
"h2", "h2",
"http 1.1.0", "http",
"http-body 1.0.0", "http-body",
"httparse", "httparse",
"httpdate", "httpdate",
"itoa", "itoa",
@ -879,8 +819,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c"
dependencies = [ dependencies = [
"futures-util", "futures-util",
"http 1.1.0", "http",
"hyper 1.3.1", "hyper",
"hyper-util", "hyper-util",
"rustls", "rustls",
"rustls-pki-types", "rustls-pki-types",
@ -897,7 +837,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
dependencies = [ dependencies = [
"bytes", "bytes",
"http-body-util", "http-body-util",
"hyper 1.3.1", "hyper",
"hyper-util", "hyper-util",
"native-tls", "native-tls",
"tokio", "tokio",
@ -914,11 +854,11 @@ dependencies = [
"bytes", "bytes",
"futures-channel", "futures-channel",
"futures-util", "futures-util",
"http 1.1.0", "http",
"http-body 1.0.0", "http-body",
"hyper 1.3.1", "hyper",
"pin-project-lite", "pin-project-lite",
"socket2 0.5.7", "socket2",
"tokio", "tokio",
"tower", "tower",
"tower-service", "tower-service",
@ -968,16 +908,6 @@ dependencies = [
"unicode-normalization", "unicode-normalization",
] ]
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
]
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "2.2.6" version = "2.2.6"
@ -985,7 +915,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown 0.14.5", "hashbrown",
] ]
[[package]] [[package]]
@ -994,7 +924,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f"
dependencies = [ dependencies = [
"socket2 0.5.7", "socket2",
"widestring", "widestring",
"windows-sys 0.48.0", "windows-sys 0.48.0",
"winreg 0.50.0", "winreg 0.50.0",
@ -1078,7 +1008,7 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc"
dependencies = [ dependencies = [
"hashbrown 0.14.5", "hashbrown",
] ]
[[package]] [[package]]
@ -1090,15 +1020,6 @@ dependencies = [
"linked-hash-map", "linked-hash-map",
] ]
[[package]]
name = "mach2"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "match_cfg" name = "match_cfg"
version = "0.1.0" version = "0.1.0"
@ -1138,24 +1059,26 @@ checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
[[package]] [[package]]
name = "metrics" name = "metrics"
version = "0.21.1" version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" checksum = "2be3cbd384d4e955b231c895ce10685e3d8260c5ccffae898c96c723b0772835"
dependencies = [ dependencies = [
"ahash", "ahash",
"metrics-macros",
"portable-atomic", "portable-atomic",
] ]
[[package]] [[package]]
name = "metrics-exporter-prometheus" name = "metrics-exporter-prometheus"
version = "0.12.2" version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" checksum = "5d58e362dc7206e9456ddbcdbd53c71ba441020e62104703075a69151e38d85f"
dependencies = [ dependencies = [
"base64 0.21.7", "base64 0.22.1",
"hyper 0.14.28", "http-body-util",
"indexmap 1.9.3", "hyper",
"hyper-tls",
"hyper-util",
"indexmap",
"ipnet", "ipnet",
"metrics", "metrics",
"metrics-util", "metrics-util",
@ -1165,28 +1088,17 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "metrics-macros"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "metrics-util" name = "metrics-util"
version = "0.15.1" version = "0.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" checksum = "8b07a5eb561b8cbc16be2d216faf7757f9baf3bfb94dbb0fae3df8387a5bb47f"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"crossbeam-epoch", "crossbeam-epoch",
"crossbeam-utils", "crossbeam-utils",
"hashbrown 0.13.1", "hashbrown",
"indexmap 1.9.3", "indexmap",
"metrics", "metrics",
"num_cpus", "num_cpus",
"ordered-float", "ordered-float",
@ -1364,9 +1276,9 @@ dependencies = [
[[package]] [[package]]
name = "ordered-float" name = "ordered-float"
version = "3.9.2" version = "4.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e"
dependencies = [ dependencies = [
"num-traits", "num-traits",
] ]
@ -1514,13 +1426,12 @@ dependencies = [
[[package]] [[package]]
name = "quanta" name = "quanta"
version = "0.11.1" version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5"
dependencies = [ dependencies = [
"crossbeam-utils", "crossbeam-utils",
"libc", "libc",
"mach2",
"once_cell", "once_cell",
"raw-cpuid", "raw-cpuid",
"wasi", "wasi",
@ -1585,18 +1496,18 @@ dependencies = [
[[package]] [[package]]
name = "raw-cpuid" name = "raw-cpuid"
version = "10.7.0" version = "11.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" checksum = "e29830cbb1290e404f24c73af91c5d8d631ce7e128691e9477556b540cd01ecd"
dependencies = [ dependencies = [
"bitflags 1.3.2", "bitflags 2.5.0",
] ]
[[package]] [[package]]
name = "redis" name = "redis"
version = "0.23.3" version = "0.25.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba" checksum = "6472825949c09872e8f2c50bde59fcefc17748b6be5c90fd67cd8b4daca73bfd"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@ -1609,7 +1520,7 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
"ryu", "ryu",
"sha1_smol", "sha1_smol",
"socket2 0.4.10", "socket2",
"tokio", "tokio",
"tokio-retry", "tokio-retry",
"tokio-util", "tokio-util",
@ -1691,10 +1602,10 @@ dependencies = [
"futures-util", "futures-util",
"h2", "h2",
"hickory-resolver", "hickory-resolver",
"http 1.1.0", "http",
"http-body 1.0.0", "http-body",
"http-body-util", "http-body-util",
"hyper 1.3.1", "hyper",
"hyper-rustls", "hyper-rustls",
"hyper-tls", "hyper-tls",
"hyper-util", "hyper-util",
@ -1922,7 +1833,7 @@ version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [ dependencies = [
"indexmap 2.2.6", "indexmap",
"itoa", "itoa",
"ryu", "ryu",
"serde", "serde",
@ -1962,7 +1873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46bdb4cc44c46a3f0f0a6d1de27c63fccd7fa3384d8d370016c21c8f4a8b89a2" checksum = "46bdb4cc44c46a3f0f0a6d1de27c63fccd7fa3384d8d370016c21c8f4a8b89a2"
dependencies = [ dependencies = [
"base64 0.21.7", "base64 0.21.7",
"http 1.1.0", "http",
"nom", "nom",
"openssl", "openssl",
"thiserror", "thiserror",
@ -2004,16 +1915,6 @@ version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "socket2"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "socket2" name = "socket2"
version = "0.5.7" version = "0.5.7"
@ -2177,7 +2078,7 @@ dependencies = [
"parking_lot", "parking_lot",
"pin-project-lite", "pin-project-lite",
"signal-hook-registry", "signal-hook-registry",
"socket2 0.5.7", "socket2",
"tokio-macros", "tokio-macros",
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
@ -2223,7 +2124,7 @@ dependencies = [
"postgres-protocol", "postgres-protocol",
"postgres-types", "postgres-types",
"rand", "rand",
"socket2 0.5.7", "socket2",
"tokio", "tokio",
"tokio-util", "tokio-util",
"whoami", "whoami",
@ -2289,8 +2190,8 @@ dependencies = [
"bitflags 2.5.0", "bitflags 2.5.0",
"bytes", "bytes",
"futures-util", "futures-util",
"http 1.1.0", "http",
"http-body 1.0.0", "http-body",
"http-body-util", "http-body-util",
"http-range-header", "http-range-header",
"httpdate", "httpdate",

View file

@ -24,11 +24,11 @@ eventsource-stream = "0.2"
futures = "0.3" futures = "0.3"
tokio-postgres = "0.7" tokio-postgres = "0.7"
systemd = "0.10" systemd = "0.10"
metrics = "0.21" metrics = "0.22"
metrics-util = "0.15" metrics-util = "0.16"
metrics-exporter-prometheus = "0.12" metrics-exporter-prometheus = "0.14"
deunicode = "1.4" deunicode = "1.4"
urlencoding = "2" urlencoding = "2"
httpdate = "1" httpdate = "1"
redis = { version = "0.23", features = ["tokio-comp", "connection-manager"] } redis = { version = "0.25", features = ["tokio-comp", "connection-manager"] }
lru = "0.12" lru = "0.12"

View file

@ -72,7 +72,8 @@ impl Database {
self.inner.client.execute(&self.inner.add_follow, &[&id, &inbox, &actor]) self.inner.client.execute(&self.inner.add_follow, &[&id, &inbox, &actor])
.await?; .await?;
let t2 = Instant::now(); let t2 = Instant::now();
histogram!("postgres_query_duration", t2 - t1, "query" => "add_follow"); histogram!("postgres_query_duration", "query" => "add_follow")
.record(t2 - t1);
Ok(()) Ok(())
} }
@ -81,7 +82,8 @@ impl Database {
self.inner.client.execute(&self.inner.del_follow, &[&id, &actor]) self.inner.client.execute(&self.inner.del_follow, &[&id, &actor])
.await?; .await?;
let t2 = Instant::now(); let t2 = Instant::now();
histogram!("postgres_query_duration", t2 - t1, "query" => "del_follow"); histogram!("postgres_query_duration", "query" => "del_follow")
.record(t2 - t1);
Ok(()) Ok(())
} }
@ -90,7 +92,8 @@ impl Database {
let rows = self.inner.client.query(&self.inner.get_following_inboxes, &[&actor]) let rows = self.inner.client.query(&self.inner.get_following_inboxes, &[&actor])
.await?; .await?;
let t2 = Instant::now(); let t2 = Instant::now();
histogram!("postgres_query_duration", t2 - t1, "query" => "get_following_inboxes"); histogram!("postgres_query_duration", "query" => "get_following_inboxes")
.record(t2 - t1);
Ok(rows.into_iter() Ok(rows.into_iter()
.map(|row| row.get(0)) .map(|row| row.get(0))
) )

View file

@ -5,7 +5,7 @@ use axum::{
routing::get, Json, Router, routing::get, Json, Router,
}; };
use tower_http::services::ServeDir; use tower_http::services::ServeDir;
use metrics::increment_counter; use metrics::counter;
use metrics_util::MetricKindMask; use metrics_util::MetricKindMask;
use metrics_exporter_prometheus::PrometheusBuilder; use metrics_exporter_prometheus::PrometheusBuilder;
use serde_json::json; use serde_json::json;
@ -33,7 +33,8 @@ use state::State;
fn track_request(method: &'static str, controller: &'static str, result: &'static str) { fn track_request(method: &'static str, controller: &'static str, result: &'static str) {
increment_counter!("api_http_requests_total", "controller" => controller, "method" => method, "result" => result); counter!("api_http_requests_total", "controller" => controller, "method" => method, "result" => result)
.increment(1);
} }
async fn webfinger( async fn webfinger(

View file

@ -1,6 +1,6 @@
use std::{sync::Arc, collections::{HashSet, HashMap}, time::{Duration, Instant}}; use std::{sync::Arc, collections::{HashSet, HashMap}, time::{Duration, Instant}};
use futures::{channel::mpsc::{channel, Sender}, StreamExt}; use futures::{channel::mpsc::{channel, Sender}, StreamExt};
use metrics::{increment_counter, histogram}; use metrics::{counter, histogram};
use serde::Deserialize; use serde::Deserialize;
use serde_json::json; use serde_json::json;
use sigh::PrivateKey; use sigh::PrivateKey;
@ -104,7 +104,7 @@ struct Job {
} }
fn spawn_worker(client: Arc<reqwest::Client>) -> Sender<Job> { fn spawn_worker(client: Arc<reqwest::Client>) -> Sender<Job> {
let (tx, mut rx) = channel(128); let (tx, mut rx) = channel(512);
tokio::spawn(async move { tokio::spawn(async move {
let mut errors = 0u32; let mut errors = 0u32;
@ -166,7 +166,8 @@ pub fn spawn(
Arc::new(url.to_string()) Arc::new(url.to_string())
} else { } else {
// skip reposts // skip reposts
increment_counter!("relay_posts_total", "action" => "skip"); counter!("relay_posts_total", "action" => "skip")
.increment(1);
continue; continue;
}; };
let mut seen_actors = HashSet::new(); let mut seen_actors = HashSet::new();
@ -226,12 +227,14 @@ pub fn spawn(
seen_actors.insert(actor); seen_actors.insert(actor);
} }
if seen_inboxes.is_empty() { if seen_inboxes.is_empty() {
increment_counter!("relay_posts_total", "action" => "no_relay"); counter!("relay_posts_total", "action" => "no_relay")
.increment(1);
} else { } else {
increment_counter!("relay_posts_total", "action" => "relay"); counter!("relay_posts_total", "action" => "relay")
.increment(1);
} }
let t2 = Instant::now(); let t2 = Instant::now();
histogram!("relay_post_duration", t2 - t1); histogram!("relay_post_duration").record(t2 - t1);
} }
}); });
} }

View file

@ -49,12 +49,15 @@ pub async fn send_raw(
let res = client.execute(req) let res = client.execute(req)
.await?; .await?;
let t3 = Instant::now(); let t3 = Instant::now();
histogram!("relay_http_request_duration", t2 - t1); histogram!("relay_http_request_duration")
.record(t2 - t1);
if res.status() >= StatusCode::OK && res.status() < StatusCode::MULTIPLE_CHOICES { if res.status() >= StatusCode::OK && res.status() < StatusCode::MULTIPLE_CHOICES {
histogram!("relay_http_response_duration", t3 - t2, "res" => "ok"); histogram!("relay_http_response_duration", "res" => "ok")
.record(t3 - t2);
Ok(()) Ok(())
} else { } else {
histogram!("relay_http_response_duration", t3 - t2, "res" => "err"); histogram!("relay_http_response_duration", "res" => "err")
.record(t3 - t2);
let response = res.text().await?; let response = res.text().await?;
Err(Error::Response(response)) Err(Error::Response(response))
} }