From 81c6e73b5f91f54687a41283f35d3e85085cd019 Mon Sep 17 00:00:00 2001 From: asonix Date: Sat, 30 Sep 2023 22:43:40 -0500 Subject: [PATCH] Simplify old_repo, remove client_pool_size --- src/config/commandline.rs | 12 ------------ src/config/defaults.rs | 19 ++++++++++++------- src/config/file.rs | 19 ++++++++++++++++--- src/lib.rs | 39 ++++++++++++++++++++++++++------------- src/repo_04.rs | 13 +++---------- src/repo_04/sled.rs | 11 ++++------- 6 files changed, 61 insertions(+), 52 deletions(-) diff --git a/src/config/commandline.rs b/src/config/commandline.rs index 310bab2..4f1aea2 100644 --- a/src/config/commandline.rs +++ b/src/config/commandline.rs @@ -52,7 +52,6 @@ impl Args { Command::Run(Run { address, api_key, - client_pool_size, client_timeout, metrics_prometheus_address, media_preprocess_steps, @@ -111,7 +110,6 @@ impl Args { }; let client = Client { - pool_size: client_pool_size, timeout: client_timeout, }; @@ -493,8 +491,6 @@ struct Server { #[derive(Debug, Default, serde::Serialize)] #[serde(rename_all = "snake_case")] struct Client { - #[serde(skip_serializing_if = "Option::is_none")] - pool_size: Option, #[serde(skip_serializing_if = "Option::is_none")] timeout: Option, } @@ -862,14 +858,6 @@ struct Run { #[arg(long)] api_key: Option, - /// Number of connections the internel HTTP client should maintain in its pool - /// - /// This number defaults to 100, and the total number is multiplied by the number of cores - /// available to the program. This means that running on a 2 core system will result in 200 - /// pooled connections, and running on a 32 core system will result in 3200 pooled connections. - #[arg(long)] - client_pool_size: Option, - /// How long (in seconds) the internel HTTP client should wait for responses /// /// This number defaults to 30 diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 8aa32bd..21d3b71 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -10,7 +10,6 @@ pub(crate) struct Defaults { server: ServerDefaults, client: ClientDefaults, tracing: TracingDefaults, - old_repo: SledDefaults, media: MediaDefaults, repo: RepoDefaults, store: StoreDefaults, @@ -27,7 +26,6 @@ struct ServerDefaults { #[derive(Clone, Debug, serde::Serialize)] #[serde(rename_all = "snake_case")] struct ClientDefaults { - pool_size: usize, timeout: u64, } @@ -138,7 +136,7 @@ enum RepoDefaults { #[derive(Clone, Debug, serde::Serialize)] #[serde(rename_all = "snake_case")] -struct SledDefaults { +pub(super) struct SledDefaults { path: PathBuf, cache_capacity: u64, export_path: PathBuf, @@ -183,10 +181,7 @@ impl Default for ServerDefaults { impl Default for ClientDefaults { fn default() -> Self { - ClientDefaults { - pool_size: 100, - timeout: 30, - } + ClientDefaults { timeout: 30 } } } @@ -351,6 +346,16 @@ impl From for crate::config::primitives: } } +impl From for crate::config::file::Sled { + fn from(defaults: SledDefaults) -> Self { + crate::config::file::Sled { + path: defaults.path, + cache_capacity: defaults.cache_capacity, + export_path: defaults.export_path, + } + } +} + impl From for crate::config::file::Sled { fn from(value: crate::config::commandline::Sled) -> Self { let defaults = SledDefaults::default(); diff --git a/src/config/file.rs b/src/config/file.rs index df35c5a..846b5cf 100644 --- a/src/config/file.rs +++ b/src/config/file.rs @@ -18,7 +18,7 @@ pub(crate) struct ConfigFile { #[serde(default)] pub(crate) metrics: Metrics, - pub(crate) old_repo: Sled, + old_repo: OldRepo, pub(crate) media: Media, @@ -27,6 +27,15 @@ pub(crate) struct ConfigFile { pub(crate) store: Store, } +impl ConfigFile { + pub(crate) fn old_repo_path(&self) -> Option<&PathBuf> { + self.old_repo.path.as_ref().or_else(|| match self.repo { + Repo::Sled(ref sled) => Some(&sled.path), + _ => None, + }) + } +} + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "snake_case")] #[serde(tag = "type")] @@ -106,8 +115,6 @@ pub(crate) struct Server { #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] pub(crate) struct Client { - pub(crate) pool_size: usize, - pub(crate) timeout: u64, } @@ -417,6 +424,12 @@ impl Media { } } +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct OldRepo { + pub(crate) path: Option, +} + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "snake_case")] pub(crate) struct Sled { diff --git a/src/lib.rs b/src/lib.rs index 5bcf6a6..7fca794 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1598,11 +1598,10 @@ fn transform_error(error: actix_form_data::Error) -> actix_web::Error { error } -fn build_client(config: &Configuration) -> Result { +fn build_client() -> Result { let client = reqwest::Client::builder() .user_agent("pict-rs v0.5.0-main") .use_rustls_tls() - .pool_max_idle_per_host(config.client.pool_size) .build() .map_err(UploadError::BuildClient)?; @@ -1979,7 +1978,7 @@ impl PictRsConfiguration { pub async fn run(self) -> color_eyre::Result<()> { let PictRsConfiguration { config, operation } = self; - let client = build_client(&config)?; + let client = build_client()?; match operation { Operation::Run => (), @@ -2073,12 +2072,19 @@ impl PictRsConfiguration { let store = FileStore::build(path, arc_repo.clone()).await?; if arc_repo.get("migrate-0.4").await?.is_none() { - if let Some(old_repo) = repo_04::open(&config.old_repo)? { - repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone()) - .await?; - arc_repo - .set("migrate-0.4", Arc::from(b"migrated".to_vec())) + if let Some(path) = config.old_repo_path() { + if let Some(old_repo) = repo_04::open(path)? { + repo::migrate_04( + old_repo, + arc_repo.clone(), + store.clone(), + config.clone(), + ) .await?; + arc_repo + .set("migrate-0.4", Arc::from(b"migrated".to_vec())) + .await?; + } } } @@ -2129,12 +2135,19 @@ impl PictRsConfiguration { .build(client.clone()); if arc_repo.get("migrate-0.4").await?.is_none() { - if let Some(old_repo) = repo_04::open(&config.old_repo)? { - repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone()) - .await?; - arc_repo - .set("migrate-0.4", Arc::from(b"migrated".to_vec())) + if let Some(path) = config.old_repo_path() { + if let Some(old_repo) = repo_04::open(path)? { + repo::migrate_04( + old_repo, + arc_repo.clone(), + store.clone(), + config.clone(), + ) .await?; + arc_repo + .set("migrate-0.4", Arc::from(b"migrated".to_vec())) + .await?; + } } } diff --git a/src/repo_04.rs b/src/repo_04.rs index 0d63d55..d921aef 100644 --- a/src/repo_04.rs +++ b/src/repo_04.rs @@ -1,24 +1,17 @@ use crate::{ - config, details::Details, repo::{Alias, DeleteToken}, }; use futures_core::Stream; -use std::{fmt::Debug, sync::Arc}; +use std::{fmt::Debug, path::PathBuf, sync::Arc}; pub(crate) use self::sled::SledRepo; mod sled; #[tracing::instrument] -pub(crate) fn open(config: &config::Sled) -> color_eyre::Result> { - let config::Sled { - path, - cache_capacity, - export_path: _, - } = config; - - SledRepo::build(path.clone(), *cache_capacity) +pub(crate) fn open(path: &PathBuf) -> color_eyre::Result> { + SledRepo::build(path.clone()) } #[derive(Debug, thiserror::Error)] diff --git a/src/repo_04/sled.rs b/src/repo_04/sled.rs index 3bef434..e60f2ea 100644 --- a/src/repo_04/sled.rs +++ b/src/repo_04/sled.rs @@ -109,8 +109,8 @@ impl OldDetails { impl SledRepo { #[tracing::instrument] - pub(crate) fn build(path: PathBuf, cache_capacity: u64) -> color_eyre::Result> { - let Some(db) = Self::open(path, cache_capacity)? else { + pub(crate) fn build(path: PathBuf) -> color_eyre::Result> { + let Some(db) = Self::open(path)? else { return Ok(None); }; @@ -129,7 +129,7 @@ impl SledRepo { })) } - fn open(mut path: PathBuf, cache_capacity: u64) -> Result, SledError> { + fn open(mut path: PathBuf) -> Result, SledError> { path.push("v0.4.0-alpha.1"); if let Err(e) = std::fs::metadata(&path) { @@ -138,10 +138,7 @@ impl SledRepo { } } - let db = ::sled::Config::new() - .cache_capacity(cache_capacity) - .path(path) - .open()?; + let db = ::sled::Config::new().path(path).open()?; Ok(Some(db)) }