Simplify old_repo, remove client_pool_size

This commit is contained in:
asonix 2023-09-30 22:43:40 -05:00
parent 47751f3875
commit 81c6e73b5f
6 changed files with 61 additions and 52 deletions

View file

@ -52,7 +52,6 @@ impl Args {
Command::Run(Run { Command::Run(Run {
address, address,
api_key, api_key,
client_pool_size,
client_timeout, client_timeout,
metrics_prometheus_address, metrics_prometheus_address,
media_preprocess_steps, media_preprocess_steps,
@ -111,7 +110,6 @@ impl Args {
}; };
let client = Client { let client = Client {
pool_size: client_pool_size,
timeout: client_timeout, timeout: client_timeout,
}; };
@ -493,8 +491,6 @@ struct Server {
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
struct Client { struct Client {
#[serde(skip_serializing_if = "Option::is_none")]
pool_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
timeout: Option<u64>, timeout: Option<u64>,
} }
@ -862,14 +858,6 @@ struct Run {
#[arg(long)] #[arg(long)]
api_key: Option<String>, api_key: Option<String>,
/// Number of connections the internel HTTP client should maintain in its pool
///
/// This number defaults to 100, and the total number is multiplied by the number of cores
/// available to the program. This means that running on a 2 core system will result in 200
/// pooled connections, and running on a 32 core system will result in 3200 pooled connections.
#[arg(long)]
client_pool_size: Option<usize>,
/// How long (in seconds) the internel HTTP client should wait for responses /// How long (in seconds) the internel HTTP client should wait for responses
/// ///
/// This number defaults to 30 /// This number defaults to 30

View file

@ -10,7 +10,6 @@ pub(crate) struct Defaults {
server: ServerDefaults, server: ServerDefaults,
client: ClientDefaults, client: ClientDefaults,
tracing: TracingDefaults, tracing: TracingDefaults,
old_repo: SledDefaults,
media: MediaDefaults, media: MediaDefaults,
repo: RepoDefaults, repo: RepoDefaults,
store: StoreDefaults, store: StoreDefaults,
@ -27,7 +26,6 @@ struct ServerDefaults {
#[derive(Clone, Debug, serde::Serialize)] #[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
struct ClientDefaults { struct ClientDefaults {
pool_size: usize,
timeout: u64, timeout: u64,
} }
@ -138,7 +136,7 @@ enum RepoDefaults {
#[derive(Clone, Debug, serde::Serialize)] #[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
struct SledDefaults { pub(super) struct SledDefaults {
path: PathBuf, path: PathBuf,
cache_capacity: u64, cache_capacity: u64,
export_path: PathBuf, export_path: PathBuf,
@ -183,10 +181,7 @@ impl Default for ServerDefaults {
impl Default for ClientDefaults { impl Default for ClientDefaults {
fn default() -> Self { fn default() -> Self {
ClientDefaults { ClientDefaults { timeout: 30 }
pool_size: 100,
timeout: 30,
}
} }
} }
@ -351,6 +346,16 @@ impl From<crate::config::commandline::Filesystem> for crate::config::primitives:
} }
} }
impl From<SledDefaults> for crate::config::file::Sled {
fn from(defaults: SledDefaults) -> Self {
crate::config::file::Sled {
path: defaults.path,
cache_capacity: defaults.cache_capacity,
export_path: defaults.export_path,
}
}
}
impl From<crate::config::commandline::Sled> for crate::config::file::Sled { impl From<crate::config::commandline::Sled> for crate::config::file::Sled {
fn from(value: crate::config::commandline::Sled) -> Self { fn from(value: crate::config::commandline::Sled) -> Self {
let defaults = SledDefaults::default(); let defaults = SledDefaults::default();

View file

@ -18,7 +18,7 @@ pub(crate) struct ConfigFile {
#[serde(default)] #[serde(default)]
pub(crate) metrics: Metrics, pub(crate) metrics: Metrics,
pub(crate) old_repo: Sled, old_repo: OldRepo,
pub(crate) media: Media, pub(crate) media: Media,
@ -27,6 +27,15 @@ pub(crate) struct ConfigFile {
pub(crate) store: Store, pub(crate) store: Store,
} }
impl ConfigFile {
pub(crate) fn old_repo_path(&self) -> Option<&PathBuf> {
self.old_repo.path.as_ref().or_else(|| match self.repo {
Repo::Sled(ref sled) => Some(&sled.path),
_ => None,
})
}
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
#[serde(tag = "type")] #[serde(tag = "type")]
@ -106,8 +115,6 @@ pub(crate) struct Server {
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub(crate) struct Client { pub(crate) struct Client {
pub(crate) pool_size: usize,
pub(crate) timeout: u64, pub(crate) timeout: u64,
} }
@ -417,6 +424,12 @@ impl Media {
} }
} }
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct OldRepo {
pub(crate) path: Option<PathBuf>,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub(crate) struct Sled { pub(crate) struct Sled {

View file

@ -1598,11 +1598,10 @@ fn transform_error(error: actix_form_data::Error) -> actix_web::Error {
error error
} }
fn build_client(config: &Configuration) -> Result<ClientWithMiddleware, Error> { fn build_client() -> Result<ClientWithMiddleware, Error> {
let client = reqwest::Client::builder() let client = reqwest::Client::builder()
.user_agent("pict-rs v0.5.0-main") .user_agent("pict-rs v0.5.0-main")
.use_rustls_tls() .use_rustls_tls()
.pool_max_idle_per_host(config.client.pool_size)
.build() .build()
.map_err(UploadError::BuildClient)?; .map_err(UploadError::BuildClient)?;
@ -1979,7 +1978,7 @@ impl PictRsConfiguration {
pub async fn run(self) -> color_eyre::Result<()> { pub async fn run(self) -> color_eyre::Result<()> {
let PictRsConfiguration { config, operation } = self; let PictRsConfiguration { config, operation } = self;
let client = build_client(&config)?; let client = build_client()?;
match operation { match operation {
Operation::Run => (), Operation::Run => (),
@ -2073,12 +2072,19 @@ impl PictRsConfiguration {
let store = FileStore::build(path, arc_repo.clone()).await?; let store = FileStore::build(path, arc_repo.clone()).await?;
if arc_repo.get("migrate-0.4").await?.is_none() { if arc_repo.get("migrate-0.4").await?.is_none() {
if let Some(old_repo) = repo_04::open(&config.old_repo)? { if let Some(path) = config.old_repo_path() {
repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone()) if let Some(old_repo) = repo_04::open(path)? {
.await?; repo::migrate_04(
arc_repo old_repo,
.set("migrate-0.4", Arc::from(b"migrated".to_vec())) arc_repo.clone(),
store.clone(),
config.clone(),
)
.await?; .await?;
arc_repo
.set("migrate-0.4", Arc::from(b"migrated".to_vec()))
.await?;
}
} }
} }
@ -2129,12 +2135,19 @@ impl PictRsConfiguration {
.build(client.clone()); .build(client.clone());
if arc_repo.get("migrate-0.4").await?.is_none() { if arc_repo.get("migrate-0.4").await?.is_none() {
if let Some(old_repo) = repo_04::open(&config.old_repo)? { if let Some(path) = config.old_repo_path() {
repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone()) if let Some(old_repo) = repo_04::open(path)? {
.await?; repo::migrate_04(
arc_repo old_repo,
.set("migrate-0.4", Arc::from(b"migrated".to_vec())) arc_repo.clone(),
store.clone(),
config.clone(),
)
.await?; .await?;
arc_repo
.set("migrate-0.4", Arc::from(b"migrated".to_vec()))
.await?;
}
} }
} }

View file

@ -1,24 +1,17 @@
use crate::{ use crate::{
config,
details::Details, details::Details,
repo::{Alias, DeleteToken}, repo::{Alias, DeleteToken},
}; };
use futures_core::Stream; use futures_core::Stream;
use std::{fmt::Debug, sync::Arc}; use std::{fmt::Debug, path::PathBuf, sync::Arc};
pub(crate) use self::sled::SledRepo; pub(crate) use self::sled::SledRepo;
mod sled; mod sled;
#[tracing::instrument] #[tracing::instrument]
pub(crate) fn open(config: &config::Sled) -> color_eyre::Result<Option<SledRepo>> { pub(crate) fn open(path: &PathBuf) -> color_eyre::Result<Option<SledRepo>> {
let config::Sled { SledRepo::build(path.clone())
path,
cache_capacity,
export_path: _,
} = config;
SledRepo::build(path.clone(), *cache_capacity)
} }
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]

View file

@ -109,8 +109,8 @@ impl OldDetails {
impl SledRepo { impl SledRepo {
#[tracing::instrument] #[tracing::instrument]
pub(crate) fn build(path: PathBuf, cache_capacity: u64) -> color_eyre::Result<Option<Self>> { pub(crate) fn build(path: PathBuf) -> color_eyre::Result<Option<Self>> {
let Some(db) = Self::open(path, cache_capacity)? else { let Some(db) = Self::open(path)? else {
return Ok(None); return Ok(None);
}; };
@ -129,7 +129,7 @@ impl SledRepo {
})) }))
} }
fn open(mut path: PathBuf, cache_capacity: u64) -> Result<Option<Db>, SledError> { fn open(mut path: PathBuf) -> Result<Option<Db>, SledError> {
path.push("v0.4.0-alpha.1"); path.push("v0.4.0-alpha.1");
if let Err(e) = std::fs::metadata(&path) { if let Err(e) = std::fs::metadata(&path) {
@ -138,10 +138,7 @@ impl SledRepo {
} }
} }
let db = ::sled::Config::new() let db = ::sled::Config::new().path(path).open()?;
.cache_capacity(cache_capacity)
.path(path)
.open()?;
Ok(Some(db)) Ok(Some(db))
} }