From 510f955cc07835718a128cf010da0657d90427d0 Mon Sep 17 00:00:00 2001 From: asonix Date: Sat, 11 Jul 2020 15:40:40 -0500 Subject: [PATCH] Migrate from 0.32.0-rc1 --- Cargo.lock | 4 +- Cargo.toml | 2 +- src/main.rs | 1 + src/migrate.rs | 173 ++++++++++++++++++++++++++++++++++++++++++ src/upload_manager.rs | 33 +++----- 5 files changed, 186 insertions(+), 27 deletions(-) create mode 100644 src/migrate.rs diff --git a/Cargo.lock b/Cargo.lock index 3726c34..9f104a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1891,9 +1891,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "sled" -version = "0.32.0-rc1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49d0ada816c19135ab4644d908b896a76e43d9d7e45bed6b25c486ac692afd7e" +checksum = "cdad3dc85d888056d3bd9954ffdf22d8a22701b6cd3aca4f6df4c436111898c4" dependencies = [ "backtrace", "crc32fast", diff --git a/Cargo.toml b/Cargo.toml index 31d281b..18f90f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ rexiv2 = { version = "0.9.0", git = "https://git.asonix.dog/asonix/rexiv2" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.9.0" -sled = "0.32.0-rc1" +sled = "0.32.0" structopt = "0.3.14" thiserror = "1.0" tracing = "0.1.15" diff --git a/src/main.rs b/src/main.rs index 44de970..598de7b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ use tracing_subscriber::EnvFilter; mod config; mod error; mod middleware; +mod migrate; mod processor; mod upload_manager; mod validate; diff --git a/src/migrate.rs b/src/migrate.rs new file mode 100644 index 0000000..47bf0ae --- /dev/null +++ b/src/migrate.rs @@ -0,0 +1,173 @@ +use crate::UploadError; +use sled as sled032; +use std::path::PathBuf; +use tracing::{debug, info, warn}; + +const SLED_032: &str = "db-0.32"; +const SLED_0320_RC1: &str = "db"; + +pub(crate) struct LatestDb { + root_dir: PathBuf, + version: DbVersion, +} + +impl LatestDb { + pub(crate) fn exists(root_dir: PathBuf) -> Self { + let version = DbVersion::exists(root_dir.clone()); + + LatestDb { root_dir, version } + } + + pub(crate) fn migrate(self) -> Result { + let LatestDb { root_dir, version } = self; + + version.migrate(root_dir) + } +} + +enum DbVersion { + Sled0320Rc1, + Sled032, + Fresh, +} + +impl DbVersion { + fn exists(root: PathBuf) -> Self { + let mut sled_dir = root.clone(); + sled_dir.push("sled"); + sled_dir.push(SLED_032); + if std::fs::metadata(sled_dir).is_ok() { + return DbVersion::Sled032; + } + + let mut sled_dir = root; + sled_dir.push(SLED_0320_RC1); + if std::fs::metadata(sled_dir).is_ok() { + return DbVersion::Sled0320Rc1; + } + + DbVersion::Fresh + } + + fn migrate(self, root: PathBuf) -> Result { + match self { + DbVersion::Sled0320Rc1 => migrate_0_32_0_rc1(root), + DbVersion::Sled032 | DbVersion::Fresh => { + let mut sled_dir = root; + sled_dir.push("sled"); + sled_dir.push(SLED_032); + Ok(sled032::open(sled_dir)?) + } + } + } +} + +fn migrate_0_32_0_rc1(root: PathBuf) -> Result { + info!("Migrating database from 0.32.0-rc1 to 0.32.0"); + let mut sled_dir = root.clone(); + sled_dir.push("db"); + + let mut new_sled_dir = root; + new_sled_dir.push("sled"); + new_sled_dir.push(SLED_032); + + let old_db = sled032::open(sled_dir)?; + let new_db = sled032::open(new_sled_dir)?; + + let old_alias_tree = old_db.open_tree("alias")?; + let new_alias_tree = new_db.open_tree("alias")?; + + let old_fname_tree = old_db.open_tree("filename")?; + let new_fname_tree = new_db.open_tree("filename")?; + + for res in old_alias_tree.iter().keys() { + let k = res?; + if let Some(v) = old_alias_tree.get(&k)? { + if !k.contains(&b"/"[0]) { + // k is an alias + migrate_main_tree(&k, &v, &old_db, &new_db)?; + debug!( + "Moving alias -> hash for alias {}", + String::from_utf8_lossy(k.as_ref()), + ); + } else { + debug!( + "Moving {}, {}", + String::from_utf8_lossy(k.as_ref()), + String::from_utf8_lossy(v.as_ref()) + ); + } + new_alias_tree.insert(k, v)?; + } else { + warn!("MISSING {}", String::from_utf8_lossy(k.as_ref())); + } + } + + for res in old_fname_tree.iter().keys() { + let k = res?; + if let Some(v) = old_fname_tree.get(&k)? { + debug!( + "Moving file -> hash for file {}", + String::from_utf8_lossy(k.as_ref()), + ); + new_fname_tree.insert(&k, &v)?; + } else { + warn!("MISSING {}", String::from_utf8_lossy(k.as_ref())); + } + } + + Ok(new_db) as Result +} + +fn migrate_main_tree( + alias: &sled032::IVec, + hash: &sled032::IVec, + old_db: &sled032::Db, + new_db: &sled032::Db, +) -> Result<(), UploadError> { + debug!( + "Migrating files for {}", + String::from_utf8_lossy(alias.as_ref()) + ); + if let Some(v) = old_db.get(&hash)? { + new_db.insert(&hash, v)?; + } else { + warn!("Missing filename"); + } + + let (start, end) = alias_key_bounds(&hash); + for res in old_db.range(start..end) { + let (k, v) = res?; + debug!("Moving alias {}", String::from_utf8_lossy(v.as_ref())); + new_db.insert(k, v)?; + } + + let (start, end) = variant_key_bounds(&hash); + for res in old_db.range(start..end) { + let (k, v) = res?; + debug!("Moving variant {}", String::from_utf8_lossy(v.as_ref())); + new_db.insert(k, v)?; + } + + Ok(()) +} + +pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec, Vec) { + let mut start = hash.to_vec(); + start.extend(&[0]); + + let mut end = hash.to_vec(); + end.extend(&[1]); + + (start, end) +} + +pub(crate) fn variant_key_bounds(hash: &[u8]) -> (Vec, Vec) { + let mut start = hash.to_vec(); + start.extend(&[2]); + + let mut end = hash.to_vec(); + end.extend(&[3]); + + (start, end) +} diff --git a/src/upload_manager.rs b/src/upload_manager.rs index 4429bd7..dd23d8e 100644 --- a/src/upload_manager.rs +++ b/src/upload_manager.rs @@ -1,4 +1,10 @@ -use crate::{config::Format, error::UploadError, to_ext, validate::validate_image}; +use crate::{ + config::Format, + error::UploadError, + migrate::{alias_key_bounds, variant_key_bounds, LatestDb}, + to_ext, + validate::validate_image, +}; use actix_web::web; use futures::stream::{Stream, StreamExt, TryStreamExt}; use sha2::Digest; @@ -84,10 +90,9 @@ impl UploadManager { mut root_dir: PathBuf, format: Option, ) -> Result { - let mut sled_dir = root_dir.clone(); - sled_dir.push("db"); + let root_clone = root_dir.clone(); // sled automatically creates it's own directories - let db = web::block(move || sled::open(sled_dir)).await?; + let db = web::block(move || LatestDb::exists(root_clone).migrate()).await?; root_dir.push("files"); @@ -684,16 +689,6 @@ fn alias_key(hash: &[u8], id: &str) -> Vec { key } -fn alias_key_bounds(hash: &[u8]) -> (Vec, Vec) { - let mut start = hash.to_vec(); - start.extend(&[0]); - - let mut end = hash.to_vec(); - end.extend(&[1]); - - (start, end) -} - fn alias_id_key(alias: &str) -> String { format!("{}/id", alias) } @@ -708,13 +703,3 @@ fn variant_key(hash: &[u8], path: &str) -> Vec { key.extend(path.as_bytes()); key } - -fn variant_key_bounds(hash: &[u8]) -> (Vec, Vec) { - let mut start = hash.to_vec(); - start.extend(&[2]); - - let mut end = hash.to_vec(); - end.extend(&[3]); - - (start, end) -}