From d8f5e643bcee95969b59c309809710a38b0661e3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 24 Apr 2020 10:10:01 +0000 Subject: [PATCH] Split code for modular compilation --- Cargo.lock | 111 +++++++++++++++++++++++-- Cargo.toml | 50 ++--------- Makefile | 2 +- src/api/Cargo.toml | 28 +++++++ src/api/api_server.rs | 18 ++-- src/api/http_util.rs | 2 +- src/api/{mod.rs => lib.rs} | 3 + src/core/Cargo.toml | 34 ++++++++ src/{store => core}/block.rs | 20 ++--- src/{store => core}/block_ref_table.rs | 10 +-- src/{store => core}/bucket_table.rs | 4 +- src/{server.rs => core/garage.rs} | 109 ++++-------------------- src/{store => core}/key_table.rs | 6 +- src/{store/mod.rs => core/lib.rs} | 5 +- src/{store => core}/object_table.rs | 12 +-- src/{store => core}/version_table.rs | 12 +-- src/garage/Cargo.toml | 36 ++++++++ src/{ => garage}/admin_rpc.rs | 18 ++-- src/{ => garage}/main.rs | 21 ++--- src/{store => garage}/repair.rs | 11 ++- src/garage/server.rs | 87 +++++++++++++++++++ src/rpc/Cargo.toml | 37 +++++++++ src/rpc/{mod.rs => lib.rs} | 3 + src/rpc/membership.rs | 10 +-- src/rpc/rpc_client.rs | 34 ++------ src/rpc/rpc_server.rs | 8 +- src/rpc/tls_util.rs | 2 +- src/table/Cargo.toml | 32 +++++++ src/table/{mod.rs => lib.rs} | 5 ++ src/table/table.rs | 20 ++--- src/table/table_fullcopy.rs | 7 +- src/table/table_sharded.rs | 7 +- src/table/table_sync.rs | 9 +- src/util/Cargo.toml | 35 ++++++++ src/{ => util}/background.rs | 0 src/{ => util}/config.rs | 14 ++-- src/{ => util}/data.rs | 0 src/{ => util}/error.rs | 21 ++++- src/util/lib.rs | 7 ++ 39 files changed, 569 insertions(+), 281 deletions(-) create mode 100644 src/api/Cargo.toml rename src/api/{mod.rs => lib.rs} (54%) create mode 100644 src/core/Cargo.toml rename src/{store => core}/block.rs (97%) rename src/{store => core}/block_ref_table.rs (90%) rename src/{store => core}/bucket_table.rs (97%) rename src/{server.rs => core/garage.rs} (57%) rename src/{store => core}/key_table.rs (97%) rename src/{store/mod.rs => core/lib.rs} (72%) rename src/{store => core}/object_table.rs (94%) rename src/{store => core}/version_table.rs (93%) create mode 100644 src/garage/Cargo.toml rename src/{ => garage}/admin_rpc.rs (97%) rename src/{ => garage}/main.rs (98%) rename src/{store => garage}/repair.rs (95%) create mode 100644 src/garage/server.rs create mode 100644 src/rpc/Cargo.toml rename src/rpc/{mod.rs => lib.rs} (70%) create mode 100644 src/table/Cargo.toml rename src/table/{mod.rs => lib.rs} (61%) create mode 100644 src/util/Cargo.toml rename src/{ => util}/background.rs (100%) rename src/{ => util}/config.rs (100%) rename src/{ => util}/data.rs (100%) rename src/{ => util}/error.rs (79%) create mode 100644 src/util/lib.rs diff --git a/Cargo.lock b/Cargo.lock index e8adba6d..46533dec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -315,32 +315,133 @@ dependencies = [ [[package]] name = "garage" version = "0.1.0" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_api 0.1.0", + "garage_core 0.1.0", + "garage_rpc 0.1.0", + "garage_table 0.1.0", + "garage_util 0.1.0", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pretty_env_logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "garage_api" +version = "0.1.0" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_core 0.1.0", + "garage_table 0.1.0", + "garage_util 0.1.0", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "garage_core" +version = "0.1.0" dependencies = [ "arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "async-trait 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "err-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_rpc 0.1.0", + "garage_table 0.1.0", + "garage_util 0.1.0", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_bytes 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "garage_rpc" +version = "0.1.0" +dependencies = [ + "arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_util 0.1.0", "gethostname 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-rustls 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "pretty_env_logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "garage_table" +version = "0.1.0" +dependencies = [ + "arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "async-trait 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_rpc 0.1.0", + "garage_util 0.1.0", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "garage_util" +version = "0.1.0" +dependencies = [ + "err-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-rustls 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "webpki 0.21.2 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/Cargo.toml b/Cargo.toml index 13033694..b3043acd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,44 +1,12 @@ -[package] -name = "garage" -version = "0.1.0" -authors = ["Alex Auvolat "] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -bytes = "0.4" -rand = "0.7" -hex = "0.3" -sha2 = "0.8" -arc-swap = "0.4" -gethostname = "0.2" -err-derive = "0.2.3" -log = "0.4" -pretty_env_logger = "0.4" - -sled = "0.31" - -structopt = { version = "0.3", default-features = false } -toml = "0.5" -rmp-serde = "0.14.3" -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" -serde_json = "1.0" - -async-trait = "0.1.30" -futures = "0.3" -futures-core = "0.3" -futures-util = "0.3" -tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } - -http = "0.2" -hyper = "0.13" -rustls = "0.17" -tokio-rustls = "0.13" -hyper-rustls = { version = "0.20", default-features = false } -webpki = "0.21" - +[workspace] +members = [ + "src/util", + "src/rpc", + "src/table", + "src/core", + "src/api", + "src/garage", +] [profile.dev] lto = "off" diff --git a/Makefile b/Makefile index 21d3e2a3..32a652aa 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,3 @@ all: cargo fmt || true - cargo build + RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml new file mode 100644 index 00000000..3606c1df --- /dev/null +++ b/src/api/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "garage_api" +version = "0.1.0" +authors = ["Alex Auvolat "] +edition = "2018" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_util = { path = "../util" } +garage_table = { path = "../table" } +garage_core = { path = "../core" } + +bytes = "0.4" +hex = "0.3" +log = "0.4" + +futures = "0.3" +futures-util = "0.3" +tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } + +http = "0.2" +hyper = "0.13" + + diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 905ba0dd..de79ffc2 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -9,18 +9,18 @@ use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; -use crate::data::*; -use crate::error::Error; -use crate::server::Garage; +use garage_util::data::*; +use garage_util::error::Error; -use crate::table::EmptyKey; +use garage_table::EmptyKey; -use crate::store::block::INLINE_THRESHOLD; -use crate::store::block_ref_table::*; -use crate::store::object_table::*; -use crate::store::version_table::*; +use garage_core::block::INLINE_THRESHOLD; +use garage_core::block_ref_table::*; +use garage_core::garage::Garage; +use garage_core::object_table::*; +use garage_core::version_table::*; -use crate::api::http_util::*; +use crate::http_util::*; type BodyType = Box + Send + Unpin>; diff --git a/src/api/http_util.rs b/src/api/http_util.rs index 228448f0..e7e74409 100644 --- a/src/api/http_util.rs +++ b/src/api/http_util.rs @@ -5,7 +5,7 @@ use futures::ready; use futures::stream::*; use hyper::body::{Bytes, HttpBody}; -use crate::error::Error; +use garage_util::error::Error; type StreamType = Pin> + Send>>; diff --git a/src/api/mod.rs b/src/api/lib.rs similarity index 54% rename from src/api/mod.rs rename to src/api/lib.rs index 8e62d1e7..b313d45d 100644 --- a/src/api/mod.rs +++ b/src/api/lib.rs @@ -1,2 +1,5 @@ +#[macro_use] +extern crate log; + pub mod api_server; pub mod http_util; diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml new file mode 100644 index 00000000..1e482d87 --- /dev/null +++ b/src/core/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "garage_core" +version = "0.1.0" +authors = ["Alex Auvolat "] +edition = "2018" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_util = { path = "../util" } +garage_rpc = { path = "../rpc" } +garage_table = { path = "../table" } + +bytes = "0.4" +rand = "0.7" +hex = "0.3" +sha2 = "0.8" +arc-swap = "0.4" +log = "0.4" + +sled = "0.31" + +rmp-serde = "0.14.3" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } +serde_bytes = "0.11" + +async-trait = "0.1.30" +futures = "0.3" +futures-util = "0.3" +tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } + diff --git a/src/store/block.rs b/src/core/block.rs similarity index 97% rename from src/store/block.rs rename to src/core/block.rs index e2ef32e0..af8b9efb 100644 --- a/src/store/block.rs +++ b/src/core/block.rs @@ -11,20 +11,20 @@ use tokio::fs; use tokio::prelude::*; use tokio::sync::{watch, Mutex, Notify}; -use crate::data; -use crate::data::*; -use crate::error::Error; +use garage_util::data; +use garage_util::data::*; +use garage_util::error::Error; -use crate::rpc::membership::System; -use crate::rpc::rpc_client::*; -use crate::rpc::rpc_server::*; +use garage_rpc::membership::System; +use garage_rpc::rpc_client::*; +use garage_rpc::rpc_server::*; -use crate::table::table_sharded::TableShardedReplication; -use crate::table::TableReplication; +use garage_table::table_sharded::TableShardedReplication; +use garage_table::TableReplication; -use crate::store::block_ref_table::*; +use crate::block_ref_table::*; -use crate::server::Garage; +use crate::garage::Garage; pub const INLINE_THRESHOLD: usize = 3072; diff --git a/src/store/block_ref_table.rs b/src/core/block_ref_table.rs similarity index 90% rename from src/store/block_ref_table.rs rename to src/core/block_ref_table.rs index c8a2a2a1..a00438c0 100644 --- a/src/store/block_ref_table.rs +++ b/src/core/block_ref_table.rs @@ -2,13 +2,13 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use crate::background::*; -use crate::data::*; -use crate::error::Error; +use garage_util::background::*; +use garage_util::data::*; +use garage_util::error::Error; -use crate::table::*; +use garage_table::*; -use crate::store::block::*; +use crate::block::*; #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct BlockRef { diff --git a/src/store/bucket_table.rs b/src/core/bucket_table.rs similarity index 97% rename from src/store/bucket_table.rs rename to src/core/bucket_table.rs index a9bdaa70..28234d82 100644 --- a/src/store/bucket_table.rs +++ b/src/core/bucket_table.rs @@ -1,8 +1,8 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; -use crate::error::Error; -use crate::table::*; +use garage_table::*; +use garage_util::error::Error; #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Bucket { diff --git a/src/server.rs b/src/core/garage.rs similarity index 57% rename from src/server.rs rename to src/core/garage.rs index 0724630a..d77b0acd 100644 --- a/src/server.rs +++ b/src/core/garage.rs @@ -1,31 +1,22 @@ -use std::path::PathBuf; use std::sync::Arc; -use futures_util::future::*; -use tokio::sync::watch; +use garage_util::background::*; +use garage_util::config::*; -use crate::background::*; -use crate::config::*; -use crate::error::Error; +use garage_rpc::membership::System; +use garage_rpc::rpc_client::RpcHttpClient; +use garage_rpc::rpc_server::RpcServer; -use crate::rpc::membership::System; -use crate::rpc::rpc_client::RpcHttpClient; -use crate::rpc::rpc_server::RpcServer; +use garage_table::table_fullcopy::*; +use garage_table::table_sharded::*; +use garage_table::*; -use crate::table::table_fullcopy::*; -use crate::table::table_sharded::*; -use crate::table::*; - -use crate::store::block::*; -use crate::store::block_ref_table::*; -use crate::store::bucket_table::*; -use crate::store::key_table::*; -use crate::store::object_table::*; -use crate::store::version_table::*; - -use crate::api::api_server; - -use crate::admin_rpc::*; +use crate::block::*; +use crate::block_ref_table::*; +use crate::bucket_table::*; +use crate::key_table::*; +use crate::object_table::*; +use crate::version_table::*; pub struct Garage { pub config: Config, @@ -166,9 +157,6 @@ impl Garage { block_ref_table, }); - info!("Crate admin RPC handler..."); - AdminRpcHandler::new(garage.clone()).register_handler(rpc_server); - info!("Start block manager background thread..."); garage.block_manager.garage.swap(Some(garage.clone())); garage.block_manager.clone().spawn_background_worker().await; @@ -176,72 +164,3 @@ impl Garage { garage } } - -async fn shutdown_signal(send_cancel: watch::Sender) -> Result<(), Error> { - // Wait for the CTRL+C signal - tokio::signal::ctrl_c() - .await - .expect("failed to install CTRL+C signal handler"); - info!("Received CTRL+C, shutting down."); - send_cancel.broadcast(true)?; - Ok(()) -} - -async fn wait_from(mut chan: watch::Receiver) -> () { - while let Some(exit_now) = chan.recv().await { - if exit_now { - return; - } - } -} - -pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { - info!("Loading configuration..."); - let config = read_config(config_file).expect("Unable to read config file"); - - info!("Opening database..."); - let mut db_path = config.metadata_dir.clone(); - db_path.push("db"); - let db = sled::open(db_path).expect("Unable to open DB"); - - info!("Initialize RPC server..."); - let mut rpc_server = RpcServer::new(config.rpc_bind_addr.clone(), config.rpc_tls.clone()); - - info!("Initializing background runner..."); - let (send_cancel, watch_cancel) = watch::channel(false); - let background = BackgroundRunner::new(16, watch_cancel.clone()); - - let garage = Garage::new(config, db, background.clone(), &mut rpc_server).await; - - info!("Initializing RPC and API servers..."); - let run_rpc_server = Arc::new(rpc_server).run(wait_from(watch_cancel.clone())); - let api_server = api_server::run_api_server(garage.clone(), wait_from(watch_cancel.clone())); - - futures::try_join!( - garage - .system - .clone() - .bootstrap(&garage.config.bootstrap_peers[..]) - .map(|rv| { - info!("Bootstrap done"); - Ok(rv) - }), - run_rpc_server.map(|rv| { - info!("RPC server exited"); - rv - }), - api_server.map(|rv| { - info!("API server exited"); - rv - }), - background.run().map(|rv| { - info!("Background runner exited"); - Ok(rv) - }), - shutdown_signal(send_cancel), - )?; - - info!("Cleaning up..."); - - Ok(()) -} diff --git a/src/store/key_table.rs b/src/core/key_table.rs similarity index 97% rename from src/store/key_table.rs rename to src/core/key_table.rs index add6ab02..76d163b5 100644 --- a/src/store/key_table.rs +++ b/src/core/key_table.rs @@ -1,9 +1,9 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; -use crate::data::*; -use crate::error::Error; -use crate::table::*; +use garage_table::*; +use garage_util::data::*; +use garage_util::error::Error; #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Key { diff --git a/src/store/mod.rs b/src/core/lib.rs similarity index 72% rename from src/store/mod.rs rename to src/core/lib.rs index 962264c4..b4a8ddb7 100644 --- a/src/store/mod.rs +++ b/src/core/lib.rs @@ -1,7 +1,10 @@ +#[macro_use] +extern crate log; + pub mod block; pub mod block_ref_table; pub mod bucket_table; +pub mod garage; pub mod key_table; pub mod object_table; -pub mod repair; pub mod version_table; diff --git a/src/store/object_table.rs b/src/core/object_table.rs similarity index 94% rename from src/store/object_table.rs rename to src/core/object_table.rs index f329a7f4..1fe2b3d4 100644 --- a/src/store/object_table.rs +++ b/src/core/object_table.rs @@ -2,14 +2,14 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use crate::background::BackgroundRunner; -use crate::data::*; -use crate::error::Error; +use garage_util::background::BackgroundRunner; +use garage_util::data::*; +use garage_util::error::Error; -use crate::table::table_sharded::*; -use crate::table::*; +use garage_table::table_sharded::*; +use garage_table::*; -use crate::store::version_table::*; +use crate::version_table::*; #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Object { diff --git a/src/store/version_table.rs b/src/core/version_table.rs similarity index 93% rename from src/store/version_table.rs rename to src/core/version_table.rs index 6d304cda..ae32e5cb 100644 --- a/src/store/version_table.rs +++ b/src/core/version_table.rs @@ -2,14 +2,14 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use crate::background::BackgroundRunner; -use crate::data::*; -use crate::error::Error; +use garage_util::background::BackgroundRunner; +use garage_util::data::*; +use garage_util::error::Error; -use crate::table::table_sharded::*; -use crate::table::*; +use garage_table::table_sharded::*; +use garage_table::*; -use crate::store::block_ref_table::*; +use crate::block_ref_table::*; #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Version { diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml new file mode 100644 index 00000000..08b55c32 --- /dev/null +++ b/src/garage/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "garage" +version = "0.1.0" +authors = ["Alex Auvolat "] +edition = "2018" + +[[bin]] +name = "garage" +path = "main.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_util = { path = "../util" } +garage_rpc = { path = "../rpc" } +garage_table = { path = "../table" } +garage_core = { path = "../core" } +garage_api = { path = "../api" } + +bytes = "0.4" +rand = "0.7" +hex = "0.3" +sha2 = "0.8" +log = "0.4" +pretty_env_logger = "0.4" + +sled = "0.31" + +structopt = { version = "0.3", default-features = false } +toml = "0.5" +rmp-serde = "0.14.3" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } + +futures = "0.3" +futures-util = "0.3" +tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } diff --git a/src/admin_rpc.rs b/src/garage/admin_rpc.rs similarity index 97% rename from src/admin_rpc.rs rename to src/garage/admin_rpc.rs index 4e5c56df..aeaf2682 100644 --- a/src/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -2,19 +2,19 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; -use crate::data::*; -use crate::error::Error; -use crate::server::Garage; +use garage_util::data::*; +use garage_util::error::Error; -use crate::table::*; +use garage_table::*; -use crate::rpc::rpc_client::*; -use crate::rpc::rpc_server::*; +use garage_rpc::rpc_client::*; +use garage_rpc::rpc_server::*; -use crate::store::bucket_table::*; -use crate::store::key_table::*; -use crate::store::repair::Repair; +use garage_core::bucket_table::*; +use garage_core::garage::Garage; +use garage_core::key_table::*; +use crate::repair::Repair; use crate::*; pub const ADMIN_RPC_TIMEOUT: Duration = Duration::from_secs(30); diff --git a/src/main.rs b/src/garage/main.rs similarity index 98% rename from src/main.rs rename to src/garage/main.rs index 2c25aadb..1185871f 100644 --- a/src/main.rs +++ b/src/garage/main.rs @@ -3,17 +3,8 @@ #[macro_use] extern crate log; -mod background; -mod config; -mod data; -mod error; - -mod api; -mod rpc; -mod store; -mod table; - mod admin_rpc; +mod repair; mod server; use std::collections::HashSet; @@ -25,12 +16,12 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use structopt::StructOpt; -use config::TlsConfig; -use data::*; -use error::Error; +use garage_util::config::TlsConfig; +use garage_util::data::*; +use garage_util::error::Error; -use rpc::membership::*; -use rpc::rpc_client::*; +use garage_rpc::membership::*; +use garage_rpc::rpc_client::*; use admin_rpc::*; diff --git a/src/store/repair.rs b/src/garage/repair.rs similarity index 95% rename from src/store/repair.rs rename to src/garage/repair.rs index 39c57fc1..4efb9e84 100644 --- a/src/store/repair.rs +++ b/src/garage/repair.rs @@ -2,12 +2,11 @@ use std::sync::Arc; use tokio::sync::watch; -use crate::error::Error; -use crate::server::Garage; -use crate::table::*; - -use crate::store::block_ref_table::*; -use crate::store::version_table::*; +use garage_core::block_ref_table::*; +use garage_core::garage::Garage; +use garage_core::version_table::*; +use garage_table::*; +use garage_util::error::Error; use crate::*; diff --git a/src/garage/server.rs b/src/garage/server.rs new file mode 100644 index 00000000..52d03464 --- /dev/null +++ b/src/garage/server.rs @@ -0,0 +1,87 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use futures_util::future::*; +use tokio::sync::watch; + +use garage_util::background::*; +use garage_util::config::*; +use garage_util::error::Error; + +use garage_api::api_server; +use garage_core::garage::Garage; +use garage_rpc::rpc_server::RpcServer; + +use crate::admin_rpc::*; + +async fn shutdown_signal(send_cancel: watch::Sender) -> Result<(), Error> { + // Wait for the CTRL+C signal + tokio::signal::ctrl_c() + .await + .expect("failed to install CTRL+C signal handler"); + info!("Received CTRL+C, shutting down."); + send_cancel.broadcast(true)?; + Ok(()) +} + +async fn wait_from(mut chan: watch::Receiver) -> () { + while let Some(exit_now) = chan.recv().await { + if exit_now { + return; + } + } +} + +pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { + info!("Loading configuration..."); + let config = read_config(config_file).expect("Unable to read config file"); + + info!("Opening database..."); + let mut db_path = config.metadata_dir.clone(); + db_path.push("db"); + let db = sled::open(db_path).expect("Unable to open DB"); + + info!("Initialize RPC server..."); + let mut rpc_server = RpcServer::new(config.rpc_bind_addr.clone(), config.rpc_tls.clone()); + + info!("Initializing background runner..."); + let (send_cancel, watch_cancel) = watch::channel(false); + let background = BackgroundRunner::new(16, watch_cancel.clone()); + + let garage = Garage::new(config, db, background.clone(), &mut rpc_server).await; + + info!("Crate admin RPC handler..."); + AdminRpcHandler::new(garage.clone()).register_handler(&mut rpc_server); + + info!("Initializing RPC and API servers..."); + let run_rpc_server = Arc::new(rpc_server).run(wait_from(watch_cancel.clone())); + let api_server = api_server::run_api_server(garage.clone(), wait_from(watch_cancel.clone())); + + futures::try_join!( + garage + .system + .clone() + .bootstrap(&garage.config.bootstrap_peers[..]) + .map(|rv| { + info!("Bootstrap done"); + Ok(rv) + }), + run_rpc_server.map(|rv| { + info!("RPC server exited"); + rv + }), + api_server.map(|rv| { + info!("API server exited"); + rv + }), + background.run().map(|rv| { + info!("Background runner exited"); + Ok(rv) + }), + shutdown_signal(send_cancel), + )?; + + info!("Cleaning up..."); + + Ok(()) +} diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml new file mode 100644 index 00000000..d7a09255 --- /dev/null +++ b/src/rpc/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "garage_rpc" +version = "0.1.0" +authors = ["Alex Auvolat "] +edition = "2018" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_util = { path = "../util" } + +bytes = "0.4" +rand = "0.7" +hex = "0.3" +sha2 = "0.8" +arc-swap = "0.4" +gethostname = "0.2" +log = "0.4" + +rmp-serde = "0.14.3" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } + +futures = "0.3" +futures-util = "0.3" +tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } + +http = "0.2" +hyper = "0.13" +rustls = "0.17" +tokio-rustls = "0.13" +hyper-rustls = { version = "0.20", default-features = false } +webpki = "0.21" + + diff --git a/src/rpc/mod.rs b/src/rpc/lib.rs similarity index 70% rename from src/rpc/mod.rs rename to src/rpc/lib.rs index 83fd0aac..3fae6c3e 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/lib.rs @@ -1,3 +1,6 @@ +#[macro_use] +extern crate log; + pub mod membership; pub mod rpc_client; pub mod rpc_server; diff --git a/src/rpc/membership.rs b/src/rpc/membership.rs index e0509536..dcda2c40 100644 --- a/src/rpc/membership.rs +++ b/src/rpc/membership.rs @@ -17,12 +17,12 @@ use tokio::prelude::*; use tokio::sync::watch; use tokio::sync::Mutex; -use crate::background::BackgroundRunner; -use crate::data::*; -use crate::error::Error; +use garage_util::background::BackgroundRunner; +use garage_util::data::*; +use garage_util::error::Error; -use crate::rpc::rpc_client::*; -use crate::rpc::rpc_server::*; +use crate::rpc_client::*; +use crate::rpc_server::*; const PING_INTERVAL: Duration = Duration::from_secs(10); const PING_TIMEOUT: Duration = Duration::from_secs(2); diff --git a/src/rpc/rpc_client.rs b/src/rpc/rpc_client.rs index 027a3cde..3f943dcc 100644 --- a/src/rpc/rpc_client.rs +++ b/src/rpc/rpc_client.rs @@ -8,7 +8,6 @@ use std::time::Duration; use arc_swap::ArcSwapOption; use bytes::IntoBuf; -use err_derive::Error; use futures::future::Future; use futures::stream::futures_unordered::FuturesUnordered; use futures::stream::StreamExt; @@ -17,36 +16,17 @@ use hyper::client::{Client, HttpConnector}; use hyper::{Body, Method, Request}; use tokio::sync::{watch, Semaphore}; -use crate::background::BackgroundRunner; -use crate::data::*; -use crate::error::Error; +use garage_util::background::BackgroundRunner; +use garage_util::config::TlsConfig; +use garage_util::data::*; +use garage_util::error::{Error, RPCError}; -use crate::rpc::membership::Status; -use crate::rpc::rpc_server::RpcMessage; -use crate::rpc::tls_util; - -use crate::config::TlsConfig; +use crate::membership::Status; +use crate::rpc_server::RpcMessage; +use crate::tls_util; const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); -#[derive(Debug, Error)] -pub enum RPCError { - #[error(display = "Node is down: {:?}.", _0)] - NodeDown(UUID), - #[error(display = "Timeout: {}", _0)] - Timeout(#[error(source)] tokio::time::Elapsed), - #[error(display = "HTTP error: {}", _0)] - HTTP(#[error(source)] http::Error), - #[error(display = "Hyper error: {}", _0)] - Hyper(#[error(source)] hyper::Error), - #[error(display = "Messagepack encode error: {}", _0)] - RMPEncode(#[error(source)] rmp_serde::encode::Error), - #[error(display = "Messagepack decode error: {}", _0)] - RMPDecode(#[error(source)] rmp_serde::decode::Error), - #[error(display = "Too many errors: {:?}", _0)] - TooManyErrors(Vec), -} - #[derive(Copy, Clone)] pub struct RequestStrategy { pub rs_timeout: Duration, diff --git a/src/rpc/rpc_server.rs b/src/rpc/rpc_server.rs index 4ee53909..4386d733 100644 --- a/src/rpc/rpc_server.rs +++ b/src/rpc/rpc_server.rs @@ -16,11 +16,11 @@ use tokio::net::{TcpListener, TcpStream}; use tokio_rustls::server::TlsStream; use tokio_rustls::TlsAcceptor; -use crate::config::TlsConfig; -use crate::data::*; -use crate::error::Error; +use garage_util::config::TlsConfig; +use garage_util::data::*; +use garage_util::error::Error; -use crate::rpc::tls_util; +use crate::tls_util; pub trait RpcMessage: Serialize + for<'de> Deserialize<'de> + Send + Sync {} diff --git a/src/rpc/tls_util.rs b/src/rpc/tls_util.rs index 52c52110..36ea7bf3 100644 --- a/src/rpc/tls_util.rs +++ b/src/rpc/tls_util.rs @@ -15,7 +15,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; use tokio_rustls::TlsConnector; use webpki::DNSNameRef; -use crate::error::Error; +use garage_util::error::Error; pub fn load_certs(filename: &str) -> Result, Error> { let certfile = fs::File::open(&filename)?; diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml new file mode 100644 index 00000000..714d0a6e --- /dev/null +++ b/src/table/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "garage_table" +version = "0.1.0" +authors = ["Alex Auvolat "] +edition = "2018" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_util = { path = "../util" } +garage_rpc = { path = "../rpc" } + +bytes = "0.4" +rand = "0.7" +hex = "0.3" +arc-swap = "0.4" +log = "0.4" + +sled = "0.31" + +rmp-serde = "0.14.3" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } +serde_bytes = "0.11" + +async-trait = "0.1.30" +futures = "0.3" +futures-util = "0.3" +tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } + diff --git a/src/table/mod.rs b/src/table/lib.rs similarity index 61% rename from src/table/mod.rs rename to src/table/lib.rs index e03b8d0b..f490b491 100644 --- a/src/table/mod.rs +++ b/src/table/lib.rs @@ -1,3 +1,8 @@ +#![recursion_limit = "1024"] + +#[macro_use] +extern crate log; + pub mod table; pub mod table_fullcopy; pub mod table_sharded; diff --git a/src/table/table.rs b/src/table/table.rs index 50e8739a..94bacc60 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -8,14 +8,14 @@ use futures::stream::*; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; -use crate::data::*; -use crate::error::Error; +use garage_util::data::*; +use garage_util::error::Error; -use crate::rpc::membership::{Ring, System}; -use crate::rpc::rpc_client::*; -use crate::rpc::rpc_server::*; +use garage_rpc::membership::{Ring, System}; +use garage_rpc::rpc_client::*; +use garage_rpc::rpc_server::*; -use crate::table::table_sync::*; +use crate::table_sync::*; const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(10); @@ -78,14 +78,14 @@ impl PartitionKey for EmptyKey { } } -impl> PartitionKey for T { +impl PartitionKey for String { fn hash(&self) -> Hash { - hash(self.as_ref().as_bytes()) + hash(self.as_bytes()) } } -impl> SortKey for T { +impl SortKey for String { fn sort_key(&self) -> &[u8] { - self.as_ref().as_bytes() + self.as_bytes() } } diff --git a/src/table/table_fullcopy.rs b/src/table/table_fullcopy.rs index 2cd2e464..a6c78a63 100644 --- a/src/table/table_fullcopy.rs +++ b/src/table/table_fullcopy.rs @@ -1,9 +1,10 @@ use arc_swap::ArcSwapOption; use std::sync::Arc; -use crate::data::*; -use crate::rpc::membership::{Ring, System}; -use crate::table::*; +use garage_rpc::membership::{Ring, System}; +use garage_util::data::*; + +use crate::*; #[derive(Clone)] pub struct TableFullReplication { diff --git a/src/table/table_sharded.rs b/src/table/table_sharded.rs index 5190f5d4..88856542 100644 --- a/src/table/table_sharded.rs +++ b/src/table/table_sharded.rs @@ -1,6 +1,7 @@ -use crate::data::*; -use crate::rpc::membership::{Ring, System}; -use crate::table::*; +use garage_rpc::membership::{Ring, System}; +use garage_util::data::*; + +use crate::*; #[derive(Clone)] pub struct TableShardedReplication { diff --git a/src/table/table_sync.rs b/src/table/table_sync.rs index 8f6582a7..145b3068 100644 --- a/src/table/table_sync.rs +++ b/src/table/table_sync.rs @@ -12,10 +12,11 @@ use serde_bytes::ByteBuf; use tokio::sync::Mutex; use tokio::sync::{mpsc, watch}; -use crate::data::*; -use crate::error::Error; -use crate::rpc::membership::Ring; -use crate::table::*; +use garage_rpc::membership::Ring; +use garage_util::data::*; +use garage_util::error::Error; + +use crate::*; const MAX_DEPTH: usize = 16; const SCAN_INTERVAL: Duration = Duration::from_secs(3600); diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml new file mode 100644 index 00000000..6f61a586 --- /dev/null +++ b/src/util/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "garage_util" +version = "0.1.0" +authors = ["Alex Auvolat "] +edition = "2018" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rand = "0.7" +hex = "0.3" +sha2 = "0.8" +err-derive = "0.2.3" +log = "0.4" + +sled = "0.31" + +toml = "0.5" +rmp-serde = "0.14.3" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } +serde_json = "1.0" + +futures = "0.3" +futures-util = "0.3" +tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } + +http = "0.2" +hyper = "0.13" +rustls = "0.17" +webpki = "0.21" + + diff --git a/src/background.rs b/src/util/background.rs similarity index 100% rename from src/background.rs rename to src/util/background.rs diff --git a/src/config.rs b/src/util/config.rs similarity index 100% rename from src/config.rs rename to src/util/config.rs index 7a6ae3f2..cb871562 100644 --- a/src/config.rs +++ b/src/util/config.rs @@ -34,6 +34,13 @@ pub struct Config { pub rpc_tls: Option, } +#[derive(Deserialize, Debug, Clone)] +pub struct TlsConfig { + pub ca_cert: String, + pub node_cert: String, + pub node_key: String, +} + fn default_max_concurrent_rpc_requests() -> usize { 12 } @@ -47,13 +54,6 @@ fn default_epidemic_factor() -> usize { 3 } -#[derive(Deserialize, Debug, Clone)] -pub struct TlsConfig { - pub ca_cert: String, - pub node_cert: String, - pub node_key: String, -} - pub fn read_config(config_file: PathBuf) -> Result { let mut file = std::fs::OpenOptions::new() .read(true) diff --git a/src/data.rs b/src/util/data.rs similarity index 100% rename from src/data.rs rename to src/util/data.rs diff --git a/src/error.rs b/src/util/error.rs similarity index 79% rename from src/error.rs rename to src/util/error.rs index 6290dc24..f73d6915 100644 --- a/src/error.rs +++ b/src/util/error.rs @@ -2,8 +2,25 @@ use err_derive::Error; use hyper::StatusCode; use std::io; -use crate::data::Hash; -use crate::rpc::rpc_client::RPCError; +use crate::data::*; + +#[derive(Debug, Error)] +pub enum RPCError { + #[error(display = "Node is down: {:?}.", _0)] + NodeDown(UUID), + #[error(display = "Timeout: {}", _0)] + Timeout(#[error(source)] tokio::time::Elapsed), + #[error(display = "HTTP error: {}", _0)] + HTTP(#[error(source)] http::Error), + #[error(display = "Hyper error: {}", _0)] + Hyper(#[error(source)] hyper::Error), + #[error(display = "Messagepack encode error: {}", _0)] + RMPEncode(#[error(source)] rmp_serde::encode::Error), + #[error(display = "Messagepack decode error: {}", _0)] + RMPDecode(#[error(source)] rmp_serde::decode::Error), + #[error(display = "Too many errors: {:?}", _0)] + TooManyErrors(Vec), +} #[derive(Debug, Error)] pub enum Error { diff --git a/src/util/lib.rs b/src/util/lib.rs new file mode 100644 index 00000000..0bf09bf6 --- /dev/null +++ b/src/util/lib.rs @@ -0,0 +1,7 @@ +#[macro_use] +extern crate log; + +pub mod background; +pub mod config; +pub mod data; +pub mod error;