diff --git a/Cargo.lock b/Cargo.lock index be38a4626..adeba9080 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3387,6 +3387,7 @@ dependencies = [ "futures-util", "i-love-jesus", "lemmy_db_schema_file", + "lemmy_db_schema_setup", "lemmy_utils", "moka", "pretty_assertions", @@ -3410,22 +3411,31 @@ dependencies = [ [[package]] name = "lemmy_db_schema_file" version = "1.0.0-alpha.5" +dependencies = [ + "diesel", + "diesel-derive-enum", + "diesel_ltree", + "serde", + "strum", + "ts-rs", +] + +[[package]] +name = "lemmy_db_schema_setup" +version = "1.0.0-alpha.5" dependencies = [ "anyhow", "chrono", "diesel", - "diesel-derive-enum", "diesel_ltree", "diesel_migrations", "diff", "itertools 0.14.0", + "lemmy_db_schema_file", "lemmy_utils", "pathfinding", - "serde", "serial_test", - "strum", "tracing", - "ts-rs", "unified-diff", ] @@ -3958,7 +3968,7 @@ dependencies = [ "lemmy_apub", "lemmy_apub_objects", "lemmy_db_schema", - "lemmy_db_schema_file", + "lemmy_db_schema_setup", "lemmy_db_views_site", "lemmy_federate", "lemmy_routes", diff --git a/Cargo.toml b/Cargo.toml index a7ab1ab36..f86c3e625 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ members = [ "crates/utils", "crates/db_schema", "crates/db_schema_file", + "crates/db_schema_setup", "crates/db_views/private_message", "crates/db_views/local_user", "crates/db_views/local_image", @@ -113,6 +114,7 @@ lemmy_apub_objects = { version = "=1.0.0-alpha.5", path = "./crates/apub_objects lemmy_utils = { version = "=1.0.0-alpha.5", path = "./crates/utils", default-features = false } lemmy_db_schema = { version = "=1.0.0-alpha.5", path = "./crates/db_schema" } lemmy_db_schema_file = { version = "=1.0.0-alpha.5", path = "./crates/db_schema_file" } +lemmy_db_schema_setup = { version = "=1.0.0-alpha.5", path = "./crates/db_schema_setup" } lemmy_api_utils = { version = "=1.0.0-alpha.5", path = "./crates/api/api_utils" } lemmy_routes = { version = "=1.0.0-alpha.5", path = "./crates/routes" } lemmy_federate = { version = "=1.0.0-alpha.5", path = "./crates/federate" } @@ -228,7 +230,7 @@ lemmy_apub = { workspace = true } lemmy_apub_objects = { workspace = true } lemmy_utils = { workspace = true } lemmy_db_schema = { workspace = true } -lemmy_db_schema_file = { workspace = true } +lemmy_db_schema_setup = { workspace = true } lemmy_api_utils = { workspace = true } lemmy_routes = { workspace = true } lemmy_federate = { workspace = true } diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index 874b02021..07e02f283 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -38,6 +38,7 @@ full = [ "tuplex", "moka", "lemmy_db_schema_file/full", + "lemmy_db_schema_setup", ] ts-rs = ["dep:ts-rs"] @@ -51,6 +52,7 @@ serde_json = { workspace = true, optional = true } activitypub_federation = { workspace = true, optional = true } lemmy_utils = { workspace = true, optional = true } lemmy_db_schema_file = { workspace = true } +lemmy_db_schema_setup = { workspace = true, optional = true } bcrypt = { workspace = true, optional = true } diesel = { workspace = true, optional = true } diesel-derive-newtype = { workspace = true, optional = true } diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 4a24facfe..a736da7dd 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -32,7 +32,6 @@ use diesel_async::{ }; use futures_util::{future::BoxFuture, FutureExt}; use i_love_jesus::{CursorKey, PaginatedQueryBuilder, SortDirection}; -use lemmy_db_schema_file::schema_setup; use lemmy_utils::{ error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult}, settings::{structs::Settings, SETTINGS}, @@ -494,7 +493,7 @@ pub fn build_db_pool() -> LemmyResult { // provide a setup function which handles creating the connection let mut config = ManagerConfig::default(); config.custom_setup = Box::new(establish_connection); - let manager = AsyncDieselConnectionManager::::new_with_config(db_url, config); + let manager = AsyncDieselConnectionManager::::new_with_config(&db_url, config); let pool = Pool::builder(manager) .max_size(SETTINGS.database.pool_size) .runtime(Runtime::Tokio1) @@ -512,7 +511,7 @@ pub fn build_db_pool() -> LemmyResult { })) .build()?; - schema_setup::run(schema_setup::Options::default().run())?; + lemmy_db_schema_setup::run(lemmy_db_schema_setup::Options::default().run(), &db_url)?; Ok(pool) } diff --git a/crates/db_schema_file/Cargo.toml b/crates/db_schema_file/Cargo.toml index bebf0698b..f746f2e63 100644 --- a/crates/db_schema_file/Cargo.toml +++ b/crates/db_schema_file/Cargo.toml @@ -18,16 +18,7 @@ doctest = false workspace = true [features] -full = [ - "diesel", - "diesel_ltree", - "diesel-derive-enum", - "lemmy_utils", - "anyhow", - "chrono", - "tracing", - "diesel_migrations", -] +full = ["diesel", "diesel_ltree", "diesel-derive-enum"] ts-rs = ["dep:ts-rs"] [dependencies] @@ -37,15 +28,3 @@ diesel = { workspace = true, optional = true } diesel_ltree = { workspace = true, optional = true } ts-rs = { workspace = true, optional = true } diesel-derive-enum = { workspace = true, optional = true } -lemmy_utils = { workspace = true, features = ["full"], optional = true } -anyhow = { workspace = true, optional = true } -chrono = { workspace = true, optional = true } -diesel_migrations = { workspace = true, optional = true } -tracing = { workspace = true, optional = true } - -[dev-dependencies] -serial_test = { workspace = true } -diff = "0.1.13" -itertools = { workspace = true } -pathfinding = "4.14.0" -unified-diff = { workspace = true } diff --git a/crates/db_schema_file/src/lib.rs b/crates/db_schema_file/src/lib.rs index 9d812b33c..63b6cb682 100644 --- a/crates/db_schema_file/src/lib.rs +++ b/crates/db_schema_file/src/lib.rs @@ -1,7 +1,3 @@ -#[cfg(feature = "full")] -pub mod diff_check; pub mod enums; #[cfg(feature = "full")] pub mod schema; -#[cfg(feature = "full")] -pub mod schema_setup; diff --git a/crates/db_schema_file/src/schema.rs b/crates/db_schema_file/src/schema.rs index bcae07f07..2039690f5 100644 --- a/crates/db_schema_file/src/schema.rs +++ b/crates/db_schema_file/src/schema.rs @@ -973,13 +973,6 @@ diesel::table! { } } -diesel::table! { - previously_run_sql (id) { - id -> Bool, - content -> Text, - } -} - diesel::table! { private_message (id) { id -> Int4, @@ -1326,7 +1319,6 @@ diesel::allow_tables_to_appear_in_same_query!( post_actions, post_report, post_tag, - previously_run_sql, private_message, private_message_report, received_activity, diff --git a/crates/db_schema_setup/Cargo.toml b/crates/db_schema_setup/Cargo.toml new file mode 100644 index 000000000..998c92f15 --- /dev/null +++ b/crates/db_schema_setup/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "lemmy_db_schema_setup" +version.workspace = true +edition.workspace = true +description.workspace = true +license.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +rust-version.workspace = true + +[lib] +name = "lemmy_db_schema_setup" +path = "src/lib.rs" +doctest = false + +[lints] +workspace = true + +[features] +full = [] + +[dependencies] +diesel = { workspace = true } +chrono = { workspace = true } +diesel_migrations = { workspace = true } +tracing = { workspace = true } +anyhow = { workspace = true } + +[dev-dependencies] +serial_test = { workspace = true } +diff = "0.1.13" +itertools = { workspace = true } +pathfinding = "4.14.0" +unified-diff = { workspace = true } +diesel_ltree = { workspace = true } +lemmy_db_schema_file = { workspace = true, features = ["full"] } +lemmy_utils = { workspace = true, features = ["full"] } diff --git a/crates/db_schema_file/replaceable_schema/triggers.sql b/crates/db_schema_setup/replaceable_schema/triggers.sql similarity index 100% rename from crates/db_schema_file/replaceable_schema/triggers.sql rename to crates/db_schema_setup/replaceable_schema/triggers.sql diff --git a/crates/db_schema_file/replaceable_schema/utils.sql b/crates/db_schema_setup/replaceable_schema/utils.sql similarity index 100% rename from crates/db_schema_file/replaceable_schema/utils.sql rename to crates/db_schema_setup/replaceable_schema/utils.sql diff --git a/crates/db_schema_file/src/diff_check.rs b/crates/db_schema_setup/src/diff_check.rs similarity index 100% rename from crates/db_schema_file/src/diff_check.rs rename to crates/db_schema_setup/src/diff_check.rs diff --git a/crates/db_schema_file/src/schema_setup.rs b/crates/db_schema_setup/src/lib.rs similarity index 93% rename from crates/db_schema_file/src/schema_setup.rs rename to crates/db_schema_setup/src/lib.rs index ac90ec986..a51526827 100644 --- a/crates/db_schema_file/src/schema_setup.rs +++ b/crates/db_schema_setup/src/lib.rs @@ -1,6 +1,4 @@ -#[cfg(test)] -use crate::diff_check; -use crate::schema::previously_run_sql; +mod diff_check; use anyhow::{anyhow, Context}; use chrono::TimeDelta; use diesel::{ @@ -18,7 +16,6 @@ use diesel::{ RunQueryDsl, }; use diesel_migrations::MigrationHarness; -use lemmy_utils::{error::LemmyResult, settings::SETTINGS}; use std::time::Instant; use tracing::debug; @@ -28,6 +25,13 @@ diesel::table! { } } +diesel::table! { + previously_run_sql (id) { + id -> Bool, + content -> Text, + } +} + fn migrations() -> diesel_migrations::EmbeddedMigrations { // Using `const` here is required by the borrow checker const MIGRATIONS: diesel_migrations::EmbeddedMigrations = diesel_migrations::embed_migrations!(); @@ -46,7 +50,7 @@ fn replaceable_schema() -> String { .join("\n") } -const REPLACEABLE_SCHEMA_PATH: &str = "crates/db_schema/replaceable_schema"; +const REPLACEABLE_SCHEMA_PATH: &str = "crates/db_schema_setup/replaceable_schema"; struct MigrationHarnessWrapper<'a> { conn: &'a mut PgConnection, @@ -178,11 +182,9 @@ pub enum Branch { ReplaceableSchemaNotRebuilt, } -pub fn run(options: Options) -> LemmyResult { - let db_url = SETTINGS.get_database_url(); - +pub fn run(options: Options, db_url: &str) -> anyhow::Result { // Migrations don't support async connection, and this function doesn't need to be async - let mut conn = PgConnection::establish(&db_url)?; + let mut conn = PgConnection::establish(db_url)?; // If possible, skip getting a lock and recreating the "r" schema, so // lemmy_server processes in a horizontally scaled setup can start without causing locks @@ -233,7 +235,7 @@ pub fn run(options: Options) -> LemmyResult { let after = diff_check::get_dump(); - diff_check::check_dump_diff([&before, &after], "The code in crates/db_schema/replaceable_schema incorrectly created or modified things outside of the `r` schema, causing these changes to be left behind after dropping the schema:"); + diff_check::check_dump_diff([&before, &after], "The code in crates/db_schema_setup/replaceable_schema incorrectly created or modified things outside of the `r` schema, causing these changes to be left behind after dropping the schema:"); diff_check::deferr_constraint_check(&after); } @@ -250,7 +252,7 @@ pub fn run(options: Options) -> LemmyResult { Ok(output) } -fn run_replaceable_schema(conn: &mut PgConnection) -> LemmyResult<()> { +fn run_replaceable_schema(conn: &mut PgConnection) -> anyhow::Result<()> { conn.transaction(|conn| { conn .batch_execute(&replaceable_schema()) @@ -266,7 +268,7 @@ fn run_replaceable_schema(conn: &mut PgConnection) -> LemmyResult<()> { }) } -fn revert_replaceable_schema(conn: &mut PgConnection) -> LemmyResult<()> { +fn revert_replaceable_schema(conn: &mut PgConnection) -> anyhow::Result<()> { conn .batch_execute("DROP SCHEMA IF EXISTS r CASCADE;") .with_context(|| format!("Failed to revert SQL files in {REPLACEABLE_SCHEMA_PATH}"))?; @@ -384,7 +386,7 @@ mod tests { // Run initial migrations to prepare basic tables assert_eq!( - run(o.run().limit(INITIAL_MIGRATIONS_COUNT))?, + run(o.run().limit(INITIAL_MIGRATIONS_COUNT), &db_url)?, ReplaceableSchemaNotRebuilt ); @@ -392,16 +394,22 @@ mod tests { insert_test_data(&mut conn)?; // Run all migrations, and make sure that changes can be correctly reverted - assert_eq!(run(o.run().enable_diff_check())?, ReplaceableSchemaRebuilt); + assert_eq!( + run(o.run().enable_diff_check(), &db_url)?, + ReplaceableSchemaRebuilt + ); // Check the test data we inserted before after running migrations check_test_data(&mut conn)?; // Check for early return - assert_eq!(run(o.run())?, EarlyReturn); + assert_eq!(run(o.run(), &db_url)?, EarlyReturn); // Test `limit` - assert_eq!(run(o.revert().limit(1))?, ReplaceableSchemaNotRebuilt); + assert_eq!( + run(o.revert().limit(1), &db_url)?, + ReplaceableSchemaNotRebuilt + ); assert_eq!( conn .pending_migrations(migrations()) @@ -409,7 +417,7 @@ mod tests { .len(), 1 ); - assert_eq!(run(o.run().limit(1))?, ReplaceableSchemaRebuilt); + assert_eq!(run(o.run().limit(1), &db_url)?, ReplaceableSchemaRebuilt); // This should throw an error saying to use lemmy_server instead of diesel CLI conn.batch_execute("DROP OWNED BY CURRENT_USER;")?; @@ -419,7 +427,7 @@ mod tests { )); // Diesel CLI's way of running migrations shouldn't break the custom migration runner - assert_eq!(run(o.run())?, ReplaceableSchemaRebuilt); + assert_eq!(run(o.run(), &db_url)?, ReplaceableSchemaRebuilt); Ok(()) } @@ -511,7 +519,7 @@ mod tests { } fn check_test_data(conn: &mut PgConnection) -> LemmyResult<()> { - use crate::schema::{comment, comment_reply, community, person, post}; + use lemmy_db_schema_file::schema::{comment, comment_reply, community, person, post}; // Check users let users: Vec<(i32, String, Option, String, String)> = person::table diff --git a/crates/db_schema_setup/src/main.rs b/crates/db_schema_setup/src/main.rs new file mode 100644 index 000000000..41eeefe05 --- /dev/null +++ b/crates/db_schema_setup/src/main.rs @@ -0,0 +1,14 @@ +/// Very minimal wrapper around `lemmy_db_schema_setup::run` to allow running migrations without +/// compiling everything. +fn main() -> anyhow::Result<()> { + if std::env::args().len() > 1 { + anyhow::bail!("To set parameters for running migrations, use the lemmy_server command."); + } + + lemmy_db_schema_setup::run( + lemmy_db_schema_setup::Options::default().run(), + &std::env::var("LEMMY_DATABASE_URL")?, + )?; + + Ok(()) +} diff --git a/diesel.toml b/diesel.toml index 7630a31c2..be92ab1cc 100644 --- a/diesel.toml +++ b/diesel.toml @@ -3,3 +3,5 @@ file = "crates/db_schema_file/src/schema.rs" patch_file = "crates/db_schema_file/diesel_ltree.patch" # Required for https://github.com/adwhit/diesel-derive-enum custom_type_derives = ["diesel::query_builder::QueryId"] +# This table is in the db_schema_setup crate instead. +filter = { except_tables = ["previously_run_sql"] } diff --git a/scripts/dump_schema.sh b/scripts/dump_schema.sh index c504de8e7..02a4ef881 100755 --- a/scripts/dump_schema.sh +++ b/scripts/dump_schema.sh @@ -9,7 +9,7 @@ cd "$CWD/../" source scripts/start_dev_db.sh -cargo run --package lemmy_server -- migration --all run +cargo run --package lemmy_db_schema_setup pg_dump --no-owner --no-privileges --no-table-access-method --schema-only --exclude-schema=r --no-sync -f schema.sqldump pg_ctl stop diff --git a/scripts/lint.sh b/scripts/lint.sh index 9da5ea54c..52224a281 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -12,6 +12,6 @@ cargo +nightly fmt taplo format # Format sql files -find migrations crates/db_schema_file/replaceable_schema -type f -name '*.sql' -print0 | xargs -0 -P 10 -L 10 pg_format -i +find migrations crates/db_schema_setup/replaceable_schema -type f -name '*.sql' -print0 | xargs -0 -P 10 -L 10 pg_format -i cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-targets --all-features -- -D warnings diff --git a/scripts/sql_format_check.sh b/scripts/sql_format_check.sh index de8241442..87c34bbd5 100755 --- a/scripts/sql_format_check.sh +++ b/scripts/sql_format_check.sh @@ -10,11 +10,11 @@ cd "$CWD/../" # Copy the files to a temp dir TMP_DIR=$(mktemp -d) cp -a migrations/. $TMP_DIR/migrations -cp -a crates/db_schema_file/replaceable_schema/. $TMP_DIR/replaceable_schema +cp -a crates/db_schema_setup/replaceable_schema/. $TMP_DIR/replaceable_schema # Format the new files find $TMP_DIR -type f -name '*.sql' -print0 | xargs -0 -P 10 -L 10 pg_format -i # Diff the directories diff -r migrations $TMP_DIR/migrations -diff -r crates/db_schema_file/replaceable_schema $TMP_DIR/replaceable_schema +diff -r crates/db_schema_setup/replaceable_schema $TMP_DIR/replaceable_schema diff --git a/src/lib.rs b/src/lib.rs index b721a6914..61ae702a9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,6 @@ use lemmy_apub::{ }; use lemmy_apub_objects::objects::{community::FETCH_COMMUNITY_COLLECTIONS, instance::ApubSite}; use lemmy_db_schema::{source::secret::Secret, utils::build_db_pool}; -use lemmy_db_schema_file::schema_setup; use lemmy_db_views_site::SiteView; use lemmy_federate::{Opts, SendManager}; use lemmy_routes::{ @@ -130,7 +129,7 @@ enum CmdSubcommand { }, } -#[derive(Subcommand, Debug)] +#[derive(Subcommand, Debug, PartialEq, Eq)] enum MigrationSubcommand { /// Run up.sql for pending migrations, oldest to newest. Run, @@ -147,8 +146,8 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> { }) = args.subcommand { let mut options = match subcommand { - MigrationSubcommand::Run => schema_setup::Options::default().run(), - MigrationSubcommand::Revert => schema_setup::Options::default().revert(), + MigrationSubcommand::Run => lemmy_db_schema_setup::Options::default().run(), + MigrationSubcommand::Revert => lemmy_db_schema_setup::Options::default().revert(), } .print_output(); @@ -156,7 +155,12 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> { options = options.limit(number); } - schema_setup::run(options)?; + lemmy_db_schema_setup::run(options, &SETTINGS.get_database_url())?; + + #[cfg(debug_assertions)] + if all && subcommand == MigrationSubcommand::Run { + println!("Warning: you probably want this command instead, which requires less crates to be compiled: cargo run --package lemmy_db_schema_setup"); + } return Ok(()); }