Allow running all migrations with minimal dependencies (#5841)

* move schema_setup to separate crate

* remove lemmy_utils dependency

* add db_schema_setup/src/main.rs

* fix diesel.toml

* run diesel print-schema

* fix replaceable_schema paths

* remove unneeded dependencies

* taplo format

* fix duplicated cfg(test) attribute

* move replaceable_schema to crates/db_schema_setup/replaceable_schema

* warn if lemmy_db_schema_setup can be used instead of lemmy_server

Cargo.lock

@@ -3387,6 +3387,7 @@ dependencies = [
  "futures-util",
  "i-love-jesus",
  "lemmy_db_schema_file",
+ "lemmy_db_schema_setup",
  "lemmy_utils",
  "moka",
  "pretty_assertions",
@@ -3410,22 +3411,31 @@ dependencies = [
 [[package]]
 name = "lemmy_db_schema_file"
 version = "1.0.0-alpha.5"
+dependencies = [
+ "diesel",
+ "diesel-derive-enum",
+ "diesel_ltree",
+ "serde",
+ "strum",
+ "ts-rs",
+]
+
+[[package]]
+name = "lemmy_db_schema_setup"
+version = "1.0.0-alpha.5"
 dependencies = [
  "anyhow",
  "chrono",
  "diesel",
- "diesel-derive-enum",
  "diesel_ltree",
  "diesel_migrations",
  "diff",
  "itertools 0.14.0",
+ "lemmy_db_schema_file",
  "lemmy_utils",
  "pathfinding",
- "serde",
  "serial_test",
- "strum",
  "tracing",
- "ts-rs",
  "unified-diff",
 ]
 
@@ -3958,7 +3968,7 @@ dependencies = [
  "lemmy_apub",
  "lemmy_apub_objects",
  "lemmy_db_schema",
- "lemmy_db_schema_file",
+ "lemmy_db_schema_setup",
  "lemmy_db_views_site",
  "lemmy_federate",
  "lemmy_routes",

Cargo.toml

@@ -51,6 +51,7 @@ members = [
   "crates/utils",
   "crates/db_schema",
   "crates/db_schema_file",
+  "crates/db_schema_setup",
   "crates/db_views/private_message",
   "crates/db_views/local_user",
   "crates/db_views/local_image",
@@ -113,6 +114,7 @@ lemmy_apub_objects = { version = "=1.0.0-alpha.5", path = "./crates/apub_objects
 lemmy_utils = { version = "=1.0.0-alpha.5", path = "./crates/utils", default-features = false }
 lemmy_db_schema = { version = "=1.0.0-alpha.5", path = "./crates/db_schema" }
 lemmy_db_schema_file = { version = "=1.0.0-alpha.5", path = "./crates/db_schema_file" }
+lemmy_db_schema_setup = { version = "=1.0.0-alpha.5", path = "./crates/db_schema_setup" }
 lemmy_api_utils = { version = "=1.0.0-alpha.5", path = "./crates/api/api_utils" }
 lemmy_routes = { version = "=1.0.0-alpha.5", path = "./crates/routes" }
 lemmy_federate = { version = "=1.0.0-alpha.5", path = "./crates/federate" }
@@ -228,7 +230,7 @@ lemmy_apub = { workspace = true }
 lemmy_apub_objects = { workspace = true }
 lemmy_utils = { workspace = true }
 lemmy_db_schema = { workspace = true }
-lemmy_db_schema_file = { workspace = true }
+lemmy_db_schema_setup = { workspace = true }
 lemmy_api_utils = { workspace = true }
 lemmy_routes = { workspace = true }
 lemmy_federate = { workspace = true }

crates/db_schema/Cargo.toml

@@ -38,6 +38,7 @@ full = [
   "tuplex",
   "moka",
   "lemmy_db_schema_file/full",
+  "lemmy_db_schema_setup",
 ]
 ts-rs = ["dep:ts-rs"]
 
@@ -51,6 +52,7 @@ serde_json = { workspace = true, optional = true }
 activitypub_federation = { workspace = true, optional = true }
 lemmy_utils = { workspace = true, optional = true }
 lemmy_db_schema_file = { workspace = true }
+lemmy_db_schema_setup = { workspace = true, optional = true }
 bcrypt = { workspace = true, optional = true }
 diesel = { workspace = true, optional = true }
 diesel-derive-newtype = { workspace = true, optional = true }

crates/db_schema/src/utils.rs

@@ -32,7 +32,6 @@ use diesel_async::{
 };
 use futures_util::{future::BoxFuture, FutureExt};
 use i_love_jesus::{CursorKey, PaginatedQueryBuilder, SortDirection};
-use lemmy_db_schema_file::schema_setup;
 use lemmy_utils::{
   error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult},
   settings::{structs::Settings, SETTINGS},
@@ -494,7 +493,7 @@ pub fn build_db_pool() -> LemmyResult<ActualDbPool> {
   // provide a setup function which handles creating the connection
   let mut config = ManagerConfig::default();
   config.custom_setup = Box::new(establish_connection);
-  let manager = AsyncDieselConnectionManager::<AsyncPgConnection>::new_with_config(db_url, config);
+  let manager = AsyncDieselConnectionManager::<AsyncPgConnection>::new_with_config(&db_url, config);
   let pool = Pool::builder(manager)
     .max_size(SETTINGS.database.pool_size)
     .runtime(Runtime::Tokio1)
@@ -512,7 +511,7 @@ pub fn build_db_pool() -> LemmyResult<ActualDbPool> {
     }))
     .build()?;
 
-  schema_setup::run(schema_setup::Options::default().run())?;
+  lemmy_db_schema_setup::run(lemmy_db_schema_setup::Options::default().run(), &db_url)?;
 
   Ok(pool)
 }

crates/db_schema_file/Cargo.toml

@@ -18,16 +18,7 @@ doctest = false
 workspace = true
 
 [features]
-full = [
-  "diesel",
-  "diesel_ltree",
-  "diesel-derive-enum",
-  "lemmy_utils",
-  "anyhow",
-  "chrono",
-  "tracing",
-  "diesel_migrations",
-]
+full = ["diesel", "diesel_ltree", "diesel-derive-enum"]
 ts-rs = ["dep:ts-rs"]
 
 [dependencies]
@@ -37,15 +28,3 @@ diesel = { workspace = true, optional = true }
 diesel_ltree = { workspace = true, optional = true }
 ts-rs = { workspace = true, optional = true }
 diesel-derive-enum = { workspace = true, optional = true }
-lemmy_utils = { workspace = true, features = ["full"], optional = true }
-anyhow = { workspace = true, optional = true }
-chrono = { workspace = true, optional = true }
-diesel_migrations = { workspace = true, optional = true }
-tracing = { workspace = true, optional = true }
-
-[dev-dependencies]
-serial_test = { workspace = true }
-diff = "0.1.13"
-itertools = { workspace = true }
-pathfinding = "4.14.0"
-unified-diff = { workspace = true }

crates/db_schema_file/src/lib.rs

@@ -1,7 +1,3 @@
-#[cfg(feature = "full")]
-pub mod diff_check;
 pub mod enums;
 #[cfg(feature = "full")]
 pub mod schema;
-#[cfg(feature = "full")]
-pub mod schema_setup;

crates/db_schema_file/src/schema.rs

@@ -973,13 +973,6 @@ diesel::table! {
     }
 }
 
-diesel::table! {
-    previously_run_sql (id) {
-        id -> Bool,
-        content -> Text,
-    }
-}
-
 diesel::table! {
     private_message (id) {
         id -> Int4,
@@ -1326,7 +1319,6 @@ diesel::allow_tables_to_appear_in_same_query!(
   post_actions,
   post_report,
   post_tag,
-  previously_run_sql,
   private_message,
   private_message_report,
   received_activity,

crates/db_schema_setup/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "lemmy_db_schema_setup"
+version.workspace = true
+edition.workspace = true
+description.workspace = true
+license.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[lib]
+name = "lemmy_db_schema_setup"
+path = "src/lib.rs"
+doctest = false
+
+[lints]
+workspace = true
+
+[features]
+full = []
+
+[dependencies]
+diesel = { workspace = true }
+chrono = { workspace = true }
+diesel_migrations = { workspace = true }
+tracing = { workspace = true }
+anyhow = { workspace = true }
+
+[dev-dependencies]
+serial_test = { workspace = true }
+diff = "0.1.13"
+itertools = { workspace = true }
+pathfinding = "4.14.0"
+unified-diff = { workspace = true }
+diesel_ltree = { workspace = true }
+lemmy_db_schema_file = { workspace = true, features = ["full"] }
+lemmy_utils = { workspace = true, features = ["full"] }

crates/db_schema_setup/src/lib.rs

@@ -1,6 +1,4 @@
-#[cfg(test)]
-use crate::diff_check;
-use crate::schema::previously_run_sql;
+mod diff_check;
 use anyhow::{anyhow, Context};
 use chrono::TimeDelta;
 use diesel::{
@@ -18,7 +16,6 @@ use diesel::{
   RunQueryDsl,
 };
 use diesel_migrations::MigrationHarness;
-use lemmy_utils::{error::LemmyResult, settings::SETTINGS};
 use std::time::Instant;
 use tracing::debug;
 
@@ -28,6 +25,13 @@ diesel::table! {
   }
 }
 
+diesel::table! {
+  previously_run_sql (id) {
+    id -> Bool,
+    content -> Text,
+  }
+}
+
 fn migrations() -> diesel_migrations::EmbeddedMigrations {
   // Using `const` here is required by the borrow checker
   const MIGRATIONS: diesel_migrations::EmbeddedMigrations = diesel_migrations::embed_migrations!();
@@ -46,7 +50,7 @@ fn replaceable_schema() -> String {
   .join("\n")
 }
 
-const REPLACEABLE_SCHEMA_PATH: &str = "crates/db_schema/replaceable_schema";
+const REPLACEABLE_SCHEMA_PATH: &str = "crates/db_schema_setup/replaceable_schema";
 
 struct MigrationHarnessWrapper<'a> {
   conn: &'a mut PgConnection,
@@ -178,11 +182,9 @@ pub enum Branch {
   ReplaceableSchemaNotRebuilt,
 }
 
-pub fn run(options: Options) -> LemmyResult<Branch> {
-  let db_url = SETTINGS.get_database_url();
-
+pub fn run(options: Options, db_url: &str) -> anyhow::Result<Branch> {
   // Migrations don't support async connection, and this function doesn't need to be async
-  let mut conn = PgConnection::establish(&db_url)?;
+  let mut conn = PgConnection::establish(db_url)?;
 
   // If possible, skip getting a lock and recreating the "r" schema, so
   // lemmy_server processes in a horizontally scaled setup can start without causing locks
@@ -233,7 +235,7 @@ pub fn run(options: Options) -> LemmyResult<Branch> {
 
       let after = diff_check::get_dump();
 
-      diff_check::check_dump_diff([&before, &after], "The code in crates/db_schema/replaceable_schema incorrectly created or modified things outside of the `r` schema, causing these changes to be left behind after dropping the schema:");
+      diff_check::check_dump_diff([&before, &after], "The code in crates/db_schema_setup/replaceable_schema incorrectly created or modified things outside of the `r` schema, causing these changes to be left behind after dropping the schema:");
 
       diff_check::deferr_constraint_check(&after);
     }
@@ -250,7 +252,7 @@ pub fn run(options: Options) -> LemmyResult<Branch> {
   Ok(output)
 }
 
-fn run_replaceable_schema(conn: &mut PgConnection) -> LemmyResult<()> {
+fn run_replaceable_schema(conn: &mut PgConnection) -> anyhow::Result<()> {
   conn.transaction(|conn| {
     conn
       .batch_execute(&replaceable_schema())
@@ -266,7 +268,7 @@ fn run_replaceable_schema(conn: &mut PgConnection) -> LemmyResult<()> {
   })
 }
 
-fn revert_replaceable_schema(conn: &mut PgConnection) -> LemmyResult<()> {
+fn revert_replaceable_schema(conn: &mut PgConnection) -> anyhow::Result<()> {
   conn
     .batch_execute("DROP SCHEMA IF EXISTS r CASCADE;")
     .with_context(|| format!("Failed to revert SQL files in {REPLACEABLE_SCHEMA_PATH}"))?;
@@ -384,7 +386,7 @@ mod tests {
 
     // Run initial migrations to prepare basic tables
     assert_eq!(
-      run(o.run().limit(INITIAL_MIGRATIONS_COUNT))?,
+      run(o.run().limit(INITIAL_MIGRATIONS_COUNT), &db_url)?,
       ReplaceableSchemaNotRebuilt
     );
 
@@ -392,16 +394,22 @@ mod tests {
     insert_test_data(&mut conn)?;
 
     // Run all migrations, and make sure that changes can be correctly reverted
-    assert_eq!(run(o.run().enable_diff_check())?, ReplaceableSchemaRebuilt);
+    assert_eq!(
+      run(o.run().enable_diff_check(), &db_url)?,
+      ReplaceableSchemaRebuilt
+    );
 
     // Check the test data we inserted before after running migrations
     check_test_data(&mut conn)?;
 
     // Check for early return
-    assert_eq!(run(o.run())?, EarlyReturn);
+    assert_eq!(run(o.run(), &db_url)?, EarlyReturn);
 
     // Test `limit`
-    assert_eq!(run(o.revert().limit(1))?, ReplaceableSchemaNotRebuilt);
+    assert_eq!(
+      run(o.revert().limit(1), &db_url)?,
+      ReplaceableSchemaNotRebuilt
+    );
     assert_eq!(
       conn
         .pending_migrations(migrations())
@@ -409,7 +417,7 @@ mod tests {
         .len(),
       1
     );
-    assert_eq!(run(o.run().limit(1))?, ReplaceableSchemaRebuilt);
+    assert_eq!(run(o.run().limit(1), &db_url)?, ReplaceableSchemaRebuilt);
 
     // This should throw an error saying to use lemmy_server instead of diesel CLI
     conn.batch_execute("DROP OWNED BY CURRENT_USER;")?;
@@ -419,7 +427,7 @@ mod tests {
     ));
 
     // Diesel CLI's way of running migrations shouldn't break the custom migration runner
-    assert_eq!(run(o.run())?, ReplaceableSchemaRebuilt);
+    assert_eq!(run(o.run(), &db_url)?, ReplaceableSchemaRebuilt);
 
     Ok(())
   }
@@ -511,7 +519,7 @@ mod tests {
   }
 
   fn check_test_data(conn: &mut PgConnection) -> LemmyResult<()> {
-    use crate::schema::{comment, comment_reply, community, person, post};
+    use lemmy_db_schema_file::schema::{comment, comment_reply, community, person, post};
 
     // Check users
     let users: Vec<(i32, String, Option<String>, String, String)> = person::table

crates/db_schema_setup/src/main.rs

@@ -0,0 +1,14 @@
+/// Very minimal wrapper around `lemmy_db_schema_setup::run` to allow running migrations without
+/// compiling everything.
+fn main() -> anyhow::Result<()> {
+  if std::env::args().len() > 1 {
+    anyhow::bail!("To set parameters for running migrations, use the lemmy_server command.");
+  }
+
+  lemmy_db_schema_setup::run(
+    lemmy_db_schema_setup::Options::default().run(),
+    &std::env::var("LEMMY_DATABASE_URL")?,
+  )?;
+
+  Ok(())
+}

diesel.toml

@@ -3,3 +3,5 @@ file = "crates/db_schema_file/src/schema.rs"
 patch_file = "crates/db_schema_file/diesel_ltree.patch"
 # Required for https://github.com/adwhit/diesel-derive-enum
 custom_type_derives = ["diesel::query_builder::QueryId"]
+# This table is in the db_schema_setup crate instead.
+filter = { except_tables = ["previously_run_sql"] }

scripts/dump_schema.sh

@@ -9,7 +9,7 @@ cd "$CWD/../"
 
 source scripts/start_dev_db.sh
 
-cargo run --package lemmy_server -- migration --all run
+cargo run --package lemmy_db_schema_setup
 pg_dump --no-owner --no-privileges --no-table-access-method --schema-only --exclude-schema=r --no-sync -f schema.sqldump
 
 pg_ctl stop

scripts/lint.sh

@@ -12,6 +12,6 @@ cargo +nightly fmt
 taplo format
 
 # Format sql files
-find migrations crates/db_schema_file/replaceable_schema -type f -name '*.sql' -print0 | xargs -0 -P 10 -L 10 pg_format -i
+find migrations crates/db_schema_setup/replaceable_schema -type f -name '*.sql' -print0 | xargs -0 -P 10 -L 10 pg_format -i
 
 cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-targets --all-features -- -D warnings

scripts/sql_format_check.sh

@@ -10,11 +10,11 @@ cd "$CWD/../"
 # Copy the files to a temp dir
 TMP_DIR=$(mktemp -d)
 cp -a migrations/. $TMP_DIR/migrations
-cp -a crates/db_schema_file/replaceable_schema/. $TMP_DIR/replaceable_schema
+cp -a crates/db_schema_setup/replaceable_schema/. $TMP_DIR/replaceable_schema
 
 # Format the new files
 find $TMP_DIR -type f -name '*.sql' -print0 | xargs -0 -P 10 -L 10 pg_format -i
 
 # Diff the directories
 diff -r migrations $TMP_DIR/migrations
-diff -r crates/db_schema_file/replaceable_schema $TMP_DIR/replaceable_schema
+diff -r crates/db_schema_setup/replaceable_schema $TMP_DIR/replaceable_schema

src/lib.rs

@@ -25,7 +25,6 @@ use lemmy_apub::{
 };
 use lemmy_apub_objects::objects::{community::FETCH_COMMUNITY_COLLECTIONS, instance::ApubSite};
 use lemmy_db_schema::{source::secret::Secret, utils::build_db_pool};
-use lemmy_db_schema_file::schema_setup;
 use lemmy_db_views_site::SiteView;
 use lemmy_federate::{Opts, SendManager};
 use lemmy_routes::{
@@ -130,7 +129,7 @@ enum CmdSubcommand {
   },
 }
 
-#[derive(Subcommand, Debug)]
+#[derive(Subcommand, Debug, PartialEq, Eq)]
 enum MigrationSubcommand {
   /// Run up.sql for pending migrations, oldest to newest.
   Run,
@@ -147,8 +146,8 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> {
   }) = args.subcommand
   {
     let mut options = match subcommand {
-      MigrationSubcommand::Run => schema_setup::Options::default().run(),
-      MigrationSubcommand::Revert => schema_setup::Options::default().revert(),
+      MigrationSubcommand::Run => lemmy_db_schema_setup::Options::default().run(),
+      MigrationSubcommand::Revert => lemmy_db_schema_setup::Options::default().revert(),
     }
     .print_output();
 
@@ -156,7 +155,12 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> {
       options = options.limit(number);
     }
 
-    schema_setup::run(options)?;
+    lemmy_db_schema_setup::run(options, &SETTINGS.get_database_url())?;
+
+    #[cfg(debug_assertions)]
+    if all && subcommand == MigrationSubcommand::Run {
+      println!("Warning: you probably want this command instead, which requires less crates to be compiled: cargo run --package lemmy_db_schema_setup");
+    }
 
     return Ok(());
   }