Speeding up comment-ltree migration, fixing index creation. Fixes #2664 (#2670)

* Speeding up comment-ltree migration, fixing index creation. Fixes #2664

* Adding some logging lines, fixing for missing posts.

* Adding more postgres config
This commit is contained in:
Dessalines 2023-01-23 09:59:25 -05:00 committed by GitHub
parent 1eaf2c8a03
commit 70c549dad8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 47 additions and 4 deletions

2
Cargo.lock generated
View file

@ -2488,6 +2488,8 @@ dependencies = [
"strum", "strum",
"strum_macros", "strum_macros",
"tokio", "tokio",
"tracing",
"tracing-error",
"typed-builder", "typed-builder",
"url", "url",
] ]

View file

@ -40,6 +40,8 @@ typed-builder = { workspace = true }
async-trait = { workspace = true } async-trait = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
bb8 = { version = "0.8.0", optional = true } bb8 = { version = "0.8.0", optional = true }
tracing = { workspace = true }
tracing-error = { workspace = true }
[dev-dependencies] [dev-dependencies]
serial_test = { workspace = true } serial_test = { workspace = true }

View file

@ -26,6 +26,7 @@ use lemmy_utils::{error::LemmyError, settings::structs::Settings};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use std::{env, env::VarError}; use std::{env, env::VarError};
use tracing::info;
use url::Url; use url::Url;
const FETCH_LIMIT_DEFAULT: i64 = 10; const FETCH_LIMIT_DEFAULT: i64 = 10;
@ -154,9 +155,11 @@ pub fn run_migrations(db_url: &str) {
// Needs to be a sync connection // Needs to be a sync connection
let mut conn = let mut conn =
PgConnection::establish(db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)); PgConnection::establish(db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url));
info!("Running Database migrations (This may take a long time)...");
let _ = &mut conn let _ = &mut conn
.run_pending_migrations(MIGRATIONS) .run_pending_migrations(MIGRATIONS)
.unwrap_or_else(|_| panic!("Couldn't run DB Migrations")); .unwrap_or_else(|_| panic!("Couldn't run DB Migrations"));
info!("Database migrations complete.");
} }
pub async fn build_db_pool(settings: &Settings) -> Result<DbPool, LemmyError> { pub async fn build_db_pool(settings: &Settings) -> Result<DbPool, LemmyError> {

View file

@ -104,17 +104,18 @@ services:
# "-c", "effective_cache_size=9GB", # "-c", "effective_cache_size=9GB",
# "-c", "maintenance_work_mem=768MB", # "-c", "maintenance_work_mem=768MB",
# "-c", "checkpoint_completion_target=0.9", # "-c", "checkpoint_completion_target=0.9",
# "-c", "checkpoint_timeout=86400",
# "-c", "wal_buffers=16MB", # "-c", "wal_buffers=16MB",
# "-c", "default_statistics_target=100", # "-c", "default_statistics_target=100",
# "-c", "random_page_cost=4", # "-c", "random_page_cost=4",
# "-c", "effective_io_concurrency=2", # "-c", "effective_io_concurrency=2",
# "-c", "work_mem=7864kB", # "-c", "work_mem=4GB",
# "-c", "min_wal_size=1GB", # "-c", "min_wal_size=1GB",
# "-c", "max_wal_size=4GB", # "-c", "max_wal_size=30GB",
# "-c", "max_worker_processes=4", # "-c", "max_worker_processes=4",
# "-c", "max_parallel_workers_per_gather=2", # "-c", "max_parallel_workers_per_gather=2",
# "-c", "max_parallel_workers=4", # "-c", "max_parallel_workers=4",
# "-c", "max_parallel_maintenance_workers=2", # "-c", "max_parallel_maintenance_workers=2"
] ]
networks: networks:
- lemmyinternal - lemmyinternal

View file

@ -1,4 +1,3 @@
-- Remove the comment.read column, and create a new comment_reply table, -- Remove the comment.read column, and create a new comment_reply table,
-- similar to the person_mention table. -- similar to the person_mention table.
-- --
@ -60,6 +59,19 @@ FROM q
ORDER BY ORDER BY
breadcrumb; breadcrumb;
-- Remove indexes and foreign key constraints, and disable triggers for faster updates
alter table comment disable trigger all;
alter table comment drop constraint if exists comment_creator_id_fkey;
alter table comment drop constraint if exists comment_parent_id_fkey;
alter table comment drop constraint if exists comment_post_id_fkey;
alter table comment drop constraint if exists idx_comment_ap_id;
drop index if exists idx_comment_creator;
drop index if exists idx_comment_parent;
drop index if exists idx_comment_post;
drop index if exists idx_comment_published;
-- Add the ltree column -- Add the ltree column
update comment c update comment c
set path = ct.ltree_path set path = ct.ltree_path
@ -75,9 +87,32 @@ from (
) as c2 ) as c2
where ca.comment_id = c2.id; where ca.comment_id = c2.id;
-- Delete comments at a depth of > 150, otherwise the index creation below will fail
delete from comment where nlevel(path) > 150;
-- Delete from comment where there is a missing post
delete from comment c where not exists (
select from post p where p.id = c.post_id
);
-- Delete from comment where there is a missing creator_id
delete from comment c where not exists (
select from person p where p.id = c.creator_id
);
-- Re-enable old constraints and indexes
alter table comment add constraint "comment_creator_id_fkey" FOREIGN KEY (creator_id) REFERENCES person(id) ON UPDATE CASCADE ON DELETE CASCADE;
alter table comment add constraint "comment_post_id_fkey" FOREIGN KEY (post_id) REFERENCES post(id) ON UPDATE CASCADE ON DELETE CASCADE;
alter table comment add constraint "idx_comment_ap_id" unique (ap_id);
create index idx_comment_creator on comment (creator_id);
create index idx_comment_post on comment (post_id);
create index idx_comment_published on comment (published desc);
-- Create the index -- Create the index
create index idx_path_gist on comment using gist (path); create index idx_path_gist on comment using gist (path);
-- Drop the parent_id column -- Drop the parent_id column
alter table comment drop column parent_id cascade; alter table comment drop column parent_id cascade;
alter table comment enable trigger all;