From 9ccd647e020ac6335639a4631422ee821eafd4cb Mon Sep 17 00:00:00 2001 From: Dessalines Date: Thu, 26 Jun 2025 03:48:39 -0400 Subject: [PATCH] Fixing docker_update.sh script to not add context. (#5820) * Fixing docker_update.sh script to not add context. Use docker build from a higher level to not load massive context. * Adding a custom postgresql for testing. * Fixing docker update * Fixing dir. * Fix rust log * Add bash / shell file format checking. * Adding shfmt package. * Running bash format. * Fixing bash fmt. * Fixing bash fmt 2. * Adding pgtune note. --- .woodpecker.yml | 9 +++ docker/customPostgresql.conf | 25 ++++++++ docker/docker-compose.yml | 7 ++- docker/docker_db_backup.sh | 2 +- docker/docker_update.sh | 28 +++++---- docker/federation/start-local-instances.bash | 2 +- scripts/compilation_benchmark.sh | 28 ++++----- scripts/db-init.sh | 41 +++++++------ scripts/install.sh | 36 +++++++----- scripts/query_testing/apache_bench_report.sh | 19 +++--- scripts/query_testing/api_benchmark.sh | 19 +++--- scripts/query_testing/post_query_hot_rank.sh | 2 +- .../views_old/generate_reports.sh | 46 +++++++-------- .../generate_reports.sh | 58 +++++++++---------- scripts/{release.sh => release.bash} | 0 scripts/start_dev_db.sh | 8 +-- scripts/update_config_defaults.sh | 2 +- 17 files changed, 192 insertions(+), 140 deletions(-) create mode 100644 docker/customPostgresql.conf rename scripts/{release.sh => release.bash} (100%) diff --git a/.woodpecker.yml b/.woodpecker.yml index 0bdab3fd9..fbbcc6fb4 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -48,6 +48,15 @@ steps: when: - event: pull_request + bash_fmt: + image: alpine:3 + commands: + - apk add shfmt + - shfmt -i 2 -d */**.bash + - shfmt -i 2 -d */**.sh + when: + - event: pull_request + toml_fmt: image: tamasfe/taplo:0.9.3 commands: diff --git a/docker/customPostgresql.conf b/docker/customPostgresql.conf new file mode 100644 index 000000000..3c2d5d49f --- /dev/null +++ b/docker/customPostgresql.conf @@ -0,0 +1,25 @@ +# You can use https://pgtune.leopard.in.ua to tune this for your system. +# DB Version: 16 +# OS Type: linux +# DB Type: web +# Total Memory (RAM): 12 GB +# CPUs num: 16 +# Data Storage: ssd + +max_connections = 200 +shared_buffers = 3GB +effective_cache_size = 9GB +maintenance_work_mem = 768MB +checkpoint_completion_target = 0.9 +wal_buffers = 16MB +default_statistics_target = 100 +random_page_cost = 1.1 +effective_io_concurrency = 200 +work_mem = 3932kB +huge_pages = try +min_wal_size = 1GB +max_wal_size = 4GB +max_worker_processes = 16 +max_parallel_workers_per_gather = 4 +max_parallel_workers = 16 +max_parallel_maintenance_workers = 4 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c1a67ab72..06986bd24 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,11 +22,13 @@ services: logging: *default-logging lemmy: - image: dessalines/lemmy:1.0.0-alpha.4 + build: + context: ../ + dockerfile: docker/Dockerfile hostname: lemmy restart: unless-stopped environment: - - RUST_LOG=extism=info,lemmy_api_common=info,error + - RUST_LOG=warn,extism=info,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug volumes: - ./lemmy.hjson:/config/config.hjson:Z - ./plugins:/plugins:Z @@ -109,5 +111,6 @@ services: - POSTGRES_DB=lemmy volumes: - ./volumes/postgres:/var/lib/postgresql/data:Z + - ./customPostgresql.conf:/etc/postgresql.conf:Z restart: unless-stopped logging: *default-logging diff --git a/docker/docker_db_backup.sh b/docker/docker_db_backup.sh index 8ec3e70c7..d453e28e1 100755 --- a/docker/docker_db_backup.sh +++ b/docker/docker_db_backup.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -docker-compose exec postgres pg_dumpall -c -U lemmy > dump_`date +%Y-%m-%d"_"%H_%M_%S`.sql +docker-compose exec postgres pg_dumpall -c -U lemmy >dump_$(date +%Y-%m-%d"_"%H_%M_%S).sql diff --git a/docker/docker_update.sh b/docker/docker_update.sh index 45411f1a3..768fa6459 100755 --- a/docker/docker_update.sh +++ b/docker/docker_update.sh @@ -1,8 +1,7 @@ #!/bin/sh set -e -Help() -{ +Help() { # Display help echo "Usage: ./docker_update.sh [OPTIONS]" echo "" @@ -15,12 +14,17 @@ Help() while getopts ":hu:" option; do case $option in - h) Help - exit;; - u) DOCKER_USER=$OPTARG - ;; - *) echo "Invalid option $OPTARG." - exit;; + h) + Help + exit + ;; + u) + DOCKER_USER=$OPTARG + ;; + *) + echo "Invalid option $OPTARG." + exit + ;; esac done @@ -37,11 +41,11 @@ if [ "$ARCH" = 'arm64' ]; then # You need a Docker account to pull images. Otherwise, you will get an error like: "error getting credentials" if [ -z "$DOCKER_USER" ]; then - echo "$LOG_PREFIX Logging into Docker Hub..." - docker login + echo "$LOG_PREFIX Logging into Docker Hub..." + docker login else - echo "$LOG_PREFIX Logging into Docker Hub. Please provide your personal access token." - docker login --username="$DOCKER_USER" + echo "$LOG_PREFIX Logging into Docker Hub. Please provide your personal access token." + docker login --username="$DOCKER_USER" fi echo "$LOG_PREFIX Initializing images in the background. Please be patient if compiling from source..." diff --git a/docker/federation/start-local-instances.bash b/docker/federation/start-local-instances.bash index 615790764..d150496ab 100755 --- a/docker/federation/start-local-instances.bash +++ b/docker/federation/start-local-instances.bash @@ -3,7 +3,7 @@ set -e sudo docker compose down -for Item in alpha beta gamma delta epsilon ; do +for Item in alpha beta gamma delta epsilon; do sudo mkdir -p volumes/pictrs_$Item sudo chown -R 991:991 volumes/pictrs_$Item done diff --git a/scripts/compilation_benchmark.sh b/scripts/compilation_benchmark.sh index 93d1b6a23..ae6cd7275 100755 --- a/scripts/compilation_benchmark.sh +++ b/scripts/compilation_benchmark.sh @@ -3,21 +3,21 @@ set -e times=3 duration=0 -for ((i=0; i < times; i++)) ; do - echo "Starting iteration $i" - echo "cargo clean" - # to benchmark incremental compilation time, do a full build with the same compiler version first, - # and use the following clean command: - cargo clean -p lemmy_utils - #cargo clean - echo "cargo build" - start=$(date +%s.%N) - RUSTC_WRAPPER='' cargo build -q - end=$(date +%s.%N) - echo "Finished iteration $i after $(bc <<< "scale=0; $end - $start") seconds" - duration=$(bc <<< "$duration + $end - $start") +for ((i = 0; i < times; i++)); do + echo "Starting iteration $i" + echo "cargo clean" + # to benchmark incremental compilation time, do a full build with the same compiler version first, + # and use the following clean command: + cargo clean -p lemmy_utils + #cargo clean + echo "cargo build" + start=$(date +%s.%N) + RUSTC_WRAPPER='' cargo build -q + end=$(date +%s.%N) + echo "Finished iteration $i after $(bc <<<"scale=0; $end - $start") seconds" + duration=$(bc <<<"$duration + $end - $start") done -average=$(bc <<< "scale=0; $duration / $times") +average=$(bc <<<"scale=0; $duration / $times") echo "Average compilation time over $times runs is $average seconds" diff --git a/scripts/db-init.sh b/scripts/db-init.sh index a63c003c1..4950f805b 100755 --- a/scripts/db-init.sh +++ b/scripts/db-init.sh @@ -25,40 +25,48 @@ ask_for_db_config() { default_config_final=0 default_config_valid=0 - while [ "$default_config_valid" == 0 ] - do + while [ "$default_config_valid" == 0 ]; do read -p "Use this configuration (y/n)? " default_config case "$default_config" in - [yY]* ) default_config_valid=1; default_config_final=1;; - [nN]* ) default_config_valid=1; default_config_final=0;; - * ) yes_no_prompt_invalid;; + [yY]*) + default_config_valid=1 + default_config_final=1 + ;; + [nN]*) + default_config_valid=1 + default_config_final=0 + ;; + *) yes_no_prompt_invalid ;; esac echo done - if [ "$default_config_final" == 0 ] - then + if [ "$default_config_final" == 0 ]; then config_ok_final=0 - while [ "$config_ok_final" == 0 ] - do + while [ "$config_ok_final" == 0 ]; do read -p "Database name: " dbname read -p "Username: " username read -p "Password: password" read -p "Port: " port #echo - + #echo "The database configuration is:" #print_config #echo - + config_ok_valid=0 - while [ "$config_ok_valid" == 0 ] - do + while [ "$config_ok_valid" == 0 ]; do read -p "Use this configuration (y/n)? " config_ok case "$config_ok" in - [yY]* ) config_ok_valid=1; config_ok_final=1;; - [nN]* ) config_ok_valid=1; config_ok_final=0;; - * ) yes_no_prompt_invalid;; + [yY]*) + config_ok_valid=1 + config_ok_final=1 + ;; + [nN]*) + config_ok_valid=1 + config_ok_final=0 + ;; + *) yes_no_prompt_invalid ;; esac echo done @@ -73,4 +81,3 @@ psql -c "CREATE DATABASE $dbname WITH OWNER $username;" -U postgres export LEMMY_DATABASE_URL=postgres://$username:$password@localhost:$port/$dbname echo "The database URL is $LEMMY_DATABASE_URL" - diff --git a/scripts/install.sh b/scripts/install.sh index 81a085121..8c1d35aec 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -17,18 +17,22 @@ yes_no_prompt_invalid() { ask_to_init_db() { init_db_valid=0 init_db_final=0 - while [ "$init_db_valid" == 0 ] - do + while [ "$init_db_valid" == 0 ]; do read -p "Initialize database (y/n)? " init_db case "$init_db" in - [yY]* ) init_db_valid=1; init_db_final=1;; - [nN]* ) init_db_valid=1; init_db_final=0;; - * ) yes_no_prompt_invalid;; + [yY]*) + init_db_valid=1 + init_db_final=1 + ;; + [nN]*) + init_db_valid=1 + init_db_final=0 + ;; + *) yes_no_prompt_invalid ;; esac echo done - if [ "$init_db_final" = 1 ] - then + if [ "$init_db_final" = 1 ]; then source ./db-init.sh read -n 1 -s -r -p "Press ANY KEY to continue execution of this script, press CTRL+C to quit..." echo @@ -38,20 +42,24 @@ ask_to_init_db() { ask_to_auto_reload() { auto_reload_valid=0 auto_reload_final=0 - while [ "$auto_reload_valid" == 0 ] - do + while [ "$auto_reload_valid" == 0 ]; do echo "Automagically reload the project when source files are changed?" echo "ONLY ENABLE THIS FOR DEVELOPMENT!" read -p "(y/n) " auto_reload case "$auto_reload" in - [yY]* ) auto_reload_valid=1; auto_reload_final=1;; - [nN]* ) auto_reload_valid=1; auto_reload_final=0;; - * ) yes_no_prompt_invalid;; + [yY]*) + auto_reload_valid=1 + auto_reload_final=1 + ;; + [nN]*) + auto_reload_valid=1 + auto_reload_final=0 + ;; + *) yes_no_prompt_invalid ;; esac echo done - if [ "$auto_reload_final" = 1 ] - then + if [ "$auto_reload_final" = 1 ]; then cd ui && pnpm dev cd server && cargo watch -x run fi diff --git a/scripts/query_testing/apache_bench_report.sh b/scripts/query_testing/apache_bench_report.sh index ae804a531..11b7c9e88 100755 --- a/scripts/query_testing/apache_bench_report.sh +++ b/scripts/query_testing/apache_bench_report.sh @@ -2,13 +2,13 @@ set -e declare -a arr=( -"https://mastodon.social/" -"https://peertube.social/" -"https://lemmy.ml/" -"https://lemmy.ml/feeds/all.xml" -"https://lemmy.ml/.well-known/nodeinfo" -"https://fediverse.blog/.well-known/nodeinfo" -"https://torrents-csv.ml/service/search?q=wheel&page=1&type_=torrent" + "https://mastodon.social/" + "https://peertube.social/" + "https://lemmy.ml/" + "https://lemmy.ml/feeds/all.xml" + "https://lemmy.ml/.well-known/nodeinfo" + "https://fediverse.blog/.well-known/nodeinfo" + "https://torrents-csv.ml/service/search?q=wheel&page=1&type_=torrent" ) ## check if ab installed @@ -18,9 +18,8 @@ if ! [ -x "$(command -v ab)" ]; then fi ## now loop through the above array -for i in "${arr[@]}" -do - ab -c 10 -t 10 "$i" > out.abtest +for i in "${arr[@]}"; do + ab -c 10 -t 10 "$i" >out.abtest grep "Server Hostname:" out.abtest grep "Document Path:" out.abtest grep "Requests per second" out.abtest diff --git a/scripts/query_testing/api_benchmark.sh b/scripts/query_testing/api_benchmark.sh index 4c1ee9e1f..08e4ebd21 100755 --- a/scripts/query_testing/api_benchmark.sh +++ b/scripts/query_testing/api_benchmark.sh @@ -6,13 +6,13 @@ set -e DOMAIN=${1:-"http://127.0.0.1:8536"} declare -a arr=( -"/api/v1/site" -"/api/v1/categories" -"/api/v1/modlog" -"/api/v1/search?q=test&type_=Posts&sort=Hot" -"/api/v1/community" -"/api/v1/community/list?sort=Hot" -"/api/v1/post/list?sort=Hot&type_=All" + "/api/v1/site" + "/api/v1/categories" + "/api/v1/modlog" + "/api/v1/search?q=test&type_=Posts&sort=Hot" + "/api/v1/community" + "/api/v1/community/list?sort=Hot" + "/api/v1/post/list?sort=Hot&type_=All" ) ## check if ab installed @@ -22,13 +22,12 @@ if ! [ -x "$(command -v ab)" ]; then fi ## now loop through the above array -for path in "${arr[@]}" -do +for path in "${arr[@]}"; do URL="$DOMAIN$path" printf "\n\n\n" echo "testing $URL" curl --show-error --fail --silent "$URL" >/dev/null - ab -c 64 -t 10 "$URL" > out.abtest + ab -c 64 -t 10 "$URL" >out.abtest grep "Server Hostname:" out.abtest grep "Document Path:" out.abtest grep "Requests per second" out.abtest diff --git a/scripts/query_testing/post_query_hot_rank.sh b/scripts/query_testing/post_query_hot_rank.sh index dad746065..790ea6a6e 100755 --- a/scripts/query_testing/post_query_hot_rank.sh +++ b/scripts/query_testing/post_query_hot_rank.sh @@ -1,3 +1,3 @@ #!/bin/bash -sudo docker exec -i docker-postgres-1 psql -Ulemmy -c "EXPLAIN (ANALYZE, COSTS, VERBOSE, BUFFERS, FORMAT JSON) SELECT post.id, post.name, post.url, post.body, post.creator_id, post.community_id, post.removed, post.locked, post.published, post.updated, post.deleted, post.nsfw, post.embed_title, post.embed_description, post.embed_video_url, post.thumbnail_url, post.ap_id, post.local, post.language_id, post.featured_community, post.featured_local, person.id, person.name, person.display_name, person.avatar, person.banned, person.published, person.updated, person.actor_id, person.bio, person.local, person.banner, person.deleted, person.inbox_url, person.shared_inbox_url, person.matrix_user_id, person.admin, person.bot_account, person.ban_expires, person.instance_id, community.id, community.name, community.title, community.description, community.removed, community.published, community.updated, community.deleted, community.nsfw, community.actor_id, community.local, community.icon, community.banner, community.hidden, community.posting_restricted_to_mods, community.instance_id, community_person_ban.id, community_person_ban.community_id, community_person_ban.person_id, community_person_ban.published, community_person_ban.expires, post_aggregates.id, post_aggregates.post_id, post_aggregates.comments, post_aggregates.score, post_aggregates.upvotes, post_aggregates.downvotes, post_aggregates.published, post_aggregates.newest_comment_time_necro, post_aggregates.newest_comment_time, post_aggregates.featured_community, post_aggregates.featured_local, community_follower.id, community_follower.community_id, community_follower.person_id, community_follower.published, community_follower.pending, post_saved.id, post_saved.post_id, post_saved.person_id, post_saved.published, post_read.id, post_read.post_id, post_read.person_id, post_read.published, person_block.id, person_block.person_id, person_block.target_id, person_block.published, post_like.score, coalesce((post_aggregates.comments - person_post_aggregates.read_comments), post_aggregates.comments) FROM ((((((((((((post INNER JOIN person ON (post.creator_id = person.id)) INNER JOIN community ON (post.community_id = community.id)) LEFT OUTER JOIN community_person_ban ON (((post.community_id = community_person_ban.community_id) AND (community_person_ban.person_id = post.creator_id)) AND ((community_person_ban.expires IS NULL) OR (community_person_ban.expires > CURRENT_TIMESTAMP)))) INNER JOIN post_aggregates ON (post_aggregates.post_id = post.id)) LEFT OUTER JOIN community_follower ON ((post.community_id = community_follower.community_id) AND (community_follower.person_id = '33517'))) LEFT OUTER JOIN post_saved ON ((post.id = post_saved.post_id) AND (post_saved.person_id = '33517'))) LEFT OUTER JOIN post_read ON ((post.id = post_read.post_id) AND (post_read.person_id = '33517'))) LEFT OUTER JOIN person_block ON ((post.creator_id = person_block.target_id) AND (person_block.person_id = '33517'))) LEFT OUTER JOIN community_block ON ((community.id = community_block.community_id) AND (community_block.person_id = '33517'))) LEFT OUTER JOIN post_like ON ((post.id = post_like.post_id) AND (post_like.person_id = '33517'))) LEFT OUTER JOIN person_post_aggregates ON ((post.id = person_post_aggregates.post_id) AND (person_post_aggregates.person_id = '33517'))) LEFT OUTER JOIN local_user_language ON ((post.language_id = local_user_language.language_id) AND (local_user_language.local_user_id = '11402'))) WHERE ((((((((((community_follower.person_id IS NOT NULL) AND (post.nsfw = 'f')) AND (community.nsfw = 'f')) AND (local_user_language.language_id IS NOT NULL)) AND (community_block.person_id IS NULL)) AND (person_block.person_id IS NULL)) AND (post.removed = 'f')) AND (post.deleted = 'f')) AND (community.removed = 'f')) AND (community.deleted = 'f')) ORDER BY post_aggregates.featured_local DESC , post_aggregates.hot_rank DESC LIMIT '40' OFFSET '0';" > query_results.json +sudo docker exec -i docker-postgres-1 psql -Ulemmy -c "EXPLAIN (ANALYZE, COSTS, VERBOSE, BUFFERS, FORMAT JSON) SELECT post.id, post.name, post.url, post.body, post.creator_id, post.community_id, post.removed, post.locked, post.published, post.updated, post.deleted, post.nsfw, post.embed_title, post.embed_description, post.embed_video_url, post.thumbnail_url, post.ap_id, post.local, post.language_id, post.featured_community, post.featured_local, person.id, person.name, person.display_name, person.avatar, person.banned, person.published, person.updated, person.actor_id, person.bio, person.local, person.banner, person.deleted, person.inbox_url, person.shared_inbox_url, person.matrix_user_id, person.admin, person.bot_account, person.ban_expires, person.instance_id, community.id, community.name, community.title, community.description, community.removed, community.published, community.updated, community.deleted, community.nsfw, community.actor_id, community.local, community.icon, community.banner, community.hidden, community.posting_restricted_to_mods, community.instance_id, community_person_ban.id, community_person_ban.community_id, community_person_ban.person_id, community_person_ban.published, community_person_ban.expires, post_aggregates.id, post_aggregates.post_id, post_aggregates.comments, post_aggregates.score, post_aggregates.upvotes, post_aggregates.downvotes, post_aggregates.published, post_aggregates.newest_comment_time_necro, post_aggregates.newest_comment_time, post_aggregates.featured_community, post_aggregates.featured_local, community_follower.id, community_follower.community_id, community_follower.person_id, community_follower.published, community_follower.pending, post_saved.id, post_saved.post_id, post_saved.person_id, post_saved.published, post_read.id, post_read.post_id, post_read.person_id, post_read.published, person_block.id, person_block.person_id, person_block.target_id, person_block.published, post_like.score, coalesce((post_aggregates.comments - person_post_aggregates.read_comments), post_aggregates.comments) FROM ((((((((((((post INNER JOIN person ON (post.creator_id = person.id)) INNER JOIN community ON (post.community_id = community.id)) LEFT OUTER JOIN community_person_ban ON (((post.community_id = community_person_ban.community_id) AND (community_person_ban.person_id = post.creator_id)) AND ((community_person_ban.expires IS NULL) OR (community_person_ban.expires > CURRENT_TIMESTAMP)))) INNER JOIN post_aggregates ON (post_aggregates.post_id = post.id)) LEFT OUTER JOIN community_follower ON ((post.community_id = community_follower.community_id) AND (community_follower.person_id = '33517'))) LEFT OUTER JOIN post_saved ON ((post.id = post_saved.post_id) AND (post_saved.person_id = '33517'))) LEFT OUTER JOIN post_read ON ((post.id = post_read.post_id) AND (post_read.person_id = '33517'))) LEFT OUTER JOIN person_block ON ((post.creator_id = person_block.target_id) AND (person_block.person_id = '33517'))) LEFT OUTER JOIN community_block ON ((community.id = community_block.community_id) AND (community_block.person_id = '33517'))) LEFT OUTER JOIN post_like ON ((post.id = post_like.post_id) AND (post_like.person_id = '33517'))) LEFT OUTER JOIN person_post_aggregates ON ((post.id = person_post_aggregates.post_id) AND (person_post_aggregates.person_id = '33517'))) LEFT OUTER JOIN local_user_language ON ((post.language_id = local_user_language.language_id) AND (local_user_language.local_user_id = '11402'))) WHERE ((((((((((community_follower.person_id IS NOT NULL) AND (post.nsfw = 'f')) AND (community.nsfw = 'f')) AND (local_user_language.language_id IS NOT NULL)) AND (community_block.person_id IS NULL)) AND (person_block.person_id IS NULL)) AND (post.removed = 'f')) AND (post.deleted = 'f')) AND (community.removed = 'f')) AND (community.deleted = 'f')) ORDER BY post_aggregates.featured_local DESC , post_aggregates.hot_rank DESC LIMIT '40' OFFSET '0';" >query_results.json diff --git a/scripts/query_testing/views_old/generate_reports.sh b/scripts/query_testing/views_old/generate_reports.sh index f52653f2d..1a3b2acb2 100755 --- a/scripts/query_testing/views_old/generate_reports.sh +++ b/scripts/query_testing/views_old/generate_reports.sh @@ -9,40 +9,40 @@ pushd reports PSQL_CMD="docker exec -i dev_postgres_1 psql -qAt -U lemmy" -echo "explain (analyze, format json) select * from user_fast limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > user_fast.json +echo "explain (analyze, format json) select * from user_fast limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >user_fast.json -echo "explain (analyze, format json) select * from post_view where user_id is null order by hot_rank desc, published desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post_view.json +echo "explain (analyze, format json) select * from post_view where user_id is null order by hot_rank desc, published desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post_view.json -echo "explain (analyze, format json) select * from post_fast_view where user_id is null order by hot_rank desc, published desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post_fast_view.json +echo "explain (analyze, format json) select * from post_fast_view where user_id is null order by hot_rank desc, published desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post_fast_view.json -echo "explain (analyze, format json) select * from comment_view where user_id is null limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > comment_view.json +echo "explain (analyze, format json) select * from comment_view where user_id is null limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >comment_view.json -echo "explain (analyze, format json) select * from comment_fast_view where user_id is null limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > comment_fast_view.json +echo "explain (analyze, format json) select * from comment_fast_view where user_id is null limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >comment_fast_view.json -echo "explain (analyze, format json) select * from community_view where user_id is null order by hot_rank desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > community_view.json +echo "explain (analyze, format json) select * from community_view where user_id is null order by hot_rank desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >community_view.json -echo "explain (analyze, format json) select * from community_fast_view where user_id is null order by hot_rank desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > community_fast_view.json +echo "explain (analyze, format json) select * from community_fast_view where user_id is null order by hot_rank desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >community_fast_view.json -echo "explain (analyze, format json) select * from site_view limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > site_view.json +echo "explain (analyze, format json) select * from site_view limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >site_view.json -echo "explain (analyze, format json) select * from reply_fast_view where user_id = 34 and recipient_id = 34 limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > reply_fast_view.json +echo "explain (analyze, format json) select * from reply_fast_view where user_id = 34 and recipient_id = 34 limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >reply_fast_view.json -echo "explain (analyze, format json) select * from user_mention_view where user_id = 34 and recipient_id = 34 limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > user_mention_view.json +echo "explain (analyze, format json) select * from user_mention_view where user_id = 34 and recipient_id = 34 limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >user_mention_view.json -echo "explain (analyze, format json) select * from user_mention_fast_view where user_id = 34 and recipient_id = 34 limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > user_mention_fast_view.json +echo "explain (analyze, format json) select * from user_mention_fast_view where user_id = 34 and recipient_id = 34 limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >user_mention_fast_view.json -grep "Execution Time" *.json > ../timings-`date +%Y-%m-%d_%H-%M-%S`.out +grep "Execution Time" *.json >../timings-$(date +%Y-%m-%d_%H-%M-%S).out rm explain.sql diff --git a/scripts/query_testing/views_to_diesel_migration/generate_reports.sh b/scripts/query_testing/views_to_diesel_migration/generate_reports.sh index aa443c1b1..fcaf03859 100755 --- a/scripts/query_testing/views_to_diesel_migration/generate_reports.sh +++ b/scripts/query_testing/views_to_diesel_migration/generate_reports.sh @@ -7,49 +7,49 @@ pushd reports PSQL_CMD="docker exec -i dev_postgres_1 psql -qAt -U lemmy" -echo "explain (analyze, format json) select * from user_ limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > user_.json +echo "explain (analyze, format json) select * from user_ limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >user_.json -echo "explain (analyze, format json) select * from post p limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post.json +echo "explain (analyze, format json) select * from post p limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post.json -echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by hot_rank(pa.score, pa.published) desc, pa.published desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post_ordered_by_rank.json +echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by hot_rank(pa.score, pa.published) desc, pa.published desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post_ordered_by_rank.json -echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.stickied desc, hot_rank(pa.score, pa.published) desc, pa.published desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post_ordered_by_stickied_then_rank.json +echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.stickied desc, hot_rank(pa.score, pa.published) desc, pa.published desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post_ordered_by_stickied_then_rank.json -echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.score desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post_ordered_by_score.json +echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.score desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post_ordered_by_score.json -echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.stickied desc, pa.score desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post_ordered_by_stickied_then_score.json +echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.stickied desc, pa.score desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post_ordered_by_stickied_then_score.json -echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.published desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post_ordered_by_published.json +echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.published desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post_ordered_by_published.json -echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.stickied desc, pa.published desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > post_ordered_by_stickied_then_published.json +echo "explain (analyze, format json) select * from post p, post_aggregates pa where p.id = pa.post_id order by pa.stickied desc, pa.published desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >post_ordered_by_stickied_then_published.json -echo "explain (analyze, format json) select * from comment limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > comment.json +echo "explain (analyze, format json) select * from comment limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >comment.json -echo "explain (analyze, format json) select * from community limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > community.json +echo "explain (analyze, format json) select * from community limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >community.json -echo "explain (analyze, format json) select * from community c, community_aggregates ca where c.id = ca.community_id order by hot_rank(ca.subscribers, ca.published) desc, ca.published desc limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > community_ordered_by_subscribers.json +echo "explain (analyze, format json) select * from community c, community_aggregates ca where c.id = ca.community_id order by hot_rank(ca.subscribers, ca.published) desc, ca.published desc limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >community_ordered_by_subscribers.json -echo "explain (analyze, format json) select * from site s" > explain.sql -cat explain.sql | $PSQL_CMD > site.json +echo "explain (analyze, format json) select * from site s" >explain.sql +cat explain.sql | $PSQL_CMD >site.json -echo "explain (analyze, format json) select * from user_mention limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > user_mention.json +echo "explain (analyze, format json) select * from user_mention limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >user_mention.json -echo "explain (analyze, format json) select * from private_message limit 100" > explain.sql -cat explain.sql | $PSQL_CMD > private_message.json +echo "explain (analyze, format json) select * from private_message limit 100" >explain.sql +cat explain.sql | $PSQL_CMD >private_message.json -grep "Execution Time" *.json > ../timings-`date +%Y-%m-%d_%H-%M-%S`.out +grep "Execution Time" *.json >../timings-$(date +%Y-%m-%d_%H-%M-%S).out rm explain.sql diff --git a/scripts/release.sh b/scripts/release.bash similarity index 100% rename from scripts/release.sh rename to scripts/release.bash diff --git a/scripts/start_dev_db.sh b/scripts/start_dev_db.sh index 1a992cb01..b03c7f441 100644 --- a/scripts/start_dev_db.sh +++ b/scripts/start_dev_db.sh @@ -12,13 +12,11 @@ export LEMMY_DATABASE_URL=$DATABASE_URL export PGDATABASE=lemmy # If cluster exists, stop the server and delete the cluster -if [[ -d $PGDATA ]] -then +if [[ -d $PGDATA ]]; then # Only stop server if it is running pg_status_exit_code=0 - (pg_ctl status > /dev/null) || pg_status_exit_code=$? - if [[ ${pg_status_exit_code} -ne 3 ]] - then + (pg_ctl status >/dev/null) || pg_status_exit_code=$? + if [[ ${pg_status_exit_code} -ne 3 ]]; then pg_ctl stop --silent fi diff --git a/scripts/update_config_defaults.sh b/scripts/update_config_defaults.sh index de7fa6479..c7eb3834f 100755 --- a/scripts/update_config_defaults.sh +++ b/scripts/update_config_defaults.sh @@ -3,4 +3,4 @@ set -e dest=${1-config/defaults.hjson} -cargo run --manifest-path crates/utils/Cargo.toml --features full > "$dest" +cargo run --manifest-path crates/utils/Cargo.toml --features full >"$dest"