diff --git a/RELEASES.md b/RELEASES.md index 5a4c7645e..25c30861a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,37 @@ +# Lemmy v0.7.0 Release (2020-06-2X) + +## Breaking Change to our image server: Pictshare to Pict-rs migration guide + +This release replaces [pictshare](https://github.com/HaschekSolutions/pictshare) with [pict-rs](https://git.asonix.dog/asonix/pict-rs), and a script must be run on your server to upgrade. + +To update, run: + +``` +cd /lemmy +wget https://raw.githubusercontent.com/dessalines/lemmy/master/docker/prod/docker-compose.yml +wget https://raw.githubusercontent.com/dessalines/lemmy/master/docker/prod/migrate-pictshare-to-pictrs.bash +sudo bash migrate-pictshare-to-pictrs.bash +``` + +You'll also have to update your nginx config, use the [one here](https://github.com/LemmyNet/lemmy/blob/master/ansible/templates/nginx.conf). + +*You'll have to log in again to pick up your avatar* + +Apart from that, we've closed [~90 issues!](https://github.com/LemmyNet/lemmy/milestone/16?closed=1), including: + +- Site-wide list of recent comments. +- Reconnecting websockets. +- Lots more themes, including a default light one. +- Expandable embeds for post links (and thumbnails), from iframely. +- Better icons. +- Emoji autocomplete to post and message bodies, and an Emoji Picker. +- Post body now searchable. +- Community title and description is now searchable. +- Simplified cross-posts. +- Better documentation. +- LOTS more languages. +- Lots of bugs squashed. + # Lemmy v0.6.0 Release (2020-01-16) `v0.6.0` is here, and we've closed [41 issues!](https://github.com/LemmyNet/lemmy/milestone/15?closed=1) diff --git a/ansible/VERSION b/ansible/VERSION index c5ee9aebd..ed2321280 100644 --- a/ansible/VERSION +++ b/ansible/VERSION @@ -1 +1 @@ -v0.6.77 +v0.6.79 diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 960a7c40f..74b6ab2f1 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -1,5 +1,6 @@ [defaults] inventory=inventory +interpreter_python=/usr/bin/python3 [ssh_connection] pipelining = True diff --git a/ansible/lemmy.yml b/ansible/lemmy.yml index bc01623fc..7b78ab8d3 100644 --- a/ansible/lemmy.yml +++ b/ansible/lemmy.yml @@ -24,10 +24,11 @@ creates: '/etc/letsencrypt/live/{{domain}}/privkey.pem' - name: create lemmy folder - file: path={{item.path}} state=directory + file: path={{item.path}} {{item.owner}} state=directory with_items: - - { path: '/lemmy/' } - - { path: '/lemmy/volumes/' } + - { path: '/lemmy/', owner: 'root' } + - { path: '/lemmy/volumes/', owner: 'root' } + - { path: '/lemmy/volumes/pictrs/', owner: '991' } - block: - name: add template files @@ -59,6 +60,7 @@ project_src: /lemmy/ state: present pull: yes + remove_orphans: yes - name: reload nginx with new config shell: nginx -s reload diff --git a/ansible/lemmy_dev.yml b/ansible/lemmy_dev.yml index e9b8364f3..7a3683610 100644 --- a/ansible/lemmy_dev.yml +++ b/ansible/lemmy_dev.yml @@ -26,10 +26,11 @@ creates: '/etc/letsencrypt/live/{{domain}}/privkey.pem' - name: create lemmy folder - file: path={{item.path}} state=directory + file: path={{item.path}} owner={{item.owner}} state=directory with_items: - - { path: '/lemmy/' } - - { path: '/lemmy/volumes/' } + - { path: '/lemmy/', owner: 'root' } + - { path: '/lemmy/volumes/', owner: 'root' } + - { path: '/lemmy/volumes/pictrs/', owner: '991' } - block: - name: add template files @@ -88,6 +89,7 @@ project_src: /lemmy/ state: present recreate: always + remove_orphans: yes ignore_errors: yes - name: reload nginx with new config diff --git a/ansible/templates/docker-compose.yml b/ansible/templates/docker-compose.yml index 9ec1bfbc2..f4c94fd71 100644 --- a/ansible/templates/docker-compose.yml +++ b/ansible/templates/docker-compose.yml @@ -12,7 +12,7 @@ services: - ./lemmy.hjson:/config/config.hjson:ro depends_on: - postgres - - pictshare + - pictrs - iframely postgres: @@ -25,12 +25,13 @@ services: - ./volumes/postgres:/var/lib/postgresql/data restart: always - pictshare: - image: hascheksolutions/pictshare:latest + pictrs: + image: asonix/pictrs:amd64-v0.1.0-r9 + user: 991:991 ports: - - "127.0.0.1:8537:80" + - "127.0.0.1:8537:8080" volumes: - - ./volumes/pictshare:/usr/share/nginx/html/data + - ./volumes/pictrs:/mnt restart: always iframely: diff --git a/ansible/templates/nginx.conf b/ansible/templates/nginx.conf index a978c1899..b710fdb30 100644 --- a/ansible/templates/nginx.conf +++ b/ansible/templates/nginx.conf @@ -48,8 +48,8 @@ server { add_header X-Frame-Options "DENY"; add_header X-XSS-Protection "1; mode=block"; - # Upload limit for pictshare - client_max_body_size 50M; + # Upload limit for pictrs + client_max_body_size 20M; location / { proxy_pass http://0.0.0.0:8536; @@ -70,15 +70,21 @@ server { proxy_cache_min_uses 5; } - location /pictshare/ { - proxy_pass http://0.0.0.0:8537/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # Redirect pictshare images to pictrs + location ~ /pictshare/(.*)$ { + return 301 /pictrs/image/$1; + } - if ($request_uri ~ \.(?:ico|gif|jpe?g|png|webp|bmp|mp4)$) { - add_header Cache-Control "public, max-age=31536000, immutable"; - } + # pict-rs images + location /pictrs { + location /pictrs/image { + proxy_pass http://0.0.0.0:8537/image; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + # Block the import + return 403; } location /iframely/ { diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index 1702f66d3..bdcb4308a 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -1,15 +1,6 @@ version: '3.3' services: - postgres: - image: postgres:12-alpine - environment: - - POSTGRES_USER=lemmy - - POSTGRES_PASSWORD=password - - POSTGRES_DB=lemmy - volumes: - - ./volumes/postgres:/var/lib/postgresql/data - restart: always lemmy: build: @@ -23,16 +14,27 @@ services: volumes: - ../lemmy.hjson:/config/config.hjson depends_on: + - pictrs - postgres - - pictshare - iframely - pictshare: - image: hascheksolutions/pictshare:latest - ports: - - "127.0.0.1:8537:80" + postgres: + image: postgres:12-alpine + environment: + - POSTGRES_USER=lemmy + - POSTGRES_PASSWORD=password + - POSTGRES_DB=lemmy volumes: - - ./volumes/pictshare:/usr/share/nginx/html/data + - ./volumes/postgres:/var/lib/postgresql/data + restart: always + + pictrs: + image: asonix/pictrs:v0.1.13-r0 + ports: + - "127.0.0.1:8537:8080" + user: 991:991 + volumes: + - ./volumes/pictrs:/mnt restart: always iframely: diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index eab5cab21..863ff593b 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -12,7 +12,7 @@ services: restart: always lemmy: - image: dessalines/lemmy:v0.6.77 + image: dessalines/lemmy:v0.6.79 ports: - "127.0.0.1:8536:8536" restart: always @@ -22,17 +22,17 @@ services: - ./lemmy.hjson:/config/config.hjson depends_on: - postgres - - pictshare + - pictrs - iframely - pictshare: - image: hascheksolutions/pictshare:latest - ports: - - "127.0.0.1:8537:80" + pictrs: + image: asonix/pictrs:v0.1.13-r0 + ports: + - "127.0.0.1:8537:8080" + user: 991:991 volumes: - - ./volumes/pictshare:/usr/share/nginx/html/data + - ./volumes/pictrs:/mnt restart: always - mem_limit: 100m iframely: image: dogbin/iframely:latest diff --git a/docker/prod/migrate-pictshare-to-pictrs.bash b/docker/prod/migrate-pictshare-to-pictrs.bash new file mode 100644 index 000000000..8229eb28c --- /dev/null +++ b/docker/prod/migrate-pictshare-to-pictrs.bash @@ -0,0 +1,60 @@ +#!/bin/bash +set -e + +if [[ $(id -u) != 0 ]]; then + echo "This migration needs to be run as root" + exit +fi + +if [[ ! -f docker-compose.yml ]]; then + echo "No docker-compose.yml found in current directory. Is this the right folder?" + exit +fi + +# Fixing pictrs permissions +mkdir -p volumes/pictrs +sudo chown -R 991:991 volumes/pictrs + +echo "Restarting docker-compose, making sure that pictrs is started and pictshare is removed" +docker-compose up -d --remove-orphans + +if [[ -z $(docker-compose ps | grep pictrs) ]]; then + echo "Pict-rs is not running, make sure you update Lemmy first" + exit +fi + +# echo "Stopping Lemmy so that users dont upload new images during the migration" +# docker-compose stop lemmy + +pushd volumes/pictshare/ +echo "Importing pictshare images to pict-rs..." +IMAGE_NAMES=* +for image in $IMAGE_NAMES; do + IMAGE_PATH="$(pwd)/$image/$image" + if [[ ! -f $IMAGE_PATH ]]; then + continue + fi + echo -e "\nImporting $IMAGE_PATH" + ret=0 + curl --silent --fail -F "images[]=@$IMAGE_PATH" http://127.0.0.1:8537/import || ret=$? + if [[ $ret != 0 ]]; then + echo "Error for $IMAGE_PATH : $ret" + fi +done + +echo "Fixing permissions on pictshare folder" +find . -type d -exec chmod 755 {} \; +find . -type f -exec chmod 644 {} \; + +popd + +echo "Rewrite image links in Lemmy database" +docker-compose exec -u postgres postgres psql -U lemmy -c "UPDATE user_ SET avatar = REPLACE(avatar, 'pictshare', 'pictrs/image') WHERE avatar is not null;" +docker-compose exec -u postgres postgres psql -U lemmy -c "UPDATE post SET url = REPLACE(url, 'pictshare', 'pictrs/image') WHERE url is not null;" + +echo "Moving pictshare data folder to pictshare_backup" +mv volumes/pictshare volumes/pictshare_backup + +echo "Migration done, starting Lemmy again" +echo "If everything went well, you can delete ./volumes/pictshare_backup/" +docker-compose start lemmy diff --git a/server/src/api/community.rs b/server/src/api/community.rs index df03546cf..618122b98 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -1,4 +1,5 @@ use super::*; +use crate::is_valid_community_name; #[derive(Serialize, Deserialize)] pub struct GetCommunity { @@ -220,6 +221,10 @@ impl Perform for Oper { } } + if !is_valid_community_name(&data.name) { + return Err(APIError::err("invalid_community_name").into()); + } + let user_id = claims.id; let conn = pool.get()?; @@ -306,6 +311,10 @@ impl Perform for Oper { Err(_e) => return Err(APIError::err("not_logged_in").into()), }; + if !is_valid_community_name(&data.name) { + return Err(APIError::err("invalid_community_name").into()); + } + let user_id = claims.id; let conn = pool.get()?; diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 3488a8c42..4f11b3278 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -18,7 +18,7 @@ use crate::db::user_mention_view::*; use crate::db::user_view::*; use crate::db::*; use crate::{ - extract_usernames, fetch_iframely_and_pictshare_data, generate_random_string, naive_from_unix, + extract_usernames, fetch_iframely_and_pictrs_data, generate_random_string, naive_from_unix, naive_now, remove_slurs, send_email, slur_check, slurs_vec_to_str, }; diff --git a/server/src/api/post.rs b/server/src/api/post.rs index 84ef89f16..9eeb51580 100644 --- a/server/src/api/post.rs +++ b/server/src/api/post.rs @@ -116,9 +116,9 @@ impl Perform for Oper { return Err(APIError::err("site_ban").into()); } - // Fetch Iframely and Pictshare cached image - let (iframely_title, iframely_description, iframely_html, pictshare_thumbnail) = - fetch_iframely_and_pictshare_data(data.url.to_owned()); + // Fetch Iframely and pictrs cached image + let (iframely_title, iframely_description, iframely_html, pictrs_thumbnail) = + fetch_iframely_and_pictrs_data(data.url.to_owned()); let post_form = PostForm { name: data.name.to_owned(), @@ -135,7 +135,7 @@ impl Perform for Oper { embed_title: iframely_title, embed_description: iframely_description, embed_html: iframely_html, - thumbnail_url: pictshare_thumbnail, + thumbnail_url: pictrs_thumbnail, }; let inserted_post = match Post::create(&conn, &post_form) { @@ -450,9 +450,9 @@ impl Perform for Oper { return Err(APIError::err("site_ban").into()); } - // Fetch Iframely and Pictshare cached image - let (iframely_title, iframely_description, iframely_html, pictshare_thumbnail) = - fetch_iframely_and_pictshare_data(data.url.to_owned()); + // Fetch Iframely and Pictrs cached image + let (iframely_title, iframely_description, iframely_html, pictrs_thumbnail) = + fetch_iframely_and_pictrs_data(data.url.to_owned()); let post_form = PostForm { name: data.name.to_owned(), @@ -469,7 +469,7 @@ impl Perform for Oper { embed_title: iframely_title, embed_description: iframely_description, embed_html: iframely_html, - thumbnail_url: pictshare_thumbnail, + thumbnail_url: pictrs_thumbnail, }; let _updated_post = match Post::update(&conn, data.edit_id, &post_form) { diff --git a/server/src/lib.rs b/server/src/lib.rs index 23d6a87a6..ebfe17d9c 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -187,25 +187,35 @@ pub fn fetch_iframely(url: &str) -> Result { Ok(res) } -#[derive(Deserialize, Debug)] -pub struct PictshareResponse { - status: String, - url: String, +#[derive(Deserialize, Debug, Clone)] +pub struct PictrsResponse { + files: Vec, + msg: String, } -pub fn fetch_pictshare(image_url: &str) -> Result { +#[derive(Deserialize, Debug, Clone)] +pub struct PictrsFile { + file: String, + delete_token: String, +} + +pub fn fetch_pictrs(image_url: &str) -> Result { is_image_content_type(image_url)?; let fetch_url = format!( - "http://pictshare/api/geturl.php?url={}", - utf8_percent_encode(image_url, NON_ALPHANUMERIC) + "http://pictrs:8080/image/download?url={}", + utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed ); let text = attohttpc::get(&fetch_url).send()?.text()?; - let res: PictshareResponse = serde_json::from_str(&text)?; - Ok(res) + let res: PictrsResponse = serde_json::from_str(&text)?; + if res.msg == "ok" { + Ok(res) + } else { + Err(format_err!("{}", &res.msg)) + } } -fn fetch_iframely_and_pictshare_data( +fn fetch_iframely_and_pictrs_data( url: Option, ) -> ( Option, @@ -225,20 +235,20 @@ fn fetch_iframely_and_pictshare_data( } }; - // Fetch pictshare thumbnail - let pictshare_thumbnail = match iframely_thumbnail_url { - Some(iframely_thumbnail_url) => match fetch_pictshare(&iframely_thumbnail_url) { - Ok(res) => Some(res.url), + // Fetch pictrs thumbnail + let pictrs_thumbnail = match iframely_thumbnail_url { + Some(iframely_thumbnail_url) => match fetch_pictrs(&iframely_thumbnail_url) { + Ok(res) => Some(res.files[0].file.to_owned()), Err(e) => { - error!("pictshare err: {}", e); + error!("pictrs err: {}", e); None } }, // Try to generate a small thumbnail if iframely is not supported - None => match fetch_pictshare(&url) { - Ok(res) => Some(res.url), + None => match fetch_pictrs(&url) { + Ok(res) => Some(res.files[0].file.to_owned()), Err(e) => { - error!("pictshare err: {}", e); + error!("pictrs err: {}", e); None } }, @@ -248,7 +258,7 @@ fn fetch_iframely_and_pictshare_data( iframely_title, iframely_description, iframely_html, - pictshare_thumbnail, + pictrs_thumbnail, ) } None => (None, None, None, None), @@ -273,11 +283,15 @@ pub fn is_valid_username(name: &str) -> bool { VALID_USERNAME_REGEX.is_match(name) } +pub fn is_valid_community_name(name: &str) -> bool { + VALID_COMMUNITY_NAME_REGEX.is_match(name) +} + #[cfg(test)] mod tests { use crate::{ - extract_usernames, is_email_regex, is_image_content_type, is_valid_username, remove_slurs, - slur_check, slurs_vec_to_str, + extract_usernames, is_email_regex, is_image_content_type, is_valid_community_name, + is_valid_username, remove_slurs, slur_check, slurs_vec_to_str, }; #[test] @@ -304,6 +318,15 @@ mod tests { assert!(!is_valid_username("")); } + #[test] + fn test_valid_community_name() { + assert!(is_valid_community_name("example")); + assert!(is_valid_community_name("example_community")); + assert!(!is_valid_community_name("Example")); + assert!(!is_valid_community_name("Ex")); + assert!(!is_valid_community_name("")); + } + #[test] fn test_slur_filter() { let test = @@ -366,4 +389,5 @@ lazy_static! { static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|nig(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btrann?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap(); static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap(); static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap(); + static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap(); } diff --git a/server/src/version.rs b/server/src/version.rs index 4c6aee148..a27f3acb9 100644 --- a/server/src/version.rs +++ b/server/src/version.rs @@ -1 +1 @@ -pub const VERSION: &str = "v0.6.77"; +pub const VERSION: &str = "v0.6.79"; diff --git a/ui/src/components/comment-form.tsx b/ui/src/components/comment-form.tsx index 5239eb2c7..79aa91bdd 100644 --- a/ui/src/components/comment-form.tsx +++ b/ui/src/components/comment-form.tsx @@ -18,6 +18,7 @@ import { setupTribute, wsJsonToRes, emojiPicker, + pictrsDeleteToast, } from '../utils'; import { WebSocketService, UserService } from '../services'; import autosize from 'autosize'; @@ -162,8 +163,9 @@ export class CommentForm extends Component { {this.state.commentForm.content && (