From 998e824bd84680fab4f815d8116fada9952ab0e4 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 15 Dec 2020 15:01:37 +0100 Subject: [PATCH 1/4] Add IP forwarding headers to nginx config (fixes #1318) --- ansible/templates/nginx.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ansible/templates/nginx.conf b/ansible/templates/nginx.conf index 0874be8f9..91fcd9318 100644 --- a/ansible/templates/nginx.conf +++ b/ansible/templates/nginx.conf @@ -67,6 +67,11 @@ server { proxy_pass $proxpass; rewrite ^(.+)/+$ $1 permanent; + + # Send actual client IP upstream + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } # backend From c947539301ad2a0cbf303809399b16ee6f244859 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Wed, 16 Dec 2020 14:03:21 +0000 Subject: [PATCH 2/4] Add docs for creating custom lemmy frontend (#1319) --- docker/dev/docker-compose.yml | 1 + docs/src/SUMMARY.md | 3 +- docs/src/contributing/custom_frontend.md | 66 ++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 docs/src/contributing/custom_frontend.md diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index a09fb8eb1..5342f8b10 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -15,6 +15,7 @@ services: - pictrs - postgres - iframely + lemmy-ui: image: dessalines/lemmy-ui:v0.8.10 ports: diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 93f6c42d0..c8b76f7c3 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -19,7 +19,8 @@ - [Docker Development](contributing/docker_development.md) - [Local Development](contributing/local_development.md) - [Theming Guide](contributing/theming.md) - - [Tests](contributing/tests.md) - [Websocket/HTTP API](contributing/websocket_http_api.md) + - [Creating a Custom Frontend](contributing/custom_frontend.md) + - [Tests](contributing/tests.md) - [Federation Development](contributing/federation_development.md) - [Lemmy Council](lemmy_council.md) diff --git a/docs/src/contributing/custom_frontend.md b/docs/src/contributing/custom_frontend.md new file mode 100644 index 000000000..085685768 --- /dev/null +++ b/docs/src/contributing/custom_frontend.md @@ -0,0 +1,66 @@ +# Creating a Custom Frontend + +The backend and frontend are completely decoupled, and run in independent Docker containers. They only communicate over the [Lemmy API](websocket_http_api.md), which makes it quite easy to write alternative frontends. + +This creates a lot of potential for custom frontends, which could change much of the design and user experience of Lemmy. For example, it would be possible to create a frontend in the style of a traditional forum like [phpBB](https://www.phpbb.com/), or a question-and-answer site like [stackoverflow](https://stackoverflow.com/). All without having to think about database queries, authentification or ActivityPub, which you essentially get for free. + +## Development + +You can use any language to create a custom frontend. The easiest option would be forking our [official frontend](https://github.com/LemmyNet/lemmy-ui), [lemmy-lite](https://github.com/IronOxidizer/lemmy-lite), or the [lemmy-frontend-example](https://github.com/LemmyNet/lemmy-front-end-example). In any case, the principle is the same: bind to `LEMMY_EXTERNAL_HOST` (default: `localhost:8536`) and handle requests using the Lemmy API at `LEMMY_INTERNAL_HOST` (default: `lemmy:8536`). Also use `LEMMY_HTTPS` to generate links with the correct protocol. + +The next step is building a Docker image from your frontend. If you forked an existing project, it should already include a `Dockerfile` and instructions to build it. Otherwise, try searching for your language on [dockerhub](https://hub.docker.com/), official images usually have build instructions in their readme. Build a Docker image with a tag, then look for the following section in `docker/dev/docker-compose.yml`: + +``` + lemmy-ui: + image: dessalines/lemmy-ui:v0.8.10 + ports: + - "1235:1234" + restart: always + environment: + - LEMMY_INTERNAL_HOST=lemmy:8536 + - LEMMY_EXTERNAL_HOST=localhost:8536 + - LEMMY_HTTPS=false + depends_on: + - lemmy +``` + +All you need to do is replace the value for `image` with the tag of your own Docker image (and possibly the environment variables if you need different ones). Then run `./docker_update.sh`, and after compilation, your frontend will be available on `http://localhost:1235`. You can also make the same change to `docker/federation/docker-compose.yml` and run `./start-local-instances.bash` to test federation with your frontend. + +## Deploy with Docker + +After building the Docker image, you need to push it to a Docker registry (such as [dockerhub](https://hub.docker.com/)). Then update the `docker-compose.yml` on your server, replacing the `image` for `lemmy-ui`, just as described above. Run `docker-compose.yml`, and after a short wait, your instance will use the new frontend. + +Note, if your instance is deployed with Ansible, it will override `docker-compose.yml` with every run, reverting back to the default frontend. In that case you should copy the `ansible/` folder from this project to your own repository, and adjust `docker-compose.yml` directly in the repo. + +It is also possible to use multiple frontends for the same Lemmy instance, either using subdomains or subfolders. To do that, don't edit the `lemmy-ui` section in `docker-compose.yml`, but duplicate it, adjusting the name, image and port so they are distinct for each. Then edit your nginx config to pass requests to the appropriate frontend, depending on the subdomain or path. + +## Translations + +You can add the [lemmy-translations](https://github.com/LemmyNet/lemmy-translations) repository to your project as a [git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules). That way you can take advantage of same translations used in the official frontend, and you will also receive new translations contributed via weblate. + +## Rate limiting + +Lemmy does rate limiting for many actions based on the client IP. But if you make any API calls on the server side (eg in the case of server-side rendering, or javascript pre-rendering), Lemmy will take the IP of the Docker container. Meaning that all requests come from the same IP, and get rate limited much earlier. To avoid this problem, you need to pass the headers `X-REAL-IP` and `X-FORWARDED-FOR` on to Lemmy (the headers are set by our nginx config). + +Here is an example snipped for NodeJS: + +```javascript +function setForwardedHeaders( + headers: IncomingHttpHeaders +): { [key: string]: string } { + let out = { + host: headers.host, + }; + if (headers['x-real-ip']) { + out['x-real-ip'] = headers['x-real-ip']; + } + if (headers['x-forwarded-for']) { + out['x-forwarded-for'] = headers['x-forwarded-for']; + } + + return out; +} + +let headers = setForwardedHeaders(req.headers); +let client = new LemmyHttp(httpUri, headers); +``` \ No newline at end of file From 036161cb38314ec3c3ac9c94de34e8d5bcc043ad Mon Sep 17 00:00:00 2001 From: eiknat <68170752+eiknat@users.noreply.github.com> Date: Wed, 16 Dec 2020 09:42:57 -0500 Subject: [PATCH 3/4] add on_conflict clauses to common unique constraint failures (#1264) * add on_conflict clauses to common unique constraint failures * user mention: change create conflict to do_update --- lemmy_db/src/comment.rs | 6 ++++++ lemmy_db/src/community.rs | 3 +++ lemmy_db/src/post.rs | 6 ++++++ lemmy_db/src/user_mention.rs | 5 +++++ lemmy_structs/src/lib.rs | 5 +---- 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/lemmy_db/src/comment.rs b/lemmy_db/src/comment.rs index f54ddd10f..3e7d06be2 100644 --- a/lemmy_db/src/comment.rs +++ b/lemmy_db/src/comment.rs @@ -209,6 +209,9 @@ impl Likeable for CommentLike { use crate::schema::comment_like::dsl::*; insert_into(comment_like) .values(comment_like_form) + .on_conflict((comment_id, user_id)) + .do_update() + .set(comment_like_form) .get_result::(conn) } fn remove(conn: &PgConnection, user_id: i32, comment_id: i32) -> Result { @@ -244,6 +247,9 @@ impl Saveable for CommentSaved { use crate::schema::comment_saved::dsl::*; insert_into(comment_saved) .values(comment_saved_form) + .on_conflict((comment_id, user_id)) + .do_update() + .set(comment_saved_form) .get_result::(conn) } fn unsave(conn: &PgConnection, comment_saved_form: &CommentSavedForm) -> Result { diff --git a/lemmy_db/src/community.rs b/lemmy_db/src/community.rs index be40da349..35d54c3f1 100644 --- a/lemmy_db/src/community.rs +++ b/lemmy_db/src/community.rs @@ -309,6 +309,9 @@ impl Followable for CommunityFollower { use crate::schema::community_follower::dsl::*; insert_into(community_follower) .values(community_follower_form) + .on_conflict((community_id, user_id)) + .do_update() + .set(community_follower_form) .get_result::(conn) } fn follow_accepted(conn: &PgConnection, community_id_: i32, user_id_: i32) -> Result diff --git a/lemmy_db/src/post.rs b/lemmy_db/src/post.rs index 530f475b4..b42c23c72 100644 --- a/lemmy_db/src/post.rs +++ b/lemmy_db/src/post.rs @@ -240,6 +240,9 @@ impl Likeable for PostLike { use crate::schema::post_like::dsl::*; insert_into(post_like) .values(post_like_form) + .on_conflict((post_id, user_id)) + .do_update() + .set(post_like_form) .get_result::(conn) } fn remove(conn: &PgConnection, user_id: i32, post_id: i32) -> Result { @@ -275,6 +278,9 @@ impl Saveable for PostSaved { use crate::schema::post_saved::dsl::*; insert_into(post_saved) .values(post_saved_form) + .on_conflict((post_id, user_id)) + .do_update() + .set(post_saved_form) .get_result::(conn) } fn unsave(conn: &PgConnection, post_saved_form: &PostSavedForm) -> Result { diff --git a/lemmy_db/src/user_mention.rs b/lemmy_db/src/user_mention.rs index 68f566332..b382a24e0 100644 --- a/lemmy_db/src/user_mention.rs +++ b/lemmy_db/src/user_mention.rs @@ -29,8 +29,13 @@ impl Crud for UserMention { fn create(conn: &PgConnection, user_mention_form: &UserMentionForm) -> Result { use crate::schema::user_mention::dsl::*; + // since the return here isnt utilized, we dont need to do an update + // but get_result doesnt return the existing row here insert_into(user_mention) .values(user_mention_form) + .on_conflict((recipient_id, comment_id)) + .do_update() + .set(user_mention_form) .get_result::(conn) } diff --git a/lemmy_structs/src/lib.rs b/lemmy_structs/src/lib.rs index 5d2e42733..a0dbdab65 100644 --- a/lemmy_structs/src/lib.rs +++ b/lemmy_structs/src/lib.rs @@ -98,10 +98,7 @@ fn do_send_local_notifs( // Allow this to fail softly, since comment edits might re-update or replace it // Let the uniqueness handle this fail - match UserMention::create(&conn, &user_mention_form) { - Ok(_mention) => (), - Err(_e) => error!("{}", &_e), - }; + let _ = UserMention::create(&conn, &user_mention_form); // Send an email to those users that have notifications on if do_send_email && mention_user.send_notifications_to_email { From db0a51de2aaa6dcab8f7d2f28b5afe39db42a325 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 16 Dec 2020 18:24:14 +0100 Subject: [PATCH 4/4] Handle long activitystreams header in nginx config (ref #1322) --- ansible/templates/nginx.conf | 3 +++ docker/federation/nginx.conf | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/ansible/templates/nginx.conf b/ansible/templates/nginx.conf index 91fcd9318..84b9c6656 100644 --- a/ansible/templates/nginx.conf +++ b/ansible/templates/nginx.conf @@ -61,6 +61,9 @@ server { if ($http_accept = "application/activity+json") { set $proxpass "http://0.0.0.0:{{ lemmy_port }}"; } + if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") { + set $proxpass "http://0.0.0.0:{{ lemmy_port }}"; + } if ($request_method = POST) { set $proxpass "http://0.0.0.0:{{ lemmy_port }}"; } diff --git a/docker/federation/nginx.conf b/docker/federation/nginx.conf index 003f88dc0..357b87c90 100644 --- a/docker/federation/nginx.conf +++ b/docker/federation/nginx.conf @@ -28,6 +28,9 @@ http { if ($http_accept = "application/activity+json") { set $proxpass http://lemmy-alpha; } + if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") { + set $proxpass http://lemmy-alpha; + } proxy_pass $proxpass; proxy_set_header X-Real-IP $remote_addr; @@ -70,6 +73,9 @@ http { if ($http_accept = "application/activity+json") { set $proxpass http://lemmy-beta; } + if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") { + set $proxpass http://lemmy-beta; + } proxy_pass $proxpass; proxy_set_header X-Real-IP $remote_addr; @@ -112,6 +118,9 @@ http { if ($http_accept = "application/activity+json") { set $proxpass http://lemmy-gamma; } + if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") { + set $proxpass http://lemmy-gamma; + } proxy_pass $proxpass; proxy_set_header X-Real-IP $remote_addr; @@ -154,6 +163,9 @@ http { if ($http_accept = "application/activity+json") { set $proxpass http://lemmy-delta; } + if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") { + set $proxpass http://lemmy-delta; + } proxy_pass $proxpass; proxy_set_header X-Real-IP $remote_addr; @@ -196,6 +208,9 @@ http { if ($http_accept = "application/activity+json") { set $proxpass http://lemmy-epsilon; } + if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") { + set $proxpass http://lemmy-epsilon; + } proxy_pass $proxpass; proxy_set_header X-Real-IP $remote_addr;