mirrors/pleroma
1
0
Fork 0
mirror of https://git.pleroma.social/pleroma/pleroma.git synced 2024-06-20 06:00:43 +00:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'pleroma/develop' into status-notification-type

Browse source 
Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
This commit is contained in:
marcin mikołajczak 2023-02-19 22:02:03 +01:00
parent 3ed39e3109 8a0162cd96
commit 92592c25c2
857 changed files with 42825 additions and 5103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.credo.exs
View file

@ -83,6 +83,7 @@
# lanodan: I think PreferImplicitTry should be consistency, and the behaviour seems
# inconsistent, see: https://github.com/rrrene/credo/issues/224
{Credo.Check.Readability.PreferImplicitTry, false},
{Credo.Check.Readability.PipeIntoAnonymousFunctions, exit_status: 0},
{Credo.Check.Readability.RedundantBlankLines},
{Credo.Check.Readability.StringSigils},
{Credo.Check.Readability.TrailingBlankLine},
@ -90,6 +91,7 @@
{Credo.Check.Readability.VariableNames},
{Credo.Check.Readability.Semicolons},
{Credo.Check.Readability.SpaceAfterCommas},
{Credo.Check.Readability.WithSingleClause, exit_status: 0},
{Credo.Check.Refactor.DoubleBooleanNegation},
{Credo.Check.Refactor.CondStatements},
{Credo.Check.Refactor.CyclomaticComplexity},

1
.gitignore vendored
View file

@ -48,6 +48,7 @@ docs/generated_config.md
# Code test coverage
/cover
/Elixir.*.coverdata
/coverage.xml
.idea
pleroma.iml

86
.gitlab-ci.yml
View file

@ -5,6 +5,7 @@ variables: &global_variables
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
DB_HOST: postgres
DB_PORT: 5432
MIX_ENV: test
cache: &global_cache_policy
@ -34,7 +35,8 @@ after_script:
build:
stage: build
only:
changes:
changes: &build_changes_policy
- ".gitlab-ci.yml"
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
@ -45,6 +47,7 @@ spec-build:
stage: test
only:
changes:
- ".gitlab-ci.yml"
- "lib/pleroma/web/api_spec/**/*.ex"
- "lib/pleroma/web/api_spec.ex"
artifacts:
@ -59,7 +62,7 @@ benchmark:
variables:
MIX_ENV: benchmark
services:
- name: postgres:9.6
- name: postgres:9.6-alpine
alias: postgres
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
script:
@ -70,38 +73,38 @@ benchmark:
unit-testing:
stage: test
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
changes: *build_changes_policy
cache: &testing_cache_policy
<<: *global_cache_policy
policy: pull
services:
- name: postgres:13
- name: postgres:13-alpine
alias: postgres
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
script:
- mix ecto.create
- mix ecto.migrate
- mix coveralls --preload-modules
- mix test --cover --preload-modules
coverage: '/^Line total: ([^ ]*%)$/'
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage.xml
unit-testing-erratic:
stage: test
retry: 2
allow_failure: true
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
changes: *build_changes_policy
cache: &testing_cache_policy
<<: *global_cache_policy
policy: pull
services:
- name: postgres:13
- name: postgres:13-alpine
alias: postgres
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
script:
@ -128,10 +131,7 @@ unit-testing-erratic:
unit-testing-rum:
stage: test
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
changes: *build_changes_policy
cache: *testing_cache_policy
services:
- name: minibikini/postgres-with-rum:12
@ -147,15 +147,14 @@ unit-testing-rum:
- mix test --preload-modules
lint:
image: elixir:1.12
image: &current_elixir elixir:1.12-alpine
stage: test
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
changes: *build_changes_policy
cache: *testing_cache_policy
before_script:
before_script: &current_bfr_script
- apk update
- apk add build-base cmake file-dev git openssl
- mix local.hex --force
- mix local.rebar --force
- mix deps.get
@ -165,29 +164,18 @@ lint:
analysis:
stage: test
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
changes: *build_changes_policy
cache: *testing_cache_policy
script:
- mix credo --strict --only=warnings,todo,fixme,consistency,readability
cycles:
image: *current_elixir
stage: test
image: elixir:1.11
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
changes: *build_changes_policy
cache: {}
before_script:
- mix local.hex --force
- mix local.rebar --force
- mix deps.get
- apt-get update
- apt-get install cmake libmagic-dev -y
before_script: *current_bfr_script
script:
- mix compile
- mix xref graph --format cycles --label compile | awk '{print $0} END{exit ($0 != "No cycles found")}'
@ -266,12 +254,14 @@ stop_review_app:
amd64:
stage: release
image: elixir:1.10.4
image: elixir:1.11.4
only: &release-only
- stable@pleroma/pleroma
- develop@pleroma/pleroma
- /^maint/.*$/@pleroma/pleroma
- /^release/.*$/@pleroma/pleroma
tags:
- amd64
artifacts: &release-artifacts
name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME"
paths:
@ -290,7 +280,7 @@ amd64:
MIX_ENV: prod
before_script: &before-release
- apt-get update && apt-get install -y cmake libmagic-dev
- echo "import Mix.Config" > config/prod.secret.exs
- echo "import Config" > config/prod.secret.exs
- mix local.hex --force
- mix local.rebar --force
script: &release
@ -304,12 +294,14 @@ amd64-musl:
stage: release
artifacts: *release-artifacts
only: *release-only
image: elixir:1.10.4-alpine
image: elixir:1.11.4-alpine
tags:
- amd64
cache: *release-cache
variables: *release-variables
before_script: &before-release-musl
- apk add git gcc g++ musl-dev make cmake file-dev
- echo "import Mix.Config" > config/prod.secret.exs
- apk add git build-base cmake file-dev openssl
- echo "import Config" > config/prod.secret.exs
- mix local.hex --force
- mix local.rebar --force
script: *release
@ -320,7 +312,7 @@ arm:
only: *release-only
tags:
- arm32-specified
image: arm32v7/elixir:1.10.4
image: arm32v7/elixir:1.11.4
cache: *release-cache
variables: *release-variables
before_script: *before-release
@ -332,7 +324,7 @@ arm-musl:
only: *release-only
tags:
- arm32-specified
image: arm32v7/elixir:1.10.4-alpine
image: arm32v7/elixir:1.11.4-alpine
cache: *release-cache
variables: *release-variables
before_script: *before-release-musl
@ -344,7 +336,7 @@ arm64:
only: *release-only
tags:
- arm
image: arm64v8/elixir:1.10.4
image: arm64v8/elixir:1.11.4
cache: *release-cache
variables: *release-variables
before_script: *before-release
@ -356,7 +348,7 @@ arm64-musl:
only: *release-only
tags:
- arm
image: arm64v8/elixir:1.10.4-alpine
image: arm64v8/elixir:1.11.4-alpine
cache: *release-cache
variables: *release-variables
before_script: *before-release-musl

10
.gitlab/merge_request_templates/Release.md
View file

@ -1,6 +1,8 @@
### Release checklist
* [ ] Bump version in `mix.exs`
* [ ] Compile a changelog
* [ ] Create an MR with an announcement to pleroma.social
* [ ] Tag the release
* [ ] Bump version in `mix.exs`
* [ ] Compile a changelog
* [ ] Create an MR with an announcement to pleroma.social
#### post-merge
* [ ] Tag the release on the merge commit
* [ ] Make the tag into a Gitlab Release™
* [ ] Merge `stable` into `develop` (in case the fixes are already in develop, use `git merge -s ours --no-commit` and manually merge the changelogs)

58
CHANGELOG.md
View file

@ -6,19 +6,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## Unreleased
### Changed
### Added
### Fixed
- rel="me" was missing its cache
### Removed
## 2.5.0 - 2022-12-23
### Removed
- MastoFE
- Quack, the logging backend that pushes to Slack channels
### Changed
- **Breaking:** Elixir >=1.11 is now required (was >= 1.9)
- Allow users to remove their emails if instance does not need email to register
- Uploadfilter `Pleroma.Upload.Filter.Exiftool` has been renamed to `Pleroma.Upload.Filter.Exiftool.StripLocation`
- **Breaking**: `/api/v1/pleroma/backups` endpoints now requires `read:backups` scope instead of `read:accounts`
- Updated the recommended pleroma.vcl configuration for Varnish to target Varnish 7.0+
- Set timeout values for Oban queues. The default is infinity and some operations may not time out on their own.
- Delete activities are federated at lowest priority
- CSP now includes wasm-unsafe-eval
### Added
- `activeMonth` and `activeHalfyear` fields in NodeInfo usage.users object
- Experimental support for Finch. Put `config :tesla, :adapter, {Tesla.Adapter.Finch, name: MyFinch}` in your secrets file to use it. Reverse Proxy will still use Hackney.
- `ForceMentionsInPostContent` MRF policy
- AdminAPI: allow moderators to manage reports, users, invites, and custom emojis
- AdminAPI: restrict moderators to access sensitive data: change user credentials, get password reset token, read private statuses and chats, etc
- PleromaAPI: Add remote follow API endpoint at `POST /api/v1/pleroma/remote_interaction`
- MastoAPI: Add `GET /api/v1/accounts/lookup`
- MastoAPI: Profile Directory support
@ -30,6 +48,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Configuration: Add `birthday_required` and `birthday_min_age` settings to provide a way to require users to enter their birth date.
- PleromaAPI: Add `GET /api/v1/pleroma/birthdays` API endpoint
- Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field.
- Add fine grained options to provide privileges to moderators and admins (e.g. delete messages, manage reports...)
- Uploadfilter `Pleroma.Upload.Filter.Exiftool.ReadDescription` returns description values to the FE so they can pre fill the image description field
- Added move account API
- Enable remote users to interact with posts
- Possibility to discover users like `user@example.org`, while Pleroma is working on `pleroma.example.org`. Additional configuration required.
### Fixed
- Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies
@ -45,8 +68,36 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Fixed crash when pinned_objects is nil
- Fixed slow timelines when there are a lot of deactivated users
- Fixed account deletion API
- Fixed lowercase HTTP HEAD method in the Media Proxy Preview code
- Removed useless notification call on Delete activities
- Improved performance for filtering out deactivated and invisible users
- RSS and Atom feeds for users work again
- TwitterCard meta tags conformance
### Removed
## 2.4.5 - 2022-11-27
## Fixed
- Image `class` attributes not being scrubbed, allowing to exploit frontend special classes [!3792](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3792)
- Delete report notifs when demoting from superuser [!3642](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3642)
- Validate `mediaType` only by it's format rather than using a list [!3597](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3597)
- Pagination: Make mutes and blocks lists behave the same as other lists [!3693](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3693)
- Compatibility with Elixir 1.14 [!3740](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3740)
- Frontend installer: FediFE build URL [!3736](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3736)
- Streaming: Don't stream ChatMessage into the home timeline [!3738](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3738)
- Streaming: Stream local-only posts in the local timeline [!3738](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3738)
- Signatures: Fix `keyId` lookup for GoToSocial [!3725](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3725)
- Validator: Fix `replies` handling for GoToSocial [!3725](https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3725)
## 2.4.4 - 2022-08-19
### Security
- Streaming API sessions will now properly disconnect if the corresponding token is revoked
## 2.4.3 - 2022-05-06
### Security
- Private `/objects/` and `/activities/` leaking if cached by authenticated user
- SweetXML library DTD bomb
## 2.4.2 - 2022-01-10
@ -91,6 +142,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Improved Twittercard and OpenGraph meta tag generation including thumbnails and image dimension metadata when available.
- AdminAPI: sort users so the newest are at the top.
- ActivityPub Client-to-Server(C2S): Limitation on the type of Activity/Object are lifted as they are now passed through ObjectValidators
- MRF (`AntiFollowbotPolicy`): Bot accounts are now also considered followbots. Users can still allow bots to follow them by first following the bot.
### Added

10
Dockerfile
View file

@ -1,18 +1,22 @@
FROM elixir:1.9-alpine as build
ARG ELIXIR_VER=1.11.4
ARG ERLANG_VER=24.2.1
ARG ALPINE_VER=3.17.0
FROM hexpm/elixir:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build
COPY . .
ENV MIX_ENV=prod
RUN apk add git gcc g++ musl-dev make cmake file-dev &&\
echo "import Mix.Config" > config/prod.secret.exs &&\
echo "import Config" > config/prod.secret.exs &&\
mix local.hex --force &&\
mix local.rebar --force &&\
mix deps.get --only prod &&\
mkdir release &&\
mix release --path release
FROM alpine:3.14
FROM alpine:${ALPINE_VER}
ARG BUILD_DATE
ARG VCS_REF

2
README.md
View file

@ -30,7 +30,7 @@ If your platform is not supported, or you just want to be able to edit the sourc
- [OpenBSD (fi)](https://docs-develop.pleroma.social/backend/installation/openbsd_fi/)
### OS/Distro packages
Currently Pleroma is packaged for [YunoHost](https://yunohost.org). If you want to package Pleroma for any OS/Distros, we can guide you through the process on our [community channels](#community-channels). If you want to change default options in your Pleroma package, please **discuss it with us first**.
Currently Pleroma is packaged for [YunoHost](https://yunohost.org) and [NixOS](https://nixos.org). If you want to package Pleroma for any OS/Distros, we can guide you through the process on our [community channels](#community-channels). If you want to change default options in your Pleroma package, please **discuss it with us first**.
### Docker
While we don’t provide docker files, other people have written very good ones. Take a look at <https://github.com/angristan/docker-pleroma> or <https://glitch.sh/sn0w/pleroma-docker>.

7
ci/Dockerfile
View file

@ -1,7 +1,8 @@
FROM elixir:1.9.4
FROM elixir:1.11.4
# Single RUN statement, otherwise intermediate images are created
# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
RUN apt-get update &&\
apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
mix local.hex --force &&\
mix local.rebar --force

12
ci/README Normal file
View file

@ -0,0 +1,12 @@
## Dependencies
Assuming an AMD64 Alpine system, you're going to need the following packages
- `qemu qemu-openrc qemu-arm qemu-aarch64` for binfmt
- `docker-cli-buildx` for building the images
## Setting up
```
docker login git.pleroma.social:5050
doas rc-service qemu-binfmt start
```

1
config/benchmark.exs
View file

@ -40,6 +40,7 @@ config :pleroma, Pleroma.Repo,
password: "postgres",
database: "pleroma_benchmark",
hostname: System.get_env("DB_HOST") || "localhost",
port: System.get_env("DB_PORT") || "5432",
pool_size: 10
# Reduce hash rounds for testing

55
config/config.exs
View file

@ -37,7 +37,7 @@
# FIGURATION! EDIT YOUR SECRET FILE (either prod.secret.exs, dev.secret.exs).
#
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
# and its dependencies with the aid of the Config module.
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
@ -160,11 +160,6 @@ config :logger, :ex_syslogger,
format: "$metadata[$level] $message",
metadata: [:request_id]
config :quack,
level: :warn,
meta: [:all],
webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE"
config :mime, :types, %{
"application/xml" => ["xml"],
"application/xrd+xml" => ["xrd+xml"],
@ -187,6 +182,7 @@ config :pleroma, :instance,
email: "example@example.com",
notify_email: "noreply@example.com",
description: "Pleroma: An efficient and flexible fediverse server",
short_description: "",
background_image: "/images/city.jpg",
instance_thumbnail: "/instance/thumbnail.jpeg",
limit: 5_000,
@ -227,6 +223,7 @@ config :pleroma, :instance,
max_pinned_statuses: 1,
attachment_links: false,
max_report_comment_size: 1000,
report_strip_status: true,
safe_dm_mentions: false,
healthcheck: false,
remote_post_retention_days: 90,
@ -256,7 +253,23 @@ config :pleroma, :instance,
show_reactions: true,
password_reset_token_validity: 60 * 60 * 24,
profile_directory: true,
privileged_staff: false,
admin_privileges: [
:users_read,
:users_manage_invites,
:users_manage_activation_state,
:users_manage_tags,
:users_manage_credentials,
:users_delete,
:messages_read,
:messages_delete,
:instances_delete,
:reports_manage_reports,
:moderation_log_read,
:announcements_manage_announcements,
:emoji_manage_emoji,
:statistics_read
],
moderator_privileges: [:messages_delete, :reports_manage_reports],
max_endorsed_users: 20,
birthday_required: false,
birthday_min_age: 0,
@ -558,8 +571,8 @@ config :pleroma, Oban,
token_expiration: 5,
filter_expiration: 1,
backup: 1,
federator_incoming: 50,
federator_outgoing: 50,
federator_incoming: 5,
federator_outgoing: 5,
ingestion_queue: 50,
web_push: 50,
mailer: 10,
@ -672,6 +685,8 @@ config :pleroma, :features, improved_hashtag_timeline: :auto
config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
config :pleroma, :delete_context_objects, fault_rate_allowance: 0.01
config :pleroma, :env, Mix.env()
config :http_signatures,
@ -740,7 +755,7 @@ config :pleroma, :frontends,
"name" => "fedi-fe",
"git" => "https://git.pleroma.social/pleroma/fedi-fe",
"build_url" =>
"https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
"https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build_release",
"ref" => "master",
"custom-http-headers" => [
{"service-worker-allowed", "/"}
@ -753,13 +768,21 @@ config :pleroma, :frontends,
"https://git.pleroma.social/pleroma/admin-fe/-/jobs/artifacts/${ref}/download?job=build",
"ref" => "develop"
},
"soapbox-fe" => %{
"name" => "soapbox-fe",
"git" => "https://gitlab.com/soapbox-pub/soapbox-fe",
"soapbox" => %{
"name" => "soapbox",
"git" => "https://gitlab.com/soapbox-pub/soapbox",
"build_url" =>
"https://gitlab.com/soapbox-pub/soapbox-fe/-/jobs/artifacts/${ref}/download?job=build-production",
"ref" => "v1.0.0",
"https://gitlab.com/soapbox-pub/soapbox/-/jobs/artifacts/${ref}/download?job=build-production",
"ref" => "v3.0.0-beta.1",
"build_dir" => "static"
},
"glitch-lily" => %{
"name" => "glitch-lily",
"git" => "https://lily-is.land/infra/glitch-lily",
"build_url" =>
"https://lily-is.land/infra/glitch-lily/-/jobs/artifacts/${ref}/download?job=build",
"ref" => "servant",
"build_dir" => "public"
}
}
@ -858,6 +881,8 @@ config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
]
config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fetch: true
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

139
config/description.exs
View file

@ -495,6 +495,27 @@ config :pleroma, :config_description, [
}
]
},
%{
group: :pleroma,
key: :delete_context_objects,
type: :group,
description: "`delete_context_objects` background migration settings",
children: [
%{
key: :fault_rate_allowance,
type: :float,
description:
"Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if context object deletion failed for all records.",
suggestions: [0.01]
},
%{
key: :sleep_interval_ms,
type: :integer,
description:
"Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
}
]
},
%{
group: :pleroma,
key: :instance,
@ -536,6 +557,15 @@ config :pleroma, :config_description, [
"Very cool instance"
]
},
%{
key: :short_description,
type: :string,
description:
"Shorter version of instance description. It can be seen on `/api/v1/instance`",
suggestions: [
"Cool instance"
]
},
%{
key: :limit,
type: :integer,
@ -785,6 +815,13 @@ config :pleroma, :config_description, [
1_000
]
},
%{
key: :report_strip_status,
label: "Report strip status",
type: :boolean,
description:
"Strip associated statuses in reports to ids when closed/resolved, otherwise keep a copy"
},
%{
key: :safe_dm_mentions,
label: "Safe DM mentions",
@ -961,10 +998,48 @@ config :pleroma, :config_description, [
description: "Enable profile directory."
},
%{
key: :privileged_staff,
type: :boolean,
key: :admin_privileges,
type: {:list, :atom},
suggestions: [
:users_read,
:users_manage_invites,
:users_manage_activation_state,
:users_manage_tags,
:users_manage_credentials,
:users_delete,
:messages_read,
:messages_delete,
:instances_delete,
:reports_manage_reports,
:moderation_log_read,
:announcements_manage_announcements,
:emoji_manage_emoji,
:statistics_read
],
description:
"Let moderators access sensitive data (e.g. updating user credentials, get password reset token, delete users, index and read private statuses and chats)"
"What extra privileges to allow admins (e.g. updating user credentials, get password reset token, delete users, index and read private statuses and chats)"
},
%{
key: :moderator_privileges,
type: {:list, :atom},
suggestions: [
:users_read,
:users_manage_invites,
:users_manage_activation_state,
:users_manage_tags,
:users_manage_credentials,
:users_delete,
:messages_read,
:messages_delete,
:instances_delete,
:reports_manage_reports,
:moderation_log_read,
:announcements_manage_announcements,
:emoji_manage_emoji,
:statistics_read
],
description:
"What extra privileges to allow moderators (e.g. updating user credentials, get password reset token, delete users, index and read private statuses and chats)"
},
%{
key: :birthday_required,
@ -975,7 +1050,17 @@ config :pleroma, :config_description, [
key: :birthday_min_age,
type: :integer,
description:
"Minimum required age for users to create account. Only used if birthday is required."
"Minimum required age (in days) for users to create account. Only used if birthday is required.",
suggestions: [6570]
},
%{
key: :languages,
type: {:list, :string},
description:
"Languages to be exposed in /api/v1/instance. Should be in the format of BCP47 language codes.",
suggestions: [
"en"
]
}
]
},
@ -1159,45 +1244,6 @@ config :pleroma, :config_description, [
}
]
},
%{
group: :quack,
type: :group,
label: "Quack Logger",
description: "Quack-related settings",
children: [
%{
key: :level,
type: {:dropdown, :atom},
description: "Log level",
suggestions: [:debug, :info, :warn, :error]
},
%{
key: :meta,
type: {:list, :atom},
description: "Configure which metadata you want to report on",
suggestions: [
:application,
:module,
:file,
:function,
:line,
:pid,
:crash_reason,
:initial_call,
:registered_name,
:all,
:none
]
},
%{
key: :webhook_url,
label: "Webhook URL",
type: :string,
description: "Configure the Slack incoming webhook",
suggestions: ["https://hooks.slack.com/services/YOUR-KEY-HERE"]
}
]
},
%{
group: :pleroma,
key: :frontend_configurations,
@ -1720,6 +1766,11 @@ config :pleroma, :config_description, [
type: :boolean,
description: "Sign object fetches with HTTP signatures"
},
%{
key: :authorized_fetch_mode,
type: :boolean,
description: "Require HTTP signatures for AP fetches"
},
%{
key: :note_replies_output_limit,
type: :integer,
@ -2726,7 +2777,7 @@ config :pleroma, :config_description, [
key: :versions,
type: {:list, :atom},
description: "List of TLS version to use",
suggestions: [:tlsv1, ":tlsv1.1", ":tlsv1.2"]
suggestions: [:tlsv1, ":tlsv1.1", ":tlsv1.2", ":tlsv1.3"]
}
]
}

1
config/docker.exs
View file

@ -18,6 +18,7 @@ config :pleroma, Pleroma.Repo,
password: System.fetch_env!("DB_PASS"),
database: System.get_env("DB_NAME", "pleroma"),
hostname: System.get_env("DB_HOST", "db"),
port: System.get_env("DB_PORT", "5432"),
pool_size: 10
# Configure web push notifications

8
config/test.exs
View file

@ -47,6 +47,7 @@ config :pleroma, Pleroma.Repo,
password: "postgres",
database: "pleroma_test",
hostname: System.get_env("DB_HOST") || "localhost",
port: System.get_env("DB_PORT") || "5432",
pool: Ecto.Adapters.SQL.Sandbox,
pool_size: 50
@ -81,10 +82,7 @@ config :web_push_encryption, :vapid_details,
"BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
config :pleroma, Oban,
queues: false,
crontab: false,
plugins: false
config :pleroma, Oban, testing: :manual
config :pleroma, Pleroma.ScheduledActivity,
daily_user_limit: 2,
@ -129,6 +127,8 @@ config :pleroma, :pipeline,
config :pleroma, :cachex, provider: Pleroma.CachexMock
config :pleroma, Pleroma.Web.WebFinger, update_nickname_on_user_fetch: false
config :pleroma, :side_effects,
ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
logger: Pleroma.LoggerMock

2
docker-entrypoint.sh
View file

@ -3,7 +3,7 @@
set -e
echo "-- Waiting for database..."
while ! pg_isready -U ${DB_USER:-pleroma} -d postgres://${DB_HOST:-db}:5432/${DB_NAME:-pleroma} -t 1; do
while ! pg_isready -U ${DB_USER:-pleroma} -d postgres://${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-pleroma} -t 1; do
sleep 1s
done

2
docs/administration/CLI_tasks/frontend.md
View file

@ -22,7 +22,7 @@ Currently, known `<frontend>` values are:
- [kenoma](http://git.pleroma.social/lambadalambda/kenoma)
- [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe)
- [fedi-fe](https://git.pleroma.social/pleroma/fedi-fe)
- [soapbox-fe](https://gitlab.com/soapbox-pub/soapbox-fe)
- [soapbox](https://gitlab.com/soapbox-pub/soapbox)
You can still install frontends that are not configured, see below.

3
docs/administration/CLI_tasks/instance.md
View file

@ -37,7 +37,8 @@ If any of the options are left unspecified, you will be prompted interactively.
- `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
- `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
- `--listen-port <port>` - the port the app should listen to, defaults to 4000
- `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
- `--strip-uploads-location <Y|N>` - use ExifTool to strip uploads of sensitive location data
- `--read-uploads-description <Y|N>` - use ExifTool to read image descriptions from uploads
- `--anonymize-uploads <Y|N>` - randomize uploaded filenames
- `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
- `--skip-release-env` - skip generation the release environment file

4
docs/administration/updating.md
View file

@ -17,11 +17,11 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
## For from source installations (using git)
1. Go to the working directory of Pleroma (default is `/opt/pleroma`)
2. Run `git pull`. This pulls the latest changes from upstream.
2. Run `git checkout <tagged release>` [^1]. e.g. `git checkout v2.4.5` This pulls the [tagged release](https://git.pleroma.social/pleroma/pleroma/-/releases) from upstream.
3. Run `mix deps.get` [^1]. This pulls in any new dependencies.
4. Stop the Pleroma service.
5. Run `mix ecto.migrate` [^1] [^2]. This task performs database migrations, if there were any.
6. Start the Pleroma service.
[^1]: Depending on which install guide you followed (for example on Debian/Ubuntu), you want to run `mix` tasks as `pleroma` user by adding `sudo -Hu pleroma` before the command.
[^1]: Depending on which install guide you followed (for example on Debian/Ubuntu), you want to run `git` and `mix` tasks as `pleroma` user by adding `sudo -Hu pleroma` before the command.
[^2]: Prefix with `MIX_ENV=prod` to run it using the production config file.

59
docs/configuration/cheatsheet.md
View file

@ -18,6 +18,7 @@ To add configuration to your config file, you can copy it from the base config.
* `email`: Email used to reach an Administrator/Moderator of the instance.
* `notify_email`: Email used for notifications.
* `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance``.
* `short_description`: Shorter version of instance description, can be seen on ``/api/v1/instance``.
* `limit`: Posts character limit (CW/Subject included in the counter).
* `description_limit`: The character limit for image descriptions.
* `remote_limit`: Hard character limit beyond which remote posts will be dropped.
@ -48,6 +49,7 @@ To add configuration to your config file, you can copy it from the base config.
* `autofollowing_nicknames`: Set to nicknames of (local) users that automatically follows every newly registered user.
* `attachment_links`: Set to true to enable automatically adding attachment link text to statuses.
* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`).
* `report_strip_status`: Strip associated statuses in reports to ids when closed/resolved, otherwise keep a copy.
* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`.
* `healthcheck`: If set to true, system data will be shown on ``/api/v1/pleroma/healthcheck``.
* `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database.
@ -64,6 +66,36 @@ To add configuration to your config file, you can copy it from the base config.
* `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
* `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
* `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
* `admin_privileges`: A list of privileges an admin has (e.g. delete messages, manage reports...)
* Possible values are:
* `:users_read`
* Allows admins to fetch users through the admin API.
* `:users_manage_invites`
* Allows admins to manage invites. This includes sending, resending, revoking and approving invites.
* `:users_manage_activation_state`
* Allows admins to activate and deactivate accounts. This also allows them to see deactivated users through the Mastodon API.
* `:users_manage_tags`
* Allows admins to set and remove tags for users. This can be useful in combination with MRF policies, such as `Pleroma.Web.ActivityPub.MRF.TagPolicy`.
* `:users_manage_credentials`
* Allows admins to trigger a password reset and set new credentials for an user.
* `:users_delete`
* Allows admins to delete accounts. Note that deleting an account is actually deactivating it and removing all data like posts, profile information, etc.
* `:messages_read`
* Allows admins to read messages through the admin API, including non-public posts and chats.
* `:messages_delete`
* Allows admins to delete messages from other users.
* `:instances_delete,`
* Allows admins to remove a whole remote instance from your instance. This will delete all users and messages from that remote instance.
* `:reports_manage_reports`
* Allows admins to see and manage reports.
* `:moderation_log_read,`
* Allows admins to read the entries in the moderation log.
* `:emoji_manage_emoji`
* Allows admins to manage custom emoji on the instance.
* `:statistics_read,`
* Allows admins to see some simple statistics about the instance.
* `moderator_privileges`: A list of privileges a moderator has (e.g. delete messages, manage reports...)
* Possible values are the same as for `admin_privileges`
## :database
* `improved_hashtag_timeline`: Setting to force toggle / force disable improved hashtags timeline. `:enabled` forces hashtags to be fetched from `hashtags` table for hashtags timeline. `:disabled` forces object-embedded hashtags to be used (slower). Keep it `:auto` for automatic behaviour (it is auto-set to `:enabled` [unless overridden] when HashtagsTableMigrator completes).
@ -125,6 +157,7 @@ To add configuration to your config file, you can copy it from the base config.
* `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
* `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
* `Pleroma.Web.ActivityPub.MRF.FollowBotPolicy`: Automatically follows newly discovered users from the specified bot account. Local accounts, locked accounts, and users with "#nobot" in their bio are respected and excluded from being followed.
* `Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy`: Drops follow requests from followbots. Users can still allow bots to follow them by first following the bot.
* `Pleroma.Web.ActivityPub.MRF.KeywordPolicy`: Rejects or removes from the federated timeline or replaces keywords. (See [`:mrf_keyword`](#mrf_keyword)).
* `Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent`: Forces every mentioned user to be reflected in the post content.
* `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
@ -202,7 +235,7 @@ config :pleroma, :mrf_user_allowlist, %{
e.g., A value of 900 results in any post with a timestamp older than 15 minutes will be acted upon.
* `actions`: A list of actions to apply to the post:
* `:delist` removes the post from public timelines
* `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines
* `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct message
* `:reject` rejects the message entirely
#### :mrf_steal_emoji
@ -626,12 +659,18 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
No specific configuration.
#### Pleroma.Upload.Filter.Exiftool
#### Pleroma.Upload.Filter.Exiftool.StripLocation
This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact.
No specific configuration.
#### Pleroma.Upload.Filter.Exiftool.ReadDescription
This filter reads the ImageDescription and iptc:Caption-Abstract fields with Exiftool so clients can prefill the media description field.
No specific configuration.
#### Pleroma.Upload.Filter.Mogrify
* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`.
@ -769,7 +808,7 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen
* ``private_key``: VAPID private key
## :logger
* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack
* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog
An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
```elixir
@ -792,10 +831,10 @@ config :logger, :ex_syslogger,
See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/)
An example of logging info to local syslog, but warn to a Slack channel:
An example of logging info to local syslog, but debug to console:
```elixir
config :logger,
backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ],
backends: [ {ExSyslogger, :ex_syslogger}, :console ],
level: :info
config :logger, :ex_syslogger,
@ -803,14 +842,12 @@ config :logger, :ex_syslogger,
ident: "pleroma",
format: "$metadata[$level] $message"
config :quack,
level: :warn,
meta: [:all],
webhook_url: "https://hooks.slack.com/services/YOUR-API-KEY-HERE"
config :logger, :console,
level: :debug,
format: "\n$time $metadata[$level] $message\n",
metadata: [:request_id]
```
See the [Quack Github](https://github.com/azohra/quack) for more details
## Database options

62
docs/configuration/how_to_serve_another_domain_for_webfinger.md Normal file
View file

@ -0,0 +1,62 @@
# How to use a different domain name for Pleroma and the users it serves
Pleroma users are primarily identified by a `user@example.org` handle, and you might want this identifier to be the same as your email or jabber account, for instance.
However, in this case, you are almost certainly serving some web content on `https://example.org` already, and you might want to use another domain (say `pleroma.example.org`) for Pleroma itself.
Pleroma supports that, but it might be tricky to set up, and any error might prevent you from federating with other instances.
*If you are already running Pleroma on `example.org`, it is no longer possible to move it to `pleroma.example.org`.*
## Account identifiers
It is important to understand that for federation purposes, a user in Pleroma has two unique identifiers associated:
- A webfinger `acct:` URI, used for discovery and as a verifiable global name for the user across Pleroma instances. In our example, our account's acct: URI is `acct:user@example.org`
- An author/actor URI, used in every other aspect of federation. This is the way in which users are identified in ActivityPub, the underlying protocol used for federation with other Pleroma instances.
In our case, it is `https://pleroma.example.org/users/user`.
Both account identifiers are unique and required for Pleroma. An important risk if you set up your Pleroma instance incorrectly is to create two users (with different acct: URIs) with conflicting author/actor URIs.
## WebFinger
As said earlier, each Pleroma user has an `acct`: URI, which is used for discovery and authentication. When you add @user@example.org, a webfinger query is performed. This is done in two steps:
1. Querying `https://example.org/.well-known/host-meta` (where the domain of the URL matches the domain part of the `acct`: URI) to get information on how to perform the query.
This file will indeed contain a URL template of the form `https://example.org/.well-known/webfinger?resource={uri}` that will be used in the second step.
2. Fill the returned template with the `acct`: URI to be queried and perform the query: `https://example.org/.well-known/webfinger?resource=acct:user@example.org`
## Configuring your Pleroma instance
**_DO NOT ATTEMPT TO CONFIGURE YOUR INSTANCE THIS WAY IF YOU DID NOT UNDERSTAND THE ABOVE_**
### Configuring Pleroma
Pleroma has a two configuration settings to enable using different domains for your users and Pleroma itself. `host` in `Pleroma.Web.Endpoint` and `domain` in `Pleroma.Web.WebFinger`. When the latter is not set, it defaults to the value of `host`.
*Be extra careful when configuring your Pleroma instance, as changing `host` may cause remote instances to register different accounts with the same author/actor URI, which will result in federation issues!*
```elixir
config :pleroma, Pleroma.Web.Endpoint,
url: [host: "pleroma.example.org"]
config :pleroma, Pleroma.Web.WebFinger, domain: "example.org"
```
- `domain` - is the domain for which your Pleroma instance has authority, it's the domain used in `acct:` URI. In our example, `domain` would be set to `example.org`. This is used in WebFinger account ids, which are the canonical account identifier in some other fediverse software like Mastodon. **If you change `domain`, the accounts on your server will be shown as different accounts in those software**.
- `host` - is the domain used for any URL generated for your instance, including the author/actor URL's. In our case, that would be `pleroma.example.org`. This is used in AP ids, which are the canonical account identifier in Pleroma and some other fediverse software. **You should not change this after you have set up the instance**.
### Configuring WebFinger domain
Now, you have Pleroma running at `https://pleroma.example.org` as well as a website at `https://example.org`. If you recall how webfinger queries work, the first step is to query `https://example.org/.well-known/host-meta`, which will contain an URL template.
Therefore, the easiest way to configure `example.org` is to redirect `/.well-known/host-meta` to `pleroma.example.org`.
With nginx, it would be as simple as adding:
```nginx
location = /.well-known/host-meta {
return 301 https://pleroma.example.org$request_uri;
}
```
in example.org's server block.

2
docs/configuration/howto_database_config.md
View file

@ -59,7 +59,7 @@ The configuration of Pleroma has traditionally been managed with a config file,
Here is an example of a server config stripped down after migration:
```
use Mix.Config
import Config
config :pleroma, Pleroma.Web.Endpoint,
url: [host: "cool.pleroma.site", scheme: "https", port: 443]

127
docs/development/API/admin_api.md
View file

@ -1064,7 +1064,6 @@ List of settings which support only full update by key:
```elixir
@full_key_update [
{:pleroma, :ecto_repos},
{:quack, :meta},
{:mime, :types},
{:cors_plug, [:max_age, :methods, :expose, :headers]},
{:auto_linker, :opts},
@ -1084,18 +1083,18 @@ List of settings which support only full update by subkey:
]
```
*Settings without explicit key must be sended in separate config object params.*
*Settings without explicit key must be sent in separate config object params.*
```elixir
config :quack,
level: :debug,
meta: [:all],
config :foo,
bar: :baz,
meta: [:data],
...
```
```json
{
"configs": [
{"group": ":quack", "key": ":level", "value": ":debug"},
{"group": ":quack", "key": ":meta", "value": [":all"]},
{"group": ":foo", "key": ":bar", "value": ":baz"},
{"group": ":foo", "key": ":meta", "value": [":data"]},
...
]
}
@ -1636,3 +1635,117 @@ Returns the content of the document
"error": "Could not install frontend"
}
```
## `GET /api/v1/pleroma/admin/announcements`
### List announcements
- Params: `offset`, `limit`
- Response: JSON, list of announcements
```json
[
{
"id": "AHDp0GBdRn1EPN5HN2",
"content": "some content",
"starts_at": null,
"ends_at": null,
"all_day": false,
"published_at": "2022-03-09T02:13:05",
"reactions": [],
"statuses": [],
"tags": [],
"emojis": [],
"updated_at": "2022-03-09T02:13:05"
}
]
```
Note that this differs from the Mastodon API variant: Mastodon API only returns *active* announcements, while this returns all.
## `GET /api/v1/pleroma/admin/announcements/:id`
### Display one announcement
- Response: JSON, one announcement
```json
{
"id": "AHDp0GBdRn1EPN5HN2",
"content": "some content",
"starts_at": null,
"ends_at": null,
"all_day": false,
"published_at": "2022-03-09T02:13:05",
"reactions": [],
"statuses": [],
"tags": [],
"emojis": [],
"updated_at": "2022-03-09T02:13:05"
}
```
## `POST /api/v1/pleroma/admin/announcements`
### Create an announcement
- Params:
- `content`: string, required, announcement content
- `starts_at`: datetime, optional, default to null, the time when the announcement will become active (displayed to users); if it is null, the announcement will be active immediately
- `ends_at`: datetime, optional, default to null, the time when the announcement will become inactive (no longer displayed to users); if it is null, the announcement will be active until an admin deletes it
- `all_day`: boolean, optional, default to false, tells the client whether to only display dates for `starts_at` and `ends_at`
- Response: JSON, created announcement
```json
{
"id": "AHDp0GBdRn1EPN5HN2",
"content": "some content",
"starts_at": null,
"ends_at": null,
"all_day": false,
"published_at": "2022-03-09T02:13:05",
"reactions": [],
"statuses": [],
"tags": [],
"emojis": [],
"updated_at": "2022-03-09T02:13:05"
}
```
## `PATCH /api/v1/pleroma/admin/announcements/:id`
### Change an announcement
- Params: same as `POST /api/v1/pleroma/admin/announcements`, except no param is required.
- Updates the announcement according to params. Missing params are kept as-is.
- Response: JSON, updated announcement
```json
{
"id": "AHDp0GBdRn1EPN5HN2",
"content": "some content",
"starts_at": null,
"ends_at": null,
"all_day": false,
"published_at": "2022-03-09T02:13:05",
"reactions": [],
"statuses": [],
"tags": [],
"emojis": [],
"updated_at": "2022-03-09T02:13:05"
}
```
## `DELETE /api/v1/pleroma/admin/announcements/:id`
### Delete an announcement
- Response: JSON, empty object
```json
{}
```

4
docs/development/API/differences_in_mastoapi_responses.md
View file

@ -40,6 +40,10 @@ Has these additional fields under the `pleroma` object:
- `parent_visible`: If the parent of this post is visible to the user or not.
- `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise.
The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes:
- `content_type`: The content type of the status source.
## Scheduled statuses
Has these additional fields in `params`:

69
docs/development/API/pleroma_api.md
View file

@ -342,6 +342,36 @@ See [Admin-API](admin_api.md)
* Response: JSON. Returns `{"status": "success"}` if the change was successful, `{"error": "[error message]"}` otherwise
* Note: Currently, Mastodon has no API for changing email. If they add it in future it might be incompatible with Pleroma.
## `/api/pleroma/move_account`
### Move account
* Method `POST`
* Authentication: required
* Params:
* `password`: user's password
* `target_account`: the nickname of the target account (e.g. `foo@example.org`)
* Response: JSON. Returns `{"status": "success"}` if the change was successful, `{"error": "[error message]"}` otherwise
* Note: This endpoint emits a `Move` activity to all followers of the current account. Some remote servers will automatically unfollow the current account and follow the target account upon seeing this, but this depends on the remote server implementation and cannot be guaranteed. For local followers , they will automatically unfollow and follow if and only if they have set the `allow_following_move` preference ("Allow auto-follow when following account moves").
## `/api/pleroma/aliases`
### Get aliases of the current account
* Method `GET`
* Authentication: required
* Response: JSON. Returns `{"aliases": [alias, ...]}`, where `alias` is the nickname of an alias, e.g. `foo@example.org`.
### Add alias to the current account
* Method `PUT`
* Authentication: required
* Params:
* `alias`: the nickname of the alias to add, e.g. `foo@example.org`.
* Response: JSON. Returns `{"status": "success"}` if the change was successful, `{"error": "[error message]"}` otherwise
### Delete alias from the current account
* Method `DELETE`
* Authentication: required
* Params:
* `alias`: the nickname of the alias to delete, e.g. `foo@example.org`.
* Response: JSON. Returns `{"status": "success"}` if the change was successful, `{"error": "[error message]"}` otherwise
# Pleroma Conversations
Pleroma Conversations have the same general structure that Mastodon Conversations have. The behavior differs in the following ways when using these endpoints:
@ -695,3 +725,42 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
* Authentication: required
* Params: none
* Response: HTTP 200 on success, 500 on error
## `/api/v1/pleroma/settings/:app`
### Gets settings for some application
* Method `GET`
* Authentication: `read:accounts`
* Response: JSON. The settings for that application, or empty object if there is none.
* Example response:
```json
{
"some key": "some value"
}
```
### Updates settings for some application
* Method `PATCH`
* Authentication: `write:accounts`
* Request body: JSON object. The object will be merged recursively with old settings. If some field is set to null, it is removed.
* Example request:
```json
{
"some key": "some value",
"key to remove": null,
"nested field": {
"some key": "some value",
"key to remove": null
}
}
```
* Response: JSON. Updated (merged) settings for that application.
* Example response:
```json
{
"some key": "some value",
"nested field": {
"some key": "some value",
}
}
```

9
docs/development/setting_up_a_gitlab_runner.md Normal file
View file

@ -0,0 +1,9 @@
# Setting up a Gitlab-runner
When you push changes, a pipeline will start some automated jobs. These are done with so called [runners](https://docs.gitlab.com/runner/), services that run somewhere on a server and run these automated jobs. These jobs typically run tests and should pass. If not, you probably need to fix something.
Generally, Pleroma provides a runner, so you don't need to set up your own. However, if for whatever reason you want to set up your own, here's some high level instructions.
1. We use docker to run the jobs, so you should install that. For Debian, you need to allow non-free packages in the [source list](https://wiki.debian.org/SourcesList). Then you can install docker with `apt install docker-compose`.
2. You can [install](https://docs.gitlab.com/runner/install/index.html) and [configure](https://docs.gitlab.com/runner/register/index.html) a Gitlab-runner. It's probably easiest to install from the packages, but there are other options as well.
3. When registering the runner, you'll need some values. You can find them in the project under your own name. Choose "Settings", "CI/CD", and then expand "Runners". For executor you can choose "docker". For default image, you can use the image used in <https://git.pleroma.social/pleroma/pleroma/-/blob/develop/.gitlab-ci.yml#L1> (although it shouldn't matter much).

2
docs/installation/generic_dependencies.include
View file

@ -1,7 +1,7 @@
## Required dependencies
* PostgreSQL 9.6+
* Elixir 1.9+
* Elixir 1.10+
* Erlang OTP 22.2+
* git
* file / libmagic

2
docs/installation/migrating_from_source_otp_en.md
View file

@ -5,7 +5,7 @@
In this guide we cover how you can migrate from a from source installation to one using OTP releases.
## Pre-requisites
You will be running commands as root. If you aren't root already, please elevate your priviledges by executing `sudo su`/`su`.
You will be running commands as root. If you aren't root already, please elevate your privileges by executing `sudo su`/`su`.
The system needs to have `curl` and `unzip` installed for downloading and unpacking release builds.

15
docs/installation/nixos_en.md Normal file
View file

@ -0,0 +1,15 @@
# Installing on NixOS
NixOS contains a source build package of pleroma and a NixOS module to install it.
For installation add this to your configuration.nix and add a config.exs next to it:
```nix
services.pleroma = {
enable = true;
configs = [ (lib.fileContents ./config.exs) ];
secretConfigFile = "/var/lib/pleroma/secret.exs";
};
```
## Questions
The nix community uses matrix for communication: [#nix:nixos.org](https://matrix.to/#/#nix:nixos.org)

15
docs/installation/optional/media_graphics_packages.md
View file

@ -1,9 +1,9 @@
# Optional software packages needed for specific functionality
For specific Pleroma functionality (which is disabled by default) some or all of the below packages are required:
* `ImageMagic`
* `ffmpeg`
* `exiftool`
* `ImageMagic`
* `ffmpeg`
* `exiftool`
Please refer to documentation in `docs/installation` on how to install them on specific OS.
@ -14,19 +14,20 @@ Note: the packages are not required with the current default settings of Pleroma
`ImageMagick` is a set of tools to create, edit, compose, or convert bitmap images.
It is required for the following Pleroma features:
* `Pleroma.Upload.Filters.Mogrify`, `Pleroma.Upload.Filters.Mogrifun` upload filters (related config: `Plaroma.Upload/filters` in `config/config.exs`)
* Media preview proxy for still images (related config: `media_preview_proxy/enabled` in `config/config.exs`)
* `Pleroma.Upload.Filters.Mogrify`, `Pleroma.Upload.Filters.Mogrifun` upload filters (related config: `Plaroma.Upload/filters` in `config/config.exs`)
* Media preview proxy for still images (related config: `media_preview_proxy/enabled` in `config/config.exs`)
## `ffmpeg`
`ffmpeg` is software to record, convert and stream audio and video.
It is required for the following Pleroma features:
* Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`)
* Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`)
## `exiftool`
`exiftool` is media files metadata reader/writer.
It is required for the following Pleroma features:
* `Pleroma.Upload.Filters.Exiftool` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)
* `Pleroma.Upload.Filters.Exiftool.StripLocation` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)
* `Pleroma.Upload.Filters.Exiftool.ReadDescription` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)

2
docs/installation/otp_en.md
View file

@ -8,7 +8,7 @@ This guide covers a installation using an OTP release. To install Pleroma from s
* A machine running Linux with GNU (e.g. Debian, Ubuntu) or musl (e.g. Alpine) libc and `x86_64`, `aarch64` or `armv7l` CPU, you have root access to. If you are not sure if it's compatible see [Detecting flavour section](#detecting-flavour) below
* A (sub)domain pointed to the machine
You will be running commands as root. If you aren't root already, please elevate your priviledges by executing `sudo su`/`su`.
You will be running commands as root. If you aren't root already, please elevate your privileges by executing `sudo su`/`su`.
While in theory OTP releases are possbile to install on any compatible machine, for the sake of simplicity this guide focuses only on Debian/Ubuntu and Alpine.

2
elixir_buildpack.config
View file

@ -1,2 +1,2 @@
elixir_version=1.9.4
elixir_version=1.10.4
erlang_version=22.3.4.1

2
installation/pleroma-mongooseim.cfg
View file

@ -466,7 +466,7 @@
%% == PostgreSQL ==
%% {rdbms, global, default, [{workers, 10}],
%% [{server, {pgsql, "server", 5432, "database", "username", "password"}}]},
%% [{server, {pgsql, "server", "port", "database", "username", "password"}}]},
%% == ODBC (MSSQL) ==
%% {rdbms, global, default, [{workers, 10}],

13
installation/pleroma.nginx
View file

@ -81,6 +81,19 @@ server {
proxy_pass http://phoenix;
}
# Uncomment this if you want notice compatibility routes for frontends like Soapbox.
# location ~ ^/@[^/]+/([^/]+)$ {
# proxy_pass http://phoenix/notice/$1;
# }
#
# location ~ ^/@[^/]+/posts/([^/]+)$ {
# proxy_pass http://phoenix/notice/$1;
# }
#
# location ~ ^/[^/]+/status/([^/]+)$ {
# proxy_pass http://phoenix/notice/$1;
# }
location ~ ^/(media|proxy) {
proxy_cache pleroma_media_cache;
slice 1m;

41
installation/pleroma.vcl
View file

@ -1,4 +1,5 @@
# Recommended varnishncsa logging format: '%h %l %u %t "%m %{X-Forwarded-Proto}i://%{Host}i%U%q %H" %s %b "%{Referer}i" "%{User-agent}i"'
# Please use Varnish 7.0+ for proper Range Requests / Chunked encoding support
vcl 4.1;
import std;
@ -22,11 +23,6 @@ sub vcl_recv {
set req.http.X-Forwarded-Proto = "https";
}
# CHUNKED SUPPORT
if (req.http.Range ~ "bytes=") {
set req.http.x-range = req.http.Range;
}
# Pipe if WebSockets request is coming through
if (req.http.upgrade ~ "(?i)websocket") {
return (pipe);
@ -35,9 +31,9 @@ sub vcl_recv {
# Allow purging of the cache
if (req.method == "PURGE") {
if (!client.ip ~ purge) {
return(synth(405,"Not allowed."));
return (synth(405,"Not allowed."));
}
return(purge);
return (purge);
}
}
@ -53,17 +49,11 @@ sub vcl_backend_response {
return (retry);
}
# CHUNKED SUPPORT
if (bereq.http.x-range ~ "bytes=" && beresp.status == 206) {
set beresp.ttl = 10m;
set beresp.http.CR = beresp.http.content-range;
}
# Bypass cache for large files
# 50000000 ~ 50MB
if (std.integer(beresp.http.content-length, 0) > 50000000) {
set beresp.uncacheable = true;
return(deliver);
return (deliver);
}
# Don't cache objects that require authentication
@ -94,7 +84,7 @@ sub vcl_synth {
if (resp.status == 750) {
set resp.status = 301;
set resp.http.Location = req.http.x-redir;
return(deliver);
return (deliver);
}
}
@ -106,25 +96,12 @@ sub vcl_pipe {
}
}
sub vcl_hash {
# CHUNKED SUPPORT
if (req.http.x-range ~ "bytes=") {
hash_data(req.http.x-range);
unset req.http.Range;
}
}
sub vcl_backend_fetch {
# Be more lenient for slow servers on the fediverse
if (bereq.url ~ "^/proxy/") {
set bereq.first_byte_timeout = 300s;
}
# CHUNKED SUPPORT
if (bereq.http.x-range) {
set bereq.http.Range = bereq.http.x-range;
}
if (bereq.retries == 0) {
# Clean up the X-Varnish-Backend-503 flag that is used internally
# to mark broken backend responses that should be retried.
@ -143,14 +120,6 @@ sub vcl_backend_fetch {
}
}
sub vcl_deliver {
# CHUNKED SUPPORT
if (resp.http.CR) {
set resp.http.Content-Range = resp.http.CR;
unset resp.http.CR;
}
}
sub vcl_backend_error {
# Retry broken backend responses.
set bereq.http.X-Varnish-Backend-503 = "1";

9
lib/mix/tasks/pleroma/config.ex
View file

@ -304,13 +304,8 @@ defmodule Mix.Tasks.Pleroma.Config do
System.cmd("mix", ["format", path])
end
if Code.ensure_loaded?(Config.Reader) do
defp config_header, do: "import Config\r\n\r\n"
defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
else
defp config_header, do: "use Mix.Config\r\n\r\n"
defp read_file(config_file), do: Mix.Config.eval!(config_file)
end
defp config_header, do: "import Config\r\n\r\n"
defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
defp write_and_delete(config, file, delete?) do
config

3
lib/mix/tasks/pleroma/database.ex
View file

@ -154,9 +154,8 @@ defmodule Mix.Tasks.Pleroma.Database do
|> join(:inner, [a], o in Object,
on:
fragment(
"(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
"(?->>'id') = associated_object_id((?))",
o.data,
a.data,
a.data
)
)

44
lib/mix/tasks/pleroma/instance.ex
View file

@ -34,7 +34,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
static_dir: :string,
listen_ip: :string,
listen_port: :string,
strip_uploads: :string,
strip_uploads_location: :string,
read_uploads_description: :string,
anonymize_uploads: :string,
dedupe_uploads: :string
],
@ -161,7 +162,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
)
|> Path.expand()
{strip_uploads_message, strip_uploads_default} =
{strip_uploads_location_message, strip_uploads_location_default} =
if Pleroma.Utils.command_available?("exiftool") do
{"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
"y"}
@ -170,12 +171,29 @@ defmodule Mix.Tasks.Pleroma.Instance do
"n"}
end
strip_uploads =
strip_uploads_location =
get_option(
options,
:strip_uploads,
strip_uploads_message,
strip_uploads_default
:strip_uploads_location,
strip_uploads_location_message,
strip_uploads_location_default
) === "y"
{read_uploads_description_message, read_uploads_description_default} =
if Pleroma.Utils.command_available?("exiftool") do
{"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as installed. (y/n)",
"y"}
else
{"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
"n"}
end
read_uploads_description =
get_option(
options,
:read_uploads_description,
read_uploads_description_message,
read_uploads_description_default
) === "y"
anonymize_uploads =
@ -229,7 +247,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
listen_port: listen_port,
upload_filters:
upload_filters(%{
strip: strip_uploads,
strip_location: strip_uploads_location,
read_description: read_uploads_description,
anonymize: anonymize_uploads,
dedupe: dedupe_uploads
})
@ -297,12 +316,19 @@ defmodule Mix.Tasks.Pleroma.Instance do
defp upload_filters(filters) when is_map(filters) do
enabled_filters =
if filters.strip do
[Pleroma.Upload.Filter.Exiftool]
if filters.strip_location do
[Pleroma.Upload.Filter.Exiftool.StripLocation]
else
[]
end
enabled_filters =
if filters.read_description do
enabled_filters ++ [Pleroma.Upload.Filter.Exiftool.ReadDescription]
else
enabled_filters
end
enabled_filters =
if filters.anonymize do
enabled_filters ++ [Pleroma.Upload.Filter.AnonymizeFilename]

67
lib/mix/tasks/pleroma/openapi_spec.ex
View file

@ -6,7 +6,70 @@ defmodule Mix.Tasks.Pleroma.OpenapiSpec do
def run([path]) do
# Load Pleroma application to get version info
Application.load(:pleroma)
spec = Pleroma.Web.ApiSpec.spec(server_specific: false) |> Jason.encode!()
File.write(path, spec)
spec_json = Pleroma.Web.ApiSpec.spec(server_specific: false) |> Jason.encode!()
# to get rid of the structs
spec_regened = spec_json |> Jason.decode!()
check_specs!(spec_regened)
File.write(path, spec_json)
end
defp check_specs!(spec) do
with :ok <- check_specs(spec) do
:ok
else
{_, errors} ->
IO.puts(IO.ANSI.format([:red, :bright, "Spec check failed, errors:"]))
Enum.map(errors, &IO.puts/1)
raise "Spec check failed"
end
end
def check_specs(spec) do
errors =
spec["paths"]
|> Enum.flat_map(fn {path, %{} = endpoints} ->
Enum.map(
endpoints,
fn {method, endpoint} ->
with :ok <- check_endpoint(spec, endpoint) do
:ok
else
error ->
"#{endpoint["operationId"]} (#{method} #{path}): #{error}"
end
end
)
|> Enum.reject(fn res -> res == :ok end)
end)
if errors == [] do
:ok
else
{:error, errors}
end
end
defp check_endpoint(spec, endpoint) do
valid_tags = available_tags(spec)
with {_, [_ | _] = tags} <- {:tags, endpoint["tags"]},
{_, []} <- {:unavailable, Enum.reject(tags, &(&1 in valid_tags))} do
:ok
else
{:tags, _} ->
"No tags specified"
{:unavailable, tags} ->
"Tags #{inspect(tags)} not available. Please add it in \"x-tagGroups\" in Pleroma.Web.ApiSpec"
end
end
defp available_tags(spec) do
spec["x-tagGroups"]
|> Enum.flat_map(fn %{"tags" => tags} -> tags end)
end
end

39
lib/mix/tasks/pleroma/user.ex
View file

@ -112,9 +112,10 @@ defmodule Mix.Tasks.Pleroma.User do
{:ok, token} <- Pleroma.PasswordResetToken.create_token(user) do
shell_info("Generated password reset token for #{user.nickname}")
IO.puts("URL: #{Pleroma.Web.Router.Helpers.reset_password_url(Pleroma.Web.Endpoint,
:reset,
token.token)}")
url =
Pleroma.Web.Router.Helpers.reset_password_url(Pleroma.Web.Endpoint, :reset, token.token)
IO.puts("URL: #{url}")
else
_ ->
shell_error("No local user #{nickname}")
@ -421,6 +422,38 @@ defmodule Mix.Tasks.Pleroma.User do
|> Stream.run()
end
def run(["fix_follow_state", local_user, remote_user]) do
start_pleroma()
with {:local, %User{} = local} <- {:local, User.get_by_nickname(local_user)},
{:remote, %User{} = remote} <- {:remote, User.get_by_nickname(remote_user)},
{:follow_data, %{data: %{"state" => request_state}}} <-
{:follow_data, Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(local, remote)} do
calculated_state = User.following?(local, remote)
shell_info(
"Request state is #{request_state}, vs calculated state of following=#{calculated_state}"
)
if calculated_state == false && request_state == "accept" do
shell_info("Discrepancy found, fixing")
Pleroma.Web.CommonAPI.reject_follow_request(local, remote)
shell_info("Relationship fixed")
else
shell_info("No discrepancy found")
end
else
{:local, _} ->
shell_error("No local user #{local_user}")
{:remote, _} ->
shell_error("No remote user #{remote_user}")
{:follow_data, _} ->
shell_error("No follow data for #{local_user} and #{remote_user}")
end
end
defp set_moderator(user, value) do
{:ok, user} =
user

13
lib/pleroma/activity.ex
View file

@ -53,7 +53,7 @@ defmodule Pleroma.Activity do
#
# ```
# |> join(:inner, [activity], o in Object,
# on: fragment("(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
# on: fragment("(?->>'id') = associated_object_id((?))",
# o.data, activity.data, activity.data))
# |> preload([activity, object], [object: object])
# ```
@ -69,9 +69,8 @@ defmodule Pleroma.Activity do
join(query, join_type, [activity], o in Object,
on:
fragment(
"(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
"(?->>'id') = associated_object_id(?)",
o.data,
activity.data,
activity.data
),
as: :object
@ -362,9 +361,11 @@ defmodule Pleroma.Activity do
end
def restrict_deactivated_users(query) do
deactivated_users_query = from(u in User.Query.build(%{deactivated: true}), select: u.ap_id)
from(activity in query, where: activity.actor not in subquery(deactivated_users_query))
query
|> join(:inner, [activity], user in User,
as: :user,
on: activity.actor == user.ap_id and user.is_active == true
)
end
defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search

36
lib/pleroma/activity/html.ex
View file

@ -8,6 +8,40 @@ defmodule Pleroma.Activity.HTML do
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
# We store a list of cache keys related to an activity in a
# separate cache, scrubber_management_cache. It has the same
# size as scrubber_cache (see application.ex). Every time we add
# a cache to scrubber_cache, we update scrubber_management_cache.
#
# The most recent write of a certain key in the management cache
# is the same as the most recent write of any record related to that
# key in the main cache.
# Assuming LRW ( https://hexdocs.pm/cachex/Cachex.Policy.LRW.html ),
# this means when the management cache is evicted by cachex, all
# related records in the main cache will also have been evicted.
defp get_cache_keys_for(activity_id) do
with {:ok, list} when is_list(list) <- @cachex.get(:scrubber_management_cache, activity_id) do
list
else
_ -> []
end
end
defp add_cache_key_for(activity_id, additional_key) do
current = get_cache_keys_for(activity_id)
unless additional_key in current do
@cachex.put(:scrubber_management_cache, activity_id, [additional_key | current])
end
end
def invalidate_cache_for(activity_id) do
keys = get_cache_keys_for(activity_id)
Enum.map(keys, &@cachex.del(:scrubber_cache, &1))
@cachex.del(:scrubber_management_cache, activity_id)
end
def get_cached_scrubbed_html_for_activity(
content,
scrubbers,
@ -19,6 +53,8 @@ defmodule Pleroma.Activity.HTML do
@cachex.fetch!(:scrubber_cache, key, fn _key ->
object = Object.normalize(activity, fetch: false)
add_cache_key_for(activity.id, key)
HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
end)
end

31
lib/pleroma/activity/ir/topics.ex
View file

@ -13,6 +13,14 @@ defmodule Pleroma.Activity.Ir.Topics do
|> List.flatten()
end
defp generate_topics(%{data: %{"type" => "ChatMessage"}}, %{data: %{"type" => "Delete"}}) do
["user", "user:pleroma_chat"]
end
defp generate_topics(%{data: %{"type" => "ChatMessage"}}, %{data: %{"type" => "Create"}}) do
[]
end
defp generate_topics(%{data: %{"type" => "Answer"}}, _) do
[]
end
@ -21,7 +29,7 @@ defmodule Pleroma.Activity.Ir.Topics do
["user", "list"] ++ visibility_tags(object, activity)
end
defp visibility_tags(object, activity) do
defp visibility_tags(object, %{data: %{"type" => type}} = activity) when type != "Announce" do
case Visibility.get_visibility(activity) do
"public" ->
if activity.local do
@ -31,6 +39,10 @@ defmodule Pleroma.Activity.Ir.Topics do
end
|> item_creation_tags(object, activity)
"local" ->
["public:local"]
|> item_creation_tags(object, activity)
"direct" ->
["direct"]
@ -39,6 +51,10 @@ defmodule Pleroma.Activity.Ir.Topics do
end
end
defp visibility_tags(_object, _activity) do
[]
end
defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
tags ++
remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
@ -63,7 +79,18 @@ defmodule Pleroma.Activity.Ir.Topics do
defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
defp attachment_topics(_object, %{local: true} = activity) do
case Visibility.get_visibility(activity) do
"public" ->
["public:media", "public:local:media"]
"local" ->
["public:local:media"]
_ ->
[]
end
end
defp attachment_topics(_object, %{actor: actor}) when is_binary(actor),
do: ["public:media", "public:remote:media:" <> URI.parse(actor).host]

6
lib/pleroma/activity/queries.ex
View file

@ -52,8 +52,7 @@ defmodule Pleroma.Activity.Queries do
activity in query,
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)",
activity.data,
"associated_object_id((?)) = ANY(?)",
activity.data,
^object_ids
)
@ -64,8 +63,7 @@ defmodule Pleroma.Activity.Queries do
from(activity in query,
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ?",
activity.data,
"associated_object_id((?)) = ?",
activity.data,
^object_id
)

16
lib/pleroma/activity/search.ex
View file

@ -30,7 +30,7 @@ defmodule Pleroma.Activity.Search do
Activity
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public()
|> restrict_public(user)
|> query_with(index_type, search_query, search_function)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
@ -57,7 +57,19 @@ defmodule Pleroma.Activity.Search do
def maybe_restrict_blocked(query, _), do: query
defp restrict_public(q) do
defp restrict_public(q, user) when not is_nil(user) do
intended_recipients = [
Pleroma.Constants.as_public(),
Pleroma.Web.ActivityPub.Utils.as_local_public()
]
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: fragment("? && ?", ^intended_recipients, a.recipients)
)
end
defp restrict_public(q, _user) do
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients

160
lib/pleroma/announcement.ex Normal file
View file

@ -0,0 +1,160 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Announcement do
use Ecto.Schema
import Ecto.Changeset, only: [cast: 3, validate_required: 2]
import Ecto.Query
alias Pleroma.AnnouncementReadRelationship
alias Pleroma.Repo
@type t :: %__MODULE__{}
@primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
schema "announcements" do
field(:data, :map)
field(:starts_at, :utc_datetime)
field(:ends_at, :utc_datetime)
field(:rendered, :map)
timestamps(type: :utc_datetime)
end
def change(struct, params \\ %{}) do
struct
|> cast(validate_params(struct, params), [:data, :starts_at, :ends_at, :rendered])
|> validate_required([:data])
end
defp validate_params(struct, params) do
base_data =
%{
"content" => "",
"all_day" => false
}
|> Map.merge((struct && struct.data) || %{})
merged_data =
Map.merge(base_data, params.data)
|> Map.take(["content", "all_day"])
params
|> Map.merge(%{data: merged_data})
|> add_rendered_properties()
end
def add_rendered_properties(params) do
{content_html, _, _} =
Pleroma.Web.CommonAPI.Utils.format_input(params.data["content"], "text/plain",
mentions_format: :full
)
rendered = %{
"content" => content_html
}
params
|> Map.put(:rendered, rendered)
end
def add(params) do
changeset = change(%__MODULE__{}, params)
Repo.insert(changeset)
end
def update(announcement, params) do
changeset = change(announcement, params)
Repo.update(changeset)
end
def list_all do
__MODULE__
|> Repo.all()
end
def list_paginated(%{limit: limited_number, offset: offset_number}) do
__MODULE__
|> limit(^limited_number)
|> offset(^offset_number)
|> Repo.all()
end
def get_by_id(id) do
Repo.get_by(__MODULE__, id: id)
end
def delete_by_id(id) do
with announcement when not is_nil(announcement) <- get_by_id(id),
{:ok, _} <- Repo.delete(announcement) do
:ok
else
_ ->
:error
end
end
def read_by?(announcement, user) do
AnnouncementReadRelationship.exists?(user, announcement)
end
def mark_read_by(announcement, user) do
AnnouncementReadRelationship.mark_read(user, announcement)
end
def render_json(announcement, opts \\ []) do
extra_params =
case Keyword.fetch(opts, :for) do
{:ok, user} when not is_nil(user) ->
%{read: read_by?(announcement, user)}
_ ->
%{}
end
admin_extra_params =
case Keyword.fetch(opts, :admin) do
{:ok, true} ->
%{pleroma: %{raw_content: announcement.data["content"]}}
_ ->
%{}
end
base = %{
id: announcement.id,
content: announcement.rendered["content"],
starts_at: announcement.starts_at,
ends_at: announcement.ends_at,
all_day: announcement.data["all_day"],
published_at: announcement.inserted_at,
updated_at: announcement.updated_at,
mentions: [],
statuses: [],
tags: [],
emojis: [],
reactions: []
}
base
|> Map.merge(extra_params)
|> Map.merge(admin_extra_params)
end
# "visible" means:
# starts_at < time < ends_at
def list_all_visible_when(time) do
__MODULE__
|> where([a], is_nil(a.starts_at) or a.starts_at < ^time)
|> where([a], is_nil(a.ends_at) or a.ends_at > ^time)
|> Repo.all()
end
def list_all_visible do
list_all_visible_when(DateTime.now("Etc/UTC") |> elem(1))
end
end

55
lib/pleroma/announcement_read_relationship.ex Normal file
View file

@ -0,0 +1,55 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.AnnouncementReadRelationship do
use Ecto.Schema
import Ecto.Changeset
alias FlakeId.Ecto.CompatType
alias Pleroma.Announcement
alias Pleroma.Repo
alias Pleroma.User
@type t :: %__MODULE__{}
schema "announcement_read_relationships" do
belongs_to(:user, User, type: CompatType)
belongs_to(:announcement, Announcement, type: CompatType)
timestamps(updated_at: false)
end
def mark_read(user, announcement) do
%__MODULE__{}
|> cast(%{user_id: user.id, announcement_id: announcement.id}, [:user_id, :announcement_id])
|> validate_required([:user_id, :announcement_id])
|> foreign_key_constraint(:user_id)
|> foreign_key_constraint(:announcement_id)
|> unique_constraint([:user_id, :announcement_id])
|> Repo.insert()
end
def mark_unread(user, announcement) do
with relationship <- get(user, announcement),
{:exists, true} <- {:exists, not is_nil(relationship)},
{:ok, _} <- Repo.delete(relationship) do
:ok
else
{:exists, false} ->
:ok
_ ->
:error
end
end
def get(user, announcement) do
Repo.get_by(__MODULE__, user_id: user.id, announcement_id: announcement.id)
end
def exists?(user, announcement) do
not is_nil(get(user, announcement))
end
end

22
lib/pleroma/application.ex
View file

@ -94,7 +94,8 @@ defmodule Pleroma.Application do
Pleroma.Repo,
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Web.Plugs.RateLimiter.Supervisor
Pleroma.Web.Plugs.RateLimiter.Supervisor,
{Task.Supervisor, name: Pleroma.TaskSupervisor}
] ++
cachex_children() ++
http_children(adapter, @mix_env) ++
@ -112,7 +113,17 @@ defmodule Pleroma.Application do
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
# If we have a lot of caches, default max_restarts can cause test
# resets to fail.
# Go for the default 3 unless we're in test
max_restarts =
if @mix_env == :test do
100
else
3
end
opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts]
result = Supervisor.start_link(children, opts)
set_postgres_server_version()
@ -189,6 +200,7 @@ defmodule Pleroma.Application do
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
build_cachex("scrubber", limit: 2500),
build_cachex("scrubber_management", limit: 2500),
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
@ -197,7 +209,8 @@ defmodule Pleroma.Application do
build_cachex("chat_message_id_idempotency_key",
expiration: chat_message_id_idempotency_key_expiration(),
limit: 500_000
)
),
build_cachex("rel_me", limit: 2500)
]
end
@ -238,7 +251,8 @@ defmodule Pleroma.Application do
defp background_migrators do
[
Pleroma.Migrators.HashtagsTableMigrator
Pleroma.Migrators.HashtagsTableMigrator,
Pleroma.Migrators.ContextObjectsDeletionMigrator
]
end

3
lib/pleroma/application_requirements.ex
View file

@ -164,7 +164,8 @@ defmodule Pleroma.ApplicationRequirements do
defp check_system_commands!(:ok) do
filter_commands_statuses = [
check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"),
check_filter(Pleroma.Upload.Filter.Exiftool.StripLocation, "exiftool"),
check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"),
check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"),

105
lib/pleroma/bbs/handler.ex
View file

@ -42,8 +42,45 @@ defmodule Pleroma.BBS.Handler do
def puts_activity(activity) do
status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity})
IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
IO.puts(HTML.strip_tags(status.content))
status.content
|> String.split("<br/>")
|> Enum.map(&HTML.strip_tags/1)
|> Enum.map(&HtmlEntities.decode/1)
|> Enum.map(&IO.puts/1)
end
def puts_notification(activity, user) do
notification =
Pleroma.Web.MastodonAPI.NotificationView.render("show.json", %{
notification: activity,
for: user
})
IO.puts(
"== (#{notification.type}) #{notification.status.id} by #{notification.account.display_name} (#{notification.account.acct})"
)
notification.status.content
|> String.split("<br/>")
|> Enum.map(&HTML.strip_tags/1)
|> Enum.map(&HtmlEntities.decode/1)
|> (fn x ->
case x do
[content] ->
"> " <> content
[head | _tail] ->
# "> " <> hd <> "..."
head
|> String.slice(1, 80)
|> (fn x -> "> " <> x <> "..." end).()
end
end).()
|> IO.puts()
IO.puts("")
end
@ -53,6 +90,11 @@ defmodule Pleroma.BBS.Handler do
IO.puts("home - Show the home timeline")
IO.puts("p <text> - Post the given text")
IO.puts("r <id> <text> - Reply to the post with the given id")
IO.puts("t <id> - Show a thread from the given id")
IO.puts("n - Show notifications")
IO.puts("n read - Mark all notifactions as read")
IO.puts("f <id> - Favourites the post with the given id")
IO.puts("R <id> - Repeat the post with the given id")
IO.puts("quit - Quit")
state
@ -73,11 +115,53 @@ defmodule Pleroma.BBS.Handler do
state
end
def handle_command(%{user: user} = state, "t " <> activity_id) do
with %Activity{} = activity <- Activity.get_by_id(activity_id) do
activities =
ActivityPub.fetch_activities_for_context(activity.data["context"], %{
blocking_user: user,
user: user,
exclude_id: activity.id
})
case activities do
[] ->
activity_id
|> Activity.get_by_id()
|> puts_activity()
_ ->
activities
|> Enum.reverse()
|> Enum.each(&puts_activity/1)
end
else
_e -> IO.puts("Could not show this thread...")
end
state
end
def handle_command(%{user: user} = state, "n read") do
Pleroma.Notification.clear(user)
IO.puts("All notifications were marked as read")
state
end
def handle_command(%{user: user} = state, "n") do
user
|> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(%{})
|> Enum.each(&puts_notification(&1, user))
state
end
def handle_command(%{user: user} = state, "p " <> text) do
text = String.trim(text)
with {:ok, _activity} <- CommonAPI.post(user, %{status: text}) do
IO.puts("Posted!")
with {:ok, activity} <- CommonAPI.post(user, %{status: text}) do
IO.puts("Posted! ID: #{activity.id}")
else
_e -> IO.puts("Could not post...")
end
@ -85,6 +169,19 @@ defmodule Pleroma.BBS.Handler do
state
end
def handle_command(%{user: user} = state, "f " <> id) do
id = String.trim(id)
with %Activity{} = activity <- Activity.get_by_id(id),
{:ok, _activity} <- CommonAPI.favorite(user, activity) do
IO.puts("Favourited!")
else
_e -> IO.puts("Could not Favourite...")
end
state
end
def handle_command(state, "home") do
user = state.user
@ -123,7 +220,7 @@ defmodule Pleroma.BBS.Handler do
loop(%{state | counter: state.counter + 1})
{:error, :interrupted} ->
{:input, ^input, {:error, :interrupted}} ->
IO.puts("Caught Ctrl+C...")
loop(%{state | counter: state.counter + 1})

42
lib/pleroma/config/deprecation_warnings.ex
View file

@ -20,6 +20,43 @@ defmodule Pleroma.Config.DeprecationWarnings do
"\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"}
]
def check_exiftool_filter do
filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
if Pleroma.Upload.Filter.Exiftool in filters do
Logger.warn("""
!!!DEPRECATION WARNING!!!
Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later:
```
config :pleroma, Pleroma.Upload,
filters: [Pleroma.Upload.Filter.Exiftool]
```
Is now
```
config :pleroma, Pleroma.Upload,
filters: [Pleroma.Upload.Filter.Exiftool.StripLocation]
```
""")
new_config =
filters
|> Enum.map(fn
Pleroma.Upload.Filter.Exiftool -> Pleroma.Upload.Filter.Exiftool.StripLocation
filter -> filter
end)
Config.put([Pleroma.Upload, :filters], new_config)
:error
else
:ok
end
end
def check_simple_policy_tuples do
has_strings =
Config.get([:mrf_simple])
@ -180,7 +217,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
check_old_chat_shoutbox(),
check_quarantined_instances_tuples(),
check_transparency_exclusions_tuples(),
check_simple_policy_tuples()
check_simple_policy_tuples(),
check_exiftool_filter()
]
|> Enum.reduce(:ok, fn
:ok, :ok -> :ok
@ -273,7 +311,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
warning_preface = """
!!!DEPRECATION WARNING!!!
Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. The setting will not take effect until updated.
"""
updated_config =

15
lib/pleroma/config/loader.ex
View file

@ -19,21 +19,10 @@ defmodule Pleroma.Config.Loader do
:tesla
]
if Code.ensure_loaded?(Config.Reader) do
@reader Config.Reader
def read(path), do: @reader.read!(path)
else
# support for Elixir less than 1.9
@reader Mix.Config
def read(path) do
path
|> @reader.eval!()
|> elem(0)
end
end
@reader Config.Reader
@spec read(Path.t()) :: keyword()
def read(path), do: @reader.read!(path)
@spec merge(keyword(), keyword()) :: keyword()
def merge(c1, c2), do: @reader.merge(c1, c2)

7
lib/pleroma/config/transfer_task.ex
View file

@ -47,7 +47,7 @@ defmodule Pleroma.Config.TransferTask do
{logger, other} =
(Repo.all(ConfigDB) ++ deleted_settings)
|> Enum.map(&merge_with_default/1)
|> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
|> Enum.split_with(fn {group, _, _, _} -> group in [:logger] end)
logger
|> Enum.sort()
@ -104,11 +104,6 @@ defmodule Pleroma.Config.TransferTask do
end
# change logger configuration in runtime, without restart
defp configure({:quack, key, _, merged}) do
Logger.configure_backend(Quack.Logger, [{key, merged}])
:ok = update_env(:quack, key, merged)
end
defp configure({_, :backends, _, merged}) do
# removing current backends
Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)

3
lib/pleroma/config_db.ex
View file

@ -163,7 +163,6 @@ defmodule Pleroma.ConfigDB do
defp only_full_update?(%ConfigDB{group: group, key: key}) do
full_key_update = [
{:pleroma, :ecto_repos},
{:quack, :meta},
{:mime, :types},
{:cors_plug, [:max_age, :methods, :expose, :headers]},
{:swarm, :node_blacklist},
@ -386,7 +385,7 @@ defmodule Pleroma.ConfigDB do
@spec module_name?(String.t()) :: boolean()
def module_name?(string) do
Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Ueberauth|Swoosh)\./, string) or
string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
end
end

42
lib/pleroma/constants.ex
View file

@ -27,4 +27,46 @@ defmodule Pleroma.Constants do
do:
~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
)
const(status_updatable_fields,
do: [
"source",
"tag",
"updated",
"emoji",
"content",
"summary",
"sensitive",
"attachment",
"generator"
]
)
const(updatable_object_types,
do: [
"Note",
"Question",
"Audio",
"Video",
"Event",
"Article",
"Page"
]
)
const(actor_types,
do: [
"Application",
"Group",
"Organization",
"Person",
"Service"
]
)
# basic regex, just there to weed out potential mistakes
# https://datatracker.ietf.org/doc/html/rfc2045#section-5.1
const(mime_regex,
do: ~r/^[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+\/[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+(; .*)?$/
)
end

1
lib/pleroma/data_migration.ex
View file

@ -42,4 +42,5 @@ defmodule Pleroma.DataMigration do
end
def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
def delete_context_objects, do: get_by_name("delete_context_objects")
end

10
lib/pleroma/docs/translator.ex Normal file
View file

@ -0,0 +1,10 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Docs.Translator do
require Pleroma.Docs.Translator.Compiler
require Pleroma.Web.Gettext
@before_compile Pleroma.Docs.Translator.Compiler
end

119
lib/pleroma/docs/translator/compiler.ex Normal file
View file

@ -0,0 +1,119 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Docs.Translator.Compiler do
@external_resource "config/description.exs"
@raw_config Pleroma.Config.Loader.read("config/description.exs")
@raw_descriptions @raw_config[:pleroma][:config_description]
defmacro __before_compile__(_env) do
strings =
__MODULE__.descriptions()
|> __MODULE__.extract_strings()
quote do
def placeholder do
unquote do
Enum.map(
strings,
fn {path, type, string} ->
ctxt = msgctxt_for(path, type)
quote do
Pleroma.Web.Gettext.dpgettext_noop(
"config_descriptions",
unquote(ctxt),
unquote(string)
)
end
end
)
end
end
end
end
def descriptions do
Pleroma.Web.ActivityPub.MRF.config_descriptions()
|> Enum.reduce(@raw_descriptions, fn description, acc -> [description | acc] end)
|> Pleroma.Docs.Generator.convert_to_strings()
end
def extract_strings(descriptions) do
descriptions
|> Enum.reduce(%{strings: [], path: []}, &process_item/2)
|> Map.get(:strings)
end
defp process_item(entity, acc) do
current_level =
acc
|> process_desc(entity)
|> process_label(entity)
process_children(entity, current_level)
end
defp process_desc(acc, %{description: desc} = item) do
%{
strings: [{acc.path ++ [key_for(item)], "description", desc} | acc.strings],
path: acc.path
}
end
defp process_desc(acc, _) do
acc
end
defp process_label(acc, %{label: label} = item) do
%{
strings: [{acc.path ++ [key_for(item)], "label", label} | acc.strings],
path: acc.path
}
end
defp process_label(acc, _) do
acc
end
defp process_children(%{children: children} = item, acc) do
current_level = Map.put(acc, :path, acc.path ++ [key_for(item)])
children
|> Enum.reduce(current_level, &process_item/2)
|> Map.put(:path, acc.path)
end
defp process_children(_, acc) do
acc
end
def msgctxt_for(path, type) do
"config #{type} at #{Enum.join(path, " > ")}"
end
defp convert_group({_, group}) do
group
end
defp convert_group(group) do
group
end
def key_for(%{group: group, key: key}) do
"#{convert_group(group)}-#{key}"
end
def key_for(%{group: group}) do
convert_group(group)
end
def key_for(%{key: key}) do
key
end
def key_for(_) do
nil
end
end

25
lib/pleroma/ecto_type/activity_pub/object_validators/mime.ex Normal file
View file

@ -0,0 +1,25 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.MIME do
use Ecto.Type
require Pleroma.Constants
def type, do: :string
def cast(mime) when is_binary(mime) do
if mime =~ Pleroma.Constants.mime_regex() do
{:ok, mime}
else
{:ok, "application/octet-stream"}
end
end
def cast(_), do: :error
def dump(data), do: {:ok, data}
def load(data), do: {:ok, data}
end

125
lib/pleroma/emoji-test.txt
View file

@ -1,13 +1,13 @@
# emoji-test.txt
# Date: 2021-08-26, 17:22:23 GMT
# © 2021 Unicode®, Inc.
# Date: 2022-08-12, 20:24:39 GMT
# © 2022 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
# For terms of use, see https://www.unicode.org/terms_of_use.html
#
# Emoji Keyboard/Display Test Data for UTS #51
# Version: 14.0
# Version: 15.0
#
# For documentation and usage, see http://www.unicode.org/reports/tr51
# For documentation and usage, see https://www.unicode.org/reports/tr51
#
# This file provides data for testing which emoji forms should be in keyboards and which should also be displayed/processed.
# Format: code points; status # emoji name
@ -92,6 +92,7 @@
1F62C ; fully-qualified # 😬 E1.0 grimacing face
1F62E 200D 1F4A8 ; fully-qualified # 😮‍💨 E13.1 face exhaling
1F925 ; fully-qualified # 🤥 E3.0 lying face
1FAE8 ; fully-qualified # 🫨 E15.0 shaking face
# subgroup: face-sleepy
1F60C ; fully-qualified # 😌 E0.6 relieved face
@ -155,7 +156,7 @@
# subgroup: face-negative
1F624 ; fully-qualified # 😤 E0.6 face with steam from nose
1F621 ; fully-qualified # 😡 E0.6 pouting face
1F621 ; fully-qualified # 😡 E0.6 enraged face
1F620 ; fully-qualified # 😠 E0.6 angry face
1F92C ; fully-qualified # 🤬 E5.0 face with symbols on mouth
1F608 ; fully-qualified # 😈 E1.0 smiling face with horns
@ -190,8 +191,7 @@
1F649 ; fully-qualified # 🙉 E0.6 hear-no-evil monkey
1F64A ; fully-qualified # 🙊 E0.6 speak-no-evil monkey
# subgroup: emotion
1F48B ; fully-qualified # 💋 E0.6 kiss mark
# subgroup: heart
1F48C ; fully-qualified # 💌 E0.6 love letter
1F498 ; fully-qualified # 💘 E0.6 heart with arrow
1F49D ; fully-qualified # 💝 E0.6 heart with ribbon
@ -210,14 +210,20 @@
2764 200D 1FA79 ; unqualified # ❤‍🩹 E13.1 mending heart
2764 FE0F ; fully-qualified # ❤️ E0.6 red heart
2764 ; unqualified # ❤ E0.6 red heart
1FA77 ; fully-qualified # 🩷 E15.0 pink heart
1F9E1 ; fully-qualified # 🧡 E5.0 orange heart
1F49B ; fully-qualified # 💛 E0.6 yellow heart
1F49A ; fully-qualified # 💚 E0.6 green heart
1F499 ; fully-qualified # 💙 E0.6 blue heart
1FA75 ; fully-qualified # 🩵 E15.0 light blue heart
1F49C ; fully-qualified # 💜 E0.6 purple heart
1F90E ; fully-qualified # 🤎 E12.0 brown heart
1F5A4 ; fully-qualified # 🖤 E3.0 black heart
1FA76 ; fully-qualified # 🩶 E15.0 grey heart
1F90D ; fully-qualified # 🤍 E12.0 white heart
# subgroup: emotion
1F48B ; fully-qualified # 💋 E0.6 kiss mark
1F4AF ; fully-qualified # 💯 E0.6 hundred points
1F4A2 ; fully-qualified # 💢 E0.6 anger symbol
1F4A5 ; fully-qualified # 💥 E0.6 collision
@ -226,21 +232,20 @@
1F4A8 ; fully-qualified # 💨 E0.6 dashing away
1F573 FE0F ; fully-qualified # 🕳️ E0.7 hole
1F573 ; unqualified # 🕳 E0.7 hole
1F4A3 ; fully-qualified # 💣 E0.6 bomb
1F4AC ; fully-qualified # 💬 E0.6 speech balloon
1F441 FE0F 200D 1F5E8 FE0F ; fully-qualified # 👁️‍🗨️ E2.0 eye in speech bubble
1F441 200D 1F5E8 FE0F ; unqualified # 👁‍🗨️ E2.0 eye in speech bubble
1F441 FE0F 200D 1F5E8 ; unqualified # 👁️‍🗨 E2.0 eye in speech bubble
1F441 FE0F 200D 1F5E8 ; minimally-qualified # 👁️‍🗨 E2.0 eye in speech bubble
1F441 200D 1F5E8 ; unqualified # 👁‍🗨 E2.0 eye in speech bubble
1F5E8 FE0F ; fully-qualified # 🗨️ E2.0 left speech bubble
1F5E8 ; unqualified # 🗨 E2.0 left speech bubble
1F5EF FE0F ; fully-qualified # 🗯️ E0.7 right anger bubble
1F5EF ; unqualified # 🗯 E0.7 right anger bubble
1F4AD ; fully-qualified # 💭 E1.0 thought balloon
1F4A4 ; fully-qualified # 💤 E0.6 zzz
1F4A4 ; fully-qualified # 💤 E0.6 ZZZ
# Smileys & Emotion subtotal: 177
# Smileys & Emotion subtotal: 177 w/o modifiers
# Smileys & Emotion subtotal: 180
# Smileys & Emotion subtotal: 180 w/o modifiers
# group: People & Body
@ -300,6 +305,18 @@
1FAF4 1F3FD ; fully-qualified # 🫴🏽 E14.0 palm up hand: medium skin tone
1FAF4 1F3FE ; fully-qualified # 🫴🏾 E14.0 palm up hand: medium-dark skin tone
1FAF4 1F3FF ; fully-qualified # 🫴🏿 E14.0 palm up hand: dark skin tone
1FAF7 ; fully-qualified # 🫷 E15.0 leftwards pushing hand
1FAF7 1F3FB ; fully-qualified # 🫷🏻 E15.0 leftwards pushing hand: light skin tone
1FAF7 1F3FC ; fully-qualified # 🫷🏼 E15.0 leftwards pushing hand: medium-light skin tone
1FAF7 1F3FD ; fully-qualified # 🫷🏽 E15.0 leftwards pushing hand: medium skin tone
1FAF7 1F3FE ; fully-qualified # 🫷🏾 E15.0 leftwards pushing hand: medium-dark skin tone
1FAF7 1F3FF ; fully-qualified # 🫷🏿 E15.0 leftwards pushing hand: dark skin tone
1FAF8 ; fully-qualified # 🫸 E15.0 rightwards pushing hand
1FAF8 1F3FB ; fully-qualified # 🫸🏻 E15.0 rightwards pushing hand: light skin tone
1FAF8 1F3FC ; fully-qualified # 🫸🏼 E15.0 rightwards pushing hand: medium-light skin tone
1FAF8 1F3FD ; fully-qualified # 🫸🏽 E15.0 rightwards pushing hand: medium skin tone
1FAF8 1F3FE ; fully-qualified # 🫸🏾 E15.0 rightwards pushing hand: medium-dark skin tone
1FAF8 1F3FF ; fully-qualified # 🫸🏿 E15.0 rightwards pushing hand: dark skin tone
# subgroup: hand-fingers-partial
1F44C ; fully-qualified # 👌 E0.6 OK hand
@ -473,11 +490,11 @@
1F932 1F3FE ; fully-qualified # 🤲🏾 E5.0 palms up together: medium-dark skin tone
1F932 1F3FF ; fully-qualified # 🤲🏿 E5.0 palms up together: dark skin tone
1F91D ; fully-qualified # 🤝 E3.0 handshake
1F91D 1F3FB ; fully-qualified # 🤝🏻 E3.0 handshake: light skin tone
1F91D 1F3FC ; fully-qualified # 🤝🏼 E3.0 handshake: medium-light skin tone
1F91D 1F3FD ; fully-qualified # 🤝🏽 E3.0 handshake: medium skin tone
1F91D 1F3FE ; fully-qualified # 🤝🏾 E3.0 handshake: medium-dark skin tone
1F91D 1F3FF ; fully-qualified # 🤝🏿 E3.0 handshake: dark skin tone
1F91D 1F3FB ; fully-qualified # 🤝🏻 E14.0 handshake: light skin tone
1F91D 1F3FC ; fully-qualified # 🤝🏼 E14.0 handshake: medium-light skin tone
1F91D 1F3FD ; fully-qualified # 🤝🏽 E14.0 handshake: medium skin tone
1F91D 1F3FE ; fully-qualified # 🤝🏾 E14.0 handshake: medium-dark skin tone
1F91D 1F3FF ; fully-qualified # 🤝🏿 E14.0 handshake: dark skin tone
1FAF1 1F3FB 200D 1FAF2 1F3FC ; fully-qualified # 🫱🏻‍🫲🏼 E14.0 handshake: light skin tone, medium-light skin tone
1FAF1 1F3FB 200D 1FAF2 1F3FD ; fully-qualified # 🫱🏻‍🫲🏽 E14.0 handshake: light skin tone, medium skin tone
1FAF1 1F3FB 200D 1FAF2 1F3FE ; fully-qualified # 🫱🏻‍🫲🏾 E14.0 handshake: light skin tone, medium-dark skin tone
@ -1455,7 +1472,7 @@
1F575 1F3FF ; fully-qualified # 🕵🏿 E2.0 detective: dark skin tone
1F575 FE0F 200D 2642 FE0F ; fully-qualified # 🕵️‍♂️ E4.0 man detective
1F575 200D 2642 FE0F ; unqualified # 🕵‍♂️ E4.0 man detective
1F575 FE0F 200D 2642 ; unqualified # 🕵️‍♂ E4.0 man detective
1F575 FE0F 200D 2642 ; minimally-qualified # 🕵️‍♂ E4.0 man detective
1F575 200D 2642 ; unqualified # 🕵‍♂ E4.0 man detective
1F575 1F3FB 200D 2642 FE0F ; fully-qualified # 🕵🏻‍♂️ E4.0 man detective: light skin tone
1F575 1F3FB 200D 2642 ; minimally-qualified # 🕵🏻‍♂ E4.0 man detective: light skin tone
@ -1469,7 +1486,7 @@
1F575 1F3FF 200D 2642 ; minimally-qualified # 🕵🏿‍♂ E4.0 man detective: dark skin tone
1F575 FE0F 200D 2640 FE0F ; fully-qualified # 🕵️‍♀️ E4.0 woman detective
1F575 200D 2640 FE0F ; unqualified # 🕵‍♀️ E4.0 woman detective
1F575 FE0F 200D 2640 ; unqualified # 🕵️‍♀ E4.0 woman detective
1F575 FE0F 200D 2640 ; minimally-qualified # 🕵️‍♀ E4.0 woman detective
1F575 200D 2640 ; unqualified # 🕵‍♀ E4.0 woman detective
1F575 1F3FB 200D 2640 FE0F ; fully-qualified # 🕵🏻‍♀️ E4.0 woman detective: light skin tone
1F575 1F3FB 200D 2640 ; minimally-qualified # 🕵🏻‍♀ E4.0 woman detective: light skin tone
@ -2302,7 +2319,7 @@
1F3CC 1F3FF ; fully-qualified # 🏌🏿 E4.0 person golfing: dark skin tone
1F3CC FE0F 200D 2642 FE0F ; fully-qualified # 🏌️‍♂️ E4.0 man golfing
1F3CC 200D 2642 FE0F ; unqualified # 🏌‍♂️ E4.0 man golfing
1F3CC FE0F 200D 2642 ; unqualified # 🏌️‍♂ E4.0 man golfing
1F3CC FE0F 200D 2642 ; minimally-qualified # 🏌️‍♂ E4.0 man golfing
1F3CC 200D 2642 ; unqualified # 🏌‍♂ E4.0 man golfing
1F3CC 1F3FB 200D 2642 FE0F ; fully-qualified # 🏌🏻‍♂️ E4.0 man golfing: light skin tone
1F3CC 1F3FB 200D 2642 ; minimally-qualified # 🏌🏻‍♂ E4.0 man golfing: light skin tone
@ -2316,7 +2333,7 @@
1F3CC 1F3FF 200D 2642 ; minimally-qualified # 🏌🏿‍♂ E4.0 man golfing: dark skin tone
1F3CC FE0F 200D 2640 FE0F ; fully-qualified # 🏌️‍♀️ E4.0 woman golfing
1F3CC 200D 2640 FE0F ; unqualified # 🏌‍♀️ E4.0 woman golfing
1F3CC FE0F 200D 2640 ; unqualified # 🏌️‍♀ E4.0 woman golfing
1F3CC FE0F 200D 2640 ; minimally-qualified # 🏌️‍♀ E4.0 woman golfing
1F3CC 200D 2640 ; unqualified # 🏌‍♀ E4.0 woman golfing
1F3CC 1F3FB 200D 2640 FE0F ; fully-qualified # 🏌🏻‍♀️ E4.0 woman golfing: light skin tone
1F3CC 1F3FB 200D 2640 ; minimally-qualified # 🏌🏻‍♀ E4.0 woman golfing: light skin tone
@ -2427,7 +2444,7 @@
26F9 1F3FF ; fully-qualified # ⛹🏿 E2.0 person bouncing ball: dark skin tone
26F9 FE0F 200D 2642 FE0F ; fully-qualified # ⛹️‍♂️ E4.0 man bouncing ball
26F9 200D 2642 FE0F ; unqualified # ⛹‍♂️ E4.0 man bouncing ball
26F9 FE0F 200D 2642 ; unqualified # ⛹️‍♂ E4.0 man bouncing ball
26F9 FE0F 200D 2642 ; minimally-qualified # ⛹️‍♂ E4.0 man bouncing ball
26F9 200D 2642 ; unqualified # ⛹‍♂ E4.0 man bouncing ball
26F9 1F3FB 200D 2642 FE0F ; fully-qualified # ⛹🏻‍♂️ E4.0 man bouncing ball: light skin tone
26F9 1F3FB 200D 2642 ; minimally-qualified # ⛹🏻‍♂ E4.0 man bouncing ball: light skin tone
@ -2441,7 +2458,7 @@
26F9 1F3FF 200D 2642 ; minimally-qualified # ⛹🏿‍♂ E4.0 man bouncing ball: dark skin tone
26F9 FE0F 200D 2640 FE0F ; fully-qualified # ⛹️‍♀️ E4.0 woman bouncing ball
26F9 200D 2640 FE0F ; unqualified # ⛹‍♀️ E4.0 woman bouncing ball
26F9 FE0F 200D 2640 ; unqualified # ⛹️‍♀ E4.0 woman bouncing ball
26F9 FE0F 200D 2640 ; minimally-qualified # ⛹️‍♀ E4.0 woman bouncing ball
26F9 200D 2640 ; unqualified # ⛹‍♀ E4.0 woman bouncing ball
26F9 1F3FB 200D 2640 FE0F ; fully-qualified # ⛹🏻‍♀️ E4.0 woman bouncing ball: light skin tone
26F9 1F3FB 200D 2640 ; minimally-qualified # ⛹🏻‍♀ E4.0 woman bouncing ball: light skin tone
@ -2462,7 +2479,7 @@
1F3CB 1F3FF ; fully-qualified # 🏋🏿 E2.0 person lifting weights: dark skin tone
1F3CB FE0F 200D 2642 FE0F ; fully-qualified # 🏋️‍♂️ E4.0 man lifting weights
1F3CB 200D 2642 FE0F ; unqualified # 🏋‍♂️ E4.0 man lifting weights
1F3CB FE0F 200D 2642 ; unqualified # 🏋️‍♂ E4.0 man lifting weights
1F3CB FE0F 200D 2642 ; minimally-qualified # 🏋️‍♂ E4.0 man lifting weights
1F3CB 200D 2642 ; unqualified # 🏋‍♂ E4.0 man lifting weights
1F3CB 1F3FB 200D 2642 FE0F ; fully-qualified # 🏋🏻‍♂️ E4.0 man lifting weights: light skin tone
1F3CB 1F3FB 200D 2642 ; minimally-qualified # 🏋🏻‍♂ E4.0 man lifting weights: light skin tone
@ -2476,7 +2493,7 @@
1F3CB 1F3FF 200D 2642 ; minimally-qualified # 🏋🏿‍♂ E4.0 man lifting weights: dark skin tone
1F3CB FE0F 200D 2640 FE0F ; fully-qualified # 🏋️‍♀️ E4.0 woman lifting weights
1F3CB 200D 2640 FE0F ; unqualified # 🏋‍♀️ E4.0 woman lifting weights
1F3CB FE0F 200D 2640 ; unqualified # 🏋️‍♀ E4.0 woman lifting weights
1F3CB FE0F 200D 2640 ; minimally-qualified # 🏋️‍♀ E4.0 woman lifting weights
1F3CB 200D 2640 ; unqualified # 🏋‍♀ E4.0 woman lifting weights
1F3CB 1F3FB 200D 2640 FE0F ; fully-qualified # 🏋🏻‍♀️ E4.0 woman lifting weights: light skin tone
1F3CB 1F3FB 200D 2640 ; minimally-qualified # 🏋🏻‍♀ E4.0 woman lifting weights: light skin tone
@ -3262,8 +3279,8 @@
1FAC2 ; fully-qualified # 🫂 E13.0 people hugging
1F463 ; fully-qualified # 👣 E0.6 footprints
# People & Body subtotal: 2986
# People & Body subtotal: 506 w/o modifiers
# People & Body subtotal: 2998
# People & Body subtotal: 508 w/o modifiers
# group: Component
@ -3306,6 +3323,8 @@
1F405 ; fully-qualified # 🐅 E1.0 tiger
1F406 ; fully-qualified # 🐆 E1.0 leopard
1F434 ; fully-qualified # 🐴 E0.6 horse face
1FACE ; fully-qualified # 🫎 E15.0 moose
1FACF ; fully-qualified # 🫏 E15.0 donkey
1F40E ; fully-qualified # 🐎 E0.6 horse
1F984 ; fully-qualified # 🦄 E1.0 unicorn
1F993 ; fully-qualified # 🦓 E5.0 zebra
@ -3373,6 +3392,9 @@
1F9A9 ; fully-qualified # 🦩 E12.0 flamingo
1F99A ; fully-qualified # 🦚 E11.0 peacock
1F99C ; fully-qualified # 🦜 E11.0 parrot
1FABD ; fully-qualified # 🪽 E15.0 wing
1F426 200D 2B1B ; fully-qualified # 🐦‍⬛ E15.0 black bird
1FABF ; fully-qualified # 🪿 E15.0 goose
# subgroup: animal-amphibian
1F438 ; fully-qualified # 🐸 E0.6 frog
@ -3399,6 +3421,7 @@
1F419 ; fully-qualified # 🐙 E0.6 octopus
1F41A ; fully-qualified # 🐚 E0.6 spiral shell
1FAB8 ; fully-qualified # 🪸 E14.0 coral
1FABC ; fully-qualified # 🪼 E15.0 jellyfish
# subgroup: animal-bug
1F40C ; fully-qualified # 🐌 E0.6 snail
@ -3433,6 +3456,7 @@
1F33B ; fully-qualified # 🌻 E0.6 sunflower
1F33C ; fully-qualified # 🌼 E0.6 blossom
1F337 ; fully-qualified # 🌷 E0.6 tulip
1FABB ; fully-qualified # 🪻 E15.0 hyacinth
# subgroup: plant-other
1F331 ; fully-qualified # 🌱 E0.6 seedling
@ -3451,9 +3475,10 @@
1F343 ; fully-qualified # 🍃 E0.6 leaf fluttering in wind
1FAB9 ; fully-qualified # 🪹 E14.0 empty nest
1FABA ; fully-qualified # 🪺 E14.0 nest with eggs
1F344 ; fully-qualified # 🍄 E0.6 mushroom
# Animals & Nature subtotal: 151
# Animals & Nature subtotal: 151 w/o modifiers
# Animals & Nature subtotal: 159
# Animals & Nature subtotal: 159 w/o modifiers
# group: Food & Drink
@ -3492,10 +3517,11 @@
1F966 ; fully-qualified # 🥦 E5.0 broccoli
1F9C4 ; fully-qualified # 🧄 E12.0 garlic
1F9C5 ; fully-qualified # 🧅 E12.0 onion
1F344 ; fully-qualified # 🍄 E0.6 mushroom
1F95C ; fully-qualified # 🥜 E3.0 peanuts
1FAD8 ; fully-qualified # 🫘 E14.0 beans
1F330 ; fully-qualified # 🌰 E0.6 chestnut
1FADA ; fully-qualified # 🫚 E15.0 ginger root
1FADB ; fully-qualified # 🫛 E15.0 pea pod
# subgroup: food-prepared
1F35E ; fully-qualified # 🍞 E0.6 bread
@ -3607,8 +3633,8 @@
1FAD9 ; fully-qualified # 🫙 E14.0 jar
1F3FA ; fully-qualified # 🏺 E1.0 amphora
# Food & Drink subtotal: 134
# Food & Drink subtotal: 134 w/o modifiers
# Food & Drink subtotal: 135
# Food & Drink subtotal: 135 w/o modifiers
# group: Travel & Places
@ -3974,11 +4000,10 @@
1F3AF ; fully-qualified # 🎯 E0.6 bullseye
1FA80 ; fully-qualified # 🪀 E12.0 yo-yo
1FA81 ; fully-qualified # 🪁 E12.0 kite
1F52B ; fully-qualified # 🔫 E0.6 water pistol
1F3B1 ; fully-qualified # 🎱 E0.6 pool 8 ball
1F52E ; fully-qualified # 🔮 E0.6 crystal ball
1FA84 ; fully-qualified # 🪄 E13.0 magic wand
1F9FF ; fully-qualified # 🧿 E11.0 nazar amulet
1FAAC ; fully-qualified # 🪬 E14.0 hamsa
1F3AE ; fully-qualified # 🎮 E0.6 video game
1F579 FE0F ; fully-qualified # 🕹️ E0.7 joystick
1F579 ; unqualified # 🕹 E0.7 joystick
@ -4013,8 +4038,8 @@
1F9F6 ; fully-qualified # 🧶 E11.0 yarn
1FAA2 ; fully-qualified # 🪢 E13.0 knot
# Activities subtotal: 97
# Activities subtotal: 97 w/o modifiers
# Activities subtotal: 96
# Activities subtotal: 96 w/o modifiers
# group: Objects
@ -4040,6 +4065,7 @@
1FA73 ; fully-qualified # 🩳 E12.0 shorts
1F459 ; fully-qualified # 👙 E0.6 bikini
1F45A ; fully-qualified # 👚 E0.6 woman’s clothes
1FAAD ; fully-qualified # 🪭 E15.0 folding hand fan
1F45B ; fully-qualified # 👛 E0.6 purse
1F45C ; fully-qualified # 👜 E0.6 handbag
1F45D ; fully-qualified # 👝 E0.6 clutch bag
@ -4055,6 +4081,7 @@
1F461 ; fully-qualified # 👡 E0.6 woman’s sandal
1FA70 ; fully-qualified # 🩰 E12.0 ballet shoes
1F462 ; fully-qualified # 👢 E0.6 woman’s boot
1FAAE ; fully-qualified # 🪮 E15.0 hair pick
1F451 ; fully-qualified # 👑 E0.6 crown
1F452 ; fully-qualified # 👒 E0.6 woman’s hat
1F3A9 ; fully-qualified # 🎩 E0.6 top hat
@ -4103,6 +4130,8 @@
1FA95 ; fully-qualified # 🪕 E12.0 banjo
1F941 ; fully-qualified # 🥁 E3.0 drum
1FA98 ; fully-qualified # 🪘 E13.0 long drum
1FA87 ; fully-qualified # 🪇 E15.0 maracas
1FA88 ; fully-qualified # 🪈 E15.0 flute
# subgroup: phone
1F4F1 ; fully-qualified # 📱 E0.6 mobile phone
@ -4275,7 +4304,7 @@
1F5E1 ; unqualified # 🗡 E0.7 dagger
2694 FE0F ; fully-qualified # ⚔️ E1.0 crossed swords
2694 ; unqualified # ⚔ E1.0 crossed swords
1F52B ; fully-qualified # 🔫 E0.6 water pistol
1F4A3 ; fully-qualified # 💣 E0.6 bomb
1FA83 ; fully-qualified # 🪃 E13.0 boomerang
1F3F9 ; fully-qualified # 🏹 E1.0 bow and arrow
1F6E1 FE0F ; fully-qualified # 🛡️ E0.7 shield
@ -4354,12 +4383,14 @@
1FAA6 ; fully-qualified # 🪦 E13.0 headstone
26B1 FE0F ; fully-qualified # ⚱️ E1.0 funeral urn
26B1 ; unqualified # ⚱ E1.0 funeral urn
1F9FF ; fully-qualified # 🧿 E11.0 nazar amulet
1FAAC ; fully-qualified # 🪬 E14.0 hamsa
1F5FF ; fully-qualified # 🗿 E0.6 moai
1FAA7 ; fully-qualified # 🪧 E13.0 placard
1FAAA ; fully-qualified # 🪪 E14.0 identification card
# Objects subtotal: 304
# Objects subtotal: 304 w/o modifiers
# Objects subtotal: 310
# Objects subtotal: 310 w/o modifiers
# group: Symbols
@ -4455,6 +4486,7 @@
262E ; unqualified # ☮ E1.0 peace symbol
1F54E ; fully-qualified # 🕎 E1.0 menorah
1F52F ; fully-qualified # 🔯 E0.6 dotted six-pointed star
1FAAF ; fully-qualified # 🪯 E15.0 khanda
# subgroup: zodiac
2648 ; fully-qualified # ♈ E0.6 Aries
@ -4503,6 +4535,7 @@
1F505 ; fully-qualified # 🔅 E1.0 dim button
1F506 ; fully-qualified # 🔆 E1.0 bright button
1F4F6 ; fully-qualified # 📶 E0.6 antenna bars
1F6DC ; fully-qualified # 🛜 E15.0 wireless
1F4F3 ; fully-qualified # 📳 E0.6 vibration mode
1F4F4 ; fully-qualified # 📴 E0.6 mobile phone off
@ -4693,8 +4726,8 @@
1F533 ; fully-qualified # 🔳 E0.6 white square button
1F532 ; fully-qualified # 🔲 E0.6 black square button
# Symbols subtotal: 302
# Symbols subtotal: 302 w/o modifiers
# Symbols subtotal: 304
# Symbols subtotal: 304 w/o modifiers
# group: Flags
@ -4709,7 +4742,7 @@
1F3F3 200D 1F308 ; unqualified # 🏳‍🌈 E4.0 rainbow flag
1F3F3 FE0F 200D 26A7 FE0F ; fully-qualified # 🏳️‍⚧️ E13.0 transgender flag
1F3F3 200D 26A7 FE0F ; unqualified # 🏳‍⚧️ E13.0 transgender flag
1F3F3 FE0F 200D 26A7 ; unqualified # 🏳️‍⚧ E13.0 transgender flag
1F3F3 FE0F 200D 26A7 ; minimally-qualified # 🏳️‍⚧ E13.0 transgender flag
1F3F3 200D 26A7 ; unqualified # 🏳‍⚧ E13.0 transgender flag
1F3F4 200D 2620 FE0F ; fully-qualified # 🏴‍☠️ E11.0 pirate flag
1F3F4 200D 2620 ; minimally-qualified # 🏴‍☠ E11.0 pirate flag
@ -4983,9 +5016,9 @@
# Flags subtotal: 275 w/o modifiers
# Status Counts
# fully-qualified : 3624
# minimally-qualified : 817
# unqualified : 252
# fully-qualified : 3655
# minimally-qualified : 827
# unqualified : 242
# component : 9
#EOF

14
lib/pleroma/emoji.ex
View file

@ -9,6 +9,7 @@ defmodule Pleroma.Emoji do
"""
use GenServer
alias Pleroma.Emoji.Combinations
alias Pleroma.Emoji.Loader
require Logger
@ -137,4 +138,17 @@ defmodule Pleroma.Emoji do
end
def is_unicode_emoji?(_), do: false
emoji_qualification_map =
emojis
|> Enum.filter(&String.contains?(&1, "\uFE0F"))
|> Combinations.variate_emoji_qualification()
for {qualified, unqualified_list} <- emoji_qualification_map do
for unqualified <- unqualified_list do
def fully_qualify_emoji(unquote(unqualified)), do: unquote(qualified)
end
end
def fully_qualify_emoji(emoji), do: emoji
end

45
lib/pleroma/emoji/combinations.ex Normal file
View file

@ -0,0 +1,45 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Emoji.Combinations do
# FE0F is the emoji variation sequence. It is used for fully-qualifying
# emoji, and that includes emoji combinations.
# This code generates combinations per emoji: for each FE0F, all possible
# combinations of the character being removed or staying will be generated.
# This is made as an attempt to find all partially-qualified and unqualified
# versions of a fully-qualified emoji.
# I have found *no cases* for which this would be a problem, after browsing
# the entire emoji list in emoji-test.txt. This is safe, and, sadly, most
# likely sane too.
defp qualification_combinations(codepoints) do
qualification_combinations([[]], codepoints)
end
defp qualification_combinations(acc, []), do: acc
defp qualification_combinations(acc, ["\uFE0F" | tail]) do
acc
|> Enum.flat_map(fn x -> [x, x ++ ["\uFE0F"]] end)
|> qualification_combinations(tail)
end
defp qualification_combinations(acc, [codepoint | tail]) do
acc
|> Enum.map(&Kernel.++(&1, [codepoint]))
|> qualification_combinations(tail)
end
def variate_emoji_qualification(emoji) when is_binary(emoji) do
emoji
|> String.codepoints()
|> qualification_combinations()
|> Enum.map(&List.to_string/1)
end
def variate_emoji_qualification(emoji) when is_list(emoji) do
emoji
|> Enum.map(fn emoji -> {emoji, variate_emoji_qualification(emoji)} end)
end
end

3
lib/pleroma/following_relationship.ex
View file

@ -194,12 +194,13 @@ defmodule Pleroma.FollowingRelationship do
|> join(:inner, [r], f in assoc(r, :follower))
|> where(following_id: ^origin.id)
|> where([r, f], f.allow_following_move == true)
|> where([r, f], f.local == true)
|> limit(50)
|> preload([:follower])
|> Repo.all()
|> Enum.map(fn following_relationship ->
Repo.delete(following_relationship)
Pleroma.Web.CommonAPI.follow(following_relationship.follower, target)
Pleroma.Web.CommonAPI.unfollow(following_relationship.follower, origin)
end)
|> case do
[] ->

9
lib/pleroma/http.ex
View file

@ -106,5 +106,12 @@ defmodule Pleroma.HTTP do
[Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool]
end
defp adapter_middlewares(_), do: []
defp adapter_middlewares(_) do
if Pleroma.Config.get(:env) == :test do
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
else
[]
end
end
end

4
lib/pleroma/http/adapter_helper/hackney.ex
View file

@ -24,10 +24,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
end
defp add_scheme_opts(opts, %URI{scheme: "https"}) do
Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
end
defp add_scheme_opts(opts, _), do: opts
defp maybe_add_with_body(opts) do

139
lib/pleroma/migrators/context_objects_deletion_migrator.ex Normal file
View file

@ -0,0 +1,139 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Migrators.ContextObjectsDeletionMigrator do
defmodule State do
use Pleroma.Migrators.Support.BaseMigratorState
@impl Pleroma.Migrators.Support.BaseMigratorState
defdelegate data_migration(), to: Pleroma.DataMigration, as: :delete_context_objects
end
use Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Object
@doc "This migration removes objects created exclusively for contexts, containing only an `id` field."
@impl BaseMigrator
def feature_config_path, do: [:features, :delete_context_objects]
@impl BaseMigrator
def fault_rate_allowance, do: Config.get([:delete_context_objects, :fault_rate_allowance], 0)
@impl BaseMigrator
def perform do
data_migration_id = data_migration_id()
max_processed_id = get_stat(:max_processed_id, 0)
Logger.info("Deleting context objects from `objects` (from oid: #{max_processed_id})...")
query()
|> where([object], object.id > ^max_processed_id)
|> Repo.chunk_stream(100, :batches, timeout: :infinity)
|> Stream.each(fn objects ->
object_ids = Enum.map(objects, & &1.id)
results = Enum.map(object_ids, &delete_context_object(&1))
failed_ids =
results
|> Enum.filter(&(elem(&1, 0) == :error))
|> Enum.map(&elem(&1, 1))
chunk_affected_count =
results
|> Enum.filter(&(elem(&1, 0) == :ok))
|> length()
for failed_id <- failed_ids do
_ =
Repo.query(
"INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
"VALUES ($1, $2) ON CONFLICT DO NOTHING;",
[data_migration_id, failed_id]
)
end
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = ANY($2)",
[data_migration_id, object_ids -- failed_ids]
)
max_object_id = Enum.at(object_ids, -1)
put_stat(:max_processed_id, max_object_id)
increment_stat(:iteration_processed_count, length(object_ids))
increment_stat(:processed_count, length(object_ids))
increment_stat(:failed_count, length(failed_ids))
increment_stat(:affected_count, chunk_affected_count)
put_stat(:records_per_second, records_per_second())
persist_state()
# A quick and dirty approach to controlling the load this background migration imposes
sleep_interval = Config.get([:delete_context_objects, :sleep_interval_ms], 0)
Process.sleep(sleep_interval)
end)
|> Stream.run()
end
@impl BaseMigrator
def query do
# Context objects have no activity type, and only one field, `id`.
# Only those context objects are without types.
from(
object in Object,
where: fragment("(?)->'type' IS NULL", object.data),
select: %{
id: object.id
}
)
end
@spec delete_context_object(integer()) :: {:ok | :error, integer()}
defp delete_context_object(id) do
result =
%Object{id: id}
|> Repo.delete()
|> elem(0)
{result, id}
end
@impl BaseMigrator
def retry_failed do
data_migration_id = data_migration_id()
failed_objects_query()
|> Repo.chunk_stream(100, :one)
|> Stream.each(fn object ->
with {res, _} when res != :error <- delete_context_object(object.id) do
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = $2",
[data_migration_id, object.id]
)
end
end)
|> Stream.run()
put_stat(:failed_count, failures_count())
persist_state()
force_continue()
end
defp failed_objects_query do
from(o in Object)
|> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
on: dmf.record_id == o.id
)
|> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
|> order_by([o], asc: o.id)
end
end

2
lib/pleroma/migrators/hashtags_table_migrator.ex
View file

@ -183,7 +183,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do
DELETE FROM hashtags_objects WHERE object_id IN
(SELECT DISTINCT objects.id FROM objects
JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
ON associated_object_id(activities) =
(objects.data->>'id')
AND activities.data->>'type' = 'Create'
WHERE activities.id IS NULL);

55
lib/pleroma/notification.ex
View file

@ -118,9 +118,8 @@ defmodule Pleroma.Notification do
|> join(:left, [n, a], object in Object,
on:
fragment(
"(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
"(?->>'id') = associated_object_id(?)",
object.data,
a.data,
a.data
)
)
@ -180,6 +179,7 @@ defmodule Pleroma.Notification do
from([_n, a, o] in query,
where:
fragment("not(?->>'content' ~* ?)", o.data, ^regex) or
fragment("?->>'content' is null", o.data) or
fragment("?->>'actor' = ?", o.data, ^user.ap_id)
)
end
@ -194,13 +194,11 @@ defmodule Pleroma.Notification do
|> join(:left, [n, a], mutated_activity in Pleroma.Activity,
on:
fragment(
"COALESCE((?->'object')->>'id', ?->>'object')",
a.data,
"associated_object_id(?)",
a.data
) ==
fragment(
"COALESCE((?->'object')->>'id', ?->>'object')",
mutated_activity.data,
"associated_object_id(?)",
mutated_activity.data
) and
fragment("(?->>'type' = 'Like' or ?->>'type' = 'Announce')", a.data, a.data) and
@ -342,14 +340,6 @@ defmodule Pleroma.Notification do
|> Repo.delete_all()
end
def destroy_multiple_from_types(%{id: user_id}, types) do
from(n in Notification,
where: n.user_id == ^user_id,
where: n.type in ^types
)
|> Repo.delete_all()
end
def dismiss(%Pleroma.Activity{} = activity) do
Notification
|> where([n], n.activity_id == ^activity.id)
@ -386,7 +376,7 @@ defmodule Pleroma.Notification do
end
def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do
when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do
do_create_notifications(activity, options)
end
@ -447,6 +437,9 @@ defmodule Pleroma.Notification do
activity
|> type_from_activity_object()
"Update" ->
"update"
t ->
raise "No notification type for activity type #{t}"
end
@ -521,7 +514,16 @@ defmodule Pleroma.Notification do
def get_notified_from_activity(activity, local_only \\ true)
def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do
when type in [
"Create",
"Like",
"Announce",
"Follow",
"Move",
"EmojiReact",
"Flag",
"Update"
] do
potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
potential_receivers =
@ -579,7 +581,24 @@ defmodule Pleroma.Notification do
end
def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag", "actor" => actor}}) do
(User.all_superusers() |> Enum.map(fn user -> user.ap_id end)) -- [actor]
(User.all_users_with_privilege(:reports_manage_reports)
|> Enum.map(fn user -> user.ap_id end)) --
[actor]
end
# Update activity: notify all who repeated this
def get_potential_receiver_ap_ids(%{data: %{"type" => "Update", "actor" => actor}} = activity) do
with %Object{data: %{"id" => object_id}} <- Object.normalize(activity, fetch: false) do
repeaters =
Activity.Queries.by_type("Announce")
|> Activity.Queries.by_object_id(object_id)
|> Activity.with_joined_user_actor()
|> where([a, u], u.local)
|> select([a, u], u.ap_id)
|> Repo.all()
repeaters -- [actor]
end
end
def get_potential_receiver_ap_ids(activity) do
@ -689,7 +708,7 @@ defmodule Pleroma.Notification do
cond do
opts[:type] == "poll" -> false
user.ap_id == actor -> false
!User.following?(follower, user) -> true
!User.following?(user, follower) -> true
true -> false
end
end

22
lib/pleroma/object.ex
View file

@ -40,8 +40,7 @@ defmodule Pleroma.Object do
join(query, join_type, [{object, object_position}], a in Activity,
on:
fragment(
"COALESCE(?->'object'->>'id', ?->>'object') = (? ->> 'id') AND (?->>'type' = ?) ",
a.data,
"associated_object_id(?) = (? ->> 'id') AND (?->>'type' = ?) ",
a.data,
object.data,
a.data,
@ -145,7 +144,7 @@ defmodule Pleroma.Object do
Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
end
def normalize(_, options \\ [fetch: false])
def normalize(_, options \\ [fetch: false, id_only: false])
# If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
# Use this whenever possible, especially when walking graphs in an O(N) loop!
@ -173,10 +172,15 @@ defmodule Pleroma.Object do
def normalize(%{"id" => ap_id}, options), do: normalize(ap_id, options)
def normalize(ap_id, options) when is_binary(ap_id) do
if Keyword.get(options, :fetch) do
Fetcher.fetch_object_from_id!(ap_id, options)
else
get_cached_by_ap_id(ap_id)
cond do
Keyword.get(options, :id_only) ->
ap_id
Keyword.get(options, :fetch) ->
Fetcher.fetch_object_from_id!(ap_id, options)
true ->
get_cached_by_ap_id(ap_id)
end
end
@ -208,10 +212,6 @@ defmodule Pleroma.Object do
end
end
def context_mapping(context) do
Object.change(%Object{}, %{data: %{"id" => context}})
end
def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
%ObjectTombstone{
id: id,

39
lib/pleroma/object/fetcher.ex
View file

@ -4,6 +4,7 @@
defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
alias Pleroma.Instances
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Object.Containment
@ -26,8 +27,42 @@ defmodule Pleroma.Object.Fetcher do
end
defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
has_history? = fn
%{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true
_ -> false
end
internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
remote_history_exists? = has_history?.(new_data)
# If the remote history exists, we treat that as the only source of truth.
new_data =
if has_history?.(old_data) and not remote_history_exists? do
Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"])
else
new_data
end
# If the remote does not have history information, we need to manage it ourselves
new_data =
if not remote_history_exists? do
changed? =
Pleroma.Constants.status_updatable_fields()
|> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end)
%{updated_object: updated_object} =
new_data
|> Object.Updater.maybe_update_history(old_data,
updated: changed?,
use_history_in_new_object?: false
)
updated_object
else
new_data
end
Map.merge(new_data, internal_fields)
end
@ -200,6 +235,10 @@ defmodule Pleroma.Object.Fetcher do
{:ok, body} <- get_object(id),
{:ok, data} <- safe_json_decode(body),
:ok <- Containment.contain_origin_from_id(id, data) do
if not Instances.reachable?(id) do
Instances.set_reachable(id)
end
{:ok, data}
else
{:scheme, _} ->

240
lib/pleroma/object/updater.ex Normal file
View file

@ -0,0 +1,240 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Object.Updater do
require Pleroma.Constants
def update_content_fields(orig_object_data, updated_object) do
Pleroma.Constants.status_updatable_fields()
|> Enum.reduce(
%{data: orig_object_data, updated: false},
fn field, %{data: data, updated: updated} ->
updated =
updated or
(field != "updated" and
Map.get(updated_object, field) != Map.get(orig_object_data, field))
data =
if Map.has_key?(updated_object, field) do
Map.put(data, field, updated_object[field])
else
Map.drop(data, [field])
end
%{data: data, updated: updated}
end
)
end
def maybe_history(object) do
with history <- Map.get(object, "formerRepresentations"),
true <- is_map(history),
"OrderedCollection" <- Map.get(history, "type"),
true <- is_list(Map.get(history, "orderedItems")),
true <- is_integer(Map.get(history, "totalItems")) do
history
else
_ -> nil
end
end
def history_for(object) do
with history when not is_nil(history) <- maybe_history(object) do
history
else
_ -> history_skeleton()
end
end
defp history_skeleton do
%{
"type" => "OrderedCollection",
"totalItems" => 0,
"orderedItems" => []
}
end
def maybe_update_history(
updated_object,
orig_object_data,
opts
) do
updated = opts[:updated]
use_history_in_new_object? = opts[:use_history_in_new_object?]
if not updated do
%{updated_object: updated_object, used_history_in_new_object?: false}
else
# Put edit history
# Note that we may have got the edit history by first fetching the object
{new_history, used_history_in_new_object?} =
with true <- use_history_in_new_object?,
updated_history when not is_nil(updated_history) <- maybe_history(opts[:new_data]) do
{updated_history, true}
else
_ ->
history = history_for(orig_object_data)
latest_history_item =
orig_object_data
|> Map.drop(["id", "formerRepresentations"])
updated_history =
history
|> Map.put("orderedItems", [latest_history_item | history["orderedItems"]])
|> Map.put("totalItems", history["totalItems"] + 1)
{updated_history, false}
end
updated_object =
updated_object
|> Map.put("formerRepresentations", new_history)
%{updated_object: updated_object, used_history_in_new_object?: used_history_in_new_object?}
end
end
defp maybe_update_poll(to_be_updated, updated_object) do
choice_key = fn data ->
if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf"
end
with true <- to_be_updated["type"] == "Question",
key <- choice_key.(updated_object),
true <- key == choice_key.(to_be_updated),
orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])),
new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])),
true <- orig_choices == new_choices do
# Choices are the same, but counts are different
to_be_updated
|> Map.put(key, updated_object[key])
else
# Choices (or vote type) have changed, do not allow this
_ -> to_be_updated
end
end
# This calculates the data to be sent as the object of an Update.
# new_data's formerRepresentations is not considered.
# formerRepresentations is added to the returned data.
def make_update_object_data(original_data, new_data, date) do
%{data: updated_data, updated: updated} =
original_data
|> update_content_fields(new_data)
if not updated do
updated_data
else
%{updated_object: updated_data} =
updated_data
|> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false)
updated_data
|> Map.put("updated", date)
end
end
# This calculates the data of the new Object from an Update.
# new_data's formerRepresentations is considered.
def make_new_object_data_from_update_object(original_data, new_data) do
update_is_reasonable =
with {_, updated} when not is_nil(updated) <- {:cur_updated, new_data["updated"]},
{_, {:ok, updated_time, _}} <- {:cur_updated, DateTime.from_iso8601(updated)},
{_, last_updated} when not is_nil(last_updated) <-
{:last_updated, original_data["updated"] || original_data["published"]},
{_, {:ok, last_updated_time, _}} <-
{:last_updated, DateTime.from_iso8601(last_updated)},
:gt <- DateTime.compare(updated_time, last_updated_time) do
:update_everything
else
# only allow poll updates
{:cur_updated, _} -> :no_content_update
:eq -> :no_content_update
# allow all updates
{:last_updated, _} -> :update_everything
# allow no updates
_ -> false
end
%{
updated_object: updated_data,
used_history_in_new_object?: used_history_in_new_object?,
updated: updated
} =
if update_is_reasonable == :update_everything do
%{data: updated_data, updated: updated} =
original_data
|> update_content_fields(new_data)
updated_data
|> maybe_update_history(original_data,
updated: updated,
use_history_in_new_object?: true,
new_data: new_data
)
|> Map.put(:updated, updated)
else
%{
updated_object: original_data,
used_history_in_new_object?: false,
updated: false
}
end
updated_data =
if update_is_reasonable != false do
updated_data
|> maybe_update_poll(new_data)
else
updated_data
end
%{
updated_data: updated_data,
updated: updated,
used_history_in_new_object?: used_history_in_new_object?
}
end
def for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do
new_items =
Enum.map(items, fun)
|> Enum.reduce_while(
{:ok, []},
fn
{:ok, item}, {:ok, acc} -> {:cont, {:ok, acc ++ [item]}}
e, _acc -> {:halt, e}
end
)
case new_items do
{:ok, items} -> {:ok, Map.put(history, "orderedItems", items)}
e -> e
end
end
def for_each_history_item(history, _, _) do
{:ok, history}
end
def do_with_history(object, fun) do
with history <- object["formerRepresentations"],
object <- Map.drop(object, ["formerRepresentations"]),
{_, {:ok, object}} <- {:main_body, fun.(object)},
{_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do
object =
if history do
Map.put(object, "formerRepresentations", history)
else
object
end
{:ok, object}
else
{:main_body, e} -> e
{:history_items, e} -> e
end
end
end

1
lib/pleroma/reverse_proxy/client/hackney.ex
View file

@ -7,7 +7,6 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
@impl true
def request(method, url, headers, body, opts \\ []) do
opts = Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
:hackney.request(method, url, headers, body, opts)
end

28
lib/pleroma/signature.ex
View file

@ -10,17 +10,14 @@ defmodule Pleroma.Signature do
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
@known_suffixes ["/publickey", "/main-key"]
def key_id_to_actor_id(key_id) do
uri =
URI.parse(key_id)
key_id
|> URI.parse()
|> Map.put(:fragment, nil)
uri =
if not is_nil(uri.path) and String.ends_with?(uri.path, "/publickey") do
Map.put(uri, :path, String.replace(uri.path, "/publickey", ""))
else
uri
end
|> remove_suffix(@known_suffixes)
maybe_ap_id = URI.to_string(uri)
@ -36,6 +33,16 @@ defmodule Pleroma.Signature do
end
end
defp remove_suffix(uri, [test | rest]) do
if not is_nil(uri.path) and String.ends_with?(uri.path, test) do
Map.put(uri, :path, String.replace(uri.path, test, ""))
else
remove_suffix(uri, rest)
end
end
defp remove_suffix(uri, []), do: uri
def fetch_public_key(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid),
@ -59,9 +66,8 @@ defmodule Pleroma.Signature do
end
end
def sign(%User{} = user, headers) do
with {:ok, %{keys: keys}} <- User.ensure_keys_present(user),
{:ok, private_key, _} <- Keys.keys_from_pem(keys) do
def sign(%User{keys: keys} = user, headers) do
with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)
end
end

27
lib/pleroma/upload.ex
View file

@ -36,6 +36,7 @@ defmodule Pleroma.Upload do
alias Ecto.UUID
alias Pleroma.Config
alias Pleroma.Maps
alias Pleroma.Web.ActivityPub.Utils
require Logger
@type source ::
@ -60,12 +61,23 @@ defmodule Pleroma.Upload do
width: integer(),
height: integer(),
blurhash: String.t(),
description: String.t(),
path: String.t()
}
defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]
defstruct [
:id,
:name,
:tempfile,
:content_type,
:width,
:height,
:blurhash,
:description,
:path
]
defp get_description(opts, upload) do
case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do
defp get_description(upload) do
case {upload.description, Pleroma.Config.get([Pleroma.Upload, :default_description])} do
{description, _} when is_binary(description) -> description
{_, :filename} -> upload.name
{_, str} when is_binary(str) -> str
@ -81,13 +93,14 @@ defmodule Pleroma.Upload do
with {:ok, upload} <- prepare_upload(upload, opts),
upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"},
{:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
description = get_description(opts, upload),
description = get_description(upload),
{_, true} <-
{:description_limit,
String.length(description) <= Pleroma.Config.get([:instance, :description_limit])},
{:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do
{:ok,
%{
"id" => Utils.generate_object_id(),
"type" => opts.activity_type,
"mediaType" => upload.content_type,
"url" => [
@ -152,7 +165,8 @@ defmodule Pleroma.Upload do
id: UUID.generate(),
name: file.filename,
tempfile: file.path,
content_type: file.content_type
content_type: file.content_type,
description: opts.description
}}
end
end
@ -172,7 +186,8 @@ defmodule Pleroma.Upload do
id: UUID.generate(),
name: hash <> "." <> ext,
tempfile: tmp_path,
content_type: content_type
content_type: content_type,
description: opts.description
}}
end
end

52
lib/pleroma/upload/filter/exiftool/read_description.ex Normal file
View file

@ -0,0 +1,52 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do
@moduledoc """
Gets a valid description from the related EXIF tags and provides them in the response if no description is provided yet.
It will first check ImageDescription, when that doesn't probide a valid description, it will check iptc:Caption-Abstract.
A valid description means the fields are filled in and not too long (see `:instance, :description_limit`).
"""
@behaviour Pleroma.Upload.Filter
@spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
def filter(%Pleroma.Upload{description: description})
when is_binary(description),
do: {:ok, :noop}
def filter(%Pleroma.Upload{tempfile: file} = upload),
do: {:ok, :filtered, upload |> Map.put(:description, read_description_from_exif_data(file))}
def filter(_, _), do: {:ok, :noop}
defp read_description_from_exif_data(file) do
nil
|> read_when_empty(file, "-ImageDescription")
|> read_when_empty(file, "-iptc:Caption-Abstract")
end
defp read_when_empty(current_description, _, _) when is_binary(current_description),
do: current_description
defp read_when_empty(_, file, tag) do
try do
{tag_content, 0} =
System.cmd("exiftool", ["-b", "-s3", tag, file],
stderr_to_stdout: false,
parallelism: true
)
tag_content = String.trim(tag_content)
if tag_content != "" and
String.length(tag_content) <=
Pleroma.Config.get([:instance, :description_limit]),
do: tag_content,
else: nil
rescue
_ in ErlangError -> nil
end
end
end

3
lib/pleroma/upload/filter/exiftool.ex → lib/pleroma/upload/filter/exiftool/strip_location.ex
View file

@ -2,7 +2,7 @@
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Upload.Filter.Exiftool do
defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
@moduledoc """
Strips GPS related EXIF tags and overwrites the file in place.
Also strips or replaces filesystem metadata e.g., timestamps.
@ -14,6 +14,7 @@ defmodule Pleroma.Upload.Filter.Exiftool do
# Formats not compatible with exiftool at this time
def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
def filter(%Pleroma.Upload{content_type: "image/svg" <> _}), do: {:ok, :noop}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do

208
lib/pleroma/user.ex
View file

@ -326,7 +326,7 @@ defmodule Pleroma.User do
end
def visible_for(%User{} = user, for_user) do
if superuser?(for_user) do
if privileged?(for_user, :users_manage_activation_state) do
:visible
else
visible_account_status(user)
@ -353,10 +353,45 @@ defmodule Pleroma.User do
end
end
@spec superuser?(User.t()) :: boolean()
def superuser?(%User{local: true, is_admin: true}), do: true
def superuser?(%User{local: true, is_moderator: true}), do: true
def superuser?(_), do: false
@spec privileged?(User.t(), atom()) :: boolean()
def privileged?(%User{is_admin: false, is_moderator: false}, _), do: false
def privileged?(
%User{local: true, is_admin: is_admin, is_moderator: is_moderator},
privilege_tag
),
do:
privileged_for?(privilege_tag, is_admin, :admin_privileges) or
privileged_for?(privilege_tag, is_moderator, :moderator_privileges)
def privileged?(_, _), do: false
defp privileged_for?(privilege_tag, true, config_role_key),
do: privilege_tag in Config.get([:instance, config_role_key])
defp privileged_for?(_, _, _), do: false
@spec privileges(User.t()) :: [atom()]
def privileges(%User{local: false}) do
[]
end
def privileges(%User{is_moderator: false, is_admin: false}) do
[]
end
def privileges(%User{local: true, is_moderator: true, is_admin: true}) do
(Config.get([:instance, :moderator_privileges]) ++ Config.get([:instance, :admin_privileges]))
|> Enum.uniq()
end
def privileges(%User{local: true, is_moderator: true, is_admin: false}) do
Config.get([:instance, :moderator_privileges])
end
def privileges(%User{local: true, is_moderator: false, is_admin: true}) do
Config.get([:instance, :admin_privileges])
end
@spec invisible?(User.t()) :: boolean()
def invisible?(%User{invisible: true}), do: true
@ -611,7 +646,13 @@ defmodule Pleroma.User do
{:ok, new_value} <- value_function.(value) do
put_change(changeset, map_field, new_value)
else
_ -> changeset
{:error, :file_too_large} ->
Ecto.Changeset.validate_change(changeset, map_field, fn map_field, _value ->
[{map_field, "file is too large"}]
end)
_ ->
changeset
end
end
@ -706,11 +747,12 @@ defmodule Pleroma.User do
])
|> validate_required([:name, :nickname])
|> unique_constraint(:nickname)
|> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
|> validate_not_restricted_nickname(:nickname)
|> validate_format(:nickname, local_nickname_regex())
|> put_ap_id()
|> unique_constraint(:ap_id)
|> put_following_and_follower_and_featured_address()
|> put_private_key()
end
def register_changeset(struct, params \\ %{}, opts \\ []) do
@ -754,17 +796,9 @@ defmodule Pleroma.User do
|> validate_confirmation(:password)
|> unique_constraint(:email)
|> validate_format(:email, @email_regex)
|> validate_change(:email, fn :email, email ->
valid? =
Config.get([User, :email_blacklist])
|> Enum.all?(fn blacklisted_domain ->
!String.ends_with?(email, ["@" <> blacklisted_domain, "." <> blacklisted_domain])
end)
if valid?, do: [], else: [email: "Invalid email"]
end)
|> validate_email_not_in_blacklisted_domain(:email)
|> unique_constraint(:nickname)
|> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
|> validate_not_restricted_nickname(:nickname)
|> validate_format(:nickname, local_nickname_regex())
|> validate_length(:bio, max: bio_limit)
|> validate_length(:name, min: 1, max: name_limit)
@ -776,6 +810,36 @@ defmodule Pleroma.User do
|> put_ap_id()
|> unique_constraint(:ap_id)
|> put_following_and_follower_and_featured_address()
|> put_private_key()
end
def validate_not_restricted_nickname(changeset, field) do
validate_change(changeset, field, fn _, value ->
valid? =
Config.get([User, :restricted_nicknames])
|> Enum.all?(fn restricted_nickname ->
String.downcase(value) != String.downcase(restricted_nickname)
end)
if valid?, do: [], else: [nickname: "Invalid nickname"]
end)
end
def validate_email_not_in_blacklisted_domain(changeset, field) do
validate_change(changeset, field, fn _, value ->
valid? =
Config.get([User, :email_blacklist])
|> Enum.all?(fn blacklisted_domain ->
blacklisted_domain_downcase = String.downcase(blacklisted_domain)
!String.ends_with?(String.downcase(value), [
"@" <> blacklisted_domain_downcase,
"." <> blacklisted_domain_downcase
])
end)
if valid?, do: [], else: [email: "Invalid email"]
end)
end
def maybe_validate_required_email(changeset, true), do: changeset
@ -825,6 +889,11 @@ defmodule Pleroma.User do
|> put_change(:featured_address, featured)
end
defp put_private_key(changeset) do
{:ok, pem} = Keys.generate_rsa_pem()
put_change(changeset, :keys, pem)
end
defp autofollow_users(user) do
candidates = Config.get([:instance, :autofollowed_nicknames])
@ -877,7 +946,7 @@ defmodule Pleroma.User do
end
end
defp send_user_approval_email(user) do
defp send_user_approval_email(%User{email: email} = user) when is_binary(email) do
user
|> Pleroma.Emails.UserEmail.approval_pending_email()
|> Pleroma.Emails.Mailer.deliver_async()
@ -885,6 +954,10 @@ defmodule Pleroma.User do
{:ok, :enqueued}
end
defp send_user_approval_email(_user) do
{:ok, :skipped}
end
defp send_admin_approval_emails(user) do
all_superusers()
|> Enum.filter(fn user -> not is_nil(user.email) end)
@ -1129,24 +1202,10 @@ defmodule Pleroma.User do
|> update_and_set_cache()
end
def update_and_set_cache(%{data: %Pleroma.User{} = user} = changeset) do
was_superuser_before_update = User.superuser?(user)
def update_and_set_cache(changeset) do
with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
set_cache(user)
end
|> maybe_remove_report_notifications(was_superuser_before_update)
end
defp maybe_remove_report_notifications({:ok, %Pleroma.User{} = user} = result, true) do
if not User.superuser?(user),
do: user |> Notification.destroy_multiple_from_types(["pleroma:report"])
result
end
defp maybe_remove_report_notifications(result, _) do
result
end
def get_user_friends_ap_ids(user) do
@ -1459,17 +1518,30 @@ defmodule Pleroma.User do
{:ok, list(UserRelationship.t())} | {:error, String.t()}
def mute(%User{} = muter, %User{} = mutee, params \\ %{}) do
notifications? = Map.get(params, :notifications, true)
expires_in = Map.get(params, :expires_in, 0)
duration = Map.get(params, :duration, 0)
with {:ok, user_mute} <- UserRelationship.create_mute(muter, mutee),
expires_at =
if duration > 0 do
DateTime.utc_now()
|> DateTime.add(duration)
else
nil
end
with {:ok, user_mute} <- UserRelationship.create_mute(muter, mutee, expires_at),
{:ok, user_notification_mute} <-
(notifications? && UserRelationship.create_notification_mute(muter, mutee)) ||
(notifications? &&
UserRelationship.create_notification_mute(
muter,
mutee,
expires_at
)) ||
{:ok, nil} do
if expires_in > 0 do
if duration > 0 do
Pleroma.Workers.MuteExpireWorker.enqueue(
"unmute_user",
%{"muter_id" => muter.id, "mutee_id" => mutee.id},
schedule_in: expires_in
scheduled_at: expires_at
)
end
@ -1540,13 +1612,19 @@ defmodule Pleroma.User do
blocker
end
# clear any requested follows as well
# clear any requested follows from both sides as well
blocked =
case CommonAPI.reject_follow_request(blocked, blocker) do
{:ok, %User{} = updated_blocked} -> updated_blocked
nil -> blocked
end
blocker =
case CommonAPI.reject_follow_request(blocker, blocked) do
{:ok, %User{} = updated_blocker} -> updated_blocker
nil -> blocker
end
unsubscribe(blocked, blocker)
unfollowing_blocked = Config.get([:activitypub, :unfollow_blocked], true)
@ -2046,6 +2124,7 @@ defmodule Pleroma.User do
follower_address: uri <> "/followers"
}
|> change
|> put_private_key()
|> unique_constraint(:nickname)
|> Repo.insert()
|> set_cache()
@ -2078,7 +2157,8 @@ defmodule Pleroma.User do
@doc "Gets or fetch a user by uri or nickname."
@spec get_or_fetch(String.t()) :: {:ok, User.t()} | {:error, String.t()}
def get_or_fetch("http" <> _host = uri), do: get_or_fetch_by_ap_id(uri)
def get_or_fetch("http://" <> _host = uri), do: get_or_fetch_by_ap_id(uri)
def get_or_fetch("https://" <> _host = uri), do: get_or_fetch_by_ap_id(uri)
def get_or_fetch(nickname), do: get_or_fetch_by_nickname(nickname)
# wait a period of time and return newest version of the User structs
@ -2206,6 +2286,11 @@ defmodule Pleroma.User do
|> Repo.all()
end
@spec all_users_with_privilege(atom()) :: [User.t()]
def all_users_with_privilege(privilege) do
User.Query.build(%{is_privileged: privilege}) |> Repo.all()
end
def muting_reblogs?(%User{} = user, %User{} = target) do
UserRelationship.reblog_mute_exists?(user, target)
end
@ -2311,17 +2396,6 @@ defmodule Pleroma.User do
}
end
def ensure_keys_present(%{keys: keys} = user) when not is_nil(keys), do: {:ok, user}
def ensure_keys_present(%User{} = user) do
with {:ok, pem} <- Keys.generate_rsa_pem() do
user
|> cast(%{keys: pem}, [:keys])
|> validate_required([:keys])
|> update_and_set_cache()
end
end
def get_ap_ids_by_nicknames(nicknames) do
from(u in User,
where: u.nickname in ^nicknames,
@ -2364,6 +2438,38 @@ defmodule Pleroma.User do
|> update_and_set_cache()
end
def alias_users(user) do
user.also_known_as
|> Enum.map(&User.get_cached_by_ap_id/1)
|> Enum.filter(fn user -> user != nil end)
end
def add_alias(user, new_alias_user) do
current_aliases = user.also_known_as || []
new_alias_ap_id = new_alias_user.ap_id
if new_alias_ap_id in current_aliases do
{:ok, user}
else
user
|> cast(%{also_known_as: current_aliases ++ [new_alias_ap_id]}, [:also_known_as])
|> update_and_set_cache()
end
end
def delete_alias(user, alias_user) do
current_aliases = user.also_known_as || []
alias_ap_id = alias_user.ap_id
if alias_ap_id in current_aliases do
user
|> cast(%{also_known_as: current_aliases -- [alias_ap_id]}, [:also_known_as])
|> update_and_set_cache()
else
{:error, :no_such_alias}
end
end
# Internal function; public one is `deactivate/2`
defp set_activation_status(user, status) do
user

18
lib/pleroma/user/backup.ex
View file

@ -32,9 +32,7 @@ defmodule Pleroma.User.Backup do
end
def create(user, admin_id \\ nil) do
with :ok <- validate_email_enabled(),
:ok <- validate_user_email(user),
:ok <- validate_limit(user, admin_id),
with :ok <- validate_limit(user, admin_id),
{:ok, backup} <- user |> new() |> Repo.insert() do
BackupWorker.process(backup, admin_id)
end
@ -86,20 +84,6 @@ defmodule Pleroma.User.Backup do
end
end
defp validate_email_enabled do
if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
:ok
else
{:error, dgettext("errors", "Backups require enabled email")}
end
end
defp validate_user_email(%User{email: nil}) do
{:error, dgettext("errors", "Email is required")}
end
defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
def get_last(user_id) do
__MODULE__
|> where(user_id: ^user_id)

39
lib/pleroma/user/query.ex
View file

@ -29,6 +29,7 @@ defmodule Pleroma.User.Query do
import Ecto.Query
import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.Config
alias Pleroma.FollowingRelationship
alias Pleroma.User
@ -49,6 +50,7 @@ defmodule Pleroma.User.Query do
is_suggested: boolean(),
is_discoverable: boolean(),
super_users: boolean(),
is_privileged: atom(),
invisible: boolean(),
internal: boolean(),
followers: User.t(),
@ -136,6 +138,43 @@ defmodule Pleroma.User.Query do
)
end
defp compose_query({:is_privileged, privilege}, query) do
moderator_privileged = privilege in Config.get([:instance, :moderator_privileges])
admin_privileged = privilege in Config.get([:instance, :admin_privileges])
query = compose_query({:active, true}, query)
query = compose_query({:local, true}, query)
case {admin_privileged, moderator_privileged} do
{false, false} ->
where(
query,
false
)
{true, true} ->
where(
query,
[u],
u.is_admin or u.is_moderator
)
{true, false} ->
where(
query,
[u],
u.is_admin
)
{false, true} ->
where(
query,
[u],
u.is_moderator
)
end
end
defp compose_query({:local, _}, query), do: location_query(query, true)
defp compose_query({:external, _}, query), do: location_query(query, false)

5
lib/pleroma/user/search.ex
View file

@ -94,6 +94,7 @@ defmodule Pleroma.User.Search do
|> subquery()
|> order_by(desc: :search_rank)
|> maybe_restrict_local(for_user)
|> filter_deactivated_users()
end
defp select_top_users(query, top_user_ids) do
@ -166,6 +167,10 @@ defmodule Pleroma.User.Search do
from(q in query, where: q.actor_type != "Application")
end
defp filter_deactivated_users(query) do
from(q in query, where: q.is_active == true)
end
defp filter_blocked_user(query, %User{} = blocker) do
query
|> join(:left, [u], b in Pleroma.UserRelationship,

44
lib/pleroma/user_relationship.ex
View file

@ -18,16 +18,17 @@ defmodule Pleroma.UserRelationship do
belongs_to(:source, User, type: FlakeId.Ecto.CompatType)
belongs_to(:target, User, type: FlakeId.Ecto.CompatType)
field(:relationship_type, Pleroma.UserRelationship.Type)
field(:expires_at, :utc_datetime)
timestamps(updated_at: false)
end
for relationship_type <- Keyword.keys(Pleroma.UserRelationship.Type.__enum_map__()) do
# `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`,
# `def create_notification_mute/2`, `def create_inverse_subscription/2`,
# `def endorsement/2`
def unquote(:"create_#{relationship_type}")(source, target),
do: create(unquote(relationship_type), source, target)
# `def create_block/3`, `def create_mute/3`, `def create_reblog_mute/3`,
# `def create_notification_mute/3`, `def create_inverse_subscription/3`,
# `def endorsement/3`
def unquote(:"create_#{relationship_type}")(source, target, expires_at \\ nil),
do: create(unquote(relationship_type), source, target, expires_at)
# `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`,
# `def delete_notification_mute/2`, `def delete_inverse_subscription/2`,
@ -37,9 +38,15 @@ defmodule Pleroma.UserRelationship do
# `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`,
# `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2`,
# `def inverse_endorsement?/2`
# `def inverse_endorsement_exists?/2`
def unquote(:"#{relationship_type}_exists?")(source, target),
do: exists?(unquote(relationship_type), source, target)
# `def get_block_expire_date/2`, `def get_mute_expire_date/2`,
# `def get_reblog_mute_expire_date/2`, `def get_notification_mute_exists?/2`,
# `def get_inverse_subscription_expire_date/2`, `def get_inverse_endorsement_expire_date/2`
def unquote(:"get_#{relationship_type}_expire_date")(source, target),
do: get_expire_date(unquote(relationship_type), source, target)
end
def user_relationship_types, do: Keyword.keys(user_relationship_mappings())
@ -48,7 +55,7 @@ defmodule Pleroma.UserRelationship do
def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do
user_relationship
|> cast(params, [:relationship_type, :source_id, :target_id])
|> cast(params, [:relationship_type, :source_id, :target_id, :expires_at])
|> validate_required([:relationship_type, :source_id, :target_id])
|> unique_constraint(:relationship_type,
name: :user_relationships_source_id_relationship_type_target_id_index
@ -62,16 +69,31 @@ defmodule Pleroma.UserRelationship do
|> Repo.exists?()
end
def create(relationship_type, %User{} = source, %User{} = target) do
def get_expire_date(relationship_type, %User{} = source, %User{} = target) do
%UserRelationship{expires_at: expires_at} =
UserRelationship
|> where(
relationship_type: ^relationship_type,
source_id: ^source.id,
target_id: ^target.id
)
|> Repo.one!()
expires_at
end
def create(relationship_type, %User{} = source, %User{} = target, expires_at \\ nil) do
%UserRelationship{}
|> changeset(%{
relationship_type: relationship_type,
source_id: source.id,
target_id: target.id
target_id: target.id,
expires_at: expires_at
})
|> Repo.insert(
on_conflict: {:replace_all_except, [:id]},
conflict_target: [:source_id, :relationship_type, :target_id]
on_conflict: {:replace_all_except, [:id, :inserted_at]},
conflict_target: [:source_id, :relationship_type, :target_id],
returning: true
)
end

124
lib/pleroma/web/activity_pub/activity_pub.ex
View file

@ -190,7 +190,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
def notify_and_stream(activity) do
Notification.create_notifications(activity)
conversation = create_or_bump_conversation(activity, activity.actor)
original_activity =
case activity do
%{data: %{"type" => "Update"}, object: %{data: %{"id" => id}}} ->
Activity.get_create_by_object_ap_id_with_object(id)
_ ->
activity
end
conversation = create_or_bump_conversation(original_activity, original_activity.actor)
participations = get_participations(conversation)
stream_out(activity)
stream_out_participations(participations)
@ -256,7 +265,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
@impl true
def stream_out(%Activity{data: %{"type" => data_type}} = activity)
when data_type in ["Create", "Announce", "Delete"] do
when data_type in ["Create", "Announce", "Delete", "Update"] do
activity
|> Topics.get_activity_topics()
|> Streamer.stream(activity)
@ -392,11 +401,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
_ <- notify_and_stream(activity),
:ok <-
maybe_federate(stripped_activity) do
User.all_superusers()
User.all_users_with_privilege(:reports_manage_reports)
|> Enum.filter(fn user -> user.ap_id != actor end)
|> Enum.filter(fn user -> not is_nil(user.email) end)
|> Enum.each(fn superuser ->
superuser
|> Enum.each(fn privileged_user ->
privileged_user
|> Pleroma.Emails.AdminEmail.report(actor, account, statuses, content)
|> Pleroma.Emails.Mailer.deliver_async()
end)
@ -413,7 +422,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
"type" => "Move",
"actor" => origin.ap_id,
"object" => origin.ap_id,
"target" => target.ap_id
"target" => target.ap_id,
"to" => [origin.follower_address]
}
with true <- origin.ap_id in target.also_known_as,
@ -501,9 +511,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
@spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
includes_local_public = Map.get(opts, :includes_local_public, false)
opts = Map.delete(opts, :user)
[Constants.as_public()]
intended_recipients =
if includes_local_public do
[Constants.as_public(), as_local_public()]
else
[Constants.as_public()]
end
intended_recipients
|> fetch_activities_query(opts)
|> restrict_unlisted(opts)
|> fetch_paginated_optimized(opts, pagination)
@ -603,9 +622,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
do: query
defp restrict_thread_visibility(query, %{user: %User{ap_id: ap_id}}, _) do
local_public = as_local_public()
from(
a in query,
where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data)
where: fragment("thread_visibility(?, (?)->>'id', ?) = true", ^ap_id, a.data, ^local_public)
)
end
@ -692,8 +713,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp user_activities_recipients(%{godmode: true}), do: []
defp user_activities_recipients(%{reading_user: reading_user}) do
if reading_user do
[Constants.as_public(), reading_user.ap_id | User.following(reading_user)]
if not is_nil(reading_user) and reading_user.local do
[
Constants.as_public(),
as_local_public(),
reading_user.ap_id | User.following(reading_user)
]
else
[Constants.as_public()]
end
@ -1134,8 +1159,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
[activity, object: o] in query,
where:
fragment(
"(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)",
activity.data,
"(?)->>'type' = 'Create' and associated_object_id((?)) = any (?)",
activity.data,
activity.data,
^ids
@ -1215,15 +1239,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
end
defp exclude_invisible_actors(query, %{type: "Flag"}), do: query
defp exclude_invisible_actors(query, %{invisible_actors: true}), do: query
defp exclude_invisible_actors(query, _opts) do
invisible_ap_ids =
User.Query.build(%{invisible: true, select: [:ap_id]})
|> Repo.all()
|> Enum.map(fn %{ap_id: ap_id} -> ap_id end)
from([activity] in query, where: activity.actor not in ^invisible_ap_ids)
query
|> join(:inner, [activity], u in User,
as: :u,
on: activity.actor == u.ap_id and u.invisible == false
)
end
defp exclude_id(query, %{exclude_id: id}) when is_binary(id) do
@ -1353,7 +1377,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> restrict_instance(opts)
|> restrict_announce_object_actor(opts)
|> restrict_filtered(opts)
|> Activity.restrict_deactivated_users()
|> maybe_restrict_deactivated_users(opts)
|> exclude_poll_votes(opts)
|> exclude_chat_messages(opts)
|> exclude_invisible_actors(opts)
@ -1458,7 +1482,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image()
defp normalize_image(_), do: nil
defp object_to_user_data(data) do
defp object_to_user_data(data, additional) do
fields =
data
|> Map.get("attachment", [])
@ -1490,15 +1514,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
public_key =
if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do
data["publicKey"]["publicKeyPem"]
else
nil
end
shared_inbox =
if is_map(data["endpoints"]) && is_binary(data["endpoints"]["sharedInbox"]) do
data["endpoints"]["sharedInbox"]
else
nil
end
birthday =
@ -1507,13 +1527,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
{:ok, date} -> date
{:error, _} -> nil
end
else
nil
end
show_birthday = !!birthday
user_data = %{
# if WebFinger request was already done, we probably have acct, otherwise
# we request WebFinger here
nickname = additional[:nickname_from_acct] || generate_nickname(data)
%{
ap_id: data["id"],
uri: get_actor_url(data["url"]),
ap_enabled: true,
@ -1535,23 +1557,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
inbox: data["inbox"],
shared_inbox: shared_inbox,
accepts_chat_messages: accepts_chat_messages,
pinned_objects: pinned_objects,
birthday: birthday,
show_birthday: show_birthday
show_birthday: show_birthday,
pinned_objects: pinned_objects,
nickname: nickname
}
end
# nickname can be nil because of virtual actors
if data["preferredUsername"] do
Map.put(
user_data,
:nickname,
"#{data["preferredUsername"]}@#{URI.parse(data["id"]).host}"
)
defp generate_nickname(%{"preferredUsername" => username} = data) when is_binary(username) do
generated = "#{username}@#{URI.parse(data["id"]).host}"
if Config.get([WebFinger, :update_nickname_on_user_fetch]) do
case WebFinger.finger(generated) do
{:ok, %{"subject" => "acct:" <> acct}} -> acct
_ -> generated
end
else
Map.put(user_data, :nickname, nil)
generated
end
end
# nickname can be nil because of virtual actors
defp generate_nickname(_), do: nil
def fetch_follow_information_for_user(user) do
with {:ok, following_data} <-
Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
@ -1623,17 +1651,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp collection_private(_data), do: {:ok, true}
def user_data_from_user_object(data) do
def user_data_from_user_object(data, additional \\ []) do
with {:ok, data} <- MRF.filter(data) do
{:ok, object_to_user_data(data)}
{:ok, object_to_user_data(data, additional)}
else
e -> {:error, e}
end
end
def fetch_and_prepare_user_from_ap_id(ap_id) do
def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do
with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
{:ok, data} <- user_data_from_user_object(data) do
{:ok, data} <- user_data_from_user_object(data, additional) do
{:ok, maybe_update_follow_information(data)}
else
# If this has been deleted, only log a debug and not an error
@ -1711,13 +1739,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
end
def make_user_from_ap_id(ap_id) do
def make_user_from_ap_id(ap_id, additional \\ []) do
user = User.get_cached_by_ap_id(ap_id)
if user && !User.ap_enabled?(user) do
Transmogrifier.upgrade_user_from_ap_id(ap_id)
else
with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
{:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
if user do
@ -1737,8 +1765,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
def make_user_from_nickname(nickname) do
with {:ok, %{"ap_id" => ap_id}} when not is_nil(ap_id) <- WebFinger.finger(nickname) do
make_user_from_ap_id(ap_id)
with {:ok, %{"ap_id" => ap_id, "subject" => "acct:" <> acct}} when not is_nil(ap_id) <-
WebFinger.finger(nickname) do
make_user_from_ap_id(ap_id, nickname_from_acct: acct)
else
_e -> {:error, "No AP id in WebFinger"}
end
@ -1760,4 +1789,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> restrict_visibility(%{visibility: "direct"})
|> order_by([activity], asc: activity.id)
end
defp maybe_restrict_deactivated_users(activity, %{type: "Flag"}), do: activity
defp maybe_restrict_deactivated_users(activity, _opts),
do: Activity.restrict_deactivated_users(activity)
end

63
lib/pleroma/web/activity_pub/activity_pub_controller.ex
View file

@ -66,8 +66,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
end
def user(conn, %{"nickname" => nickname}) do
with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
{:ok, user} <- User.ensure_keys_present(user) do
with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
conn
|> put_resp_content_type("application/activity+json")
|> put_view(UserView)
@ -84,6 +83,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
user <- Map.get(assigns, :user, nil),
{_, true} <- {:visible?, Visibility.visible_for_user?(object, user)} do
conn
|> maybe_skip_cache(user)
|> assign(:tracking_fun_data, object.id)
|> set_cache_ttl_for(object)
|> put_resp_content_type("application/activity+json")
@ -112,6 +112,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
user <- Map.get(assigns, :user, nil),
{_, true} <- {:visible?, Visibility.visible_for_user?(activity, user)} do
conn
|> maybe_skip_cache(user)
|> maybe_set_tracking_data(activity)
|> set_cache_ttl_for(activity)
|> put_resp_content_type("application/activity+json")
@ -151,6 +152,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
assign(conn, :cache_ttl, ttl)
end
def maybe_skip_cache(conn, user) do
if user do
conn
|> assign(:skip_cache, true)
else
conn
end
end
# GET /relay/following
def relay_following(conn, _params) do
with %{halted: false} = conn <- FederatingPlug.call(conn, []) do
@ -163,7 +173,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
def following(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "page" => page}) do
with %User{} = user <- User.get_cached_by_nickname(nickname),
{user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user),
{:show_follows, true} <-
{:show_follows, (for_user && for_user == user) || !user.hide_follows} do
{page, _} = Integer.parse(page)
@ -181,8 +190,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
end
def following(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname}) do
with %User{} = user <- User.get_cached_by_nickname(nickname),
{user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
with %User{} = user <- User.get_cached_by_nickname(nickname) do
conn
|> put_resp_content_type("application/activity+json")
|> put_view(UserView)
@ -202,7 +210,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
def followers(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "page" => page}) do
with %User{} = user <- User.get_cached_by_nickname(nickname),
{user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user),
{:show_followers, true} <-
{:show_followers, (for_user && for_user == user) || !user.hide_followers} do
{page, _} = Integer.parse(page)
@ -220,8 +227,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
end
def followers(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname}) do
with %User{} = user <- User.get_cached_by_nickname(nickname),
{user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
with %User{} = user <- User.get_cached_by_nickname(nickname) do
conn
|> put_resp_content_type("application/activity+json")
|> put_view(UserView)
@ -234,8 +240,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
%{"nickname" => nickname, "page" => page?} = params
)
when page? in [true, "true"] do
with %User{} = user <- User.get_cached_by_nickname(nickname),
{:ok, user} <- User.ensure_keys_present(user) do
with %User{} = user <- User.get_cached_by_nickname(nickname) do
# "include_poll_votes" is a hack because postgres generates inefficient
# queries when filtering by 'Answer', poll votes will be hidden by the
# visibility filter in this case anyway
@ -259,8 +264,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
end
def outbox(conn, %{"nickname" => nickname}) do
with %User{} = user <- User.get_cached_by_nickname(nickname),
{:ok, user} <- User.ensure_keys_present(user) do
with %User{} = user <- User.get_cached_by_nickname(nickname) do
conn
|> put_resp_content_type("application/activity+json")
|> put_view(UserView)
@ -317,14 +321,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
end
defp represent_service_actor(%User{} = user, conn) do
with {:ok, user} <- User.ensure_keys_present(user) do
conn
|> put_resp_content_type("application/activity+json")
|> put_view(UserView)
|> render("user.json", %{user: user})
else
nil -> {:error, :not_found}
end
conn
|> put_resp_content_type("application/activity+json")
|> put_view(UserView)
|> render("user.json", %{user: user})
end
defp represent_service_actor(nil, _), do: {:error, :not_found}
@ -377,12 +377,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
def read_inbox(%{assigns: %{user: %User{nickname: nickname} = user}} = conn, %{
"nickname" => nickname
}) do
with {:ok, user} <- User.ensure_keys_present(user) do
conn
|> put_resp_content_type("application/activity+json")
|> put_view(UserView)
|> render("activity_collection.json", %{iri: "#{user.ap_id}/inbox"})
end
conn
|> put_resp_content_type("application/activity+json")
|> put_view(UserView)
|> render("activity_collection.json", %{iri: "#{user.ap_id}/inbox"})
end
def read_inbox(%{assigns: %{user: %User{nickname: as_nickname}}} = conn, %{
@ -519,19 +517,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
conn
end
defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
{:ok, new_user} = User.ensure_keys_present(user)
for_user =
if new_user != user and match?(%User{}, for_user) do
User.get_cached_by_nickname(for_user.nickname)
else
for_user
end
{new_user, for_user}
end
def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
with {:ok, object} <-
ActivityPub.upload(

13
lib/pleroma/web/activity_pub/builder.ex
View file

@ -218,10 +218,16 @@ defmodule Pleroma.Web.ActivityPub.Builder do
end
end
# Retricted to user updates for now, always public
@spec update(User.t(), Object.t()) :: {:ok, map(), keyword()}
def update(actor, object) do
to = [Pleroma.Constants.as_public(), actor.follower_address]
{to, cc} =
if object["type"] in Pleroma.Constants.actor_types() do
# User updates, always public
{[Pleroma.Constants.as_public(), actor.follower_address], []}
else
# Status updates, follow the recipients in the object
{object["to"] || [], object["cc"] || []}
end
{:ok,
%{
@ -229,7 +235,8 @@ defmodule Pleroma.Web.ActivityPub.Builder do
"type" => "Update",
"actor" => actor.ap_id,
"object" => object,
"to" => to
"to" => to,
"cc" => cc
}, []}
end

45
lib/pleroma/web/activity_pub/mrf.ex
View file

@ -53,10 +53,53 @@ defmodule Pleroma.Web.ActivityPub.MRF do
@required_description_keys [:key, :related_policy]
def filter_one(policy, message) do
should_plug_history? =
if function_exported?(policy, :history_awareness, 0) do
policy.history_awareness()
else
:manual
end
|> Kernel.==(:auto)
if not should_plug_history? do
policy.filter(message)
else
main_result = policy.filter(message)
with {_, {:ok, main_message}} <- {:main, main_result},
{_,
%{
"formerRepresentations" => %{
"orderedItems" => [_ | _]
}
}} = {_, object} <- {:object, message["object"]},
{_, {:ok, new_history}} <-
{:history,
Pleroma.Object.Updater.for_each_history_item(
object["formerRepresentations"],
object,
fn item ->
with {:ok, filtered} <- policy.filter(Map.put(message, "object", item)) do
{:ok, filtered["object"]}
else
e -> e
end
end
)} do
{:ok, put_in(main_message, ["object", "formerRepresentations"], new_history)}
else
{:main, _} -> main_result
{:object, _} -> main_result
{:history, e} -> e
end
end
end
def filter(policies, %{} = message) do
policies
|> Enum.reduce({:ok, message}, fn
policy, {:ok, message} -> policy.filter(message)
policy, {:ok, message} -> filter_one(policy, message)
_, error -> error
end)
end

21
lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
View file

@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
defp score_displayname("fedibot"), do: 1.0
defp score_displayname(_), do: 0.0
defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
defp determine_if_followbot(%User{nickname: nickname, name: displayname, actor_type: actor_type}) do
# nickname will be a binary string except when following a relay
nick_score =
if is_binary(nickname) do
@ -45,19 +45,32 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
0.0
end
nick_score + name_score
# actor_type "Service" is a Bot account
actor_type_score =
if actor_type == "Service" do
1.0
else
0.0
end
nick_score + name_score + actor_type_score
end
defp determine_if_followbot(_), do: 0.0
defp bot_allowed?(%{"object" => target}, bot_actor) do
%User{} = user = normalize_by_ap_id(target)
User.following?(user, bot_actor)
end
@impl true
def filter(%{"type" => "Follow", "actor" => actor_id} = message) do
%User{} = actor = normalize_by_ap_id(actor_id)
score = determine_if_followbot(actor)
# TODO: scan biography data for keywords and score it somehow.
if score < 0.8 do
if score < 0.8 || bot_allowed?(message, actor) do
{:ok, message}
else
{:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"}

3
lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
View file

@ -9,6 +9,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
require Logger
@impl true
def history_awareness, do: :auto
# has the user successfully posted before?
defp old_user?(%User{} = u) do
u.note_count > 0 || u.follower_count > 0

6
lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex
View file

@ -10,6 +10,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
@reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless])
def history_awareness, do: :auto
def filter_by_summary(
%{data: %{"summary" => parent_summary}} = _in_reply_to,
%{"summary" => child_summary} = child
@ -27,8 +29,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
def filter_by_summary(_in_reply_to, child), do: child
def filter(%{"type" => "Create", "object" => child_object} = object)
when is_map(child_object) do
def filter(%{"type" => type, "object" => child_object} = object)
when type in ["Create", "Update"] and is_map(child_object) do
child =
child_object["inReplyTo"]
|> Object.normalize(fetch: false)

7
lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex
View file

@ -11,6 +11,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def history_awareness, do: :auto
defp do_extract({:a, attrs, _}, acc) do
if Enum.find(attrs, fn {name, value} ->
name == "class" && value in ["mention", "u-url mention", "mention u-url"]
@ -74,11 +77,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do
@impl true
def filter(
%{
"type" => "Create",
"type" => type,
"object" => %{"type" => "Note", "to" => to, "inReplyTo" => in_reply_to}
} = object
)
when is_list(to) and is_binary(in_reply_to) do
when type in ["Create", "Update"] and is_list(to) and is_binary(in_reply_to) do
# image-only posts from pleroma apparently reach this MRF without the content field
content = object["object"]["content"] || ""

47
lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex
View file

@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def history_awareness, do: :manual
defp check_reject(message, hashtags) do
if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do
{:reject, "[HashtagPolicy] Matches with rejected keyword"}
@ -47,22 +50,46 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
defp check_ftl_removal(message, _hashtags), do: {:ok, message}
defp check_sensitive(message, hashtags) do
if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
{:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
else
{:ok, message}
end
defp check_sensitive(message) do
{:ok, new_object} =
Object.Updater.do_with_history(message["object"], fn object ->
hashtags = Object.hashtags(%Object{data: object})
if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
{:ok, Map.put(object, "sensitive", true)}
else
{:ok, object}
end
end)
{:ok, Map.put(message, "object", new_object)}
end
@impl true
def filter(%{"type" => "Create", "object" => object} = message) do
hashtags = Object.hashtags(%Object{data: object})
def filter(%{"type" => type, "object" => object} = message) when type in ["Create", "Update"] do
history_items =
with %{"formerRepresentations" => %{"orderedItems" => items}} <- object do
items
else
_ -> []
end
historical_hashtags =
Enum.reduce(history_items, [], fn item, acc ->
acc ++ Object.hashtags(%Object{data: item})
end)
hashtags = Object.hashtags(%Object{data: object}) ++ historical_hashtags
if hashtags != [] do
with {:ok, message} <- check_reject(message, hashtags),
{:ok, message} <- check_ftl_removal(message, hashtags),
{:ok, message} <- check_sensitive(message, hashtags) do
{:ok, message} <-
(if "type" == "Create" do
check_ftl_removal(message, hashtags)
else
{:ok, message}
end),
{:ok, message} <- check_sensitive(message) do
{:ok, message}
end
else

55
lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
View file

@ -27,24 +27,46 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
end
defp check_reject(%{"object" => %{} = object} = message) do
payload = object_payload(object)
with {:ok, _new_object} <-
Pleroma.Object.Updater.do_with_history(object, fn object ->
payload = object_payload(object)
if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
string_matches?(payload, pattern)
end) do
{:reject, "[KeywordPolicy] Matches with rejected keyword"}
else
if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
string_matches?(payload, pattern)
end) do
{:reject, "[KeywordPolicy] Matches with rejected keyword"}
else
{:ok, message}
end
end) do
{:ok, message}
else
e -> e
end
end
defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do
payload = object_payload(object)
defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = message) do
check_keyword = fn object ->
payload = object_payload(object)
if Pleroma.Constants.as_public() in to and
Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
if Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
string_matches?(payload, pattern)
end) do
{:should_delist, nil}
else
{:ok, %{}}
end
end
should_delist? = fn object ->
with {:ok, _} <- Pleroma.Object.Updater.do_with_history(object, check_keyword) do
false
else
_ -> true
end
end
if Pleroma.Constants.as_public() in to and should_delist?.(object) do
to = List.delete(to, Pleroma.Constants.as_public())
cc = [Pleroma.Constants.as_public() | message["cc"] || []]
@ -59,8 +81,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
end
end
defp check_ftl_removal(message) do
{:ok, message}
end
defp check_replace(%{"object" => %{} = object} = message) do
object =
replace_kw = fn object ->
["content", "name", "summary"]
|> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
|> Enum.reduce(object, fn field, object ->
@ -73,6 +99,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
Map.put(object, field, data)
end)
|> (fn object -> {:ok, object} end).()
end
{:ok, object} = Pleroma.Object.Updater.do_with_history(object, replace_kw)
message = Map.put(message, "object", object)
@ -80,7 +110,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
end
@impl true
def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do
def filter(%{"type" => type, "object" => %{"content" => _content}} = message)
when type in ["Create", "Update"] do
with {:ok, message} <- check_reject(message),
{:ok, message} <- check_ftl_removal(message),
{:ok, message} <- check_replace(message) do

9
lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
View file

@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
recv_timeout: 10_000
]
@impl true
def history_awareness, do: :auto
defp prefetch(url) do
# Fetching only proxiable resources
if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
@ -54,10 +57,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
end
@impl true
def filter(
%{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
)
when is_list(attachments) and length(attachments) > 0 do
def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = message)
when type in ["Create", "Update"] and is_list(attachments) and length(attachments) > 0 do
preload(message)
{:ok, message}

15
lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex
View file

@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
@impl true
def filter(%{"actor" => actor} = object) do
with true <- is_local?(actor),
true <- is_eligible_type?(object),
true <- is_note?(object),
false <- has_attachment?(object),
true <- only_mentions?(object) do
@ -32,7 +33,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
end
defp has_attachment?(%{
"type" => "Create",
"object" => %{"type" => "Note", "attachment" => attachments}
})
when length(attachments) > 0,
@ -40,7 +40,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
defp has_attachment?(_), do: false
defp only_mentions?(%{"type" => "Create", "object" => %{"type" => "Note", "source" => source}}) do
defp only_mentions?(%{"object" => %{"type" => "Note", "source" => source}}) do
source =
case source do
%{"content" => text} -> text
_ -> source
end
non_mentions =
source |> String.split() |> Enum.filter(&(not String.starts_with?(&1, "@"))) |> length
@ -53,9 +59,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
defp only_mentions?(_), do: false
defp is_note?(%{"type" => "Create", "object" => %{"type" => "Note"}}), do: true
defp is_note?(%{"object" => %{"type" => "Note"}}), do: true
defp is_note?(_), do: false
defp is_eligible_type?(%{"type" => type}) when type in ["Create", "Update"], do: true
defp is_eligible_type?(_), do: false
@impl true
def describe, do: {:ok, %{}}
end

7
lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex
View file

@ -6,14 +6,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
@moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)"
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def history_awareness, do: :auto
@impl true
def filter(
%{
"type" => "Create",
"type" => type,
"object" => %{"content" => content, "attachment" => _} = _child_object
} = object
)
when content in [".", "<p>.</p>"] do
when type in ["Create", "Update"] and content in [".", "<p>.</p>"] do
{:ok, put_in(object, ["object", "content"], "")}
end

6
lib/pleroma/web/activity_pub/mrf/normalize_markup.ex
View file

@ -9,7 +9,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true
def filter(%{"type" => "Create", "object" => child_object} = object) do
def history_awareness, do: :auto
@impl true
def filter(%{"type" => type, "object" => child_object} = object)
when type in ["Create", "Update"] do
scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy])
content =

2
lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
View file

@ -131,7 +131,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
type: {:list, :atom},
description:
"A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
"`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <>
"`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct message; " <>
"`:reject` rejects the message entirely",
suggestions: [:delist, :strip_followers, :reject]
}

3
lib/pleroma/web/activity_pub/mrf/policy.ex
View file

@ -12,5 +12,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do
label: String.t(),
description: String.t()
}
@optional_callbacks config_description: 0
@callback history_awareness() :: :auto | :manual
@optional_callbacks config_description: 0, history_awareness: 0
end

9
lib/pleroma/web/activity_pub/mrf/simple_policy.ex
View file

@ -40,9 +40,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
defp check_media_removal(
%{host: actor_host} = _actor_info,
%{"type" => "Create", "object" => %{"attachment" => child_attachment}} = object
%{"type" => type, "object" => %{"attachment" => child_attachment}} = object
)
when length(child_attachment) > 0 do
when length(child_attachment) > 0 and type in ["Create", "Update"] do
media_removal =
instance_list(:media_removal)
|> MRF.subdomains_regex()
@ -63,10 +63,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
defp check_media_nsfw(
%{host: actor_host} = _actor_info,
%{
"type" => "Create",
"type" => type,
"object" => %{} = _child_object
} = object
) do
)
when type in ["Create", "Update"] do
media_nsfw =
instance_list(:media_nsfw)
|> MRF.subdomains_regex()

18
lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
View file

@ -12,6 +12,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
defp shortcode_matches?(shortcode, pattern) when is_binary(pattern) do
shortcode == pattern
end
defp shortcode_matches?(shortcode, pattern) do
String.match?(shortcode, pattern)
end
defp steal_emoji({shortcode, url}, emoji_dir_path) do
url = Pleroma.Web.MediaProxy.url(url)
@ -72,7 +80,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
reject_emoji? =
[:mrf_steal_emoji, :rejected_shortcodes]
|> Config.get([])
|> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
|> Enum.find(false, fn pattern -> shortcode_matches?(shortcode, pattern) end)
!reject_emoji?
end)
@ -122,8 +130,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
%{
key: :rejected_shortcodes,
type: {:list, :string},
description: "Regex-list of shortcodes to reject",
suggestions: [""]
description: """
A list of patterns or matches to reject shortcodes with.
Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
""",
suggestions: ["foo", ~r/foo/]
},
%{
key: :size_limit,

10
lib/pleroma/web/activity_pub/mrf/tag_policy.ex
View file

@ -27,22 +27,22 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
defp process_tag(
"mrf_tag:media-force-nsfw",
%{
"type" => "Create",
"type" => type,
"object" => %{"attachment" => child_attachment}
} = message
)
when length(child_attachment) > 0 do
when length(child_attachment) > 0 and type in ["Create", "Update"] do
{:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
end
defp process_tag(
"mrf_tag:media-strip",
%{
"type" => "Create",
"type" => type,
"object" => %{"attachment" => child_attachment} = object
} = message
)
when length(child_attachment) > 0 do
when length(child_attachment) > 0 and type in ["Create", "Update"] do
object = Map.delete(object, "attachment")
message = Map.put(message, "object", object)
@ -152,7 +152,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
do: filter_message(target_actor, message)
@impl true
def filter(%{"actor" => actor, "type" => "Create"} = message),
def filter(%{"actor" => actor, "type" => type} = message) when type in ["Create", "Update"],
do: filter_message(actor, message)
@impl true

114
lib/pleroma/web/activity_pub/object_validator.ex
View file

@ -103,8 +103,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
meta
)
when objtype in ~w[Question Answer Audio Video Event Article Note Page] do
with {:ok, object_data} <- cast_and_apply(object),
meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
with {:ok, object_data} <- cast_and_apply_and_stringify_with_history(object),
meta = Keyword.put(meta, :object_data, object_data),
{:ok, create_activity} <-
create_activity
|> CreateGenericValidator.cast_and_validate(meta)
@ -128,19 +128,53 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
end
with {:ok, object} <-
object
|> validator.cast_and_validate()
|> Ecto.Changeset.apply_action(:insert) do
object = stringify_keys(object)
do_separate_with_history(object, fn object ->
with {:ok, object} <-
object
|> validator.cast_and_validate()
|> Ecto.Changeset.apply_action(:insert) do
object = stringify_keys(object)
# Insert copy of hashtags as strings for the non-hashtag table indexing
tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object})
object = Map.put(object, "tag", tag)
# Insert copy of hashtags as strings for the non-hashtag table indexing
tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object})
object = Map.put(object, "tag", tag)
{:ok, object}
end
end) do
{:ok, object, meta}
end
end
def validate(
%{"type" => "Update", "object" => %{"type" => objtype} = object} = update_activity,
meta
)
when objtype in ~w[Question Answer Audio Video Event Article Note Page] do
with {_, false} <- {:local, Access.get(meta, :local, false)},
{_, {:ok, object_data, _}} <- {:object_validation, validate(object, meta)},
meta = Keyword.put(meta, :object_data, object_data),
{:ok, update_activity} <-
update_activity
|> UpdateValidator.cast_and_validate()
|> Ecto.Changeset.apply_action(:insert) do
update_activity = stringify_keys(update_activity)
{:ok, update_activity, meta}
else
{:local, _} ->
with {:ok, object} <-
update_activity
|> UpdateValidator.cast_and_validate()
|> Ecto.Changeset.apply_action(:insert) do
object = stringify_keys(object)
{:ok, object, meta}
end
{:object_validation, e} ->
e
end
end
def validate(%{"type" => type} = object, meta)
when type in ~w[Accept Reject Follow Update Like EmojiReact Announce
ChatMessage Answer] do
@ -178,6 +212,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
def validate(o, m), do: {:error, {:validator_not_set, {o, m}}}
def cast_and_apply_and_stringify_with_history(object) do
do_separate_with_history(object, fn object ->
with {:ok, object_data} <- cast_and_apply(object),
object_data <- object_data |> stringify_keys() do
{:ok, object_data}
end
end)
end
def cast_and_apply(%{"type" => "ChatMessage"} = object) do
ChatMessageValidator.cast_and_apply(object)
end
@ -204,8 +247,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
# is_struct/1 appears in Elixir 1.11
def stringify_keys(%{__struct__: _} = object) do
def stringify_keys(object) when is_struct(object) do
object
|> Map.from_struct()
|> stringify_keys
@ -236,4 +278,54 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
Object.normalize(object["object"], fetch: true)
:ok
end
defp for_each_history_item(
%{"type" => "OrderedCollection", "orderedItems" => items} = history,
object,
fun
) do
processed_items =
Enum.map(items, fn item ->
with item <- Map.put(item, "id", object["id"]),
{:ok, item} <- fun.(item) do
item
else
_ -> nil
end
end)
if Enum.all?(processed_items, &(not is_nil(&1))) do
{:ok, Map.put(history, "orderedItems", processed_items)}
else
{:error, :invalid_history}
end
end
defp for_each_history_item(nil, _object, _fun) do
{:ok, nil}
end
defp for_each_history_item(_, _object, _fun) do
{:error, :invalid_history}
end
# fun is (object -> {:ok, validated_object_with_string_keys})
defp do_separate_with_history(object, fun) do
with history <- object["formerRepresentations"],
object <- Map.drop(object, ["formerRepresentations"]),
{_, {:ok, object}} <- {:main_body, fun.(object)},
{_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do
object =
if history do
Map.put(object, "formerRepresentations", history)
else
object
end
{:ok, object}
else
{:main_body, e} -> e
{:history_items, e} -> e
end
end
end

Some files were not shown because too many files have changed in this diff Show more