From fe2dceb66d056809d9a145773a8053ac9fb02658 Mon Sep 17 00:00:00 2001 From: Wim Vanderbauwhede Date: Fri, 4 Jan 2019 15:22:02 +0000 Subject: [PATCH 01/75] Patch to support image descriptions in Pleroma FE --- lib/pleroma/web/common_api/utils.ex | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 3ff9f9452..51e74ac8f 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -1,8 +1,9 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2018 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.CommonAPI.Utils do + require Logger alias Calendar.Strftime alias Comeonin.Pbkdf2 alias Pleroma.{Activity, Formatter, Object, Repo} @@ -32,9 +33,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_replied_to_activity(_), do: nil - def attachments_from_ids(ids) do - Enum.map(ids || [], fn media_id -> + def attachments_from_ids(ids, descs) do + Enum.map(ids || [], fn media_id -> do + Logger.warn(descs[media_id]) Repo.get(Object, media_id).data + end end) end From 4c95545d194e8a807e9e3514ed75347d78ec0856 Mon Sep 17 00:00:00 2001 From: Wim Vanderbauwhede Date: Fri, 4 Jan 2019 15:35:41 +0000 Subject: [PATCH 02/75] Patch to support image descriptions in Pleroma FE --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index e474653ff..50074b8b0 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -90,7 +90,7 @@ defmodule Pleroma.Web.CommonAPI do limit = Pleroma.Config.get([:instance, :limit]) with status <- String.trim(status), - attachments <- attachments_from_ids(data["media_ids"]), + attachments <- attachments_from_ids(data["media_ids"], data["descriptions"]), mentions <- Formatter.parse_mentions(status), inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]), {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 51e74ac8f..5fe21fb99 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -1,9 +1,8 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.CommonAPI.Utils do - require Logger alias Calendar.Strftime alias Comeonin.Pbkdf2 alias Pleroma.{Activity, Formatter, Object, Repo} @@ -33,11 +32,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_replied_to_activity(_), do: nil - def attachments_from_ids(ids, descs) do - Enum.map(ids || [], fn media_id -> do - Logger.warn(descs[media_id]) - Repo.get(Object, media_id).data - end + def attachments_from_ids(ids, descs_str) do + {_, descs} = Jason.decode(descs_str) + + Enum.map(ids || [], fn media_id -> + Map.put(Repo.get(Object, media_id).data, "name", descs[media_id]) end) end From ba93396649f65a1f32eeedfd9ccd32cf308e7210 Mon Sep 17 00:00:00 2001 From: Wim Vanderbauwhede Date: Fri, 4 Jan 2019 16:27:46 +0000 Subject: [PATCH 03/75] Patch to support image descriptions in Pleroma FE --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 16 +++++++++++++++- priv/static/index.html | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 50074b8b0..ef79b9c5d 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -90,7 +90,7 @@ defmodule Pleroma.Web.CommonAPI do limit = Pleroma.Config.get([:instance, :limit]) with status <- String.trim(status), - attachments <- attachments_from_ids(data["media_ids"], data["descriptions"]), + attachments <- attachments_from_ids(data), mentions <- Formatter.parse_mentions(status), inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]), {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 5fe21fb99..59df48ed6 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -32,7 +32,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_replied_to_activity(_), do: nil - def attachments_from_ids(ids, descs_str) do + def attachments_from_ids(data) do + if Map.has_key?(data, "descriptions") do + attachments_from_ids_descs(data["media_ids"], data["descriptions"]) + else + attachments_from_ids_no_descs(data["media_ids"]) + end + end + + def attachments_from_ids_no_descs(ids) do + Enum.map(ids || [], fn media_id -> + Repo.get(Object, media_id).data + end) + end + + def attachments_from_ids_descs(ids, descs_str) do {_, descs} = Jason.decode(descs_str) Enum.map(ids || [], fn media_id -> diff --git a/priv/static/index.html b/priv/static/index.html index 83c11118c..068d2faec 100644 --- a/priv/static/index.html +++ b/priv/static/index.html @@ -1 +1 @@ -Pleroma
\ No newline at end of file +Pleroma
\ No newline at end of file From bf5b1c7f06c9f8882c40cf2385439bee3b74522c Mon Sep 17 00:00:00 2001 From: Wim Vanderbauwhede Date: Sun, 20 Jan 2019 11:19:50 +0000 Subject: [PATCH 04/75] Removed file as requested --- priv/static/index.html | 1 - 1 file changed, 1 deletion(-) delete mode 100644 priv/static/index.html diff --git a/priv/static/index.html b/priv/static/index.html deleted file mode 100644 index 068d2faec..000000000 --- a/priv/static/index.html +++ /dev/null @@ -1 +0,0 @@ -Pleroma
\ No newline at end of file From 55affbca7fcb214c71b3f8378b0de869c4d4d072 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 28 Jan 2019 22:17:17 +0700 Subject: [PATCH 05/75] add a job queue --- config/config.exs | 6 +- config/test.exs | 2 + docs/config.md | 31 +++- lib/pleroma/application.ex | 5 +- lib/pleroma/jobs.ex | 153 ++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 20 ++- .../activity_pub/activity_pub_controller.ex | 4 +- lib/pleroma/web/activity_pub/utils.ex | 2 +- lib/pleroma/web/federator/federator.ex | 139 ++++++---------- lib/pleroma/web/ostatus/ostatus_controller.ex | 2 +- lib/pleroma/web/websub/websub.ex | 8 +- lib/pleroma/web/websub/websub_controller.ex | 2 +- test/jobs_test.exs | 83 ++++++++++ test/support/jobs_worker_mock.ex | 19 +++ test/web/federator_test.exs | 24 +-- 15 files changed, 358 insertions(+), 142 deletions(-) create mode 100644 lib/pleroma/jobs.ex create mode 100644 test/jobs_test.exs create mode 100644 test/support/jobs_worker_mock.ex diff --git a/config/config.exs b/config/config.exs index e8cf2ed3a..98dd8eb65 100644 --- a/config/config.exs +++ b/config/config.exs @@ -271,14 +271,16 @@ config :pleroma, Pleroma.User, "web" ] -config :pleroma, Pleroma.Web.Federator, max_jobs: 50 - config :pleroma, Pleroma.Web.Federator.RetryQueue, enabled: false, max_jobs: 20, initial_timeout: 30, max_retries: 5 +config :pleroma, Pleroma.Jobs, + federator_incoming: [max_jobs: 50], + federator_outgoing: [max_jobs: 50] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/test.exs b/config/test.exs index 67ed4737f..7e833fbb3 100644 --- a/config/test.exs +++ b/config/test.exs @@ -43,6 +43,8 @@ config :web_push_encryption, :vapid_details, "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4", private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA" +config :pleroma, Pleroma.Jobs, testing: [max_jobs: 2] + try do import_config "test.secret.exs" rescue diff --git a/docs/config.md b/docs/config.md index 5464fa90d..84a1e3607 100644 --- a/docs/config.md +++ b/docs/config.md @@ -36,14 +36,15 @@ This filter replaces the filename (not the path) of an upload. For complete obfu An example for Sendgrid adapter: -``` +```exs config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendgrid, api_key: "YOUR_API_KEY" ``` An example for SMTP adapter: -``` + +```exs config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.SMTP, relay: "smtp.gmail.com", @@ -163,7 +164,7 @@ their ActivityPub ID. An example: -``` +```exs config :pleroma, :mrf_user_allowlist, "example.org": ["https://example.org/users/admin"] ``` @@ -192,18 +193,34 @@ the source code is here: https://github.com/koto-bank/kocaptcha. The default end Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: -``` +```exs config :pleroma, :admin_token, "somerandomtoken" ``` You can then do -``` + +```sh curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" ``` -## Pleroma.Web.Federator +## Pleroma.Jobs + +A list of job queues and their settings. + +Job queue settings: + +* `max_jobs`: The maximum amount of parallel jobs running at the same time. + +Example: + +```exs +config :pleroma, Pleroma.Jobs, + federator_incoming: [max_jobs: 50], + federator_outgoing: [max_jobs: 50] +``` + +This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. -* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. ## Pleroma.Web.Federator.RetryQueue diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 47c0e5b68..60cba1580 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -101,9 +101,10 @@ defmodule Pleroma.Application do ), worker(Pleroma.FlakeId, []), worker(Pleroma.Web.Federator.RetryQueue, []), - worker(Pleroma.Web.Federator, []), worker(Pleroma.Stats, []), - worker(Pleroma.Web.Push, []) + worker(Pleroma.Web.Push, []), + worker(Pleroma.Jobs, []), + worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary) ] ++ streamer_child() ++ chat_child() ++ diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex new file mode 100644 index 000000000..dff0f2197 --- /dev/null +++ b/lib/pleroma/jobs.ex @@ -0,0 +1,153 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Jobs do + @moduledoc """ + A basic job queue + """ + use GenServer + + require Logger + + def init(args) do + {:ok, args} + end + + def start_link do + queues = + Pleroma.Config.get(Pleroma.Jobs) + |> Enum.map(fn {name, _} -> create_queue(name) end) + |> Enum.into(%{}) + + state = %{ + queues: queues, + refs: %{} + } + + GenServer.start_link(__MODULE__, state, name: __MODULE__) + end + + def create_queue(name) do + {name, {:sets.new(), []}} + end + + @doc """ + Enqueues a job. + + Returns `:ok`. + + ## Arguments + + - `queue_name` - a queue name(must be specified in the config). + - `mod` - a worker module, must have `perform` function. + - `args` - a list of arguments for the `perform` function of the worker module. + - `priority` - a job priority (`0` by default). + + ## Examples + + Enqueue `Module.perform/0` with `priority=1`: + + iex> Pleroma.Jobs.enqueue(:example_queue, Module, []) + :ok + + Enqueue `Module.perform(:job_name)` with `priority=5`: + + iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:job_name], 5) + :ok + + Enqueue `Module.perform(:another_job, data)` with `priority=1`: + + iex> data = "foobar" + iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:another_job, data]) + :ok + + Enqueue `Module.perform(:foobar_job, :foo, :bar, 42)` with `priority=1`: + + iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:foobar_job, :foo, :bar, 42]) + :ok + + """ + + def enqueue(queue_name, mod, args, priority \\ 1) + + if Mix.env() == :test do + def enqueue(_queue_name, mod, args, _priority) do + apply(mod, :perform, args) + end + else + @spec enqueue(atom(), atom(), [any()], integer()) :: :ok + def enqueue(queue_name, mod, args, priority \\ 1) do + GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority}) + end + end + + def handle_cast({:enqueue, queue_name, mod, args, priority}, state) do + {running_jobs, queue} = state[:queues][queue_name] + + queue = enqueue_sorted(queue, {mod, args}, priority) + + state = + state + |> update_queue(queue_name, {running_jobs, queue}) + |> maybe_start_job(queue_name, running_jobs, queue) + + {:noreply, state} + end + + def handle_cast(m, state) do + IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}") + {:noreply, state} + end + + def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do + queue_name = state.refs[ref] + + {running_jobs, queue} = state[:queues][queue_name] + + running_jobs = :sets.del_element(ref, running_jobs) + + state = state |> remove_ref(ref) |> maybe_start_job(queue_name, running_jobs, queue) + + {:noreply, state} + end + + def maybe_start_job(state, queue_name, running_jobs, queue) do + if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, queue_name, :max_jobs]) && + queue != [] do + {{mod, args}, queue} = queue_pop(queue) + {:ok, pid} = Task.start(fn -> apply(mod, :perform, args) end) + mref = Process.monitor(pid) + + state + |> add_ref(queue_name, mref) + |> update_queue(queue_name, {:sets.add_element(mref, running_jobs), queue}) + else + update_queue(state, queue_name, {running_jobs, queue}) + end + end + + def enqueue_sorted(queue, element, priority) do + [%{item: element, priority: priority} | queue] + |> Enum.sort_by(fn %{priority: priority} -> priority end) + end + + def queue_pop([%{item: element} | queue]) do + {element, queue} + end + + defp add_ref(state, queue_name, ref) do + refs = Map.put(state[:refs], ref, queue_name) + Map.put(state, :refs, refs) + end + + defp remove_ref(state, ref) do + refs = Map.delete(state[:refs], ref) + Map.put(state, :refs, refs) + end + + defp update_queue(state, queue_name, data) do + queues = Map.put(state[:queues], queue_name, data) + Map.put(state, :queues, queues) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6b4682e35..5be359887 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -711,20 +711,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do public = is_public?(activity) - remote_inboxes = - (Pleroma.Web.Salmon.remote_users(activity) ++ followers) - |> Enum.filter(fn user -> User.ap_enabled?(user) end) - |> Enum.map(fn %{info: %{source_data: data}} -> - (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] - end) - |> Enum.uniq() - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) - Enum.each(remote_inboxes, fn inbox -> - Federator.enqueue(:publish_single_ap, %{ + (Pleroma.Web.Salmon.remote_users(activity) ++ followers) + |> Enum.filter(fn user -> User.ap_enabled?(user) end) + |> Enum.map(fn %{info: %{source_data: data}} -> + (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] + end) + |> Enum.uniq() + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Enum.each(fn inbox -> + Federator.publish_single_ap(%{ inbox: inbox, json: json, actor: actor, diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 7eed0a600..04c6fef3f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -150,13 +150,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do with %User{} = user <- User.get_cached_by_nickname(nickname), true <- Utils.recipient_in_message(user.ap_id, params), params <- Utils.maybe_splice_recipient(user.ap_id, params) do - Federator.enqueue(:incoming_ap_doc, params) + Federator.incoming_ap_doc(params) json(conn, "ok") end end def inbox(%{assigns: %{valid_signature: true}} = conn, params) do - Federator.enqueue(:incoming_ap_doc, params) + Federator.incoming_ap_doc(params) json(conn, "ok") end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index e40d05fcd..2ff8a1e8a 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -156,7 +156,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do _ -> 5 end - Pleroma.Web.Federator.enqueue(:publish, activity, priority) + Pleroma.Web.Federator.publish(activity, priority) :ok end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f3a0e18b8..4d03b4622 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -3,9 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator do - use GenServer alias Pleroma.User alias Pleroma.Activity + alias Pleroma.Jobs alias Pleroma.Web.{WebFinger, Websub} alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.ActivityPub.ActivityPub @@ -18,39 +18,60 @@ defmodule Pleroma.Web.Federator do @websub Application.get_env(:pleroma, :websub) @ostatus Application.get_env(:pleroma, :ostatus) - def init(args) do - {:ok, args} + def init() do + # 1 minute + Process.sleep(1000 * 60 * 1) + refresh_subscriptions() end - def start_link do - spawn(fn -> - # 1 minute - Process.sleep(1000 * 60 * 1) - enqueue(:refresh_subscriptions, nil) - end) + # Client API - GenServer.start_link( - __MODULE__, - %{ - in: {:sets.new(), []}, - out: {:sets.new(), []} - }, - name: __MODULE__ - ) + def incoming_doc(doc) do + Jobs.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc]) end - def handle(:refresh_subscriptions, _) do + def incoming_ap_doc(params) do + Jobs.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params]) + end + + def publish(activity, priority \\ 1) do + Jobs.enqueue(:federator_out, __MODULE__, [:publish, activity], priority) + end + + def publish_single_ap(params) do + Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_ap, params]) + end + + def publish_single_websub(websub) do + Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_websub, websub]) + end + + def verify_websub(websub) do + Jobs.enqueue(:federator_out, __MODULE__, [:verify_websub, websub]) + end + + def request_subscription(sub) do + Jobs.enqueue(:federator_out, __MODULE__, [:request_subscription, sub]) + end + + def refresh_subscriptions() do + Jobs.enqueue(:federator_out, __MODULE__, [:refresh_subscriptions]) + end + + # Job Worker Callbacks + + def perform(:refresh_subscriptions) do Logger.debug("Federator running refresh subscriptions") Websub.refresh_subscriptions() spawn(fn -> # 6 hours Process.sleep(1000 * 60 * 60 * 6) - enqueue(:refresh_subscriptions, nil) + refresh_subscriptions() end) end - def handle(:request_subscription, websub) do + def perform(:request_subscription, websub) do Logger.debug("Refreshing #{websub.topic}") with {:ok, websub} <- Websub.request_subscription(websub) do @@ -60,7 +81,7 @@ defmodule Pleroma.Web.Federator do end end - def handle(:publish, activity) do + def perform(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do @@ -86,7 +107,7 @@ defmodule Pleroma.Web.Federator do end end - def handle(:verify_websub, websub) do + def perform(:verify_websub, websub) do Logger.debug(fn -> "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" end) @@ -94,12 +115,12 @@ defmodule Pleroma.Web.Federator do @websub.verify(websub) end - def handle(:incoming_doc, doc) do + def perform(:incoming_doc, doc) do Logger.info("Got document, trying to parse") @ostatus.handle_incoming(doc) end - def handle(:incoming_ap_doc, params) do + def perform(:incoming_ap_doc, params) do Logger.info("Handling incoming AP activity") params = Utils.normalize_params(params) @@ -124,7 +145,7 @@ defmodule Pleroma.Web.Federator do end end - def handle(:publish_single_ap, params) do + def perform(:publish_single_ap, params) do case ActivityPub.publish_one(params) do {:ok, _} -> :ok @@ -134,7 +155,7 @@ defmodule Pleroma.Web.Federator do end end - def handle( + def perform( :publish_single_websub, %{xml: _xml, topic: _topic, callback: _callback, secret: _secret} = params ) do @@ -147,75 +168,11 @@ defmodule Pleroma.Web.Federator do end end - def handle(type, _) do + def perform(type, _) do Logger.debug(fn -> "Unknown task: #{type}" end) {:error, "Don't know what to do with this"} end - if Mix.env() == :test do - def enqueue(type, payload, _priority \\ 1) do - if Pleroma.Config.get([:instance, :federating]) do - handle(type, payload) - end - end - else - def enqueue(type, payload, priority \\ 1) do - if Pleroma.Config.get([:instance, :federating]) do - GenServer.cast(__MODULE__, {:enqueue, type, payload, priority}) - end - end - end - - def maybe_start_job(running_jobs, queue) do - if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, :max_jobs]) && queue != [] do - {{type, payload}, queue} = queue_pop(queue) - {:ok, pid} = Task.start(fn -> handle(type, payload) end) - mref = Process.monitor(pid) - {:sets.add_element(mref, running_jobs), queue} - else - {running_jobs, queue} - end - end - - def handle_cast({:enqueue, type, payload, _priority}, state) - when type in [:incoming_doc, :incoming_ap_doc] do - %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - i_queue = enqueue_sorted(i_queue, {type, payload}, 1) - {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) - {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} - end - - def handle_cast({:enqueue, type, payload, _priority}, state) do - %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - o_queue = enqueue_sorted(o_queue, {type, payload}, 1) - {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) - {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} - end - - def handle_cast(m, state) do - IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}") - {:noreply, state} - end - - def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do - %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - i_running_jobs = :sets.del_element(ref, i_running_jobs) - o_running_jobs = :sets.del_element(ref, o_running_jobs) - {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) - {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) - - {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} - end - - def enqueue_sorted(queue, element, priority) do - [%{item: element, priority: priority} | queue] - |> Enum.sort_by(fn %{priority: priority} -> priority end) - end - - def queue_pop([%{item: element} | queue]) do - {element, queue} - end - def ap_enabled_actor(id) do user = User.get_by_ap_id(id) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 823619edb..845bc60bb 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -82,7 +82,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do {:ok, body, _conn} = read_body(conn) {:ok, doc} = decode_or_retry(body) - Federator.enqueue(:incoming_doc, doc) + Federator.incoming_doc(doc) conn |> send_resp(200, "") diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 7ca62c83b..652ffd92c 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Repo alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter - alias Pleroma.Web.{XML, Endpoint, OStatus} + alias Pleroma.Web.{XML, Endpoint, OStatus, Federator} alias Pleroma.Web.Router.Helpers require Logger @@ -77,7 +77,7 @@ defmodule Pleroma.Web.Websub do secret: sub.secret } - Pleroma.Web.Federator.enqueue(:publish_single_websub, data) + Federator.publish_single_websub(data) end) end @@ -109,7 +109,7 @@ defmodule Pleroma.Web.Websub do websub = Repo.update!(change) - Pleroma.Web.Federator.enqueue(:verify_websub, websub) + Federator.verify_websub(websub) {:ok, websub} else @@ -259,7 +259,7 @@ defmodule Pleroma.Web.Websub do subs = Repo.all(query) Enum.each(subs, fn sub -> - Pleroma.Web.Federator.enqueue(:request_subscription, sub) + Federator.request_subscription(sub) end) end diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index e58f144e5..eb10227cb 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -80,7 +80,7 @@ defmodule Pleroma.Web.Websub.WebsubController do %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), {:ok, body, _conn} = read_body(conn), ^signature <- Websub.sign(websub.secret, body) do - Federator.enqueue(:incoming_doc, body) + Federator.incoming_doc(body) conn |> send_resp(200, "OK") diff --git a/test/jobs_test.exs b/test/jobs_test.exs new file mode 100644 index 000000000..ccb518dec --- /dev/null +++ b/test/jobs_test.exs @@ -0,0 +1,83 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.JobsTest do + use ExUnit.Case, async: true + + alias Pleroma.Jobs + alias Jobs.WorkerMock + + setup do + state = %{ + queues: Enum.into([Jobs.create_queue(:testing)], %{}), + refs: %{} + } + + [state: state] + end + + test "creates queue" do + queue = Jobs.create_queue(:foobar) + + assert {:foobar, set} = queue + assert :set == elem(set, 0) |> elem(0) + end + + test "enqueues an element according to priority" do + queue = [%{item: 1, priority: 2}] + + new_queue = Jobs.enqueue_sorted(queue, 2, 1) + assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + new_queue = Jobs.enqueue_sorted(queue, 2, 3) + assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}] + end + + test "pop first item" do + queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + assert {2, [%{item: 1, priority: 2}]} = Jobs.queue_pop(queue) + end + + test "enqueue a job", %{state: state} do + assert {:noreply, new_state} = + Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state) + + assert %{queues: %{testing: {running_jobs, []}}, refs: _} = new_state + assert :sets.size(running_jobs) == 1 + assert [ref] = :sets.to_list(running_jobs) + assert %{refs: %{^ref => :testing}} = new_state + end + + test "max jobs setting", %{state: state} do + max_jobs = Pleroma.Config.get([Jobs, :testing, :max_jobs]) + + {:noreply, state} = + Enum.reduce(1..(max_jobs + 1), {:noreply, state}, fn _, {:noreply, state} -> + Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state) + end) + + assert %{ + queues: %{ + testing: + {running_jobs, [%{item: {WorkerMock, [:test_job, :foo, :bar]}, priority: 3}]} + } + } = state + + assert :sets.size(running_jobs) == max_jobs + end + + test "remove job after it finished", %{state: state} do + {:noreply, new_state} = + Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state) + + %{queues: %{testing: {running_jobs, []}}} = new_state + [ref] = :sets.to_list(running_jobs) + + assert {:noreply, %{queues: %{testing: {running_jobs, []}}, refs: %{}}} = + Jobs.handle_info({:DOWN, ref, :process, nil, nil}, new_state) + + assert :sets.size(running_jobs) == 0 + end +end diff --git a/test/support/jobs_worker_mock.ex b/test/support/jobs_worker_mock.ex new file mode 100644 index 000000000..0fb976d05 --- /dev/null +++ b/test/support/jobs_worker_mock.ex @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Jobs.WorkerMock do + require Logger + + def perform(:test_job, arg, arg2) do + Logger.debug({:perform, :test_job, arg, arg2}) + end + + def perform(:test_job, payload) do + Logger.debug({:perform, :test_job, payload}) + end + + def test_job(payload) do + Pleroma.Jobs.enqueue(:testing, __MODULE__, [:test_job, payload]) + end +end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index a49265c0c..d58621cc3 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -14,22 +14,6 @@ defmodule Pleroma.Web.FederatorTest do :ok end - test "enqueues an element according to priority" do - queue = [%{item: 1, priority: 2}] - - new_queue = Federator.enqueue_sorted(queue, 2, 1) - assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}] - - new_queue = Federator.enqueue_sorted(queue, 2, 3) - assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}] - end - - test "pop first item" do - queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}] - - assert {2, [%{item: 1, priority: 2}]} = Federator.queue_pop(queue) - end - describe "Publish an activity" do setup do user = insert(:user) @@ -49,7 +33,7 @@ defmodule Pleroma.Web.FederatorTest do relay_mock: relay_mock } do with_mocks([relay_mock]) do - Federator.handle(:publish, activity) + Federator.publish(activity) end assert_received :relay_publish @@ -62,7 +46,7 @@ defmodule Pleroma.Web.FederatorTest do Pleroma.Config.put([:instance, :allow_relay], false) with_mocks([relay_mock]) do - Federator.handle(:publish, activity) + Federator.publish(activity) end refute_received :relay_publish @@ -87,7 +71,7 @@ defmodule Pleroma.Web.FederatorTest do "to" => ["https://www.w3.org/ns/activitystreams#Public"] } - {:ok, _activity} = Federator.handle(:incoming_ap_doc, params) + {:ok, _activity} = Federator.incoming_ap_doc(params) end test "rejects incoming AP docs with incorrect origin" do @@ -105,7 +89,7 @@ defmodule Pleroma.Web.FederatorTest do "to" => ["https://www.w3.org/ns/activitystreams#Public"] } - :error = Federator.handle(:incoming_ap_doc, params) + :error = Federator.incoming_ap_doc(params) end end end From b2e9700785124a8ba4cfb037ae92a6bfc7c3b224 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 28 Jan 2019 23:12:02 +0700 Subject: [PATCH 06/75] cleanup --- lib/pleroma/jobs.ex | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex index dff0f2197..2a75ff529 100644 --- a/lib/pleroma/jobs.ex +++ b/lib/pleroma/jobs.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Jobs do ## Arguments - `queue_name` - a queue name(must be specified in the config). - - `mod` - a worker module, must have `perform` function. + - `mod` - a worker module (must have `perform` function). - `args` - a list of arguments for the `perform` function of the worker module. - `priority` - a job priority (`0` by default). @@ -76,7 +76,6 @@ defmodule Pleroma.Jobs do apply(mod, :perform, args) end else - @spec enqueue(atom(), atom(), [any()], integer()) :: :ok def enqueue(queue_name, mod, args, priority \\ 1) do GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority}) end @@ -95,11 +94,6 @@ defmodule Pleroma.Jobs do {:noreply, state} end - def handle_cast(m, state) do - IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}") - {:noreply, state} - end - def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do queue_name = state.refs[ref] From 56533495b53c12223678de3f2fc6df8ee1165c19 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 29 Jan 2019 00:33:08 +0700 Subject: [PATCH 07/75] fix defaults --- lib/pleroma/jobs.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex index 2a75ff529..d42688193 100644 --- a/lib/pleroma/jobs.ex +++ b/lib/pleroma/jobs.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Jobs do apply(mod, :perform, args) end else - def enqueue(queue_name, mod, args, priority \\ 1) do + def enqueue(queue_name, mod, args, priority) do GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority}) end end From 1724a6b34b099dc83b94687d27d2492aaf97013e Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 29 Jan 2019 00:35:57 +0700 Subject: [PATCH 08/75] add spec back --- lib/pleroma/jobs.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex index d42688193..16dccb682 100644 --- a/lib/pleroma/jobs.ex +++ b/lib/pleroma/jobs.ex @@ -76,6 +76,7 @@ defmodule Pleroma.Jobs do apply(mod, :perform, args) end else + @spec enqueue(atom(), atom(), [any()], integer()) :: :ok def enqueue(queue_name, mod, args, priority) do GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority}) end From ab31adf15bbec1597a9b7cf065898fb3f712eef3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 30 Jan 2019 22:56:59 +0700 Subject: [PATCH 09/75] tiny improve --- lib/pleroma/jobs.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex index 16dccb682..24b7e5e46 100644 --- a/lib/pleroma/jobs.ex +++ b/lib/pleroma/jobs.ex @@ -102,7 +102,11 @@ defmodule Pleroma.Jobs do running_jobs = :sets.del_element(ref, running_jobs) - state = state |> remove_ref(ref) |> maybe_start_job(queue_name, running_jobs, queue) + state = + state + |> remove_ref(ref) + |> update_queue(queue_name, {running_jobs, queue}) + |> maybe_start_job(queue_name, running_jobs, queue) {:noreply, state} end @@ -118,7 +122,7 @@ defmodule Pleroma.Jobs do |> add_ref(queue_name, mref) |> update_queue(queue_name, {:sets.add_element(mref, running_jobs), queue}) else - update_queue(state, queue_name, {running_jobs, queue}) + state end end From 58e250d9d27205116df5634d909ec1e43ea55286 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 31 Jan 2019 15:23:50 +0700 Subject: [PATCH 10/75] fix merge --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e81816e35..e44c48c35 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -109,7 +109,7 @@ defmodule Pleroma.Application do worker(Pleroma.Stats, []), worker(Pleroma.Web.Push, []), worker(Pleroma.Jobs, []), - worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary) + worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary) ] ++ streamer_child() ++ chat_child() ++ From 5b1d7c3c5672af065af503891d156b6e0cf5a8c1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 6 Feb 2019 12:17:41 +0700 Subject: [PATCH 11/75] fix tests --- test/web/federator_test.exs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 147086918..b56694a8b 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -87,11 +87,9 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) - assert called( - Federator.enqueue(:publish_single_ap, %{inbox: inbox1, unreachable_since: dt}) - ) + assert called(Federator.publish_single_ap(%{inbox: inbox1, unreachable_since: dt})) - refute called(Federator.enqueue(:publish_single_ap, %{inbox: inbox2})) + refute called(Federator.publish_single_ap(%{inbox: inbox2})) end test_with_mock "it federates only to reachable instances via Websub", @@ -123,13 +121,13 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"}) assert called( - Federator.enqueue(:publish_single_websub, %{ + Federator.publish_single_websub(%{ callback: sub2.callback, unreachable_since: dt }) ) - refute called(Federator.enqueue(:publish_single_websub, %{callback: sub1.callback})) + refute called(Federator.publish_single_websub(%{callback: sub1.callback})) end test_with_mock "it federates only to reachable instances via Salmon", @@ -163,13 +161,13 @@ defmodule Pleroma.Web.FederatorTest do CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) assert called( - Federator.enqueue(:publish_single_salmon, %{ + Federator.publish_single_salmon(%{ recipient: remote_user2, unreachable_since: dt }) ) - refute called(Federator.enqueue(:publish_single_websub, %{recipient: remote_user1})) + refute called(Federator.publish_single_websub(%{recipient: remote_user1})) end end From 89b6f255b4f3732182bd7b167c95fa87ccc31d33 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 10 Feb 2019 22:56:47 +0100 Subject: [PATCH 12/75] .credo.exs: Add test directory as well --- .credo.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.credo.exs b/.credo.exs index 94e19c4b5..580cb2705 100644 --- a/.credo.exs +++ b/.credo.exs @@ -19,7 +19,7 @@ # # You can give explicit globs or simply directories. # In the latter case `**/*.{ex,exs}` will be used. - included: ["lib/", "src/", "web/", "apps/"], + included: ["lib/", "src/", "web/", "apps/", "test/"], excluded: [~r"/_build/", ~r"/deps/"] }, # From 8bb7e19b3814e261e66c2d3592d146f72d4ce623 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 10 Feb 2019 22:57:38 +0100 Subject: [PATCH 13/75] test: de-group alias/es --- test/notification_test.exs | 3 ++- test/object_test.exs | 3 ++- test/support/builders/user_builder.ex | 3 ++- test/tasks/relay_test.exs | 4 +++- test/user_test.exs | 4 +++- test/web/activity_pub/activity_pub_controller_test.exs | 9 +++++++-- test/web/activity_pub/activity_pub_test.exs | 5 ++++- test/web/admin_api/admin_api_controller_test.exs | 3 ++- test/web/common_api/common_api_utils_test.exs | 2 +- test/web/federator_test.exs | 3 ++- test/web/mastodon_api/mastodon_api_controller_test.exs | 9 +++++++-- test/web/mastodon_api/status_view_test.exs | 3 ++- test/web/oauth/authorization_test.exs | 3 ++- test/web/oauth/oauth_controller_test.exs | 3 ++- test/web/oauth/token_test.exs | 4 +++- test/web/ostatus/activity_representer_test.exs | 4 +++- test/web/ostatus/feed_representer_test.exs | 4 +++- .../ostatus/incoming_documents/delete_handling_test.exs | 4 +++- test/web/ostatus/ostatus_controller_test.exs | 4 +++- test/web/ostatus/ostatus_test.exs | 6 +++++- test/web/salmon/salmon_test.exs | 4 +++- .../representers/activity_representer_test.exs | 7 +++++-- test/web/twitter_api/twitter_api_controller_test.exs | 9 +++++++-- test/web/twitter_api/twitter_api_test.exs | 9 +++++++-- test/web/twitter_api/views/notification_view_test.exs | 3 ++- test/web/websub/websub_controller_test.exs | 3 ++- test/web/websub/websub_test.exs | 3 ++- 27 files changed, 89 insertions(+), 32 deletions(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index 94fb0ab15..d36a92a7c 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -6,7 +6,8 @@ defmodule Pleroma.NotificationTest do use Pleroma.DataCase alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Web.CommonAPI - alias Pleroma.{User, Notification} + alias Pleroma.User + alias Pleroma.Notification alias Pleroma.Web.ActivityPub.Transmogrifier import Pleroma.Factory diff --git a/test/object_test.exs b/test/object_test.exs index ab6431012..9f5283d2d 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -5,7 +5,8 @@ defmodule Pleroma.ObjectTest do use Pleroma.DataCase import Pleroma.Factory - alias Pleroma.{Repo, Object} + alias Pleroma.Repo + alias Pleroma.Object test "returns an object by it's AP id" do object = insert(:note) diff --git a/test/support/builders/user_builder.ex b/test/support/builders/user_builder.ex index 7a1ca79b5..611a5be18 100644 --- a/test/support/builders/user_builder.ex +++ b/test/support/builders/user_builder.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Builders.UserBuilder do - alias Pleroma.{User, Repo} + alias Pleroma.User + alias Pleroma.Repo def build(data \\ %{}) do user = %User{ diff --git a/test/tasks/relay_test.exs b/test/tasks/relay_test.exs index 96fac4811..64ff07753 100644 --- a/test/tasks/relay_test.exs +++ b/test/tasks/relay_test.exs @@ -4,7 +4,9 @@ defmodule Mix.Tasks.Pleroma.RelayTest do alias Pleroma.Activity - alias Pleroma.Web.ActivityPub.{ActivityPub, Relay, Utils} + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.Relay alias Pleroma.User use Pleroma.DataCase diff --git a/test/user_test.exs b/test/user_test.exs index 523ab1ea4..1fd54b944 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -4,7 +4,9 @@ defmodule Pleroma.UserTest do alias Pleroma.Builders.UserBuilder - alias Pleroma.{User, Repo, Activity} + alias Pleroma.Activity + alias Pleroma.Repo + alias Pleroma.User alias Pleroma.Web.CommonAPI use Pleroma.DataCase diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 570bee6b3..acd295988 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -5,8 +5,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory - alias Pleroma.Web.ActivityPub.{UserView, ObjectView} - alias Pleroma.{Object, Repo, Activity, User, Instances} + alias Pleroma.Web.ActivityPub.UserView + alias Pleroma.Web.ActivityPub.ObjectView + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Instances setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index a55961ac4..a6f8b822a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -7,7 +7,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI - alias Pleroma.{Activity, Object, User, Instances} + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Instances alias Pleroma.Builders.ActivityBuilder import Pleroma.Factory diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 42450a7b6..a27c26f95 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -5,7 +5,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do use Pleroma.Web.ConnCase - alias Pleroma.{Repo, User} + alias Pleroma.Repo + alias Pleroma.User import Pleroma.Factory describe "/api/pleroma/admin/user" do diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index 754bc7255..faed6b685 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.Endpoint - alias Pleroma.Builders.{UserBuilder} + alias Pleroma.Builders.UserBuilder use Pleroma.DataCase test "it adds attachment links to a given text and attachment set" do diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 05f813291..9f8d71454 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -3,7 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.FederatorTest do - alias Pleroma.Web.{CommonAPI, Federator} + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Federator alias Pleroma.Instances use Pleroma.DataCase import Pleroma.Factory diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index f8da86004..9fd505f84 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -6,8 +6,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Web.TwitterAPI.TwitterAPI - alias Pleroma.{Repo, User, Object, Activity, Notification} - alias Pleroma.Web.{OStatus, CommonAPI} + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Object + alias Pleroma.Activity + alias Pleroma.Notification + alias Pleroma.Web.OStatus + alias Pleroma.Web.CommonAPI alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.MastodonAPI.FilterView alias Ecto.Changeset diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 2106253f2..0dc9c538c 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -5,7 +5,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do use Pleroma.DataCase - alias Pleroma.Web.MastodonAPI.{StatusView, AccountView} + alias Pleroma.Web.MastodonAPI.AccountView + alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.User alias Pleroma.Web.OStatus alias Pleroma.Web.CommonAPI diff --git a/test/web/oauth/authorization_test.exs b/test/web/oauth/authorization_test.exs index 3b1ddada8..81618e935 100644 --- a/test/web/oauth/authorization_test.exs +++ b/test/web/oauth/authorization_test.exs @@ -4,7 +4,8 @@ defmodule Pleroma.Web.OAuth.AuthorizationTest do use Pleroma.DataCase - alias Pleroma.Web.OAuth.{Authorization, App} + alias Pleroma.Web.OAuth.Authorization + alias Pleroma.Web.OAuth.App import Pleroma.Factory test "create an authorization token for a valid app" do diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index e0d3cb55f..2315f9a34 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -7,7 +7,8 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do import Pleroma.Factory alias Pleroma.Repo - alias Pleroma.Web.OAuth.{Authorization, Token} + alias Pleroma.Web.OAuth.Authorization + alias Pleroma.Web.OAuth.Token test "redirects with oauth authorization" do user = insert(:user) diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs index 9a241d61a..4dab4a308 100644 --- a/test/web/oauth/token_test.exs +++ b/test/web/oauth/token_test.exs @@ -4,7 +4,9 @@ defmodule Pleroma.Web.OAuth.TokenTest do use Pleroma.DataCase - alias Pleroma.Web.OAuth.{App, Token, Authorization} + alias Pleroma.Web.OAuth.App + alias Pleroma.Web.OAuth.Authorization + alias Pleroma.Web.OAuth.Token alias Pleroma.Repo import Pleroma.Factory diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 0869f2fd5..eebc5c040 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -6,7 +6,9 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do use Pleroma.DataCase alias Pleroma.Web.OStatus.ActivityRepresenter - alias Pleroma.{User, Activity, Object} + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Object alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.OStatus diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index 55717dec7..efd4e7217 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -6,7 +6,9 @@ defmodule Pleroma.Web.OStatus.FeedRepresenterTest do use Pleroma.DataCase import Pleroma.Factory alias Pleroma.User - alias Pleroma.Web.OStatus.{FeedRepresenter, UserRepresenter, ActivityRepresenter} + alias Pleroma.Web.OStatus.ActivityRepresenter + alias Pleroma.Web.OStatus.FeedRepresenter + alias Pleroma.Web.OStatus.UserRepresenter alias Pleroma.Web.OStatus test "returns a feed of the last 20 items of the user" do diff --git a/test/web/ostatus/incoming_documents/delete_handling_test.exs b/test/web/ostatus/incoming_documents/delete_handling_test.exs index d97cd79f4..d295cc539 100644 --- a/test/web/ostatus/incoming_documents/delete_handling_test.exs +++ b/test/web/ostatus/incoming_documents/delete_handling_test.exs @@ -4,7 +4,9 @@ defmodule Pleroma.Web.OStatus.DeleteHandlingTest do import Pleroma.Factory import Tesla.Mock - alias Pleroma.{Repo, Activity, Object} + alias Pleroma.Repo + alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Web.OStatus setup do diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 3145ca9a1..da9c72be8 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -5,7 +5,9 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory - alias Pleroma.{User, Repo, Object} + alias Pleroma.User + alias Pleroma.Repo + alias Pleroma.Object alias Pleroma.Web.CommonAPI alias Pleroma.Web.OStatus.ActivityRepresenter diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index dbe5de2e2..b4b19ab05 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -6,7 +6,11 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus alias Pleroma.Web.XML - alias Pleroma.{Object, Repo, User, Activity, Instances} + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Activity + alias Pleroma.Instances import Pleroma.Factory import ExUnit.CaptureLog diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index c539a28b2..9e583ba40 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -5,7 +5,9 @@ defmodule Pleroma.Web.Salmon.SalmonTest do use Pleroma.DataCase alias Pleroma.Web.Salmon - alias Pleroma.{Repo, Activity, User} + alias Pleroma.Activity + alias Pleroma.Repo + alias Pleroma.User import Pleroma.Factory @magickey "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwQhh-1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" diff --git a/test/web/twitter_api/representers/activity_representer_test.exs b/test/web/twitter_api/representers/activity_representer_test.exs index ea5813733..365c7f659 100644 --- a/test/web/twitter_api/representers/activity_representer_test.exs +++ b/test/web/twitter_api/representers/activity_representer_test.exs @@ -4,8 +4,11 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenterTest do use Pleroma.DataCase - alias Pleroma.{User, Activity, Object} - alias Pleroma.Web.TwitterAPI.Representers.{ActivityRepresenter, ObjectRepresenter} + alias Pleroma.User + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter + alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.TwitterAPI.UserView import Pleroma.Factory diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 855ae1526..acb03b146 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -5,8 +5,13 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter - alias Pleroma.Builders.{ActivityBuilder, UserBuilder} - alias Pleroma.{Repo, Activity, User, Object, Notification} + alias Pleroma.Builders.ActivityBuilder + alias Pleroma.Builders.UserBuilder + alias Pleroma.Repo + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Object + alias Pleroma.Notification alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.TwitterAPI.UserView alias Pleroma.Web.TwitterAPI.NotificationView diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 48ddbcf50..aa2a4d650 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -4,8 +4,13 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do use Pleroma.DataCase - alias Pleroma.Web.TwitterAPI.{TwitterAPI, UserView} - alias Pleroma.{Activity, User, Object, Repo, UserInviteToken} + alias Pleroma.Web.TwitterAPI.TwitterAPI + alias Pleroma.Web.TwitterAPI.UserView + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.UserInviteToken alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.TwitterAPI.ActivityView diff --git a/test/web/twitter_api/views/notification_view_test.exs b/test/web/twitter_api/views/notification_view_test.exs index 8367fc6c7..3a67f7292 100644 --- a/test/web/twitter_api/views/notification_view_test.exs +++ b/test/web/twitter_api/views/notification_view_test.exs @@ -5,7 +5,8 @@ defmodule Pleroma.Web.TwitterAPI.NotificationViewTest do use Pleroma.DataCase - alias Pleroma.{User, Notification} + alias Pleroma.User + alias Pleroma.Notification alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Web.TwitterAPI.NotificationView alias Pleroma.Web.TwitterAPI.UserView diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 6492df2a0..04fb4a5a3 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -6,7 +6,8 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory alias Pleroma.Web.Websub.WebsubClientSubscription - alias Pleroma.{Repo, Activity} + alias Pleroma.Activity + alias Pleroma.Repo alias Pleroma.Web.Websub test "websub subscription request", %{conn: conn} do diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 9751d161d..9a9b9df02 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -5,7 +5,8 @@ defmodule Pleroma.Web.WebsubTest do use Pleroma.DataCase alias Pleroma.Web.Websub - alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} + alias Pleroma.Web.Websub.WebsubServerSubscription + alias Pleroma.Web.Websub.WebsubClientSubscription import Pleroma.Factory alias Pleroma.Web.Router.Helpers import Tesla.Mock From 74579115a73d697aed67abe2dc8ea1a664c89c5b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 11 Feb 2019 00:08:48 +0100 Subject: [PATCH 14/75] =?UTF-8?q?test:=20Change=20`lenght(=E2=80=A6)=20=3D?= =?UTF-8?q?=3D=200`=20to=20`Enum.empty=3F(=E2=80=A6)`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/formatter_test.exs | 2 +- test/notification_test.exs | 30 +++++++++---------- test/tasks/user_test.exs | 2 +- .../mastodon_api_controller_test.exs | 2 +- test/web/websub/websub_controller_test.exs | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 2e717194b..f14077d25 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -197,7 +197,7 @@ defmodule Pleroma.FormatterTest do {subs, text} = Formatter.add_user_links({[], text}, mentions) - assert length(subs) == 0 + assert Enum.empty?(subs) Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end) expected_text = "@a hi" diff --git a/test/notification_test.exs b/test/notification_test.exs index d36a92a7c..755874a3d 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -300,7 +300,7 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) @@ -308,7 +308,7 @@ defmodule Pleroma.NotificationTest do {:ok, _} = CommonAPI.delete(activity.id, user) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) end test "liking an activity results in 1 notification, then 0 if the activity is unliked" do @@ -317,7 +317,7 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) @@ -325,7 +325,7 @@ defmodule Pleroma.NotificationTest do {:ok, _, _, _} = CommonAPI.unfavorite(activity.id, other_user) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) end test "repeating an activity results in 1 notification, then 0 if the activity is deleted" do @@ -334,7 +334,7 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) {:ok, _, _} = CommonAPI.repeat(activity.id, other_user) @@ -342,7 +342,7 @@ defmodule Pleroma.NotificationTest do {:ok, _} = CommonAPI.delete(activity.id, user) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) end test "repeating an activity results in 1 notification, then 0 if the activity is unrepeated" do @@ -351,7 +351,7 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) {:ok, _, _} = CommonAPI.repeat(activity.id, other_user) @@ -359,7 +359,7 @@ defmodule Pleroma.NotificationTest do {:ok, _, _} = CommonAPI.unrepeat(activity.id, other_user) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) end test "liking an activity which is already deleted does not generate a notification" do @@ -368,15 +368,15 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) {:ok, _deletion_activity} = CommonAPI.delete(activity.id, user) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) {:error, _} = CommonAPI.favorite(activity.id, other_user) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) end test "repeating an activity which is already deleted does not generate a notification" do @@ -385,15 +385,15 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) {:ok, _deletion_activity} = CommonAPI.delete(activity.id, user) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) {:error, _} = CommonAPI.repeat(activity.id, other_user) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) end test "replying to a deleted post without tagging does not generate a notification" do @@ -409,7 +409,7 @@ defmodule Pleroma.NotificationTest do "in_reply_to_status_id" => activity.id }) - assert length(Notification.for_user(user)) == 0 + assert Enum.empty?(Notification.for_user(user)) end end end diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index 44271898c..7b814d171 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -151,7 +151,7 @@ defmodule Mix.Tasks.Pleroma.UserTest do assert message =~ "Successfully unsubscribed" user = User.get_by_nickname(user.nickname) - assert length(user.following) == 0 + assert Enum.empty?(user.following) assert user.info.deactivated end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 9fd505f84..19393f355 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -36,7 +36,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do |> assign(:user, user) |> get("/api/v1/timelines/home") - assert length(json_response(conn, 200)) == 0 + assert Enum.empty?(json_response(conn, 200)) {:ok, user} = User.follow(user, following) diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 04fb4a5a3..87b01d89b 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -81,7 +81,7 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do assert response(conn, 500) == "Error" - assert length(Repo.all(Activity)) == 0 + assert Enum.empty?(Repo.all(Activity)) end end end From ea1058929c4dd569c00864f5292ec0a689acd1c6 Mon Sep 17 00:00:00 2001 From: shibayashi Date: Tue, 12 Feb 2019 00:08:52 +0100 Subject: [PATCH 15/75] Use url[:scheme] instead of protocol to determine if https is enabled --- lib/pleroma/plugs/http_security_plug.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 2a266c407..3c8e6a18f 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -33,7 +33,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do end defp csp_string do - protocol = Config.get([Pleroma.Web.Endpoint, :protocol]) + scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] [ "default-src 'none'", @@ -46,7 +46,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do "script-src 'self'", "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"), "manifest-src 'self'", - if protocol == "https" do + if scheme == "https" do "upgrade-insecure-requests" end ] From 84f22d1cb8cf953bf8f48c04d82ae05f780ec407 Mon Sep 17 00:00:00 2001 From: Hakaba Hitoyo Date: Tue, 12 Feb 2019 02:35:15 +0000 Subject: [PATCH 16/75] Mark streaming feature for mobile/web apps in Clients.md --- docs/Clients.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/Clients.md b/docs/Clients.md index 057f12392..043d04a0f 100644 --- a/docs/Clients.md +++ b/docs/Clients.md @@ -26,60 +26,71 @@ Feel free to contact us to be added to this list! - Source Code: - Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy) - Platforms: iOS +- Features: No Streaming ### Nekonium - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/) - Source: - Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin) - Platforms: Android +- Features: Streaming Ready ### Mastalab - Source Code: - Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79) - Platforms: Android +- Features: Streaming Ready ### Roma - Homepage: - Source Code: ??? - Platforms: iOS, Android +- Features: No Streaming ### Tootdon - Homepage: , - Source Code: ??? - Contact: [@tootdon@mstdn.jp](https://mstdn.jp/users/tootdon) - Platforms: Android, iOS +- Features: No Streaming ### Tusky - Homepage: - Source Code: - Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck) - Platforms: Android +- Features: No Streaming ### Twidere - Homepage: - Source Code: , - Contact: - Platform: Android, iOS +- Features: No Streaming ## Alternative Web Interfaces ### Brutaldon - Homepage: - Source Code: - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc) +- Features: No Streaming ### Feather - Source Code: - Contact: [@kaniini@pleroma.site](https://pleroma.site/kaniini) +- Features: No Streaming ### Halcyon - Source Code: - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon) +- Features: Streaming Ready ### Pinafore - Homepage: - Source Code: - Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore) - Note: Pleroma support is a secondary goal +- Features: No Streaming ### Sengi - Source Code: From 71ce564ecc442614995c021281a8f1e1a67fabc1 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 22 Nov 2018 07:12:13 +0100 Subject: [PATCH 17/75] =?UTF-8?q?config/dev.exs:=20Don=E2=80=99t=20put=20s?= =?UTF-8?q?ecure=20cookies=20on=20dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/dev.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/dev.exs b/config/dev.exs index 8f89aa03c..f77bb9976 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -16,7 +16,8 @@ config :pleroma, Pleroma.Web.Endpoint, debug_errors: true, code_reloader: true, check_origin: false, - watchers: [] + watchers: [], + secure_cookie_flag: false config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Local From 00e8f0b07dd3dced84b0317e1c5c4156d249dec4 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 1 Feb 2019 13:10:50 +0100 Subject: [PATCH 18/75] Plugs.HTTPSecurityPlug: Add unsafe-eval to script-src when in dev mode This is needed to run dev mode mastofe at the same time --- lib/pleroma/plugs/http_security_plug.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 3c8e6a18f..05e935f2c 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -43,9 +43,11 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do "media-src 'self' https:", "style-src 'self' 'unsafe-inline'", "font-src 'self'", - "script-src 'self'", "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"), "manifest-src 'self'", + if Mix.env() == :dev do + "script-src 'self' 'unsafe-eval'" + end, if scheme == "https" do "upgrade-insecure-requests" end From da4c662af31a2c45c767f2a9ed136272ee9fc2c8 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 2 Feb 2019 19:06:26 +0100 Subject: [PATCH 19/75] Plugs.HTTPSecurityPlug: Add webpacker to connect-src --- lib/pleroma/plugs/http_security_plug.ex | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 05e935f2c..057553e24 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -34,6 +34,21 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do defp csp_string do scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] + websocket_url = String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws") + + connect_src = + if Mix.env() == :dev do + "connect-src 'self' http://localhost:3035/ " <> websocket_url + else + "connect-src 'self' " <> websocket_url + end + + script_src = + if Mix.env() == :dev do + "script-src 'self' 'unsafe-eval'" + else + "script-src 'self'" + end [ "default-src 'none'", @@ -43,11 +58,9 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do "media-src 'self' https:", "style-src 'self' 'unsafe-inline'", "font-src 'self'", - "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"), "manifest-src 'self'", - if Mix.env() == :dev do - "script-src 'self' 'unsafe-eval'" - end, + connect_src, + script_src, if scheme == "https" do "upgrade-insecure-requests" end From 1d727cd0691fdedcd78d5ef22c07a1b280531037 Mon Sep 17 00:00:00 2001 From: Karen Konou Date: Tue, 12 Feb 2019 23:25:09 +0100 Subject: [PATCH 20/75] added checks for public url and follower collections --- .../web/activity_pub/mrf/hellthread_policy.ex | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 4c6e612b2..9be382972 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -14,6 +14,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) end + defp get_recipient_count(message) do + recipients = (message["to"] || []) ++ (message["cc"] || []) + + cond do + Enum.member?(recipients, "https://www.w3.org/ns/activitystreams#Public") && + Enum.find(recipients, &String.ends_with?(&1, "/followers")) -> + length(recipients) - 2 + + Enum.member?(recipients, "https://www.w3.org/ns/activitystreams#Public") || + Enum.find(recipients, &String.ends_with?(&1, "/followers")) -> + length(recipients) - 1 + + true -> + length(recipients) + end + end + @impl true def filter(%{"type" => "Create"} = message) do delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) @@ -24,13 +41,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do Pleroma.Config.get([:mrf_hellthread, :threshold]) ) - recipients = (message["to"] || []) ++ (message["cc"] || []) + recipients = get_recipient_count(message) cond do - length(recipients) > reject_threshold and reject_threshold > 0 -> + recipients > reject_threshold and reject_threshold > 0 -> {:reject, nil} - length(recipients) > delist_threshold and delist_threshold > 0 -> + recipients > delist_threshold and delist_threshold > 0 -> if Enum.member?(message["to"], "https://www.w3.org/ns/activitystreams#Public") or Enum.member?(message["cc"], "https://www.w3.org/ns/activitystreams#Public") do {:ok, delist_message(message)} From b7bc666200d6cd113365af7456b4135c86f1db03 Mon Sep 17 00:00:00 2001 From: hakabahitoyo Date: Wed, 13 Feb 2019 15:46:42 +0900 Subject: [PATCH 21/75] bugfix mdii uploader --- lib/pleroma/uploaders/mdii.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/uploaders/mdii.ex b/lib/pleroma/uploaders/mdii.ex index 320b07abd..190ed9f3a 100644 --- a/lib/pleroma/uploaders/mdii.ex +++ b/lib/pleroma/uploaders/mdii.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Uploaders.MDII do query = "#{cgi}?#{extension}" with {:ok, %{status: 200, body: body}} <- - @httpoison.post(query, file_data, adapter: [pool: :default]) do + @httpoison.post(query, file_data, [], adapter: [pool: :default]) do remote_file_name = String.split(body) |> List.first() public_url = "#{files}/#{remote_file_name}.#{extension}" {:ok, {:url, public_url}} From 16b7c07115ae5359541ac07752dd7d433035046d Mon Sep 17 00:00:00 2001 From: Hakaba Hitoyo Date: Wed, 13 Feb 2019 07:51:14 +0000 Subject: [PATCH 22/75] Mark streaming feature for desktop apps in Clients.md --- docs/Clients.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Clients.md b/docs/Clients.md index 043d04a0f..c3d776488 100644 --- a/docs/Clients.md +++ b/docs/Clients.md @@ -7,6 +7,7 @@ Feel free to contact us to be added to this list! - Homepage: - Source Code: ??? - Platforms: Windows, Mac, (Linux?) +- Features: Streaming Ready ### Social - Source Code: @@ -19,6 +20,7 @@ Feel free to contact us to be added to this list! - Source Code: - Contact: [@h3poteto@pleroma.io](https://pleroma.io/users/h3poteto) - Platforms: Windows, Mac, Linux +- Features: Streaming Ready ## Handheld ### Amaroq From 61a4bc50952b11a59dce7f655c883de59306adcd Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 10 Feb 2019 22:41:06 +0300 Subject: [PATCH 23/75] Add OAuth tokens endpoint --- lib/pleroma/web/oauth/token.ex | 8 +++++++ lib/pleroma/web/router.ex | 2 ++ .../web/twitter_api/twitter_api_controller.ex | 12 ++++++++++ .../web/twitter_api/views/token_view.ex | 22 +++++++++++++++++++ test/support/factory.ex | 13 +++++++++++ .../twitter_api_controller_test.exs | 18 +++++++++++++++ 6 files changed, 75 insertions(+) create mode 100644 lib/pleroma/web/twitter_api/views/token_view.ex diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index b0bbeeb69..40bf0ac6b 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -52,4 +52,12 @@ defmodule Pleroma.Web.OAuth.Token do ) |> Repo.delete_all() end + + def get_user_tokens(%User{id: user_id}) do + from( + t in Pleroma.Web.OAuth.Token, + where: t.user_id == ^user_id + ) + |> Repo.all() + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5b5627ce8..a394900b2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -389,6 +389,8 @@ defmodule Pleroma.Web.Router do get("/qvitter/mutes", TwitterAPI.Controller, :raw_empty_array) get("/externalprofile/show", TwitterAPI.Controller, :external_profile) + + get("/oauth_tokens", TwitterAPI.Controller, :oauth_tokens) end pipeline :ap_relay do diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index c2f0dc2a9..1a43e9a60 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -8,6 +8,10 @@ defmodule Pleroma.Web.TwitterAPI.Controller do import Pleroma.Web.ControllerHelper, only: [json_response: 3] alias Ecto.Changeset + alias Pleroma.Web.TwitterAPI.{TwitterAPI, UserView, ActivityView, NotificationView, TokenView} + alias Pleroma.Web.CommonAPI + alias Pleroma.{Repo, Activity, Object, User, Notification} + alias Pleroma.Web.OAuth.Token alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI @@ -542,6 +546,14 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end end + def oauth_tokens(%{assigns: %{user: user}} = conn, _params) do + with oauth_tokens <- Token.get_user_tokens(user) do + conn + |> put_view(TokenView) + |> render("index.json", %{tokens: oauth_tokens}) + end + end + def blocks(%{assigns: %{user: user}} = conn, _params) do with blocked_users <- User.blocked_users(user) do conn diff --git a/lib/pleroma/web/twitter_api/views/token_view.ex b/lib/pleroma/web/twitter_api/views/token_view.ex new file mode 100644 index 000000000..96b8526a4 --- /dev/null +++ b/lib/pleroma/web/twitter_api/views/token_view.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.TwitterAPI.TokenView do + use Pleroma.Web, :view + + def render("index.json", %{tokens: tokens}) do + tokens + |> render_many(Pleroma.Web.TwitterAPI.TokenView, "show.json") + |> Enum.filter(&Enum.any?/1) + end + + def render("show.json", %{token: token_entry}) do + %{ + id: token_entry.id, + token: token_entry.token, + refresh_token: token_entry.refresh_token, + valid_until: token_entry.valid_until + } + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index 0c21093ce..7a91549f5 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -227,4 +227,17 @@ defmodule Pleroma.Factory do unreachable_since: nil } end + + def oauth_token_factory do + user = insert(:user) + oauth_app = insert(:oauth_app) + + %Pleroma.Web.OAuth.Token{ + token: :crypto.strong_rand_bytes(32) |> Base.url_encode64(), + refresh_token: :crypto.strong_rand_bytes(32) |> Base.url_encode64(), + user_id: user.id, + app_id: oauth_app.id, + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10) + } + end end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 855ae1526..c50d82def 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1876,4 +1876,22 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do ActivityRepresenter.to_map(activity, %{user: user, for: user}) end end + + describe "GET /api/oauth_tokens" do + test "renders list" do + token = insert(:oauth_token) + + response = + build_conn() + |> assign(:user, Repo.get(User, token.user_id)) + |> get("/api/oauth_tokens") + + keys = + json_response(response, 200) + |> hd() + |> Map.keys() + + assert keys -- ["id", "refresh_token", "token", "valid_until"] == [] + end + end end From 62a45bdc11bc98ca4c24b0b8aa54c9d2958f81a1 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 11 Feb 2019 00:49:56 +0300 Subject: [PATCH 24/75] Add revoke token --- lib/pleroma/web/oauth/token.ex | 11 ++++++++- lib/pleroma/web/router.ex | 1 + .../web/twitter_api/twitter_api_controller.ex | 6 +++++ .../twitter_api_controller_test.exs | 23 ++++++++++++++++--- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 40bf0ac6b..380a39360 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -53,9 +53,18 @@ defmodule Pleroma.Web.OAuth.Token do |> Repo.delete_all() end - def get_user_tokens(%User{id: user_id}) do + def delete_user_token(%User{id: user_id}, token_id) do from( t in Pleroma.Web.OAuth.Token, + where: t.user_id == ^user_id, + where: t.id == ^token_id + ) + |> Repo.delete_all() + end + + def get_user_tokens(%User{id: user_id}) do + from( + t in Token, where: t.user_id == ^user_id ) |> Repo.all() diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a394900b2..d45fa526e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -391,6 +391,7 @@ defmodule Pleroma.Web.Router do get("/externalprofile/show", TwitterAPI.Controller, :external_profile) get("/oauth_tokens", TwitterAPI.Controller, :oauth_tokens) + delete("/oauth_tokens/:id", TwitterAPI.Controller, :revoke_token) end pipeline :ap_relay do diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 1a43e9a60..fac05f288 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -554,6 +554,12 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end end + def revoke_token(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do + Token.delete_user_token(user, id) + + json_reply(conn, 201, "") + end + def blocks(%{assigns: %{user: user}} = conn, _params) do with blocked_users <- User.blocked_users(user) do conn diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index c50d82def..527a920fb 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do alias Pleroma.Builders.{ActivityBuilder, UserBuilder} alias Pleroma.{Repo, Activity, User, Object, Notification} alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.OAuth.Token alias Pleroma.Web.TwitterAPI.UserView alias Pleroma.Web.TwitterAPI.NotificationView alias Pleroma.Web.CommonAPI @@ -1878,12 +1879,16 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do end describe "GET /api/oauth_tokens" do - test "renders list" do - token = insert(:oauth_token) + setup do + token = insert(:oauth_token) |> Repo.preload(:user) + %{token: token} + end + + test "renders list", %{token: token} do response = build_conn() - |> assign(:user, Repo.get(User, token.user_id)) + |> assign(:user, token.user) |> get("/api/oauth_tokens") keys = @@ -1893,5 +1898,17 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert keys -- ["id", "refresh_token", "token", "valid_until"] == [] end + + test "revoke token", %{token: token} do + response = + build_conn() + |> assign(:user, token.user) + |> delete("/api/oauth_tokens/#{token.id}") + + tokens = Token.get_user_tokens(token.user) + + assert tokens == [] + assert response.status == 201 + end end end From 760fec4cb85c7ddf2ec4fa5578ab4a1ceafc1e84 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 11 Feb 2019 09:48:24 +0000 Subject: [PATCH 25/75] Update token.ex --- lib/pleroma/web/oauth/token.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 380a39360..f5594f834 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -47,7 +47,7 @@ defmodule Pleroma.Web.OAuth.Token do def delete_user_tokens(%User{id: user_id}) do from( - t in Pleroma.Web.OAuth.Token, + t in Token, where: t.user_id == ^user_id ) |> Repo.delete_all() @@ -55,7 +55,7 @@ defmodule Pleroma.Web.OAuth.Token do def delete_user_token(%User{id: user_id}, token_id) do from( - t in Pleroma.Web.OAuth.Token, + t in Token, where: t.user_id == ^user_id, where: t.id == ^token_id ) From 88a4de24f9bc6fc73696cb5c986440c2c659b636 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 13 Feb 2019 13:52:27 +0100 Subject: [PATCH 26/75] User.follow_all: Respect blocks in both directions. --- lib/pleroma/user.ex | 4 ++-- test/user_test.exs | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0060d966b..3232cb842 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -311,12 +311,12 @@ defmodule Pleroma.User do end end - @doc "A mass follow for local users. Respects blocks but does not create activities." + @doc "A mass follow for local users. Respects blocks in both directions but does not create activities." @spec follow_all(User.t(), list(User.t())) :: {atom(), User.t()} def follow_all(follower, followeds) do followed_addresses = followeds - |> Enum.reject(fn %{ap_id: ap_id} -> ap_id in follower.info.blocks end) + |> Enum.reject(fn followed -> blocks?(follower, followed) || blocks?(followed, follower) end) |> Enum.map(fn %{follower_address: fa} -> fa end) q = diff --git a/test/user_test.exs b/test/user_test.exs index 523ab1ea4..a282274ce 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -55,18 +55,21 @@ defmodule Pleroma.UserTest do followed_two = insert(:user) blocked = insert(:user) not_followed = insert(:user) + reverse_blocked = insert(:user) {:ok, user} = User.block(user, blocked) + {:ok, reverse_blocked} = User.block(reverse_blocked, user) {:ok, user} = User.follow(user, followed_zero) - {:ok, user} = User.follow_all(user, [followed_one, followed_two, blocked]) + {:ok, user} = User.follow_all(user, [followed_one, followed_two, blocked, reverse_blocked]) assert User.following?(user, followed_one) assert User.following?(user, followed_two) assert User.following?(user, followed_zero) refute User.following?(user, not_followed) refute User.following?(user, blocked) + refute User.following?(user, reverse_blocked) end test "follow_all follows mutliple users without duplicating" do From bef9b9cb669d6c1081eb83c624af89dbf9a8b9da Mon Sep 17 00:00:00 2001 From: Karen Konou Date: Wed, 13 Feb 2019 16:23:09 +0100 Subject: [PATCH 27/75] refactored code --- .../web/activity_pub/mrf/hellthread_policy.ex | 86 +++++++++++-------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 9be382972..1f09b4e66 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -7,56 +7,66 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do @behaviour Pleroma.Web.ActivityPub.MRF defp delist_message(message) do + delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address - message - |> Map.put("to", [follower_collection]) - |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) + message = + with {:public, recipients} <- get_recipient_count(message) do + if recipients > delist_threshold and delist_threshold > 0 do + message + |> Map.put("to", [follower_collection]) + |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) + else + message + end + else + _ -> message + end + + {:ok, message} end - defp get_recipient_count(message) do - recipients = (message["to"] || []) ++ (message["cc"] || []) - - cond do - Enum.member?(recipients, "https://www.w3.org/ns/activitystreams#Public") && - Enum.find(recipients, &String.ends_with?(&1, "/followers")) -> - length(recipients) - 2 - - Enum.member?(recipients, "https://www.w3.org/ns/activitystreams#Public") || - Enum.find(recipients, &String.ends_with?(&1, "/followers")) -> - length(recipients) - 1 - - true -> - length(recipients) - end - end - - @impl true - def filter(%{"type" => "Create"} = message) do - delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) - + defp reject_message(message) do reject_threshold = Pleroma.Config.get( [:mrf_hellthread, :reject_threshold], Pleroma.Config.get([:mrf_hellthread, :threshold]) ) - recipients = get_recipient_count(message) - - cond do - recipients > reject_threshold and reject_threshold > 0 -> + with {_, recipients} <- get_recipient_count(message) do + if recipients > reject_threshold and reject_threshold > 0 do {:reject, nil} - - recipients > delist_threshold and delist_threshold > 0 -> - if Enum.member?(message["to"], "https://www.w3.org/ns/activitystreams#Public") or - Enum.member?(message["cc"], "https://www.w3.org/ns/activitystreams#Public") do - {:ok, delist_message(message)} - else - {:ok, message} - end - - true -> + else {:ok, message} + end + end + end + + defp get_recipient_count(message) do + recipients = (message["to"] || []) ++ (message["cc"] || []) + follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address + + if Enum.member?(recipients, "https://www.w3.org/ns/activitystreams#Public") do + recipients + |> List.delete("https://www.w3.org/ns/activitystreams#Public") + |> List.delete(follower_collection) + + {:public, length(recipients)} + else + recipients + |> List.delete(follower_collection) + + {:not_public, length(recipients)} + end + end + + @impl true + def filter(%{"type" => "Create"} = message) do + with {:ok, message} <- reject_message(message), + {:ok, message} <- delist_message(message) do + {:ok, message} + else + _e -> {:reject, nil} end end From 90facd359813197060f6c33f2389fce772550fc3 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 12 Feb 2019 21:28:11 +0000 Subject: [PATCH 28/75] user view: add AP C2S oauth endpoints to local user profiles --- .../web/activity_pub/views/user_view.ex | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 15e6c1f68..0d880212e 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -15,6 +15,20 @@ defmodule Pleroma.Web.ActivityPub.UserView do import Ecto.Query + def render("endpoints.json", %{user: %{local: true} = _user}) do + %{ + "oauthAuthorizationEndpoint" => "#{Pleroma.Web.Endpoint.url()}/oauth/authorize", + "oauthTokenEndpoint" => "#{Pleroma.Web.Endpoint.url()}/oauth/token" + } + |> Map.merge(render("endpoints.json", nil)) + end + + def render("endpoints.json", _) do + %{ + "sharedInbox" => "#{Pleroma.Web.Endpoint.url()}/inbox" + } + end + # the instance itself is not a Person, but instead an Application def render("user.json", %{user: %{nickname: nil} = user}) do {:ok, user} = WebFinger.ensure_keys_present(user) @@ -22,6 +36,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) public_key = :public_key.pem_encode([public_key]) + endpoints = render("endpoints.json", %{user: user}) + %{ "id" => user.ap_id, "type" => "Application", @@ -37,9 +53,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "owner" => user.ap_id, "publicKeyPem" => public_key }, - "endpoints" => %{ - "sharedInbox" => "#{Pleroma.Web.Endpoint.url()}/inbox" - } + "endpoints" => endpoints } |> Map.merge(Utils.make_json_ld_header()) end @@ -50,6 +64,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) public_key = :public_key.pem_encode([public_key]) + endpoints = render("endpoints.json", %{user: user}) + %{ "id" => user.ap_id, "type" => "Person", @@ -67,9 +83,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "owner" => user.ap_id, "publicKeyPem" => public_key }, - "endpoints" => %{ - "sharedInbox" => "#{Pleroma.Web.Endpoint.url()}/inbox" - }, + "endpoints" => endpoints, "icon" => %{ "type" => "Image", "url" => User.avatar_url(user) From dd989962e681a126ff086064d22cbcbd9bfaf7a2 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 12 Feb 2019 21:37:37 +0000 Subject: [PATCH 29/75] litepub schema: add oauthRegistrationEndpoint [ci skip] --- priv/static/schemas/litepub-0.1.jsonld | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld index 15645646a..f36b231c5 100644 --- a/priv/static/schemas/litepub-0.1.jsonld +++ b/priv/static/schemas/litepub-0.1.jsonld @@ -19,7 +19,11 @@ "value": "schema:value", "sensitive": "as:sensitive", "litepub": "http://litepub.social/ns#", - "directMessage": "litepub:directMessage" + "directMessage": "litepub:directMessage", + "oauthRegistrationEndpoint": { + "@id": "litepub:oauthRegistrationEndpoint", + "@type": "@id" + } } ] } From db8abd958dc2266262def048352466280c12d3a7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 12 Feb 2019 21:42:32 +0000 Subject: [PATCH 30/75] activitypub: user view: fix up endpoints rendering --- lib/pleroma/web/activity_pub/views/user_view.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 0d880212e..44beee728 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -15,12 +15,12 @@ defmodule Pleroma.Web.ActivityPub.UserView do import Ecto.Query - def render("endpoints.json", %{user: %{local: true} = _user}) do + def render("endpoints.json", %{user: %User{local: true} = _user}) do %{ "oauthAuthorizationEndpoint" => "#{Pleroma.Web.Endpoint.url()}/oauth/authorize", "oauthTokenEndpoint" => "#{Pleroma.Web.Endpoint.url()}/oauth/token" } - |> Map.merge(render("endpoints.json", nil)) + |> Map.merge(render("endpoints.json", %{user: nil})) end def render("endpoints.json", _) do From 29e946ace43f5dd3342e2bd3699004e9c56e711d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 12 Feb 2019 21:49:48 +0000 Subject: [PATCH 31/75] activitypub: user view: add oauthRegistrationEndpoint to user profiles --- lib/pleroma/web/activity_pub/views/user_view.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 44beee728..af75546dd 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -18,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do def render("endpoints.json", %{user: %User{local: true} = _user}) do %{ "oauthAuthorizationEndpoint" => "#{Pleroma.Web.Endpoint.url()}/oauth/authorize", + "oauthRegistrationEndpoint" => "#{Pleroma.Web.Endpoint.url()}/api/v1/apps", "oauthTokenEndpoint" => "#{Pleroma.Web.Endpoint.url()}/oauth/token" } |> Map.merge(render("endpoints.json", %{user: nil})) From 9bd6ed975ec57f46ff6796fadb8822faec262bbc Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 13 Feb 2019 19:20:41 +0000 Subject: [PATCH 32/75] activitypub: user view: use route helpers instead of hardcoded URIs --- .../web/activity_pub/views/user_view.ex | 18 ++++++++---------- lib/pleroma/web/router.ex | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index af75546dd..035463de2 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -12,23 +12,21 @@ defmodule Pleroma.Web.ActivityPub.UserView do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.Router.Helpers + alias Pleroma.Web.Endpoint import Ecto.Query - def render("endpoints.json", %{user: %User{local: true} = _user}) do + def render("endpoints.json", %{user: %User{nickname: _nickname, local: true} = _user}) do %{ - "oauthAuthorizationEndpoint" => "#{Pleroma.Web.Endpoint.url()}/oauth/authorize", - "oauthRegistrationEndpoint" => "#{Pleroma.Web.Endpoint.url()}/api/v1/apps", - "oauthTokenEndpoint" => "#{Pleroma.Web.Endpoint.url()}/oauth/token" + "oauthAuthorizationEndpoint" => Helpers.o_auth_url(Endpoint, :authorize), + "oauthRegistrationEndpoint" => Helpers.mastodon_api_url(Endpoint, :create_app), + "oauthTokenEndpoint" => Helpers.o_auth_url(Endpoint, :token_exchange), + "sharedInbox" => Helpers.activity_pub_url(Endpoint, :inbox) } - |> Map.merge(render("endpoints.json", %{user: nil})) end - def render("endpoints.json", _) do - %{ - "sharedInbox" => "#{Pleroma.Web.Endpoint.url()}/inbox" - } - end + def render("endpoints.json", _), do: %{} # the instance itself is not a Person, but instead an Application def render("user.json", %{user: %{nickname: nil} = user}) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5b5627ce8..d66a1c2a1 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -468,8 +468,8 @@ defmodule Pleroma.Web.Router do scope "/", Pleroma.Web.ActivityPub do pipe_through(:activitypub) - post("/users/:nickname/inbox", ActivityPubController, :inbox) post("/inbox", ActivityPubController, :inbox) + post("/users/:nickname/inbox", ActivityPubController, :inbox) end scope "/.well-known", Pleroma.Web do From d54c483964692e1ca6b813d6b35a0635d3c0abf9 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 13 Feb 2019 19:48:24 +0000 Subject: [PATCH 33/75] tests: add tests for endpoints --- .../web/activity_pub/views/user_view_test.exs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 7fc870e96..95d736c50 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -15,4 +15,32 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert String.contains?(result["publicKey"]["publicKeyPem"], "BEGIN PUBLIC KEY") end + + describe "endpoints" do + test "local users have a usable endpoints structure" do + user = insert(:user) + {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + + result = UserView.render("user.json", %{user: user}) + + assert result["id"] == user.ap_id + + %{ + "sharedInbox" => _, + "oauthAuthorizationEndpoint" => _, + "oauthRegistrationEndpoint" => _, + "oauthTokenEndpoint" => _ + } = result["endpoints"] + end + + test "remote users have an empty endpoints structure" do + user = insert(:user, local: false) + {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + + result = UserView.render("user.json", %{user: user}) + + assert result["id"] == user.ap_id + assert result["endpoints"] == %{} + end + end end From f62c1d6266be1af59aa5e0fe05e438c54c330e74 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 13 Feb 2019 22:33:22 +0000 Subject: [PATCH 34/75] Improve login error for OAuth flow --- lib/pleroma/web/templates/layout/app.html.eex | 26 +++++++++++++++++++ .../web/templates/o_auth/o_auth/show.html.eex | 4 +++ 2 files changed, 30 insertions(+) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 8dd3284d6..520e4b3d5 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -67,6 +67,32 @@ font-weight: 500; font-size: 16px; } + + .alert-danger { + box-sizing: border-box; + width: 100%; + color: #D8000C; + background-color: #FFD2D2; + border-radius: 4px; + border: none; + padding: 10px; + margin-top: 20px; + font-weight: 500; + font-size: 16px; + } + + .alert-info { + box-sizing: border-box; + width: 100%; + color: #00529B; + background-color: #BDE5F8; + border-radius: 4px; + border: none; + padding: 10px; + margin-top: 20px; + font-weight: 500; + font-size: 16px; + } diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index de2241ec9..32c458f0c 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -1,5 +1,9 @@ +<%= if get_flash(@conn, :info) do %> +<% end %> +<%= if get_flash(@conn, :error) do %> +<% end %>

OAuth Authorization

<%= form_for @conn, o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> <%= label f, :name, "Name or email" %> From 94cbbb0e3a047c1d2851e2214b408dc19f569fbd Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 00:27:35 +0000 Subject: [PATCH 35/75] activitypub: transmogrifier: do not attempt to expand pre-existing AS2 tag objects --- lib/pleroma/web/activity_pub/transmogrifier.ex | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 98a2af819..5da65fa39 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -765,12 +765,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def add_hashtags(object) do tags = (object["tag"] || []) - |> Enum.map(fn tag -> - %{ - "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}", - "name" => "##{tag}", - "type" => "Hashtag" - } + |> Enum.map(fn + # Expand internal representation tags into AS2 tags. + tag when is_binary(tag) -> + %{ + "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}", + "name" => "##{tag}", + "type" => "Hashtag" + } + + # Do not process tags which are already AS2 tag objects. + tag when is_map(tag) -> + tag end) object From e05bf2940f764e8182edcf58659eeee1db751118 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 00:34:20 +0000 Subject: [PATCH 36/75] activitypub: transmogrifier: correctly handle nil inReplyTo value --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5da65fa39..26b2dd575 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -649,7 +649,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do if object = Object.normalize(id), do: {:ok, object}, else: nil end - def set_reply_to_uri(%{"inReplyTo" => inReplyTo} = object) do + def set_reply_to_uri(%{"inReplyTo" => inReplyTo} = object) when is_binary(inReplyTo) do with false <- String.starts_with?(inReplyTo, "http"), {:ok, %{data: replied_to_object}} <- get_obj_helper(inReplyTo) do Map.put(object, "inReplyTo", replied_to_object["external_url"] || inReplyTo) From 889ad95a2a766b82d17aa148d92754ecda244bf7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 00:59:18 +0000 Subject: [PATCH 37/75] tests: add some reserialization tests based on IR differences --- test/web/activity_pub/transmogrifier_test.exs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index e5e3c8d33..86c66deff 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1128,4 +1128,58 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do ) end end + + describe "reserialization" do + test "successfully reserializes a message with inReplyTo == nil" do + user = insert(:user) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Create", + "object" => %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Note", + "content" => "Hi", + "inReplyTo" => nil, + "attributedTo" => user.ap_id + }, + "actor" => user.ap_id + } + + {:ok, activity} = Transmogrifier.handle_incoming(message) + + {:ok, _} = Transmogrifier.prepare_outgoing(activity.data) + end + + test "successfully reserializes a message with AS2 objects in IR" do + user = insert(:user) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Create", + "object" => %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Note", + "content" => "Hi", + "inReplyTo" => nil, + "attributedTo" => user.ap_id, + "tag" => [ + %{"name" => "#2hu", "href" => "http://example.com/2hu", "type" => "Hashtag"}, + %{"name" => "Bob", "href" => "http://example.com/bob", "type" => "Mention"} + ] + }, + "actor" => user.ap_id + } + + {:ok, activity} = Transmogrifier.handle_incoming(message) + + {:ok, _} = Transmogrifier.prepare_outgoing(activity.data) + end + end end From e9ef4b8da627e516d5f1a2b742c6dafa65232098 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 01:05:25 +0000 Subject: [PATCH 38/75] oauth: never use base64 padding when returning tokens to applications The normal Base64 alphabet uses the equals sign (=) as a padding character. Since Base64 strings are self-synchronizing, padding characters are unnecessary, so don't generate them in the first place. --- lib/pleroma/web/oauth/app.ex | 10 ++++++++-- lib/pleroma/web/oauth/authorization.ex | 2 +- lib/pleroma/web/oauth/oauth_controller.ex | 2 +- lib/pleroma/web/oauth/token.ex | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/oauth/app.ex b/lib/pleroma/web/oauth/app.ex index 3e8acde31..8b61bf3a4 100644 --- a/lib/pleroma/web/oauth/app.ex +++ b/lib/pleroma/web/oauth/app.ex @@ -25,8 +25,14 @@ defmodule Pleroma.Web.OAuth.App do if changeset.valid? do changeset - |> put_change(:client_id, :crypto.strong_rand_bytes(32) |> Base.url_encode64()) - |> put_change(:client_secret, :crypto.strong_rand_bytes(32) |> Base.url_encode64()) + |> put_change( + :client_id, + :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + ) + |> put_change( + :client_secret, + :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + ) else changeset end diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index 75c9ab9aa..9039b8b45 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Web.OAuth.Authorization do end def create_authorization(%App{} = app, %User{} = user) do - token = :crypto.strong_rand_bytes(32) |> Base.url_encode64() + token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) authorization = %Authorization{ token: token, diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index e4d0601f8..dddfcf299 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -173,7 +173,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do token |> URI.decode() |> Base.url_decode64!(padding: false) - |> Base.url_encode64() + |> Base.url_encode64(padding: false) end defp get_app_from_request(conn, params) do diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index b0bbeeb69..ca9e718ac 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -31,8 +31,8 @@ defmodule Pleroma.Web.OAuth.Token do end def create_token(%App{} = app, %User{} = user) do - token = :crypto.strong_rand_bytes(32) |> Base.url_encode64() - refresh_token = :crypto.strong_rand_bytes(32) |> Base.url_encode64() + token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + refresh_token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) token = %Token{ token: token, From 64620d8980e3e93791d3f880296be2060ffc4d39 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 02:41:21 +0000 Subject: [PATCH 39/75] activitypub: user view: do not expose oAuth endpoints for instance users --- lib/pleroma/web/activity_pub/views/user_view.ex | 6 +++++- test/web/activity_pub/views/user_view_test.exs | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 035463de2..b363a3dc4 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -17,7 +17,11 @@ defmodule Pleroma.Web.ActivityPub.UserView do import Ecto.Query - def render("endpoints.json", %{user: %User{nickname: _nickname, local: true} = _user}) do + def render("endpoints.json", %{user: %User{nickname: nil, local: true} = _user}) do + %{"sharedInbox" => Helpers.activity_pub_url(Endpoint, :inbox)} + end + + def render("endpoints.json", %{user: %User{local: true} = _user}) do %{ "oauthAuthorizationEndpoint" => Helpers.o_auth_url(Endpoint, :authorize), "oauthRegistrationEndpoint" => Helpers.mastodon_api_url(Endpoint, :create_app), diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 95d736c50..0bc1d4728 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -42,5 +42,16 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert result["id"] == user.ap_id assert result["endpoints"] == %{} end + + test "instance users do not expose oAuth endpoints" do + user = insert(:user, nickname: nil, local: true) + {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + + result = UserView.render("user.json", %{user: user}) + + refute result["endpoints"]["oauthAuthorizationEndpoint"] + refute result["endpoints"]["oauthRegistrationEndpoint"] + refute result["endpoints"]["oauthTokenEndpoint"] + end end end From ee2fa1a31475259575a267e67cf5d7da04a30616 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 03:01:39 +0000 Subject: [PATCH 40/75] activitypub: user view: remove totalInbox from user inbox view It is not really feasible to quickly calculate the totalItems value and it shouldn't be trusted anyway. --- lib/pleroma/web/activity_pub/views/user_view.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index b363a3dc4..1e16f7ebb 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -224,7 +224,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do "id" => "#{iri}?max_id=#{max_id}", "type" => "OrderedCollectionPage", "partOf" => iri, - "totalItems" => -1, "orderedItems" => collection, "next" => "#{iri}?max_id=#{min_id}" } @@ -233,7 +232,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do %{ "id" => iri, "type" => "OrderedCollection", - "totalItems" => -1, "first" => page } |> Map.merge(Utils.make_json_ld_header()) From 6542b8629203cd3b3ef4f0a08a5ad67451366a72 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 03:02:45 +0000 Subject: [PATCH 41/75] activitypub: user view: remove totalItems from user outbox (this is based on a counter in User.Info, but the counter is not reliable.) --- lib/pleroma/web/activity_pub/views/user_view.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 1e16f7ebb..506fa5ea3 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -177,7 +177,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do "id" => "#{iri}?max_id=#{max_id}", "type" => "OrderedCollectionPage", "partOf" => iri, - "totalItems" => info.note_count, "orderedItems" => collection, "next" => "#{iri}?max_id=#{min_id}" } @@ -186,7 +185,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do %{ "id" => iri, "type" => "OrderedCollection", - "totalItems" => info.note_count, "first" => page } |> Map.merge(Utils.make_json_ld_header()) From 5307c211b8ca00def779221ecbde2cc4bea6d2d5 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 03:03:41 +0000 Subject: [PATCH 42/75] activitypub: user view: report totalItems=0 for follows/followers when hidden --- .../web/activity_pub/views/user_view.ex | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 506fa5ea3..dd1ab3d2b 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -104,8 +104,14 @@ defmodule Pleroma.Web.ActivityPub.UserView do query = User.get_friends_query(user) query = from(user in query, select: [:ap_id]) following = Repo.all(query) + total = + if !user.info.hide_follows do + length(following) + else + 0 + end - collection(following, "#{user.ap_id}/following", page, !user.info.hide_follows) + collection(following, "#{user.ap_id}/following", page, !user.info.hide_follows, total) |> Map.merge(Utils.make_json_ld_header()) end @@ -113,11 +119,17 @@ defmodule Pleroma.Web.ActivityPub.UserView do query = User.get_friends_query(user) query = from(user in query, select: [:ap_id]) following = Repo.all(query) + total = + if !user.info.hide_follows do + length(following) + else + 0 + end %{ "id" => "#{user.ap_id}/following", "type" => "OrderedCollection", - "totalItems" => length(following), + "totalItems" => total, "first" => collection(following, "#{user.ap_id}/following", 1, !user.info.hide_follows) } |> Map.merge(Utils.make_json_ld_header()) @@ -127,8 +139,14 @@ defmodule Pleroma.Web.ActivityPub.UserView do query = User.get_followers_query(user) query = from(user in query, select: [:ap_id]) followers = Repo.all(query) + total = + if !user.info.hide_followers do + length(followers) + else + 0 + end - collection(followers, "#{user.ap_id}/followers", page, !user.info.hide_followers) + collection(followers, "#{user.ap_id}/followers", page, !user.info.hide_followers, total) |> Map.merge(Utils.make_json_ld_header()) end @@ -136,20 +154,23 @@ defmodule Pleroma.Web.ActivityPub.UserView do query = User.get_followers_query(user) query = from(user in query, select: [:ap_id]) followers = Repo.all(query) + total = + if !user.info.hide_followers do + length(followers) + else + 0 + end %{ "id" => "#{user.ap_id}/followers", "type" => "OrderedCollection", - "totalItems" => length(followers), - "first" => collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_followers) + "totalItems" => total, + "first" => collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_followers, total) } |> Map.merge(Utils.make_json_ld_header()) end def render("outbox.json", %{user: user, max_id: max_qid}) do - # XXX: technically note_count is wrong for this, but it's better than nothing - info = User.user_info(user) - params = %{ "limit" => "10" } From 72ba5b4ab73ff725b91888e38af612082f8df5ad Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 03:13:07 +0000 Subject: [PATCH 43/75] activitypub: user view: formatting --- lib/pleroma/web/activity_pub/views/user_view.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index dd1ab3d2b..c8e154989 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -104,6 +104,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do query = User.get_friends_query(user) query = from(user in query, select: [:ap_id]) following = Repo.all(query) + total = if !user.info.hide_follows do length(following) @@ -119,6 +120,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do query = User.get_friends_query(user) query = from(user in query, select: [:ap_id]) following = Repo.all(query) + total = if !user.info.hide_follows do length(following) @@ -139,6 +141,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do query = User.get_followers_query(user) query = from(user in query, select: [:ap_id]) followers = Repo.all(query) + total = if !user.info.hide_followers do length(followers) @@ -154,6 +157,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do query = User.get_followers_query(user) query = from(user in query, select: [:ap_id]) followers = Repo.all(query) + total = if !user.info.hide_followers do length(followers) @@ -165,7 +169,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do "id" => "#{user.ap_id}/followers", "type" => "OrderedCollection", "totalItems" => total, - "first" => collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_followers, total) + "first" => + collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_followers, total) } |> Map.merge(Utils.make_json_ld_header()) end From e031cc6473e12cae7249324fe6fdea5287b6304a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 03:22:54 +0000 Subject: [PATCH 44/75] tests: update tests for totalItems leak fix --- test/web/activity_pub/activity_pub_controller_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 570bee6b3..9f6d87caa 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -397,7 +397,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> json_response(200) assert result["first"]["orderedItems"] == [] - assert result["totalItems"] == 1 + assert result["totalItems"] == 0 end test "it works for more than 10 users", %{conn: conn} do @@ -452,7 +452,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> json_response(200) assert result["first"]["orderedItems"] == [] - assert result["totalItems"] == 1 + assert result["totalItems"] == 0 end test "it works for more than 10 users", %{conn: conn} do From 907306174b082cccd823894c855194a4fc1e8305 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 14 Feb 2019 15:55:21 +0700 Subject: [PATCH 45/75] fix S3 links encoding in Mediaproxy --- lib/pleroma/web/media_proxy/media_proxy.ex | 13 +++++++++++++ test/media_proxy_test.exs | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 1e9da7283..39a725a69 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -19,11 +19,16 @@ defmodule Pleroma.Web.MediaProxy do else secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base] + # Must preserve `%2F` for compatibility with S3 (https://git.pleroma.social/pleroma/pleroma/issues/580) + replacement = get_replacement(url, ":2F:") + # The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice. base64 = url + |> String.replace("%2F", replacement) |> URI.decode() |> URI.encode() + |> String.replace(replacement, "%2F") |> Base.url_encode64(@base64_opts) sig = :crypto.hmac(:sha, secret, base64) @@ -60,4 +65,12 @@ defmodule Pleroma.Web.MediaProxy do |> Enum.filter(fn value -> value end) |> Path.join() end + + defp get_replacement(url, replacement) do + if String.contains?(url, replacement) do + get_replacement(url, replacement <> replacement) + else + replacement + end + end end diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index 05d927422..ddbadfbf5 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -140,6 +140,15 @@ defmodule Pleroma.MediaProxyTest do assert String.starts_with?(encoded, Pleroma.Config.get([:media_proxy, :base_url])) end + + # https://git.pleroma.social/pleroma/pleroma/issues/580 + test "encoding S3 links (must preserve `%2F`)" do + url = + "https://s3.amazonaws.com/example/test.png?X-Amz-Credential=your-access-key-id%2F20130721%2Fus-east-1%2Fs3%2Faws4_request" + + encoded = url(url) + assert decode_result(encoded) == url + end end describe "when disabled" do From 3f32d7b937a2368707794b55d1256bfa9fa508f4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 14 Feb 2019 17:02:47 +0700 Subject: [PATCH 46/75] Fix queue name --- lib/pleroma/web/federator/federator.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 7df75aca6..d4e2a9742 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -38,31 +38,31 @@ defmodule Pleroma.Web.Federator do end def publish(activity, priority \\ 1) do - Jobs.enqueue(:federator_out, __MODULE__, [:publish, activity], priority) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority) end def publish_single_ap(params) do - Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_ap, params]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_ap, params]) end def publish_single_websub(websub) do - Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_websub, websub]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_websub, websub]) end def verify_websub(websub) do - Jobs.enqueue(:federator_out, __MODULE__, [:verify_websub, websub]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub]) end def request_subscription(sub) do - Jobs.enqueue(:federator_out, __MODULE__, [:request_subscription, sub]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub]) end def refresh_subscriptions() do - Jobs.enqueue(:federator_out, __MODULE__, [:refresh_subscriptions]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions]) end def publish_single_salmon(params) do - Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_salmon, params]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_salmon, params]) end # Job Worker Callbacks From 56862f4ce1275689416661847b0734129f5471ae Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 19:42:33 +0000 Subject: [PATCH 47/75] activitypub: clean up logging statements a little --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c46d8233e..ab2872f56 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -818,8 +818,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if object = Object.get_cached_by_ap_id(id) do {:ok, object} else - Logger.info("Fetching #{id} via AP") - with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), nil <- Object.normalize(data), params <- %{ @@ -851,7 +849,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def fetch_and_contain_remote_object_from_id(id) do - Logger.info("Fetching #{id} via AP") + Logger.info("Fetching object #{id} via AP") with true <- String.starts_with?(id, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- From da44cdd3812fabb777c738b162c704de22e4b2f4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 19:58:24 +0000 Subject: [PATCH 48/75] user: search: use get_or_fetch() instead of get_or_fetch_by_nickname() get_or_fetch() handles the nickname verses URI differences transparently. --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3232cb842..29d2b3d89 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -731,7 +731,7 @@ defmodule Pleroma.User do # Strip the beginning @ off if there is a query query = String.trim_leading(query, "@") - if resolve, do: User.get_or_fetch_by_nickname(query) + if resolve, do: get_or_fetch(query) fts_results = do_search(fts_search_subquery(query), for_user) From 32b164943433ddebfdb04494bbd4d4e8a90578d4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 14 Feb 2019 19:59:12 +0000 Subject: [PATCH 49/75] test: user: add a test for whether user search returns a user or not --- test/user_test.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/user_test.exs b/test/user_test.exs index 58587bd82..a99b79a0d 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -878,6 +878,16 @@ defmodule Pleroma.UserTest do assert [] == User.search(query) end) end + + test "works with URIs" do + results = User.search("http://mastodon.example.org/users/admin", true) + result = results |> List.first() + + user = User.get_by_ap_id("http://mastodon.example.org/users/admin") + + assert length(results) == 1 + assert user == result |> Map.put(:search_rank, nil) + end end test "auth_active?/1 works correctly" do From ecdf0657ba4a90d821d3874c827593963e0ff041 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sun, 10 Feb 2019 02:26:29 +0300 Subject: [PATCH 50/75] Add logic for keeping follow_request_count up-to-date on the `follow`, `approve_friend_request`, and `deny_friend_request` actions. Add follow_request_count to the user view. --- lib/pleroma/user.ex | 26 +++++++++++++++++++ lib/pleroma/user/info.ex | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 16 +++++++----- .../web/activity_pub/transmogrifier.ex | 6 ++--- .../mastodon_api/mastodon_api_controller.ex | 4 +-- .../web/twitter_api/twitter_api_controller.ex | 4 +-- .../web/twitter_api/views/user_view.ex | 18 ++++++++++--- .../mastodon_api_controller_test.exs | 8 +++++- .../twitter_api_controller_test.exs | 26 +++++++++++++++++++ 9 files changed, 91 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3232cb842..854787a2b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -618,6 +618,32 @@ defmodule Pleroma.User do ) end + def update_follow_request_count(%User{} = user) do + subquery = + user + |> User.get_follow_requests_query() + |> select([a], %{count: count(a.id)}) + + User + |> where(id: ^user.id) + |> join(:inner, [u], s in subquery(subquery)) + |> update([u, s], + set: [ + info: + fragment( + "jsonb_set(?, '{follow_request_count}', ?::varchar::jsonb, true)", + u.info, + s.count + ) + ] + ) + |> Repo.update_all([], returning: true) + |> case do + {1, [user]} -> {:ok, user} + _ -> {:error, user} + end + end + def get_follow_requests(%User{} = user) do q = get_follow_requests_query(user) reqs = Repo.all(q) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 9d8779fab..c59e74c45 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -12,6 +12,7 @@ defmodule Pleroma.User.Info do field(:source_data, :map, default: %{}) field(:note_count, :integer, default: 0) field(:follower_count, :integer, default: 0) + field(:follow_request_count, :integer, default: 0) field(:locked, :boolean, default: false) field(:confirmation_pending, :boolean, default: false) field(:confirmation_token, :string, default: nil) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c46d8233e..975f9fde3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -172,9 +172,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # only accept false as false value local = !(params[:local] == false) - with data <- %{"to" => to, "type" => "Accept", "actor" => actor, "object" => object}, + with data <- %{"to" => to, "type" => "Accept", "actor" => actor.ap_id, "object" => object}, {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.update_follow_request_count(actor) do {:ok, activity} end end @@ -183,9 +184,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # only accept false as false value local = !(params[:local] == false) - with data <- %{"to" => to, "type" => "Reject", "actor" => actor, "object" => object}, + with data <- %{"to" => to, "type" => "Reject", "actor" => actor.ap_id, "object" => object}, {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.update_follow_request_count(actor) do {:ok, activity} end end @@ -283,7 +285,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def follow(follower, followed, activity_id \\ nil, local \\ true) do with data <- make_follow_data(follower, followed, activity_id), {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.update_follow_request_count(followed) do {:ok, activity} end end @@ -293,7 +296,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, follow_activity} <- update_follow_state(follow_activity, "cancelled"), unfollow_data <- make_unfollow_data(follower, followed, follow_activity, activity_id), {:ok, activity} <- insert(unfollow_data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.update_follow_request_count(followed) do {:ok, activity} end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 26b2dd575..41d89a02b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -406,7 +406,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do if not User.locked?(followed) do ActivityPub.accept(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: data, local: true }) @@ -432,7 +432,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do ActivityPub.accept(%{ to: follow_activity.data["to"], type: "Accept", - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], local: false }) do @@ -458,7 +458,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do ActivityPub.reject(%{ to: follow_activity.data["to"], type: "Reject", - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], local: false }) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index dcaeccac6..f0bbe5b3f 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -680,7 +680,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do {:ok, _activity} <- ActivityPub.accept(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], type: "Accept" }) do @@ -702,7 +702,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do {:ok, _activity} <- ActivityPub.reject(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], type: "Reject" }) do diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index c2f0dc2a9..70ae4068a 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -570,7 +570,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do {:ok, _activity} <- ActivityPub.accept(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], type: "Accept" }) do @@ -590,7 +590,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do {:ok, _activity} <- ActivityPub.reject(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], type: "Reject" }) do diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index a09450df7..df7384476 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -113,10 +113,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do "fields" => fields, # Pleroma extension - "pleroma" => %{ - "confirmation_pending" => user_info.confirmation_pending, - "tags" => user.tags - } + "pleroma" => + %{ + "confirmation_pending" => user_info.confirmation_pending, + "tags" => user.tags + } + |> maybe_with_follow_request_count(user, for_user) } data = @@ -132,6 +134,14 @@ defmodule Pleroma.Web.TwitterAPI.UserView do end end + defp maybe_with_follow_request_count(data, %User{id: id, info: %{locked: true}} = user, %User{ + id: id + }) do + Map.put(data, "follow_request_count", user.info.follow_request_count) + end + + defp maybe_with_follow_request_count(data, _, _), do: data + defp maybe_with_role(data, %User{id: id} = user, %User{id: id}) do Map.merge(data, %{"role" => role(user), "show_role" => user.info.show_role}) end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 26c9c25a6..7749c5ded 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -937,7 +937,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "/api/v1/follow_requests/:id/authorize works" do - user = insert(:user, %{info: %Pleroma.User.Info{locked: true}}) + user = insert(:user, %{info: %User.Info{locked: true}}) other_user = insert(:user) {:ok, _activity} = ActivityPub.follow(other_user, user) @@ -946,6 +946,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == false + assert user.info.follow_request_count == 1 conn = build_conn() @@ -959,6 +960,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == true + assert user.info.follow_request_count == 0 end test "verify_credentials", %{conn: conn} do @@ -979,6 +981,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do {:ok, _activity} = ActivityPub.follow(other_user, user) + user = Repo.get(User, user.id) + assert user.info.follow_request_count == 1 + conn = build_conn() |> assign(:user, user) @@ -991,6 +996,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == false + assert user.info.follow_request_count == 0 end end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index acb03b146..50b19fd86 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -640,6 +640,24 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert json_response(conn, 200) == UserView.render("show.json", %{user: followed, for: current_user}) end + + test "for restricted account", %{conn: conn, user: current_user} do + followed = insert(:user, info: %User.Info{locked: true}) + + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/friendships/create.json", %{user_id: followed.id}) + + current_user = Repo.get(User, current_user.id) + followed = Repo.get(User, followed.id) + + refute User.ap_followers(followed) in current_user.following + assert followed.info.follow_request_count == 1 + + assert json_response(conn, 200) == + UserView.render("show.json", %{user: followed, for: current_user}) + end end describe "POST /friendships/destroy.json" do @@ -1676,15 +1694,19 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == false + assert user.info.follow_request_count == 1 conn = build_conn() |> assign(:user, user) |> post("/api/pleroma/friendships/approve", %{"user_id" => other_user.id}) + user = Repo.get(User, user.id) + assert relationship = json_response(conn, 200) assert other_user.id == relationship["id"] assert relationship["follows_you"] == true + assert user.info.follow_request_count == 0 end end @@ -1699,15 +1721,19 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == false + assert user.info.follow_request_count == 1 conn = build_conn() |> assign(:user, user) |> post("/api/pleroma/friendships/deny", %{"user_id" => other_user.id}) + user = Repo.get(User, user.id) + assert relationship = json_response(conn, 200) assert other_user.id == relationship["id"] assert relationship["follows_you"] == false + assert user.info.follow_request_count == 0 end end From d943c90249e0a598e57a0dbdf41d387b53916092 Mon Sep 17 00:00:00 2001 From: Karen Konou Date: Fri, 15 Feb 2019 12:47:50 +0100 Subject: [PATCH 51/75] Add tests, change default config values, fix a bug --- config/config.exs | 4 +- .../web/activity_pub/mrf/hellthread_policy.ex | 12 +++-- .../mrf/hellthread_policy_test.exs | 50 +++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 test/web/activity_pub/mrf/hellthread_policy_test.exs diff --git a/config/config.exs b/config/config.exs index 5db0ea9aa..e2a239a76 100644 --- a/config/config.exs +++ b/config/config.exs @@ -228,8 +228,8 @@ config :pleroma, :mrf_rejectnonpublic, allow_direct: false config :pleroma, :mrf_hellthread, - delist_threshold: 5, - reject_threshold: 10 + delist_threshold: 10, + reject_threshold: 20 config :pleroma, :mrf_simple, media_removal: [], diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 1f09b4e66..95211c596 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -47,14 +47,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address if Enum.member?(recipients, "https://www.w3.org/ns/activitystreams#Public") do - recipients - |> List.delete("https://www.w3.org/ns/activitystreams#Public") - |> List.delete(follower_collection) + recipients = + recipients + |> List.delete("https://www.w3.org/ns/activitystreams#Public") + |> List.delete(follower_collection) {:public, length(recipients)} else - recipients - |> List.delete(follower_collection) + recipients = + recipients + |> List.delete(follower_collection) {:not_public, length(recipients)} end diff --git a/test/web/activity_pub/mrf/hellthread_policy_test.exs b/test/web/activity_pub/mrf/hellthread_policy_test.exs new file mode 100644 index 000000000..b5bdd35cc --- /dev/null +++ b/test/web/activity_pub/mrf/hellthread_policy_test.exs @@ -0,0 +1,50 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do + use Pleroma.DataCase + import Pleroma.Factory + + import Pleroma.Web.ActivityPub.MRF.HellthreadPolicy + + describe "hellthread filter tests" do + setup do + user = insert(:user) + + message = %{ + "actor" => user.ap_id, + "cc" => [user.follower_address], + "type" => "Create", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "https://instace.tld/users/user1", + "https://instace.tld/users/user2", + "https://instace.tld/users/user3" + ] + } + + [user: user, message: message] + end + + test "reject test", %{message: message} do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 2}) + + {:reject, nil} = filter(message) + end + + test "delist test", %{user: user, message: message} do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 2, reject_threshold: 0}) + + {:ok, message} = filter(message) + assert user.follower_address in message["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in message["cc"] + end + + test "threshold test", %{message: message} do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 3}) + + {:ok, _} = filter(message) + end + end +end From dca6bee2f7eba1dc366cc65d3087f20678549739 Mon Sep 17 00:00:00 2001 From: Karen Konou Date: Fri, 15 Feb 2019 13:43:14 +0100 Subject: [PATCH 52/75] Rename test, add check for follower collection when delisting --- .../web/activity_pub/mrf/hellthread_policy.ex | 17 +++++++++++------ .../activity_pub/mrf/hellthread_policy_test.exs | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 95211c596..1fd7b9c67 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -10,17 +10,22 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address + follower_collection? = Enum.member?(message["to"] ++ message["cc"], follower_collection) + message = - with {:public, recipients} <- get_recipient_count(message) do - if recipients > delist_threshold and delist_threshold > 0 do + case recipients = get_recipient_count(message) do + {:public, _} when follower_collection? and recipients > delist_threshold -> message |> Map.put("to", [follower_collection]) |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) - else + + {:public, _} when recipients > delist_threshold -> + message + |> Map.put("to", []) + |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) + + _ -> message - end - else - _ -> message end {:ok, message} diff --git a/test/web/activity_pub/mrf/hellthread_policy_test.exs b/test/web/activity_pub/mrf/hellthread_policy_test.exs index b5bdd35cc..ebf9997cd 100644 --- a/test/web/activity_pub/mrf/hellthread_policy_test.exs +++ b/test/web/activity_pub/mrf/hellthread_policy_test.exs @@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do assert "https://www.w3.org/ns/activitystreams#Public" in message["cc"] end - test "threshold test", %{message: message} do + test "excludes follower collection and public URI from threshold count", %{message: message} do Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 3}) {:ok, _} = filter(message) From c2e0a0c8d44f8697160d4597db45c5c4afd0d8a6 Mon Sep 17 00:00:00 2001 From: Karen Konou Date: Fri, 15 Feb 2019 14:05:20 +0100 Subject: [PATCH 53/75] Readd threshold is not 0 check, optmization? --- .../web/activity_pub/mrf/hellthread_policy.ex | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 1fd7b9c67..8ab1dd4e5 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -6,20 +6,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do alias Pleroma.User @behaviour Pleroma.Web.ActivityPub.MRF - defp delist_message(message) do - delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) + defp delist_message(message, threshold) when threshold > 0 do follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address follower_collection? = Enum.member?(message["to"] ++ message["cc"], follower_collection) message = case recipients = get_recipient_count(message) do - {:public, _} when follower_collection? and recipients > delist_threshold -> + {:public, _} + when follower_collection? and recipients > threshold -> message |> Map.put("to", [follower_collection]) |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) - {:public, _} when recipients > delist_threshold -> + {:public, _} when recipients > threshold -> message |> Map.put("to", []) |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) @@ -31,15 +31,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do {:ok, message} end - defp reject_message(message) do - reject_threshold = - Pleroma.Config.get( - [:mrf_hellthread, :reject_threshold], - Pleroma.Config.get([:mrf_hellthread, :threshold]) - ) + defp delist_message(message, _threshold), do: {:ok, message} + defp reject_message(message, threshold) when threshold > 0 do with {_, recipients} <- get_recipient_count(message) do - if recipients > reject_threshold and reject_threshold > 0 do + if recipients > threshold do {:reject, nil} else {:ok, message} @@ -47,6 +43,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do end end + defp reject_message(message, _threshold), do: {:ok, message} + defp get_recipient_count(message) do recipients = (message["to"] || []) ++ (message["cc"] || []) follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address @@ -69,8 +67,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do @impl true def filter(%{"type" => "Create"} = message) do - with {:ok, message} <- reject_message(message), - {:ok, message} <- delist_message(message) do + reject_threshold = + Pleroma.Config.get( + [:mrf_hellthread, :reject_threshold], + Pleroma.Config.get([:mrf_hellthread, :threshold]) + ) + + delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) + + with {:ok, message} <- reject_message(message, reject_threshold), + {:ok, message} <- delist_message(message, delist_threshold) do {:ok, message} else _e -> {:reject, nil} From d812a347ca936dba764eb223fde029d83ca3fba0 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 16 Feb 2019 16:42:34 +0100 Subject: [PATCH 54/75] Add optional welcome message. --- config/config.exs | 4 +++- docs/config.md | 2 ++ lib/pleroma/user.ex | 1 + lib/pleroma/user/welcome_message.ex | 33 +++++++++++++++++++++++++++++ test/user_test.exs | 19 +++++++++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/user/welcome_message.ex diff --git a/config/config.exs b/config/config.exs index e2a239a76..271224e85 100644 --- a/config/config.exs +++ b/config/config.exs @@ -162,7 +162,9 @@ config :pleroma, :instance, mrf_transparency: true, autofollowed_nicknames: [], max_pinned_statuses: 1, - no_attachment_links: false + no_attachment_links: false, + welcome_user_nickname: nil, + welcome_message: nil config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because diff --git a/docs/config.md b/docs/config.md index 74badd0da..78daa488e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -97,6 +97,8 @@ config :pleroma, Pleroma.Mailer, * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. * `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses +* `welcome_message`: A message that will be send to a newly registered users as a direct message. +* `welcome_user_nickname`: The nickname of the user that sends the welcome message. ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 29d2b3d89..3c6a9953d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -261,6 +261,7 @@ defmodule Pleroma.User do def register(%Ecto.Changeset{} = changeset) do with {:ok, user} <- Repo.insert(changeset), {:ok, user} <- autofollow_users(user), + {:ok, _} <- Pleroma.User.WelcomeMessage.post_welcome_message_to_user(user), {:ok, _} <- try_send_confirmation_email(user) do {:ok, user} end diff --git a/lib/pleroma/user/welcome_message.ex b/lib/pleroma/user/welcome_message.ex new file mode 100644 index 000000000..6a0ec084f --- /dev/null +++ b/lib/pleroma/user/welcome_message.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.User.WelcomeMessage do + alias Pleroma.User + alias Pleroma.Web.CommonAPI + import Ecto.Query + + def post_welcome_message_to_user(user) do + with %User{} = sender_user <- welcome_user(), + message when is_binary(message) <- welcome_message() do + CommonAPI.post(sender_user, %{ + "visibility" => "direct", + "status" => "@#{user.nickname}\n#{message}" + }) + else + _ -> {:ok, nil} + end + end + + defp welcome_user() do + if nickname = Pleroma.Config.get([:instance, :welcome_user_nickname]) do + from(u in User, + where: u.local == true, + where: u.nickname == ^nickname + ) + |> Pleroma.Repo.one() + else + nil + end + end + + defp welcome_message() do + Pleroma.Config.get([:instance, :welcome_message]) + end +end diff --git a/test/user_test.exs b/test/user_test.exs index a99b79a0d..d89fe379f 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -196,6 +196,25 @@ defmodule Pleroma.UserTest do assert User.following?(registered_user, user) refute User.following?(registered_user, remote_user) + + Pleroma.Config.put([:instance, :autofollowed_nicknames], []) + end + + test "it sends a welcome message if it is set" do + welcome_user = insert(:user) + + Pleroma.Config.put([:instance, :welcome_user_nickname], welcome_user.nickname) + Pleroma.Config.put([:instance, :welcome_message], "Hello, this is a cool site") + + cng = User.register_changeset(%User{}, @full_user_data) + {:ok, registered_user} = User.register(cng) + + activity = Repo.one(Pleroma.Activity) + assert registered_user.ap_id in activity.recipients + assert activity.data["object"]["content"] =~ "cool site" + + Pleroma.Config.put([:instance, :welcome_user_nickname], nil) + Pleroma.Config.put([:instance, :welcome_message], nil) end test "it requires an email, name, nickname and password, bio is optional" do From 38e15930cb7e8aec4742eb85da26955b4c08e8ce Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 16 Feb 2019 17:01:15 +0100 Subject: [PATCH 55/75] Add option to return all friends in twitter api. Mainly useful for user export. --- lib/pleroma/web/twitter_api/twitter_api_controller.ex | 3 +++ test/web/twitter_api/twitter_api_controller_test.exs | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index c2f0dc2a9..a8ef0a8ca 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -524,6 +524,9 @@ defmodule Pleroma.Web.TwitterAPI.Controller do def friends(%{assigns: %{user: for_user}} = conn, params) do {:ok, page} = Ecto.Type.cast(:integer, params["page"] || 1) + {:ok, export} = Ecto.Type.cast(:boolean, params["all"] || false) + + page = if export, do: nil, else: page with {:ok, user} <- TwitterAPI.get_user(conn.assigns[:user], params), {:ok, friends} <- User.get_friends(user, page) do diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index acb03b146..f7e40e0d3 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1218,7 +1218,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert Enum.sort(expected) == Enum.sort(result) end - test "it returns 20 friends per page", %{conn: conn} do + test "it returns 20 friends per page, except if 'export' is set to true", %{conn: conn} do user = insert(:user) followeds = insert_list(21, :user) @@ -1242,6 +1242,14 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do result = json_response(res_conn, 200) assert length(result) == 1 + + res_conn = + conn + |> assign(:user, user) + |> get("/api/statuses/friends", %{all: true}) + + result = json_response(res_conn, 200) + assert length(result) == 21 end test "it returns a given user's friends with user_id", %{conn: conn} do From f469a8610f47d6d36b2bcaa1974a1744990db7b4 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 16 Feb 2019 17:24:31 +0100 Subject: [PATCH 56/75] Check that the welcome message is sent from the correct user. --- test/user_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/user_test.exs b/test/user_test.exs index d89fe379f..92991d063 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -212,6 +212,7 @@ defmodule Pleroma.UserTest do activity = Repo.one(Pleroma.Activity) assert registered_user.ap_id in activity.recipients assert activity.data["object"]["content"] =~ "cool site" + assert activity.actor == welcome_user.ap_id Pleroma.Config.put([:instance, :welcome_user_nickname], nil) Pleroma.Config.put([:instance, :welcome_message], nil) From 269d3e1ca6c1d01feb995a108852963ce5bc32fc Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 16 Feb 2019 17:24:48 +0100 Subject: [PATCH 57/75] WelcomeMessage: Get rid of Ecto reference. --- lib/pleroma/user/welcome_message.ex | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/user/welcome_message.ex b/lib/pleroma/user/welcome_message.ex index 6a0ec084f..8018ac22f 100644 --- a/lib/pleroma/user/welcome_message.ex +++ b/lib/pleroma/user/welcome_message.ex @@ -1,7 +1,6 @@ defmodule Pleroma.User.WelcomeMessage do alias Pleroma.User alias Pleroma.Web.CommonAPI - import Ecto.Query def post_welcome_message_to_user(user) do with %User{} = sender_user <- welcome_user(), @@ -16,14 +15,12 @@ defmodule Pleroma.User.WelcomeMessage do end defp welcome_user() do - if nickname = Pleroma.Config.get([:instance, :welcome_user_nickname]) do - from(u in User, - where: u.local == true, - where: u.nickname == ^nickname - ) - |> Pleroma.Repo.one() + with nickname when is_binary(nickname) <- + Pleroma.Config.get([:instance, :welcome_user_nickname]), + %User{local: true} = user <- User.get_cached_by_nickname(nickname) do + user else - nil + _ -> nil end end From 27375e55757a7034aa2ad6c94a8d6c82b1128b34 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 16 Feb 2019 17:25:06 +0100 Subject: [PATCH 58/75] WelcomeMessage: specify that the user has to be local. --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 78daa488e..0c1051dee 100644 --- a/docs/config.md +++ b/docs/config.md @@ -98,7 +98,7 @@ config :pleroma, Pleroma.Mailer, * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. * `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses * `welcome_message`: A message that will be send to a newly registered users as a direct message. -* `welcome_user_nickname`: The nickname of the user that sends the welcome message. +* `welcome_user_nickname`: The nickname of the local user that sends the welcome message. ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog From 96c725328b556833d59de23093fb4aeba9bb5684 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 16 Feb 2019 20:38:25 +0300 Subject: [PATCH 59/75] Remove a limit on attachments in Mastodon API and document the changes in responses from vanilla mastodon --- docs/Differences-in-MastodonAPI-Responses.md | 9 +++++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 docs/Differences-in-MastodonAPI-Responses.md diff --git a/docs/Differences-in-MastodonAPI-Responses.md b/docs/Differences-in-MastodonAPI-Responses.md new file mode 100644 index 000000000..cfa34593e --- /dev/null +++ b/docs/Differences-in-MastodonAPI-Responses.md @@ -0,0 +1,9 @@ +# Differences in Mastodon API responses from vanilla Mastodon + +## Flake IDs + +Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are sortable strings + +## Attachment cap + +Some apps operate under the assumption that no more than 4 attachments ccan be returned, however Pleroma can return any amount of attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 69f5f992c..a49b381c9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -166,7 +166,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do sensitive: sensitive, spoiler_text: object["summary"] || "", visibility: get_visibility(object), - media_attachments: attachments |> Enum.take(4), + media_attachments: attachments, mentions: mentions, tags: build_tags(tags), application: %{ From c788f1543c4a2436c93409423d93284467e431e2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 16 Feb 2019 21:14:07 +0300 Subject: [PATCH 60/75] Add a section on how to identify a pleroma instance, clarify that post upload limit is not capped too --- docs/Differences-in-MastodonAPI-Responses.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/Differences-in-MastodonAPI-Responses.md b/docs/Differences-in-MastodonAPI-Responses.md index cfa34593e..488dc9389 100644 --- a/docs/Differences-in-MastodonAPI-Responses.md +++ b/docs/Differences-in-MastodonAPI-Responses.md @@ -1,9 +1,11 @@ # Differences in Mastodon API responses from vanilla Mastodon +A Pleroma instance can be identified by " (compatible; Pleroma )" present in `version` field in response from `/api/v1/instance` + ## Flake IDs Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are sortable strings ## Attachment cap -Some apps operate under the assumption that no more than 4 attachments ccan be returned, however Pleroma can return any amount of attachments +Some apps operate under the assumption that no more than 4 attachments ccan be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting. From 25ab90edeaae53b6ce084d1ba9a02df5505b5041 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 16 Feb 2019 19:39:03 +0100 Subject: [PATCH 61/75] mix.exs: Add docs/Clients.md to docs.extras [ci skip] --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index d46998891..eba3295dd 100644 --- a/mix.exs +++ b/mix.exs @@ -21,7 +21,7 @@ defmodule Pleroma.Mixfile do homepage_url: "https://pleroma.social/", docs: [ logo: "priv/static/static/logo.png", - extras: ["README.md", "docs/config.md", "docs/Pleroma-API.md", "docs/Admin-API.md"], + extras: ["README.md", "docs/Admin-API.md", "docs/Clients.md", "docs/config.md", "docs/Pleroma-API.md"], main: "readme", output: "priv/static/doc" ] From 4df455f69bef5270c7e6a57022237ff75f13687c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 3 Feb 2019 12:31:12 +0100 Subject: [PATCH 62/75] [MastoAPI] Add switching of frontend flavours --- docs/Pleroma-API.md | 14 +++++++ lib/pleroma/user/info.ex | 9 ++++ .../mastodon_api/mastodon_api_controller.ex | 41 ++++++++++++++++++- lib/pleroma/web/router.ex | 3 ++ .../mastodon_api/mastodon/index.html.eex | 8 ++-- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/docs/Pleroma-API.md b/docs/Pleroma-API.md index e1448d3f0..379d3dbed 100644 --- a/docs/Pleroma-API.md +++ b/docs/Pleroma-API.md @@ -94,3 +94,17 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi ## `/api/pleroma/admin/`… See [Admin-API](Admin-API.md) + +## `/api/v1/pleroma/flavour/:flavour` +* Method `POST` +* Authentication: required +* Response: JSON string. Returns the user flavour or the default one on success, otherwise returns `{"error": "error_msg"}` +* Example response: "glitch" +* Note: This is intended to be used only by mastofe + +## `/api/v1/pleroma/flavour` +* Method `GET` +* Authentication: required +* Response: JSON string. Returns the user flavour or the default one. +* Example response: "glitch" +* Note: This is intended to be used only by mastofe diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 9d8779fab..e33ec816b 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -34,6 +34,7 @@ defmodule Pleroma.User.Info do field(:hide_followers, :boolean, default: false) field(:hide_follows, :boolean, default: false) field(:pinned_activities, {:array, :string}, default: []) + field(:flavour, :string, default: nil) # Found in the wild # ap_id -> Where is this used? @@ -186,6 +187,14 @@ defmodule Pleroma.User.Info do |> validate_required([:settings]) end + def mastodon_flavour_update(info, flavour) do + params = %{flavour: flavour} + + info + |> cast(params, [:flavour]) + |> validate_required([:flavour]) + end + def set_source_data(info, source_data) do params = %{source_data: source_data} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index dcaeccac6..0150f18f8 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1051,6 +1051,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do accounts = Map.put(%{}, user.id, AccountView.render("account.json", %{user: user, for: user})) + flavour = get_user_flavour(user) + initial_state = %{ meta: %{ @@ -1135,7 +1137,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do conn |> put_layout(false) |> put_view(MastodonView) - |> render("index.html", %{initial_state: initial_state}) + |> render("index.html", %{initial_state: initial_state, flavour: flavour}) else conn |> redirect(to: "/web/login") @@ -1157,6 +1159,43 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + @supported_flavours ["glitch", "vanilla"] + + def set_flavour(%{assigns: %{user: user}} = conn, %{"flavour" => flavour} = _params) + when flavour in @supported_flavours do + flavour_cng = User.Info.mastodon_flavour_update(user.info, flavour) + + with changeset <- Ecto.Changeset.change(user), + changeset <- Ecto.Changeset.put_embed(changeset, :info, flavour_cng), + {:ok, user} <- User.update_and_set_cache(changeset), + flavour <- user.info.flavour do + json(conn, flavour) + else + e -> + conn + |> put_resp_content_type("application/json") + |> send_resp(500, Jason.encode!(%{"error" => inspect(e)})) + end + end + + def set_flavour(conn, _params) do + conn + |> put_status(400) + |> json(%{error: "Unsupported flavour"}) + end + + def get_flavour(%{assigns: %{user: user}} = conn, _params) do + json(conn, get_user_flavour(user)) + end + + defp get_user_flavour(%User{info: %{flavour: flavour}}) when flavour in @supported_flavours do + flavour + end + + defp get_user_flavour(_) do + "glitch" + end + def login(conn, %{"code" => code}) do with {:ok, app} <- get_or_make_app(), %Authorization{} = auth <- Repo.get_by(Authorization, token: code, app_id: app.id), diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d66a1c2a1..664f93c1c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -236,6 +236,9 @@ defmodule Pleroma.Web.Router do get("/suggestions", MastodonAPIController, :suggestions) get("/endorsements", MastodonAPIController, :empty_array) + + post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour) + get("/pleroma/flavour", MastodonAPIController, :get_flavour) end scope "/api/web", Pleroma.Web.MastodonAPI do diff --git a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex index 9a725e420..5659c7828 100644 --- a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex +++ b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex @@ -8,7 +8,7 @@ - + @@ -19,10 +19,10 @@ - - + + - +
From 72a4272d84a68ceb4d9a39ddaa4d3f45779bfebf Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 10 Feb 2019 23:11:12 +0100 Subject: [PATCH 63/75] Web.MastodonAPI.MastodonAPIControllerTest: Add testing of the flavour switching --- .../mastodon_api_controller_test.exs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 26c9c25a6..450bf10a3 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1786,4 +1786,29 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do |> json_response(200) end end + + test "flavours switching (Pleroma Extension)", %{conn: conn} do + user = insert(:user) + + get_old_flavour = + conn + |> assign(:user, user) + |> get("/api/v1/pleroma/flavour") + + assert "glitch" == json_response(get_old_flavour, 200) + + set_flavour = + conn + |> assign(:user, user) + |> post("/api/v1/pleroma/flavour/vanilla") + + assert "vanilla" == json_response(set_flavour, 200) + + get_new_flavour = + conn + |> assign(:user, user) + |> post("/api/v1/pleroma/flavour/vanilla") + + assert json_response(set_flavour, 200) == json_response(get_new_flavour, 200) + end end From 006bec8c6a8addb57427937f71ad05ca691fe7b5 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 Feb 2019 10:34:00 +0300 Subject: [PATCH 64/75] Add differences in MastoAPI responses to mix.exs extras --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index d46998891..e06501e5b 100644 --- a/mix.exs +++ b/mix.exs @@ -21,7 +21,7 @@ defmodule Pleroma.Mixfile do homepage_url: "https://pleroma.social/", docs: [ logo: "priv/static/static/logo.png", - extras: ["README.md", "docs/config.md", "docs/Pleroma-API.md", "docs/Admin-API.md"], + extras: ["README.md", "docs/config.md", "docs/Pleroma-API.md", "docs/Admin-API.md", "docs/Differences-in-MastodonAPI-Responses.md"], main: "readme", output: "priv/static/doc" ] From ccd30a187541697dabbcb6449708dc5c4555aabc Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 Feb 2019 12:07:49 +0300 Subject: [PATCH 65/75] Fix formating --- mix.exs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index e06501e5b..ed6ca6135 100644 --- a/mix.exs +++ b/mix.exs @@ -21,7 +21,13 @@ defmodule Pleroma.Mixfile do homepage_url: "https://pleroma.social/", docs: [ logo: "priv/static/static/logo.png", - extras: ["README.md", "docs/config.md", "docs/Pleroma-API.md", "docs/Admin-API.md", "docs/Differences-in-MastodonAPI-Responses.md"], + extras: [ + "README.md", + "docs/config.md", + "docs/Pleroma-API.md", + "docs/Admin-API.md", + "docs/Differences-in-MastodonAPI-Responses.md" + ], main: "readme", output: "priv/static/doc" ] From 8f98d970c105bec3205c6ce524750583ad5fb502 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 Feb 2019 13:46:40 +0300 Subject: [PATCH 66/75] Fix recipient count in hellthread policy --- lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 8ab1dd4e5..6736f3cb9 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -12,14 +12,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do follower_collection? = Enum.member?(message["to"] ++ message["cc"], follower_collection) message = - case recipients = get_recipient_count(message) do - {:public, _} + case get_recipient_count(message) do + {:public, recipients} when follower_collection? and recipients > threshold -> message |> Map.put("to", [follower_collection]) |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) - {:public, _} when recipients > threshold -> + {:public, recipients} when recipients > threshold -> message |> Map.put("to", []) |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) From fa191a658b373eb53ab0a44ab0529fb8635cfd74 Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 17 Feb 2019 11:36:14 +0100 Subject: [PATCH 67/75] Speed up docker postgres. --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b59445895..eeddce69e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,8 @@ image: elixir:1.7.2 services: - - postgres:9.6.2 + - name: postgres:9.6.2 + command: ["postgres", "-c", "fsync=off"] variables: POSTGRES_DB: pleroma_test From d0a94f98e030dd700c56e8cc4576fb45cf449f53 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 Feb 2019 14:33:44 +0300 Subject: [PATCH 68/75] more tests for HellthreadPolicy --- .../mrf/hellthread_policy_test.exs | 65 +++++++++++++------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/test/web/activity_pub/mrf/hellthread_policy_test.exs b/test/web/activity_pub/mrf/hellthread_policy_test.exs index ebf9997cd..eb6ee4d04 100644 --- a/test/web/activity_pub/mrf/hellthread_policy_test.exs +++ b/test/web/activity_pub/mrf/hellthread_policy_test.exs @@ -8,32 +8,47 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do import Pleroma.Web.ActivityPub.MRF.HellthreadPolicy - describe "hellthread filter tests" do - setup do - user = insert(:user) + setup do + user = insert(:user) - message = %{ - "actor" => user.ap_id, - "cc" => [user.follower_address], - "type" => "Create", - "to" => [ - "https://www.w3.org/ns/activitystreams#Public", - "https://instace.tld/users/user1", - "https://instace.tld/users/user2", - "https://instace.tld/users/user3" - ] - } + message = %{ + "actor" => user.ap_id, + "cc" => [user.follower_address], + "type" => "Create", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "https://instance.tld/users/user1", + "https://instance.tld/users/user2", + "https://instance.tld/users/user3" + ] + } - [user: user, message: message] - end + [user: user, message: message] + end - test "reject test", %{message: message} do + describe "reject" do + test "rejects the message if the recipient count is above reject_threshold", %{ + message: message + } do Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 2}) {:reject, nil} = filter(message) end - test "delist test", %{user: user, message: message} do + test "does not reject the message if the recipient count is below reject_threshold", %{ + message: message + } do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 3}) + + assert {:ok, ^message} = filter(message) + end + end + + describe "delist" do + test "delists the message if the recipient count is above delist_threshold", %{ + user: user, + message: message + } do Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 2, reject_threshold: 0}) {:ok, message} = filter(message) @@ -41,10 +56,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do assert "https://www.w3.org/ns/activitystreams#Public" in message["cc"] end - test "excludes follower collection and public URI from threshold count", %{message: message} do - Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 3}) + test "does not delist the message if the recipient count is below delist_threshold", %{ + message: message + } do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 4, reject_threshold: 0}) - {:ok, _} = filter(message) + assert {:ok, ^message} = filter(message) end end + + test "excludes follower collection and public URI from threshold count", %{message: message} do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 3}) + + assert {:ok, ^message} = filter(message) + end end From 71c8c60ded8dce402dbe69545afebd55c63927e5 Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 17 Feb 2019 17:47:24 +0100 Subject: [PATCH 69/75] More speedup, test fixes. --- .gitlab-ci.yml | 4 ++-- test/web/common_api/common_api_test.exs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eeddce69e..6deb0a1de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ image: elixir:1.7.2 services: - name: postgres:9.6.2 - command: ["postgres", "-c", "fsync=off"] + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] variables: POSTGRES_DB: pleroma_test @@ -36,4 +36,4 @@ lint: unit-testing: stage: test script: - - mix test --trace + - mix test --trace --preload-modules diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index d26b6e49c..870648fb5 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -2,7 +2,7 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Web.CommonAPI.Test do +defmodule Pleroma.Web.CommonAPITest do use Pleroma.DataCase alias Pleroma.Web.CommonAPI alias Pleroma.User From 6a8a8e90dafdd905fbcec3dd0159b7931b8642c2 Mon Sep 17 00:00:00 2001 From: lambda Date: Sun, 17 Feb 2019 17:01:22 +0000 Subject: [PATCH 70/75] Update Differences-in-MastodonAPI-Responses.md --- docs/Differences-in-MastodonAPI-Responses.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Differences-in-MastodonAPI-Responses.md b/docs/Differences-in-MastodonAPI-Responses.md index 488dc9389..f6a5b6461 100644 --- a/docs/Differences-in-MastodonAPI-Responses.md +++ b/docs/Differences-in-MastodonAPI-Responses.md @@ -8,4 +8,4 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas ## Attachment cap -Some apps operate under the assumption that no more than 4 attachments ccan be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting. +Some apps operate under the assumption that no more than 4 attachments can be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting. From 94708d63705f1e4e7f3aaebb6744634237b0cf21 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 17 Feb 2019 23:57:35 +0300 Subject: [PATCH 71/75] Render only "id", "valid_until" and "app_name" in TokenView --- lib/pleroma/web/oauth/token.ex | 1 + lib/pleroma/web/twitter_api/views/token_view.ex | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index f5594f834..7fe58f6a2 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -68,5 +68,6 @@ defmodule Pleroma.Web.OAuth.Token do where: t.user_id == ^user_id ) |> Repo.all() + |> Repo.preload(:app) end end diff --git a/lib/pleroma/web/twitter_api/views/token_view.ex b/lib/pleroma/web/twitter_api/views/token_view.ex index 96b8526a4..3ff314913 100644 --- a/lib/pleroma/web/twitter_api/views/token_view.ex +++ b/lib/pleroma/web/twitter_api/views/token_view.ex @@ -14,9 +14,8 @@ defmodule Pleroma.Web.TwitterAPI.TokenView do def render("show.json", %{token: token_entry}) do %{ id: token_entry.id, - token: token_entry.token, - refresh_token: token_entry.refresh_token, - valid_until: token_entry.valid_until + valid_until: token_entry.valid_until, + app_name: token_entry.app.client_name } end end From fd17a0cc9b78d1338e1fee51aa452858172639fe Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 18 Feb 2019 00:10:48 +0300 Subject: [PATCH 72/75] Fix test --- test/web/twitter_api/twitter_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 527a920fb..3922b3c5e 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1896,7 +1896,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do |> hd() |> Map.keys() - assert keys -- ["id", "refresh_token", "token", "valid_until"] == [] + assert keys -- ["id", "app_name", "valid_until"] == [] end test "revoke token", %{token: token} do From fc35481445c4fdfea9fab58a74c5f4231b56885b Mon Sep 17 00:00:00 2001 From: eugenijm Date: Tue, 19 Feb 2019 10:43:37 +0300 Subject: [PATCH 73/75] Update user cache when user tags are updated --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ff84e7b0a..322c338cd 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1200,7 +1200,7 @@ defmodule Pleroma.User do {:ok, updated_user} = user |> change(%{tags: new_tags}) - |> Repo.update() + |> update_and_set_cache() updated_user end From 96dcacade1fb5efd3c5118a1b8510e0cedbfeb85 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 19 Feb 2019 13:23:13 +0300 Subject: [PATCH 74/75] properly check for follower address in is_private? --- lib/pleroma/web/activity_pub/activity_pub.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index ab2872f56..839b6ce2a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -876,7 +876,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def is_private?(activity) do - !is_public?(activity) && Enum.any?(activity.data["to"], &String.contains?(&1, "/followers")) + unless is_public?(activity) do + follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address + Enum.any?(activity.data["to"], &(&1 == follower_address)) + else + false + end end def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true From 109b01a6318e01566a3a148c9c9ecd0313546f5f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 19 Feb 2019 13:52:15 +0300 Subject: [PATCH 75/75] mark ap_id unique_constraint --- lib/pleroma/user.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 29d2b3d89..c2ca58fb9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -237,6 +237,7 @@ defmodule Pleroma.User do changeset |> put_change(:password_hash, hashed) |> put_change(:ap_id, ap_id) + |> unique_constraint(:ap_id) |> put_change(:following, [followers]) |> put_change(:follower_address, followers) else