From 016d8d6c560cb81dfe67cc660e12d2e70d0bc6af Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 3 Aug 2020 12:37:31 -0500 Subject: [PATCH 01/12] Consolidate construction of Rich Media Parser HTTP requests --- lib/pleroma/web/rich_media/helpers.ex | 21 +++++++++++++++++++ lib/pleroma/web/rich_media/parser.ex | 20 +----------------- .../web/rich_media/parsers/oembed_parser.ex | 2 +- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 5c7daf1a5..6210f2c5a 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -9,6 +9,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @rich_media_options [ + pool: :media, + max_body: 2_000_000 + ] + @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) @@ -77,4 +82,20 @@ defmodule Pleroma.Web.RichMedia.Helpers do fetch_data_for_activity(activity) :ok end + + def rich_media_get(url) do + headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] + + options = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@rich_media_options, + recv_timeout: 2_000, + with_body: true + ) + else + @rich_media_options + end + + Pleroma.HTTP.get(url, headers, options) + end end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index c8a767935..ca592833f 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,11 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @options [ - pool: :media, - max_body: 2_000_000 - ] - defp parsers do Pleroma.Config.get([:rich_media, :parsers]) end @@ -75,21 +70,8 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp parse_url(url) do - opts = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.merge(@options, - recv_timeout: 2_000, - with_body: true - ) - else - @options - end - try do - rich_media_agent = Pleroma.Application.user_agent() <> "; Bot" - - {:ok, %Tesla.Env{body: html}} = - Pleroma.HTTP.get(url, [{"user-agent", rich_media_agent}], adapter: opts) + {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url) html |> parse_html() diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex index 6bdeac89c..1fe6729c3 100644 --- a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex +++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do end defp get_oembed_data(url) do - with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do + with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do Jason.decode(json) end end From 953f71bcfa25569d8b92d4047f4bdbee97e0077c Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 13:38:30 +0200 Subject: [PATCH 02/12] App Test: Make more resilient --- test/tasks/app_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/tasks/app_test.exs b/test/tasks/app_test.exs index b8f03566d..71a84ac8e 100644 --- a/test/tasks/app_test.exs +++ b/test/tasks/app_test.exs @@ -50,13 +50,13 @@ defmodule Mix.Tasks.Pleroma.AppTest do defp assert_app(name, redirect, scopes) do app = Repo.get_by(Pleroma.Web.OAuth.App, client_name: name) - assert_received {:mix_shell, :info, [message]} + assert_receive {:mix_shell, :info, [message]} assert message == "#{name} successfully created:" - assert_received {:mix_shell, :info, [message]} + assert_receive {:mix_shell, :info, [message]} assert message == "App client_id: #{app.client_id}" - assert_received {:mix_shell, :info, [message]} + assert_receive {:mix_shell, :info, [message]} assert message == "App client_secret: #{app.client_secret}" assert app.scopes == scopes From 988ca4ab6a0d299308d96e84aa45ef63341128bf Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 14:07:10 +0200 Subject: [PATCH 03/12] Test Config: Don't have any MRFs by default --- config/test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/test.exs b/config/test.exs index db0655e73..413c7f0b9 100644 --- a/config/test.exs +++ b/config/test.exs @@ -120,6 +120,8 @@ config :pleroma, Pleroma.Uploaders.S3, config :tzdata, :autoupdate, :disabled +config :pleroma, :mrf, policies: [] + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else From e92c040ad3d0cc568ea0dc4b79f207a392c7c90f Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 14:08:12 +0200 Subject: [PATCH 04/12] CommonAPITest: Add test that deactivated users can't post. --- test/web/common_api/common_api_test.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 313dda21b..4ba6232dc 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -458,6 +458,11 @@ defmodule Pleroma.Web.CommonAPITest do end describe "posting" do + test "deactivated users can't post" do + user = insert(:user, deactivated: true) + assert {:error, _} = CommonAPI.post(user, %{status: "ye"}) + end + test "it supports explicit addressing" do user = insert(:user) user_two = insert(:user) From 0cfadcf2caf84e2db944036576bad888a9707ff1 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 14:15:32 +0200 Subject: [PATCH 05/12] TransmogrifierTest: Add test for deactivated users --- test/web/activity_pub/transmogrifier_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 7d33feaf2..828964a36 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -163,6 +163,14 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil" end + test "it does not work for deactivated users" do + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() + + insert(:user, ap_id: data["actor"], deactivated: true) + + assert {:error, _} = Transmogrifier.handle_incoming(data) + end + test "it works for incoming notices" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() From 1a00713744803824b16efd575c9c6880b1d1a57e Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 14:17:03 +0200 Subject: [PATCH 06/12] CommonValidations: Treat deactivated users as not present. --- .../object_validators/common_validations.ex | 13 +++++++++---- .../transmogrifier/chat_message_test.exs | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index aeef31945..bd46f8034 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -34,10 +34,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do cng |> validate_change(field_name, fn field_name, actor -> - if User.get_cached_by_ap_id(actor) do - [] - else - [{field_name, "can't find user"}] + case User.get_cached_by_ap_id(actor) do + %User{deactivated: true} -> + [{field_name, "user is deactivated"}] + + %User{} -> + [] + + _ -> + [{field_name, "can't find user"}] end end) end diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs index d6736dc3e..31274c067 100644 --- a/test/web/activity_pub/transmogrifier/chat_message_test.exs +++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs @@ -124,6 +124,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageTest do {:ok, %Activity{} = _activity} = Transmogrifier.handle_incoming(data) end + test "it doesn't work for deactivated users" do + data = + File.read!("test/fixtures/create-chat-message.json") + |> Poison.decode!() + + _author = + insert(:user, + ap_id: data["actor"], + local: false, + last_refreshed_at: DateTime.utc_now(), + deactivated: true + ) + + _recipient = insert(:user, ap_id: List.first(data["to"]), local: true) + + assert {:error, _} = Transmogrifier.handle_incoming(data) + end + test "it inserts it and creates a chat" do data = File.read!("test/fixtures/create-chat-message.json") From 36aa34a1a8c489f74a9821095d823f8060afac5f Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 4 Aug 2020 15:08:51 +0200 Subject: [PATCH 07/12] MastodonAPITest: Do the needful --- test/web/mastodon_api/mastodon_api_test.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_test.exs b/test/web/mastodon_api/mastodon_api_test.exs index c08be37d4..0c5a38bf6 100644 --- a/test/web/mastodon_api/mastodon_api_test.exs +++ b/test/web/mastodon_api/mastodon_api_test.exs @@ -17,8 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPITest do test "returns error when followed user is deactivated" do follower = insert(:user) user = insert(:user, local: true, deactivated: true) - {:error, error} = MastodonAPI.follow(follower, user) - assert error == :rejected + assert {:error, _error} = MastodonAPI.follow(follower, user) end test "following for user" do From 697e3db01c0a1ee1e18fe25946a4ef56527828e7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 4 Aug 2020 08:55:40 -0500 Subject: [PATCH 08/12] Add analyze mix alias to run the same credo checks we use in CI --- mix.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 0e723c15f..63142dee7 100644 --- a/mix.exs +++ b/mix.exs @@ -214,7 +214,8 @@ defmodule Pleroma.Mixfile do "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.reset": ["ecto.drop", "ecto.setup"], test: ["ecto.create --quiet", "ecto.migrate", "test"], - docs: ["pleroma.docs", "docs"] + docs: ["pleroma.docs", "docs"], + analyze: ["credo --strict --only=warnings,todo,fixme,consistency,readability"] ] end From 91fbb5b21f9d8f098c9796eb4dd917bcd1e92404 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 4 Aug 2020 18:26:37 +0400 Subject: [PATCH 09/12] Fix ActivityExpirationPolicy --- .../web/activity_pub/mrf/activity_expiration_policy.ex | 4 ++-- .../activity_pub/mrf/activity_expiration_policy_test.exs | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex index 8e47f1e02..7b4c78e0f 100644 --- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex @@ -21,8 +21,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do @impl true def describe, do: {:ok, %{}} - defp local?(%{"id" => id}) do - String.starts_with?(id, Pleroma.Web.Endpoint.url()) + defp local?(%{"actor" => actor}) do + String.starts_with?(actor, Pleroma.Web.Endpoint.url()) end defp note?(activity) do diff --git a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs index 8babf49e7..f25cf8b12 100644 --- a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs +++ b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs @@ -7,11 +7,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do alias Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy @id Pleroma.Web.Endpoint.url() <> "/activities/cofe" + @local_actor Pleroma.Web.Endpoint.url() <> "/users/cofe" test "adds `expires_at` property" do assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} = ActivityExpirationPolicy.filter(%{ "id" => @id, + "actor" => @local_actor, "type" => "Create", "object" => %{"type" => "Note"} }) @@ -25,6 +27,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, %{"type" => "Create", "expires_at" => ^expires_at}} = ActivityExpirationPolicy.filter(%{ "id" => @id, + "actor" => @local_actor, "type" => "Create", "expires_at" => expires_at, "object" => %{"type" => "Note"} @@ -37,6 +40,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} = ActivityExpirationPolicy.filter(%{ "id" => @id, + "actor" => @local_actor, "type" => "Create", "expires_at" => too_distant_future, "object" => %{"type" => "Note"} @@ -49,6 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, activity} = ActivityExpirationPolicy.filter(%{ "id" => "https://example.com/123", + "actor" => "https://example.com/users/cofe", "type" => "Create", "object" => %{"type" => "Note"} }) @@ -60,6 +65,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, activity} = ActivityExpirationPolicy.filter(%{ "id" => "https://example.com/123", + "actor" => "https://example.com/users/cofe", "type" => "Follow" }) @@ -68,6 +74,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do assert {:ok, activity} = ActivityExpirationPolicy.filter(%{ "id" => "https://example.com/123", + "actor" => "https://example.com/users/cofe", "type" => "Create", "object" => %{"type" => "Cofe"} }) From 184742af5eed2c48ba8518f1e114cbe0655ad467 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 Aug 2020 22:32:51 -0500 Subject: [PATCH 10/12] Unique apps.client_id for new installations, fixes #2022 --- ...200804183107_add_unique_index_to_app_client_id.exs | 7 +++++++ test/web/oauth/app_test.exs | 11 +++++++++++ 2 files changed, 18 insertions(+) create mode 100644 priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs diff --git a/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs b/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs new file mode 100644 index 000000000..83de18096 --- /dev/null +++ b/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAppClientId do + use Ecto.Migration + + def change do + create(unique_index(:apps, [:client_id])) + end +end diff --git a/test/web/oauth/app_test.exs b/test/web/oauth/app_test.exs index 899af648e..993a490e0 100644 --- a/test/web/oauth/app_test.exs +++ b/test/web/oauth/app_test.exs @@ -29,5 +29,16 @@ defmodule Pleroma.Web.OAuth.AppTest do assert exist_app.id == app.id assert exist_app.scopes == ["read", "write", "follow", "push"] end + + test "has unique client_id" do + insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop") + + error = + catch_error(insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop")) + + assert %Ecto.ConstraintError{} = error + assert error.constraint == "apps_client_id_index" + assert error.type == :unique + end end end From 079e410d6efcb39e72a238c13e52bd1898b442a2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 4 Aug 2020 13:12:23 -0500 Subject: [PATCH 11/12] Add a migration to clean up activity_expirations table --- ...0804180322_remove_nonlocal_expirations.exs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs diff --git a/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs b/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs new file mode 100644 index 000000000..389935f0d --- /dev/null +++ b/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.RemoveNonlocalExpirations do + use Ecto.Migration + + def up do + statement = """ + DELETE FROM + activity_expirations A USING activities B + WHERE + A.activity_id = B.id + AND B.local = false; + """ + + execute(statement) + end + + def down do + :ok + end +end From 577b11167cb55203d30c43773f40108a87b2be6d Mon Sep 17 00:00:00 2001 From: Karol Kosek Date: Wed, 5 Aug 2020 00:01:30 +0200 Subject: [PATCH 12/12] templates/layout/app.html.eex: fix link color --- lib/pleroma/web/templates/layout/app.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 5836ec1e0..51603fe0c 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -37,7 +37,7 @@ } a { - color: color: #d8a070; + color: #d8a070; text-decoration: none; }