From 6b74a5527410c5e98a6c4906a8e6f9c4d9f5e5d1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Jul 2022 15:40:40 -0500 Subject: [PATCH 01/50] InstanceStatic should have reasonable caching While here fix the naming convention of the old module attribute restricting caching and add a new one for the default cache value All frontends should be shipped with versioned assets. There be dragons if you don't. --- lib/pleroma/web/endpoint.ex | 41 +++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index d8d40cceb..17bd956d8 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Web.Plugs.HTTPSecurityPlug) plug(Pleroma.Web.Plugs.UploadedMedia) - @static_cache_control "public, no-cache" + @static_cache_control "public, max-age=1209600" + @static_cache_disabled "public, no-cache" # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well @@ -30,22 +31,32 @@ defmodule Pleroma.Web.Endpoint do from: :pleroma, only: ["emoji", "images"], gzip: true, - cache_control_for_etags: "public, max-age=1209600", - headers: %{ - "cache-control" => "public, max-age=1209600" - } - ) - - plug(Pleroma.Web.Plugs.InstanceStatic, - at: "/", - gzip: true, cache_control_for_etags: @static_cache_control, headers: %{ "cache-control" => @static_cache_control } ) - # Careful! No `only` restriction here, as we don't know what frontends contain. + plug(Pleroma.Web.Plugs.InstanceStatic, + at: "/", + gzip: true, + cache_control_for_etags: @static_cache_disabled, + headers: %{ + "cache-control" => @static_cache_disabled + } + ) + + plug(Pleroma.Web.Plugs.FrontendStatic, + at: "/", + frontend_type: :primary, + only: ["index.html"], + gzip: true, + cache_control_for_etags: @static_cache_disabled, + headers: %{ + "cache-control" => @static_cache_disabled + } + ) + plug(Pleroma.Web.Plugs.FrontendStatic, at: "/", frontend_type: :primary, @@ -62,9 +73,9 @@ defmodule Pleroma.Web.Endpoint do at: "/pleroma/admin", frontend_type: :admin, gzip: true, - cache_control_for_etags: @static_cache_control, + cache_control_for_etags: @static_cache_disabled, headers: %{ - "cache-control" => @static_cache_control + "cache-control" => @static_cache_disabled } ) @@ -79,9 +90,9 @@ defmodule Pleroma.Web.Endpoint do only: Pleroma.Constants.static_only_files(), # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength gzip: true, - cache_control_for_etags: @static_cache_control, + cache_control_for_etags: @static_cache_disabled, headers: %{ - "cache-control" => @static_cache_control + "cache-control" => @static_cache_disabled } ) From 0de1a7629c5dbf3f6fba69a41bbeff5947558899 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 25 Jan 2024 00:26:55 +0100 Subject: [PATCH 02/50] Maps: Add filter_empty_values/1 --- lib/pleroma/maps.ex | 15 ++++++++++++++- test/pleroma/maps_test.exs | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 test/pleroma/maps_test.exs diff --git a/lib/pleroma/maps.ex b/lib/pleroma/maps.ex index 6d586e53e..5020a8ff8 100644 --- a/lib/pleroma/maps.ex +++ b/lib/pleroma/maps.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors +# Copyright © 2017-2024 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Maps do @@ -18,4 +18,17 @@ defmodule Pleroma.Maps do rescue _ -> data end + + def filter_empty_values(data) do + # TODO: Change to Map.filter in Elixir 1.13+ + data + |> Enum.filter(fn + {_k, nil} -> false + {_k, ""} -> false + {_k, []} -> false + {_k, %{} = v} -> Map.keys(v) != [] + {_k, _v} -> true + end) + |> Map.new() + end end diff --git a/test/pleroma/maps_test.exs b/test/pleroma/maps_test.exs new file mode 100644 index 000000000..05f1b18b2 --- /dev/null +++ b/test/pleroma/maps_test.exs @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.MapsTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Maps + + describe "filter_empty_values/1" do + assert %{"bar" => "b", "ray" => ["foo"], "objs" => %{"a" => "b"}} == + Maps.filter_empty_values(%{ + "foo" => nil, + "fooz" => "", + "bar" => "b", + "rei" => [], + "ray" => ["foo"], + "obj" => %{}, + "objs" => %{"a" => "b"} + }) + end +end From acef2a4a407058c6dfa4e18aaa9920ab1a82eb94 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 25 Jan 2024 00:27:15 +0100 Subject: [PATCH 03/50] CommonFixes: Use Maps.filter_empty_values on fix_object_defaults --- lib/pleroma/web/activity_pub/object_validators/common_fixes.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 4d9be0bdd..99c62cb4e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.User @@ -24,6 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do end def fix_object_defaults(data) do + data = Maps.filter_empty_values(data) + context = Utils.maybe_create_context( data["context"] || data["conversation"] || data["inReplyTo"] || data["id"] From 558b4210798657606bcb65debfe1845a9042927c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 25 Jan 2024 10:14:45 +0100 Subject: [PATCH 04/50] Test incoming federation from Convergence AP Bridge --- changelog.d/bugfix-ccworks.fix | 1 + test/fixtures/ccworld-ap-bridge_note.json | 1 + .../article_note_page_validator_test.exs | 11 +++++++++++ 3 files changed, 13 insertions(+) create mode 100644 changelog.d/bugfix-ccworks.fix create mode 100644 test/fixtures/ccworld-ap-bridge_note.json diff --git a/changelog.d/bugfix-ccworks.fix b/changelog.d/bugfix-ccworks.fix new file mode 100644 index 000000000..658e27b86 --- /dev/null +++ b/changelog.d/bugfix-ccworks.fix @@ -0,0 +1 @@ +Fix federation with Convergence AP Bridge \ No newline at end of file diff --git a/test/fixtures/ccworld-ap-bridge_note.json b/test/fixtures/ccworld-ap-bridge_note.json new file mode 100644 index 000000000..4b13b4bc1 --- /dev/null +++ b/test/fixtures/ccworld-ap-bridge_note.json @@ -0,0 +1 @@ +{"@context":"https://www.w3.org/ns/activitystreams","type":"Note","id":"https://cc.mkdir.uk/ap/note/e5d1d0a1-1ab3-4498-9949-588e3fdea286","attributedTo":"https://cc.mkdir.uk/ap/acct/hiira","inReplyTo":"","quoteUrl":"","content":"おはコンー","published":"2024-01-19T22:08:05Z","to":["https://www.w3.org/ns/activitystreams#Public"],"tag":null,"attachment":[],"object":null} diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 4703c3801..2b33950d6 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -93,6 +93,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) end + test "a Note from Convergence AP Bridge validates" do + insert(:user, ap_id: "https://cc.mkdir.uk/ap/acct/hiira") + + note = + "test/fixtures/ccworld-ap-bridge_note.json" + |> File.read!() + |> Jason.decode!() + + %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) + end + test "a note with an attachment should work", _ do insert(:user, %{ap_id: "https://owncast.localhost.localdomain/federation/user/streamer"}) From 799891d359363ab66fc0e351ad8140b689027fc4 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jan 2024 17:10:10 +0100 Subject: [PATCH 05/50] Transmogrifier: Cleanup obsolete handling of `"contentMap": null` --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a4cf395f2..80f64cf1e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -336,10 +336,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_tag(object), do: object - def fix_content_map(%{"contentMap" => nil} = object) do - Map.drop(object, ["contentMap"]) - end - # content map usually only has one language so this will do for now. def fix_content_map(%{"contentMap" => content_map} = object) do content_groups = Map.to_list(content_map) From c6f783c5519cc2cd519be4406366ecbad57a8c40 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 11:01:37 -0500 Subject: [PATCH 06/50] Pleroma.Web.ControllerHelper: fix @spec to resolve dialyzer errors lib/pleroma/web/admin_api/controllers/user_controller.ex:333:no_return Function index/2 has no local return. ________________________________________________________________________________ lib/pleroma/web/admin_api/controllers/user_controller.ex:357:unused_fun Function maybe_parse_filters/1 will never be called. ________________________________________________________________________________ lib/pleroma/web/admin_api/controllers/user_controller.ex:366:no_return Function page_params/1 has no local return. ________________________________________________________________________________ lib/pleroma/web/admin_api/controllers/user_controller.ex:368:call The function call will not succeed. Pleroma.Web.ControllerHelper.fetch_integer_param(_params :: any(), :page, 1) breaks the contract (map(), String.t(), integer() | nil) :: integer() | nil --- lib/pleroma/web/controller_helper.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index d3cf372dc..1caf0f7e6 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.ControllerHelper do |> json(json) end - @spec fetch_integer_param(map(), String.t(), integer() | nil) :: integer() | nil + @spec fetch_integer_param(map(), String.t() | atom(), integer() | nil) :: integer() | nil def fetch_integer_param(params, name, default \\ nil) do params |> Map.get(name, default) From ed2f8e45e9118177727b22f103f3c20476a891cf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 11:12:41 -0500 Subject: [PATCH 07/50] Pleroma.Web.MastodonAPI.SearchController: fix dialyzer errors Add a separate Pagination.paginate_list/2 function instead of overloading paginate/4 and complicating its matching and @spec --- lib/pleroma/pagination.ex | 9 +++++---- .../web/mastodon_api/controllers/search_controller.ex | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index f12ca2819..8db732cc9 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -61,15 +61,16 @@ defmodule Pleroma.Pagination do |> Repo.all() end - @spec paginate(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()] - def paginate(query, options, method \\ :keyset, table_binding \\ nil) - - def paginate(list, options, _method, _table_binding) when is_list(list) do + @spec paginate_list(list(), keyword()) :: list() + def paginate_list(list, options) do offset = options[:offset] || 0 limit = options[:limit] || 0 Enum.slice(list, offset, limit) end + @spec paginate(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()] + def paginate(query, options, method \\ :keyset, table_binding \\ nil) + def paginate(query, options, :keyset, table_binding) do query |> restrict(:min_id, options, table_binding) diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 7fdf684d2..628aa311b 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -156,7 +156,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do tags end - Pleroma.Pagination.paginate(tags, options) + Pleroma.Pagination.paginate_list(tags, options) end defp add_joined_tag(tags) do From 5e8bedcca0ebac6fc92733c4e3a95842c850165b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 11:15:48 -0500 Subject: [PATCH 08/50] Pleroma.Web.PleromaAPI.MascotController: fix dialyzer error due to bad error match lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:37:pattern_match The pattern can never match the type. Pattern: {:content_type, _} Type: {:error, _} ________________________________________________________________________________ lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:40:pattern_match The pattern can never match the type. Pattern: {:upload, {:error, _}} Type: {:error, _} --- lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex b/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex index 5e412196b..3208cde98 100644 --- a/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.PleromaAPI.MascotController do _ ) do with {:content_type, "image" <> _} <- {:content_type, file.content_type}, - {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)) do + {_, {:ok, object}} <- {:upload, ActivityPub.upload(file, actor: User.ap_id(user))} do attachment = render_attachment(object) {:ok, _user} = User.mascot_update(user, attachment) From 92992c022d454f9489ce54aa8f6189456435a48a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 11:29:06 -0500 Subject: [PATCH 09/50] Pleroma.Web.OAuth.OAuthController: dialyzer error validate_scopes/2 can never receive a map as it is only called in one place with a guard requiring a list lib/pleroma/web/o_auth/o_auth_controller.ex:615:guard_fail The guard test: is_map(_params :: maybe_improper_list()) can never succeed. --- lib/pleroma/web/o_auth/o_auth_controller.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 2bbe5d5fa..47b03215f 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -610,13 +610,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do end end - @spec validate_scopes(App.t(), map() | list()) :: + @spec validate_scopes(App.t(), list()) :: {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes} - defp validate_scopes(%App{} = app, params) when is_map(params) do - requested_scopes = Scopes.fetch_scopes(params, app.scopes) - validate_scopes(app, requested_scopes) - end - defp validate_scopes(%App{} = app, requested_scopes) when is_list(requested_scopes) do Scopes.validate(requested_scopes, app.scopes) end From 97c4d3bcc987b6e60c09ab6e2911661cf43051c3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 13:12:56 -0500 Subject: [PATCH 10/50] Pleroma.Web.Plugs.RateLimiter.Supervisor: dialyzer error lib/pleroma/web/plugs/rate_limiter/supervisor.ex:12:no_return Function init/1 has no local return. --- lib/pleroma/web/plugs/rate_limiter/supervisor.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/plugs/rate_limiter/supervisor.ex b/lib/pleroma/web/plugs/rate_limiter/supervisor.ex index f00f3d95e..5f79a3e3e 100644 --- a/lib/pleroma/web/plugs/rate_limiter/supervisor.ex +++ b/lib/pleroma/web/plugs/rate_limiter/supervisor.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.Plugs.RateLimiter.Supervisor do Pleroma.Web.Plugs.RateLimiter.LimiterSupervisor ] - opts = [strategy: :one_for_one, name: Pleroma.Web.Streamer.Supervisor] + opts = [strategy: :one_for_one] Supervisor.init(children, opts) end end From 518a38577b6bf3fdf04036e697d81f98009bbec9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 13:19:34 -0500 Subject: [PATCH 11/50] Fix dialyzer errors due to deprecated usage of put_layout/2 --- lib/pleroma/web/embed_controller.ex | 2 -- lib/pleroma/web/o_status/o_status_controller.ex | 1 - lib/pleroma/web/static_fe/static_fe_controller.ex | 1 - 3 files changed, 4 deletions(-) diff --git a/lib/pleroma/web/embed_controller.ex b/lib/pleroma/web/embed_controller.ex index ab0df9c5a..2ca4501a6 100644 --- a/lib/pleroma/web/embed_controller.ex +++ b/lib/pleroma/web/embed_controller.ex @@ -11,8 +11,6 @@ defmodule Pleroma.Web.EmbedController do alias Pleroma.Web.ActivityPub.Visibility - plug(:put_layout, :embed) - def show(conn, %{"id" => id}) do with %Activity{local: true} = activity <- Activity.get_by_id_with_object(id), diff --git a/lib/pleroma/web/o_status/o_status_controller.ex b/lib/pleroma/web/o_status/o_status_controller.ex index 4f2cf02c3..ee7ef4a5d 100644 --- a/lib/pleroma/web/o_status/o_status_controller.ex +++ b/lib/pleroma/web/o_status/o_status_controller.ex @@ -112,7 +112,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object, true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do conn - |> put_layout(:metadata_player) |> put_resp_header("x-frame-options", "ALLOW") |> put_resp_header( "content-security-policy", diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 3f2dbcd02..012f8e464 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -13,7 +13,6 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do alias Pleroma.Web.Metadata alias Pleroma.Web.Router.Helpers - plug(:put_layout, :static_fe) plug(:assign_id) @page_keys ["max_id", "min_id", "limit", "since_id", "order"] From f933d24b02472352a2928c2987451ceef7208b34 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 13:34:17 -0500 Subject: [PATCH 12/50] Pleroma.Config.DeprecationWarnings: fix type errors detected by gradient lib/pleroma/config/deprecation_warnings.ex: The atom :error on line 278 is expected to have type :ok | nil but it has type :error lib/pleroma/config/deprecation_warnings.ex: The atom :error on line 292 is expected to have type :ok | nil but it has type :error lib/pleroma/config/deprecation_warnings.ex: The atom :error on line 390 is expected to have type :ok | nil but it has type :error lib/pleroma/config/deprecation_warnings.ex: The atom :error on line 413 is expected to have type :ok | nil but it has type :error --- lib/pleroma/config/deprecation_warnings.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index a77923264..d26c5db72 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -256,7 +256,7 @@ defmodule Pleroma.Config.DeprecationWarnings do move_namespace_and_warn(@mrf_config_map, warning_preface) end - @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | nil + @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | nil | :error def move_namespace_and_warn(config_map, warning_preface) do warning = Enum.reduce(config_map, "", fn From 7745ee27bcedbe54be0479cd0561b74218acb8fe Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 13:43:07 -0500 Subject: [PATCH 13/50] Pleroma.MFA.Totp.provisioning_uri/3: add @spec --- lib/pleroma/mfa/totp.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/mfa/totp.ex b/lib/pleroma/mfa/totp.ex index 429c4b700..96fa8d71d 100644 --- a/lib/pleroma/mfa/totp.ex +++ b/lib/pleroma/mfa/totp.ex @@ -14,6 +14,7 @@ defmodule Pleroma.MFA.TOTP do @doc """ https://github.com/google/google-authenticator/wiki/Key-Uri-Format """ + @spec provisioning_uri(String.t(), String.t(), list()) :: String.t() def provisioning_uri(secret, label, opts \\ []) do query = %{ @@ -27,7 +28,7 @@ defmodule Pleroma.MFA.TOTP do |> URI.encode_query() %URI{scheme: "otpauth", host: "totp", path: "/" <> label, query: query} - |> URI.to_string() + |> to_string() end defp default_period, do: Config.get(@config_ns ++ [:period]) From cccfdde14c6742b7bbad0d4e0e25d4b641c721be Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 13:50:43 -0500 Subject: [PATCH 14/50] Pleroma.MFA: fix gradient error lib/pleroma/mfa.ex: The map %{error: msg} on line 80 is expected to have type {:ok, list(binary())} | {:error, String.t()} but it has type %{required(:error) => any()} --- lib/pleroma/mfa.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/mfa.ex b/lib/pleroma/mfa.ex index 01b730c76..ce30cd4ad 100644 --- a/lib/pleroma/mfa.ex +++ b/lib/pleroma/mfa.ex @@ -77,7 +77,7 @@ defmodule Pleroma.MFA do {:ok, codes} else {:error, msg} -> - %{error: msg} + {:error, msg} end end From 15621b72842ad7dc1b1be1a480045498cf815f9b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 13:58:26 -0500 Subject: [PATCH 15/50] Pleroma.HTTP.RequestBuilder: fix gradient error lib/pleroma/http/request_builder.ex: The variable key on line 69 is expected to have type String.t() but it has type atom() --- lib/pleroma/http/request_builder.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index f16fb3b35..0a028a64c 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -54,12 +54,12 @@ defmodule Pleroma.HTTP.RequestBuilder do @doc """ Add optional parameters to the request """ - @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t() + @spec add_param(Request.t(), atom(), atom() | String.t(), any()) :: Request.t() def add_param(request, :query, :query, values), do: %{request | query: values} def add_param(request, :body, :body, value), do: %{request | body: value} - def add_param(request, :body, key, value) do + def add_param(request, :body, key, value) when is_binary(key) do request |> Map.put(:body, Multipart.new()) |> Map.update!( From ac7f2cf105204123c420799bb867005eef14053b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 14:08:29 -0500 Subject: [PATCH 16/50] Pleroma Emoji mix task: fix gradient error lib/mix/tasks/pleroma/emoji.ex: The tuple {:cwd, pack_path} on line 114 is expected to have type :cooked | :keep_old_files | :memory | :verbose | {:cwd, list(char())} | {:file_filter, (record(:zip_file) -> boolean())} | {:file_list, list(:file.name())} but it has type {:cwd, binary()} --- lib/mix/tasks/pleroma/emoji.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 537f0715e..8b9c921c8 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -111,7 +111,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do {:ok, _} = :zip.unzip(binary_archive, - cwd: pack_path, + cwd: String.to_charlist(pack_path), file_list: files_to_unzip ) From bff04da0f32becc34921dceec57b3a65231adc07 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 14:08:54 -0500 Subject: [PATCH 17/50] Pleroma.Emoji.Pack: fix gradient error lib/pleroma/emoji/pack.ex: The tuple {:cwd, tmp_dir} on line 103 is expected to have type :cooked | :keep_old_files | :memory | :verbose | {:cwd, list(char())} | {:file_filter, (record(:zip_file) -> boolean())} | {:file_list, list(:file.name())} but it has type {:cwd, binary()} --- lib/pleroma/emoji/pack.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 85f7e1877..afc341853 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -100,7 +100,7 @@ defmodule Pleroma.Emoji.Pack do {:ok, _emoji_files} = :zip.unzip( to_charlist(file.path), - [{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, tmp_dir}] + [{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, String.to_charlist(tmp_dir)}] ) {_, updated_pack} = From d42b0eb29bdd1e48afa06d6bcf9eee37bc3a4b08 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 14:15:01 -0500 Subject: [PATCH 18/50] Pleroma.Config.DeprecationWarnings: fix gradient errors lib/pleroma/config/deprecation_warnings.ex: The atom :error on line 292 is expected to have type :ok | nil but it has type :error lib/pleroma/config/deprecation_warnings.ex: The function call move_namespace_and_warn( [ {Pleroma.ActivityExpiration, Pleroma.Workers.PurgeExpiredActivity, " * `config :pleroma, Pleroma.ActivityExpiration` is now `config :pleroma, Pleroma.Workers.PurgeExpiredActivity`"} ], warning_preface ) on line 350 is expected to have type :ok | nil but it has type :ok | nil | :error lib/pleroma/config/deprecation_warnings.ex: The function call move_namespace_and_warn( [ {Pleroma.Plugs.RemoteIp, Pleroma.Web.Plugs.RemoteIp, " * `config :pleroma, Pleroma.Plugs.RemoteIp` is now `config :pleroma, Pleroma.Web.Plugs.RemoteIp`"} ], warning_preface ) on line 366 is expected to have type :ok | nil but it has type :ok | nil | :error lib/pleroma/config/deprecation_warnings.ex: The atom :error on line 390 is expected to have type :ok | nil but it has type :error lib/pleroma/config/deprecation_warnings.ex: The atom :error on line 413 is expected to have type :ok | nil but it has type :error --- lib/pleroma/config/deprecation_warnings.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index d26c5db72..58d164dc7 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -256,7 +256,7 @@ defmodule Pleroma.Config.DeprecationWarnings do move_namespace_and_warn(@mrf_config_map, warning_preface) end - @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | nil | :error + @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | :error def move_namespace_and_warn(config_map, warning_preface) do warning = Enum.reduce(config_map, "", fn @@ -279,7 +279,7 @@ defmodule Pleroma.Config.DeprecationWarnings do end end - @spec check_media_proxy_whitelist_config() :: :ok | nil + @spec check_media_proxy_whitelist_config() :: :ok | :error def check_media_proxy_whitelist_config do whitelist = Config.get([:media_proxy, :whitelist]) @@ -340,7 +340,7 @@ defmodule Pleroma.Config.DeprecationWarnings do end end - @spec check_activity_expiration_config() :: :ok | nil + @spec check_activity_expiration_config() :: :ok | :error def check_activity_expiration_config do warning_preface = """ !!!DEPRECATION WARNING!!! @@ -356,7 +356,7 @@ defmodule Pleroma.Config.DeprecationWarnings do ) end - @spec check_remote_ip_plug_name() :: :ok | nil + @spec check_remote_ip_plug_name() :: :ok | :error def check_remote_ip_plug_name do warning_preface = """ !!!DEPRECATION WARNING!!! @@ -372,7 +372,7 @@ defmodule Pleroma.Config.DeprecationWarnings do ) end - @spec check_uploaders_s3_public_endpoint() :: :ok | nil + @spec check_uploaders_s3_public_endpoint() :: :ok | :error def check_uploaders_s3_public_endpoint do s3_config = Pleroma.Config.get([Pleroma.Uploaders.S3]) @@ -393,7 +393,7 @@ defmodule Pleroma.Config.DeprecationWarnings do end end - @spec check_old_chat_shoutbox() :: :ok | nil + @spec check_old_chat_shoutbox() :: :ok | :error def check_old_chat_shoutbox do instance_config = Pleroma.Config.get([:instance]) chat_config = Pleroma.Config.get([:chat]) || [] From a2c686e16c570791e754af7ed8505e77d29549de Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 14:22:17 -0500 Subject: [PATCH 19/50] Pleroma.Filter: fix gradient error lib/pleroma/filter.ex: The clause on line 220 cannot be reached --- lib/pleroma/filter.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index db88bc021..e827d3cbc 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -216,9 +216,6 @@ defmodule Pleroma.Filter do :re -> ~r/\b#{phrases}\b/i - - _ -> - nil end end From 0ffeb84f0cb834228ed878ff63c444b5d1ccf760 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jan 2024 14:58:59 -0500 Subject: [PATCH 20/50] Changelog --- changelog.d/dialyzer4.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/dialyzer4.skip diff --git a/changelog.d/dialyzer4.skip b/changelog.d/dialyzer4.skip new file mode 100644 index 000000000..e69de29bb From 04fc4eddaa534185d9784351e70f59f30bc1476f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 4 Feb 2024 19:24:52 -0500 Subject: [PATCH 21/50] Fix Rich Media Previews for updated activities The Rich Media Previews were not regenerated when a post was updated due to a cache invalidation issue. They are now cached by the activity id so they can be evicted with the other activity cache objects in the :scrubber_cache. --- changelog.d/rich_media.fix | 1 + lib/pleroma/activity/html.ex | 2 +- lib/pleroma/html.ex | 31 +++++++------------- lib/pleroma/web/rich_media/helpers.ex | 21 ++++++++++++- test/fixtures/rich_media/google.html | 12 ++++++++ test/fixtures/rich_media/yahoo.html | 12 ++++++++ test/pleroma/web/rich_media/helpers_test.exs | 28 ++++++++++++++++++ test/support/http_request_mock.ex | 12 +++++++- 8 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 changelog.d/rich_media.fix create mode 100644 test/fixtures/rich_media/google.html create mode 100644 test/fixtures/rich_media/yahoo.html diff --git a/changelog.d/rich_media.fix b/changelog.d/rich_media.fix new file mode 100644 index 000000000..08f119550 --- /dev/null +++ b/changelog.d/rich_media.fix @@ -0,0 +1 @@ +Rich Media Preview cache eviction when the activity is updated. diff --git a/lib/pleroma/activity/html.ex b/lib/pleroma/activity/html.ex index 706b2d36c..ba284b4d5 100644 --- a/lib/pleroma/activity/html.ex +++ b/lib/pleroma/activity/html.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Activity.HTML do end end - defp add_cache_key_for(activity_id, additional_key) do + def add_cache_key_for(activity_id, additional_key) do current = get_cache_keys_for(activity_id) unless additional_key in current do diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 5bf735c4f..84ff2f129 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -6,8 +6,6 @@ defmodule Pleroma.HTML do # Scrubbers are compiled on boot so they can be configured in OTP releases # @on_load :compile_scrubbers - @cachex Pleroma.Config.get([:cachex, :provider], Cachex) - def compile_scrubbers do dir = Path.join(:code.priv_dir(:pleroma), "scrubbers") @@ -67,27 +65,20 @@ defmodule Pleroma.HTML do end end - def extract_first_external_url_from_object(%{data: %{"content" => content}} = object) + @spec extract_first_external_url_from_object(Pleroma.Object.t()) :: + {:ok, String.t()} | {:error, :no_content} + def extract_first_external_url_from_object(%{data: %{"content" => content}}) when is_binary(content) do - unless object.data["fake"] do - key = "URL|#{object.id}" + url = + content + |> Floki.parse_fragment!() + |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") + |> Enum.take(1) + |> Floki.attribute("href") + |> Enum.at(0) - @cachex.fetch!(:scrubber_cache, key, fn _key -> - {:commit, {:ok, extract_first_external_url(content)}} - end) - else - {:ok, extract_first_external_url(content)} - end + {:ok, url} end def extract_first_external_url_from_object(_), do: {:error, :no_content} - - def extract_first_external_url(content) do - content - |> Floki.parse_fragment!() - |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") - |> Enum.take(1) - |> Floki.attribute("href") - |> Enum.at(0) - end end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 61000bb9b..ee12a9dfb 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) @options [ @@ -71,7 +73,24 @@ defmodule Pleroma.Web.RichMedia.Helpers do def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do with true <- @config_impl.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity, fetch: false) do - fetch_data_for_object(object) + if object.data["fake"] do + fetch_data_for_object(object) + else + key = "URL|#{activity.id}" + + @cachex.fetch!(:scrubber_cache, key, fn _ -> + result = fetch_data_for_object(object) + + cond do + match?(%{page_url: _, rich_media: _}, result) -> + Activity.HTML.add_cache_key_for(activity.id, key) + {:commit, result} + + true -> + {:ignore, %{}} + end + end) + end else _ -> %{} end diff --git a/test/fixtures/rich_media/google.html b/test/fixtures/rich_media/google.html new file mode 100644 index 000000000..c068397a5 --- /dev/null +++ b/test/fixtures/rich_media/google.html @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test/fixtures/rich_media/yahoo.html b/test/fixtures/rich_media/yahoo.html new file mode 100644 index 000000000..41d8c5cd9 --- /dev/null +++ b/test/fixtures/rich_media/yahoo.html @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test/pleroma/web/rich_media/helpers_test.exs b/test/pleroma/web/rich_media/helpers_test.exs index 3ef5705ce..8f6713ef8 100644 --- a/test/pleroma/web/rich_media/helpers_test.exs +++ b/test/pleroma/web/rich_media/helpers_test.exs @@ -83,6 +83,34 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end + test "recrawls URLs on updates" do + original_url = "https://google.com/" + updated_url = "https://yahoo.com/" + + Pleroma.StaticStubbedConfigMock + |> stub(:get, fn + [:rich_media, :enabled] -> true + path -> Pleroma.Test.StaticConfig.get(path) + end) + + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{status: "I like this site #{original_url}"}) + + assert match?( + %{page_url: ^original_url, rich_media: _}, + Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + ) + + {:ok, _} = CommonAPI.update(user, activity, %{status: "I like this site #{updated_url}"}) + + activity = Pleroma.Activity.get_by_id(activity.id) + + assert match?( + %{page_url: ^updated_url, rich_media: _}, + Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + ) + end + # This does not seem to work. The urls are being fetched. @tag skip: true test "refuses to crawl URLs of private network from posts" do diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index f76128312..b220fd051 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1464,6 +1464,14 @@ defmodule HttpRequestMock do }} end + def get("https://google.com/", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/google.html")}} + end + + def get("https://yahoo.com/", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/yahoo.html")}} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} @@ -1539,7 +1547,9 @@ defmodule HttpRequestMock do @rich_media_mocks [ "https://example.com/ogp", "https://example.com/ogp-missing-data", - "https://example.com/twitter-card" + "https://example.com/twitter-card", + "https://google.com/", + "https://yahoo.com/" ] def head(url, _query, _body, _headers) when url in @rich_media_mocks do {:ok, %Tesla.Env{status: 404, body: ""}} From 579561e97ba83183022d4bd2658522be6b6ae202 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 4 Feb 2024 23:40:11 -0500 Subject: [PATCH 22/50] URI.authority is deprecated --- lib/pleroma/web/rich_media/helpers.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index ee12a9dfb..9d6b8a38b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -27,8 +27,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do |> parse_uri(page_url) end - defp validate_page_url(%URI{host: host, scheme: "https", authority: authority}) - when is_binary(authority) do + defp validate_page_url(%URI{host: host, scheme: "https"}) do cond do host in @config_impl.get([:rich_media, :ignore_hosts], []) -> :error From 0cc038b67c231090827c1b4e71a32f65ee7c3d88 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 5 Feb 2024 00:09:37 -0500 Subject: [PATCH 23/50] Ensure URLs with IP addresses for the host do not generate previews --- lib/pleroma/web/rich_media/helpers.ex | 3 +++ test/pleroma/web/rich_media/helpers_test.exs | 12 +++++------- test/support/http_request_mock.ex | 3 ++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 9d6b8a38b..1501776d9 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -29,6 +29,9 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp validate_page_url(%URI{host: host, scheme: "https"}) do cond do + Linkify.Parser.ip?(host) -> + :error + host in @config_impl.get([:rich_media, :ignore_hosts], []) -> :error diff --git a/test/pleroma/web/rich_media/helpers_test.exs b/test/pleroma/web/rich_media/helpers_test.exs index 8f6713ef8..bf7372476 100644 --- a/test/pleroma/web/rich_media/helpers_test.exs +++ b/test/pleroma/web/rich_media/helpers_test.exs @@ -111,8 +111,6 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do ) end - # This does not seem to work. The urls are being fetched. - @tag skip: true test "refuses to crawl URLs of private network from posts" do user = insert(:user) @@ -130,10 +128,10 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do path -> Pleroma.Test.StaticConfig.get(path) end) - assert %{} = Helpers.fetch_data_for_activity(activity) - assert %{} = Helpers.fetch_data_for_activity(activity2) - assert %{} = Helpers.fetch_data_for_activity(activity3) - assert %{} = Helpers.fetch_data_for_activity(activity4) - assert %{} = Helpers.fetch_data_for_activity(activity5) + assert %{} == Helpers.fetch_data_for_activity(activity) + assert %{} == Helpers.fetch_data_for_activity(activity2) + assert %{} == Helpers.fetch_data_for_activity(activity3) + assert %{} == Helpers.fetch_data_for_activity(activity4) + assert %{} == Helpers.fetch_data_for_activity(activity5) end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index b220fd051..df3371a75 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1549,7 +1549,8 @@ defmodule HttpRequestMock do "https://example.com/ogp-missing-data", "https://example.com/twitter-card", "https://google.com/", - "https://yahoo.com/" + "https://yahoo.com/", + "https://pleroma.local/notice/9kCP7V" ] def head(url, _query, _body, _headers) when url in @rich_media_mocks do {:ok, %Tesla.Env{status: 404, body: ""}} From 6b7b443ff95587b33f4b666e68ed82dc6fb485a5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Feb 2024 14:34:59 -0500 Subject: [PATCH 24/50] Pleroma.Web.RichMedia.Parser: Remove test-specific codepaths Also consolidate Tesla mocks into the HttpRequestMock module. Tests were not exercising the real codepaths. The Rich Media Preview only works with https, but most of these tests were only mocking http. --- lib/pleroma/caching.ex | 3 + lib/pleroma/web/rich_media/parser.ex | 101 ++++++++--------- test/fixtures/rich_media/oembed.html | 2 +- .../controllers/status_controller_test.exs | 8 +- .../chat_message_reference_view_test.exs | 1 + test/pleroma/web/rich_media/helpers_test.exs | 4 +- test/pleroma/web/rich_media/parser_test.exs | 105 +++--------------- test/support/cachex_proxy.ex | 6 + test/support/http_request_mock.ex | 61 +++++++++- test/support/null_cache.ex | 6 + 10 files changed, 144 insertions(+), 153 deletions(-) diff --git a/lib/pleroma/caching.ex b/lib/pleroma/caching.ex index eb0588708..796a465af 100644 --- a/lib/pleroma/caching.ex +++ b/lib/pleroma/caching.ex @@ -8,10 +8,13 @@ defmodule Pleroma.Caching do @callback put(Cachex.cache(), any(), any(), Keyword.t()) :: {Cachex.status(), boolean()} @callback put(Cachex.cache(), any(), any()) :: {Cachex.status(), boolean()} @callback fetch!(Cachex.cache(), any(), function() | nil) :: any() + @callback fetch(Cachex.cache(), any(), function() | nil) :: + {atom(), any()} | {atom(), any(), any()} # @callback del(Cachex.cache(), any(), Keyword.t()) :: {Cachex.status(), boolean()} @callback del(Cachex.cache(), any()) :: {Cachex.status(), boolean()} @callback stream!(Cachex.cache(), any()) :: Enumerable.t() @callback expire_at(Cachex.cache(), binary(), number()) :: {Cachex.status(), boolean()} + @callback expire(Cachex.cache(), binary(), number()) :: {Cachex.status(), boolean()} @callback exists?(Cachex.cache(), any()) :: {Cachex.status(), boolean()} @callback execute!(Cachex.cache(), function()) :: any() @callback get_and_update(Cachex.cache(), any(), function()) :: diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 837182d8a..a791b2bab 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -13,70 +13,65 @@ defmodule Pleroma.Web.RichMedia.Parser do def parse(nil), do: {:error, "No URL provided"} - if Pleroma.Config.get(:env) == :test do - @spec parse(String.t()) :: {:ok, map()} | {:error, any()} - def parse(url), do: parse_url(url) - else - @spec parse(String.t()) :: {:ok, map()} | {:error, any()} - def parse(url) do - with {:ok, data} <- get_cached_or_parse(url), - {:ok, _} <- set_ttl_based_on_image(data, url) do - {:ok, data} - end + @spec parse(String.t()) :: {:ok, map()} | {:error, any()} + def parse(url) do + with {:ok, data} <- get_cached_or_parse(url), + {:ok, _} <- set_ttl_based_on_image(data, url) do + {:ok, data} end + end - defp get_cached_or_parse(url) do - case @cachex.fetch(:rich_media_cache, url, fn -> - case parse_url(url) do - {:ok, _} = res -> - {:commit, res} + defp get_cached_or_parse(url) do + case @cachex.fetch(:rich_media_cache, url, fn -> + case parse_url(url) do + {:ok, _} = res -> + {:commit, res} - {:error, reason} = e -> - # Unfortunately we have to log errors here, instead of doing that - # along with ttl setting at the bottom. Otherwise we can get log spam - # if more than one process was waiting for the rich media card - # while it was generated. Ideally we would set ttl here as well, - # so we don't override it number_of_waiters_on_generation - # times, but one, obviously, can't set ttl for not-yet-created entry - # and Cachex doesn't support returning ttl from the fetch callback. - log_error(url, reason) - {:commit, e} - end - end) do - {action, res} when action in [:commit, :ok] -> - case res do - {:ok, _data} = res -> - res + {:error, reason} = e -> + # Unfortunately we have to log errors here, instead of doing that + # along with ttl setting at the bottom. Otherwise we can get log spam + # if more than one process was waiting for the rich media card + # while it was generated. Ideally we would set ttl here as well, + # so we don't override it number_of_waiters_on_generation + # times, but one, obviously, can't set ttl for not-yet-created entry + # and Cachex doesn't support returning ttl from the fetch callback. + log_error(url, reason) + {:commit, e} + end + end) do + {action, res} when action in [:commit, :ok] -> + case res do + {:ok, _data} = res -> + res - {:error, reason} = e -> - if action == :commit, do: set_error_ttl(url, reason) - e - end + {:error, reason} = e -> + if action == :commit, do: set_error_ttl(url, reason) + e + end - {:error, e} -> - {:error, {:cachex_error, e}} - end + {:error, e} -> + {:error, {:cachex_error, e}} end + end - defp set_error_ttl(_url, :body_too_large), do: :ok - defp set_error_ttl(_url, {:content_type, _}), do: :ok + defp set_error_ttl(_url, :body_too_large), do: :ok + defp set_error_ttl(_url, {:content_type, _}), do: :ok - # The TTL is not set for the errors above, since they are unlikely to change - # with time + # The TTL is not set for the errors above, since they are unlikely to change + # with time - defp set_error_ttl(url, _reason) do - ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) - @cachex.expire(:rich_media_cache, url, ttl) - :ok - end + defp set_error_ttl(url, _reason) do + ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) + @cachex.expire(:rich_media_cache, url, ttl) + :ok + end - defp log_error(url, {:invalid_metadata, data}) do - Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end) - end + defp log_error(url, {:invalid_metadata, data}) do + Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end) + end - defp log_error(url, reason) do - Logger.warning(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) - end + defp log_error(url, reason) do + Logger.warning(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) end @doc """ diff --git a/test/fixtures/rich_media/oembed.html b/test/fixtures/rich_media/oembed.html index 55f17004b..5429630d0 100644 --- a/test/fixtures/rich_media/oembed.html +++ b/test/fixtures/rich_media/oembed.html @@ -1,3 +1,3 @@ diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 7bbe46cbe..f95f15ec3 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -336,13 +336,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do path -> Pleroma.Test.StaticConfig.get(path) end) - Tesla.Mock.mock(fn - %{ - method: :get, - url: "https://example.com/twitter-card" - } -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")} - + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) diff --git a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs index 44d40269c..c8b3cb391 100644 --- a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs +++ b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs @@ -49,6 +49,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do :chat_message_id_idempotency_key_cache, ^id -> {:ok, "123"} cache, key -> NullCache.get(cache, key) end) + |> stub(:fetch, fn :rich_media_cache, _, _ -> {:ok, {:ok, %{}}} end) chat_message = MessageReferenceView.render("show.json", chat_message_reference: cm_ref) diff --git a/test/pleroma/web/rich_media/helpers_test.exs b/test/pleroma/web/rich_media/helpers_test.exs index bf7372476..13d2341ad 100644 --- a/test/pleroma/web/rich_media/helpers_test.exs +++ b/test/pleroma/web/rich_media/helpers_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.HelpersTest do - use Pleroma.DataCase, async: true + use Pleroma.DataCase, async: false alias Pleroma.StaticStubbedConfigMock, as: ConfigMock alias Pleroma.Web.CommonAPI @@ -14,7 +14,7 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do import Tesla.Mock setup do - mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) ConfigMock |> stub(:get, fn diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs index 9064138a6..a05b89a2b 100644 --- a/test/pleroma/web/rich_media/parser_test.exs +++ b/test/pleroma/web/rich_media/parser_test.exs @@ -3,95 +3,26 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.ParserTest do - use ExUnit.Case, async: true + use Pleroma.DataCase, async: false alias Pleroma.Web.RichMedia.Parser + import Tesla.Mock + setup do - Tesla.Mock.mock(fn - %{ - method: :get, - url: "http://example.com/ogp" - } -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")} - - %{ - method: :get, - url: "http://example.com/non-ogp" - } -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")} - - %{ - method: :get, - url: "http://example.com/ogp-missing-title" - } -> - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/rich_media/ogp-missing-title.html") - } - - %{ - method: :get, - url: "http://example.com/twitter-card" - } -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")} - - %{ - method: :get, - url: "http://example.com/oembed" - } -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.html")} - - %{ - method: :get, - url: "http://example.com/oembed.json" - } -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.json")} - - %{method: :get, url: "http://example.com/empty"} -> - %Tesla.Env{status: 200, body: "hello"} - - %{method: :get, url: "http://example.com/malformed"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")} - - %{method: :get, url: "http://example.com/error"} -> - {:error, :overload} - - %{ - method: :head, - url: "http://example.com/huge-page" - } -> - %Tesla.Env{ - status: 200, - headers: [{"content-length", "2000001"}, {"content-type", "text/html"}] - } - - %{ - method: :head, - url: "http://example.com/pdf-file" - } -> - %Tesla.Env{ - status: 200, - headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}] - } - - %{method: :head} -> - %Tesla.Env{status: 404, body: "", headers: []} - end) - - :ok + mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) end test "returns error when no metadata present" do - assert {:error, _} = Parser.parse("http://example.com/empty") + assert {:error, _} = Parser.parse("https://example.com/empty") end test "doesn't just add a title" do - assert {:error, {:invalid_metadata, _}} = Parser.parse("http://example.com/non-ogp") + assert {:error, {:invalid_metadata, _}} = Parser.parse("https://example.com/non-ogp") end test "parses ogp" do - assert Parser.parse("http://example.com/ogp") == + assert Parser.parse("https://example.com/ogp") == {:ok, %{ "image" => "http://ia.media-imdb.com/images/rock.jpg", @@ -99,12 +30,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do "description" => "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.", "type" => "video.movie", - "url" => "http://example.com/ogp" + "url" => "https://example.com/ogp" }} end test "falls back to when ogp:title is missing" do - assert Parser.parse("http://example.com/ogp-missing-title") == + assert Parser.parse("https://example.com/ogp-missing-title") == {:ok, %{ "image" => "http://ia.media-imdb.com/images/rock.jpg", @@ -112,12 +43,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do "description" => "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.", "type" => "video.movie", - "url" => "http://example.com/ogp-missing-title" + "url" => "https://example.com/ogp-missing-title" }} end test "parses twitter card" do - assert Parser.parse("http://example.com/twitter-card") == + assert Parser.parse("https://example.com/twitter-card") == {:ok, %{ "card" => "summary", @@ -125,12 +56,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do "image" => "https://farm6.staticflickr.com/5510/14338202952_93595258ff_z.jpg", "title" => "Small Island Developing States Photo Submission", "description" => "View the album on Flickr.", - "url" => "http://example.com/twitter-card" + "url" => "https://example.com/twitter-card" }} end test "parses OEmbed and filters HTML tags" do - assert Parser.parse("http://example.com/oembed") == + assert Parser.parse("https://example.com/oembed") == {:ok, %{ "author_name" => "\u202E\u202D\u202Cbees\u202C", @@ -150,7 +81,7 @@ defmodule Pleroma.Web.RichMedia.ParserTest do "thumbnail_width" => 150, "title" => "Bacon Lollys", "type" => "photo", - "url" => "http://example.com/oembed", + "url" => "https://example.com/oembed", "version" => "1.0", "web_page" => "https://www.flickr.com/photos/bees/2362225867/", "web_page_short_url" => "https://flic.kr/p/4AK2sc", @@ -159,18 +90,18 @@ defmodule Pleroma.Web.RichMedia.ParserTest do end test "rejects invalid OGP data" do - assert {:error, _} = Parser.parse("http://example.com/malformed") + assert {:error, _} = Parser.parse("https://example.com/malformed") end test "returns error if getting page was not successful" do - assert {:error, :overload} = Parser.parse("http://example.com/error") + assert {:error, :overload} = Parser.parse("https://example.com/error") end test "does a HEAD request to check if the body is too large" do - assert {:error, :body_too_large} = Parser.parse("http://example.com/huge-page") + assert {:error, :body_too_large} = Parser.parse("https://example.com/huge-page") end test "does a HEAD request to check if the body is html" do - assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file") + assert {:error, {:content_type, _}} = Parser.parse("https://example.com/pdf-file") end end diff --git a/test/support/cachex_proxy.ex b/test/support/cachex_proxy.ex index 83ae5610f..8f27986a9 100644 --- a/test/support/cachex_proxy.ex +++ b/test/support/cachex_proxy.ex @@ -26,9 +26,15 @@ defmodule Pleroma.CachexProxy do @impl true defdelegate fetch!(cache, key, func), to: Cachex + @impl true + defdelegate fetch(cache, key, func), to: Cachex + @impl true defdelegate expire_at(cache, str, num), to: Cachex + @impl true + defdelegate expire(cache, str, num), to: Cachex + @impl true defdelegate exists?(cache, key), to: Cachex diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index df3371a75..f4b6f1f9f 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1059,7 +1059,7 @@ defmodule HttpRequestMock do }} end - def get("http://example.com/malformed", _, _, _) do + def get("https://example.com/malformed", _, _, _) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}} end @@ -1472,6 +1472,37 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/yahoo.html")}} end + def get("https://example.com/error", _, _, _), do: {:error, :overload} + + def get("https://example.com/ogp-missing-title", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/rich_media/ogp-missing-title.html") + }} + end + + def get("https://example.com/oembed", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.html")}} + end + + def get("https://example.com/oembed.json", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.json")}} + end + + def get("https://example.com/twitter-card", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}} + end + + def get("https://example.com/non-ogp", _, _, _) do + {:ok, + %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")}} + end + + def get("https://example.com/empty", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: "hello"}} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} @@ -1545,17 +1576,41 @@ defmodule HttpRequestMock do # Most of the rich media mocks are missing HEAD requests, so we just return 404. @rich_media_mocks [ + "https://example.com/empty", + "https://example.com/error", + "https://example.com/malformed", + "https://example.com/non-ogp", + "https://example.com/oembed", + "https://example.com/oembed.json", "https://example.com/ogp", "https://example.com/ogp-missing-data", + "https://example.com/ogp-missing-title", "https://example.com/twitter-card", "https://google.com/", - "https://yahoo.com/", - "https://pleroma.local/notice/9kCP7V" + "https://pleroma.local/notice/9kCP7V", + "https://yahoo.com/" ] + def head(url, _query, _body, _headers) when url in @rich_media_mocks do {:ok, %Tesla.Env{status: 404, body: ""}} end + def head("https://example.com/pdf-file", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}] + }} + end + + def head("https://example.com/huge-page", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + headers: [{"content-length", "2000001"}, {"content-type", "text/html"}] + }} + end + def head(url, query, body, headers) do {:error, "Mock response not implemented for HEAD #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} diff --git a/test/support/null_cache.ex b/test/support/null_cache.ex index 9f1d45f1d..47c84174e 100644 --- a/test/support/null_cache.ex +++ b/test/support/null_cache.ex @@ -28,6 +28,9 @@ defmodule Pleroma.NullCache do end end + @impl true + def fetch(_, key, func), do: func.(key) + @impl true def get_and_update(_, _, func) do func.(nil) @@ -36,6 +39,9 @@ defmodule Pleroma.NullCache do @impl true def expire_at(_, _, _), do: {:ok, true} + @impl true + def expire(_, _, _), do: {:ok, true} + @impl true def exists?(_, _), do: {:ok, false} From 9f2319e50dc0516bde4bfa3b117ec4792e553bd2 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Tue, 6 Feb 2024 16:54:52 -0500 Subject: [PATCH 25/50] RichMedia.Helpers: move the validate_page_url/1 function to the Parser module This will ensure that the page validation happens in Parser.parse/1 so it can be called from anywhere and still filter invalid URLs. --- changelog.d/rich_media_tests.skip | 0 lib/pleroma/web/rich_media/helpers.ex | 43 ------------------------- lib/pleroma/web/rich_media/parser.ex | 46 ++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 44 deletions(-) create mode 100644 changelog.d/rich_media_tests.skip diff --git a/changelog.d/rich_media_tests.skip b/changelog.d/rich_media_tests.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 1501776d9..a711dc436 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -18,53 +18,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do recv_timeout: 2_000 ] - @spec validate_page_url(URI.t() | binary()) :: :ok | :error - defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = @config_impl.get([Pleroma.Formatter, :validate_tld]) - - page_url - |> Linkify.Parser.url?(validate_tld: validate_tld) - |> parse_uri(page_url) - end - - defp validate_page_url(%URI{host: host, scheme: "https"}) do - cond do - Linkify.Parser.ip?(host) -> - :error - - host in @config_impl.get([:rich_media, :ignore_hosts], []) -> - :error - - get_tld(host) in @config_impl.get([:rich_media, :ignore_tld], []) -> - :error - - true -> - :ok - end - end - - defp validate_page_url(_), do: :error - - defp parse_uri(true, url) do - url - |> URI.parse() - |> validate_page_url - end - - defp parse_uri(_, _), do: :error - - defp get_tld(host) do - host - |> String.split(".") - |> Enum.reverse() - |> hd - end - def fetch_data_for_object(object) do with true <- @config_impl.get([:rich_media, :enabled]), {:ok, page_url} <- HTML.extract_first_external_url_from_object(object), - :ok <- validate_page_url(page_url), {:ok, rich_media} <- Parser.parse(page_url) do %{page_url: page_url, rich_media: rich_media} else diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index a791b2bab..a73fbc4b9 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.RichMedia.Parser do require Logger @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) defp parsers do Pleroma.Config.get([:rich_media, :parsers]) @@ -15,7 +16,8 @@ defmodule Pleroma.Web.RichMedia.Parser do @spec parse(String.t()) :: {:ok, map()} | {:error, any()} def parse(url) do - with {:ok, data} <- get_cached_or_parse(url), + with :ok <- validate_page_url(url), + {:ok, data} <- get_cached_or_parse(url), {:ok, _} <- set_ttl_based_on_image(data, url) do {:ok, data} end @@ -161,4 +163,46 @@ defmodule Pleroma.Web.RichMedia.Parser do end) |> Map.new() end + + @spec validate_page_url(URI.t() | binary()) :: :ok | :error + defp validate_page_url(page_url) when is_binary(page_url) do + validate_tld = @config_impl.get([Pleroma.Formatter, :validate_tld]) + + page_url + |> Linkify.Parser.url?(validate_tld: validate_tld) + |> parse_uri(page_url) + end + + defp validate_page_url(%URI{host: host, scheme: "https"}) do + cond do + Linkify.Parser.ip?(host) -> + :error + + host in @config_impl.get([:rich_media, :ignore_hosts], []) -> + :error + + get_tld(host) in @config_impl.get([:rich_media, :ignore_tld], []) -> + :error + + true -> + :ok + end + end + + defp validate_page_url(_), do: :error + + defp parse_uri(true, url) do + url + |> URI.parse() + |> validate_page_url + end + + defp parse_uri(_, _), do: :error + + defp get_tld(host) do + host + |> String.split(".") + |> Enum.reverse() + |> hd + end end From 0fcdcc2300cc23eee27f1d20bf0a8008581329d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= <git@mkljczk.pl> Date: Wed, 7 Feb 2024 17:55:52 +0100 Subject: [PATCH 26/50] Use User.full_nickname/1 in oauth html template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak <git@mkljczk.pl> --- changelog.d/oauth-nickname.skip | 1 + lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/oauth-nickname.skip diff --git a/changelog.d/oauth-nickname.skip b/changelog.d/oauth-nickname.skip new file mode 100644 index 000000000..02f16e06c --- /dev/null +++ b/changelog.d/oauth-nickname.skip @@ -0,0 +1 @@ +Use User.full_nickname/1 in oauth html template \ No newline at end of file diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 5b38f7142..6bc8eb602 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -13,7 +13,7 @@ <div class="account-header__avatar" style="background-image: url('<%= Pleroma.User.avatar_url(@user) %>')"></div> <div class="account-header__meta"> <div class="account-header__display-name"><%= @user.name %></div> - <div class="account-header__nickname">@<%= @user.nickname %>@<%= Pleroma.User.get_host(@user) %></div> + <div class="account-header__nickname">@<%= Pleroma.User.full_nickname(@user.nickname) %></div> </div> </div> <% end %> From 0eca3e38ebd6c5b1e7135275959d984af7acfd26 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Fri, 9 Feb 2024 10:36:58 -0500 Subject: [PATCH 27/50] Fix Gun connection supervisor logic error This was recently changed to solve a Dialyzer error, but the replacement logic was faulty as "retry" would only be compared to :error and not have its truthiness evaluated. The original logic was also faulty as it returned {:error, :pool_full} even retry was true. It never retried when the pool was full. --- changelog.d/gun_pool.fix | 1 + lib/pleroma/gun/connection_pool/worker_supervisor.ex | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/gun_pool.fix diff --git a/changelog.d/gun_pool.fix b/changelog.d/gun_pool.fix new file mode 100644 index 000000000..94ec9103d --- /dev/null +++ b/changelog.d/gun_pool.fix @@ -0,0 +1 @@ +Fix logic error in Gun connection pooling which prevented retries even when the worker was launched with retry = true diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index b2be4ff87..24ad61117 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -21,7 +21,9 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do def start_worker(opts, retry \\ false) do case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do {:error, :max_children} -> - if Enum.any?([retry, free_pool()], &match?(&1, :error)) do + funs = [fn -> !retry end, fn -> match?(:error, free_pool()) end] + + if Enum.any?(funs, fn fun -> fun.() end) do :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) {:error, :pool_full} else From 8daf19ec0faac26569bf078b140fc4f7ad0d6b2b Mon Sep 17 00:00:00 2001 From: Alex Gleason <alex@alexgleason.me> Date: Mon, 12 Feb 2024 12:39:47 -0600 Subject: [PATCH 28/50] Fix notifications index --- lib/pleroma/notification.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 48d467c59..368e609d2 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Notification do where: q.seen == true, select: type(q.id, :string), limit: 1, - order_by: [desc: :id] + order_by: fragment("? desc nulls last", q.id) ) end From cb4d3db8c69104960f18be740cb7253add8c00e4 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Mon, 12 Feb 2024 14:14:38 -0500 Subject: [PATCH 29/50] Changelog for notifications fix pulled in from Rebased --- changelog.d/.search_config.fix.swp | Bin 0 -> 12288 bytes changelog.d/notifications-index.fix | 1 + 2 files changed, 1 insertion(+) create mode 100644 changelog.d/.search_config.fix.swp create mode 100644 changelog.d/notifications-index.fix diff --git a/changelog.d/.search_config.fix.swp b/changelog.d/.search_config.fix.swp new file mode 100644 index 0000000000000000000000000000000000000000..e310fa87e26a0ad016283c2dc8e69c6ae7819dcd GIT binary patch literal 12288 zcmeI%Jx;?g7=U4?I|7vp1Uh4vGO;3-4w1Tq6}gF%T21Z9$<PVJDYzL2;RGb5-3n62 zsIMj4@~`|ok_XGn>B~HcALUeZ>8dY&)5oU}g{ksG`@cHTVcpy%PYY>fzeaA%(dlpf zpek*vY7-jobfxOb1i#TLTWMFC%9k=%jnN@ny=LAOwv>h49^H&i1Q0k;;39pP-S4jY z<n|`M9;YX+XF381Ab<b@2q1s}0tgIQpsgeE-UTq)2NC^I4>?B8jsOA(Ab<b@2q1s} z0tg_0z_|<P?uY;DQi$&^_y7Ng_y4Ya?l{Dc00IagfB*srAb<b@2q1vKU<ICi?0g$r I*O<_XA5R1=AOHXW literal 0 HcmV?d00001 diff --git a/changelog.d/notifications-index.fix b/changelog.d/notifications-index.fix new file mode 100644 index 000000000..4617cbec0 --- /dev/null +++ b/changelog.d/notifications-index.fix @@ -0,0 +1 @@ +Fix notifications query which was not using the index properly From 67c3acde34bf92e033b9c65b210e3f6fb3e97778 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Mon, 12 Feb 2024 17:22:57 -0500 Subject: [PATCH 30/50] Update .gitignore --- .gitignore | 5 +++-- changelog.d/.search_config.fix.swp | Bin 12288 -> 0 bytes 2 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/.search_config.fix.swp diff --git a/.gitignore b/.gitignore index 4009bd844..3b672184e 100644 --- a/.gitignore +++ b/.gitignore @@ -57,5 +57,6 @@ pleroma.iml .tool-versions # Editor temp files -/*~ -/*# +*~ +*# +*.swp diff --git a/changelog.d/.search_config.fix.swp b/changelog.d/.search_config.fix.swp deleted file mode 100644 index e310fa87e26a0ad016283c2dc8e69c6ae7819dcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI%Jx;?g7=U4?I|7vp1Uh4vGO;3-4w1Tq6}gF%T21Z9$<PVJDYzL2;RGb5-3n62 zsIMj4@~`|ok_XGn>B~HcALUeZ>8dY&)5oU}g{ksG`@cHTVcpy%PYY>fzeaA%(dlpf zpek*vY7-jobfxOb1i#TLTWMFC%9k=%jnN@ny=LAOwv>h49^H&i1Q0k;;39pP-S4jY z<n|`M9;YX+XF381Ab<b@2q1s}0tgIQpsgeE-UTq)2NC^I4>?B8jsOA(Ab<b@2q1s} z0tg_0z_|<P?uY;DQi$&^_y7Ng_y4Ya?l{Dc00IagfB*srAb<b@2q1vKU<ICi?0g$r I*O<_XA5R1=AOHXW From 3b82864bccee1af625dd19faed511d5b76f66f9d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Wed, 14 Feb 2024 18:16:54 +0100 Subject: [PATCH 31/50] =?UTF-8?q?Config:=20Check=20the=20permissions=20of?= =?UTF-8?q?=20the=20linked=20file=20instead=20of=20the=20symlink=E2=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- changelog.d/config-stat-symlink.fix | 1 + lib/pleroma/config/release_runtime_provider.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/config-stat-symlink.fix diff --git a/changelog.d/config-stat-symlink.fix b/changelog.d/config-stat-symlink.fix new file mode 100644 index 000000000..c8b98225d --- /dev/null +++ b/changelog.d/config-stat-symlink.fix @@ -0,0 +1 @@ +- Config: Check the permissions of the linked file instead of the symlink diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 9ec0f975e..351639836 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do with_runtime_config = if File.exists?(config_path) do # <https://git.pleroma.social/pleroma/pleroma/-/issues/3135> - %File.Stat{mode: mode} = File.lstat!(config_path) + %File.Stat{mode: mode} = File.stat!(config_path) if Bitwise.band(mode, 0o007) > 0 do raise "Configuration at #{config_path} has world-permissions, execute the following: chmod o= #{config_path}" From 60ba6fd244b4deb8c69d4cd6b3114dcefeee4075 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 14 Feb 2024 13:25:52 -0500 Subject: [PATCH 32/50] MediaProxy RFC compliance --- changelog.d/content-length.fix | 1 + lib/pleroma/reverse_proxy.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/content-length.fix diff --git a/changelog.d/content-length.fix b/changelog.d/content-length.fix new file mode 100644 index 000000000..dee906a9d --- /dev/null +++ b/changelog.d/content-length.fix @@ -0,0 +1 @@ +MediaProxy was setting the content-length header which is not permitted by RFC9112§6.2 when we are chunking the reply as it conflicts with the existence of the transfer-encoding header. diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index cc4530010..4d13e51fc 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -8,7 +8,7 @@ defmodule Pleroma.ReverseProxy do ~w(if-unmodified-since if-none-match) ++ @range_headers @resp_cache_headers ~w(etag date last-modified) @keep_resp_headers @resp_cache_headers ++ - ~w(content-length content-type content-disposition content-encoding) ++ + ~w(content-type content-disposition content-encoding) ++ ~w(content-range accept-ranges vary) @default_cache_control_header "public, max-age=1209600" @valid_resp_codes [200, 206, 304] From 9a4c8e2316b59e0c369486c3c2f758162af1b72e Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 14 Feb 2024 13:28:32 -0500 Subject: [PATCH 33/50] Change some Gun connection pool logs to debug level --- changelog.d/gun-logs.skip | 0 lib/pleroma/telemetry/logger.ex | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/gun-logs.skip diff --git a/changelog.d/gun-logs.skip b/changelog.d/gun-logs.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 92d395394..9998d8185 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Telemetry.Logger do _, _ ) do - Logger.error(fn -> + Logger.debug(fn -> "Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion" end) end @@ -81,7 +81,7 @@ defmodule Pleroma.Telemetry.Logger do %{key: key, protocol: :http}, _ ) do - Logger.info(fn -> + Logger.debug(fn -> "Pool worker for #{key}: #{length(clients)} clients are using an HTTP1 connection at the same time, head-of-line blocking might occur." end) end From 64ad451a7b8a2ac89079a1bc32680e9cf08ef24e Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Tue, 13 Feb 2024 19:21:45 -0500 Subject: [PATCH 34/50] Websocket refactor to use Phoenix.Socket.Transport This will make us compatible with Cowboy and Bandit --- config/config.exs | 9 +- lib/phoenix/transports/web_socket/raw.ex | 93 ------- lib/pleroma/web/endpoint.ex | 9 + .../web/mastodon_api/websocket_handler.ex | 256 +++++++++--------- 4 files changed, 133 insertions(+), 234 deletions(-) delete mode 100644 lib/phoenix/transports/web_socket/raw.ex diff --git a/config/config.exs b/config/config.exs index bb17ab145..435387a64 100644 --- a/config/config.exs +++ b/config/config.exs @@ -114,14 +114,7 @@ config :pleroma, :uri_schemes, config :pleroma, Pleroma.Web.Endpoint, url: [host: "localhost"], http: [ - ip: {127, 0, 0, 1}, - dispatch: [ - {:_, - [ - {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, - {:_, Plug.Cowboy.Handler, {Pleroma.Web.Endpoint, []}} - ]} - ] + ip: {127, 0, 0, 1} ], protocol: "https", secret_key_base: "aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl", diff --git a/lib/phoenix/transports/web_socket/raw.ex b/lib/phoenix/transports/web_socket/raw.ex deleted file mode 100644 index cf4fda79f..000000000 --- a/lib/phoenix/transports/web_socket/raw.ex +++ /dev/null @@ -1,93 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Phoenix.Transports.WebSocket.Raw do - import Plug.Conn, - only: [ - fetch_query_params: 1, - send_resp: 3 - ] - - alias Phoenix.Socket.Transport - - def default_config do - [ - timeout: 60_000, - transport_log: false, - cowboy: Phoenix.Endpoint.CowboyWebSocket - ] - end - - def init(%Plug.Conn{method: "GET"} = conn, {endpoint, handler, transport}) do - {_, opts} = handler.__transport__(transport) - - conn = - conn - |> fetch_query_params - |> Transport.transport_log(opts[:transport_log]) - |> Transport.check_origin(handler, endpoint, opts) - - case conn do - %{halted: false} = conn -> - case handler.connect(%{ - endpoint: endpoint, - transport: transport, - options: [serializer: nil], - params: conn.params - }) do - {:ok, socket} -> - {:ok, conn, {__MODULE__, {socket, opts}}} - - :error -> - send_resp(conn, :forbidden, "") - {:error, conn} - end - - _ -> - {:error, conn} - end - end - - def init(conn, _) do - send_resp(conn, :bad_request, "") - {:error, conn} - end - - def ws_init({socket, config}) do - Process.flag(:trap_exit, true) - {:ok, %{socket: socket}, config[:timeout]} - end - - def ws_handle(op, data, state) do - state.socket.handler - |> apply(:handle, [op, data, state]) - |> case do - {op, data} -> - {:reply, {op, data}, state} - - {op, data, state} -> - {:reply, {op, data}, state} - - %{} = state -> - {:ok, state} - - _ -> - {:ok, state} - end - end - - def ws_info({_, _} = tuple, state) do - {:reply, tuple, state} - end - - def ws_info(_tuple, state), do: {:ok, state} - - def ws_close(state) do - ws_handle(:closed, :normal, state) - end - - def ws_terminate(reason, state) do - ws_handle(:closed, reason, state) - end -end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 307fa069e..c56362049 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,6 +9,15 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config + socket("/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, + longpoll: false, + websocket: [ + path: "/", + compress: false, + error_handler: {Pleroma.Web.MastodonAPI.WebsocketHandler, :handle_error, []} + ] + ) + socket("/socket", Pleroma.Web.UserSocket, websocket: [ path: "/websocket", diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 07c2b62e3..bb27d806d 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -11,28 +11,21 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.Web.Streamer alias Pleroma.Web.StreamerView - @behaviour :cowboy_websocket + @behaviour Phoenix.Socket.Transport # Client ping period. @tick :timer.seconds(30) - # Cowboy timeout period. - @timeout :timer.seconds(60) - # Hibernate every X messages - @hibernate_every 100 - def init(%{qs: qs} = req, state) do - with params <- Enum.into(:cow_qs.parse_qs(qs), %{}), - sec_websocket <- :cowboy_req.header("sec-websocket-protocol", req, nil), - access_token <- Map.get(params, "access_token"), - {:ok, user, oauth_token} <- authenticate_request(access_token, sec_websocket), - {:ok, topic} <- Streamer.get_topic(params["stream"], user, oauth_token, params) do - req = - if sec_websocket do - :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req) - else - req - end + @impl Phoenix.Socket.Transport + def child_spec(_opts), do: :ignore + # This only prepares the connection and is not in the process yet + @impl Phoenix.Socket.Transport + def connect(%{params: params} = transport_info) do + with access_token <- Map.get(params, "access_token"), + {:ok, user, oauth_token} <- authenticate_request(access_token), + {:ok, topic} <- + Streamer.get_topic(params["stream"], user, oauth_token, params) do topics = if topic do [topic] @@ -40,41 +33,40 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do [] end - {:cowboy_websocket, req, - %{user: user, topics: topics, oauth_token: oauth_token, count: 0, timer: nil}, - %{idle_timeout: @timeout}} + state = %{ + user: user, + topics: topics, + oauth_token: oauth_token, + count: 0, + timer: nil + } + + {:ok, state} else {:error, :bad_topic} -> - Logger.debug("#{__MODULE__} bad topic #{inspect(req)}") - req = :cowboy_req.reply(404, req) - {:ok, req, state} + Logger.debug("#{__MODULE__} bad topic #{inspect(transport_info)}") + + {:error, :bad_topic} {:error, :unauthorized} -> - Logger.debug("#{__MODULE__} authentication error: #{inspect(req)}") - req = :cowboy_req.reply(401, req) - {:ok, req, state} + Logger.debug("#{__MODULE__} authentication error: #{inspect(transport_info)}") + {:error, :unauthorized} end end - def websocket_init(state) do - Logger.debug( - "#{__MODULE__} accepted websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics}" - ) - + # All subscriptions/links and messages cannot be created + # until the processed is launched with init/1 + @impl Phoenix.Socket.Transport + def init(state) do Enum.each(state.topics, fn topic -> Streamer.add_socket(topic, state.oauth_token) end) - {:ok, %{state | timer: timer()}} + + Process.send_after(self(), :ping, @tick) + + {:ok, state} end - # Client's Pong frame. - def websocket_handle(:pong, state) do - if state.timer, do: Process.cancel_timer(state.timer) - {:ok, %{state | timer: timer()}} - end - - # We only receive pings for now - def websocket_handle(:ping, state), do: {:ok, state} - - def websocket_handle({:text, text}, state) do + @impl Phoenix.Socket.Transport + def handle_in({text, [opcode: :text]}, state) do with {:ok, %{} = event} <- Jason.decode(text) do handle_client_event(event, state) else @@ -84,50 +76,47 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end end - def websocket_handle(frame, state) do + def handle_in(frame, state) do Logger.error("#{__MODULE__} received frame: #{inspect(frame)}") {:ok, state} end - def websocket_info({:render_with_user, view, template, item, topic}, state) do + @impl Phoenix.Socket.Transport + def handle_info({:render_with_user, view, template, item, topic}, state) do user = %User{} = User.get_cached_by_ap_id(state.user.ap_id) unless Streamer.filtered_by_user?(user, item) do - websocket_info({:text, view.render(template, item, user, topic)}, %{state | user: user}) + message = view.render(template, item, user, topic) + {:push, {:text, message}, %{state | user: user}} else {:ok, state} end end - def websocket_info({:text, message}, state) do - # If the websocket processed X messages, force an hibernate/GC. - # We don't hibernate at every message to balance CPU usage/latency with RAM usage. - if state.count > @hibernate_every do - {:reply, {:text, message}, %{state | count: 0}, :hibernate} - else - {:reply, {:text, message}, %{state | count: state.count + 1}} - end + def handle_info({:text, text}, state) do + {:push, {:text, text}, state} end - # Ping tick. We don't re-queue a timer there, it is instead queued when :pong is received. - # As we hibernate there, reset the count to 0. - # If the client misses :pong, Cowboy will automatically timeout the connection after - # `@idle_timeout`. - def websocket_info(:tick, state) do - {:reply, :ping, %{state | timer: nil, count: 0}, :hibernate} + def handle_info(:ping, state) do + Process.send_after(self(), :ping, @tick) + + {:push, {:ping, ""}, state} end - def websocket_info(:close, state) do - {:stop, state} + def handle_info(:close, state) do + {:stop, {:closed, 'connection closed by server'}, state} end - # State can be `[]` only in case we terminate before switching to websocket, - # we already log errors for these cases in `init/1`, so just do nothing here - def terminate(_reason, _req, []), do: :ok + def handle_info(msg, state) do + Logger.debug("#{__MODULE__} received info: #{inspect(msg)}") - def terminate(reason, _req, state) do + {:ok, state} + end + + @impl Phoenix.Socket.Transport + def terminate(reason, state) do Logger.debug( - "#{__MODULE__} terminating websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics || "?"}: #{inspect(reason)}" + "#{__MODULE__} terminating websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics || "?"}: #{inspect(reason)})" ) Enum.each(state.topics, fn topic -> Streamer.remove_socket(topic) end) @@ -135,16 +124,13 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end # Public streams without authentication. - defp authenticate_request(nil, nil) do + defp authenticate_request(nil) do {:ok, nil, nil} end # Authenticated streams. - defp authenticate_request(access_token, sec_websocket) do - token = access_token || sec_websocket - - with true <- is_bitstring(token), - oauth_token = %Token{user_id: user_id} <- Repo.get_by(Token, token: token), + defp authenticate_request(access_token) do + with oauth_token = %Token{user_id: user_id} <- Repo.get_by(Token, token: access_token), user = %User{} <- User.get_cached_by_id(user_id) do {:ok, user, oauth_token} else @@ -152,36 +138,32 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end end - defp timer do - Process.send_after(self(), :tick, @tick) - end - defp handle_client_event(%{"type" => "subscribe", "stream" => _topic} = params, state) do with {_, {:ok, topic}} <- {:topic, Streamer.get_topic(params["stream"], state.user, state.oauth_token, params)}, {_, false} <- {:subscribed, topic in state.topics} do Streamer.add_socket(topic, state.oauth_token) - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "success"})} - ], %{state | topics: [topic | state.topics]}} + message = + StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "success"}) + + {:reply, :ok, {:text, message}, %{state | topics: [topic | state.topics]}} else {:subscribed, true} -> - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "ignored"})} - ], state} + message = + StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "ignored"}) + + {:reply, :error, {:text, message}, state} {:topic, {:error, error}} -> - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{ - type: "subscribe", - result: "error", - error: error - })} - ], state} + message = + StreamerView.render("pleroma_respond.json", %{ + type: "subscribe", + result: "error", + error: error + }) + + {:reply, :error, {:text, message}, state} end end @@ -191,26 +173,26 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {_, true} <- {:subscribed, topic in state.topics} do Streamer.remove_socket(topic) - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "success"})} - ], %{state | topics: List.delete(state.topics, topic)}} + message = + StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "success"}) + + {:reply, :ok, {:text, message}, %{state | topics: List.delete(state.topics, topic)}} else {:subscribed, false} -> - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "ignored"})} - ], state} + message = + StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "ignored"}) + + {:reply, :error, {:text, message}, state} {:topic, {:error, error}} -> - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{ - type: "unsubscribe", - result: "error", - error: error - })} - ], state} + message = + StreamerView.render("pleroma_respond.json", %{ + type: "unsubscribe", + result: "error", + error: error + }) + + {:reply, :error, {:text, message}, state} end end @@ -219,39 +201,47 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do state ) do with {:auth, nil, nil} <- {:auth, state.user, state.oauth_token}, - {:ok, user, oauth_token} <- authenticate_request(access_token, nil) do - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{ - type: "pleroma:authenticate", - result: "success" - })} - ], %{state | user: user, oauth_token: oauth_token}} + {:ok, user, oauth_token} <- authenticate_request(access_token) do + message = + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma:authenticate", + result: "success" + }) + + {:reply, :ok, {:text, message}, %{state | user: user, oauth_token: oauth_token}} else {:auth, _, _} -> - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{ - type: "pleroma:authenticate", - result: "error", - error: :already_authenticated - })} - ], state} + message = + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma:authenticate", + result: "error", + error: :already_authenticated + }) + + {:reply, :error, {:text, message}, state} _ -> - {[ - {:text, - StreamerView.render("pleroma_respond.json", %{ - type: "pleroma:authenticate", - result: "error", - error: :unauthorized - })} - ], state} + message = + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma:authenticate", + result: "error", + error: :unauthorized + }) + + {:reply, :error, {:text, message}, state} end end defp handle_client_event(params, state) do Logger.error("#{__MODULE__} received unknown event: #{inspect(params)}") - {[], state} + {:ok, state} + end + + def handle_error(conn, :unauthorized) do + Plug.Conn.send_resp(conn, 401, "Unauthorized") + end + + def handle_error(conn, _reason) do + Plug.Conn.send_resp(conn, 404, "Not Found") end end From d0f4b2b02fc3aee3f08239d9c188ca5a2e8ad482 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 14 Feb 2024 14:15:24 -0500 Subject: [PATCH 35/50] Remove invalid test It is not allowed to use the Sec-WebSocket-Protocol header for arbitrary values. This was possible due to the raw websocket handling we were doing with Cowboy, but Phoenix.Socket.Transport does not allow this as the value of this header is compared against a static list of subprotocols. https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#socket/3-websocket-configuration Additionally I cannot find anywhere that we depended on this behavior. Setting the Sec-WebSocket-Protocol header does not appear to be a part of PleromaFE. --- test/pleroma/integration/mastodon_websocket_test.exs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index a2c20f0a6..a0ffddf8d 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -268,17 +268,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end - test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do - assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) - - capture_log(fn -> - assert {:error, %WebSockex.RequestError{code: 401}} = - start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) - - Process.sleep(30) - end) - end - test "accepts valid token on client-sent event", %{token: token} do assert {:ok, pid} = start_socket() From 6be129ead2ff5d6a19edf7230d102aa51a731b03 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 14 Feb 2024 14:19:24 -0500 Subject: [PATCH 36/50] Websocket refactor changelog --- changelog.d/websocket-refactor.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/websocket-refactor.change diff --git a/changelog.d/websocket-refactor.change b/changelog.d/websocket-refactor.change new file mode 100644 index 000000000..3c447832b --- /dev/null +++ b/changelog.d/websocket-refactor.change @@ -0,0 +1 @@ +Refactor the Mastodon /api/v1/streaming websocket handler to use Phoenix.Socket.Transport From 86e6d395d931f532b18fccdeb65c300b22fbce8a Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 14 Feb 2024 17:54:56 -0500 Subject: [PATCH 37/50] Fix atom leak in password digest functionality The value here gets passesd to :crypto.pbkdf2_hmac and it expects one of these atoms: :sha | :sha224 | :sha256 | :sha384 | :sha512 so it will always exist --- lib/pleroma/password/pbkdf2.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/password/pbkdf2.ex b/lib/pleroma/password/pbkdf2.ex index 92e9e1952..9c6d2e381 100644 --- a/lib/pleroma/password/pbkdf2.ex +++ b/lib/pleroma/password/pbkdf2.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Password.Pbkdf2 do iterations = String.to_integer(iterations) - digest = String.to_atom(digest) + digest = String.to_existing_atom(digest) binary_hash = KeyGenerator.generate(password, salt, digest: digest, iterations: iterations, length: 64) From 91c83a82a052ec73c82b9b5576fd5b05c7dc8a74 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 14 Feb 2024 17:58:36 -0500 Subject: [PATCH 38/50] Fix atom leak in background worker The only permitted values are "blocks_import", "follow_import", "mutes_import" of which we already have the equivalent atoms defined. --- lib/pleroma/workers/background_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 794417612..7a2210dc1 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Workers.BackgroundWorker do def perform(%Job{args: %{"op" => op, "user_id" => user_id, "identifiers" => identifiers}}) when op in ["blocks_import", "follow_import", "mutes_import"] do user = User.get_cached_by_id(user_id) - {:ok, User.Import.perform(String.to_atom(op), user, identifiers)} + {:ok, User.Import.perform(String.to_existing_atom(op), user, identifiers)} end def perform(%Job{ From 9138754b0acaac9714bbf12d9d00a22870b2af6e Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 14 Feb 2024 18:04:39 -0500 Subject: [PATCH 39/50] Changelog --- changelog.d/atom-leak.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/atom-leak.skip diff --git a/changelog.d/atom-leak.skip b/changelog.d/atom-leak.skip new file mode 100644 index 000000000..e69de29bb From 0c5bec0493bc948a778ea5253e42713f1422742c Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Thu, 15 Feb 2024 09:45:48 -0500 Subject: [PATCH 40/50] Support Bandit as an alternate HTTP backend to Cowboy. This is currently considered experimental, but may improve performance and resource usage. --- changelog.d/bandit.change | 1 + mix.exs | 1 + mix.lock | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 changelog.d/bandit.change diff --git a/changelog.d/bandit.change b/changelog.d/bandit.change new file mode 100644 index 000000000..7a1104314 --- /dev/null +++ b/changelog.d/bandit.change @@ -0,0 +1 @@ +Support Bandit as an alternative to Cowboy for the HTTP server. diff --git a/mix.exs b/mix.exs index a13c89c04..580f0aefc 100644 --- a/mix.exs +++ b/mix.exs @@ -188,6 +188,7 @@ defmodule Pleroma.Mixfile do {:exile, git: "https://git.pleroma.social/pleroma/elixir-libraries/exile.git", ref: "0d6337cf68e7fbc8a093cae000955aa93b067f91"}, + {:bandit, "~> 1.2"}, ## dev & test {:ex_doc, "~> 0.22", only: :dev, runtime: false}, diff --git a/mix.lock b/mix.lock index 60d0d51d9..2da3ecda1 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,6 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, + "bandit": {:hex, :bandit, "1.2.1", "aa485b4ac175065b8e0fb5864ddd5dd7b50d52336b36f61c82f484c3718b3d15", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "27393e590a407f1b7d51c5fee4737f139fe224a30449ce25061eac70f763896b"}, "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"}, "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "19851074419a5fedb4ef49e1f01b30df504bb5dbb6d6adfc135238063bebd1c3"}, "bcrypt_elixir": {:hex, :bcrypt_elixir, "2.3.1", "5114d780459a04f2b4aeef52307de23de961b69e13a5cd98a911e39fda13f420", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "42182d5f46764def15bf9af83739e3bf4ad22661b1c34fc3e88558efced07279"}, @@ -133,6 +134,7 @@ "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.0", "b583c3f18508f5c5561b674d16cf5d9afd2ea3c04505b7d92baaeac93c1b8260", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "9cba950e1c4733468efbe3f821841f34ac05d28e7af7798622f88ecdbbe63ea3"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"}, + "thousand_island": {:hex, :thousand_island, "1.3.2", "bc27f9afba6e1a676dd36507d42e429935a142cf5ee69b8e3f90bff1383943cd", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0e085b93012cd1057b378fce40cbfbf381ff6d957a382bfdd5eca1a98eec2535"}, "timex": {:hex, :timex, "3.7.7", "3ed093cae596a410759104d878ad7b38e78b7c2151c6190340835515d4a46b8a", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "0ec4b09f25fe311321f9fc04144a7e3affe48eb29481d7a5583849b6c4dfa0a7"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, From 202721e80c3dbd1aa5607d86ba8f86f7c95efdf1 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Thu, 15 Feb 2024 09:46:08 -0500 Subject: [PATCH 41/50] Remove Cowboy-specific HTTP options These were only used in dev and served no specific purpose. The equivalent settings for Bandit are under a key called :http1_options and the default values are set to 10_000. --- config/dev.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index ab3e83c12..fe8de5045 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -8,8 +8,7 @@ import Config # with brunch.io to recompile .js and .css sources. config :pleroma, Pleroma.Web.Endpoint, http: [ - port: 4000, - protocol_options: [max_request_line_length: 8192, max_header_value_length: 8192] + port: 4000 ], protocol: "http", debug_errors: true, From 4648997a1017da68dcf5235e527e861f2c85cf91 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Thu, 15 Feb 2024 10:06:37 -0500 Subject: [PATCH 42/50] Support a new changelog entry type: deps --- mix.exs | 2 +- mix.lock | 4 ++-- tools/check-changelog | 2 +- tools/collect-changelog | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index 580f0aefc..582092aef 100644 --- a/mix.exs +++ b/mix.exs @@ -137,7 +137,7 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 1.0"}, {:cachex, "~> 3.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, "~> 1.4.0", override: true}, + {:tesla, "~> 1.8.0"}, {:castore, "~> 0.1"}, {:cowlib, "~> 2.9", override: true}, {:gun, "~> 2.0.0-rc.1", override: true}, diff --git a/mix.lock b/mix.lock index 2da3ecda1..a5fccbec4 100644 --- a/mix.lock +++ b/mix.lock @@ -52,7 +52,7 @@ "fast_html": {:hex, :fast_html, "2.2.0", "6c5ef1be087a4ed613b0379c13f815c4d11742b36b67bb52cee7859847c84520", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "064c4f23b4a6168f9187dac8984b056f2c531bb0787f559fd6a8b34b38aefbae"}, "fast_sanitize": {:hex, :fast_sanitize, "0.2.3", "67b93dfb34e302bef49fec3aaab74951e0f0602fd9fa99085987af05bd91c7a5", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e8ad286d10d0386e15d67d0ee125245ebcfbc7d7290b08712ba9013c8c5e56e2"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, - "finch": {:hex, :finch, "0.17.0", "17d06e1d44d891d20dbd437335eebe844e2426a0cd7e3a3e220b461127c73f70", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8d014a661bb6a437263d4b5abf0bcbd3cf0deb26b1e8596f2a271d22e48934c7"}, + "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, "floki": {:hex, :floki, "0.35.2", "87f8c75ed8654b9635b311774308b2760b47e9a579dabf2e4d5f1e1d42c39e0b", [:mix], [], "hexpm", "6b05289a8e9eac475f644f09c2e4ba7e19201fd002b89c28c1293e7bd16773d9"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, @@ -133,7 +133,7 @@ "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.0", "b583c3f18508f5c5561b674d16cf5d9afd2ea3c04505b7d92baaeac93c1b8260", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "9cba950e1c4733468efbe3f821841f34ac05d28e7af7798622f88ecdbbe63ea3"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, - "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"}, + "tesla": {:hex, :tesla, "1.8.0", "d511a4f5c5e42538d97eef7c40ec4f3e44effdc5068206f42ed859e09e51d1fd", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "10501f360cd926a309501287470372af1a6e1cbed0f43949203a4c13300bc79f"}, "thousand_island": {:hex, :thousand_island, "1.3.2", "bc27f9afba6e1a676dd36507d42e429935a142cf5ee69b8e3f90bff1383943cd", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0e085b93012cd1057b378fce40cbfbf381ff6d957a382bfdd5eca1a98eec2535"}, "timex": {:hex, :timex, "3.7.7", "3ed093cae596a410759104d878ad7b38e78b7c2151c6190340835515d4a46b8a", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "0ec4b09f25fe311321f9fc04144a7e3affe48eb29481d7a5583849b6c4dfa0a7"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, diff --git a/tools/check-changelog b/tools/check-changelog index d053ed577..28ef8454e 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -6,7 +6,7 @@ git remote add upstream https://git.pleroma.social/pleroma/pleroma.git git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}:refs/remotes/upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME git diff --raw --no-renames upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ - grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\|change\)$' + grep ' A\t' | grep '\.\(add\|change\|deps\|fix\|remove\|security\)$' ret=$? if [ $ret -eq 0 ]; then diff --git a/tools/collect-changelog b/tools/collect-changelog index 1e12d5640..d0e4905d4 100755 --- a/tools/collect-changelog +++ b/tools/collect-changelog @@ -23,5 +23,6 @@ collectType change Changed collectType add Added collectType fix Fixed collectType remove Removed +collectType deps Dependencies rm changelog.d/* From 772f8d08cf349e0357d264065d80b9019ac70284 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Thu, 15 Feb 2024 10:07:30 -0500 Subject: [PATCH 43/50] Tesla changelog --- changelog.d/tesla.deps | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/tesla.deps diff --git a/changelog.d/tesla.deps b/changelog.d/tesla.deps new file mode 100644 index 000000000..799bbc670 --- /dev/null +++ b/changelog.d/tesla.deps @@ -0,0 +1 @@ +Update Tesla HTTP client middleware to 1.8.0 From 2a4fa4c408d874d8b938f780337d2956b6f0101f Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Thu, 15 Feb 2024 10:20:52 -0500 Subject: [PATCH 44/50] Add support for a "deps" changelog type and document deps changes since 2.6.1 release --- changelog.d/bandit.change | 1 - changelog.d/bandit.deps | 1 + changelog.d/eblurhash.deps | 1 + changelog.d/exile.deps | 1 + changelog.d/floki.deps | 1 + changelog.d/http_signatures.deps | 1 + changelog.d/phoenix.deps | 1 + changelog.d/{promex.change => promex.deps} | 0 changelog.d/vix.deps | 1 + 9 files changed, 7 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/bandit.change create mode 100644 changelog.d/bandit.deps create mode 100644 changelog.d/eblurhash.deps create mode 100644 changelog.d/exile.deps create mode 100644 changelog.d/floki.deps create mode 100644 changelog.d/http_signatures.deps create mode 100644 changelog.d/phoenix.deps rename changelog.d/{promex.change => promex.deps} (100%) create mode 100644 changelog.d/vix.deps diff --git a/changelog.d/bandit.change b/changelog.d/bandit.change deleted file mode 100644 index 7a1104314..000000000 --- a/changelog.d/bandit.change +++ /dev/null @@ -1 +0,0 @@ -Support Bandit as an alternative to Cowboy for the HTTP server. diff --git a/changelog.d/bandit.deps b/changelog.d/bandit.deps new file mode 100644 index 000000000..b60d1d39c --- /dev/null +++ b/changelog.d/bandit.deps @@ -0,0 +1 @@ +Support Bandit as an alternative to Cowboy for the HTTP server diff --git a/changelog.d/eblurhash.deps b/changelog.d/eblurhash.deps new file mode 100644 index 000000000..0f7f7ed5f --- /dev/null +++ b/changelog.d/eblurhash.deps @@ -0,0 +1 @@ +Replaced eblurhash with rinpatch_blurhash diff --git a/changelog.d/exile.deps b/changelog.d/exile.deps new file mode 100644 index 000000000..fa1b7a004 --- /dev/null +++ b/changelog.d/exile.deps @@ -0,0 +1 @@ +Add exile to replace custom process streaming implementation for calls to ffmpeg diff --git a/changelog.d/floki.deps b/changelog.d/floki.deps new file mode 100644 index 000000000..35f70f6f7 --- /dev/null +++ b/changelog.d/floki.deps @@ -0,0 +1 @@ +Update Floki to 0.3.5 diff --git a/changelog.d/http_signatures.deps b/changelog.d/http_signatures.deps new file mode 100644 index 000000000..9ae1cd75a --- /dev/null +++ b/changelog.d/http_signatures.deps @@ -0,0 +1 @@ +Update http-signatures to 0.1.2 diff --git a/changelog.d/phoenix.deps b/changelog.d/phoenix.deps new file mode 100644 index 000000000..62dd9d1b8 --- /dev/null +++ b/changelog.d/phoenix.deps @@ -0,0 +1 @@ +Update Phoenix to 1.7.3 diff --git a/changelog.d/promex.change b/changelog.d/promex.deps similarity index 100% rename from changelog.d/promex.change rename to changelog.d/promex.deps diff --git a/changelog.d/vix.deps b/changelog.d/vix.deps new file mode 100644 index 000000000..68c361760 --- /dev/null +++ b/changelog.d/vix.deps @@ -0,0 +1 @@ +Add Vix for image processing with libvips to replace some usage of ImageMagick From c9cd449bba73375c25f9ef4c31caf80d0506ff63 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Fri, 16 Feb 2024 12:51:39 -0500 Subject: [PATCH 45/50] Revert "Support a new changelog entry type: deps" This reverts commit 4648997a1017da68dcf5235e527e861f2c85cf91. --- tools/check-changelog | 2 +- tools/collect-changelog | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/check-changelog b/tools/check-changelog index 28ef8454e..d053ed577 100644 --- a/tools/check-changelog +++ b/tools/check-changelog @@ -6,7 +6,7 @@ git remote add upstream https://git.pleroma.social/pleroma/pleroma.git git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}:refs/remotes/upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME git diff --raw --no-renames upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD -- changelog.d | \ - grep ' A\t' | grep '\.\(add\|change\|deps\|fix\|remove\|security\)$' + grep ' A\t' | grep '\.\(skip\|add\|remove\|fix\|security\|change\)$' ret=$? if [ $ret -eq 0 ]; then diff --git a/tools/collect-changelog b/tools/collect-changelog index d0e4905d4..1e12d5640 100755 --- a/tools/collect-changelog +++ b/tools/collect-changelog @@ -23,6 +23,5 @@ collectType change Changed collectType add Added collectType fix Fixed collectType remove Removed -collectType deps Dependencies rm changelog.d/* From 1951d56ed9651abc8f6a9d4dc02b7f976274415f Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Fri, 16 Feb 2024 12:53:18 -0500 Subject: [PATCH 46/50] Revert "Add support for a "deps" changelog type and document deps changes since 2.6.1 release" This reverts commit 2a4fa4c408d874d8b938f780337d2956b6f0101f. --- changelog.d/bandit.change | 1 + changelog.d/bandit.deps | 1 - changelog.d/eblurhash.deps | 1 - changelog.d/exile.deps | 1 - changelog.d/floki.deps | 1 - changelog.d/http_signatures.deps | 1 - changelog.d/phoenix.deps | 1 - changelog.d/{promex.deps => promex.change} | 0 changelog.d/vix.deps | 1 - 9 files changed, 1 insertion(+), 7 deletions(-) create mode 100644 changelog.d/bandit.change delete mode 100644 changelog.d/bandit.deps delete mode 100644 changelog.d/eblurhash.deps delete mode 100644 changelog.d/exile.deps delete mode 100644 changelog.d/floki.deps delete mode 100644 changelog.d/http_signatures.deps delete mode 100644 changelog.d/phoenix.deps rename changelog.d/{promex.deps => promex.change} (100%) delete mode 100644 changelog.d/vix.deps diff --git a/changelog.d/bandit.change b/changelog.d/bandit.change new file mode 100644 index 000000000..7a1104314 --- /dev/null +++ b/changelog.d/bandit.change @@ -0,0 +1 @@ +Support Bandit as an alternative to Cowboy for the HTTP server. diff --git a/changelog.d/bandit.deps b/changelog.d/bandit.deps deleted file mode 100644 index b60d1d39c..000000000 --- a/changelog.d/bandit.deps +++ /dev/null @@ -1 +0,0 @@ -Support Bandit as an alternative to Cowboy for the HTTP server diff --git a/changelog.d/eblurhash.deps b/changelog.d/eblurhash.deps deleted file mode 100644 index 0f7f7ed5f..000000000 --- a/changelog.d/eblurhash.deps +++ /dev/null @@ -1 +0,0 @@ -Replaced eblurhash with rinpatch_blurhash diff --git a/changelog.d/exile.deps b/changelog.d/exile.deps deleted file mode 100644 index fa1b7a004..000000000 --- a/changelog.d/exile.deps +++ /dev/null @@ -1 +0,0 @@ -Add exile to replace custom process streaming implementation for calls to ffmpeg diff --git a/changelog.d/floki.deps b/changelog.d/floki.deps deleted file mode 100644 index 35f70f6f7..000000000 --- a/changelog.d/floki.deps +++ /dev/null @@ -1 +0,0 @@ -Update Floki to 0.3.5 diff --git a/changelog.d/http_signatures.deps b/changelog.d/http_signatures.deps deleted file mode 100644 index 9ae1cd75a..000000000 --- a/changelog.d/http_signatures.deps +++ /dev/null @@ -1 +0,0 @@ -Update http-signatures to 0.1.2 diff --git a/changelog.d/phoenix.deps b/changelog.d/phoenix.deps deleted file mode 100644 index 62dd9d1b8..000000000 --- a/changelog.d/phoenix.deps +++ /dev/null @@ -1 +0,0 @@ -Update Phoenix to 1.7.3 diff --git a/changelog.d/promex.deps b/changelog.d/promex.change similarity index 100% rename from changelog.d/promex.deps rename to changelog.d/promex.change diff --git a/changelog.d/vix.deps b/changelog.d/vix.deps deleted file mode 100644 index 68c361760..000000000 --- a/changelog.d/vix.deps +++ /dev/null @@ -1 +0,0 @@ -Add Vix for image processing with libvips to replace some usage of ImageMagick From 2c9fed9b73766cb581577014e70e3599a7c376f4 Mon Sep 17 00:00:00 2001 From: SyoBoN <syobon@syobon.net> Date: Fri, 16 Feb 2024 02:41:14 +0000 Subject: [PATCH 47/50] Translated using Weblate (Japanese) Currently translated at 63.1% (60 of 95 strings) Translation: Pleroma/Pleroma Backend (domain errors) Translate-URL: https://translate.pleroma.social/projects/pleroma/pleroma-backend-domain-errors/ja/ --- priv/gettext/ja/LC_MESSAGES/errors.po | 90 +++++++++++++-------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/priv/gettext/ja/LC_MESSAGES/errors.po b/priv/gettext/ja/LC_MESSAGES/errors.po index 5d9986485..c3c467a67 100644 --- a/priv/gettext/ja/LC_MESSAGES/errors.po +++ b/priv/gettext/ja/LC_MESSAGES/errors.po @@ -3,16 +3,16 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2021-09-18 09:07+0000\n" -"PO-Revision-Date: 2021-09-19 09:45+0000\n" -"Last-Translator: Ryo Ueno <r.ueno.nfive@gmail.com>\n" +"PO-Revision-Date: 2024-02-16 11:49+0000\n" +"Last-Translator: SyoBoN <syobon@syobon.net>\n" "Language-Team: Japanese <https://translate.pleroma.social/projects/pleroma/" -"pleroma/ja/>\n" +"pleroma-backend-domain-errors/ja/>\n" "Language: ja\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=1; plural=0;\n" -"X-Generator: Weblate 4.6.2\n" +"X-Generator: Weblate 4.13.1\n" ## This file is a PO Template file. ## @@ -25,11 +25,11 @@ msgstr "" ## effect: edit them in PO (`.po`) files instead. ## From Ecto.Changeset.cast/4 msgid "can't be blank" -msgstr "" +msgstr "空にすることはできません" ## From Ecto.Changeset.unique_constraint/3 msgid "has already been taken" -msgstr "" +msgstr "すでに使用されています" ## From Ecto.Changeset.put_change/3 msgid "is invalid" @@ -37,7 +37,7 @@ msgstr "" ## From Ecto.Changeset.validate_format/3 msgid "has invalid format" -msgstr "" +msgstr "書式が正しくありません" ## From Ecto.Changeset.validate_subset/3 msgid "has an invalid entry" @@ -45,7 +45,7 @@ msgstr "" ## From Ecto.Changeset.validate_exclusion/3 msgid "is reserved" -msgstr "" +msgstr "予約されています" ## From Ecto.Changeset.validate_confirmation/3 msgid "does not match confirmation" @@ -61,23 +61,23 @@ msgstr "" ## From Ecto.Changeset.validate_length/3 msgid "should be %{count} character(s)" msgid_plural "should be %{count} character(s)" -msgstr[0] "" +msgstr[0] "%{count}文字である必要があります" msgid "should have %{count} item(s)" msgid_plural "should have %{count} item(s)" -msgstr[0] "" +msgstr[0] "%{count}個の要素を含む必要があります" msgid "should be at least %{count} character(s)" msgid_plural "should be at least %{count} character(s)" -msgstr[0] "" +msgstr[0] "%{count}文字以上である必要があります" msgid "should have at least %{count} item(s)" msgid_plural "should have at least %{count} item(s)" -msgstr[0] "" +msgstr[0] "%{count}個以上の要素を含む必要があります" msgid "should be at most %{count} character(s)" msgid_plural "should be at most %{count} character(s)" -msgstr[0] "" +msgstr[0] "%{count}文字以下である必要があります" msgid "should have at most %{count} item(s)" msgid_plural "should have at most %{count} item(s)" @@ -102,7 +102,7 @@ msgstr "%{number}でなければなりません" #: lib/pleroma/web/common_api/common_api.ex:505 #, elixir-format msgid "Account not found" -msgstr "アカウントがありません" +msgstr "アカウントが見つかりません" #: lib/pleroma/web/common_api/common_api.ex:339 #, elixir-format @@ -226,7 +226,7 @@ msgstr "" #: lib/pleroma/web/common_api/utils.ex:414 #, elixir-format msgid "Invalid password." -msgstr "" +msgstr "パスワードが間違っています。" #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:220 #, elixir-format @@ -236,7 +236,7 @@ msgstr "" #: lib/pleroma/web/twitter_api/twitter_api.ex:109 #, elixir-format msgid "Kocaptcha service unavailable" -msgstr "" +msgstr "Kocaptchaが利用できません" #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:112 #, elixir-format @@ -259,12 +259,12 @@ msgstr "" #: lib/pleroma/web/feed/user_controller.ex:71 lib/pleroma/web/ostatus/ostatus_controller.ex:143 #, elixir-format msgid "Not found" -msgstr "" +msgstr "見つかりません" #: lib/pleroma/web/common_api/common_api.ex:331 #, elixir-format msgid "Poll's author can't vote" -msgstr "" +msgstr "作成者は投票できません" #: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:37 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:49 @@ -284,17 +284,17 @@ msgstr "" #: lib/pleroma/web/common_api/activity_draft.ex:107 #, elixir-format msgid "The message visibility must be direct" -msgstr "" +msgstr "公開範囲は「ダイレクト」でなければいけません" #: lib/pleroma/web/common_api/utils.ex:573 #, elixir-format msgid "The status is over the character limit" -msgstr "" +msgstr "文字数制限を超過しています" #: lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex:31 #, elixir-format msgid "This resource requires authentication." -msgstr "" +msgstr "認証が必要です。" #: lib/pleroma/plugs/rate_limiter/rate_limiter.ex:206 #, elixir-format @@ -304,7 +304,7 @@ msgstr "" #: lib/pleroma/web/common_api/common_api.ex:356 #, elixir-format msgid "Too many choices" -msgstr "" +msgstr "選択肢が多すぎます" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:443 #, elixir-format @@ -320,7 +320,7 @@ msgstr "" #: lib/pleroma/web/oauth/oauth_controller.ex:308 #, elixir-format msgid "Your account is currently disabled" -msgstr "" +msgstr "あなたのアカウントは無効化されています" #: lib/pleroma/web/oauth/oauth_controller.ex:183 #: lib/pleroma/web/oauth/oauth_controller.ex:331 @@ -331,23 +331,23 @@ msgstr "" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:390 #, elixir-format msgid "can't read inbox of %{nickname} as %{as_nickname}" -msgstr "" +msgstr "%{nickname}のinboxを%{as_nickname}として閲覧することはできません" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:473 #, elixir-format msgid "can't update outbox of %{nickname} as %{as_nickname}" -msgstr "" +msgstr "%{nickname}のoutboxを%{as_nickname}として更新することはできません" #: lib/pleroma/web/common_api/common_api.ex:471 #, elixir-format msgid "conversation is already muted" -msgstr "" +msgstr "この会話はミュート済みです" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:314 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:492 #, elixir-format msgid "error" -msgstr "" +msgstr "エラー" #: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:32 #, elixir-format @@ -357,7 +357,7 @@ msgstr "" #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:62 #, elixir-format msgid "not found" -msgstr "" +msgstr "見つかりません" #: lib/pleroma/web/oauth/oauth_controller.ex:394 #, elixir-format @@ -382,28 +382,28 @@ msgstr "" #: lib/pleroma/web/oauth/oauth_controller.ex:410 #, elixir-format msgid "Failed to authenticate: %{message}." -msgstr "" +msgstr "認証に失敗しました: %{message}。" #: lib/pleroma/web/oauth/oauth_controller.ex:441 #, elixir-format msgid "Failed to set up user account." -msgstr "" +msgstr "ユーザーアカウントの設定に失敗しました。" #: lib/pleroma/plugs/oauth_scopes_plug.ex:38 #, elixir-format msgid "Insufficient permissions: %{permissions}." -msgstr "" +msgstr "%{permissions}の権限が必要です。" #: lib/pleroma/plugs/uploaded_media.ex:104 #, elixir-format msgid "Internal Error" -msgstr "" +msgstr "内部エラー" #: lib/pleroma/web/oauth/fallback_controller.ex:22 #: lib/pleroma/web/oauth/fallback_controller.ex:29 #, elixir-format msgid "Invalid Username/Password" -msgstr "" +msgstr "ユーザー名/パスワードが間違っています" #: lib/pleroma/web/twitter_api/twitter_api.ex:118 #, elixir-format @@ -423,7 +423,7 @@ msgstr "" #: lib/pleroma/web/oauth/fallback_controller.ex:14 #, elixir-format msgid "Unknown error, please check the details and try again." -msgstr "" +msgstr "不明なエラーです。詳細を確認し、再度試行してください。" #: lib/pleroma/web/oauth/oauth_controller.ex:119 #: lib/pleroma/web/oauth/oauth_controller.ex:158 @@ -449,17 +449,17 @@ msgstr "" #: lib/pleroma/web/twitter_api/twitter_api.ex:103 #, elixir-format msgid "CAPTCHA Error" -msgstr "" +msgstr "CAPTCHAエラー" #: lib/pleroma/web/common_api/common_api.ex:290 #, elixir-format msgid "Could not add reaction emoji" -msgstr "" +msgstr "リアクションを追加できません" #: lib/pleroma/web/common_api/common_api.ex:301 #, elixir-format msgid "Could not remove reaction emoji" -msgstr "" +msgstr "リアクションを解除できません" #: lib/pleroma/web/twitter_api/twitter_api.ex:129 #, elixir-format @@ -469,7 +469,7 @@ msgstr "" #: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:92 #, elixir-format msgid "List not found" -msgstr "" +msgstr "リストが見つかりません" #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:123 #, elixir-format @@ -480,7 +480,7 @@ msgstr "" #: lib/pleroma/web/oauth/oauth_controller.ex:321 #, elixir-format msgid "Password reset is required" -msgstr "" +msgstr "パスワードのリセットが必要です" #: lib/pleroma/tests/auth_test_controller.ex:9 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:6 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:6 @@ -522,7 +522,7 @@ msgstr "" #: lib/pleroma/plugs/ensure_authenticated_plug.ex:28 #, elixir-format msgid "Two-factor authentication enabled, you must use a access token." -msgstr "" +msgstr "二段階認証が有効になっています。アクセストークンが必要です。" #: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:210 #, elixir-format @@ -552,7 +552,7 @@ msgstr "" #: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:61 #, elixir-format msgid "Web push subscription is disabled on this Pleroma instance" -msgstr "" +msgstr "このPleromaインスタンスではプッシュ通知が利用できません" #: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:451 #, elixir-format @@ -562,19 +562,19 @@ msgstr "" #: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:126 #, elixir-format msgid "authorization required for timeline view" -msgstr "" +msgstr "タイムラインを閲覧するには認証が必要です" #: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:24 #, elixir-format msgid "Access denied" -msgstr "" +msgstr "アクセスが拒否されました" #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:282 #, elixir-format msgid "This API requires an authenticated user" -msgstr "" +msgstr "このAPIを利用するには認証が必要です" #: lib/pleroma/plugs/user_is_admin_plug.ex:21 #, elixir-format msgid "User is not an admin." -msgstr "" +msgstr "ユーザーは管理者ではありません。" From 7e99d0619d02835fab66134f14dd99fb4c313d26 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Mon, 12 Feb 2024 17:25:02 -0500 Subject: [PATCH 48/50] Force more frequent full_sweep GC runs on the Websocket processes Websocket processes seem to be the primary culprit for Binary memory allocation bloat. --- changelog.d/memleak.fix | 1 + lib/pleroma/web/endpoint.ex | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog.d/memleak.fix diff --git a/changelog.d/memleak.fix b/changelog.d/memleak.fix new file mode 100644 index 000000000..2465921c0 --- /dev/null +++ b/changelog.d/memleak.fix @@ -0,0 +1 @@ +Fix a memory leak caused by Websocket connections that would not enter a state where a full garbage collection run could be triggered. diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 51a77ed60..2e2104904 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -14,7 +14,8 @@ defmodule Pleroma.Web.Endpoint do websocket: [ path: "/", compress: false, - error_handler: {Pleroma.Web.MastodonAPI.WebsocketHandler, :handle_error, []} + error_handler: {Pleroma.Web.MastodonAPI.WebsocketHandler, :handle_error, []}, + fullsweep_after: 20 ] ) @@ -27,7 +28,8 @@ defmodule Pleroma.Web.Endpoint do ], timeout: 60_000, transport_log: false, - compress: false + compress: false, + fullsweep_after: 20 ], longpoll: false ) From 7d624c4750dcf53d48cc65874c832513f2b03fbc Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Tue, 20 Feb 2024 08:45:48 +0100 Subject: [PATCH 49/50] StealEmojiPolicy: Sanitize shortcodes Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3245 --- changelog.d/mrf_steal_emoji-path.security | 1 + .../activity_pub/mrf/steal_emoji_policy.ex | 2 ++ .../mrf/steal_emoji_policy_test.exs | 26 +++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 changelog.d/mrf_steal_emoji-path.security diff --git a/changelog.d/mrf_steal_emoji-path.security b/changelog.d/mrf_steal_emoji-path.security new file mode 100644 index 000000000..d880680d1 --- /dev/null +++ b/changelog.d/mrf_steal_emoji-path.security @@ -0,0 +1 @@ +StealEmojiPolicy: Sanitize shortcodes (thanks to Hazel K for the report) \ No newline at end of file diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 237dfefa5..fa6b595ea 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -36,6 +36,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do extension = if extension == "", do: ".png", else: extension + shortcode = Path.basename(shortcode) file_path = Path.join(emoji_dir_path, shortcode <> extension) case File.write(file_path, response.body) do @@ -78,6 +79,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do new_emojis = foreign_emojis |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end) + |> Enum.reject(fn {shortcode, _url} -> String.contains?(shortcode, ["/", "\\"]) end) |> Enum.filter(fn {shortcode, _url} -> reject_emoji? = [:mrf_steal_emoji, :rejected_shortcodes] diff --git a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs index c477a093d..2c7497da5 100644 --- a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs @@ -87,6 +87,32 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do assert File.exists?(fullpath) end + test "rejects invalid shortcodes", %{path: path} do + message = %{ + "type" => "Create", + "object" => %{ + "emoji" => [{"fired/fox", "https://example.org/emoji/firedfox"}], + "actor" => "https://example.org/users/admin" + } + } + + fullpath = Path.join(path, "fired/fox.png") + + Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")} + end) + + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) + + refute "firedfox" in installed() + refute File.exists?(path) + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + refute "fired/fox" in installed() + refute File.exists?(fullpath) + end + test "reject regex shortcode", %{message: message} do refute "firedfox" in installed() From e149ee6e225667ca964e30a08179ca84b8537aaf Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Tue, 20 Feb 2024 09:16:36 +0100 Subject: [PATCH 50/50] Mergeback of security release 2.6.2 --- CHANGELOG.md | 5 +++++ changelog.d/mergeback-2.6.2.skip | 0 changelog.d/mrf_steal_emoji-path.security | 1 - mix.exs | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changelog.d/mergeback-2.6.2.skip delete mode 100644 changelog.d/mrf_steal_emoji-path.security diff --git a/CHANGELOG.md b/CHANGELOG.md index e071fe693..063d51d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## 2.6.2 + +### Security +- MRF StealEmojiPolicy: Sanitize shortcodes (thanks to Hazel K for the report + ## 2.6.1 ### Changed - - Document maximum supported version of Erlang & Elixir diff --git a/changelog.d/mergeback-2.6.2.skip b/changelog.d/mergeback-2.6.2.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/mrf_steal_emoji-path.security b/changelog.d/mrf_steal_emoji-path.security deleted file mode 100644 index d880680d1..000000000 --- a/changelog.d/mrf_steal_emoji-path.security +++ /dev/null @@ -1 +0,0 @@ -StealEmojiPolicy: Sanitize shortcodes (thanks to Hazel K for the report) \ No newline at end of file diff --git a/mix.exs b/mix.exs index 582092aef..b707a20a8 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("2.6.51"), + version: version("2.6.52"), elixir: "~> 1.11", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(),