From 889dc17abd95bd1f414646e54d7e3cdadd9afbc9 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivant.business@gmail.com>
Date: Tue, 16 Jul 2019 19:18:30 +0300
Subject: [PATCH 01/40] [#1094] Rate-limited follow & unfollow actions.

---
 config/config.exs                                       | 2 ++
 docs/config.md                                          | 2 ++
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 8 ++++++++
 3 files changed, 12 insertions(+)

diff --git a/config/config.exs b/config/config.exs
index 7d539f994..03e0341c8 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -528,6 +528,8 @@ config :http_signatures,
 config :pleroma, :rate_limit,
   search: [{1000, 10}, {1000, 30}],
   app_account_creation: {1_800_000, 25},
+  relations_actions: {10_000, 10},
+  relation_id_action: {60_000, 2},
   statuses_actions: {10_000, 15},
   status_id_action: {60_000, 3}
 
diff --git a/docs/config.md b/docs/config.md
index 9a64f0ed7..f595caba5 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -647,5 +647,7 @@ Supported rate limiters:
 
 * `:search` for the search requests (account & status search etc.)
 * `:app_account_creation` for registering user accounts from the same IP address
+* `:relations_actions` for actions on relations with other users (follow, unfollow)
+* `:relation_id_action` for actions on relation with specific another user (follow, unfollow)
 * `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses
 * `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index f4aa576f7..a732a6990 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -47,6 +47,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   require Logger
 
+  @rate_limited_relations_actions ~w(follow unfollow)a
+
   @rate_limited_status_actions ~w(reblog_status unreblog_status fav_status unfav_status
     post_status delete_status)a
 
@@ -62,6 +64,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     when action in ~w(fav_status unfav_status)a
   )
 
+  plug(
+    RateLimiter,
+    {:relations_id_action, params: ["id", "uri"]} when action in @rate_limited_relations_actions
+  )
+
+  plug(RateLimiter, :relations_actions when action in @rate_limited_relations_actions)
   plug(RateLimiter, :statuses_actions when action in @rate_limited_status_actions)
   plug(RateLimiter, :app_account_creation when action == :account_register)
   plug(RateLimiter, :search when action in [:search, :search2, :account_search])

From 2ba07b63f4557554cd4acc63dc8e0424612554a0 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantbusiness@gmail.com>
Date: Tue, 16 Jul 2019 16:59:02 +0000
Subject: [PATCH 02/40] Apply suggestion to docs/config.md

---
 docs/config.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/config.md b/docs/config.md
index f595caba5..346b8cda2 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -647,7 +647,7 @@ Supported rate limiters:
 
 * `:search` for the search requests (account & status search etc.)
 * `:app_account_creation` for registering user accounts from the same IP address
-* `:relations_actions` for actions on relations with other users (follow, unfollow)
-* `:relation_id_action` for actions on relation with specific another user (follow, unfollow)
+* `:relations_actions` for actions on relations with all users (follow, unfollow)
+* `:relation_id_action` for actions on relation with a specific user (follow, unfollow)
 * `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses
 * `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user

From 18234cc44e6bc989e3e3cf15714c54b4fa05b9dd Mon Sep 17 00:00:00 2001
From: Sachin Joshi <satchin.joshi@gmail.com>
Date: Tue, 16 Jul 2019 22:37:36 +0545
Subject: [PATCH 03/40] add the rich media ttl based on image exp time

---
 CHANGELOG.md                                  |  1 +
 config/config.exs                             |  3 +-
 ..._set_richmedia_cache_ttl_based_on_image.md | 32 +++++++++++
 lib/pleroma/web/rich_media/parser.ex          | 41 ++++++++++++++
 .../rich_media/parsers/ttl/aws_signed_url.ex  | 54 +++++++++++++++++++
 test/fixtures/rich_media/amz.html             |  5 ++
 test/web/rich_media/aws_signed_url_test.exs   | 37 +++++++++++++
 7 files changed, 172 insertions(+), 1 deletion(-)
 create mode 100644 docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
 create mode 100644 lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
 create mode 100644 test/fixtures/rich_media/amz.html
 create mode 100644 test/web/rich_media/aws_signed_url_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f3630a1c5..4e58b0a9f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
 - Admin API: changed json structure for saving config settings.
 - RichMedia: parsers and their order are configured in `rich_media` config.
+- RichMedia: add the rich media ttl based on image expiration time.
 
 ## [1.0.1] - 2019-07-14
 ### Security
diff --git a/config/config.exs b/config/config.exs
index 7d539f994..aa5bd0da9 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -344,7 +344,8 @@ config :pleroma, :rich_media,
     Pleroma.Web.RichMedia.Parsers.TwitterCard,
     Pleroma.Web.RichMedia.Parsers.OGP,
     Pleroma.Web.RichMedia.Parsers.OEmbed
-  ]
+  ],
+  ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]
 
 config :pleroma, :media_proxy,
   enabled: false,
diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
new file mode 100644
index 000000000..489f9ece8
--- /dev/null
+++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
@@ -0,0 +1,32 @@
+# How to set rich media cache ttl based on image ttl
+## Explanation
+
+Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url.
+In such cases the old image url (expired) is returned from the media cache.
+
+So to avoid such situation we can define a moddule that will set ttl based no image.
+
+The module must have a `run` function and it should be registered in the config.
+
+### Example
+
+```exs
+defmodule MyModule do
+  def run(data, url) do
+    image_url = Map.get(data, :image)
+    # do some parsing in the url and get the ttl of the image
+    # ttl is unix time
+    ttl = parse_ttl_from_url(image_url)  
+    Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
+  end
+end
+```
+
+And update the config
+
+```exs
+config :pleroma, :rich_media,
+  ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl, MyModule]
+```
+
+> For reference there is a parser for AWS signed URL `Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl`, it's enabled by default.
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 0d2523338..ba8dc6f2a 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -24,6 +24,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
         Cachex.fetch!(:rich_media_cache, url, fn _ ->
           {:commit, parse_url(url)}
         end)
+        |> set_ttl_based_on_image(url)
       rescue
         e ->
           {:error, "Cachex error: #{inspect(e)}"}
@@ -31,6 +32,46 @@ defmodule Pleroma.Web.RichMedia.Parser do
     end
   end
 
+  @doc """
+  Set the rich media cache based on the expiration time of image.
+
+  Define a module that has `run` function
+
+  ## Example
+
+      defmodule MyModule do
+        def run(data, url) do
+          image_url = Map.get(data, :image)
+          # do some parsing in the url and get the ttl of the image
+          # ttl is unix time
+          ttl = parse_ttl_from_url(image_url)  
+          Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
+        end
+      end
+
+  Define the module in the config
+
+      config :pleroma, :rich_media,
+        ttl_setters: [MyModule]
+  """
+  def set_ttl_based_on_image({:ok, data}, url) do
+    case Cachex.ttl(:rich_media_cache, url) do
+      {:ok, nil} ->
+        modules = Pleroma.Config.get([:rich_media, :ttl_setters])
+
+        if Enum.count(modules) > 0 do
+          Enum.each(modules, & &1.run(data, url))
+        end
+
+        {:ok, data}
+
+      _ ->
+        {:ok, data}
+    end
+  end
+
+  def set_ttl_based_on_image(data, _url), do: data
+
   defp parse_url(url) do
     try do
       {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
new file mode 100644
index 000000000..d57107939
--- /dev/null
+++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
@@ -0,0 +1,54 @@
+defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
+  def run(data, url) do
+    image = Map.get(data, :image)
+
+    if is_aws_signed_url(image) do
+      image
+      |> parse_query_params()
+      |> format_query_params()
+      |> get_expiration_timestamp()
+      |> set_ttl(url)
+    end
+  end
+
+  defp is_aws_signed_url(""), do: nil
+  defp is_aws_signed_url(nil), do: nil
+
+  defp is_aws_signed_url(image) when is_binary(image) do
+    %URI{host: host, query: query} = URI.parse(image)
+
+    if String.contains?(host, "amazonaws.com") and
+         String.contains?(query, "X-Amz-Expires") do
+      image
+    else
+      nil
+    end
+  end
+
+  defp is_aws_signed_url(_), do: nil
+
+  defp parse_query_params(image) do
+    %URI{query: query} = URI.parse(image)
+    query
+  end
+
+  defp format_query_params(query) do
+    query
+    |> String.split(~r/&|=/)
+    |> Enum.chunk_every(2)
+    |> Map.new(fn [k, v] -> {k, v} end)
+  end
+
+  defp get_expiration_timestamp(params) when is_map(params) do
+    {:ok, date} =
+      params
+      |> Map.get("X-Amz-Date")
+      |> Timex.parse("{ISO:Basic:Z}")
+
+    Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))
+  end
+
+  defp set_ttl(ttl, url) do
+    Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
+  end
+end
diff --git a/test/fixtures/rich_media/amz.html b/test/fixtures/rich_media/amz.html
new file mode 100644
index 000000000..d4f8bd1a3
--- /dev/null
+++ b/test/fixtures/rich_media/amz.html
@@ -0,0 +1,5 @@
+<meta name="twitter:card" content="summary" />
+<meta name="twitter:site" content="@flickr" />
+<meta name="twitter:title" content="Small Island Developing States Photo Submission" />
+<meta name="twitter:description" content="View the album on Flickr." />
+<meta name="twitter:image" content="https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=20190716T175105Z&X-Amz-Expires=300000&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host" />
diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs
new file mode 100644
index 000000000..75bf6c6df
--- /dev/null
+++ b/test/web/rich_media/aws_signed_url_test.exs
@@ -0,0 +1,37 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do
+  use ExUnit.Case, async: true
+
+  test "amazon signed url is parsed and correct ttl is set for rich media" do
+    url = "https://pleroma.social/amz"
+
+    {:ok, timestamp} =
+      Timex.now()
+      |> DateTime.truncate(:second)
+      |> Timex.format("{ISO:Basic:Z}")
+
+    # in seconds
+    valid_till = 30
+
+    data = %{
+      image:
+        "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{
+          timestamp
+        }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host",
+      locale: "en_US",
+      site_name: "Pleroma",
+      title: "PLeroma",
+      url: url
+    }
+
+    Cachex.put(:rich_media_cache, url, data)
+    assert {:ok, _} = Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.run(data, url)
+    {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)
+
+    # as there is delay in setting and pulling the data from cache we ignore 1 second
+    assert_in_delta(valid_till * 1000, cache_ttl, 1000)
+  end
+end

From 21e3f9ac69e258d7c46e0f2acdcf011f74cba8e3 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Tue, 16 Jul 2019 21:35:43 +0000
Subject: [PATCH 04/40] added tests for Pleroma.Upload.Filter

---
 lib/pleroma/upload/filter/dedupe.ex   | 15 ++++++--
 lib/pleroma/upload/filter/mogrifun.ex | 24 ++-----------
 lib/pleroma/upload/filter/mogrify.ex  | 13 ++++---
 test/upload/filter/dedupe_test.exs    | 31 ++++++++++++++++
 test/upload/filter/mogrifun_test.exs  | 44 +++++++++++++++++++++++
 test/upload/filter/mogrify_test.exs   | 51 +++++++++++++++++++++++++++
 test/upload/filter_test.exs           | 39 ++++++++++++++++++++
 7 files changed, 187 insertions(+), 30 deletions(-)
 create mode 100644 test/upload/filter/dedupe_test.exs
 create mode 100644 test/upload/filter/mogrifun_test.exs
 create mode 100644 test/upload/filter/mogrify_test.exs
 create mode 100644 test/upload/filter_test.exs

diff --git a/lib/pleroma/upload/filter/dedupe.ex b/lib/pleroma/upload/filter/dedupe.ex
index e4c225833..14928c355 100644
--- a/lib/pleroma/upload/filter/dedupe.ex
+++ b/lib/pleroma/upload/filter/dedupe.ex
@@ -6,10 +6,19 @@ defmodule Pleroma.Upload.Filter.Dedupe do
   @behaviour Pleroma.Upload.Filter
   alias Pleroma.Upload
 
-  def filter(%Upload{name: name} = upload) do
-    extension = String.split(name, ".") |> List.last()
-    shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower)
+  def filter(%Upload{name: name, tempfile: tempfile} = upload) do
+    extension =
+      name
+      |> String.split(".")
+      |> List.last()
+
+    shasum =
+      :crypto.hash(:sha256, File.read!(tempfile))
+      |> Base.encode16(case: :lower)
+
     filename = shasum <> "." <> extension
     {:ok, %Upload{upload | id: shasum, path: filename}}
   end
+
+  def filter(_), do: :ok
 end
diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex
index 35a5a1381..fee49fb51 100644
--- a/lib/pleroma/upload/filter/mogrifun.ex
+++ b/lib/pleroma/upload/filter/mogrifun.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Upload.Filter.Mogrifun do
   @behaviour Pleroma.Upload.Filter
+  alias Pleroma.Upload.Filter
 
   @filters [
     {"implode", "1"},
@@ -34,31 +35,10 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
   ]
 
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
-    filter = Enum.random(@filters)
-
-    file
-    |> Mogrify.open()
-    |> mogrify_filter(filter)
-    |> Mogrify.save(in_place: true)
+    Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
 
     :ok
   end
 
   def filter(_), do: :ok
-
-  defp mogrify_filter(mogrify, [filter | rest]) do
-    mogrify
-    |> mogrify_filter(filter)
-    |> mogrify_filter(rest)
-  end
-
-  defp mogrify_filter(mogrify, []), do: mogrify
-
-  defp mogrify_filter(mogrify, {action, options}) do
-    Mogrify.custom(mogrify, action, options)
-  end
-
-  defp mogrify_filter(mogrify, string) when is_binary(string) do
-    Mogrify.custom(mogrify, string)
-  end
 end
diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex
index f459eeecb..91bfdd4f5 100644
--- a/lib/pleroma/upload/filter/mogrify.ex
+++ b/lib/pleroma/upload/filter/mogrify.ex
@@ -11,16 +11,19 @@ defmodule Pleroma.Upload.Filter.Mogrify do
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     filters = Pleroma.Config.get!([__MODULE__, :args])
 
-    file
-    |> Mogrify.open()
-    |> mogrify_filter(filters)
-    |> Mogrify.save(in_place: true)
-
+    do_filter(file, filters)
     :ok
   end
 
   def filter(_), do: :ok
 
+  def do_filter(file, filters) do
+    file
+    |> Mogrify.open()
+    |> mogrify_filter(filters)
+    |> Mogrify.save(in_place: true)
+  end
+
   defp mogrify_filter(mogrify, nil), do: mogrify
 
   defp mogrify_filter(mogrify, [filter | rest]) do
diff --git a/test/upload/filter/dedupe_test.exs b/test/upload/filter/dedupe_test.exs
new file mode 100644
index 000000000..fddd594dc
--- /dev/null
+++ b/test/upload/filter/dedupe_test.exs
@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.DedupeTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Upload
+  alias Pleroma.Upload.Filter.Dedupe
+
+  @shasum "e30397b58d226d6583ab5b8b3c5defb0c682bda5c31ef07a9f57c1c4986e3781"
+
+  test "adds shasum" do
+    File.cp!(
+      "test/fixtures/image.jpg",
+      "test/fixtures/image_tmp.jpg"
+    )
+
+    upload = %Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image_tmp.jpg"),
+      tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+    }
+
+    assert {
+             :ok,
+             %Pleroma.Upload{id: @shasum, path: "#{@shasum}.jpg"}
+           } = Dedupe.filter(upload)
+  end
+end
diff --git a/test/upload/filter/mogrifun_test.exs b/test/upload/filter/mogrifun_test.exs
new file mode 100644
index 000000000..d5a8751cc
--- /dev/null
+++ b/test/upload/filter/mogrifun_test.exs
@@ -0,0 +1,44 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.MogrifunTest do
+  use Pleroma.DataCase
+  import Mock
+
+  alias Pleroma.Upload
+  alias Pleroma.Upload.Filter
+
+  test "apply mogrify filter" do
+    File.cp!(
+      "test/fixtures/image.jpg",
+      "test/fixtures/image_tmp.jpg"
+    )
+
+    upload = %Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image_tmp.jpg"),
+      tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+    }
+
+    task =
+      Task.async(fn ->
+        assert_receive {:apply_filter, {}}, 4_000
+      end)
+
+    with_mocks([
+      {Mogrify, [],
+       [
+         open: fn _f -> %Mogrify.Image{} end,
+         custom: fn _m, _a -> send(task.pid, {:apply_filter, {}}) end,
+         custom: fn _m, _a, _o -> send(task.pid, {:apply_filter, {}}) end,
+         save: fn _f, _o -> :ok end
+       ]}
+    ]) do
+      assert Filter.Mogrifun.filter(upload) == :ok
+    end
+
+    Task.await(task)
+  end
+end
diff --git a/test/upload/filter/mogrify_test.exs b/test/upload/filter/mogrify_test.exs
new file mode 100644
index 000000000..c301440fd
--- /dev/null
+++ b/test/upload/filter/mogrify_test.exs
@@ -0,0 +1,51 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.MogrifyTest do
+  use Pleroma.DataCase
+  import Mock
+
+  alias Pleroma.Config
+  alias Pleroma.Upload
+  alias Pleroma.Upload.Filter
+
+  setup do
+    filter = Config.get([Filter.Mogrify, :args])
+
+    on_exit(fn ->
+      Config.put([Filter.Mogrify, :args], filter)
+    end)
+  end
+
+  test "apply mogrify filter" do
+    Config.put([Filter.Mogrify, :args], [{"tint", "40"}])
+
+    File.cp!(
+      "test/fixtures/image.jpg",
+      "test/fixtures/image_tmp.jpg"
+    )
+
+    upload = %Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image_tmp.jpg"),
+      tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+    }
+
+    task =
+      Task.async(fn ->
+        assert_receive {:apply_filter, {_, "tint", "40"}}, 4_000
+      end)
+
+    with_mock Mogrify,
+      open: fn _f -> %Mogrify.Image{} end,
+      custom: fn _m, _a -> :ok end,
+      custom: fn m, a, o -> send(task.pid, {:apply_filter, {m, a, o}}) end,
+      save: fn _f, _o -> :ok end do
+      assert Filter.Mogrify.filter(upload) == :ok
+    end
+
+    Task.await(task)
+  end
+end
diff --git a/test/upload/filter_test.exs b/test/upload/filter_test.exs
new file mode 100644
index 000000000..640cd7107
--- /dev/null
+++ b/test/upload/filter_test.exs
@@ -0,0 +1,39 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.FilterTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Config
+  alias Pleroma.Upload.Filter
+
+  setup do
+    custom_filename = Config.get([Pleroma.Upload.Filter.AnonymizeFilename, :text])
+
+    on_exit(fn ->
+      Config.put([Pleroma.Upload.Filter.AnonymizeFilename, :text], custom_filename)
+    end)
+  end
+
+  test "applies filters" do
+    Config.put([Pleroma.Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
+
+    File.cp!(
+      "test/fixtures/image.jpg",
+      "test/fixtures/image_tmp.jpg"
+    )
+
+    upload = %Pleroma.Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image_tmp.jpg"),
+      tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+    }
+
+    assert Filter.filter([], upload) == {:ok, upload}
+
+    assert {:ok, upload} = Filter.filter([Pleroma.Upload.Filter.AnonymizeFilename], upload)
+    assert upload.name == "custom-file.png"
+  end
+end

From 10f82c88b88fa4d26f6fa57f9cf36439012b8d0c Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 16 Jul 2019 21:44:50 +0000
Subject: [PATCH 05/40] mastoapi password reset

added rate limit to password reset

configure rate limit in runtime
---
 CHANGELOG.md                                  |  2 +
 config/config.exs                             |  3 +-
 config/test.exs                               |  3 +-
 .../mastodon_api/mastodon_api_controller.ex   | 17 ++++++
 lib/pleroma/web/router.ex                     |  2 +
 .../web/twitter_api/twitter_api_controller.ex |  7 +++
 .../mastodon_api_controller_test.exs          | 52 +++++++++++++++++++
 .../twitter_api_controller_test.exs           | 10 ++--
 8 files changed, 90 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f3630a1c5..c5632f273 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Add support for muting/unmuting notifications
 - Mastodon API: Add support for the `blocked_by` attribute in the relationship API (`GET /api/v1/accounts/relationships`). <https://github.com/tootsuite/mastodon/pull/10373>
 - Mastodon API: Add `pleroma.deactivated` to the Account entity
+- Mastodon API: added `/auth/password` endpoint for password reset with rate limit.
 - Admin API: Return users' tags when querying reports
 - Admin API: Return avatar and display name when querying users
 - Admin API: Allow querying user by ID
@@ -40,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `enabled` option for `Pleroma.Emails.Mailer`, defaulting to `false`.
 - Configuration: Pleroma.Plugs.RateLimiter `bucket_name`, `params` options.
 - Addressable lists
+- Twitter API: added rate limit for `/api/account/password_reset` endpoint.
 
 ### Changed
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
diff --git a/config/config.exs b/config/config.exs
index 03e0341c8..38780eef7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -531,7 +531,8 @@ config :pleroma, :rate_limit,
   relations_actions: {10_000, 10},
   relation_id_action: {60_000, 2},
   statuses_actions: {10_000, 15},
-  status_id_action: {60_000, 3}
+  status_id_action: {60_000, 3},
+  password_reset: {1_800_000, 5}
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
diff --git a/config/test.exs b/config/test.exs
index 96ecf3592..0af62aa14 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -67,7 +67,8 @@ config :pleroma, Pleroma.ScheduledActivity,
 
 config :pleroma, :rate_limit,
   search: [{1000, 30}, {1000, 30}],
-  app_account_creation: {10_000, 5}
+  app_account_creation: {10_000, 5},
+  password_reset: {1000, 30}
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index a732a6990..aff76e2ea 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -73,6 +73,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   plug(RateLimiter, :statuses_actions when action in @rate_limited_status_actions)
   plug(RateLimiter, :app_account_creation when action == :account_register)
   plug(RateLimiter, :search when action in [:search, :search2, :account_search])
+  plug(RateLimiter, :password_reset when action == :password_reset)
 
   @local_mastodon_name "Mastodon-Local"
 
@@ -1816,6 +1817,22 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
+  def password_reset(conn, params) do
+    nickname_or_email = params["email"] || params["nickname"]
+
+    with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do
+      conn
+      |> put_status(:no_content)
+      |> json("")
+    else
+      {:error, "unknown user"} ->
+        put_status(conn, :not_found)
+
+      {:error, _} ->
+        put_status(conn, :bad_request)
+    end
+  end
+
   def try_render(conn, target, params)
       when is_binary(target) do
     case render(conn, target, params) do
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 3e5142e8a..52b8dc0bf 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -691,6 +691,8 @@ defmodule Pleroma.Web.Router do
     get("/web/login", MastodonAPIController, :login)
     delete("/auth/sign_out", MastodonAPIController, :logout)
 
+    post("/auth/password", MastodonAPIController, :password_reset)
+
     scope [] do
       pipe_through(:oauth_read_or_public)
       get("/web/*path", MastodonAPIController, :index)
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 0313560a8..8cb703501 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -27,6 +27,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   require Logger
 
+  plug(Pleroma.Plugs.RateLimiter, :password_reset when action == :password_reset)
   plug(:only_if_public_instance when action in [:public_timeline, :public_and_external_timeline])
   action_fallback(:errors)
 
@@ -437,6 +438,12 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
     with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do
       json_response(conn, :no_content, "")
+    else
+      {:error, "unknown user"} ->
+        put_status(conn, :not_found)
+
+      {:error, _} ->
+        put_status(conn, :bad_request)
     end
   end
 
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index 85b4ad024..d9d8dafdb 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -23,6 +23,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
   import Pleroma.Factory
   import ExUnit.CaptureLog
   import Tesla.Mock
+  import Swoosh.TestAssertions
 
   @image "data:image/gif;base64,R0lGODlhEAAQAMQAAORHHOVSKudfOulrSOp3WOyDZu6QdvCchPGolfO0o/XBs/fNwfjZ0frl3/zy7////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH5BAkAABAALAAAAAAQABAAAAVVICSOZGlCQAosJ6mu7fiyZeKqNKToQGDsM8hBADgUXoGAiqhSvp5QAnQKGIgUhwFUYLCVDFCrKUE1lBavAViFIDlTImbKC5Gm2hB0SlBCBMQiB0UjIQA7"
 
@@ -3807,4 +3808,55 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
       assert Enum.empty?(response)
     end
   end
+
+  describe "POST /auth/password, with valid parameters" do
+    setup %{conn: conn} do
+      user = insert(:user)
+      conn = post(conn, "/auth/password?email=#{user.email}")
+      %{conn: conn, user: user}
+    end
+
+    test "it returns 204", %{conn: conn} do
+      assert json_response(conn, :no_content)
+    end
+
+    test "it creates a PasswordResetToken record for user", %{user: user} do
+      token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id)
+      assert token_record
+    end
+
+    test "it sends an email to user", %{user: user} do
+      token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id)
+
+      email = Pleroma.Emails.UserEmail.password_reset_email(user, token_record.token)
+      notify_email = Pleroma.Config.get([:instance, :notify_email])
+      instance_name = Pleroma.Config.get([:instance, :name])
+
+      assert_email_sent(
+        from: {instance_name, notify_email},
+        to: {user.name, user.email},
+        html_body: email.html_body
+      )
+    end
+  end
+
+  describe "POST /auth/password, with invalid parameters" do
+    setup do
+      user = insert(:user)
+      {:ok, user: user}
+    end
+
+    test "it returns 404 when user is not found", %{conn: conn, user: user} do
+      conn = post(conn, "/auth/password?email=nonexisting_#{user.email}")
+      assert conn.status == 404
+      refute conn.resp_body
+    end
+
+    test "it returns 400 when user is not local", %{conn: conn, user: user} do
+      {:ok, user} = Repo.update(Changeset.change(user, local: false))
+      conn = post(conn, "/auth/password?email=#{user.email}")
+      assert conn.status == 400
+      refute conn.resp_body
+    end
+  end
 end
diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs
index de6177575..622bf510e 100644
--- a/test/web/twitter_api/twitter_api_controller_test.exs
+++ b/test/web/twitter_api/twitter_api_controller_test.exs
@@ -1116,15 +1116,17 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
   describe "POST /api/account/password_reset, with invalid parameters" do
     setup [:valid_user]
 
-    test "it returns 500 when user is not found", %{conn: conn, user: user} do
+    test "it returns 404 when user is not found", %{conn: conn, user: user} do
       conn = post(conn, "/api/account/password_reset?email=nonexisting_#{user.email}")
-      assert json_response(conn, :internal_server_error)
+      assert conn.status == 404
+      refute conn.resp_body
     end
 
-    test "it returns 500 when user is not local", %{conn: conn, user: user} do
+    test "it returns 400 when user is not local", %{conn: conn, user: user} do
       {:ok, user} = Repo.update(Changeset.change(user, local: false))
       conn = post(conn, "/api/account/password_reset?email=#{user.email}")
-      assert json_response(conn, :internal_server_error)
+      assert conn.status == 400
+      refute conn.resp_body
     end
   end
 

From 96a2890a9ecca3a6392edfaaaed4487303a920d7 Mon Sep 17 00:00:00 2001
From: RX14 <steph@rx14.co.uk>
Date: Wed, 17 Jul 2019 14:55:47 +0100
Subject: [PATCH 06/40] Add MRF MentionPolicy for dropping posts which mention
 specific actors

---
 CHANGELOG.md                                  |  1 +
 docs/config.md                                |  4 +
 .../web/activity_pub/mrf/mention_policy.ex    | 24 +++++
 .../activity_pub/mrf/mention_policy_test.exs  | 92 +++++++++++++++++++
 4 files changed, 121 insertions(+)
 create mode 100644 lib/pleroma/web/activity_pub/mrf/mention_policy.ex
 create mode 100644 test/web/activity_pub/mrf/mention_policy_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f3630a1c5..0bc72a3fc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`)
 - MRF: Support for excluding specific domains from Transparency.
+- MRF: Support for filtering posts based on who they mention (`Pleroma.Web.ActivityPub.MRF.MentionPolicy`)
 - Configuration: `federation_incoming_replies_max_depth` option
 - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses)
 - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header
diff --git a/docs/config.md b/docs/config.md
index 9a64f0ed7..5f69f8daf 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -101,6 +101,7 @@ config :pleroma, Pleroma.Emails.Mailer,
   * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:.
   * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links.
   * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed.
+  * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (see `:mrf_mention` section)
 * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network.
 * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send.
 * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json``
@@ -271,6 +272,9 @@ config :pleroma, :mrf_subchain,
 * `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html)
 * `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html)
 
+## :mrf_mention
+* `actors`: A list of actors, for which to drop any posts mentioning.
+
 ## :media_proxy
 * `enabled`: Enables proxying of remote media to the instance’s proxy
 * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
new file mode 100644
index 000000000..1842e1aeb
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do
+  @moduledoc "Block messages which mention a user"
+
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  @impl true
+  def filter(%{"type" => "Create"} = message) do
+    reject_actors = Pleroma.Config.get([:mrf_mention, :actors], [])
+    recipients = (message["to"] || []) ++ (message["cc"] || [])
+
+    if Enum.any?(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do
+      {:reject, nil}
+    else
+      {:ok, message}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end
diff --git a/test/web/activity_pub/mrf/mention_policy_test.exs b/test/web/activity_pub/mrf/mention_policy_test.exs
new file mode 100644
index 000000000..9fd9c31df
--- /dev/null
+++ b/test/web/activity_pub/mrf/mention_policy_test.exs
@@ -0,0 +1,92 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicyTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Web.ActivityPub.MRF.MentionPolicy
+
+  test "pass filter if allow list is empty" do
+    Pleroma.Config.delete([:mrf_mention])
+
+    message = %{
+      "type" => "Create",
+      "to" => ["https://example.com/ok"],
+      "cc" => ["https://example.com/blocked"]
+    }
+
+    assert MentionPolicy.filter(message) == {:ok, message}
+  end
+
+  describe "allow" do
+    test "empty" do
+      Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]})
+
+      message = %{
+        "type" => "Create"
+      }
+
+      assert MentionPolicy.filter(message) == {:ok, message}
+    end
+
+    test "to" do
+      Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]})
+
+      message = %{
+        "type" => "Create",
+        "to" => ["https://example.com/ok"]
+      }
+
+      assert MentionPolicy.filter(message) == {:ok, message}
+    end
+
+    test "cc" do
+      Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]})
+
+      message = %{
+        "type" => "Create",
+        "cc" => ["https://example.com/ok"]
+      }
+
+      assert MentionPolicy.filter(message) == {:ok, message}
+    end
+
+    test "both" do
+      Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]})
+
+      message = %{
+        "type" => "Create",
+        "to" => ["https://example.com/ok"],
+        "cc" => ["https://example.com/ok2"]
+      }
+
+      assert MentionPolicy.filter(message) == {:ok, message}
+    end
+  end
+
+  describe "deny" do
+    test "to" do
+      Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]})
+
+      message = %{
+        "type" => "Create",
+        "to" => ["https://example.com/blocked"]
+      }
+
+      assert MentionPolicy.filter(message) == {:reject, nil}
+    end
+
+    test "cc" do
+      Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]})
+
+      message = %{
+        "type" => "Create",
+        "to" => ["https://example.com/ok"],
+        "cc" => ["https://example.com/blocked"]
+      }
+
+      assert MentionPolicy.filter(message) == {:reject, nil}
+    end
+  end
+end

From 4885473be2704d4d370fdb96e1473bc4eb9368f4 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 15:48:51 +0000
Subject: [PATCH 07/40] user: refactor get_or_create_instance_user() into
 get_or_create_service_actor_by_id()

---
 lib/pleroma/user.ex                   | 13 ++++++-------
 lib/pleroma/web/activity_pub/relay.ex |  3 ++-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index ffba3f390..463bb9ad4 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1157,19 +1157,18 @@ defmodule Pleroma.User do
     end
   end
 
-  def get_or_create_instance_user do
-    relay_uri = "#{Pleroma.Web.Endpoint.url()}/relay"
-
-    if user = get_cached_by_ap_id(relay_uri) do
+  @doc "Creates an internal service actor by URI if missing.  Optionally takes nickname for addressing."
+  def get_or_create_service_actor_by_ap_id(uri, nickname \\ nil) do
+    if user = get_cached_by_ap_id(uri) do
       user
     else
       changes =
         %User{info: %User.Info{}}
         |> cast(%{}, [:ap_id, :nickname, :local])
-        |> put_change(:ap_id, relay_uri)
-        |> put_change(:nickname, nil)
+        |> put_change(:ap_id, uri)
+        |> put_change(:nickname, nickname)
         |> put_change(:local, true)
-        |> put_change(:follower_address, relay_uri <> "/followers")
+        |> put_change(:follower_address, uri <> "/followers")
 
       {:ok, user} = Repo.insert(changes)
       user
diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex
index 93808517b..1ebfcdd86 100644
--- a/lib/pleroma/web/activity_pub/relay.ex
+++ b/lib/pleroma/web/activity_pub/relay.ex
@@ -10,7 +10,8 @@ defmodule Pleroma.Web.ActivityPub.Relay do
   require Logger
 
   def get_actor do
-    User.get_or_create_instance_user()
+    "#{Pleroma.Web.Endpoint.url()}/relay"
+    |> User.get_or_create_service_actor_by_ap_id()
   end
 
   def follow(target_instance) do

From a9d6a12bb3e71bafc11fe7cf5e44ad5bc9981cf9 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 16:22:57 +0000
Subject: [PATCH 08/40] activitypub: controller: rework the way the relay actor
 is presented so the code can be reused

---
 .../web/activity_pub/activity_pub_controller.ex      | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index e2af4ad1a..dc9ef066d 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -206,9 +206,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     json(conn, dgettext("errors", "error"))
   end
 
-  def relay(conn, _params) do
-    with %User{} = user <- Relay.get_actor(),
-         {:ok, user} <- User.ensure_keys_present(user) do
+  defp represent_service_actor(%User{} = user, conn) do
+    with {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("user.json", %{user: user}))
@@ -217,6 +216,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
+  defp represent_service_actor(nil, _), do: {:error, :not_found}
+
+  def relay(conn, _params) do
+    Relay.get_actor()
+    |> represent_service_actor(conn)
+  end
+
   def whoami(%{assigns: %{user: %User{} = user}} = conn, _params) do
     conn
     |> put_resp_header("content-type", "application/activity+json")

From 0a6f6e1b5bf863fc23a90e6ef358129364bcacce Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 16:59:29 +0000
Subject: [PATCH 09/40] webfinger: allow resolution of usernames with dots in
 them (internal actors)

---
 lib/pleroma/web/web_finger/web_finger.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index 3fca72de8..fa34c7ced 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -32,7 +32,7 @@ defmodule Pleroma.Web.WebFinger do
 
   def webfinger(resource, fmt) when fmt in ["XML", "JSON"] do
     host = Pleroma.Web.Endpoint.host()
-    regex = ~r/(acct:)?(?<username>\w+)@#{host}/
+    regex = ~r/(acct:)?(?<username>[a-z0-9A-Z_\.-]+)@#{host}/
 
     with %{"username" => username} <- Regex.named_captures(regex, resource),
          %User{} = user <- User.get_cached_by_nickname(username) do

From 62e5ff624e1984b48eecaf2a6c14ad092013a13e Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 17:12:42 +0000
Subject: [PATCH 10/40] user: add is_internal_user? helper function

---
 lib/pleroma/user.ex |  4 ++++
 test/user_test.exs  | 17 +++++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 463bb9ad4..c91fbb68a 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1410,4 +1410,8 @@ defmodule Pleroma.User do
   end
 
   defp put_password_hash(changeset), do: changeset
+
+  def is_internal_user?(%User{nickname: nil}), do: true
+  def is_internal_user?(%User{local: true, nickname: "internal." <> _}), do: true
+  def is_internal_user?(_), do: false
 end
diff --git a/test/user_test.exs b/test/user_test.exs
index 264b7a40e..908f72a0e 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1310,4 +1310,21 @@ defmodule Pleroma.UserTest do
       assert following == 0
     end
   end
+
+  describe "is_internal_user?/1" do
+    test "non-internal user returns false" do
+      user = insert(:user)
+      refute User.is_internal_user?(user)
+    end
+
+    test "user with no nickname returns true" do
+      user = insert(:user, %{nickname: nil})
+      assert User.is_internal_user?(user)
+    end
+
+    test "user with internal-prefixed nickname returns true" do
+      user = insert(:user, %{nickname: "internal.test"})
+      assert User.is_internal_user?(user)
+    end
+  end
 end

From d930e5d5c322a7c4b3a080dfa3e318d97994aaf1 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 17:14:08 +0000
Subject: [PATCH 11/40] activitypub: introduce internal fetch service actor

---
 lib/pleroma/application.ex                    |  5 +++++
 .../web/activity_pub/internal_fetch_actor.ex  | 20 +++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 lib/pleroma/web/activity_pub/internal_fetch_actor.ex

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index ba4cf8486..035331491 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -140,6 +140,11 @@ defmodule Pleroma.Application do
             id: :federator_init,
             start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
             restart: :temporary
+          },
+          %{
+            id: :internal_fetch_init,
+            start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
+            restart: :temporary
           }
         ] ++
         streamer_child() ++
diff --git a/lib/pleroma/web/activity_pub/internal_fetch_actor.ex b/lib/pleroma/web/activity_pub/internal_fetch_actor.ex
new file mode 100644
index 000000000..9213ddde7
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/internal_fetch_actor.ex
@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.InternalFetchActor do
+  alias Pleroma.User
+
+  require Logger
+
+  def init do
+    # Wait for everything to settle.
+    Process.sleep(1000 * 5)
+    get_actor()
+  end
+
+  def get_actor do
+    "#{Pleroma.Web.Endpoint.url()}/internal/fetch"
+    |> User.get_or_create_service_actor_by_ap_id("internal.fetch")
+  end
+end

From cf9cb953d5c341bb13e4b86272ff4ef405aeab92 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 17:34:57 +0000
Subject: [PATCH 12/40] activitypub: represent internal fetch actor

---
 .../web/activity_pub/activity_pub_controller.ex     |  6 ++++++
 lib/pleroma/web/activity_pub/views/user_view.ex     | 13 ++++++++++---
 lib/pleroma/web/router.ex                           | 13 +++++++++++--
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index dc9ef066d..133a726c5 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Object.Fetcher
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.ActivityPub.ObjectView
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -223,6 +224,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> represent_service_actor(conn)
   end
 
+  def internal_fetch(conn, _params) do
+    InternalFetchActor.get_actor()
+    |> represent_service_actor(conn)
+  end
+
   def whoami(%{assigns: %{user: %User{} = user}} = conn, _params) do
     conn
     |> put_resp_header("content-type", "application/activity+json")
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index d9c1bcb2c..639519e0a 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -31,8 +31,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
 
   def render("endpoints.json", _), do: %{}
 
-  # the instance itself is not a Person, but instead an Application
-  def render("user.json", %{user: %{nickname: nil} = user}) do
+  def render("service.json", %{user: user}) do
     {:ok, user} = User.ensure_keys_present(user)
     {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys)
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
@@ -47,7 +46,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "followers" => "#{user.ap_id}/followers",
       "inbox" => "#{user.ap_id}/inbox",
       "name" => "Pleroma",
-      "summary" => "Virtual actor for Pleroma relay",
+      "summary" =>
+        "An internal service actor for this Pleroma instance.  No user-serviceable parts inside.",
       "url" => user.ap_id,
       "manuallyApprovesFollowers" => false,
       "publicKey" => %{
@@ -60,6 +60,13 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     |> Map.merge(Utils.make_json_ld_header())
   end
 
+  # the instance itself is not a Person, but instead an Application
+  def render("user.json", %{user: %User{nickname: nil} = user}),
+    do: render("service.json", %{user: user})
+
+  def render("user.json", %{user: %User{nickname: "internal." <> _} = user}),
+    do: render("service.json", %{user: user})
+
   def render("user.json", %{user: user}) do
     {:ok, user} = User.ensure_keys_present(user)
     {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 52b8dc0bf..8095ac4b1 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -586,7 +586,7 @@ defmodule Pleroma.Web.Router do
     end
   end
 
-  pipeline :ap_relay do
+  pipeline :ap_service_actor do
     plug(:accepts, ["activity+json", "json"])
   end
 
@@ -663,8 +663,17 @@ defmodule Pleroma.Web.Router do
   end
 
   scope "/relay", Pleroma.Web.ActivityPub do
-    pipe_through(:ap_relay)
+    pipe_through(:ap_service_actor)
+
     get("/", ActivityPubController, :relay)
+    post("/inbox", ActivityPubController, :inbox)
+  end
+
+  scope "/internal/fetch", Pleroma.Web.ActivityPub do
+    pipe_through(:ap_service_actor)
+
+    get("/", ActivityPubController, :internal_fetch)
+    post("/inbox", ActivityPubController, :inbox)
   end
 
   scope "/", Pleroma.Web.ActivityPub do

From 3d23a12d75fc159d3ec25424245847fe703b7bd6 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 17:48:08 +0000
Subject: [PATCH 13/40] tests: add test for fetching the internal fetch actor

---
 .../web/activity_pub/activity_pub_controller_test.exs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index 452172bb4..40344f17e 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -48,6 +48,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
     end
   end
 
+  describe "/internal/fetch" do
+    test "it returns the internal fetch user", %{conn: conn} do
+      res =
+        conn
+        |> get(activity_pub_path(conn, :internal_fetch))
+        |> json_response(200)
+
+      assert res["id"] =~ "/fetch"
+    end
+  end
+
   describe "/users/:nickname" do
     test "it returns a json representation of the user with accept application/json", %{
       conn: conn

From c1198351022ead54b8de3bc535df32e333eb9b4d Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 17:49:21 +0000
Subject: [PATCH 14/40] update changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d91ad817..654d208dd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: Pleroma.Plugs.RateLimiter `bucket_name`, `params` options.
 - Addressable lists
 - Twitter API: added rate limit for `/api/account/password_reset` endpoint.
+- ActivityPub: Add an internal service actor for fetching ActivityPub objects.
 
 ### Changed
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text

From 4bf2bb9cff2d263e1b022f5c40128ffcbd372746 Mon Sep 17 00:00:00 2001
From: Eugenij <eugenijm@protonmail.com>
Date: Wed, 17 Jul 2019 18:09:31 +0000
Subject: [PATCH 15/40] Fix password reset for non-test env

Fixes `Plug.Conn.NotSentError` that causes a 5xx error in response
instead of 404 and 400.

Fixes pattern matching error caused by different response format
in test and non-test env: `Pleroma.Emails.Mailer.deliver_async` returns
:ok when PleromaJobQueue is enabled and `{:ok, _}` when it's disabled.
In tests, it's disabled.
---
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 ++--
 lib/pleroma/web/twitter_api/twitter_api.ex              | 2 ++
 lib/pleroma/web/twitter_api/twitter_api_controller.ex   | 4 ++--
 test/web/mastodon_api/mastodon_api_controller_test.exs  | 4 ++--
 test/web/twitter_api/twitter_api_controller_test.exs    | 4 ++--
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index aff76e2ea..877430a1d 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1826,10 +1826,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> json("")
     else
       {:error, "unknown user"} ->
-        put_status(conn, :not_found)
+        send_resp(conn, :not_found, "")
 
       {:error, _} ->
-        put_status(conn, :bad_request)
+        send_resp(conn, :bad_request, "")
     end
   end
 
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index 41e1c2877..bb5dda204 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -221,6 +221,8 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
       user
       |> UserEmail.password_reset_email(token_record.token)
       |> Mailer.deliver_async()
+
+      {:ok, :enqueued}
     else
       false ->
         {:error, "bad user identifier"}
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 8cb703501..5dfab6a6c 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -440,10 +440,10 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
       json_response(conn, :no_content, "")
     else
       {:error, "unknown user"} ->
-        put_status(conn, :not_found)
+        send_resp(conn, :not_found, "")
 
       {:error, _} ->
-        put_status(conn, :bad_request)
+        send_resp(conn, :bad_request, "")
     end
   end
 
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index d9d8dafdb..b4b1dd785 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -3849,14 +3849,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
     test "it returns 404 when user is not found", %{conn: conn, user: user} do
       conn = post(conn, "/auth/password?email=nonexisting_#{user.email}")
       assert conn.status == 404
-      refute conn.resp_body
+      assert conn.resp_body == ""
     end
 
     test "it returns 400 when user is not local", %{conn: conn, user: user} do
       {:ok, user} = Repo.update(Changeset.change(user, local: false))
       conn = post(conn, "/auth/password?email=#{user.email}")
       assert conn.status == 400
-      refute conn.resp_body
+      assert conn.resp_body == ""
     end
   end
 end
diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs
index 622bf510e..8bb8aa36d 100644
--- a/test/web/twitter_api/twitter_api_controller_test.exs
+++ b/test/web/twitter_api/twitter_api_controller_test.exs
@@ -1119,14 +1119,14 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
     test "it returns 404 when user is not found", %{conn: conn, user: user} do
       conn = post(conn, "/api/account/password_reset?email=nonexisting_#{user.email}")
       assert conn.status == 404
-      refute conn.resp_body
+      assert conn.resp_body == ""
     end
 
     test "it returns 400 when user is not local", %{conn: conn, user: user} do
       {:ok, user} = Repo.update(Changeset.change(user, local: false))
       conn = post(conn, "/api/account/password_reset?email=#{user.email}")
       assert conn.status == 400
-      refute conn.resp_body
+      assert conn.resp_body == ""
     end
   end
 

From 1e3aff6ef18c774783c4fc7eb46c245e7d8fb9b2 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 19:00:26 +0000
Subject: [PATCH 16/40] mix: bump http_signatures dependency to
 a2a5982fa167fb1352fbd518ce6b606ba233a989

---
 mix.exs  | 2 +-
 mix.lock | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index a26bb6202..e4ea19bb6 100644
--- a/mix.exs
+++ b/mix.exs
@@ -138,7 +138,7 @@ defmodule Pleroma.Mixfile do
        ref: "95e8188490e97505c56636c1379ffdf036c1fdde"},
       {:http_signatures,
        git: "https://git.pleroma.social/pleroma/http_signatures.git",
-       ref: "9789401987096ead65646b52b5a2ca6bf52fc531"},
+       ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"},
       {:pleroma_job_queue, "~> 0.2.0"},
       {:telemetry, "~> 0.3"},
       {:prometheus_ex, "~> 3.0"},
diff --git a/mix.lock b/mix.lock
index dcbf80f01..6477c1ed5 100644
--- a/mix.lock
+++ b/mix.lock
@@ -38,7 +38,7 @@
   "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
   "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"},
   "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
-  "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "9789401987096ead65646b52b5a2ca6bf52fc531", [ref: "9789401987096ead65646b52b5a2ca6bf52fc531"]},
+  "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "a2a5982fa167fb1352fbd518ce6b606ba233a989", [ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"]},
   "httpoison": {:hex, :httpoison, "1.2.0", "2702ed3da5fd7a8130fc34b11965c8cfa21ade2f232c00b42d96d4967c39a3a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
   "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},

From f84fb340b7358df195734f2db199e76a819e45bf Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 19:18:19 +0000
Subject: [PATCH 17/40] http signatures: derive actor ID from key ID.

Almost all AP servers return their key ID as the actor URI with #main-key
added.  Hubzilla, which doesn't, uses a URL which refers to the actor
anyway, so worst case, Hubzilla users get refetched.
---
 lib/pleroma/signature.ex | 13 ++++++++++---
 test/signature_test.exs  | 16 ++++++++++------
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index 1a4d54c62..a45c70a9d 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -8,10 +8,16 @@ defmodule Pleroma.Signature do
   alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Utils
+
+  defp key_id_to_actor_id(key_id) do
+    URI.parse(key_id)
+    |> Map.put(:fragment, nil)
+    |> URI.to_string()
+  end
 
   def fetch_public_key(conn) do
-    with actor_id <- Utils.get_ap_id(conn.params["actor"]),
+    with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
+         actor_id <- key_id_to_actor_id(kid),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
@@ -21,7 +27,8 @@ defmodule Pleroma.Signature do
   end
 
   def refetch_public_key(conn) do
-    with actor_id <- Utils.get_ap_id(conn.params["actor"]),
+    with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
+         actor_id <- key_id_to_actor_id(kid),
          {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
diff --git a/test/signature_test.exs b/test/signature_test.exs
index 4920196c7..840987cd6 100644
--- a/test/signature_test.exs
+++ b/test/signature_test.exs
@@ -31,25 +31,29 @@ defmodule Pleroma.SignatureTest do
     65_537
   }
 
+  defp make_fake_signature(key_id), do: "keyId=\"#{key_id}\""
+
+  defp make_fake_conn(key_id),
+    do: %Plug.Conn{req_headers: %{"signature" => make_fake_signature(key_id <> "#main-key")}}
+
   describe "fetch_public_key/1" do
     test "it returns key" do
       expected_result = {:ok, @rsa_public_key}
 
       user = insert(:user, %{info: %{source_data: %{"publicKey" => @public_key}}})
 
-      assert Signature.fetch_public_key(%Plug.Conn{params: %{"actor" => user.ap_id}}) ==
-               expected_result
+      assert Signature.fetch_public_key(make_fake_conn(user.ap_id)) == expected_result
     end
 
     test "it returns error when not found user" do
-      assert Signature.fetch_public_key(%Plug.Conn{params: %{"actor" => "test-ap_id"}}) ==
+      assert Signature.fetch_public_key(make_fake_conn("test-ap_id")) ==
                {:error, :error}
     end
 
     test "it returns error if public key is empty" do
       user = insert(:user, %{info: %{source_data: %{"publicKey" => %{}}}})
 
-      assert Signature.fetch_public_key(%Plug.Conn{params: %{"actor" => user.ap_id}}) ==
+      assert Signature.fetch_public_key(make_fake_conn(user.ap_id)) ==
                {:error, :error}
     end
   end
@@ -58,12 +62,12 @@ defmodule Pleroma.SignatureTest do
     test "it returns key" do
       ap_id = "https://mastodon.social/users/lambadalambda"
 
-      assert Signature.refetch_public_key(%Plug.Conn{params: %{"actor" => ap_id}}) ==
+      assert Signature.refetch_public_key(make_fake_conn(ap_id)) ==
                {:ok, @rsa_public_key}
     end
 
     test "it returns error when not found user" do
-      assert Signature.refetch_public_key(%Plug.Conn{params: %{"actor" => "test-ap_id"}}) ==
+      assert Signature.refetch_public_key(make_fake_conn("test-ap_id")) ==
                {:error, {:error, :ok}}
     end
   end

From b2a8ccf37fac6e40e55ce9b224c66ef1fd655614 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 21:38:06 +0000
Subject: [PATCH 18/40] config: add sign_object_fetches option

---
 config/config.exs | 3 ++-
 docs/config.md    | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 38780eef7..bda92e45d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -305,7 +305,8 @@ config :pleroma, :activitypub,
   accept_blocks: true,
   unfollow_blocked: true,
   outgoing_blocks: true,
-  follow_handshake_timeout: 500
+  follow_handshake_timeout: 500,
+  sign_object_fetches: true
 
 config :pleroma, :user, deny_follow_blocked: true
 
diff --git a/docs/config.md b/docs/config.md
index 42556a0be..02f86dc16 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -332,6 +332,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 * ``unfollow_blocked``: Whether blocks result in people getting unfollowed
 * ``outgoing_blocks``: Whether to federate blocks to other instances
 * ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question
+* ``sign_object_fetches``: Sign object fetches with HTTP signatures
 
 ## :http_security
 * ``enabled``: Whether the managed content security policy is enabled

From cdecef2a084eca629d2c6f72d083362a2113dbd2 Mon Sep 17 00:00:00 2001
From: "Bradley D. Thornton" <bradley@northtech.us>
Date: Wed, 17 Jul 2019 22:35:16 +0000
Subject: [PATCH 19/40] Update otp_en.md - Added creation of initial admin user

---
 docs/installation/otp_en.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 9b851e395..5b50e1838 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -242,6 +242,14 @@ So for example, if the task is `mix pleroma.user set admin --admin`, you should
 ```sh
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl user set admin --admin"
 ```
+
+## Create your first user and set as admin
+```sh
+cd /opt/pleroma/bin
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin"
+```
+This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password.
+
 ### Updating
 Generally, doing the following is enough:
 ```sh

From 399acd4c42ae3f1c072c097ff9fabc9761c3d3c1 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 22:41:42 +0000
Subject: [PATCH 20/40] fetcher: sign object fetches if configured

---
 lib/pleroma/object/fetcher.ex | 49 +++++++++++++++++++++++++++++++----
 1 file changed, 44 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 96b34ae9f..305ce8357 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -6,6 +6,8 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.HTTP
   alias Pleroma.Object
   alias Pleroma.Object.Containment
+  alias Pleroma.Signature
+  alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.OStatus
 
@@ -82,15 +84,52 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
+  defp make_signature(id, date) do
+    uri = URI.parse(id)
+
+    signature =
+      InternalFetchActor.get_actor()
+      |> Signature.sign(%{
+        "(request-target)": "get #{uri.path}",
+        host: uri.host,
+        date: date
+      })
+
+    [{:Signature, signature}]
+  end
+
+  defp sign_fetch(headers, id, date) do
+    if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
+      headers ++ make_signature(id, date)
+    else
+      headers
+    end
+  end
+
+  defp maybe_date_fetch(headers, date) do
+    if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
+      headers ++ [{:Date, date}]
+    else
+      headers
+    end
+  end
+
   def fetch_and_contain_remote_object_from_id(id) do
     Logger.info("Fetching object #{id} via AP")
 
+    date =
+      NaiveDateTime.utc_now()
+      |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
+
+    headers =
+      [{:Accept, "application/activity+json"}]
+      |> maybe_date_fetch(date)
+      |> sign_fetch(id, date)
+
+    Logger.debug("Fetch headers: #{inspect(headers)}")
+
     with true <- String.starts_with?(id, "http"),
-         {:ok, %{body: body, status: code}} when code in 200..299 <-
-           HTTP.get(
-             id,
-             [{:Accept, "application/activity+json"}]
-           ),
+         {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get(id, headers),
          {:ok, data} <- Jason.decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}

From cffe8229a570ac883d6966dae97c97dba19412f0 Mon Sep 17 00:00:00 2001
From: tallship <bradley@northtech.us>
Date: Wed, 17 Jul 2019 23:03:46 +0000
Subject: [PATCH 21/40] Update clients.md - added Homepage for Fedilab Android
 app

---
 docs/clients.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/clients.md b/docs/clients.md
index 30358c210..91096970e 100644
--- a/docs/clients.md
+++ b/docs/clients.md
@@ -31,6 +31,7 @@ Feel free to contact us to be added to this list!
 - Features: No Streaming
 
 ### Fedilab
+- Homepage: <https://fedilab.app/>
 - Source Code: <https://gitlab.com/tom79/mastalab/>
 - Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79)
 - Platforms: Android

From 1345e0c2bf51c7eb8add41a25683e121c83c1ff8 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 22:58:52 +0000
Subject: [PATCH 22/40] tests: add tests for signed object fetches

---
 config/test.exs              |  2 ++
 test/object/fetcher_test.exs | 30 ++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/config/test.exs b/config/test.exs
index 0af62aa14..92dca18bc 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -31,6 +31,8 @@ config :pleroma, :instance,
   skip_thread_containment: false,
   federating: false
 
+config :pleroma, :activitypub, sign_object_fetches: false
+
 # Configure your database
 config :pleroma, Pleroma.Repo,
   adapter: Ecto.Adapters.Postgres,
diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs
index 56a9d775f..482252cff 100644
--- a/test/object/fetcher_test.exs
+++ b/test/object/fetcher_test.exs
@@ -150,4 +150,34 @@ defmodule Pleroma.Object.FetcherTest do
       assert object.id != object_two.id
     end
   end
+
+  describe "signed fetches" do
+    test_with_mock "it signs fetches when configured to do so",
+                   Pleroma.Signature,
+                   [:passthrough],
+                   [] do
+      option = Pleroma.Config.get([:activitypub, :sign_object_fetches])
+      Pleroma.Config.put([:activitypub, :sign_object_fetches], true)
+
+      Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
+
+      assert called(Pleroma.Signature.sign(:_, :_))
+
+      Pleroma.Config.put([:activitypub, :sign_object_fetches], option)
+    end
+
+    test_with_mock "it doesn't sign fetches when not configured to do so",
+                   Pleroma.Signature,
+                   [:passthrough],
+                   [] do
+      option = Pleroma.Config.get([:activitypub, :sign_object_fetches])
+      Pleroma.Config.put([:activitypub, :sign_object_fetches], false)
+
+      Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
+
+      refute called(Pleroma.Signature.sign(:_, :_))
+
+      Pleroma.Config.put([:activitypub, :sign_object_fetches], option)
+    end
+  end
 end

From 7bb50bfcccdc08f72113fda5732b87876eb01df8 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Wed, 17 Jul 2019 23:07:53 +0000
Subject: [PATCH 23/40] update changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 654d208dd..6f268d110 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Addressable lists
 - Twitter API: added rate limit for `/api/account/password_reset` endpoint.
 - ActivityPub: Add an internal service actor for fetching ActivityPub objects.
+- ActivityPub: Optional signing of ActivityPub object fetches.
 
 ### Changed
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text

From 129078eddc1422e3654514477fe438b56dcff562 Mon Sep 17 00:00:00 2001
From: aries <aries@asterism.xyz>
Date: Thu, 18 Jul 2019 21:08:31 +0900
Subject: [PATCH 24/40] Fix redirect setting not working

---
 docs/config/howto_mediaproxy.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/config/howto_mediaproxy.md b/docs/config/howto_mediaproxy.md
index fb731112b..ed70c3ed4 100644
--- a/docs/config/howto_mediaproxy.md
+++ b/docs/config/howto_mediaproxy.md
@@ -24,7 +24,9 @@ If you came here from one of the installation guides, take a look at the example
 ```
 config :pleroma, :media_proxy,
       enabled: true,
-      redirect_on_failure: true
+      proxy_opts: [
+            redirect_on_failure: true
+      ]
       #base_url: "https://cache.pleroma.social"
 ```
 If you want to use a subdomain to serve the files, uncomment `base_url`, change the url and add a comma after `true` in the previous line.

From b6b748d3e7f3383303a2fcccb17b7b0f7054100f Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Thu, 18 Jul 2019 12:30:18 +0000
Subject: [PATCH 25/40] tests for Uploader with webhook

---
 lib/pleroma/uploaders/uploader.ex      |  9 ++-
 lib/pleroma/web/uploader_controller.ex |  4 --
 test/support/data_case.ex              | 17 +++--
 test/upload_test.exs                   | 89 +++++++++++++++++++++++++-
 test/web/uploader_controller_test.exs  | 43 +++++++++++++
 5 files changed, 147 insertions(+), 15 deletions(-)
 create mode 100644 test/web/uploader_controller_test.exs

diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex
index 0af76bc59..c0b22c28a 100644
--- a/lib/pleroma/uploaders/uploader.ex
+++ b/lib/pleroma/uploaders/uploader.ex
@@ -68,7 +68,14 @@ defmodule Pleroma.Uploaders.Uploader do
             {:error, error}
         end
     after
-      30_000 -> {:error, dgettext("errors", "Uploader callback timeout")}
+      callback_timeout() -> {:error, dgettext("errors", "Uploader callback timeout")}
+    end
+  end
+
+  defp callback_timeout do
+    case Mix.env() do
+      :test -> 1_000
+      _ -> 30_000
     end
   end
 end
diff --git a/lib/pleroma/web/uploader_controller.ex b/lib/pleroma/web/uploader_controller.ex
index bf09775e6..0cc172698 100644
--- a/lib/pleroma/web/uploader_controller.ex
+++ b/lib/pleroma/web/uploader_controller.ex
@@ -11,10 +11,6 @@ defmodule Pleroma.Web.UploaderController do
     process_callback(conn, :global.whereis_name({Uploader, upload_path}), params)
   end
 
-  def callbacks(conn, _) do
-    render_error(conn, :bad_request, "bad request")
-  end
-
   defp process_callback(conn, pid, params) when is_pid(pid) do
     send(pid, {Uploader, self(), conn, params})
 
diff --git a/test/support/data_case.ex b/test/support/data_case.ex
index df260bd3f..f3d98e7e3 100644
--- a/test/support/data_case.ex
+++ b/test/support/data_case.ex
@@ -42,19 +42,18 @@ defmodule Pleroma.DataCase do
     :ok
   end
 
-  def ensure_local_uploader(_context) do
+  def ensure_local_uploader(context) do
+    test_uploader = Map.get(context, :uploader, Pleroma.Uploaders.Local)
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
     filters = Pleroma.Config.get([Pleroma.Upload, :filters])
 
-    unless uploader == Pleroma.Uploaders.Local || filters != [] do
-      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
-      Pleroma.Config.put([Pleroma.Upload, :filters], [])
+    Pleroma.Config.put([Pleroma.Upload, :uploader], test_uploader)
+    Pleroma.Config.put([Pleroma.Upload, :filters], [])
 
-      on_exit(fn ->
-        Pleroma.Config.put([Pleroma.Upload, :uploader], uploader)
-        Pleroma.Config.put([Pleroma.Upload, :filters], filters)
-      end)
-    end
+    on_exit(fn ->
+      Pleroma.Config.put([Pleroma.Upload, :uploader], uploader)
+      Pleroma.Config.put([Pleroma.Upload, :filters], filters)
+    end)
 
     :ok
   end
diff --git a/test/upload_test.exs b/test/upload_test.exs
index 946ebcb5a..f7b1893ad 100644
--- a/test/upload_test.exs
+++ b/test/upload_test.exs
@@ -3,9 +3,96 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.UploadTest do
-  alias Pleroma.Upload
   use Pleroma.DataCase
 
+  alias Pleroma.Upload
+  alias Pleroma.Uploaders.Uploader
+
+  @upload_file %Plug.Upload{
+    content_type: "image/jpg",
+    path: Path.absname("test/fixtures/image_tmp.jpg"),
+    filename: "image.jpg"
+  }
+
+  defmodule TestUploaderBase do
+    def put_file(%{path: path} = _upload, module_name) do
+      task_pid =
+        Task.async(fn ->
+          :timer.sleep(10)
+
+          {Uploader, path}
+          |> :global.whereis_name()
+          |> send({Uploader, self(), {:test}, %{}})
+
+          assert_receive {Uploader, {:test}}, 4_000
+        end)
+
+      Agent.start(fn -> task_pid end, name: module_name)
+
+      :wait_callback
+    end
+  end
+
+  describe "Tried storing a file when http callback response success result" do
+    defmodule TestUploaderSuccess do
+      def http_callback(conn, _params),
+        do: {:ok, conn, {:file, "post-process-file.jpg"}}
+
+      def put_file(upload), do: TestUploaderBase.put_file(upload, __MODULE__)
+    end
+
+    setup do: [uploader: TestUploaderSuccess]
+    setup [:ensure_local_uploader]
+
+    test "it returns file" do
+      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+
+      assert Upload.store(@upload_file) ==
+               {:ok,
+                %{
+                  "name" => "image.jpg",
+                  "type" => "Document",
+                  "url" => [
+                    %{
+                      "href" => "http://localhost:4001/media/post-process-file.jpg",
+                      "mediaType" => "image/jpeg",
+                      "type" => "Link"
+                    }
+                  ]
+                }}
+
+      Task.await(Agent.get(TestUploaderSuccess, fn task_pid -> task_pid end))
+    end
+  end
+
+  describe "Tried storing a file when http callback response error" do
+    defmodule TestUploaderError do
+      def http_callback(conn, _params), do: {:error, conn, "Errors"}
+
+      def put_file(upload), do: TestUploaderBase.put_file(upload, __MODULE__)
+    end
+
+    setup do: [uploader: TestUploaderError]
+    setup [:ensure_local_uploader]
+
+    test "it returns error" do
+      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+      assert Upload.store(@upload_file) == {:error, "Errors"}
+      Task.await(Agent.get(TestUploaderError, fn task_pid -> task_pid end))
+    end
+  end
+
+  describe "Tried storing a file when http callback doesn't response by timeout" do
+    defmodule(TestUploader, do: def(put_file(_upload), do: :wait_callback))
+    setup do: [uploader: TestUploader]
+    setup [:ensure_local_uploader]
+
+    test "it returns error" do
+      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+      assert Upload.store(@upload_file) == {:error, "Uploader callback timeout"}
+    end
+  end
+
   describe "Storing a file with the Local uploader" do
     setup [:ensure_local_uploader]
 
diff --git a/test/web/uploader_controller_test.exs b/test/web/uploader_controller_test.exs
new file mode 100644
index 000000000..70028df1c
--- /dev/null
+++ b/test/web/uploader_controller_test.exs
@@ -0,0 +1,43 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.UploaderControllerTest do
+  use Pleroma.Web.ConnCase
+  alias Pleroma.Uploaders.Uploader
+
+  describe "callback/2" do
+    test "it returns 400 response when process callback isn't alive", %{conn: conn} do
+      res =
+        conn
+        |> post(uploader_path(conn, :callback, "test-path"))
+
+      assert res.status == 400
+      assert res.resp_body == "{\"error\":\"bad request\"}"
+    end
+
+    test "it returns success result", %{conn: conn} do
+      task =
+        Task.async(fn ->
+          receive do
+            {Uploader, pid, conn, _params} ->
+              conn =
+                conn
+                |> put_status(:ok)
+                |> Phoenix.Controller.json(%{upload_path: "test-path"})
+
+              send(pid, {Uploader, conn})
+          end
+        end)
+
+      :global.register_name({Uploader, "test-path"}, task.pid)
+
+      res =
+        conn
+        |> post(uploader_path(conn, :callback, "test-path"))
+        |> json_response(200)
+
+      assert res == %{"upload_path" => "test-path"}
+    end
+  end
+end

From 88d064d80e4a3272a2a7101089b5f924fd175866 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 18 Jul 2019 15:06:58 +0000
Subject: [PATCH 26/40] http signature plug: remove redundant checks handled by
 HTTPSignatures library

the redundant checks assumed a POST request, which will not work for signed GETs.
this check was originally needed because the HTTPSignatures adapter assumed that
the requests were also POST requests.  but now, the adapter has been corrected.
---
 lib/pleroma/plugs/http_signature.ex     | 51 ++++++++++---------------
 test/plugs/http_signature_plug_test.exs | 18 ---------
 2 files changed, 21 insertions(+), 48 deletions(-)

diff --git a/lib/pleroma/plugs/http_signature.ex b/lib/pleroma/plugs/http_signature.ex
index e2874c469..d87fa52fa 100644
--- a/lib/pleroma/plugs/http_signature.ex
+++ b/lib/pleroma/plugs/http_signature.ex
@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
-  alias Pleroma.Web.ActivityPub.Utils
   import Plug.Conn
   require Logger
 
@@ -16,38 +15,30 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   end
 
   def call(conn, _opts) do
-    user = Utils.get_ap_id(conn.params["actor"])
-    Logger.debug("Checking sig for #{user}")
     [signature | _] = get_req_header(conn, "signature")
 
-    cond do
-      signature && String.contains?(signature, user) ->
-        # set (request-target) header to the appropriate value
-        # we also replace the digest header with the one we computed
-        conn =
-          conn
-          |> put_req_header(
-            "(request-target)",
-            String.downcase("#{conn.method}") <> " #{conn.request_path}"
-          )
-
-        conn =
-          if conn.assigns[:digest] do
-            conn
-            |> put_req_header("digest", conn.assigns[:digest])
-          else
-            conn
-          end
-
-        assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn))
-
-      signature ->
-        Logger.debug("Signature not from actor")
-        assign(conn, :valid_signature, false)
-
-      true ->
-        Logger.debug("No signature header!")
+    if signature do
+      # set (request-target) header to the appropriate value
+      # we also replace the digest header with the one we computed
+      conn =
         conn
+        |> put_req_header(
+          "(request-target)",
+          String.downcase("#{conn.method}") <> " #{conn.request_path}"
+        )
+
+      conn =
+        if conn.assigns[:digest] do
+          conn
+          |> put_req_header("digest", conn.assigns[:digest])
+        else
+          conn
+        end
+
+      assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn))
+    else
+      Logger.debug("No signature header!")
+      conn
     end
   end
 end
diff --git a/test/plugs/http_signature_plug_test.exs b/test/plugs/http_signature_plug_test.exs
index efd811df7..d6fd9ea81 100644
--- a/test/plugs/http_signature_plug_test.exs
+++ b/test/plugs/http_signature_plug_test.exs
@@ -26,22 +26,4 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do
       assert called(HTTPSignatures.validate_conn(:_))
     end
   end
-
-  test "bails out early if the signature isn't by the activity actor" do
-    params = %{"actor" => "https://mst3k.interlinked.me/users/luciferMysticus"}
-    conn = build_conn(:get, "/doesntmattter", params)
-
-    with_mock HTTPSignatures, validate_conn: fn _ -> false end do
-      conn =
-        conn
-        |> put_req_header(
-          "signature",
-          "keyId=\"http://mastodon.example.org/users/admin#main-key"
-        )
-        |> HTTPSignaturePlug.call(%{})
-
-      assert conn.assigns.valid_signature == false
-      refute called(HTTPSignatures.validate_conn(:_))
-    end
-  end
 end

From 18d8d12d53567b7c0c246bb793ee724d0d2e4c77 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 18 Jul 2019 15:35:42 +0000
Subject: [PATCH 27/40] signature: make key_id_to_actor_id() public

---
 lib/pleroma/signature.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index a45c70a9d..2a0823ecf 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Signature do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
 
-  defp key_id_to_actor_id(key_id) do
+  def key_id_to_actor_id(key_id) do
     URI.parse(key_id)
     |> Map.put(:fragment, nil)
     |> URI.to_string()

From 184fa61fb3a1bc8c5d5515bb7748c12816b11ebf Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 18 Jul 2019 15:38:45 +0000
Subject: [PATCH 28/40] plugs: add MappedSignatureToIdentityPlug

---
 .../mapped_signature_to_identity_plug.ex      | 64 +++++++++++++++++++
 lib/pleroma/web/router.ex                     |  1 +
 2 files changed, 65 insertions(+)
 create mode 100644 lib/pleroma/plugs/mapped_signature_to_identity_plug.ex

diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
new file mode 100644
index 000000000..ae9339595
--- /dev/null
+++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
+  alias Pleroma.Signature
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Utils
+
+  import Plug.Conn
+  require Logger
+
+  def init(options), do: options
+
+  defp key_id_from_conn(conn) do
+    with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn) do
+      Signature.key_id_to_actor_id(key_id)
+    else
+      _ ->
+        nil
+    end
+  end
+
+  defp user_from_key_id(conn) do
+    with key_actor_id when is_binary(key_actor_id) <- key_id_from_conn(conn),
+         %User{} = user <- User.get_or_fetch_by_ap_id(key_actor_id) do
+      user
+    else
+      _ ->
+        nil
+    end
+  end
+
+  def call(%{assigns: %{mapped_identity: _}} = conn, _opts), do: conn
+
+  # if this has payload make sure it is signed by the same actor that made it
+  def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do
+    with actor_id <- Utils.get_ap_id(actor),
+         %User{} = user <- user_from_key_id(conn),
+         true <- user.ap_id == actor_id do
+      assign(conn, :mapped_identity, user)
+    else
+      _ ->
+        Logger.debug("Failed to map identity from signature (payload actor mismatch?)")
+        Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}")
+        conn
+    end
+  end
+
+  # no payload, probably a signed fetch
+  def call(%{assigns: %{valid_signature: true}} = conn, _opts) do
+    with %User{} = user <- user_from_key_id(conn) do
+      assign(conn, :mapped_identity, user)
+    else
+      _ ->
+        Logger.debug("Failed to map identity from signature (no payload actor mismatch)")
+        Logger.debug("key_id=#{key_id_from_conn(conn)}")
+        conn
+    end
+  end
+
+  # no signature at all
+  def call(conn, _opts), do: conn
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 8095ac4b1..518720d38 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -617,6 +617,7 @@ defmodule Pleroma.Web.Router do
   pipeline :activitypub do
     plug(:accepts, ["activity+json", "json"])
     plug(Pleroma.Web.Plugs.HTTPSignaturePlug)
+    plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug)
   end
 
   scope "/", Pleroma.Web.ActivityPub do

From cdf0038d0f5d56cf78f640fb149e9141e74800fc Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 18 Jul 2019 15:51:58 +0000
Subject: [PATCH 29/40] mix: update http signatures dependency

---
 mix.exs  | 2 +-
 mix.lock | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index e4ea19bb6..c12b0a500 100644
--- a/mix.exs
+++ b/mix.exs
@@ -138,7 +138,7 @@ defmodule Pleroma.Mixfile do
        ref: "95e8188490e97505c56636c1379ffdf036c1fdde"},
       {:http_signatures,
        git: "https://git.pleroma.social/pleroma/http_signatures.git",
-       ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"},
+       ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"},
       {:pleroma_job_queue, "~> 0.2.0"},
       {:telemetry, "~> 0.3"},
       {:prometheus_ex, "~> 3.0"},
diff --git a/mix.lock b/mix.lock
index 6477c1ed5..45142ba8f 100644
--- a/mix.lock
+++ b/mix.lock
@@ -38,7 +38,7 @@
   "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
   "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"},
   "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
-  "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "a2a5982fa167fb1352fbd518ce6b606ba233a989", [ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"]},
+  "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]},
   "httpoison": {:hex, :httpoison, "1.2.0", "2702ed3da5fd7a8130fc34b11965c8cfa21ade2f232c00b42d96d4967c39a3a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
   "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},

From 5ea0cd69f7457086fc486f13e072f13d2c1ef547 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 18 Jul 2019 16:01:21 +0000
Subject: [PATCH 30/40] mapped signature plug: don't invalidate in cases where
 a signature is actually not present (testsuite)

---
 .../plugs/mapped_signature_to_identity_plug.ex   | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
index ae9339595..2a8ed4470 100644
--- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
+++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
@@ -36,12 +36,18 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
   # if this has payload make sure it is signed by the same actor that made it
   def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do
     with actor_id <- Utils.get_ap_id(actor),
-         %User{} = user <- user_from_key_id(conn),
-         true <- user.ap_id == actor_id do
+         {:user, %User{} = user} <- {:user, user_from_key_id(conn)},
+         {:user_match, true} <- {:user_match, user.ap_id == actor_id} do
       assign(conn, :mapped_identity, user)
     else
-      _ ->
-        Logger.debug("Failed to map identity from signature (payload actor mismatch?)")
+      {:user_match, false} ->
+        Logger.debug("Failed to map identity from signature (payload actor mismatch)")
+        Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}")
+        assign(conn, :valid_signature, false)
+
+      # remove me once testsuite uses mapped capabilities instead of what we do now
+      {:user, nil} ->
+        Logger.debug("Failed to map identity from signature (lookup failure)")
         Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}")
         conn
     end
@@ -55,7 +61,7 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
       _ ->
         Logger.debug("Failed to map identity from signature (no payload actor mismatch)")
         Logger.debug("key_id=#{key_id_from_conn(conn)}")
-        conn
+        assign(conn, :valid_signature, false)
     end
   end
 

From a8af0ac053713102204418fe7a28d322f81eb3ea Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 18 Jul 2019 16:27:50 +0000
Subject: [PATCH 31/40] mapped signature plug: fix user lookup

---
 lib/pleroma/plugs/mapped_signature_to_identity_plug.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
index 2a8ed4470..1e7da4f50 100644
--- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
+++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
@@ -23,7 +23,7 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
 
   defp user_from_key_id(conn) do
     with key_actor_id when is_binary(key_actor_id) <- key_id_from_conn(conn),
-         %User{} = user <- User.get_or_fetch_by_ap_id(key_actor_id) do
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(key_actor_id) do
       user
     else
       _ ->

From 621cacf667e7d5519a2731fadf86c429f70f4489 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 18 Jul 2019 16:28:36 +0000
Subject: [PATCH 32/40] tests: add tests for mapped signature plug

---
 ...mapped_identity_to_signature_plug_test.exs | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 test/plugs/mapped_identity_to_signature_plug_test.exs

diff --git a/test/plugs/mapped_identity_to_signature_plug_test.exs b/test/plugs/mapped_identity_to_signature_plug_test.exs
new file mode 100644
index 000000000..9aca534e1
--- /dev/null
+++ b/test/plugs/mapped_identity_to_signature_plug_test.exs
@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlugTest do
+  use Pleroma.Web.ConnCase
+  alias Pleroma.Web.Plugs.MappedSignatureToIdentityPlug
+
+  import Tesla.Mock
+  import Plug.Conn
+
+  setup do
+    mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  defp set_signature(conn, key_id) do
+    conn
+    |> put_req_header("signature", "keyId=\"#{key_id}\"")
+    |> assign(:valid_signature, true)
+  end
+
+  test "it successfully maps a valid identity with a valid signature" do
+    conn =
+      build_conn(:get, "/doesntmattter")
+      |> set_signature("http://mastodon.example.org/users/admin")
+      |> MappedSignatureToIdentityPlug.call(%{})
+
+    refute is_nil(conn.assigns.mapped_identity)
+  end
+
+  test "it successfully maps a valid identity with a valid signature with payload" do
+    conn =
+      build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"})
+      |> set_signature("http://mastodon.example.org/users/admin")
+      |> MappedSignatureToIdentityPlug.call(%{})
+
+    refute is_nil(conn.assigns.mapped_identity)
+  end
+
+  test "it considers a mapped identity to be invalid when it mismatches a payload" do
+    conn =
+      build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"})
+      |> set_signature("https://niu.moe/users/rye")
+      |> MappedSignatureToIdentityPlug.call(%{})
+
+    assert %{valid_signature: false} == conn.assigns
+  end
+
+  @tag skip: "known breakage; the testsuite presently depends on it"
+  test "it considers a mapped identity to be invalid when the identity cannot be found" do
+    conn =
+      build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"})
+      |> set_signature("http://niu.moe/users/rye")
+      |> MappedSignatureToIdentityPlug.call(%{})
+
+    assert %{valid_signature: false} == conn.assigns
+  end
+end

From f435217e5003dac5d749e4bb905572d51c383b29 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Thu, 18 Jul 2019 20:29:51 +0000
Subject: [PATCH 33/40] tests for Plugs.AuthenticationPlug

---
 lib/pleroma/plugs/authentication_plug.ex | 23 ++++++++++------------
 test/plugs/authentication_plug_test.exs  | 25 ++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/lib/pleroma/plugs/authentication_plug.ex b/lib/pleroma/plugs/authentication_plug.ex
index eec514892..567674a0b 100644
--- a/lib/pleroma/plugs/authentication_plug.ex
+++ b/lib/pleroma/plugs/authentication_plug.ex
@@ -8,22 +8,19 @@ defmodule Pleroma.Plugs.AuthenticationPlug do
   alias Pleroma.User
   require Logger
 
-  def init(options) do
-    options
+  def init(options), do: options
+
+  def checkpw(password, "$6" <> _ = password_hash) do
+    :crypt.crypt(password, password_hash) == password_hash
   end
 
-  def checkpw(password, password_hash) do
-    cond do
-      String.starts_with?(password_hash, "$pbkdf2") ->
-        Pbkdf2.checkpw(password, password_hash)
+  def checkpw(password, "$pbkdf2" <> _ = password_hash) do
+    Pbkdf2.checkpw(password, password_hash)
+  end
 
-      String.starts_with?(password_hash, "$6") ->
-        :crypt.crypt(password, password_hash) == password_hash
-
-      true ->
-        Logger.error("Password hash not recognized")
-        false
-    end
+  def checkpw(_password, _password_hash) do
+    Logger.error("Password hash not recognized")
+    false
   end
 
   def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs
index 6158086ea..b55e746f8 100644
--- a/test/plugs/authentication_plug_test.exs
+++ b/test/plugs/authentication_plug_test.exs
@@ -54,4 +54,29 @@ defmodule Pleroma.Plugs.AuthenticationPlugTest do
 
     assert conn == ret_conn
   end
+
+  describe "checkpw/2" do
+    test "check pbkdf2 hash" do
+      hash =
+        "$pbkdf2-sha512$160000$loXqbp8GYls43F0i6lEfIw$AY.Ep.2pGe57j2hAPY635sI/6w7l9Q9u9Bp02PkPmF3OrClDtJAI8bCiivPr53OKMF7ph6iHhN68Rom5nEfC2A"
+
+      assert AuthenticationPlug.checkpw("test-password", hash)
+      refute AuthenticationPlug.checkpw("test-password1", hash)
+    end
+
+    test "check sha512-crypt hash" do
+      hash =
+        "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1"
+
+      assert AuthenticationPlug.checkpw("password", hash)
+      refute AuthenticationPlug.checkpw("password1", hash)
+    end
+
+    test "it returns false when hash invalid" do
+      hash =
+        "psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1"
+
+      refute Pleroma.Plugs.AuthenticationPlug.checkpw("password", hash)
+    end
+  end
 end

From c947cfec5ab49f90fb2de83f61bda77568298d6f Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 18 Jul 2019 20:31:25 +0000
Subject: [PATCH 34/40] mapped signature plug: use `user` assign like
 authentication plug

---
 lib/pleroma/plugs/mapped_signature_to_identity_plug.ex | 6 +++---
 test/plugs/mapped_identity_to_signature_plug_test.exs  | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
index 1e7da4f50..ce8494b9d 100644
--- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
+++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
@@ -31,14 +31,14 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
     end
   end
 
-  def call(%{assigns: %{mapped_identity: _}} = conn, _opts), do: conn
+  def call(%{assigns: %{user: _}} = conn, _opts), do: conn
 
   # if this has payload make sure it is signed by the same actor that made it
   def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do
     with actor_id <- Utils.get_ap_id(actor),
          {:user, %User{} = user} <- {:user, user_from_key_id(conn)},
          {:user_match, true} <- {:user_match, user.ap_id == actor_id} do
-      assign(conn, :mapped_identity, user)
+      assign(conn, :user, user)
     else
       {:user_match, false} ->
         Logger.debug("Failed to map identity from signature (payload actor mismatch)")
@@ -56,7 +56,7 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
   # no payload, probably a signed fetch
   def call(%{assigns: %{valid_signature: true}} = conn, _opts) do
     with %User{} = user <- user_from_key_id(conn) do
-      assign(conn, :mapped_identity, user)
+      assign(conn, :user, user)
     else
       _ ->
         Logger.debug("Failed to map identity from signature (no payload actor mismatch)")
diff --git a/test/plugs/mapped_identity_to_signature_plug_test.exs b/test/plugs/mapped_identity_to_signature_plug_test.exs
index 9aca534e1..bb45d9edf 100644
--- a/test/plugs/mapped_identity_to_signature_plug_test.exs
+++ b/test/plugs/mapped_identity_to_signature_plug_test.exs
@@ -26,7 +26,7 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlugTest do
       |> set_signature("http://mastodon.example.org/users/admin")
       |> MappedSignatureToIdentityPlug.call(%{})
 
-    refute is_nil(conn.assigns.mapped_identity)
+    refute is_nil(conn.assigns.user)
   end
 
   test "it successfully maps a valid identity with a valid signature with payload" do
@@ -35,7 +35,7 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlugTest do
       |> set_signature("http://mastodon.example.org/users/admin")
       |> MappedSignatureToIdentityPlug.call(%{})
 
-    refute is_nil(conn.assigns.mapped_identity)
+    refute is_nil(conn.assigns.user)
   end
 
   test "it considers a mapped identity to be invalid when it mismatches a payload" do

From de9906ad56bd25d6c8c38bef1307192df2e95445 Mon Sep 17 00:00:00 2001
From: Sachin Joshi <satchin.joshi@gmail.com>
Date: Fri, 19 Jul 2019 11:43:42 +0545
Subject: [PATCH 35/40] change the structure of image ttl parsar

---
 ..._set_richmedia_cache_ttl_based_on_image.md |  2 +-
 lib/pleroma/web/rich_media/parser.ex          | 36 +++++-----
 .../rich_media/parsers/ttl/aws_signed_url.ex  | 10 ++-
 lib/pleroma/web/rich_media/parsers/ttl/ttl.ex |  3 +
 test/web/rich_media/aws_signed_url_test.exs   | 70 +++++++++++++++----
 5 files changed, 85 insertions(+), 36 deletions(-)
 create mode 100644 lib/pleroma/web/rich_media/parsers/ttl/ttl.ex

diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
index 489f9ece8..5846b6ab0 100644
--- a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
+++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
@@ -4,7 +4,7 @@
 Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url.
 In such cases the old image url (expired) is returned from the media cache.
 
-So to avoid such situation we can define a moddule that will set ttl based no image.
+So to avoid such situation we can define a moddule that will set ttl based on image.
 
 The module must have a `run` function and it should be registered in the config.
 
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index ba8dc6f2a..b69b2be61 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -35,17 +35,17 @@ defmodule Pleroma.Web.RichMedia.Parser do
   @doc """
   Set the rich media cache based on the expiration time of image.
 
-  Define a module that has `run` function
+  Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
 
   ## Example
 
       defmodule MyModule do
-        def run(data, url) do
+        @behaviour Pleroma.Web.RichMedia.Parser.TTL
+        def ttl(data, url) do
           image_url = Map.get(data, :image)
           # do some parsing in the url and get the ttl of the image
-          # ttl is unix time
-          ttl = parse_ttl_from_url(image_url)  
-          Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
+          # and return ttl is unix time
+          parse_ttl_from_url(image_url)
         end
       end
 
@@ -55,22 +55,26 @@ defmodule Pleroma.Web.RichMedia.Parser do
         ttl_setters: [MyModule]
   """
   def set_ttl_based_on_image({:ok, data}, url) do
-    case Cachex.ttl(:rich_media_cache, url) do
-      {:ok, nil} ->
-        modules = Pleroma.Config.get([:rich_media, :ttl_setters])
-
-        if Enum.count(modules) > 0 do
-          Enum.each(modules, & &1.run(data, url))
-        end
-
-        {:ok, data}
-
+    with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url) do
+      ttl = get_ttl_from_image(data, url)
+      Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
+      {:ok, data}
+    else
       _ ->
         {:ok, data}
     end
   end
 
-  def set_ttl_based_on_image(data, _url), do: data
+  defp get_ttl_from_image(data, url) do
+    Pleroma.Config.get([:rich_media, :ttl_setters])
+    |> Enum.reduce({:ok, nil}, fn
+      module, {:ok, _ttl} ->
+        module.ttl(data, url)
+
+      _, error ->
+        error
+    end)
+  end
 
   defp parse_url(url) do
     try do
diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
index d57107939..014c0935f 100644
--- a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
+++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
@@ -1,5 +1,8 @@
 defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
-  def run(data, url) do
+  @behaviour Pleroma.Web.RichMedia.Parser.TTL
+
+  @impl Pleroma.Web.RichMedia.Parser.TTL
+  def ttl(data, _url) do
     image = Map.get(data, :image)
 
     if is_aws_signed_url(image) do
@@ -7,7 +10,6 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
       |> parse_query_params()
       |> format_query_params()
       |> get_expiration_timestamp()
-      |> set_ttl(url)
     end
   end
 
@@ -47,8 +49,4 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
 
     Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))
   end
-
-  defp set_ttl(ttl, url) do
-    Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
-  end
 end
diff --git a/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex b/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex
new file mode 100644
index 000000000..6b3ec6d30
--- /dev/null
+++ b/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex
@@ -0,0 +1,3 @@
+defmodule Pleroma.Web.RichMedia.Parser.TTL do
+  @callback ttl(Map.t(), String.t()) :: {:ok, Integer.t()} | {:error, String.t()}
+end
diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs
index 75bf6c6df..122787bc2 100644
--- a/test/web/rich_media/aws_signed_url_test.exs
+++ b/test/web/rich_media/aws_signed_url_test.exs
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do
   use ExUnit.Case, async: true
 
-  test "amazon signed url is parsed and correct ttl is set for rich media" do
+  test "s3 signed url is parsed correct for expiration time" do
     url = "https://pleroma.social/amz"
 
     {:ok, timestamp} =
@@ -16,22 +16,66 @@ defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do
     # in seconds
     valid_till = 30
 
-    data = %{
-      image:
-        "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{
-          timestamp
-        }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host",
-      locale: "en_US",
-      site_name: "Pleroma",
-      title: "PLeroma",
-      url: url
-    }
+    metadata = construct_metadata(timestamp, valid_till, url)
+
+    expire_time =
+      Timex.parse!(timestamp, "{ISO:Basic:Z}") |> Timex.to_unix() |> Kernel.+(valid_till)
+
+    assert expire_time == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url)
+  end
+
+  test "s3 signed url is parsed and correct ttl is set for rich media" do
+    url = "https://pleroma.social/amz"
+
+    {:ok, timestamp} =
+      Timex.now()
+      |> DateTime.truncate(:second)
+      |> Timex.format("{ISO:Basic:Z}")
+
+    # in seconds
+    valid_till = 30
+
+    metadata = construct_metadata(timestamp, valid_till, url)
+
+    body = """
+    <meta name="twitter:card" content="Pleroma" />
+    <meta name="twitter:site" content="Pleroma" />
+    <meta name="twitter:title" content="Pleroma" />
+    <meta name="twitter:description" content="Pleroma" />
+    <meta name="twitter:image" content="#{Map.get(metadata, :image)}" />
+    """
+
+    Tesla.Mock.mock(fn
+      %{
+        method: :get,
+        url: "https://pleroma.social/amz"
+      } ->
+        %Tesla.Env{status: 200, body: body}
+    end)
+
+    Cachex.put(:rich_media_cache, url, metadata)
+
+    Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image({:ok, metadata}, url)
 
-    Cachex.put(:rich_media_cache, url, data)
-    assert {:ok, _} = Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.run(data, url)
     {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)
 
     # as there is delay in setting and pulling the data from cache we ignore 1 second
     assert_in_delta(valid_till * 1000, cache_ttl, 1000)
   end
+
+  defp construct_s3_url(timestamp, valid_till) do
+    "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{
+      timestamp
+    }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host"
+  end
+
+  defp construct_metadata(timestamp, valid_till, url) do
+    %{
+      image: construct_s3_url(timestamp, valid_till),
+      site: "Pleroma",
+      title: "Pleroma",
+      description: "Pleroma",
+      url: url
+    }
+  end
 end

From 581756ccc50cc08823957a2f24f506bf23c7cd22 Mon Sep 17 00:00:00 2001
From: Sachin Joshi <satchin.joshi@gmail.com>
Date: Fri, 19 Jul 2019 11:50:47 +0545
Subject: [PATCH 36/40] update the docs

---
 ...owto_set_richmedia_cache_ttl_based_on_image.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
index 5846b6ab0..bfee5a9e6 100644
--- a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
+++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md
@@ -4,20 +4,21 @@
 Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url.
 In such cases the old image url (expired) is returned from the media cache.
 
-So to avoid such situation we can define a moddule that will set ttl based on image.
-
-The module must have a `run` function and it should be registered in the config.
+So to avoid such situation we can define a module that will set ttl based on image.
+The module must adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
 
 ### Example
 
 ```exs
 defmodule MyModule do
-  def run(data, url) do
+  @behaviour Pleroma.Web.RichMedia.Parser.TTL
+
+  @impl Pleroma.Web.RichMedia.Parser.TTL
+  def ttl(data, url) do
     image_url = Map.get(data, :image)
     # do some parsing in the url and get the ttl of the image
-    # ttl is unix time
-    ttl = parse_ttl_from_url(image_url)  
-    Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
+    # return ttl is unix time
+    parse_ttl_from_url(image_url)
   end
 end
 ```

From b0b9eca37d43c8cc5e9e0e01fa4b1325a71ce4d2 Mon Sep 17 00:00:00 2001
From: tom79 <tschneider.ac@gmail.com>
Date: Fri, 19 Jul 2019 08:39:22 +0000
Subject: [PATCH 37/40] Update clients.md for Fedilab - Change owner
 (@fedilab@framapiaf.org), Source code: Framagit, Add some other supported
 features

---
 docs/clients.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/clients.md b/docs/clients.md
index 91096970e..dc4ab37b1 100644
--- a/docs/clients.md
+++ b/docs/clients.md
@@ -32,10 +32,10 @@ Feel free to contact us to be added to this list!
 
 ### Fedilab
 - Homepage: <https://fedilab.app/>
-- Source Code: <https://gitlab.com/tom79/mastalab/>
-- Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79)
+- Source Code: <https://framagit.org/tom79/fedilab/>
+- Contact: [@fedilab@mastodon.social](https://framapiaf.org/users/fedilab)
 - Platforms: Android
-- Features: Streaming Ready
+- Features: Streaming Ready, Moderation, Text Formatting 
 
 ### Nekonium
 - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/)

From 5c4a555b1da1d2cb6649a9384e386a48bbcf905f Mon Sep 17 00:00:00 2001
From: tom79 <tschneider.ac@gmail.com>
Date: Fri, 19 Jul 2019 08:40:47 +0000
Subject: [PATCH 38/40] Fix domain for the contact clients.md

---
 docs/clients.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/clients.md b/docs/clients.md
index dc4ab37b1..9029361f8 100644
--- a/docs/clients.md
+++ b/docs/clients.md
@@ -33,7 +33,7 @@ Feel free to contact us to be added to this list!
 ### Fedilab
 - Homepage: <https://fedilab.app/>
 - Source Code: <https://framagit.org/tom79/fedilab/>
-- Contact: [@fedilab@mastodon.social](https://framapiaf.org/users/fedilab)
+- Contact: [@fedilab@framapiaf.org](https://framapiaf.org/users/fedilab)
 - Platforms: Android
 - Features: Streaming Ready, Moderation, Text Formatting 
 

From c2e2aadc4254fe931ea519a9813854ccdac456b8 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Fri, 19 Jul 2019 16:20:23 +0000
Subject: [PATCH 39/40] #1110 fixed /api/pleroma/healthcheck

---
 .../controllers/util_controller.ex            | 32 ++++++----
 test/web/twitter_api/util_controller_test.exs | 64 ++++++++++++++++++-
 2 files changed, 79 insertions(+), 17 deletions(-)

diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index c10c66ff2..9e4da7dca 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -8,7 +8,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   require Logger
 
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Emoji
+  alias Pleroma.Healthcheck
   alias Pleroma.Notification
   alias Pleroma.Plugs.AuthenticationPlug
   alias Pleroma.User
@@ -23,7 +25,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   end
 
   def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do
-    with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do
+    with %User{} = user <- User.get_cached_by_nickname(nick),
+         avatar = User.avatar_url(user) do
       conn
       |> render("subscribe.html", %{nickname: nick, avatar: avatar, error: false})
     else
@@ -338,20 +341,21 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   end
 
   def healthcheck(conn, _params) do
-    info =
-      if Pleroma.Config.get([:instance, :healthcheck]) do
-        Pleroma.Healthcheck.system_info()
-      else
-        %{}
-      end
+    with true <- Config.get([:instance, :healthcheck]),
+         %{healthy: true} = info <- Healthcheck.system_info() do
+      json(conn, info)
+    else
+      %{healthy: false} = info ->
+        service_unavailable(conn, info)
 
-    conn =
-      if info[:healthy] do
-        conn
-      else
-        Plug.Conn.put_status(conn, :service_unavailable)
-      end
+      _ ->
+        service_unavailable(conn, %{})
+    end
+  end
 
-    json(conn, info)
+  defp service_unavailable(conn, info) do
+    conn
+    |> put_status(:service_unavailable)
+    |> json(info)
   end
 end
diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs
index 21324399f..3d699e1df 100644
--- a/test/web/twitter_api/util_controller_test.exs
+++ b/test/web/twitter_api/util_controller_test.exs
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   import Pleroma.Factory
+  import Mock
 
   setup do
     Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -231,10 +232,67 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
     end
   end
 
-  test "GET /api/pleroma/healthcheck", %{conn: conn} do
-    conn = get(conn, "/api/pleroma/healthcheck")
+  describe "GET /api/pleroma/healthcheck" do
+    setup do
+      config_healthcheck = Pleroma.Config.get([:instance, :healthcheck])
 
-    assert conn.status in [200, 503]
+      on_exit(fn ->
+        Pleroma.Config.put([:instance, :healthcheck], config_healthcheck)
+      end)
+
+      :ok
+    end
+
+    test "returns 503 when healthcheck disabled", %{conn: conn} do
+      Pleroma.Config.put([:instance, :healthcheck], false)
+
+      response =
+        conn
+        |> get("/api/pleroma/healthcheck")
+        |> json_response(503)
+
+      assert response == %{}
+    end
+
+    test "returns 200 when healthcheck enabled and all ok", %{conn: conn} do
+      Pleroma.Config.put([:instance, :healthcheck], true)
+
+      with_mock Pleroma.Healthcheck,
+        system_info: fn -> %Pleroma.Healthcheck{healthy: true} end do
+        response =
+          conn
+          |> get("/api/pleroma/healthcheck")
+          |> json_response(200)
+
+        assert %{
+                 "active" => _,
+                 "healthy" => true,
+                 "idle" => _,
+                 "memory_used" => _,
+                 "pool_size" => _
+               } = response
+      end
+    end
+
+    test "returns 503 when healthcheck enabled and  health is false", %{conn: conn} do
+      Pleroma.Config.put([:instance, :healthcheck], true)
+
+      with_mock Pleroma.Healthcheck,
+        system_info: fn -> %Pleroma.Healthcheck{healthy: false} end do
+        response =
+          conn
+          |> get("/api/pleroma/healthcheck")
+          |> json_response(503)
+
+        assert %{
+                 "active" => _,
+                 "healthy" => false,
+                 "idle" => _,
+                 "memory_used" => _,
+                 "pool_size" => _
+               } = response
+      end
+    end
   end
 
   describe "POST /api/pleroma/disable_account" do

From 9a8eb2c94d2243fadd69786eb74d94cc6116468f Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 19 Jul 2019 19:11:04 +0000
Subject: [PATCH 40/40] mix: add pleroma.user unsubscribe_all_from_instance

---
 lib/mix/tasks/pleroma/user.ex | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 8a78b4fe6..c9b84b8f9 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -62,6 +62,10 @@ defmodule Mix.Tasks.Pleroma.User do
 
       mix pleroma.user unsubscribe NICKNAME
 
+  ## Unsubscribe local users from an entire instance and deactivate all accounts
+
+      mix pleroma.user unsubscribe_all_from_instance INSTANCE
+
   ## Create a password reset link.
 
       mix pleroma.user reset_password NICKNAME
@@ -246,6 +250,20 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["unsubscribe_all_from_instance", instance]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{nickname: "@#{instance}"})
+    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user ->
+        run(["unsubscribe", user.nickname])
+      end)
+    end)
+    |> Stream.run()
+  end
+
   def run(["set", nickname | rest]) do
     start_pleroma()