Merge develop

CHANGELOG.md

@@ -5,10 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
-### Added
-- Experimental websocket-based federation between Pleroma instances.
-- User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
-
 ### Changed
 
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
@@ -19,6 +15,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 - Media preview proxy (requires media proxy be enabled; see `:media_preview_proxy` config for more details).
+- Pleroma API: Importing the mutes users from CSV files.
+- Experimental websocket-based federation between Pleroma instances.
+- Admin API: Importing emoji from a zip file
+- User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
 
 ### Removed
 
@@ -28,6 +28,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were
 switched to a new configuration mechanism, however it was not officially removed until now.
 
+### Fixed
+- Allow sending out emails again.
+
 ## [2.1.2] - 2020-09-17
 
 ### Security

README.md

@@ -18,15 +18,16 @@ If you are running Linux (glibc or musl) on x86/arm, the recommended way to inst
 ### From Source
 If your platform is not supported, or you just want to be able to edit the source code easily, you may install Pleroma from source.
 
-- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
-- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
 - [Alpine Linux](https://docs-develop.pleroma.social/backend/installation/alpine_linux_en/)
 - [Arch Linux](https://docs-develop.pleroma.social/backend/installation/arch_linux_en/)
+- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
+- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
+- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
+- [FreeBSD](https://docs-develop.pleroma.social/backend/installation/freebsd_en/)
 - [Gentoo Linux](https://docs-develop.pleroma.social/backend/installation/gentoo_en/)
 - [NetBSD](https://docs-develop.pleroma.social/backend/installation/netbsd_en/)
 - [OpenBSD](https://docs-develop.pleroma.social/backend/installation/openbsd_en/)
 - [OpenBSD (fi)](https://docs-develop.pleroma.social/backend/installation/openbsd_fi/)
-- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
 
 ### OS/Distro packages
 Currently Pleroma is not packaged by any OS/Distros, but if you want to package it for one, we can guide you through the process on our [community channels](#community-channels). If you want to change default options in your Pleroma package, please **discuss it with us first**.

config/description.exs

@@ -2445,7 +2445,7 @@ config :pleroma, :config_description, [
   %{
     group: :pleroma,
     key: Pleroma.Formatter,
-    label: "Auto Linker",
+    label: "Linkify",
     type: :group,
     description:
       "Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.",

docs/API/pleroma_api.md

@@ -44,6 +44,22 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: HTTP 200 on success, 500 on error
 * Note: Users that can't be followed are silently skipped.
 
+## `/api/pleroma/blocks_import`
+### Imports your blocks.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to block
+* Response: HTTP 200 on success, 500 on error
+
+## `/api/pleroma/mutes_import`
+### Imports your mutes.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to mute
+* Response: HTTP 200 on success, 500 on error
+
 ## `/api/pleroma/captcha`
 ### Get a new captcha
 * Method: `GET`

installation/pleroma.nginx

@@ -9,6 +9,12 @@
 proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m max_size=10g
                  inactive=720m use_temp_path=off;
 
+# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+# and `localhost.` resolves to [::0] on some systems: see issue #930
+upstream phoenix {
+    server 127.0.0.1:4000 max_fails=5 fail_timeout=60s;
+}
+
 server {
     server_name    example.tld;
 
@@ -63,19 +69,16 @@ server {
 
     # the nginx default is 1m, not enough for large media uploads
     client_max_body_size 16m;
+    ignore_invalid_headers off;
 
-    location / {
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
     proxy_set_header Host $http_host;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
-        # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+    location / {
-        # and `localhost.` resolves to [::0] on some systems: see issue #930
+        proxy_pass http://phoenix;
-        proxy_pass http://127.0.0.1:4000;
-
-        client_max_body_size 16m;
     }
 
     location ~ ^/(media|proxy) {
@@ -83,12 +86,16 @@ server {
         slice              1m;
         proxy_cache_key    $host$uri$is_args$args$slice_range;
         proxy_set_header   Range $slice_range;
-        proxy_http_version 1.1;
         proxy_cache_valid  200 206 301 304 1h;
         proxy_cache_lock   on;
         proxy_ignore_client_abort on;
         proxy_buffering    on;
         chunked_transfer_encoding on;
-        proxy_pass         http://127.0.0.1:4000;
+        proxy_pass         http://phoenix;
+    }
+
+    location /api/fedsocket/v1 {
+        proxy_request_buffering off;
+        proxy_pass http://phoenix/api/fedsocket/v1;
     }
 }

lib/pleroma/emails/mailer.ex

@@ -35,6 +35,11 @@ defmodule Pleroma.Emails.Mailer do
   def deliver(email, config \\ [])
 
   def deliver(email, config) do
+    # temporary hackney fix until hackney max_connections bug is fixed
+    # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+    email =
+      Swoosh.Email.put_private(email, :hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+
     case enabled?() do
       true -> Swoosh.Mailer.deliver(email, parse_config(config))
       false -> {:error, :deliveries_disabled}

lib/pleroma/emoji.ex

@@ -56,6 +56,9 @@ defmodule Pleroma.Emoji do
     end
   end
 
+  @spec exist?(String.t()) :: boolean()
+  def exist?(name), do: not is_nil(get(name))
+
   @doc "Returns all the emojos!!"
   @spec get_all() :: list({String.t(), String.t(), String.t()})
   def get_all do

lib/pleroma/emoji/pack.ex

@@ -17,6 +17,7 @@ defmodule Pleroma.Emoji.Pack do
         }
 
   alias Pleroma.Emoji
+  alias Pleroma.Emoji.Pack
 
   @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
@@ -64,24 +65,93 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
+  @spec unpack_zip_emojies(list(tuple())) :: list(map())
-          {:ok, t()} | {:error, File.posix() | atom()}
+  defp unpack_zip_emojies(zip_files) do
-  def add_file(name, shortcode, filename, file) do
+    Enum.reduce(zip_files, [], fn
-    with :ok <- validate_not_empty([name, shortcode, filename]),
+      {_, path, s, _, _, _}, acc when elem(s, 2) == :regular ->
+        with(
+          filename <- Path.basename(path),
+          shortcode <- Path.basename(filename, Path.extname(filename)),
+          false <- Emoji.exist?(shortcode)
+        ) do
+          [%{path: path, filename: path, shortcode: shortcode} | acc]
+        else
+          _ -> acc
+        end
+
+      _, acc ->
+        acc
+    end)
+  end
+
+  @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
+          {:ok, t()}
+          | {:error, File.posix() | atom()}
+  def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
+    with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
+         [_ | _] = emojies <- unpack_zip_emojies(zip_files),
+         {:ok, tmp_dir} <- Pleroma.Utils.tmp_dir("emoji") do
+      try do
+        {:ok, _emoji_files} =
+          :zip.unzip(
+            to_charlist(file.path),
+            [{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, tmp_dir}]
+          )
+
+        {_, updated_pack} =
+          Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
+            emoji_file = %Plug.Upload{
+              filename: item[:filename],
+              path: Path.join(tmp_dir, item[:path])
+            }
+
+            {:ok, updated_pack} =
+              do_add_file(
+                emoji_pack,
+                item[:shortcode],
+                to_string(item[:filename]),
+                emoji_file
+              )
+
+            {item, updated_pack}
+          end)
+
+        Emoji.reload()
+
+        {:ok, updated_pack}
+      after
+        File.rm_rf(tmp_dir)
+      end
+    else
+      {:error, _} = error ->
+        error
+
+      _ ->
+        {:ok, pack}
+    end
+  end
+
+  def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do
+    with :ok <- validate_not_empty([shortcode, filename]),
          :ok <- validate_emoji_not_exists(shortcode),
-         {:ok, pack} <- load_pack(name),
+         {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
-         :ok <- save_file(file, pack, filename),
-         {:ok, updated_pack} <- pack |> put_emoji(shortcode, filename) |> save_pack() do
       Emoji.reload()
       {:ok, updated_pack}
     end
   end
 
-  @spec delete_file(String.t(), String.t()) ::
+  defp do_add_file(pack, shortcode, filename, file) do
+    with :ok <- save_file(file, pack, filename) do
+      pack
+      |> put_emoji(shortcode, filename)
+      |> save_pack()
+    end
+  end
+
+  @spec delete_file(t(), String.t()) ::
           {:ok, t()} | {:error, File.posix() | atom()}
-  def delete_file(name, shortcode) do
+  def delete_file(%Pack{} = pack, shortcode) do
-    with :ok <- validate_not_empty([name, shortcode]),
+    with :ok <- validate_not_empty([shortcode]),
-         {:ok, pack} <- load_pack(name),
          :ok <- remove_file(pack, shortcode),
          {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
       Emoji.reload()
@@ -89,11 +159,10 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) ::
+  @spec update_file(t(), String.t(), String.t(), String.t(), boolean()) ::
           {:ok, t()} | {:error, File.posix() | atom()}
-  def update_file(name, shortcode, new_shortcode, new_filename, force) do
+  def update_file(%Pack{} = pack, shortcode, new_shortcode, new_filename, force) do
-    with :ok <- validate_not_empty([name, shortcode, new_shortcode, new_filename]),
+    with :ok <- validate_not_empty([shortcode, new_shortcode, new_filename]),
-         {:ok, pack} <- load_pack(name),
          {:ok, filename} <- get_filename(pack, shortcode),
          :ok <- validate_emoji_not_exists(new_shortcode, force),
          :ok <- rename_file(pack, filename, new_filename),
@@ -243,9 +312,10 @@ defmodule Pleroma.Emoji.Pack do
   defp validate_emoji_not_exists(_shortcode, true), do: :ok
 
   defp validate_emoji_not_exists(shortcode, _) do
-    case Emoji.get(shortcode) do
+    if Emoji.exist?(shortcode) do
-      nil -> :ok
+      {:error, :already_exists}
-      _ -> {:error, :already_exists}
+    else
+      :ok
     end
   end
 
@@ -386,25 +456,18 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  defp save_file(file, pack, filename) do
+  defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
     file_path = Path.join(pack.path, filename)
     create_subdirs(file_path)
 
-    case file do
+    with {:ok, _} <- File.copy(upload_path, file_path) do
-      %Plug.Upload{path: upload_path} ->
+      :ok
-        # Copy the uploaded file from the temporary directory
-        with {:ok, _} <- File.copy(upload_path, file_path), do: :ok
-
-      url when is_binary(url) ->
-        # Download and write the file
-        file_contents = Tesla.get!(url).body
-        File.write(file_path, file_contents)
     end
   end
 
   defp put_emoji(pack, shortcode, filename) do
     files = Map.put(pack.files, shortcode, filename)
-    %{pack | files: files}
+    %{pack | files: files, files_count: length(Map.keys(files))}
   end
 
   defp delete_emoji(pack, shortcode) do

lib/pleroma/plugs/oauth_scopes_plug.ex

@@ -53,7 +53,7 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
     |> assign(:token, nil)
   end
 
-  @doc "Filters descendants of supported scopes"
+  @doc "Keeps those of `scopes` which are descendants of `supported_scopes`"
   def filter_descendants(scopes, supported_scopes) do
     Enum.filter(
       scopes,

lib/pleroma/user.ex

@@ -1719,42 +1719,6 @@ defmodule Pleroma.User do
 
   def perform(:deactivate_async, user, status), do: deactivate(user, status)
 
-  @spec perform(atom(), User.t(), list()) :: list() | {:error, any()}
-  def perform(:blocks_import, %User{} = blocker, blocked_identifiers)
-      when is_list(blocked_identifiers) do
-    Enum.map(
-      blocked_identifiers,
-      fn blocked_identifier ->
-        with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
-             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
-          blocked
-        else
-          err ->
-            Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
-  def perform(:follow_import, %User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    Enum.map(
-      followed_identifiers,
-      fn followed_identifier ->
-        with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier),
-             {:ok, follower} <- maybe_direct_follow(follower, followed),
-             {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
-          followed
-        else
-          err ->
-            Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
   @spec external_users_query() :: Ecto.Query.t()
   def external_users_query do
     User.Query.build(%{
@@ -1783,21 +1747,6 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
-    BackgroundWorker.enqueue("blocks_import", %{
-      "blocker_id" => blocker.id,
-      "blocked_identifiers" => blocked_identifiers
-    })
-  end
-
-  def follow_import(%User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    BackgroundWorker.enqueue("follow_import", %{
-      "follower_id" => follower.id,
-      "followed_identifiers" => followed_identifiers
-    })
-  end
-
   def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
     Notification
     |> join(:inner, [n], activity in assoc(n, :activity))

lib/pleroma/user/import.ex

@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Import do
+  use Ecto.Schema
+
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Workers.BackgroundWorker
+
+  require Logger
+
+  @spec perform(atom(), User.t(), list()) :: :ok | list() | {:error, any()}
+  def perform(:mutes_import, %User{} = user, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = muted_user} <- User.get_or_fetch(identifier),
+             {:ok, _} <- User.mute(user, muted_user) do
+          muted_user
+        else
+          error -> handle_error(:mutes_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(:blocks_import, %User{} = blocker, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = blocked} <- User.get_or_fetch(identifier),
+             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
+          blocked
+        else
+          error -> handle_error(:blocks_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(:follow_import, %User{} = follower, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = followed} <- User.get_or_fetch(identifier),
+             {:ok, follower} <- User.maybe_direct_follow(follower, followed),
+             {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
+          followed
+        else
+          error -> handle_error(:follow_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(_, _, _), do: :ok
+
+  defp handle_error(op, user_id, error) do
+    Logger.debug("#{op} failed for #{user_id} with: #{inspect(error)}")
+    error
+  end
+
+  def blocks_import(%User{} = blocker, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "blocks_import",
+      %{"user_id" => blocker.id, "identifiers" => identifiers}
+    )
+  end
+
+  def follow_import(%User{} = follower, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "follow_import",
+      %{"user_id" => follower.id, "identifiers" => identifiers}
+    )
+  end
+
+  def mutes_import(%User{} = user, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "mutes_import",
+      %{"user_id" => user.id, "identifiers" => identifiers}
+    )
+  end
+end

lib/pleroma/utils.ex

@@ -24,4 +24,24 @@ defmodule Pleroma.Utils do
   def command_available?(command) do
     match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
   end
+
+  @doc "creates the uniq temporary directory"
+  @spec tmp_dir(String.t()) :: {:ok, String.t()} | {:error, :file.posix()}
+  def tmp_dir(prefix \\ "") do
+    sub_dir =
+      [
+        prefix,
+        Timex.to_unix(Timex.now()),
+        :os.getpid(),
+        String.downcase(Integer.to_string(:rand.uniform(0x100000000), 36))
+      ]
+      |> Enum.join("-")
+
+    tmp_dir = Path.join(System.tmp_dir!(), sub_dir)
+
+    case File.mkdir(tmp_dir) do
+      :ok -> {:ok, tmp_dir}
+      error -> error
+    end
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -841,7 +841,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     from(
       [activity, object: o] in query,
       where: fragment("not (? = ANY(?))", activity.actor, ^blocked_ap_ids),
-      where: fragment("not (? && ?)", activity.recipients, ^blocked_ap_ids),
+      where:
+        fragment(
+          "((not (? && ?)) or ? = ?)",
+          activity.recipients,
+          ^blocked_ap_ids,
+          activity.actor,
+          ^user.ap_id
+        ),
       where:
         fragment(
           "recipients_contain_blocked_domains(?, ?) = false",

lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex

@@ -0,0 +1,139 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaEmojiFileOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Add new file to the pack",
+      operationId: "PleromaAPI.EmojiPackController.add_file",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", create_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp create_request do
+    %Schema{
+      type: :object,
+      required: [:file],
+      properties: %{
+        file: %Schema{
+          description:
+            "File needs to be uploaded with the multipart request or link to remote file",
+          anyOf: [
+            %Schema{type: :string, format: :binary},
+            %Schema{type: :string, format: :uri}
+          ]
+        },
+        shortcode: %Schema{
+          type: :string,
+          description:
+            "Shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename."
+        },
+        filename: %Schema{
+          type: :string,
+          description:
+            "New emoji file name. If not specified will be taken from original filename."
+        }
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Add new file to the pack",
+      operationId: "PleromaAPI.EmojiPackController.update_file",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", update_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      type: :object,
+      required: [:shortcode, :new_shortcode, :new_filename],
+      properties: %{
+        shortcode: %Schema{
+          type: :string,
+          description: "Emoji file shortcode"
+        },
+        new_shortcode: %Schema{
+          type: :string,
+          description: "New emoji file shortcode"
+        },
+        new_filename: %Schema{
+          type: :string,
+          description: "New filename for emoji file"
+        },
+        force: %Schema{
+          type: :boolean,
+          description: "With true value to overwrite existing emoji with new shortcode",
+          default: false
+        }
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Delete emoji file from pack",
+      operationId: "PleromaAPI.EmojiPackController.delete_file",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        name_param(),
+        Operation.parameter(:shortcode, :query, :string, "File shortcode",
+          example: "cofe",
+          required: true
+        )
+      ],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp name_param do
+    Operation.parameter(:name, :path, :string, "Pack Name", example: "cofe", required: true)
+  end
+
+  defp files_object do
+    %Schema{
+      type: :object,
+      additionalProperties: %Schema{type: :string},
+      description: "Object with emoji names as keys and filenames as values"
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex

@@ -175,111 +175,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
     }
   end
 
-  def add_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Add new file to the pack",
-      operationId: "PleromaAPI.EmojiPackController.add_file",
-      security: [%{"oAuth" => ["write"]}],
-      requestBody: request_body("Parameters", add_file_request(), required: true),
-      parameters: [name_param()],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError),
-        409 => Operation.response("Conflict", "application/json", ApiError)
-      }
-    }
-  end
-
-  defp add_file_request do
-    %Schema{
-      type: :object,
-      required: [:file],
-      properties: %{
-        file: %Schema{
-          description:
-            "File needs to be uploaded with the multipart request or link to remote file",
-          anyOf: [
-            %Schema{type: :string, format: :binary},
-            %Schema{type: :string, format: :uri}
-          ]
-        },
-        shortcode: %Schema{
-          type: :string,
-          description:
-            "Shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename."
-        },
-        filename: %Schema{
-          type: :string,
-          description:
-            "New emoji file name. If not specified will be taken from original filename."
-        }
-      }
-    }
-  end
-
-  def update_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Add new file to the pack",
-      operationId: "PleromaAPI.EmojiPackController.update_file",
-      security: [%{"oAuth" => ["write"]}],
-      requestBody: request_body("Parameters", update_file_request(), required: true),
-      parameters: [name_param()],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError),
-        409 => Operation.response("Conflict", "application/json", ApiError)
-      }
-    }
-  end
-
-  defp update_file_request do
-    %Schema{
-      type: :object,
-      required: [:shortcode, :new_shortcode, :new_filename],
-      properties: %{
-        shortcode: %Schema{
-          type: :string,
-          description: "Emoji file shortcode"
-        },
-        new_shortcode: %Schema{
-          type: :string,
-          description: "New emoji file shortcode"
-        },
-        new_filename: %Schema{
-          type: :string,
-          description: "New filename for emoji file"
-        },
-        force: %Schema{
-          type: :boolean,
-          description: "With true value to overwrite existing emoji with new shortcode",
-          default: false
-        }
-      }
-    }
-  end
-
-  def delete_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Delete emoji file from pack",
-      operationId: "PleromaAPI.EmojiPackController.delete_file",
-      security: [%{"oAuth" => ["write"]}],
-      parameters: [
-        name_param(),
-        Operation.parameter(:shortcode, :query, :string, "File shortcode",
-          example: "cofe",
-          required: true
-        )
-      ],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError)
-      }
-    }
-  end
-
   def import_from_filesystem_operation do
     %Operation{
       tags: ["Emoji Packs"],

lib/pleroma/web/api_spec/operations/user_import_operation.ex

@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.UserImportOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  @spec open_api_operation(atom) :: Operation.t()
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def follow_operation do
+    %Operation{
+      tags: ["follow_import"],
+      summary: "Imports your follows.",
+      operationId: "UserImportController.follow",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:follow"]}]
+    }
+  end
+
+  def blocks_operation do
+    %Operation{
+      tags: ["blocks_import"],
+      summary: "Imports your blocks.",
+      operationId: "UserImportController.blocks",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:blocks"]}]
+    }
+  end
+
+  def mutes_operation do
+    %Operation{
+      tags: ["mutes_import"],
+      summary: "Imports your mutes.",
+      operationId: "UserImportController.mutes",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:mutes"]}]
+    }
+  end
+
+  defp import_request do
+    %Schema{
+      type: :object,
+      required: [:list],
+      properties: %{
+        list: %Schema{
+          description:
+            "STRING or FILE containing a whitespace-separated list of accounts to import.",
+          anyOf: [
+            %Schema{type: :string, format: :binary},
+            %Schema{type: :string}
+          ]
+        }
+      }
+    }
+  end
+
+  defp ok_response do
+    Operation.response("Ok", "application/json", %Schema{type: :string, example: "ok"})
+  end
+end

lib/pleroma/web/fed_sockets/outgoing_handler.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.FedSockets.OutgoingHandler do
 
   require Logger
 
+  alias Pleroma.Application
   alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.FedSockets
   alias Pleroma.Web.FedSockets.FedRegistry
@@ -85,9 +86,12 @@ defmodule Pleroma.Web.FedSockets.OutgoingHandler do
 
     %{host: host, port: port, path: path} = URI.parse(ws_uri)
 
-    with {:ok, conn_pid} <- :gun.open(to_charlist(host), port),
+    with {:ok, conn_pid} <- :gun.open(to_charlist(host), port, %{protocols: [:http]}),
          {:ok, _} <- :gun.await_up(conn_pid),
-         reference <- :gun.get(conn_pid, to_charlist(path)),
+         reference <-
+           :gun.get(conn_pid, to_charlist(path), [
+             {'user-agent', to_charlist(Application.user_agent())}
+           ]),
          {:response, :fin, 204, _} <- :gun.await(conn_pid, reference),
          headers <- build_headers(uri),
          ref <- :gun.ws_upgrade(conn_pid, to_charlist(path), headers, %{silence_pings: false}) do
@@ -132,7 +136,8 @@ defmodule Pleroma.Web.FedSockets.OutgoingHandler do
       {'date', date},
       {'digest', to_charlist(digest)},
       {'content-length', to_charlist("#{shake_size}")},
-      {to_charlist("(request-target)"), to_charlist(shake)}
+      {to_charlist("(request-target)"), to_charlist(shake)},
+      {'user-agent', to_charlist(Application.user_agent())}
     ]
   end
 

lib/pleroma/web/federator/federator.ex

@@ -66,14 +66,17 @@ defmodule Pleroma.Web.Federator do
   def perform(:incoming_ap_doc, params) do
     Logger.debug("Handling incoming AP activity")
 
-    params = Utils.normalize_params(params)
+    actor =
+      params
+      |> Map.get("actor")
+      |> Utils.get_ap_id()
 
     # NOTE: we use the actor ID to do the containment, this is fine because an
     # actor shouldn't be acting on objects outside their own AP server.
-    with {:ok, _user} <- ap_enabled_actor(params["actor"]),
+    with {_, {:ok, _user}} <- {:actor, ap_enabled_actor(actor)},
          nil <- Activity.normalize(params["id"]),
          {_, :ok} <-
-           {:correct_origin?, Containment.contain_origin_from_id(params["actor"], params)},
+           {:correct_origin?, Containment.contain_origin_from_id(actor, params)},
          {:ok, activity} <- Transmogrifier.handle_incoming(params) do
       {:ok, activity}
     else
@@ -85,10 +88,13 @@ defmodule Pleroma.Web.Federator do
         Logger.debug("Already had #{params["id"]}")
         {:error, :already_present}
 
+      {:actor, e} ->
+        Logger.debug("Unhandled actor #{actor}, #{inspect(e)}")
+        {:error, e}
+
       e ->
         # Just drop those for now
-        Logger.debug("Unhandled activity")
+        Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end)
-        Logger.debug(Jason.encode!(params, pretty: true))
         {:error, e}
     end
   end

lib/pleroma/web/mastodon_api/controllers/auth_controller.ex

@@ -5,6 +5,8 @@
 defmodule Pleroma.Web.MastodonAPI.AuthController do
   use Pleroma.Web, :controller
 
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
   alias Pleroma.User
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
@@ -61,9 +63,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do
 
     TwitterAPI.password_reset(nickname_or_email)
 
-    conn
+    json_response(conn, :no_content, "")
-    |> put_status(:no_content)
-    |> json("")
   end
 
   defp local_mastodon_root_path(conn) do

lib/pleroma/web/mastodon_api/websocket_handler.ex

@@ -23,8 +23,8 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     with params <- Enum.into(:cow_qs.parse_qs(qs), %{}),
          sec_websocket <- :cowboy_req.header("sec-websocket-protocol", req, nil),
          access_token <- Map.get(params, "access_token"),
-         {:ok, user} <- authenticate_request(access_token, sec_websocket),
+         {:ok, user, oauth_token} <- authenticate_request(access_token, sec_websocket),
-         {:ok, topic} <- Streamer.get_topic(Map.get(params, "stream"), user, params) do
+         {:ok, topic} <- Streamer.get_topic(params["stream"], user, oauth_token, params) do
       req =
         if sec_websocket do
           :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
@@ -117,7 +117,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
 
   # Public streams without authentication.
   defp authenticate_request(nil, nil) do
-    {:ok, nil}
+    {:ok, nil, nil}
   end
 
   # Authenticated streams.
@@ -125,9 +125,9 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     token = access_token || sec_websocket
 
     with true <- is_bitstring(token),
-         %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
+         oauth_token = %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
          user = %User{} <- User.get_cached_by_id(user_id) do
-      {:ok, user}
+      {:ok, user, oauth_token}
     else
       _ -> {:error, :unauthorized}
     end

lib/pleroma/web/pleroma_api/controllers/emoji_file_controller.ex

@@ -0,0 +1,133 @@
+defmodule Pleroma.Web.PleromaAPI.EmojiFileController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Emoji.Pack
+  alias Pleroma.Web.ApiSpec
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    Pleroma.Plugs.OAuthScopesPlug,
+    %{scopes: ["write"], admin: true}
+    when action in [
+           :create,
+           :update,
+           :delete
+         ]
+  )
+
+  defdelegate open_api_operation(action), to: ApiSpec.PleromaEmojiFileOperation
+
+  def create(%{body_params: params} = conn, %{name: pack_name}) do
+    filename = params[:filename] || get_filename(params[:file])
+    shortcode = params[:shortcode] || Path.basename(filename, Path.extname(filename))
+
+    with {:ok, pack} <- Pack.load_pack(pack_name),
+         {:ok, file} <- get_file(params[:file]),
+         {:ok, pack} <- Pack.add_file(pack, shortcode, filename, file) do
+      json(conn, pack.files)
+    else
+      {:error, :already_exists} ->
+        conn
+        |> put_status(:conflict)
+        |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"})
+
+      {:error, :empty_values} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: "pack name, shortcode or filename cannot be empty"})
+
+      {:error, _} = error ->
+        handle_error(conn, error, %{pack_name: pack_name})
+    end
+  end
+
+  def update(%{body_params: %{shortcode: shortcode} = params} = conn, %{name: pack_name}) do
+    new_shortcode = params[:new_shortcode]
+    new_filename = params[:new_filename]
+    force = params[:force]
+
+    with {:ok, pack} <- Pack.load_pack(pack_name),
+         {:ok, pack} <- Pack.update_file(pack, shortcode, new_shortcode, new_filename, force) do
+      json(conn, pack.files)
+    else
+      {:error, :already_exists} ->
+        conn
+        |> put_status(:conflict)
+        |> json(%{
+          error:
+            "New shortcode \"#{new_shortcode}\" is already used. If you want to override emoji use 'force' option"
+        })
+
+      {:error, :empty_values} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: "new_shortcode or new_filename cannot be empty"})
+
+      {:error, _} = error ->
+        handle_error(conn, error, %{pack_name: pack_name, code: shortcode})
+    end
+  end
+
+  def delete(conn, %{name: pack_name, shortcode: shortcode}) do
+    with {:ok, pack} <- Pack.load_pack(pack_name),
+         {:ok, pack} <- Pack.delete_file(pack, shortcode) do
+      json(conn, pack.files)
+    else
+      {:error, :empty_values} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: "pack name or shortcode cannot be empty"})
+
+      {:error, _} = error ->
+        handle_error(conn, error, %{pack_name: pack_name, code: shortcode})
+    end
+  end
+
+  defp handle_error(conn, {:error, :doesnt_exist}, %{code: emoji_code}) do
+    conn
+    |> put_status(:bad_request)
+    |> json(%{error: "Emoji \"#{emoji_code}\" does not exist"})
+  end
+
+  defp handle_error(conn, {:error, :not_found}, %{pack_name: pack_name}) do
+    conn
+    |> put_status(:not_found)
+    |> json(%{error: "pack \"#{pack_name}\" is not found"})
+  end
+
+  defp handle_error(conn, {:error, _}, _) do
+    render_error(
+      conn,
+      :internal_server_error,
+      "Unexpected error occurred while adding file to pack."
+    )
+  end
+
+  defp get_filename(%Plug.Upload{filename: filename}), do: filename
+  defp get_filename(url) when is_binary(url), do: Path.basename(url)
+
+  def get_file(%Plug.Upload{} = file), do: {:ok, file}
+
+  def get_file(url) when is_binary(url) do
+    with {:ok, %Tesla.Env{body: body, status: code, headers: headers}}
+         when code in 200..299 <- Pleroma.HTTP.get(url) do
+      path = Plug.Upload.random_file!("emoji")
+
+      content_type =
+        case List.keyfind(headers, "content-type", 0) do
+          {"content-type", value} -> value
+          nil -> nil
+        end
+
+      File.write(path, body)
+
+      {:ok,
+       %Plug.Upload{
+         filename: Path.basename(url),
+         path: path,
+         content_type: content_type
+       }}
+    end
+  end
+end

lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex

@@ -14,10 +14,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
            :download,
            :create,
            :update,
-           :delete,
+           :delete
-           :add_file,
-           :update_file,
-           :delete_file
          ]
   )
 
@@ -184,105 +181,6 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
     end
   end
 
-  def add_file(%{body_params: params} = conn, %{name: name}) do
-    filename = params[:filename] || get_filename(params[:file])
-    shortcode = params[:shortcode] || Path.basename(filename, Path.extname(filename))
-
-    with {:ok, pack} <- Pack.add_file(name, shortcode, filename, params[:file]) do
-      json(conn, pack.files)
-    else
-      {:error, :already_exists} ->
-        conn
-        |> put_status(:conflict)
-        |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"})
-
-      {:error, :not_found} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack \"#{name}\" is not found"})
-
-      {:error, :empty_values} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack name, shortcode or filename cannot be empty"})
-
-      {:error, _} ->
-        render_error(
-          conn,
-          :internal_server_error,
-          "Unexpected error occurred while adding file to pack."
-        )
-    end
-  end
-
-  def update_file(%{body_params: %{shortcode: shortcode} = params} = conn, %{name: name}) do
-    new_shortcode = params[:new_shortcode]
-    new_filename = params[:new_filename]
-    force = params[:force]
-
-    with {:ok, pack} <- Pack.update_file(name, shortcode, new_shortcode, new_filename, force) do
-      json(conn, pack.files)
-    else
-      {:error, :doesnt_exist} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "Emoji \"#{shortcode}\" does not exist"})
-
-      {:error, :already_exists} ->
-        conn
-        |> put_status(:conflict)
-        |> json(%{
-          error:
-            "New shortcode \"#{new_shortcode}\" is already used. If you want to override emoji use 'force' option"
-        })
-
-      {:error, :not_found} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack \"#{name}\" is not found"})
-
-      {:error, :empty_values} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "new_shortcode or new_filename cannot be empty"})
-
-      {:error, _} ->
-        render_error(
-          conn,
-          :internal_server_error,
-          "Unexpected error occurred while updating file in pack."
-        )
-    end
-  end
-
-  def delete_file(conn, %{name: name, shortcode: shortcode}) do
-    with {:ok, pack} <- Pack.delete_file(name, shortcode) do
-      json(conn, pack.files)
-    else
-      {:error, :doesnt_exist} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "Emoji \"#{shortcode}\" does not exist"})
-
-      {:error, :not_found} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack \"#{name}\" is not found"})
-
-      {:error, :empty_values} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack name or shortcode cannot be empty"})
-
-      {:error, _} ->
-        render_error(
-          conn,
-          :internal_server_error,
-          "Unexpected error occurred while removing file from pack."
-        )
-    end
-  end
-
   def import_from_filesystem(conn, _params) do
     with {:ok, names} <- Pack.import_from_filesystem() do
       json(conn, names)
@@ -298,7 +196,4 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
         |> json(%{error: "Error accessing emoji pack directory"})
     end
   end
-
-  defp get_filename(%Plug.Upload{filename: filename}), do: filename
-  defp get_filename(url) when is_binary(url), do: Path.basename(url)
 end

lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex

@@ -0,0 +1,61 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.UserImportController do
+  use Pleroma.Web, :controller
+
+  require Logger
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.User
+  alias Pleroma.Web.ApiSpec
+
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:follows"]} when action == :follow)
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks)
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action == :mutes)
+
+  plug(OpenApiSpex.Plug.CastAndValidate)
+  defdelegate open_api_operation(action), to: ApiSpec.UserImportOperation
+
+  def follow(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    follow(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
+  end
+
+  def follow(%{assigns: %{user: follower}, body_params: %{list: list}} = conn, _) do
+    identifiers =
+      list
+      |> String.split("\n")
+      |> Enum.map(&(&1 |> String.split(",") |> List.first()))
+      |> List.delete("Account address")
+      |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@")))
+      |> Enum.reject(&(&1 == ""))
+
+    User.Import.follow_import(follower, identifiers)
+    json(conn, "job started")
+  end
+
+  def blocks(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    blocks(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
+  end
+
+  def blocks(%{assigns: %{user: blocker}, body_params: %{list: list}} = conn, _) do
+    User.Import.blocks_import(blocker, prepare_user_identifiers(list))
+    json(conn, "job started")
+  end
+
+  def mutes(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    mutes(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
+  end
+
+  def mutes(%{assigns: %{user: user}, body_params: %{list: list}} = conn, _) do
+    User.Import.mutes_import(user, prepare_user_identifiers(list))
+    json(conn, "job started")
+  end
+
+  defp prepare_user_identifiers(list) do
+    list
+    |> String.split()
+    |> Enum.map(&String.trim_leading(&1, "@"))
+  end
+end

lib/pleroma/web/router.ex

@@ -238,9 +238,9 @@ defmodule Pleroma.Web.Router do
       patch("/:name", EmojiPackController, :update)
       delete("/:name", EmojiPackController, :delete)
 
-      post("/:name/files", EmojiPackController, :add_file)
+      post("/:name/files", EmojiFileController, :create)
-      patch("/:name/files", EmojiPackController, :update_file)
+      patch("/:name/files", EmojiFileController, :update)
-      delete("/:name/files", EmojiPackController, :delete_file)
+      delete("/:name/files", EmojiFileController, :delete)
     end
 
     # Pack info / downloading
@@ -269,14 +269,15 @@ defmodule Pleroma.Web.Router do
     post("/delete_account", UtilController, :delete_account)
     put("/notification_settings", UtilController, :update_notificaton_settings)
     post("/disable_account", UtilController, :disable_account)
-
-    post("/blocks_import", UtilController, :blocks_import)
-    post("/follow_import", UtilController, :follow_import)
   end
 
   scope "/api/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:authenticated_api)
 
+    post("/mutes_import", UserImportController, :mutes)
+    post("/blocks_import", UserImportController, :blocks)
+    post("/follow_import", UserImportController, :follow)
+
     get("/accounts/mfa", TwoFactorAuthenticationController, :settings)
     get("/accounts/mfa/backup_codes", TwoFactorAuthenticationController, :backup_codes)
     get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup)

lib/pleroma/web/streamer/streamer.ex

@@ -11,10 +11,12 @@ defmodule Pleroma.Web.Streamer do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.StreamerView
 
   @mix_env Mix.env()
@@ -26,53 +28,87 @@ defmodule Pleroma.Web.Streamer do
   @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"]
 
   @doc "Expands and authorizes a stream, and registers the process for streaming."
-  @spec get_topic_and_add_socket(stream :: String.t(), User.t() | nil, Map.t() | nil) ::
+  @spec get_topic_and_add_socket(
+          stream :: String.t(),
+          User.t() | nil,
+          Token.t() | nil,
+          Map.t() | nil
+        ) ::
           {:ok, topic :: String.t()} | {:error, :bad_topic} | {:error, :unauthorized}
-  def get_topic_and_add_socket(stream, user, params \\ %{}) do
+  def get_topic_and_add_socket(stream, user, oauth_token, params \\ %{}) do
-    case get_topic(stream, user, params) do
+    case get_topic(stream, user, oauth_token, params) do
       {:ok, topic} -> add_socket(topic, user)
       error -> error
     end
   end
 
   @doc "Expand and authorizes a stream"
-  @spec get_topic(stream :: String.t(), User.t() | nil, Map.t()) ::
+  @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) ::
           {:ok, topic :: String.t()} | {:error, :bad_topic}
-  def get_topic(stream, user, params \\ %{})
+  def get_topic(stream, user, oauth_token, params \\ %{})
 
   # Allow all public steams.
-  def get_topic(stream, _, _) when stream in @public_streams do
+  def get_topic(stream, _user, _oauth_token, _params) when stream in @public_streams do
     {:ok, stream}
   end
 
   # Allow all hashtags streams.
-  def get_topic("hashtag", _, %{"tag" => tag}) do
+  def get_topic("hashtag", _user, _oauth_token, %{"tag" => tag} = _params) do
     {:ok, "hashtag:" <> tag}
   end
 
   # Expand user streams.
-  def get_topic(stream, %User{} = user, _) when stream in @user_streams do
+  def get_topic(
-    {:ok, stream <> ":" <> to_string(user.id)}
+        stream,
+        %User{id: user_id} = user,
+        %Token{user_id: token_user_id} = oauth_token,
+        _params
+      )
+      when stream in @user_streams and user_id == token_user_id do
+    # Note: "read" works for all user streams (not mentioning it since it's an ancestor scope)
+    required_scopes =
+      if stream == "user:notification" do
+        ["read:notifications"]
+      else
+        ["read:statuses"]
       end
 
-  def get_topic(stream, _, _) when stream in @user_streams do
+    if OAuthScopesPlug.filter_descendants(required_scopes, oauth_token.scopes) == [] do
+      {:error, :unauthorized}
+    else
+      {:ok, stream <> ":" <> to_string(user.id)}
+    end
+  end
+
+  def get_topic(stream, _user, _oauth_token, _params) when stream in @user_streams do
     {:error, :unauthorized}
   end
 
   # List streams.
-  def get_topic("list", %User{} = user, %{"list" => id}) do
+  def get_topic(
-    if Pleroma.List.get(id, user) do
+        "list",
+        %User{id: user_id} = user,
+        %Token{user_id: token_user_id} = oauth_token,
+        %{"list" => id}
+      )
+      when user_id == token_user_id do
+    cond do
+      OAuthScopesPlug.filter_descendants(["read", "read:lists"], oauth_token.scopes) == [] ->
+        {:error, :unauthorized}
+
+      Pleroma.List.get(id, user) ->
         {:ok, "list:" <> to_string(id)}
-    else
+
+      true ->
         {:error, :bad_topic}
     end
   end
 
-  def get_topic("list", _, _) do
+  def get_topic("list", _user, _oauth_token, _params) do
     {:error, :unauthorized}
   end
 
-  def get_topic(_, _, _) do
+  def get_topic(_stream, _user, _oauth_token, _params) do
     {:error, :bad_topic}
   end
 

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -18,14 +18,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
 
   plug(Pleroma.Web.FederatingPlug when action == :remote_subscribe)
 
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["follow", "write:follows"]}
-    when action == :follow_import
-  )
-
-  plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks_import)
-
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:accounts"]}
@@ -104,33 +96,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
     end
   end
 
-  def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
-    follow_import(conn, %{"list" => File.read!(listfile.path)})
-  end
-
-  def follow_import(%{assigns: %{user: follower}} = conn, %{"list" => list}) do
-    followed_identifiers =
-      list
-      |> String.split("\n")
-      |> Enum.map(&(&1 |> String.split(",") |> List.first()))
-      |> List.delete("Account address")
-      |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@")))
-      |> Enum.reject(&(&1 == ""))
-
-    User.follow_import(follower, followed_identifiers)
-    json(conn, "job started")
-  end
-
-  def blocks_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
-    blocks_import(conn, %{"list" => File.read!(listfile.path)})
-  end
-
-  def blocks_import(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do
-    blocked_identifiers = list |> String.split() |> Enum.map(&String.trim_leading(&1, "@"))
-    User.blocks_import(blocker, blocked_identifiers)
-    json(conn, "job started")
-  end
-
   def change_password(%{assigns: %{user: user}} = conn, params) do
     case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
       {:ok, user} ->

lib/pleroma/workers/background_worker.ex

@@ -26,26 +26,10 @@ defmodule Pleroma.Workers.BackgroundWorker do
     User.perform(:force_password_reset, user)
   end
 
-  def perform(%Job{
+  def perform(%Job{args: %{"op" => op, "user_id" => user_id, "identifiers" => identifiers}})
-        args: %{
+      when op in ["blocks_import", "follow_import", "mutes_import"] do
-          "op" => "blocks_import",
+    user = User.get_cached_by_id(user_id)
-          "blocker_id" => blocker_id,
+    {:ok, User.Import.perform(String.to_atom(op), user, identifiers)}
-          "blocked_identifiers" => blocked_identifiers
-        }
-      }) do
-    blocker = User.get_cached_by_id(blocker_id)
-    {:ok, User.perform(:blocks_import, blocker, blocked_identifiers)}
-  end
-
-  def perform(%Job{
-        args: %{
-          "op" => "follow_import",
-          "follower_id" => follower_id,
-          "followed_identifiers" => followed_identifiers
-        }
-      }) do
-    follower = User.get_cached_by_id(follower_id)
-    {:ok, User.perform(:follow_import, follower, followed_identifiers)}
   end
 
   def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do

test/emoji/pack_test.exs

@@ -0,0 +1,93 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Emoji.PackTest do
+  use ExUnit.Case, async: true
+  alias Pleroma.Emoji.Pack
+
+  @emoji_path Path.join(
+                Pleroma.Config.get!([:instance, :static_dir]),
+                "emoji"
+              )
+
+  setup do
+    pack_path = Path.join(@emoji_path, "dump_pack")
+    File.mkdir(pack_path)
+
+    File.write!(Path.join(pack_path, "pack.json"), """
+    {
+    "files": { },
+    "pack": {
+    "description": "Dump pack", "homepage": "https://pleroma.social",
+    "license": "Test license", "share-files": true
+    }}
+    """)
+
+    {:ok, pack} = Pleroma.Emoji.Pack.load_pack("dump_pack")
+
+    on_exit(fn ->
+      File.rm_rf!(pack_path)
+    end)
+
+    {:ok, pack: pack}
+  end
+
+  describe "add_file/4" do
+    test "add emojies from zip file", %{pack: pack} do
+      file = %Plug.Upload{
+        content_type: "application/zip",
+        filename: "emojis.zip",
+        path: Path.absname("test/fixtures/emojis.zip")
+      }
+
+      {:ok, updated_pack} = Pack.add_file(pack, nil, nil, file)
+
+      assert updated_pack.files == %{
+               "a_trusted_friend-128" => "128px/a_trusted_friend-128.png",
+               "auroraborealis" => "auroraborealis.png",
+               "baby_in_a_box" => "1000px/baby_in_a_box.png",
+               "bear" => "1000px/bear.png",
+               "bear-128" => "128px/bear-128.png"
+             }
+
+      assert updated_pack.files_count == 5
+    end
+  end
+
+  test "returns error when zip file is bad", %{pack: pack} do
+    file = %Plug.Upload{
+      content_type: "application/zip",
+      filename: "emojis.zip",
+      path: Path.absname("test/instance_static/emoji/test_pack/blank.png")
+    }
+
+    assert Pack.add_file(pack, nil, nil, file) == {:error, :einval}
+  end
+
+  test "returns pack when zip file is empty", %{pack: pack} do
+    file = %Plug.Upload{
+      content_type: "application/zip",
+      filename: "emojis.zip",
+      path: Path.absname("test/fixtures/empty.zip")
+    }
+
+    {:ok, updated_pack} = Pack.add_file(pack, nil, nil, file)
+    assert updated_pack == pack
+  end
+
+  test "add emoji file", %{pack: pack} do
+    file = %Plug.Upload{
+      filename: "blank.png",
+      path: "#{@emoji_path}/test_pack/blank.png"
+    }
+
+    {:ok, updated_pack} = Pack.add_file(pack, "test_blank", "test_blank.png", file)
+
+    assert updated_pack.files == %{
+             "test_blank" => "test_blank.png"
+           }
+
+    assert updated_pack.files_count == 1
+  end
+end

test/integration/mastodon_websocket_test.exs

@@ -78,7 +78,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do
         Pleroma.Repo.insert(
           OAuth.App.register_changeset(%OAuth.App{}, %{
             client_name: "client",
-            scopes: ["scope"],
+            scopes: ["read"],
             redirect_uris: "url"
           })
         )

test/notification_test.exs

@@ -179,17 +179,19 @@ defmodule Pleroma.NotificationTest do
   describe "create_notification" do
     @tag needs_streamer: true
     test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do
-      user = insert(:user)
+      %{user: user, token: oauth_token} = oauth_access(["read"])
 
       task =
         Task.async(fn ->
-          Streamer.get_topic_and_add_socket("user", user)
+          {:ok, _topic} = Streamer.get_topic_and_add_socket("user", user, oauth_token)
           assert_receive {:render_with_user, _, _, _}, 4_000
         end)
 
       task_user_notification =
         Task.async(fn ->
-          Streamer.get_topic_and_add_socket("user:notification", user)
+          {:ok, _topic} =
+            Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
+
           assert_receive {:render_with_user, _, _, _}, 4_000
         end)
 

test/support/data_case.ex

@@ -27,6 +27,21 @@ defmodule Pleroma.DataCase do
       import Ecto.Query
       import Pleroma.DataCase
       use Pleroma.Tests.Helpers
+
+      # Sets up OAuth access with specified scopes
+      defp oauth_access(scopes, opts \\ []) do
+        user =
+          Keyword.get_lazy(opts, :user, fn ->
+            Pleroma.Factory.insert(:user)
+          end)
+
+        token =
+          Keyword.get_lazy(opts, :oauth_token, fn ->
+            Pleroma.Factory.insert(:oauth_token, user: user, scopes: scopes)
+          end)
+
+        %{user: user, token: token}
+      end
     end
   end
 

test/user/import_test.exs

@@ -0,0 +1,76 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.ImportTest do
+  alias Pleroma.Repo
+  alias Pleroma.Tests.ObanHelpers
+  alias Pleroma.User
+
+  use Pleroma.DataCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Pleroma.Factory
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  describe "follow_import" do
+    test "it imports user followings from list" do
+      [user1, user2, user3] = insert_list(3, :user)
+
+      identifiers = [
+        user2.ap_id,
+        user3.nickname
+      ]
+
+      {:ok, job} = User.Import.follow_import(user1, identifiers)
+
+      assert {:ok, result} = ObanHelpers.perform(job)
+      assert is_list(result)
+      assert result == [user2, user3]
+      assert User.following?(user1, user2)
+      assert User.following?(user1, user3)
+    end
+  end
+
+  describe "blocks_import" do
+    test "it imports user blocks from list" do
+      [user1, user2, user3] = insert_list(3, :user)
+
+      identifiers = [
+        user2.ap_id,
+        user3.nickname
+      ]
+
+      {:ok, job} = User.Import.blocks_import(user1, identifiers)
+
+      assert {:ok, result} = ObanHelpers.perform(job)
+      assert is_list(result)
+      assert result == [user2, user3]
+      assert User.blocks?(user1, user2)
+      assert User.blocks?(user1, user3)
+    end
+  end
+
+  describe "mutes_import" do
+    test "it imports user mutes from list" do
+      [user1, user2, user3] = insert_list(3, :user)
+
+      identifiers = [
+        user2.ap_id,
+        user3.nickname
+      ]
+
+      {:ok, job} = User.Import.mutes_import(user1, identifiers)
+
+      assert {:ok, result} = ObanHelpers.perform(job)
+      assert is_list(result)
+      assert result == [user2, user3]
+      assert User.mutes?(user1, user2)
+      assert User.mutes?(user1, user3)
+    end
+  end
+end

test/user_test.exs

@@ -509,7 +509,12 @@ defmodule Pleroma.UserTest do
       cng = User.register_changeset(%User{}, @full_user_data)
       {:ok, registered_user} = User.register(cng)
       ObanHelpers.perform_all()
-      assert_email_sent(Pleroma.Emails.UserEmail.account_confirmation_email(registered_user))
+
+      Pleroma.Emails.UserEmail.account_confirmation_email(registered_user)
+      # temporary hackney fix until hackney max_connections bug is fixed
+      # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+      |> Swoosh.Email.put_private(:hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+      |> assert_email_sent()
     end
 
     test "it requires an email, name, nickname and password, bio is optional when account_activation_required is enabled" do
@@ -971,23 +976,6 @@ defmodule Pleroma.UserTest do
     end
   end
 
-  describe "follow_import" do
-    test "it imports user followings from list" do
-      [user1, user2, user3] = insert_list(3, :user)
-
-      identifiers = [
-        user2.ap_id,
-        user3.nickname
-      ]
-
-      {:ok, job} = User.follow_import(user1, identifiers)
-
-      assert {:ok, result} = ObanHelpers.perform(job)
-      assert is_list(result)
-      assert result == [user2, user3]
-    end
-  end
-
   describe "mutes" do
     test "it mutes people" do
       user = insert(:user)
@@ -1226,23 +1214,6 @@ defmodule Pleroma.UserTest do
     end
   end
 
-  describe "blocks_import" do
-    test "it imports user blocks from list" do
-      [user1, user2, user3] = insert_list(3, :user)
-
-      identifiers = [
-        user2.ap_id,
-        user3.nickname
-      ]
-
-      {:ok, job} = User.blocks_import(user1, identifiers)
-
-      assert {:ok, result} = ObanHelpers.perform(job)
-      assert is_list(result)
-      assert result == [user2, user3]
-    end
-  end
-
   describe "get_recipients_from_activity" do
     test "works for announces" do
       actor = insert(:user)

test/utils_test.exs

@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.UtilsTest do
+  use ExUnit.Case, async: true
+
+  describe "tmp_dir/1" do
+    test "returns unique temporary directory" do
+      {:ok, path} = Pleroma.Utils.tmp_dir("emoji")
+      assert path =~ ~r/\/tmp\/emoji-(.*)-#{:os.getpid()}-(.*)/
+      File.rm_rf(path)
+    end
+  end
+end

test/web/admin_api/controllers/admin_api_controller_test.exs

@@ -1977,7 +1977,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
                }"
 
       ObanHelpers.perform_all()
-      assert_email_sent(Pleroma.Emails.UserEmail.account_confirmation_email(first_user))
+
+      Pleroma.Emails.UserEmail.account_confirmation_email(first_user)
+      # temporary hackney fix until hackney max_connections bug is fixed
+      # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+      |> Swoosh.Email.put_private(:hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+      |> assert_email_sent()
     end
   end
 

test/web/mastodon_api/controllers/auth_controller_test.exs

@@ -61,7 +61,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthControllerTest do
     end
 
     test "it returns 204", %{conn: conn} do
-      assert json_response(conn, :no_content)
+      assert empty_json_response(conn)
     end
 
     test "it creates a PasswordResetToken record for user", %{user: user} do
@@ -91,7 +91,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthControllerTest do
 
       assert conn
              |> post("/auth/password?nickname=#{user.nickname}")
-             |> json_response(:no_content)
+             |> empty_json_response()
 
       ObanHelpers.perform_all()
       token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id)
@@ -112,7 +112,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthControllerTest do
 
       assert conn
              |> post("/auth/password?nickname=#{user.nickname}")
-             |> json_response(:no_content)
+             |> empty_json_response()
     end
   end
 
@@ -125,24 +125,21 @@ defmodule Pleroma.Web.MastodonAPI.AuthControllerTest do
     test "it returns 204 when user is not found", %{conn: conn, user: user} do
       conn = post(conn, "/auth/password?email=nonexisting_#{user.email}")
 
-      assert conn
+      assert empty_json_response(conn)
-             |> json_response(:no_content)
     end
 
     test "it returns 204 when user is not local", %{conn: conn, user: user} do
       {:ok, user} = Repo.update(Ecto.Changeset.change(user, local: false))
       conn = post(conn, "/auth/password?email=#{user.email}")
 
-      assert conn
+      assert empty_json_response(conn)
-             |> json_response(:no_content)
     end
 
     test "it returns 204 when user is deactivated", %{conn: conn, user: user} do
       {:ok, user} = Repo.update(Ecto.Changeset.change(user, deactivated: true, local: true))
       conn = post(conn, "/auth/password?email=#{user.email}")
 
-      assert conn
+      assert empty_json_response(conn)
-             |> json_response(:no_content)
     end
   end
 

test/web/mastodon_api/controllers/timeline_controller_test.exs

@@ -114,8 +114,16 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
       {:ok, _reply_from_friend} =
         CommonAPI.post(friend, %{status: "status", in_reply_to_status_id: reply_from_blockee})
 
-      res_conn = get(conn, "/api/v1/timelines/public")
+      # Still shows replies from yourself
-      [%{"id" => ^activity_id}] = json_response_and_validate_schema(res_conn, 200)
+      {:ok, %{id: reply_from_me}} =
+        CommonAPI.post(blocker, %{status: "status", in_reply_to_status_id: reply_from_blockee})
+
+      response =
+        get(conn, "/api/v1/timelines/public")
+        |> json_response_and_validate_schema(200)
+
+      assert length(response) == 2
+      [%{"id" => ^reply_from_me}, %{"id" => ^activity_id}] = response
     end
 
     test "doesn't return replies if follow is posting with users from blocked domain" do

test/web/pleroma_api/controllers/emoji_file_controller_test.exs

@@ -0,0 +1,357 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.EmojiFileControllerTest do
+  use Pleroma.Web.ConnCase
+
+  import Tesla.Mock
+  import Pleroma.Factory
+
+  @emoji_path Path.join(
+                Pleroma.Config.get!([:instance, :static_dir]),
+                "emoji"
+              )
+  setup do: clear_config([:auth, :enforce_oauth_admin_scope_usage], false)
+
+  setup do: clear_config([:instance, :public], true)
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    admin_conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    Pleroma.Emoji.reload()
+    {:ok, %{admin_conn: admin_conn}}
+  end
+
+  describe "POST/PATCH/DELETE /api/pleroma/emoji/packs/:name/files" do
+    setup do
+      pack_file = "#{@emoji_path}/test_pack/pack.json"
+      original_content = File.read!(pack_file)
+
+      on_exit(fn ->
+        File.write!(pack_file, original_content)
+      end)
+
+      :ok
+    end
+
+    test "upload zip file with emojies", %{admin_conn: admin_conn} do
+      on_exit(fn ->
+        [
+          "128px/a_trusted_friend-128.png",
+          "auroraborealis.png",
+          "1000px/baby_in_a_box.png",
+          "1000px/bear.png",
+          "128px/bear-128.png"
+        ]
+        |> Enum.each(fn path -> File.rm_rf!("#{@emoji_path}/test_pack/#{path}") end)
+      end)
+
+      resp =
+        admin_conn
+        |> put_req_header("content-type", "multipart/form-data")
+        |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+          file: %Plug.Upload{
+            content_type: "application/zip",
+            filename: "emojis.zip",
+            path: Path.absname("test/fixtures/emojis.zip")
+          }
+        })
+        |> json_response_and_validate_schema(200)
+
+      assert resp == %{
+               "a_trusted_friend-128" => "128px/a_trusted_friend-128.png",
+               "auroraborealis" => "auroraborealis.png",
+               "baby_in_a_box" => "1000px/baby_in_a_box.png",
+               "bear" => "1000px/bear.png",
+               "bear-128" => "128px/bear-128.png",
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      Enum.each(Map.values(resp), fn path ->
+        assert File.exists?("#{@emoji_path}/test_pack/#{path}")
+      end)
+    end
+
+    test "create shortcode exists", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(:conflict) == %{
+               "error" => "An emoji with the \"blank\" shortcode already exists"
+             }
+    end
+
+    test "don't rewrite old emoji", %{admin_conn: admin_conn} do
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir/") end)
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank3",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png",
+               "blank3" => "dir/blank.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank",
+               new_shortcode: "blank2",
+               new_filename: "dir_2/blank_3.png"
+             })
+             |> json_response_and_validate_schema(:conflict) == %{
+               "error" =>
+                 "New shortcode \"blank2\" is already used. If you want to override emoji use 'force' option"
+             }
+    end
+
+    test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir_2/") end)
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank3",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png",
+               "blank3" => "dir/blank.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank3",
+               new_shortcode: "blank4",
+               new_filename: "dir_2/blank_3.png",
+               force: true
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png",
+               "blank4" => "dir_2/blank_3.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
+    end
+
+    test "with empty filename", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank2",
+               filename: "",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(422) == %{
+               "error" => "pack name, shortcode or filename cannot be empty"
+             }
+    end
+
+    test "add file with not loaded pack", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/not_loaded/files", %{
+               shortcode: "blank3",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(:not_found) == %{
+               "error" => "pack \"not_loaded\" is not found"
+             }
+    end
+
+    test "remove file with not loaded pack", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=blank3")
+             |> json_response_and_validate_schema(:not_found) == %{
+               "error" => "pack \"not_loaded\" is not found"
+             }
+    end
+
+    test "remove file with empty shortcode", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=")
+             |> json_response_and_validate_schema(:not_found) == %{
+               "error" => "pack \"not_loaded\" is not found"
+             }
+    end
+
+    test "update file with not loaded pack", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/not_loaded/files", %{
+               shortcode: "blank4",
+               new_shortcode: "blank3",
+               new_filename: "dir_2/blank_3.png"
+             })
+             |> json_response_and_validate_schema(:not_found) == %{
+               "error" => "pack \"not_loaded\" is not found"
+             }
+    end
+
+    test "new with shortcode as file with update", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank4",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank4" => "dir/blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank4",
+               new_shortcode: "blank3",
+               new_filename: "dir_2/blank_3.png"
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank3" => "dir_2/blank_3.png",
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      refute File.exists?("#{@emoji_path}/test_pack/dir/")
+      assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
+
+      assert admin_conn
+             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      refute File.exists?("#{@emoji_path}/test_pack/dir_2/")
+
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir") end)
+    end
+
+    test "new with shortcode from url", %{admin_conn: admin_conn} do
+      mock(fn
+        %{
+          method: :get,
+          url: "https://test-blank/blank_url.png"
+        } ->
+          text(File.read!("#{@emoji_path}/test_pack/blank.png"))
+      end)
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank_url",
+               file: "https://test-blank/blank_url.png"
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank_url" => "blank_url.png",
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/blank_url.png")
+
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/blank_url.png") end)
+    end
+
+    test "new without shortcode", %{admin_conn: admin_conn} do
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/shortcode.png") end)
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               file: %Plug.Upload{
+                 filename: "shortcode.png",
+                 path: "#{Pleroma.Config.get([:instance, :static_dir])}/add/shortcode.png"
+               }
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "shortcode" => "shortcode.png",
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+    end
+
+    test "remove non existing shortcode in pack.json", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
+             |> json_response_and_validate_schema(:bad_request) == %{
+               "error" => "Emoji \"blank3\" does not exist"
+             }
+    end
+
+    test "update non existing emoji", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank3",
+               new_shortcode: "blank4",
+               new_filename: "dir_2/blank_3.png"
+             })
+             |> json_response_and_validate_schema(:bad_request) == %{
+               "error" => "Emoji \"blank3\" does not exist"
+             }
+    end
+
+    test "update with empty shortcode", %{admin_conn: admin_conn} do
+      assert %{
+               "error" => "Missing field: new_shortcode."
+             } =
+               admin_conn
+               |> put_req_header("content-type", "multipart/form-data")
+               |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+                 shortcode: "blank",
+                 new_filename: "dir_2/blank_3.png"
+               })
+               |> json_response_and_validate_schema(:bad_request)
+    end
+  end
+end

test/web/pleroma_api/controllers/emoji_pack_controller_test.exs

@@ -411,293 +411,6 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerTest do
     end
   end
 
-  describe "POST/PATCH/DELETE /api/pleroma/emoji/packs/:name/files" do
-    setup do
-      pack_file = "#{@emoji_path}/test_pack/pack.json"
-      original_content = File.read!(pack_file)
-
-      on_exit(fn ->
-        File.write!(pack_file, original_content)
-      end)
-
-      :ok
-    end
-
-    test "create shortcode exists", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(:conflict) == %{
-               "error" => "An emoji with the \"blank\" shortcode already exists"
-             }
-    end
-
-    test "don't rewrite old emoji", %{admin_conn: admin_conn} do
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir/") end)
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank3",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank2" => "blank2.png",
-               "blank3" => "dir/blank.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank",
-               new_shortcode: "blank2",
-               new_filename: "dir_2/blank_3.png"
-             })
-             |> json_response_and_validate_schema(:conflict) == %{
-               "error" =>
-                 "New shortcode \"blank2\" is already used. If you want to override emoji use 'force' option"
-             }
-    end
-
-    test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir_2/") end)
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank3",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank2" => "blank2.png",
-               "blank3" => "dir/blank.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank3",
-               new_shortcode: "blank4",
-               new_filename: "dir_2/blank_3.png",
-               force: true
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank2" => "blank2.png",
-               "blank4" => "dir_2/blank_3.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
-    end
-
-    test "with empty filename", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
-               filename: "",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack name, shortcode or filename cannot be empty"
-             }
-    end
-
-    test "add file with not loaded pack", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/not_loaded/files", %{
-               shortcode: "blank3",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack \"not_loaded\" is not found"
-             }
-    end
-
-    test "remove file with not loaded pack", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=blank3")
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack \"not_loaded\" is not found"
-             }
-    end
-
-    test "remove file with empty shortcode", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=")
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack name or shortcode cannot be empty"
-             }
-    end
-
-    test "update file with not loaded pack", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/not_loaded/files", %{
-               shortcode: "blank4",
-               new_shortcode: "blank3",
-               new_filename: "dir_2/blank_3.png"
-             })
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack \"not_loaded\" is not found"
-             }
-    end
-
-    test "new with shortcode as file with update", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank4",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank4" => "dir/blank.png",
-               "blank2" => "blank2.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank4",
-               new_shortcode: "blank3",
-               new_filename: "dir_2/blank_3.png"
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank3" => "dir_2/blank_3.png",
-               "blank" => "blank.png",
-               "blank2" => "blank2.png"
-             }
-
-      refute File.exists?("#{@emoji_path}/test_pack/dir/")
-      assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
-
-      assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank2" => "blank2.png"
-             }
-
-      refute File.exists?("#{@emoji_path}/test_pack/dir_2/")
-
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir") end)
-    end
-
-    test "new with shortcode from url", %{admin_conn: admin_conn} do
-      mock(fn
-        %{
-          method: :get,
-          url: "https://test-blank/blank_url.png"
-        } ->
-          text(File.read!("#{@emoji_path}/test_pack/blank.png"))
-      end)
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank_url",
-               file: "https://test-blank/blank_url.png"
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank_url" => "blank_url.png",
-               "blank" => "blank.png",
-               "blank2" => "blank2.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/blank_url.png")
-
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/blank_url.png") end)
-    end
-
-    test "new without shortcode", %{admin_conn: admin_conn} do
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/shortcode.png") end)
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               file: %Plug.Upload{
-                 filename: "shortcode.png",
-                 path: "#{Pleroma.Config.get([:instance, :static_dir])}/add/shortcode.png"
-               }
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "shortcode" => "shortcode.png",
-               "blank" => "blank.png",
-               "blank2" => "blank2.png"
-             }
-    end
-
-    test "remove non existing shortcode in pack.json", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "Emoji \"blank3\" does not exist"
-             }
-    end
-
-    test "update non existing emoji", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank3",
-               new_shortcode: "blank4",
-               new_filename: "dir_2/blank_3.png"
-             })
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "Emoji \"blank3\" does not exist"
-             }
-    end
-
-    test "update with empty shortcode", %{admin_conn: admin_conn} do
-      assert %{
-               "error" => "Missing field: new_shortcode."
-             } =
-               admin_conn
-               |> put_req_header("content-type", "multipart/form-data")
-               |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-                 shortcode: "blank",
-                 new_filename: "dir_2/blank_3.png"
-               })
-               |> json_response_and_validate_schema(:bad_request)
-    end
-  end
-
   describe "POST/DELETE /api/pleroma/emoji/packs/:name" do
     test "creating and deleting a pack", %{admin_conn: admin_conn} do
       assert admin_conn

test/web/pleroma_api/controllers/user_import_controller_test.exs

@@ -0,0 +1,235 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do
+  use Pleroma.Web.ConnCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  alias Pleroma.Config
+  alias Pleroma.Tests.ObanHelpers
+
+  import Pleroma.Factory
+  import Mock
+
+  setup do
+    Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  describe "POST /api/pleroma/follow_import" do
+    setup do: oauth_access(["follow"])
+
+    test "it returns HTTP 200", %{conn: conn} do
+      user2 = insert(:user)
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/follow_import", %{"list" => "#{user2.ap_id}"})
+               |> json_response_and_validate_schema(200)
+    end
+
+    test "it imports follow lists from file", %{conn: conn} do
+      user2 = insert(:user)
+
+      with_mocks([
+        {File, [],
+         read!: fn "follow_list.txt" ->
+           "Account address,Show boosts\n#{user2.ap_id},true"
+         end}
+      ]) do
+        assert "job started" ==
+                 conn
+                 |> put_req_header("content-type", "application/json")
+                 |> post("/api/pleroma/follow_import", %{
+                   "list" => %Plug.Upload{path: "follow_list.txt"}
+                 })
+                 |> json_response_and_validate_schema(200)
+
+        assert [{:ok, job_result}] = ObanHelpers.perform_all()
+        assert job_result == [user2]
+      end
+    end
+
+    test "it imports new-style mastodon follow lists", %{conn: conn} do
+      user2 = insert(:user)
+
+      response =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/pleroma/follow_import", %{
+          "list" => "Account address,Show boosts\n#{user2.ap_id},true"
+        })
+        |> json_response_and_validate_schema(200)
+
+      assert response == "job started"
+    end
+
+    test "requires 'follow' or 'write:follows' permissions" do
+      token1 = insert(:oauth_token, scopes: ["read", "write"])
+      token2 = insert(:oauth_token, scopes: ["follow"])
+      token3 = insert(:oauth_token, scopes: ["something"])
+      another_user = insert(:user)
+
+      for token <- [token1, token2, token3] do
+        conn =
+          build_conn()
+          |> put_req_header("authorization", "Bearer #{token.token}")
+          |> put_req_header("content-type", "application/json")
+          |> post("/api/pleroma/follow_import", %{"list" => "#{another_user.ap_id}"})
+
+        if token == token3 do
+          assert %{"error" => "Insufficient permissions: follow | write:follows."} ==
+                   json_response(conn, 403)
+        else
+          assert json_response(conn, 200)
+        end
+      end
+    end
+
+    test "it imports follows with different nickname variations", %{conn: conn} do
+      users = [user2, user3, user4, user5, user6] = insert_list(5, :user)
+
+      identifiers =
+        [
+          user2.ap_id,
+          user3.nickname,
+          "  ",
+          "@" <> user4.nickname,
+          user5.nickname <> "@localhost",
+          "@" <> user6.nickname <> "@localhost"
+        ]
+        |> Enum.join("\n")
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/follow_import", %{"list" => identifiers})
+               |> json_response_and_validate_schema(200)
+
+      assert [{:ok, job_result}] = ObanHelpers.perform_all()
+      assert job_result == users
+    end
+  end
+
+  describe "POST /api/pleroma/blocks_import" do
+    # Note: "follow" or "write:blocks" permission is required
+    setup do: oauth_access(["write:blocks"])
+
+    test "it returns HTTP 200", %{conn: conn} do
+      user2 = insert(:user)
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/blocks_import", %{"list" => "#{user2.ap_id}"})
+               |> json_response_and_validate_schema(200)
+    end
+
+    test "it imports blocks users from file", %{conn: conn} do
+      users = [user2, user3] = insert_list(2, :user)
+
+      with_mocks([
+        {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}
+      ]) do
+        assert "job started" ==
+                 conn
+                 |> put_req_header("content-type", "application/json")
+                 |> post("/api/pleroma/blocks_import", %{
+                   "list" => %Plug.Upload{path: "blocks_list.txt"}
+                 })
+                 |> json_response_and_validate_schema(200)
+
+        assert [{:ok, job_result}] = ObanHelpers.perform_all()
+        assert job_result == users
+      end
+    end
+
+    test "it imports blocks with different nickname variations", %{conn: conn} do
+      users = [user2, user3, user4, user5, user6] = insert_list(5, :user)
+
+      identifiers =
+        [
+          user2.ap_id,
+          user3.nickname,
+          "@" <> user4.nickname,
+          user5.nickname <> "@localhost",
+          "@" <> user6.nickname <> "@localhost"
+        ]
+        |> Enum.join(" ")
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/blocks_import", %{"list" => identifiers})
+               |> json_response_and_validate_schema(200)
+
+      assert [{:ok, job_result}] = ObanHelpers.perform_all()
+      assert job_result == users
+    end
+  end
+
+  describe "POST /api/pleroma/mutes_import" do
+    # Note: "follow" or "write:mutes" permission is required
+    setup do: oauth_access(["write:mutes"])
+
+    test "it returns HTTP 200", %{user: user, conn: conn} do
+      user2 = insert(:user)
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/mutes_import", %{"list" => "#{user2.ap_id}"})
+               |> json_response_and_validate_schema(200)
+
+      assert [{:ok, job_result}] = ObanHelpers.perform_all()
+      assert job_result == [user2]
+      assert Pleroma.User.mutes?(user, user2)
+    end
+
+    test "it imports mutes users from file", %{user: user, conn: conn} do
+      users = [user2, user3] = insert_list(2, :user)
+
+      with_mocks([
+        {File, [], read!: fn "mutes_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}
+      ]) do
+        assert "job started" ==
+                 conn
+                 |> put_req_header("content-type", "application/json")
+                 |> post("/api/pleroma/mutes_import", %{
+                   "list" => %Plug.Upload{path: "mutes_list.txt"}
+                 })
+                 |> json_response_and_validate_schema(200)
+
+        assert [{:ok, job_result}] = ObanHelpers.perform_all()
+        assert job_result == users
+        assert Enum.all?(users, &Pleroma.User.mutes?(user, &1))
+      end
+    end
+
+    test "it imports mutes with different nickname variations", %{user: user, conn: conn} do
+      users = [user2, user3, user4, user5, user6] = insert_list(5, :user)
+
+      identifiers =
+        [
+          user2.ap_id,
+          user3.nickname,
+          "@" <> user4.nickname,
+          user5.nickname <> "@localhost",
+          "@" <> user6.nickname <> "@localhost"
+        ]
+        |> Enum.join(" ")
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/mutes_import", %{"list" => identifiers})
+               |> json_response_and_validate_schema(200)
+
+      assert [{:ok, job_result}] = ObanHelpers.perform_all()
+      assert job_result == users
+      assert Enum.all?(users, &Pleroma.User.mutes?(user, &1))
+    end
+  end
+end

test/web/streamer/streamer_test.exs

@@ -21,92 +21,148 @@ defmodule Pleroma.Web.StreamerTest do
 
   setup do: clear_config([:instance, :skip_thread_containment])
 
-  describe "get_topic without an user" do
+  describe "get_topic/_ (unauthenticated)" do
     test "allows public" do
-      assert {:ok, "public"} = Streamer.get_topic("public", nil)
+      assert {:ok, "public"} = Streamer.get_topic("public", nil, nil)
-      assert {:ok, "public:local"} = Streamer.get_topic("public:local", nil)
+      assert {:ok, "public:local"} = Streamer.get_topic("public:local", nil, nil)
-      assert {:ok, "public:media"} = Streamer.get_topic("public:media", nil)
+      assert {:ok, "public:media"} = Streamer.get_topic("public:media", nil, nil)
-      assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil)
+      assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil, nil)
     end
 
     test "allows hashtag streams" do
-      assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", nil, %{"tag" => "cofe"})
+      assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", nil, nil, %{"tag" => "cofe"})
     end
 
     test "disallows user streams" do
-      assert {:error, _} = Streamer.get_topic("user", nil)
+      assert {:error, _} = Streamer.get_topic("user", nil, nil)
-      assert {:error, _} = Streamer.get_topic("user:notification", nil)
+      assert {:error, _} = Streamer.get_topic("user:notification", nil, nil)
-      assert {:error, _} = Streamer.get_topic("direct", nil)
+      assert {:error, _} = Streamer.get_topic("direct", nil, nil)
     end
 
     test "disallows list streams" do
-      assert {:error, _} = Streamer.get_topic("list", nil, %{"list" => 42})
+      assert {:error, _} = Streamer.get_topic("list", nil, nil, %{"list" => 42})
     end
   end
 
-  describe "get_topic with an user" do
+  describe "get_topic/_ (authenticated)" do
-    setup do
+    setup do: oauth_access(["read"])
-      user = insert(:user)
+
-      {:ok, %{user: user}}
+    test "allows public streams (regardless of OAuth token scopes)", %{
+      user: user,
+      token: read_oauth_token
+    } do
+      with oauth_token <- [nil, read_oauth_token] do
+        assert {:ok, "public"} = Streamer.get_topic("public", user, oauth_token)
+        assert {:ok, "public:local"} = Streamer.get_topic("public:local", user, oauth_token)
+        assert {:ok, "public:media"} = Streamer.get_topic("public:media", user, oauth_token)
+
+        assert {:ok, "public:local:media"} =
+                 Streamer.get_topic("public:local:media", user, oauth_token)
+      end
     end
 
-    test "allows public streams", %{user: user} do
+    test "allows user streams (with proper OAuth token scopes)", %{
-      assert {:ok, "public"} = Streamer.get_topic("public", user)
+      user: user,
-      assert {:ok, "public:local"} = Streamer.get_topic("public:local", user)
+      token: read_oauth_token
-      assert {:ok, "public:media"} = Streamer.get_topic("public:media", user)
+    } do
-      assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", user)
+      %{token: read_notifications_token} = oauth_access(["read:notifications"], user: user)
-    end
+      %{token: read_statuses_token} = oauth_access(["read:statuses"], user: user)
+      %{token: badly_scoped_token} = oauth_access(["irrelevant:scope"], user: user)
 
-    test "allows user streams", %{user: user} do
       expected_user_topic = "user:#{user.id}"
-      expected_notif_topic = "user:notification:#{user.id}"
+      expected_notification_topic = "user:notification:#{user.id}"
       expected_direct_topic = "direct:#{user.id}"
-      assert {:ok, ^expected_user_topic} = Streamer.get_topic("user", user)
+      expected_pleroma_chat_topic = "user:pleroma_chat:#{user.id}"
-      assert {:ok, ^expected_notif_topic} = Streamer.get_topic("user:notification", user)
+
-      assert {:ok, ^expected_direct_topic} = Streamer.get_topic("direct", user)
+      for valid_user_token <- [read_oauth_token, read_statuses_token] do
+        assert {:ok, ^expected_user_topic} = Streamer.get_topic("user", user, valid_user_token)
+
+        assert {:ok, ^expected_direct_topic} =
+                 Streamer.get_topic("direct", user, valid_user_token)
+
+        assert {:ok, ^expected_pleroma_chat_topic} =
+                 Streamer.get_topic("user:pleroma_chat", user, valid_user_token)
       end
 
-    test "allows hashtag streams", %{user: user} do
+      for invalid_user_token <- [read_notifications_token, badly_scoped_token],
-      assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", user, %{"tag" => "cofe"})
+          user_topic <- ["user", "direct", "user:pleroma_chat"] do
+        assert {:error, :unauthorized} = Streamer.get_topic(user_topic, user, invalid_user_token)
       end
 
-    test "disallows registering to an user stream", %{user: user} do
+      for valid_notification_token <- [read_oauth_token, read_notifications_token] do
+        assert {:ok, ^expected_notification_topic} =
+                 Streamer.get_topic("user:notification", user, valid_notification_token)
+      end
+
+      for invalid_notification_token <- [read_statuses_token, badly_scoped_token] do
+        assert {:error, :unauthorized} =
+                 Streamer.get_topic("user:notification", user, invalid_notification_token)
+      end
+    end
+
+    test "allows hashtag streams (regardless of OAuth token scopes)", %{
+      user: user,
+      token: read_oauth_token
+    } do
+      for oauth_token <- [nil, read_oauth_token] do
+        assert {:ok, "hashtag:cofe"} =
+                 Streamer.get_topic("hashtag", user, oauth_token, %{"tag" => "cofe"})
+      end
+    end
+
+    test "disallows registering to another user's stream", %{user: user, token: read_oauth_token} do
       another_user = insert(:user)
-      assert {:error, _} = Streamer.get_topic("user:#{another_user.id}", user)
+      assert {:error, _} = Streamer.get_topic("user:#{another_user.id}", user, read_oauth_token)
-      assert {:error, _} = Streamer.get_topic("user:notification:#{another_user.id}", user)
+
-      assert {:error, _} = Streamer.get_topic("direct:#{another_user.id}", user)
+      assert {:error, _} =
+               Streamer.get_topic("user:notification:#{another_user.id}", user, read_oauth_token)
+
+      assert {:error, _} = Streamer.get_topic("direct:#{another_user.id}", user, read_oauth_token)
     end
 
-    test "allows list stream that are owned by the user", %{user: user} do
+    test "allows list stream that are owned by the user (with `read` or `read:lists` scopes)", %{
+      user: user,
+      token: read_oauth_token
+    } do
+      %{token: read_lists_token} = oauth_access(["read:lists"], user: user)
+      %{token: invalid_token} = oauth_access(["irrelevant:scope"], user: user)
       {:ok, list} = List.create("Test", user)
-      assert {:error, _} = Streamer.get_topic("list:#{list.id}", user)
+
-      assert {:ok, _} = Streamer.get_topic("list", user, %{"list" => list.id})
+      assert {:error, _} = Streamer.get_topic("list:#{list.id}", user, read_oauth_token)
+
+      for valid_token <- [read_oauth_token, read_lists_token] do
+        assert {:ok, _} = Streamer.get_topic("list", user, valid_token, %{"list" => list.id})
       end
 
-    test "disallows list stream that are not owned by the user", %{user: user} do
+      assert {:error, _} = Streamer.get_topic("list", user, invalid_token, %{"list" => list.id})
+    end
+
+    test "disallows list stream that are not owned by the user", %{user: user, token: oauth_token} do
       another_user = insert(:user)
       {:ok, list} = List.create("Test", another_user)
-      assert {:error, _} = Streamer.get_topic("list:#{list.id}", user)
+
-      assert {:error, _} = Streamer.get_topic("list", user, %{"list" => list.id})
+      assert {:error, _} = Streamer.get_topic("list:#{list.id}", user, oauth_token)
+      assert {:error, _} = Streamer.get_topic("list", user, oauth_token, %{"list" => list.id})
     end
   end
 
   describe "user streams" do
     setup do
-      user = insert(:user)
+      %{user: user, token: token} = oauth_access(["read"])
       notify = insert(:notification, user: user, activity: build(:note_activity))
-      {:ok, %{user: user, notify: notify}}
+      {:ok, %{user: user, notify: notify, token: token}}
     end
 
-    test "it streams the user's post in the 'user' stream", %{user: user} do
+    test "it streams the user's post in the 'user' stream", %{user: user, token: oauth_token} do
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
       {:ok, activity} = CommonAPI.post(user, %{status: "hey"})
+
       assert_receive {:render_with_user, _, _, ^activity}
       refute Streamer.filtered_by_user?(user, activity)
     end
 
-    test "it streams boosts of the user in the 'user' stream", %{user: user} do
+    test "it streams boosts of the user in the 'user' stream", %{user: user, token: oauth_token} do
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
 
       other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(other_user, %{status: "hey"})
@@ -117,9 +173,10 @@ defmodule Pleroma.Web.StreamerTest do
     end
 
     test "it does not stream announces of the user's own posts in the 'user' stream", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
 
       other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(user, %{status: "hey"})
@@ -129,9 +186,10 @@ defmodule Pleroma.Web.StreamerTest do
     end
 
     test "it does stream notifications announces of the user's own posts in the 'user' stream", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
 
       other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(user, %{status: "hey"})
@@ -145,8 +203,11 @@ defmodule Pleroma.Web.StreamerTest do
       refute Streamer.filtered_by_user?(user, notification)
     end
 
-    test "it streams boosts of mastodon user in the 'user' stream", %{user: user} do
+    test "it streams boosts of mastodon user in the 'user' stream", %{
-      Streamer.get_topic_and_add_socket("user", user)
+      user: user,
+      token: oauth_token
+    } do
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
 
       other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(other_user, %{status: "hey"})
@@ -164,21 +225,34 @@ defmodule Pleroma.Web.StreamerTest do
       refute Streamer.filtered_by_user?(user, announce)
     end
 
-    test "it sends notify to in the 'user' stream", %{user: user, notify: notify} do
+    test "it sends notify to in the 'user' stream", %{
-      Streamer.get_topic_and_add_socket("user", user)
+      user: user,
+      token: oauth_token,
+      notify: notify
+    } do
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
       Streamer.stream("user", notify)
+
       assert_receive {:render_with_user, _, _, ^notify}
       refute Streamer.filtered_by_user?(user, notify)
     end
 
-    test "it sends notify to in the 'user:notification' stream", %{user: user, notify: notify} do
+    test "it sends notify to in the 'user:notification' stream", %{
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      user: user,
+      token: oauth_token,
+      notify: notify
+    } do
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       Streamer.stream("user:notification", notify)
+
       assert_receive {:render_with_user, _, _, ^notify}
       refute Streamer.filtered_by_user?(user, notify)
     end
 
-    test "it sends chat messages to the 'user:pleroma_chat' stream", %{user: user} do
+    test "it sends chat messages to the 'user:pleroma_chat' stream", %{
+      user: user,
+      token: oauth_token
+    } do
       other_user = insert(:user)
 
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
@@ -187,7 +261,7 @@ defmodule Pleroma.Web.StreamerTest do
       cm_ref = MessageReference.for_chat_and_object(chat, object)
       cm_ref = %{cm_ref | chat: chat, object: object}
 
-      Streamer.get_topic_and_add_socket("user:pleroma_chat", user)
+      Streamer.get_topic_and_add_socket("user:pleroma_chat", user, oauth_token)
       Streamer.stream("user:pleroma_chat", {user, cm_ref})
 
       text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
@@ -196,7 +270,7 @@ defmodule Pleroma.Web.StreamerTest do
       assert_receive {:text, ^text}
     end
 
-    test "it sends chat messages to the 'user' stream", %{user: user} do
+    test "it sends chat messages to the 'user' stream", %{user: user, token: oauth_token} do
       other_user = insert(:user)
 
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
@@ -205,7 +279,7 @@ defmodule Pleroma.Web.StreamerTest do
       cm_ref = MessageReference.for_chat_and_object(chat, object)
       cm_ref = %{cm_ref | chat: chat, object: object}
 
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
       Streamer.stream("user", {user, cm_ref})
 
       text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
@@ -214,7 +288,10 @@ defmodule Pleroma.Web.StreamerTest do
       assert_receive {:text, ^text}
     end
 
-    test "it sends chat message notifications to the 'user:notification' stream", %{user: user} do
+    test "it sends chat message notifications to the 'user:notification' stream", %{
+      user: user,
+      token: oauth_token
+    } do
       other_user = insert(:user)
 
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey")
@@ -223,19 +300,21 @@ defmodule Pleroma.Web.StreamerTest do
         Repo.get_by(Pleroma.Notification, user_id: user.id, activity_id: create_activity.id)
         |> Repo.preload(:activity)
 
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       Streamer.stream("user:notification", notify)
+
       assert_receive {:render_with_user, _, _, ^notify}
       refute Streamer.filtered_by_user?(user, notify)
     end
 
     test "it doesn't send notify to the 'user:notification' stream when a user is blocked", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       blocked = insert(:user)
       {:ok, _user_relationship} = User.block(user, blocked)
 
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
 
       {:ok, activity} = CommonAPI.post(user, %{status: ":("})
       {:ok, _} = CommonAPI.favorite(blocked, activity.id)
@@ -244,14 +323,15 @@ defmodule Pleroma.Web.StreamerTest do
     end
 
     test "it doesn't send notify to the 'user:notification' stream when a thread is muted", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       user2 = insert(:user)
 
       {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
       {:ok, _} = CommonAPI.add_mute(user, activity)
 
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
 
       {:ok, favorite_activity} = CommonAPI.favorite(user2, activity.id)
 
@@ -260,12 +340,13 @@ defmodule Pleroma.Web.StreamerTest do
     end
 
     test "it sends favorite to 'user:notification' stream'", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       user2 = insert(:user, %{ap_id: "https://hecking-lewd-place.com/user/meanie"})
 
       {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       {:ok, favorite_activity} = CommonAPI.favorite(user2, activity.id)
 
       assert_receive {:render_with_user, _, "notification.json", notif}
@@ -274,13 +355,14 @@ defmodule Pleroma.Web.StreamerTest do
     end
 
     test "it doesn't send the 'user:notification' stream' when a domain is blocked", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       user2 = insert(:user, %{ap_id: "https://hecking-lewd-place.com/user/meanie"})
 
       {:ok, user} = User.block_domain(user, "hecking-lewd-place.com")
       {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       {:ok, favorite_activity} = CommonAPI.favorite(user2, activity.id)
 
       refute_receive _
@@ -288,7 +370,8 @@ defmodule Pleroma.Web.StreamerTest do
     end
 
     test "it sends follow activities to the 'user:notification' stream", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       user_url = user.ap_id
       user2 = insert(:user)
@@ -303,7 +386,7 @@ defmodule Pleroma.Web.StreamerTest do
           %Tesla.Env{status: 200, body: body}
       end)
 
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       {:ok, _follower, _followed, follow_activity} = CommonAPI.follow(user2, user)
 
       assert_receive {:render_with_user, _, "notification.json", notif}
@@ -312,34 +395,22 @@ defmodule Pleroma.Web.StreamerTest do
     end
   end
 
-  test "it sends to public authenticated" do
+  describe "public streams" do
-    user = insert(:user)
+    test "it sends to public (authenticated)" do
+      %{user: user, token: oauth_token} = oauth_access(["read"])
       other_user = insert(:user)
 
-    Streamer.get_topic_and_add_socket("public", other_user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
 
-    {:ok, activity} = CommonAPI.post(user, %{status: "Test"})
-    assert_receive {:render_with_user, _, _, ^activity}
-    refute Streamer.filtered_by_user?(user, activity)
-  end
-
-  test "works for deletions" do
-    user = insert(:user)
-    other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(other_user, %{status: "Test"})
-
+      assert_receive {:render_with_user, _, _, ^activity}
-    Streamer.get_topic_and_add_socket("public", user)
+      refute Streamer.filtered_by_user?(other_user, activity)
-
-    {:ok, _} = CommonAPI.delete(activity.id, other_user)
-    activity_id = activity.id
-    assert_receive {:text, event}
-    assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event)
     end
 
-  test "it sends to public unauthenticated" do
+    test "it sends to public (unauthenticated)" do
       user = insert(:user)
 
-    Streamer.get_topic_and_add_socket("public", nil)
+      Streamer.get_topic_and_add_socket("public", nil, nil)
 
       {:ok, activity} = CommonAPI.post(user, %{status: "Test"})
       activity_id = activity.id
@@ -352,11 +423,25 @@ defmodule Pleroma.Web.StreamerTest do
       assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event)
     end
 
-  describe "thread_containment" do
+    test "handles deletions" do
+      %{user: user, token: oauth_token} = oauth_access(["read"])
+      other_user = insert(:user)
+      {:ok, activity} = CommonAPI.post(other_user, %{status: "Test"})
+
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
+
+      {:ok, _} = CommonAPI.delete(activity.id, other_user)
+      activity_id = activity.id
+      assert_receive {:text, event}
+      assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event)
+    end
+  end
+
+  describe "thread_containment/2" do
     test "it filters to user if recipients invalid and thread containment is enabled" do
       Pleroma.Config.put([:instance, :skip_thread_containment], false)
       author = insert(:user)
-      user = insert(:user)
+      %{user: user, token: oauth_token} = oauth_access(["read"])
       User.follow(user, author, :follow_accept)
 
       activity =
@@ -368,7 +453,7 @@ defmodule Pleroma.Web.StreamerTest do
             )
         )
 
-      Streamer.get_topic_and_add_socket("public", user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
       Streamer.stream("public", activity)
       assert_receive {:render_with_user, _, _, ^activity}
       assert Streamer.filtered_by_user?(user, activity)
@@ -377,7 +462,7 @@ defmodule Pleroma.Web.StreamerTest do
     test "it sends message if recipients invalid and thread containment is disabled" do
       Pleroma.Config.put([:instance, :skip_thread_containment], true)
       author = insert(:user)
-      user = insert(:user)
+      %{user: user, token: oauth_token} = oauth_access(["read"])
       User.follow(user, author, :follow_accept)
 
       activity =
@@ -389,7 +474,7 @@ defmodule Pleroma.Web.StreamerTest do
             )
         )
 
-      Streamer.get_topic_and_add_socket("public", user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
       Streamer.stream("public", activity)
 
       assert_receive {:render_with_user, _, _, ^activity}
@@ -400,6 +485,7 @@ defmodule Pleroma.Web.StreamerTest do
       Pleroma.Config.put([:instance, :skip_thread_containment], false)
       author = insert(:user)
       user = insert(:user, skip_thread_containment: true)
+      %{token: oauth_token} = oauth_access(["read"], user: user)
       User.follow(user, author, :follow_accept)
 
       activity =
@@ -411,7 +497,7 @@ defmodule Pleroma.Web.StreamerTest do
             )
         )
 
-      Streamer.get_topic_and_add_socket("public", user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
       Streamer.stream("public", activity)
 
       assert_receive {:render_with_user, _, _, ^activity}
@@ -420,23 +506,26 @@ defmodule Pleroma.Web.StreamerTest do
   end
 
   describe "blocks" do
-    test "it filters messages involving blocked users" do
+    setup do: oauth_access(["read"])
-      user = insert(:user)
+
+    test "it filters messages involving blocked users", %{user: user, token: oauth_token} do
       blocked_user = insert(:user)
       {:ok, _user_relationship} = User.block(user, blocked_user)
 
-      Streamer.get_topic_and_add_socket("public", user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
       {:ok, activity} = CommonAPI.post(blocked_user, %{status: "Test"})
       assert_receive {:render_with_user, _, _, ^activity}
       assert Streamer.filtered_by_user?(user, activity)
     end
 
-    test "it filters messages transitively involving blocked users" do
+    test "it filters messages transitively involving blocked users", %{
-      blocker = insert(:user)
+      user: blocker,
+      token: blocker_token
+    } do
       blockee = insert(:user)
       friend = insert(:user)
 
-      Streamer.get_topic_and_add_socket("public", blocker)
+      Streamer.get_topic_and_add_socket("public", blocker, blocker_token)
 
       {:ok, _user_relationship} = User.block(blocker, blockee)
 
@@ -458,8 +547,9 @@ defmodule Pleroma.Web.StreamerTest do
   end
 
   describe "lists" do
-    test "it doesn't send unwanted DMs to list" do
+    setup do: oauth_access(["read"])
-      user_a = insert(:user)
+
+    test "it doesn't send unwanted DMs to list", %{user: user_a, token: user_a_token} do
       user_b = insert(:user)
       user_c = insert(:user)
 
@@ -468,7 +558,7 @@ defmodule Pleroma.Web.StreamerTest do
       {:ok, list} = List.create("Test", user_a)
       {:ok, list} = List.follow(list, user_b)
 
-      Streamer.get_topic_and_add_socket("list", user_a, %{"list" => list.id})
+      Streamer.get_topic_and_add_socket("list", user_a, user_a_token, %{"list" => list.id})
 
       {:ok, _activity} =
         CommonAPI.post(user_b, %{
@@ -479,14 +569,13 @@ defmodule Pleroma.Web.StreamerTest do
       refute_receive _
     end
 
-    test "it doesn't send unwanted private posts to list" do
+    test "it doesn't send unwanted private posts to list", %{user: user_a, token: user_a_token} do
-      user_a = insert(:user)
       user_b = insert(:user)
 
       {:ok, list} = List.create("Test", user_a)
       {:ok, list} = List.follow(list, user_b)
 
-      Streamer.get_topic_and_add_socket("list", user_a, %{"list" => list.id})
+      Streamer.get_topic_and_add_socket("list", user_a, user_a_token, %{"list" => list.id})
 
       {:ok, _activity} =
         CommonAPI.post(user_b, %{
@@ -497,8 +586,7 @@ defmodule Pleroma.Web.StreamerTest do
       refute_receive _
     end
 
-    test "it sends wanted private posts to list" do
+    test "it sends wanted private posts to list", %{user: user_a, token: user_a_token} do
-      user_a = insert(:user)
       user_b = insert(:user)
 
       {:ok, user_a} = User.follow(user_a, user_b)
@@ -506,7 +594,7 @@ defmodule Pleroma.Web.StreamerTest do
       {:ok, list} = List.create("Test", user_a)
       {:ok, list} = List.follow(list, user_b)
 
-      Streamer.get_topic_and_add_socket("list", user_a, %{"list" => list.id})
+      Streamer.get_topic_and_add_socket("list", user_a, user_a_token, %{"list" => list.id})
 
       {:ok, activity} =
         CommonAPI.post(user_b, %{
@@ -520,8 +608,9 @@ defmodule Pleroma.Web.StreamerTest do
   end
 
   describe "muted reblogs" do
-    test "it filters muted reblogs" do
+    setup do: oauth_access(["read"])
-      user1 = insert(:user)
+
+    test "it filters muted reblogs", %{user: user1, token: user1_token} do
       user2 = insert(:user)
       user3 = insert(:user)
       CommonAPI.follow(user1, user2)
@@ -529,34 +618,38 @@ defmodule Pleroma.Web.StreamerTest do
 
       {:ok, create_activity} = CommonAPI.post(user3, %{status: "I'm kawen"})
 
-      Streamer.get_topic_and_add_socket("user", user1)
+      Streamer.get_topic_and_add_socket("user", user1, user1_token)
       {:ok, announce_activity} = CommonAPI.repeat(create_activity.id, user2)
       assert_receive {:render_with_user, _, _, ^announce_activity}
       assert Streamer.filtered_by_user?(user1, announce_activity)
     end
 
-    test "it filters reblog notification for reblog-muted actors" do
+    test "it filters reblog notification for reblog-muted actors", %{
-      user1 = insert(:user)
+      user: user1,
+      token: user1_token
+    } do
       user2 = insert(:user)
       CommonAPI.follow(user1, user2)
       CommonAPI.hide_reblogs(user1, user2)
 
       {:ok, create_activity} = CommonAPI.post(user1, %{status: "I'm kawen"})
-      Streamer.get_topic_and_add_socket("user", user1)
+      Streamer.get_topic_and_add_socket("user", user1, user1_token)
       {:ok, _announce_activity} = CommonAPI.repeat(create_activity.id, user2)
 
       assert_receive {:render_with_user, _, "notification.json", notif}
       assert Streamer.filtered_by_user?(user1, notif)
     end
 
-    test "it send non-reblog notification for reblog-muted actors" do
+    test "it send non-reblog notification for reblog-muted actors", %{
-      user1 = insert(:user)
+      user: user1,
+      token: user1_token
+    } do
       user2 = insert(:user)
       CommonAPI.follow(user1, user2)
       CommonAPI.hide_reblogs(user1, user2)
 
       {:ok, create_activity} = CommonAPI.post(user1, %{status: "I'm kawen"})
-      Streamer.get_topic_and_add_socket("user", user1)
+      Streamer.get_topic_and_add_socket("user", user1, user1_token)
       {:ok, _favorite_activity} = CommonAPI.favorite(user2, create_activity.id)
 
       assert_receive {:render_with_user, _, "notification.json", notif}
@@ -564,27 +657,28 @@ defmodule Pleroma.Web.StreamerTest do
     end
   end
 
+  describe "muted threads" do
     test "it filters posts from muted threads" do
       user = insert(:user)
-    user2 = insert(:user)
+      %{user: user2, token: user2_token} = oauth_access(["read"])
-    Streamer.get_topic_and_add_socket("user", user2)
+      Streamer.get_topic_and_add_socket("user", user2, user2_token)
+
       {:ok, user2, user, _activity} = CommonAPI.follow(user2, user)
       {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
       {:ok, _} = CommonAPI.add_mute(user2, activity)
+
       assert_receive {:render_with_user, _, _, ^activity}
       assert Streamer.filtered_by_user?(user2, activity)
     end
-
-  describe "direct streams" do
-    setup do
-      :ok
   end
 
-    test "it sends conversation update to the 'direct' stream", %{} do
+  describe "direct streams" do
-      user = insert(:user)
+    setup do: oauth_access(["read"])
+
+    test "it sends conversation update to the 'direct' stream", %{user: user, token: oauth_token} do
       another_user = insert(:user)
 
-      Streamer.get_topic_and_add_socket("direct", user)
+      Streamer.get_topic_and_add_socket("direct", user, oauth_token)
 
       {:ok, _create_activity} =
         CommonAPI.post(another_user, %{
@@ -602,11 +696,11 @@ defmodule Pleroma.Web.StreamerTest do
       assert last_status["pleroma"]["direct_conversation_id"] == participation.id
     end
 
-    test "it doesn't send conversation update to the 'direct' stream when the last message in the conversation is deleted" do
+    test "it doesn't send conversation update to the 'direct' stream when the last message in the conversation is deleted",
-      user = insert(:user)
+         %{user: user, token: oauth_token} do
       another_user = insert(:user)
 
-      Streamer.get_topic_and_add_socket("direct", user)
+      Streamer.get_topic_and_add_socket("direct", user, oauth_token)
 
       {:ok, create_activity} =
         CommonAPI.post(another_user, %{
@@ -629,10 +723,12 @@ defmodule Pleroma.Web.StreamerTest do
       refute_receive _
     end
 
-    test "it sends conversation update to the 'direct' stream when a message is deleted" do
+    test "it sends conversation update to the 'direct' stream when a message is deleted", %{
-      user = insert(:user)
+      user: user,
+      token: oauth_token
+    } do
       another_user = insert(:user)
-      Streamer.get_topic_and_add_socket("direct", user)
+      Streamer.get_topic_and_add_socket("direct", user, oauth_token)
 
       {:ok, create_activity} =
         CommonAPI.post(another_user, %{

test/web/twitter_api/util_controller_test.exs

@@ -21,170 +21,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
   setup do: clear_config([:instance])
   setup do: clear_config([:frontend_configurations, :pleroma_fe])
 
-  describe "POST /api/pleroma/follow_import" do
-    setup do: oauth_access(["follow"])
-
-    test "it returns HTTP 200", %{conn: conn} do
-      user2 = insert(:user)
-
-      response =
-        conn
-        |> post("/api/pleroma/follow_import", %{"list" => "#{user2.ap_id}"})
-        |> json_response(:ok)
-
-      assert response == "job started"
-    end
-
-    test "it imports follow lists from file", %{user: user1, conn: conn} do
-      user2 = insert(:user)
-
-      with_mocks([
-        {File, [],
-         read!: fn "follow_list.txt" ->
-           "Account address,Show boosts\n#{user2.ap_id},true"
-         end}
-      ]) do
-        response =
-          conn
-          |> post("/api/pleroma/follow_import", %{"list" => %Plug.Upload{path: "follow_list.txt"}})
-          |> json_response(:ok)
-
-        assert response == "job started"
-
-        assert ObanHelpers.member?(
-                 %{
-                   "op" => "follow_import",
-                   "follower_id" => user1.id,
-                   "followed_identifiers" => [user2.ap_id]
-                 },
-                 all_enqueued(worker: Pleroma.Workers.BackgroundWorker)
-               )
-      end
-    end
-
-    test "it imports new-style mastodon follow lists", %{conn: conn} do
-      user2 = insert(:user)
-
-      response =
-        conn
-        |> post("/api/pleroma/follow_import", %{
-          "list" => "Account address,Show boosts\n#{user2.ap_id},true"
-        })
-        |> json_response(:ok)
-
-      assert response == "job started"
-    end
-
-    test "requires 'follow' or 'write:follows' permissions" do
-      token1 = insert(:oauth_token, scopes: ["read", "write"])
-      token2 = insert(:oauth_token, scopes: ["follow"])
-      token3 = insert(:oauth_token, scopes: ["something"])
-      another_user = insert(:user)
-
-      for token <- [token1, token2, token3] do
-        conn =
-          build_conn()
-          |> put_req_header("authorization", "Bearer #{token.token}")
-          |> post("/api/pleroma/follow_import", %{"list" => "#{another_user.ap_id}"})
-
-        if token == token3 do
-          assert %{"error" => "Insufficient permissions: follow | write:follows."} ==
-                   json_response(conn, 403)
-        else
-          assert json_response(conn, 200)
-        end
-      end
-    end
-
-    test "it imports follows with different nickname variations", %{conn: conn} do
-      [user2, user3, user4, user5, user6] = insert_list(5, :user)
-
-      identifiers =
-        [
-          user2.ap_id,
-          user3.nickname,
-          "  ",
-          "@" <> user4.nickname,
-          user5.nickname <> "@localhost",
-          "@" <> user6.nickname <> "@localhost"
-        ]
-        |> Enum.join("\n")
-
-      response =
-        conn
-        |> post("/api/pleroma/follow_import", %{"list" => identifiers})
-        |> json_response(:ok)
-
-      assert response == "job started"
-      assert [{:ok, job_result}] = ObanHelpers.perform_all()
-      assert job_result == [user2, user3, user4, user5, user6]
-    end
-  end
-
-  describe "POST /api/pleroma/blocks_import" do
-    # Note: "follow" or "write:blocks" permission is required
-    setup do: oauth_access(["write:blocks"])
-
-    test "it returns HTTP 200", %{conn: conn} do
-      user2 = insert(:user)
-
-      response =
-        conn
-        |> post("/api/pleroma/blocks_import", %{"list" => "#{user2.ap_id}"})
-        |> json_response(:ok)
-
-      assert response == "job started"
-    end
-
-    test "it imports blocks users from file", %{user: user1, conn: conn} do
-      user2 = insert(:user)
-      user3 = insert(:user)
-
-      with_mocks([
-        {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}
-      ]) do
-        response =
-          conn
-          |> post("/api/pleroma/blocks_import", %{"list" => %Plug.Upload{path: "blocks_list.txt"}})
-          |> json_response(:ok)
-
-        assert response == "job started"
-
-        assert ObanHelpers.member?(
-                 %{
-                   "op" => "blocks_import",
-                   "blocker_id" => user1.id,
-                   "blocked_identifiers" => [user2.ap_id, user3.ap_id]
-                 },
-                 all_enqueued(worker: Pleroma.Workers.BackgroundWorker)
-               )
-      end
-    end
-
-    test "it imports blocks with different nickname variations", %{conn: conn} do
-      [user2, user3, user4, user5, user6] = insert_list(5, :user)
-
-      identifiers =
-        [
-          user2.ap_id,
-          user3.nickname,
-          "@" <> user4.nickname,
-          user5.nickname <> "@localhost",
-          "@" <> user6.nickname <> "@localhost"
-        ]
-        |> Enum.join(" ")
-
-      response =
-        conn
-        |> post("/api/pleroma/blocks_import", %{"list" => identifiers})
-        |> json_response(:ok)
-
-      assert response == "job started"
-      assert [{:ok, job_result}] = ObanHelpers.perform_all()
-      assert job_result == [user2, user3, user4, user5, user6]
-    end
-  end
-
   describe "PUT /api/pleroma/notification_settings" do
     setup do: oauth_access(["write:accounts"])