Merge branch 'feature/addressable-lists' into 'develop'

[#802] Add addressable lists

See merge request pleroma/pleroma!1113
This commit is contained in:
kaniini 2019-07-15 13:00:33 +00:00
commit d725ccfd3a
18 changed files with 373 additions and 53 deletions

View file

@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Added synchronization of following/followers counters for external users
- Configuration: `enabled` option for `Pleroma.Emails.Mailer`, defaulting to `false`.
- Configuration: Pleroma.Plugs.RateLimiter `bucket_name`, `params` options.
- Addressable lists
### Changed
- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text

View file

@ -16,9 +16,11 @@ Adding the parameter `with_muted=true` to the timeline queries will also return
## Statuses
- `visibility`: has an additional possible value `list`
Has these additional fields under the `pleroma` object:
- `local`: true if the post was made on the local instance.
- `local`: true if the post was made on the local instance
- `conversation_id`: the ID of the conversation the status is associated with (if any)
- `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any)
- `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
@ -72,6 +74,7 @@ Additional parameters can be added to the JSON body/Form data:
- `preview`: boolean, if set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example.
- `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
- `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply.
- `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
## PATCH `/api/v1/update_credentials`

View file

@ -16,6 +16,7 @@ defmodule Pleroma.List do
belongs_to(:user, User, type: Pleroma.FlakeId)
field(:title, :string)
field(:following, {:array, :string}, default: [])
field(:ap_id, :string)
timestamps()
end
@ -55,6 +56,10 @@ defmodule Pleroma.List do
Repo.one(query)
end
def get_by_ap_id(ap_id) do
Repo.get_by(__MODULE__, ap_id: ap_id)
end
def get_following(%Pleroma.List{following: following} = _list) do
q =
from(
@ -105,7 +110,14 @@ defmodule Pleroma.List do
def create(title, %User{} = creator) do
list = %Pleroma.List{user_id: creator.id, title: title}
Repo.insert(list)
Repo.transaction(fn ->
list = Repo.insert!(list)
list
|> change(ap_id: "#{creator.ap_id}/lists/#{list.id}")
|> Repo.update!()
end)
end
def follow(%Pleroma.List{following: following} = list, %User{} = followed) do
@ -125,4 +137,19 @@ defmodule Pleroma.List do
|> follow_changeset(attrs)
|> Repo.update()
end
def memberships(%User{follower_address: follower_address}) do
Pleroma.List
|> where([l], ^follower_address in l.following)
|> select([l], l.ap_id)
|> Repo.all()
end
def memberships(_), do: []
def member?(%Pleroma.List{following: following}, %User{follower_address: follower_address}) do
Enum.member?(following, follower_address)
end
def member?(_, _), do: false
end

View file

@ -27,19 +27,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
# For Announce activities, we filter the recipients based on following status for any actors
# that match actual users. See issue #164 for more information about why this is necessary.
defp get_recipients(%{"type" => "Announce"} = data) do
to = data["to"] || []
cc = data["cc"] || []
to = Map.get(data, "to", [])
cc = Map.get(data, "cc", [])
bcc = Map.get(data, "bcc", [])
actor = User.get_cached_by_ap_id(data["actor"])
recipients =
(to ++ cc)
|> Enum.filter(fn recipient ->
Enum.filter(Enum.concat([to, cc, bcc]), fn recipient ->
case User.get_cached_by_ap_id(recipient) do
nil ->
true
user ->
User.following?(user, actor)
nil -> true
user -> User.following?(user, actor)
end
end)
@ -47,17 +44,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
defp get_recipients(%{"type" => "Create"} = data) do
to = data["to"] || []
cc = data["cc"] || []
actor = data["actor"] || []
recipients = (to ++ cc ++ [actor]) |> Enum.uniq()
to = Map.get(data, "to", [])
cc = Map.get(data, "cc", [])
bcc = Map.get(data, "bcc", [])
actor = Map.get(data, "actor", [])
recipients = [to, cc, bcc, [actor]] |> Enum.concat() |> Enum.uniq()
{recipients, to, cc}
end
defp get_recipients(data) do
to = data["to"] || []
cc = data["cc"] || []
recipients = to ++ cc
to = Map.get(data, "to", [])
cc = Map.get(data, "cc", [])
bcc = Map.get(data, "bcc", [])
recipients = Enum.concat([to, cc, bcc])
{recipients, to, cc}
end
@ -898,13 +897,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp maybe_order(query, _), do: query
def fetch_activities_query(recipients, opts \\ %{}) do
base_query = from(activity in Activity)
config = %{
skip_thread_containment: Config.get([:instance, :skip_thread_containment])
}
base_query
Activity
|> maybe_preload_objects(opts)
|> maybe_preload_bookmarks(opts)
|> maybe_set_thread_muted_field(opts)
@ -933,11 +930,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
def fetch_activities(recipients, opts \\ %{}) do
fetch_activities_query(recipients, opts)
list_memberships = Pleroma.List.memberships(opts["user"])
fetch_activities_query(recipients ++ list_memberships, opts)
|> Pagination.fetch_paginated(opts)
|> Enum.reverse()
|> maybe_update_cc(list_memberships, opts["user"])
end
defp maybe_update_cc(activities, list_memberships, %User{ap_id: user_ap_id})
when is_list(list_memberships) and length(list_memberships) > 0 do
Enum.map(activities, fn
%{data: %{"bcc" => bcc}} = activity when is_list(bcc) and length(bcc) > 0 ->
if Enum.any?(bcc, &(&1 in list_memberships)) do
update_in(activity.data["cc"], &[user_ap_id | &1])
else
activity
end
activity ->
activity
end)
end
defp maybe_update_cc(activities, _, _), do: activities
def fetch_activities_bounded_query(query, recipients, recipients_with_public) do
from(activity in query,
where:

View file

@ -92,18 +92,68 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
end
end
@doc """
Publishes an activity to all relevant peers.
"""
def publish(%User{} = actor, %Activity{} = activity) do
remote_followers =
defp recipients(actor, activity) do
followers =
if actor.follower_address in activity.recipients do
{:ok, followers} = User.get_followers(actor)
followers |> Enum.filter(&(!&1.local))
Enum.filter(followers, &(!&1.local))
else
[]
end
Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers
end
defp get_cc_ap_ids(ap_id, recipients) do
host = Map.get(URI.parse(ap_id), :host)
recipients
|> Enum.filter(fn %User{ap_id: ap_id} -> Map.get(URI.parse(ap_id), :host) == host end)
|> Enum.map(& &1.ap_id)
end
@doc """
Publishes an activity with BCC to all relevant peers.
"""
def publish(actor, %{data: %{"bcc" => bcc}} = activity) when is_list(bcc) and bcc != [] do
public = is_public?(activity)
{:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
recipients = recipients(actor, activity)
recipients
|> Enum.filter(&User.ap_enabled?/1)
|> Enum.map(fn %{info: %{source_data: data}} -> data["inbox"] end)
|> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
|> Instances.filter_reachable()
|> Enum.each(fn {inbox, unreachable_since} ->
%User{ap_id: ap_id} =
Enum.find(recipients, fn %{info: %{source_data: data}} -> data["inbox"] == inbox end)
# Get all the recipients on the same host and add them to cc. Otherwise it a remote
# instance would only accept a first message for the first recipient and ignore the rest.
cc = get_cc_ap_ids(ap_id, recipients)
json =
data
|> Map.put("cc", cc)
|> Jason.encode!()
Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{
inbox: inbox,
json: json,
actor: actor,
id: activity.data["id"],
unreachable_since: unreachable_since
})
end)
end
@doc """
Publishes an activity to all relevant peers.
"""
def publish(%User{} = actor, %Activity{} = activity) do
public = is_public?(activity)
if public && Config.get([:instance, :allow_relay]) do
@ -114,7 +164,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
{:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
json = Jason.encode!(data)
(Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
recipients(actor, activity)
|> Enum.filter(fn user -> User.ap_enabled?(user) end)
|> Enum.map(fn %{info: %{source_data: data}} ->
(is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]

View file

@ -814,13 +814,16 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def prepare_outgoing(%{"type" => "Create", "object" => object_id} = data) do
object =
Object.normalize(object_id).data
object_id
|> Object.normalize()
|> Map.get(:data)
|> prepare_object
data =
data
|> Map.put("object", object)
|> Map.merge(Utils.make_json_ld_header())
|> Map.delete("bcc")
{:ok, data}
end

View file

@ -34,6 +34,20 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
!is_public?(activity) && !is_private?(activity)
end
def is_list?(%{data: %{"listMessage" => _}}), do: true
def is_list?(_), do: false
def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true
def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do
user.ap_id in activity.data["to"] ||
list_ap_id
|> Pleroma.List.get_by_ap_id()
|> Pleroma.List.member?(user)
end
def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false
def visible_for_user?(activity, nil) do
is_public?(activity)
end
@ -73,6 +87,9 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
object.data["directMessage"] == true ->
"direct"
is_binary(object.data["listMessage"]) ->
"list"
length(cc) > 0 ->
"private"

View file

@ -176,6 +176,11 @@ defmodule Pleroma.Web.CommonAPI do
when visibility in ~w{public unlisted private direct},
do: {visibility, get_replied_to_visibility(in_reply_to)}
def get_visibility(%{"visibility" => "list:" <> list_id}, in_reply_to) do
visibility = {:list, String.to_integer(list_id)}
{visibility, get_replied_to_visibility(in_reply_to)}
end
def get_visibility(_, in_reply_to) when not is_nil(in_reply_to) do
visibility = get_replied_to_visibility(in_reply_to)
{visibility, visibility}
@ -236,19 +241,18 @@ defmodule Pleroma.Web.CommonAPI do
"emoji",
Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji)
) do
res =
ActivityPub.create(
preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
direct? = visibility == "direct"
%{
to: to,
actor: user,
context: context,
object: object,
additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
},
Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
)
res
additional: %{"cc" => cc, "directMessage" => direct?}
}
|> maybe_add_list_data(user, visibility)
|> ActivityPub.create(preview?)
else
{:private_to_public, true} ->
{:error, dgettext("errors", "The message visibility must be direct")}

View file

@ -100,12 +100,29 @@ defmodule Pleroma.Web.CommonAPI.Utils do
end
end
def get_to_and_cc(_user, mentions, _inReplyTo, {:list, _}), do: {mentions, []}
def get_addressed_users(_, to) when is_list(to) do
User.get_ap_ids_by_nicknames(to)
end
def get_addressed_users(mentioned_users, _), do: mentioned_users
def maybe_add_list_data(activity_params, user, {:list, list_id}) do
case Pleroma.List.get(list_id, user) do
%Pleroma.List{} = list ->
activity_params
|> put_in([:additional, "bcc"], [list.ap_id])
|> put_in([:additional, "listMessage"], list.ap_id)
|> put_in([:object, "listMessage"], list.ap_id)
_ ->
activity_params
end
end
def maybe_add_list_data(activity_params, _, _), do: activity_params
def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data)
when is_list(options) do
%{max_expiration: max_expiration, min_expiration: min_expiration} =

View file

@ -123,11 +123,26 @@ defmodule Pleroma.Web.Salmon do
{:ok, salmon}
end
def remote_users(%{data: %{"to" => to} = data}) do
to = to ++ (data["cc"] || [])
def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do
cc = Map.get(data, "cc", [])
to
|> Enum.map(fn id -> User.get_cached_by_ap_id(id) end)
bcc =
data
|> Map.get("bcc", [])
|> Enum.reduce([], fn ap_id, bcc ->
case Pleroma.List.get_by_ap_id(ap_id) do
%Pleroma.List{user_id: ^user_id} = list ->
{:ok, following} = Pleroma.List.get_following(list)
bcc ++ Enum.map(following, & &1.ap_id)
_ ->
bcc
end
end)
[to, cc, bcc]
|> Enum.concat()
|> Enum.map(&User.get_cached_by_ap_id/1)
|> Enum.filter(fn user -> user && !user.local end)
end
@ -191,7 +206,7 @@ defmodule Pleroma.Web.Salmon do
{:ok, private, _} = Keys.keys_from_pem(keys)
{:ok, feed} = encode(private, feed)
remote_users = remote_users(activity)
remote_users = remote_users(user, activity)
salmon_urls = Enum.map(remote_users, & &1.info.salmon)
reachable_urls_metadata = Instances.filter_reachable(salmon_urls)

View file

@ -0,0 +1,26 @@
defmodule Pleroma.Repo.Migrations.AddApIdToLists do
use Ecto.Migration
def up do
alter table(:lists) do
add(:ap_id, :string)
end
execute("""
UPDATE lists
SET ap_id = u.ap_id || '/lists/' || lists.id
FROM users AS u
WHERE lists.user_id = u.id
""")
create(unique_index(:lists, :ap_id))
end
def down do
drop(index(:lists, [:ap_id]))
alter table(:lists) do
remove(:ap_id)
end
end
end

View file

@ -20,6 +20,10 @@
"sensitive": "as:sensitive",
"litepub": "http://litepub.social/ns#",
"directMessage": "litepub:directMessage",
"listMessage": {
"@id": "litepub:listMessage",
"@type": "@id"
},
"oauthRegistrationEndpoint": {
"@id": "litepub:oauthRegistrationEndpoint",
"@type": "@id"

View file

@ -113,4 +113,30 @@ defmodule Pleroma.ListTest do
assert owned_list in lists_2
refute not_owned_list in lists_2
end
test "get by ap_id" do
user = insert(:user)
{:ok, list} = Pleroma.List.create("foo", user)
assert Pleroma.List.get_by_ap_id(list.ap_id) == list
end
test "memberships" do
user = insert(:user)
member = insert(:user)
{:ok, list} = Pleroma.List.create("foo", user)
{:ok, list} = Pleroma.List.follow(list, member)
assert Pleroma.List.memberships(member) == [list.ap_id]
end
test "member?" do
user = insert(:user)
member = insert(:user)
{:ok, list} = Pleroma.List.create("foo", user)
{:ok, list} = Pleroma.List.follow(list, member)
assert Pleroma.List.member?(list, member)
refute Pleroma.List.member?(list, user)
end
end

View file

@ -1190,6 +1190,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
end
end
test "fetch_activities/2 returns activities addressed to a list " do
user = insert(:user)
member = insert(:user)
{:ok, list} = Pleroma.List.create("foo", user)
{:ok, list} = Pleroma.List.follow(list, member)
{:ok, activity} =
CommonAPI.post(user, %{"status" => "foobar", "visibility" => "list:#{list.id}"})
activity = Repo.preload(activity, :bookmark)
activity = %Activity{activity | thread_muted?: !!activity.thread_muted?}
assert ActivityPub.fetch_activities([], %{"user" => user}) == [activity]
end
def data_uri do
File.read!("test/fixtures/avatar_data_uri")
end

View file

@ -1098,6 +1098,18 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert modified["directMessage"] == true
end
test "it strips BCC field" do
user = insert(:user)
{:ok, list} = Pleroma.List.create("foo", user)
{:ok, activity} =
CommonAPI.post(user, %{"status" => "foobar", "visibility" => "list:#{list.id}"})
{:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
assert is_nil(modified["bcc"])
end
end
describe "user upgrade" do

View file

@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
following = insert(:user)
unrelated = insert(:user)
{:ok, following} = Pleroma.User.follow(following, user)
{:ok, list} = Pleroma.List.create("foo", user)
Pleroma.List.follow(list, unrelated)
{:ok, public} =
CommonAPI.post(user, %{"status" => "@#{mentioned.nickname}", "visibility" => "public"})
@ -29,6 +32,12 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
{:ok, unlisted} =
CommonAPI.post(user, %{"status" => "@#{mentioned.nickname}", "visibility" => "unlisted"})
{:ok, list} =
CommonAPI.post(user, %{
"status" => "@#{mentioned.nickname}",
"visibility" => "list:#{list.id}"
})
%{
public: public,
private: private,
@ -37,29 +46,65 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
user: user,
mentioned: mentioned,
following: following,
unrelated: unrelated
unrelated: unrelated,
list: list
}
end
test "is_direct?", %{public: public, private: private, direct: direct, unlisted: unlisted} do
test "is_direct?", %{
public: public,
private: private,
direct: direct,
unlisted: unlisted,
list: list
} do
assert Visibility.is_direct?(direct)
refute Visibility.is_direct?(public)
refute Visibility.is_direct?(private)
refute Visibility.is_direct?(unlisted)
assert Visibility.is_direct?(list)
end
test "is_public?", %{public: public, private: private, direct: direct, unlisted: unlisted} do
test "is_public?", %{
public: public,
private: private,
direct: direct,
unlisted: unlisted,
list: list
} do
refute Visibility.is_public?(direct)
assert Visibility.is_public?(public)
refute Visibility.is_public?(private)
assert Visibility.is_public?(unlisted)
refute Visibility.is_public?(list)
end
test "is_private?", %{public: public, private: private, direct: direct, unlisted: unlisted} do
test "is_private?", %{
public: public,
private: private,
direct: direct,
unlisted: unlisted,
list: list
} do
refute Visibility.is_private?(direct)
refute Visibility.is_private?(public)
assert Visibility.is_private?(private)
refute Visibility.is_private?(unlisted)
refute Visibility.is_private?(list)
end
test "is_list?", %{
public: public,
private: private,
direct: direct,
unlisted: unlisted,
list: list
} do
refute Visibility.is_list?(direct)
refute Visibility.is_list?(public)
refute Visibility.is_list?(private)
refute Visibility.is_list?(unlisted)
assert Visibility.is_list?(list)
end
test "visible_for_user?", %{
@ -70,7 +115,8 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
user: user,
mentioned: mentioned,
following: following,
unrelated: unrelated
unrelated: unrelated,
list: list
} do
# All visible to author
@ -78,6 +124,7 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
assert Visibility.visible_for_user?(private, user)
assert Visibility.visible_for_user?(unlisted, user)
assert Visibility.visible_for_user?(direct, user)
assert Visibility.visible_for_user?(list, user)
# All visible to a mentioned user
@ -85,6 +132,7 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
assert Visibility.visible_for_user?(private, mentioned)
assert Visibility.visible_for_user?(unlisted, mentioned)
assert Visibility.visible_for_user?(direct, mentioned)
assert Visibility.visible_for_user?(list, mentioned)
# DM not visible for just follower
@ -92,6 +140,7 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
assert Visibility.visible_for_user?(private, following)
assert Visibility.visible_for_user?(unlisted, following)
refute Visibility.visible_for_user?(direct, following)
refute Visibility.visible_for_user?(list, following)
# Public and unlisted visible for unrelated user
@ -99,6 +148,9 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
assert Visibility.visible_for_user?(unlisted, unrelated)
refute Visibility.visible_for_user?(private, unrelated)
refute Visibility.visible_for_user?(direct, unrelated)
# Visible for a list member
assert Visibility.visible_for_user?(list, unrelated)
end
test "doesn't die when the user doesn't exist",
@ -115,18 +167,24 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do
public: public,
private: private,
direct: direct,
unlisted: unlisted
unlisted: unlisted,
list: list
} do
assert Visibility.get_visibility(public) == "public"
assert Visibility.get_visibility(private) == "private"
assert Visibility.get_visibility(direct) == "direct"
assert Visibility.get_visibility(unlisted) == "unlisted"
assert Visibility.get_visibility(list) == "list"
end
test "get_visibility with directMessage flag" do
assert Visibility.get_visibility(%{data: %{"directMessage" => true}}) == "direct"
end
test "get_visibility with listMessage flag" do
assert Visibility.get_visibility(%{data: %{"listMessage" => ""}}) == "list"
end
describe "entire_thread_visible_for_user?/2" do
test "returns false if not found activity", %{user: user} do
refute Visibility.entire_thread_visible_for_user?(%Activity{}, user)

View file

@ -129,6 +129,18 @@ defmodule Pleroma.Web.CommonAPITest do
})
end)
end
test "it allows to address a list" do
user = insert(:user)
{:ok, list} = Pleroma.List.create("foo", user)
{:ok, activity} =
CommonAPI.post(user, %{"status" => "foobar", "visibility" => "list:#{list.id}"})
assert activity.data["bcc"] == [list.ap_id]
assert activity.recipients == [list.ap_id, user.ap_id]
assert activity.data["listMessage"] == list.ap_id
end
end
describe "reactions" do

View file

@ -541,4 +541,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
assert result[:reblog][:account][:pleroma][:relationship] ==
AccountView.render("relationship.json", %{user: user, target: user})
end
test "visibility/list" do
user = insert(:user)
{:ok, list} = Pleroma.List.create("foo", user)
{:ok, activity} =
CommonAPI.post(user, %{"status" => "foobar", "visibility" => "list:#{list.id}"})
status = StatusView.render("status.json", activity: activity)
assert status.visibility == "list"
end
end