Merge branch 'features/ingestion-ecto-tag' into 'develop'

TagValidator: New

See merge request pleroma/pleroma!3244

lib/pleroma/web/activity_pub/object_validator.ex

@@ -114,9 +114,34 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(%{"type" => type} = object, meta)
+      when type in ~w[Event Question Audio Video Article] do
+    validator =
+      case type do
+        "Event" -> EventValidator
+        "Question" -> QuestionValidator
+        "Audio" -> AudioVideoValidator
+        "Video" -> AudioVideoValidator
+        "Article" -> ArticleNoteValidator
+      end
+
+    with {:ok, object} <-
+           object
+           |> validator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+
+      # Insert copy of hashtags as strings for the non-hashtag table indexing
+      tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object})
+      object = Map.put(object, "tag", tag)
+
+      {:ok, object, meta}
+    end
+  end
+
   def validate(%{"type" => type} = object, meta)
       when type in ~w[Accept Reject Follow Update Like EmojiReact Announce
-      Event ChatMessage Question Audio Video Article Answer] do
+      ChatMessage Answer] do
     validator =
       case type do
         "Accept" -> AcceptRejectValidator
@@ -126,12 +151,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         "Like" -> LikeValidator
         "EmojiReact" -> EmojiReactValidator
         "Announce" -> AnnounceValidator
-        "Event" -> EventValidator
         "ChatMessage" -> ChatMessageValidator
-        "Question" -> QuestionValidator
-        "Audio" -> AudioVideoValidator
-        "Video" -> AudioVideoValidator
-        "Article" -> ArticleNoteValidator
         "Answer" -> AnswerValidator
       end
 
@@ -180,7 +200,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
-  # is_struct/1 isn't present in Elixir 1.8.x
+  # is_struct/1 appears in Elixir 1.11
   def stringify_keys(%{__struct__: _} = object) do
     object
     |> Map.from_struct()

lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex

@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Accept", "Reject"])

lib/pleroma/web/activity_pub/object_validators/announce_validator.ex

@@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
     cng
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Announce"])
     |> validate_required([:id, :type, :object, :actor, :to, :cc])

lib/pleroma/web/activity_pub/object_validators/answer_validator.ex

@@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Answer"])
     |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor])

lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
@@ -22,8 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:bto, ObjectValidators.Recipients, default: [])
     field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
+    embeds_many(:tag, TagValidator)
-    field(:tag, {:array, :map}, default: [])
     field(:type, :string)
 
     field(:name, :string)
@@ -90,11 +90,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast(data, __schema__(:fields) -- [:attachment, :tag])
     |> cast_embed(:attachment)
+    |> cast_embed(:tag)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Article", "Note"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])

lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex

@@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator
 
   import Ecto.Changeset
 
@@ -90,7 +89,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     end
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_inclusion(:type, ~w[Document Audio Image Video])
     |> validate_required([:mediaType, :url, :type])

lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
@@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:bto, ObjectValidators.Recipients, default: [])
     field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
+    embeds_many(:tag, TagValidator)
-    field(:tag, {:array, :map}, default: [])
     field(:type, :string)
 
     field(:name, :string)
@@ -132,11 +132,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast(data, __schema__(:fields) -- [:attachment, :tag])
     |> cast_embed(:attachment)
+    |> cast_embed(:tag)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Audio", "Video"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])

lib/pleroma/web/activity_pub/object_validators/block_validator.ex

@@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Block"])

lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex

@@ -67,7 +67,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
     |> cast_embed(:attachment)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["ChatMessage"])
     |> validate_required([:id, :actor, :to, :type, :published])

lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex

@@ -39,7 +39,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
     |> validate_data(meta)
   end
 
-  def validate_data(cng, meta \\ []) do
+  defp validate_data(cng, meta) do
     cng
     |> validate_required([:id, :actor, :to, :type, :object])
     |> validate_inclusion(:type, ["Create"])

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -79,7 +79,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     |> CommonFixes.fix_actor()
   end
 
-  def validate_data(cng, meta \\ []) do
+  defp validate_data(cng, meta) do
     cng
     |> validate_required([:actor, :type, :object])
     |> validate_inclusion(:type, ["Create"])

lib/pleroma/web/activity_pub/object_validators/delete_validator.ex

@@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
     Tombstone
     Video
   }
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Delete"])

lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex

@@ -70,7 +70,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
     end
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["EmojiReact"])
     |> validate_required([:id, :type, :object, :actor, :context, :to, :cc, :content])

lib/pleroma/web/activity_pub/object_validators/event_validator.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
@@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:bto, ObjectValidators.Recipients, default: [])
     field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
+    embeds_many(:tag, TagValidator)
-    field(:tag, {:array, :map}, default: [])
     field(:type, :string)
 
     field(:name, :string)
@@ -81,11 +81,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast(data, __schema__(:fields) -- [:attachment, :tag])
     |> cast_embed(:attachment)
+    |> cast_embed(:tag)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Event"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])

lib/pleroma/web/activity_pub/object_validators/follow_validator.ex

@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Follow"])

lib/pleroma/web/activity_pub/object_validators/like_validator.ex

@@ -76,7 +76,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do
     end
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Like"])
     |> validate_required([:id, :type, :object, :actor, :context, :to, :cc])

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
@@ -24,8 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:bto, ObjectValidators.Recipients, default: [])
     field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
+    embeds_many(:tag, TagValidator)
-    field(:tag, {:array, :map}, default: [])
     field(:type, :string)
     field(:content, :string)
     field(:context, :string)
@@ -93,13 +93,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment])
+    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment, :tag])
     |> cast_embed(:attachment)
     |> cast_embed(:anyOf)
     |> cast_embed(:oneOf)
+    |> cast_embed(:tag)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Question"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])

lib/pleroma/web/activity_pub/object_validators/tag_validator.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+
+  import Ecto.Changeset
+
+  @primary_key false
+  embedded_schema do
+    # Common
+    field(:type, :string)
+    field(:name, :string)
+
+    # Mention, Hashtag
+    field(:href, ObjectValidators.Uri)
+
+    # Emoji
+    embeds_one :icon, IconObjectValidator, primary_key: false do
+      field(:type, :string)
+      field(:url, ObjectValidators.Uri)
+    end
+
+    field(:updated, ObjectValidators.DateTime)
+    field(:id, ObjectValidators.Uri)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, %{"type" => "Mention"} = data) do
+    struct
+    |> cast(data, [:type, :name, :href])
+    |> validate_required([:type, :href])
+  end
+
+  def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do
+    name =
+      cond do
+        "#" <> name -> name
+        name -> name
+      end
+      |> String.downcase()
+
+    data = Map.put(data, "name", name)
+
+    struct
+    |> cast(data, [:type, :name, :href])
+    |> validate_required([:type, :name])
+  end
+
+  def changeset(struct, %{"type" => "Emoji"} = data) do
+    data = Map.put(data, "name", String.trim(data["name"], ":"))
+
+    struct
+    |> cast(data, [:type, :name, :updated, :id])
+    |> cast_embed(:icon, with: &icon_changeset/2)
+    |> validate_required([:type, :name, :icon])
+  end
+
+  def icon_changeset(struct, data) do
+    struct
+    |> cast(data, [:type, :url])
+    |> validate_inclusion(:type, ~w[Image])
+    |> validate_required([:type, :url])
+  end
+end

lib/pleroma/web/activity_pub/object_validators/undo_validator.ex

@@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Undo"])
     |> validate_required([:id, :type, :object, :actor, :to, :cc])

lib/pleroma/web/activity_pub/object_validators/update_validator.ex

@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Update"])

lib/pleroma/web/activity_pub/pipeline.ex

@@ -40,19 +40,17 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
     end
   end
 
-  def do_common_pipeline(object, meta) do
+  def do_common_pipeline(%{__struct__: _}, _meta), do: {:error, :is_struct}
-    with {_, {:ok, validated_object, meta}} <-
+
-           {:validate_object, @object_validator.validate(object, meta)},
+  def do_common_pipeline(message, meta) do
-         {_, {:ok, mrfd_object, meta}} <-
+    with {_, {:ok, message, meta}} <- {:validate, @object_validator.validate(message, meta)},
-           {:mrf_object, @mrf.pipeline_filter(validated_object, meta)},
+         {_, {:ok, message, meta}} <- {:mrf, @mrf.pipeline_filter(message, meta)},
-         {_, {:ok, activity, meta}} <-
+         {_, {:ok, message, meta}} <- {:persist, @activity_pub.persist(message, meta)},
-           {:persist_object, @activity_pub.persist(mrfd_object, meta)},
+         {_, {:ok, message, meta}} <- {:side_effects, @side_effects.handle(message, meta)},
-         {_, {:ok, activity, meta}} <-
+         {_, {:ok, _}} <- {:federation, maybe_federate(message, meta)} do
-           {:execute_side_effects, @side_effects.handle(activity, meta)},
+      {:ok, message, meta}
-         {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
-      {:ok, activity, meta}
     else
-      {:mrf_object, {:reject, message, _}} -> {:reject, message}
+      {:mrf, {:reject, message, _}} -> {:reject, message}
       e -> {:error, e}
     end
   end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -564,7 +564,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
            Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
     else
-      {:error, {:validate_object, _}} = e ->
+      {:error, {:validate, _}} = e ->
         # Check if we have a create activity for this
         with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]),
              %Activity{data: %{"actor" => actor}} <-

lib/pleroma/web/common_api.ex

@@ -228,17 +228,7 @@ defmodule Pleroma.Web.CommonAPI do
       {:find_object, _} ->
         {:error, :not_found}
 
-      {:common_pipeline,
+      {:common_pipeline, {:error, {:validate, {:error, changeset}}}} = e ->
-       {
-         :error,
-         {
-           :validate_object,
-           {
-             :error,
-             changeset
-           }
-         }
-       }} = e ->
         if {:object, {"already liked by this actor", []}} in changeset.errors do
           {:ok, :already_liked}
         else

test/pleroma/web/activity_pub/pipeline_test.exs

@@ -25,9 +25,6 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
       MRFMock
       |> expect(:pipeline_filter, fn o, m -> {:ok, o, m} end)
 
-      ActivityPubMock
-      |> expect(:persist, fn o, m -> {:ok, o, m} end)
-
       SideEffectsMock
       |> expect(:handle, fn o, m -> {:ok, o, m} end)
       |> expect(:handle_after_transaction, fn m -> m end)
@@ -42,6 +39,9 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
 
       activity_with_object = %{activity | data: Map.put(activity.data, "object", object)}
 
+      ActivityPubMock
+      |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
       FederatorMock
       |> expect(:publish, fn ^activity_with_object -> :ok end)
 
@@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
 
       assert {:ok, ^activity, ^meta} =
                Pleroma.Web.ActivityPub.Pipeline.common_pipeline(
-                 activity,
+                 activity.data,
                  meta
                )
     end
@@ -59,6 +59,9 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
       activity = insert(:note_activity)
       meta = [local: true]
 
+      ActivityPubMock
+      |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
       FederatorMock
       |> expect(:publish, fn ^activity -> :ok end)
 
@@ -66,29 +69,35 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
       |> expect(:get, fn [:instance, :federating] -> true end)
 
       assert {:ok, ^activity, ^meta} =
-               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
     end
 
     test "it goes through validation, filtering, persisting, side effects without federation for remote activities" do
       activity = insert(:note_activity)
       meta = [local: false]
 
+      ActivityPubMock
+      |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
       ConfigMock
       |> expect(:get, fn [:instance, :federating] -> true end)
 
       assert {:ok, ^activity, ^meta} =
-               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
     end
 
     test "it goes through validation, filtering, persisting, side effects without federation for local activities if federation is deactivated" do
       activity = insert(:note_activity)
       meta = [local: true]
 
+      ActivityPubMock
+      |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
       ConfigMock
       |> expect(:get, fn [:instance, :federating] -> false end)
 
       assert {:ok, ^activity, ^meta} =
-               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
     end
   end
 end