Allow to specify post language

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>

changelog.d/post-languages.add

@@ -0,0 +1 @@
+Allow to specify post language

lib/pleroma/constants.ex

@@ -19,7 +19,8 @@ defmodule Pleroma.Constants do
       "context_id",
       "deleted_activity_id",
       "pleroma_internal",
-      "generator"
+      "generator",
+      "language"
     ]
   )
 
@@ -38,7 +39,8 @@ defmodule Pleroma.Constants do
       "summary",
       "sensitive",
       "attachment",
-      "generator"
+      "generator",
+      "language"
     ]
   )
 

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -86,6 +86,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
     |> fix_attachments()
     |> Transmogrifier.fix_emoji()
     |> Transmogrifier.fix_content_map()
+    |> Transmogrifier.maybe_add_language()
   end
 
   def changeset(struct, data) do

lib/pleroma/web/activity_pub/object_validators/common_fields.ex

@@ -57,6 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
       field(:replies_count, :integer, default: 0)
       field(:like_count, :integer, default: 0)
       field(:announcement_count, :integer, default: 0)
+      field(:language, :string)
       field(:inReplyTo, ObjectValidators.ObjectID)
       field(:url, ObjectValidators.BareUri)
 

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -22,6 +22,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Web.Federator
 
   import Ecto.Query
+  import Pleroma.Web.CommonAPI.Utils, only: [is_good_locale_code?: 1]
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
 
   require Logger
   require Pleroma.Constants
@@ -42,6 +44,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_content_map()
     |> fix_addressing()
     |> fix_summary()
+    |> maybe_add_language()
   end
 
   def fix_summary(%{"summary" => nil} = object) do
@@ -318,6 +321,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_tag(object), do: object
 
+  def fix_content_map(%{"content" => content} = object) when not_empty_string(content), do: object
+
   # content map usually only has one language so this will do for now.
   def fix_content_map(%{"contentMap" => content_map} = object) do
     content_groups = Map.to_list(content_map)
@@ -454,6 +459,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       |> strip_internal_fields()
       |> fix_type(fetch_options)
       |> fix_in_reply_to(fetch_options)
+      |> maybe_add_language_from_activity(data)
 
     data = Map.put(data, "object", object)
     options = Keyword.put(options, :local, false)
@@ -679,6 +685,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> add_mention_tags
     |> add_emoji_tags
     |> add_attributed_to
+    |> maybe_add_content_map
     |> prepare_attachments
     |> set_conversation
     |> set_reply_to_uri
@@ -722,7 +729,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     data =
       data
       |> Map.put("object", object)
-      |> Map.merge(Utils.make_json_ld_header())
+      |> Map.merge(Utils.make_json_ld_header(data))
       |> Map.delete("bcc")
 
     {:ok, data}
@@ -737,7 +744,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     data =
       data
       |> Map.put("object", object)
-      |> Map.merge(Utils.make_json_ld_header())
+      |> Map.merge(Utils.make_json_ld_header(data))
       |> Map.delete("bcc")
 
     {:ok, data}
@@ -758,7 +765,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     data =
       data
       |> strip_internal_fields
-      |> Map.merge(Utils.make_json_ld_header())
+      |> Map.merge(Utils.make_json_ld_header(data))
       |> Map.delete("bcc")
 
     {:ok, data}
@@ -778,7 +785,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       data =
         data
         |> Map.put("object", object)
-        |> Map.merge(Utils.make_json_ld_header())
+        |> Map.merge(Utils.make_json_ld_header(data))
 
       {:ok, data}
     end
@@ -796,7 +803,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       data =
         data
         |> Map.put("object", object)
-        |> Map.merge(Utils.make_json_ld_header())
+        |> Map.merge(Utils.make_json_ld_header(data))
 
       {:ok, data}
     end
@@ -807,7 +814,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       data
       |> strip_internal_fields
       |> maybe_fix_object_url
-      |> Map.merge(Utils.make_json_ld_header())
+      |> Map.merge(Utils.make_json_ld_header(data))
 
     {:ok, data}
   end
@@ -952,4 +959,63 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def maybe_fix_user_url(data), do: data
 
   def maybe_fix_user_object(data), do: maybe_fix_user_url(data)
+
+  defp maybe_add_content_map(%{"language" => language, "content" => content} = object)
+       when not_empty_string(language) do
+    Map.put(object, "contentMap", Map.put(%{}, language, content))
+  end
+
+  defp maybe_add_content_map(object), do: object
+
+  def maybe_add_language(object) do
+    language =
+      [
+        get_language_from_context(object),
+        get_language_from_content_map(object),
+        get_language_from_content(object)
+      ]
+      |> Enum.find(&is_good_locale_code?(&1))
+
+    if language do
+      Map.put(object, "language", language)
+    else
+      object
+    end
+  end
+
+  def maybe_add_language_from_activity(object, activity) do
+    language = get_language_from_context(activity)
+
+    if is_good_locale_code?(language) do
+      Map.put(object, "language", language)
+    else
+      object
+    end
+  end
+
+  defp get_language_from_context(%{"@context" => context}) when is_list(context) do
+    case context
+         |> Enum.find(fn
+           %{"@language" => language} -> language != "und"
+           _ -> nil
+         end) do
+      %{"@language" => language} -> language
+      _ -> nil
+    end
+  end
+
+  defp get_language_from_context(_), do: nil
+
+  defp get_language_from_content_map(%{"contentMap" => content_map, "content" => source_content}) do
+    content_groups = Map.to_list(content_map)
+
+    case Enum.find(content_groups, fn {_, content} -> content == source_content end) do
+      {language, _} -> language
+      _ -> nil
+    end
+  end
+
+  defp get_language_from_content_map(_), do: nil
+
+  defp get_language_from_content(_), do: nil
 end

lib/pleroma/web/activity_pub/utils.ex

@@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   alias Pleroma.Web.Router.Helpers
 
   import Ecto.Query
+  import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
 
   require Logger
   require Pleroma.Constants
@@ -108,18 +109,24 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     end
   end
 
-  def make_json_ld_header do
+  def make_json_ld_header(data \\ %{}) do
     %{
       "@context" => [
         "https://www.w3.org/ns/activitystreams",
         "#{Endpoint.url()}/schemas/litepub-0.1.jsonld",
         %{
-          "@language" => "und"
+          "@language" => get_language(data)
         }
       ]
     }
   end
 
+  defp get_language(%{"language" => language}) when not_empty_string(language) do
+    language
+  end
+
+  defp get_language(_), do: "und"
+
   def make_date do
     DateTime.utc_now() |> DateTime.to_iso8601()
   end

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   def render("object.json", %{object: %Object{} = object}) do
-    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(object.data)
 
     additional = Transmogrifier.prepare_object(object.data)
     Map.merge(base, additional)
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{data: %{"type" => activity_type}} = activity})
       when activity_type in ["Create", "Listen"] do
-    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(activity.data)
     object = Object.normalize(activity, fetch: false)
 
     additional =
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
   end
 
   def render("object.json", %{object: %Activity{} = activity}) do
-    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(activity.data)
     object_id = Object.normalize(activity, id_only: true)
 
     additional =

lib/pleroma/web/common_api/activity_draft.ex

@@ -33,6 +33,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
             cc: [],
             context: nil,
             sensitive: false,
+            language: nil,
             object: nil,
             preview?: false,
             changes: %{}
@@ -57,6 +58,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     |> content()
     |> with_valid(&to_and_cc/1)
     |> with_valid(&context/1)
+    |> with_valid(&language/1)
     |> sensitive()
     |> with_valid(&object/1)
     |> preview?()
@@ -190,6 +192,16 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
     %__MODULE__{draft | sensitive: sensitive}
   end
 
+  defp language(draft) do
+    language = draft.params[:language]
+
+    if Utils.is_good_locale_code?(language) do
+      %__MODULE__{draft | language: language}
+    else
+      draft
+    end
+  end
+
   defp object(draft) do
     emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji)
 
@@ -229,6 +241,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
         "mediaType" => Utils.get_content_type(draft.params[:content_type])
       })
       |> Map.put("generator", draft.params[:generator])
+      |> Map.put("language", draft.language)
 
     %__MODULE__{draft | object: object}
   end

lib/pleroma/web/common_api/utils.ex

@@ -494,4 +494,20 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       {:error, dgettext("errors", "Too many attachments")}
     end
   end
+
+  def is_good_locale_code?(code) when is_binary(code) do
+    code
+    |> String.codepoints()
+    |> Enum.all?(&valid_char?/1)
+  end
+
+  def is_good_locale_code?(_code), do: false
+
+  # [a-zA-Z0-9-]
+  defp valid_char?(char) do
+    ("a" <= char and char <= "z") or
+      ("A" <= char and char <= "Z") or
+      ("0" <= char and char <= "9") or
+      char == "-"
+  end
 end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -200,7 +200,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       mentions: mentions,
       tags: reblogged[:tags] || [],
       application: build_application(object.data["generator"]),
-      language: nil,
+      language: object.data["language"],
       emojis: [],
       pleroma: %{
         local: activity.local,
@@ -391,7 +391,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       mentions: mentions,
       tags: build_tags(tags),
       application: build_application(object.data["generator"]),
-      language: nil,
+      language: object.data["language"],
       emojis: build_emojis(object.data["emoji"]),
       pleroma: %{
         local: activity.local,

test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs

@@ -221,6 +221,36 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
                "<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>"
     end
 
+    test "it only uses contentMap if content is not present" do
+      user = insert(:user)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "cc" => [],
+        "type" => "Create",
+        "object" => %{
+          "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+          "cc" => [],
+          "id" => Utils.generate_object_id(),
+          "type" => "Note",
+          "content" => "Hi",
+          "contentMap" => %{
+            "de" => "Hallo",
+            "uk" => "Привіт"
+          },
+          "inReplyTo" => nil,
+          "attributedTo" => user.ap_id
+        },
+        "actor" => user.ap_id
+      }
+
+      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+      object = Object.normalize(data["object"], fetch: false)
+
+      assert object.data["content"] == "Hi"
+    end
+
     test "it works for incoming notices with to/cc not being an array (kroeg)" do
       data = File.read!("test/fixtures/kroeg-post-activity.json") |> Jason.decode!()
 
@@ -358,6 +388,87 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
     end
   end
 
+  test "it detects language from context" do
+    user = insert(:user)
+
+    message = %{
+      "@context" => ["https://www.w3.org/ns/activitystreams", %{"@language" => "pl"}],
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "cc" => [],
+      "type" => "Create",
+      "object" => %{
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "cc" => [],
+        "id" => Utils.generate_object_id(),
+        "type" => "Note",
+        "content" => "Szczęść Boże",
+        "attributedTo" => user.ap_id
+      },
+      "actor" => user.ap_id
+    }
+
+    {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+    object = Object.normalize(data["object"], fetch: false)
+
+    assert object.data["language"] == "pl"
+  end
+
+  test "it detects language from contentMap" do
+    user = insert(:user)
+
+    message = %{
+      "@context" => "https://www.w3.org/ns/activitystreams",
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "cc" => [],
+      "type" => "Create",
+      "object" => %{
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "cc" => [],
+        "id" => Utils.generate_object_id(),
+        "type" => "Note",
+        "content" => "Szczęść Boże",
+        "contentMap" => %{
+          "de" => "Gott segne",
+          "pl" => "Szczęść Boże"
+        },
+        "attributedTo" => user.ap_id
+      },
+      "actor" => user.ap_id
+    }
+
+    {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+    object = Object.normalize(data["object"], fetch: false)
+
+    assert object.data["language"] == "pl"
+  end
+
+  test "it detects language from content" do
+    clear_config([Pleroma.Language.LanguageDetector, :provider], LanguageDetectorMock)
+
+    user = insert(:user)
+
+    message = %{
+      "@context" => ["https://www.w3.org/ns/activitystreams"],
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "cc" => [],
+      "type" => "Create",
+      "object" => %{
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "cc" => [],
+        "id" => Utils.generate_object_id(),
+        "type" => "Note",
+        "content" => "Dieu vous bénisse, Fédivers.",
+        "attributedTo" => user.ap_id
+      },
+      "actor" => user.ap_id
+    }
+
+    {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+    object = Object.normalize(data["object"], fetch: false)
+
+    assert object.data["language"] == "fr"
+  end
+
   describe "`handle_incoming/2`, Mastodon format `replies` handling" do
     setup do: clear_config([:activitypub, :note_replies_output_limit], 5)
     setup do: clear_config([:instance, :federation_incoming_replies_max_depth])

test/pleroma/web/activity_pub/transmogrifier_test.exs

@@ -352,6 +352,18 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
     end
   end
 
+  test "it adds contentMap if language is specified" do
+    user = insert(:user)
+
+    {:ok, activity} = CommonAPI.post(user, %{status: "тест", language: "uk"})
+
+    {:ok, prepared} = Transmogrifier.prepare_outgoing(activity.data)
+
+    assert prepared["object"]["contentMap"] == %{
+             "uk" => "тест"
+           }
+  end
+
   describe "actor rewriting" do
     test "it fixes the actor URL property to be a proper URI" do
       data = %{

test/pleroma/web/activity_pub/utils_test.exs

@@ -138,16 +138,30 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
     end
   end
 
-  test "make_json_ld_header/0" do
-    assert Utils.make_json_ld_header() == %{
-             "@context" => [
-               "https://www.w3.org/ns/activitystreams",
-               "http://localhost:4001/schemas/litepub-0.1.jsonld",
-               %{
-                 "@language" => "und"
-               }
-             ]
-           }
+  describe "make_json_ld_header/1" do
+    test "makes jsonld header" do
+      assert Utils.make_json_ld_header() == %{
+               "@context" => [
+                 "https://www.w3.org/ns/activitystreams",
+                 "http://localhost:4001/schemas/litepub-0.1.jsonld",
+                 %{
+                   "@language" => "und"
+                 }
+               ]
+             }
+    end
+
+    test "includes language if specified" do
+      assert Utils.make_json_ld_header(%{"language" => "pl"}) == %{
+               "@context" => [
+                 "https://www.w3.org/ns/activitystreams",
+                 "http://localhost:4001/schemas/litepub-0.1.jsonld",
+                 %{
+                   "@language" => "pl"
+                 }
+               ]
+             }
+    end
   end
 
   describe "get_existing_votes" do

test/pleroma/web/mastodon_api/views/status_view_test.exs

@@ -770,6 +770,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
     assert status.edited_at
   end
 
+  test "it shows post language" do
+    user = insert(:user)
+
+    {:ok, post} = CommonAPI.post(user, %{status: "Szczęść Boże", language: "pl"})
+
+    status = StatusView.render("show.json", activity: post)
+
+    assert status.language == "pl"
+  end
+
   test "with a source object" do
     note =
       insert(:note,