Rename Activity.Search to Search.DatabaseSearch

This commit is contained in:
Ekaterina Vaartis 2021-11-17 22:29:49 +03:00
parent 0fae71f88d
commit a6946048fb
12 changed files with 181 additions and 193 deletions

View file

@ -870,7 +870,7 @@ config :pleroma, ConcurrentLimiter, [
{Pleroma.Search, [max_running: 30, max_waiting: 50]} {Pleroma.Search, [max_running: 30, max_waiting: 50]}
] ]
config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil

View file

@ -133,7 +133,7 @@ config :pleroma, :side_effects,
ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock, ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
logger: Pleroma.LoggerMock logger: Pleroma.LoggerMock
config :pleroma, Pleroma.Search, module: Pleroma.Activity config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil

View file

@ -6,7 +6,7 @@
To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`: To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`:
> config :pleroma, Pleroma.Search, module: Pleroma.Activity > config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
While it has no external dependencies, it has problems with performance and relevancy. While it has no external dependencies, it has problems with performance and relevancy.

View file

@ -366,7 +366,7 @@ defmodule Pleroma.Activity do
from(activity in query, where: activity.actor not in subquery(deactivated_users_query)) from(activity in query, where: activity.actor not in subquery(deactivated_users_query))
end end
defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch
def direct_conversation_id(activity, for_user) do def direct_conversation_id(activity, for_user) do
alias Pleroma.Conversation.Participation alias Pleroma.Conversation.Participation

View file

@ -1,165 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Activity.Search do
alias Pleroma.Activity
alias Pleroma.Object.Fetcher
alias Pleroma.Pagination
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Visibility
require Pleroma.Constants
import Ecto.Query
def search(user, search_query, options \\ []) do
index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
limit = Enum.min([Keyword.get(options, :limit), 40])
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
search_function =
if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
:websearch
else
:plain
end
try do
Activity
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public(user)
|> query_with(index_type, search_query, search_function)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> Pagination.fetch_paginated(
%{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
:offset
)
|> maybe_fetch(user, search_query)
rescue
_ -> maybe_fetch([], user, search_query)
end
end
def add_to_index(_activity), do: nil
def remove_from_index(_object), do: nil
def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author)
end
def maybe_restrict_author(query, _), do: query
def maybe_restrict_blocked(query, %User{} = user) do
Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
end
def maybe_restrict_blocked(query, _), do: query
defp restrict_public(q, user) when not is_nil(user) do
intended_recipients = [
Pleroma.Constants.as_public(),
Pleroma.Web.ActivityPub.Utils.as_local_public()
]
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: fragment("? && ?", ^intended_recipients, a.recipients)
)
end
defp restrict_public(q, _user) do
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients
)
end
defp query_with(q, :gin, search_query, :plain) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :gin, search_query, :websearch) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :rum, search_query, :plain) do
from([a, o] in q,
where:
fragment(
"? @@ plainto_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
defp query_with(q, :rum, search_query, :websearch) do
from([a, o] in q,
where:
fragment(
"? @@ websearch_to_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
def maybe_restrict_local(q, user) do
limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
case {limit, user} do
{:all, _} -> restrict_local(q)
{:unauthenticated, %User{}} -> q
{:unauthenticated, _} -> restrict_local(q)
{false, _} -> q
end
end
defp restrict_local(q), do: where(q, local: true)
def maybe_fetch(activities, user, search_query) do
with true <- Regex.match?(~r/https?:/, search_query),
{:ok, object} <- Fetcher.fetch_object_from_id(search_query),
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
true <- Visibility.visible_for_user?(activity, user) do
[activity | activities]
else
_ -> activities
end
end
end

24
lib/pleroma/search.ex Normal file
View file

@ -0,0 +1,24 @@
defmodule Pleroma.Search do
def add_to_index(activity) do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
ConcurrentLimiter.limit(Pleroma.Search, fn ->
Task.start(fn -> search_module.add_to_index(activity) end)
end)
end
def remove_from_index(object) do
# Also delete from search index
search_module = Pleroma.Config.get([Pleroma.Search, :module])
ConcurrentLimiter.limit(Pleroma.Search, fn ->
Task.start(fn -> search_module.remove_from_index(object) end)
end)
end
def search(query, options) do
search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
search_module.search(options[:for_user], query, options)
end
end

View file

@ -1,24 +1,153 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.DatabaseSearch do defmodule Pleroma.Search.DatabaseSearch do
def add_to_index(activity) do alias Pleroma.Activity
search_module = Pleroma.Config.get([Pleroma.Search, :module]) alias Pleroma.Object.Fetcher
alias Pleroma.Pagination
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Visibility
ConcurrentLimiter.limit(Pleroma.Search, fn -> require Pleroma.Constants
Task.start(fn -> search_module.add_to_index(activity) end)
end) import Ecto.Query
def search(user, search_query, options \\ []) do
index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
limit = Enum.min([Keyword.get(options, :limit), 40])
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
search_function =
if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
:websearch
else
:plain
end
try do
Activity
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public()
|> query_with(index_type, search_query, search_function)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> Pagination.fetch_paginated(
%{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
:offset
)
|> maybe_fetch(user, search_query)
rescue
_ -> maybe_fetch([], user, search_query)
end
end end
def remove_from_index(object) do def add_to_index(_activity), do: nil
# Also delete from search index def remove_from_index(_object), do: nil
search_module = Pleroma.Config.get([Pleroma.Search, :module])
ConcurrentLimiter.limit(Pleroma.Search, fn -> def maybe_restrict_author(query, %User{} = author) do
Task.start(fn -> search_module.remove_from_index(object) end) Activity.Queries.by_author(query, author)
end)
end end
def search(query, options) do def maybe_restrict_author(query, _), do: query
search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
search_module.search(options[:for_user], query, options) def maybe_restrict_blocked(query, %User{} = user) do
Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
end
def maybe_restrict_blocked(query, _), do: query
def restrict_public(q) do
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients
)
end
defp query_with(q, :gin, search_query, :plain) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :gin, search_query, :websearch) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :rum, search_query, :plain) do
from([a, o] in q,
where:
fragment(
"? @@ plainto_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
defp query_with(q, :rum, search_query, :websearch) do
from([a, o] in q,
where:
fragment(
"? @@ websearch_to_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
def maybe_restrict_local(q, user) do
limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
case {limit, user} do
{:all, _} -> restrict_local(q)
{:unauthenticated, %User{}} -> q
{:unauthenticated, _} -> restrict_local(q)
{false, _} -> q
end
end
defp restrict_local(q), do: where(q, local: true)
def maybe_fetch(activities, user, search_query) do
with true <- Regex.match?(~r/https?:/, search_query),
{:ok, object} <- Fetcher.fetch_object_from_id(search_query),
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
true <- Visibility.visible_for_user?(activity, user) do
[activity | activities]
else
_ -> activities
end
end end
end end

View file

@ -4,7 +4,7 @@ defmodule Pleroma.Search.Meilisearch do
alias Pleroma.Activity alias Pleroma.Activity
import Pleroma.Activity.Search import Pleroma.Search.DatabaseSearch
import Ecto.Query import Ecto.Query
defp meili_headers do defp meili_headers do

View file

@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end) end)
# Add local posts to search index # Add local posts to search index
if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity) if local, do: Pleroma.Search.add_to_index(activity)
{:ok, activity} {:ok, activity}
else else

View file

@ -227,7 +227,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end) end)
Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object)) Pleroma.Search.add_to_index(Map.put(activity, :object, object))
meta = meta =
meta meta
@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
if result == :ok do if result == :ok do
Notification.create_notifications(object) Notification.create_notifications(object)
Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object) Pleroma.Search.remove_from_index(deleted_object)
{:ok, object, meta} {:ok, object, meta}
else else

View file

@ -99,7 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
end end
defp resource_search(_, "statuses", query, options) do defp resource_search(_, "statuses", query, options) do
statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end) statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end)
StatusView.render("index.json", StatusView.render("index.json",
activities: statuses, activities: statuses,

View file

@ -2,8 +2,8 @@
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Activity.SearchTest do defmodule Pleroma.Search.DatabaseSearchTest do
alias Pleroma.Activity.Search alias Pleroma.Search.DatabaseSearch
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI
import Pleroma.Factory import Pleroma.Factory
@ -13,7 +13,7 @@ defmodule Pleroma.Activity.SearchTest do
user = insert(:user) user = insert(:user)
{:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
[result] = Search.search(nil, "wednesday") [result] = DatabaseSearch.search(nil, "wednesday")
assert result.id == post.id assert result.id == post.id
end end
@ -45,7 +45,7 @@ defmodule Pleroma.Activity.SearchTest do
{:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
# plainto doesn't understand complex queries # plainto doesn't understand complex queries
assert [result] = Search.search(nil, "wednesday -dudes") assert [result] = DatabaseSearch.search(nil, "wednesday -dudes")
assert result.id == post.id assert result.id == post.id
end end
@ -55,7 +55,7 @@ defmodule Pleroma.Activity.SearchTest do
{:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
{:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
assert [result] = Search.search(nil, "wednesday -dudes") assert [result] = DatabaseSearch.search(nil, "wednesday -dudes")
assert result.id == other_post.id assert result.id == other_post.id
end end