From 8bec926bebe855e0968f5b71368876cbf2439333 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 8 Nov 2022 17:06:16 -0600 Subject: [PATCH] LanguageDetector: strip non-language text to (hopefully) improve accuracy --- lib/pleroma/language/language_detector.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex index b19eb4571..0be69d220 100644 --- a/lib/pleroma/language/language_detector.ex +++ b/lib/pleroma/language/language_detector.ex @@ -15,10 +15,18 @@ defmodule Pleroma.Language.LanguageDetector do end end + # Strip tags from text, etc. + defp prepare_text(text) do + text + |> Floki.parse_fragment!() + |> Floki.filter_out(".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre") + |> Floki.text() + end + def detect(text) do provider = get_provider() - {:ok, text} = text |> FastSanitize.strip_tags() + text = prepare_text(text) word_count = text |> String.split(~r/\s+/) |> Enum.count() if word_count < @words_threshold or !provider or !provider.configured? do