mirror of
https://github.com/searxng/searxng.git
synced 2024-11-29 06:01:02 +00:00
cdb4927b8b
This patch fixes a bug reported by CI "Fetch traits" [1] (brave) and improves
other fetch traits functions (google, annas_archive & radio_browser).
brave:
File "/home/runner/work/searxng/searxng/searx/engines/brave.py", line 434, in fetch_traits
sxng_tag = region_tag(babel.Locale.parse(ui_lang, sep='-'))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/searxng/searxng/searx/locales.py", line 155, in region_tag
Error: raise ValueError('%s missed a territory')
google:
change ERROR message about unknow UI language to INFO message
radio_browser:
country_list contains duplicates that differ only in upper/lower case
annas_archive:
for better diff; sort the persistence of the traits
[1] https://github.com/searxng/searxng/actions/runs/10606312371/job/29433352518#step:6:41
Signed-off-by: Markus <markus@venom.fritz.box>
460 lines
15 KiB
Python
460 lines
15 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||
"""
|
||
SearXNG’s locale data
|
||
=====================
|
||
|
||
The variables :py:obj:`RTL_LOCALES` and :py:obj:`LOCALE_NAMES` are loaded from
|
||
:origin:`searx/data/locales.json` / see :py:obj:`locales_initialize` and
|
||
:ref:`update_locales.py`.
|
||
|
||
.. hint::
|
||
|
||
Whenever the value of :py:obj:`ADDITIONAL_TRANSLATIONS` or
|
||
:py:obj:`LOCALE_BEST_MATCH` is modified, the
|
||
:origin:`searx/data/locales.json` needs to be rebuild::
|
||
|
||
./manage data.locales
|
||
|
||
SearXNG's locale codes
|
||
======================
|
||
|
||
.. automodule:: searx.sxng_locales
|
||
:members:
|
||
|
||
|
||
SearXNG’s locale implementations
|
||
================================
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from pathlib import Path
|
||
|
||
import babel
|
||
from babel.support import Translations
|
||
import babel.languages
|
||
import babel.core
|
||
import flask_babel
|
||
import flask
|
||
from flask.ctx import has_request_context
|
||
from searx import (
|
||
data,
|
||
logger,
|
||
searx_dir,
|
||
)
|
||
|
||
logger = logger.getChild('locales')
|
||
|
||
|
||
# safe before monkey patching flask_babel.get_translations
|
||
_flask_babel_get_translations = flask_babel.get_translations
|
||
|
||
LOCALE_NAMES = {}
|
||
"""Mapping of locales and their description. Locales e.g. 'fr' or 'pt-BR' (see
|
||
:py:obj:`locales_initialize`).
|
||
|
||
:meta hide-value:
|
||
"""
|
||
|
||
RTL_LOCALES: set[str] = set()
|
||
"""List of *Right-To-Left* locales e.g. 'he' or 'fa-IR' (see
|
||
:py:obj:`locales_initialize`)."""
|
||
|
||
ADDITIONAL_TRANSLATIONS = {
|
||
"dv": "ދިވެހި (Dhivehi)",
|
||
"oc": "Occitan",
|
||
"szl": "Ślōnski (Silesian)",
|
||
"pap": "Papiamento",
|
||
}
|
||
"""Additional languages SearXNG has translations for but not supported by
|
||
python-babel (see :py:obj:`locales_initialize`)."""
|
||
|
||
LOCALE_BEST_MATCH = {
|
||
"dv": "si",
|
||
"oc": 'fr-FR',
|
||
"szl": "pl",
|
||
"nl-BE": "nl",
|
||
"zh-HK": "zh-Hant-TW",
|
||
"pap": "pt-BR",
|
||
}
|
||
"""Map a locale we do not have a translations for to a locale we have a
|
||
translation for. By example: use Taiwan version of the translation for Hong
|
||
Kong."""
|
||
|
||
|
||
def localeselector():
|
||
locale = 'en'
|
||
if has_request_context():
|
||
value = flask.request.preferences.get_value('locale')
|
||
if value:
|
||
locale = value
|
||
|
||
# first, set the language that is not supported by babel
|
||
if locale in ADDITIONAL_TRANSLATIONS:
|
||
flask.request.form['use-translation'] = locale
|
||
|
||
# second, map locale to a value python-babel supports
|
||
locale = LOCALE_BEST_MATCH.get(locale, locale)
|
||
|
||
if locale == '':
|
||
# if there is an error loading the preferences
|
||
# the locale is going to be ''
|
||
locale = 'en'
|
||
|
||
# babel uses underscore instead of hyphen.
|
||
locale = locale.replace('-', '_')
|
||
return locale
|
||
|
||
|
||
def get_translations():
|
||
"""Monkey patch of :py:obj:`flask_babel.get_translations`"""
|
||
if has_request_context():
|
||
use_translation = flask.request.form.get('use-translation')
|
||
if use_translation in ADDITIONAL_TRANSLATIONS:
|
||
babel_ext = flask_babel.current_app.extensions['babel']
|
||
return Translations.load(babel_ext.translation_directories[0], use_translation)
|
||
return _flask_babel_get_translations()
|
||
|
||
|
||
_TR_LOCALES: list[str] = []
|
||
|
||
|
||
def get_translation_locales() -> list[str]:
|
||
"""Returns the list of transaltion locales (*underscore*). The list is
|
||
generated from the translation folders in :origin:`searx/translations`"""
|
||
|
||
global _TR_LOCALES # pylint:disable=global-statement
|
||
if _TR_LOCALES:
|
||
return _TR_LOCALES
|
||
|
||
tr_locales = []
|
||
for folder in (Path(searx_dir) / 'translations').iterdir():
|
||
if not folder.is_dir():
|
||
continue
|
||
if not (folder / 'LC_MESSAGES').is_dir():
|
||
continue
|
||
tr_locales.append(folder.name)
|
||
_TR_LOCALES = sorted(tr_locales)
|
||
return _TR_LOCALES
|
||
|
||
|
||
def locales_initialize():
|
||
"""Initialize locales environment of the SearXNG session.
|
||
|
||
- monkey patch :py:obj:`flask_babel.get_translations` by :py:obj:`get_translations`
|
||
- init global names :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`
|
||
"""
|
||
flask_babel.get_translations = get_translations
|
||
LOCALE_NAMES.update(data.LOCALES["LOCALE_NAMES"])
|
||
RTL_LOCALES.update(data.LOCALES["RTL_LOCALES"])
|
||
|
||
|
||
def region_tag(locale: babel.Locale) -> str:
|
||
"""Returns SearXNG's region tag from the locale (e.g. zh-TW , en-US)."""
|
||
if not locale.territory:
|
||
raise ValueError('babel.Locale %s: missed a territory' % locale)
|
||
return locale.language + '-' + locale.territory
|
||
|
||
|
||
def language_tag(locale: babel.Locale) -> str:
|
||
"""Returns SearXNG's language tag from the locale and if exits, the tag
|
||
includes the script name (e.g. en, zh_Hant).
|
||
"""
|
||
sxng_lang = locale.language
|
||
if locale.script:
|
||
sxng_lang += '_' + locale.script
|
||
return sxng_lang
|
||
|
||
|
||
def get_locale(locale_tag: str) -> babel.Locale | None:
|
||
"""Returns a :py:obj:`babel.Locale` object parsed from argument
|
||
``locale_tag``"""
|
||
try:
|
||
locale = babel.Locale.parse(locale_tag, sep='-')
|
||
return locale
|
||
|
||
except babel.core.UnknownLocaleError:
|
||
return None
|
||
|
||
|
||
def get_official_locales(
|
||
territory: str, languages=None, regional: bool = False, de_facto: bool = True
|
||
) -> set[babel.Locale]:
|
||
"""Returns a list of :py:obj:`babel.Locale` with languages from
|
||
:py:obj:`babel.languages.get_official_languages`.
|
||
|
||
:param territory: The territory (country or region) code.
|
||
|
||
:param languages: A list of language codes the languages from
|
||
:py:obj:`babel.languages.get_official_languages` should be in
|
||
(intersection). If this argument is ``None``, all official languages in
|
||
this territory are used.
|
||
|
||
:param regional: If the regional flag is set, then languages which are
|
||
regionally official are also returned.
|
||
|
||
:param de_facto: If the de_facto flag is set to `False`, then languages
|
||
which are “de facto” official are not returned.
|
||
|
||
"""
|
||
ret_val = set()
|
||
o_languages = babel.languages.get_official_languages(territory, regional=regional, de_facto=de_facto)
|
||
|
||
if languages:
|
||
languages = [l.lower() for l in languages]
|
||
o_languages = set(l for l in o_languages if l.lower() in languages)
|
||
|
||
for lang in o_languages:
|
||
try:
|
||
locale = babel.Locale.parse(lang + '_' + territory)
|
||
ret_val.add(locale)
|
||
except babel.UnknownLocaleError:
|
||
continue
|
||
|
||
return ret_val
|
||
|
||
|
||
def get_engine_locale(searxng_locale, engine_locales, default=None):
|
||
"""Return engine's language (aka locale) string that best fits to argument
|
||
``searxng_locale``.
|
||
|
||
Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to
|
||
corresponding *engine locales*::
|
||
|
||
<engine>: {
|
||
# SearXNG string : engine-string
|
||
'ca-ES' : 'ca_ES',
|
||
'fr-BE' : 'fr_BE',
|
||
'fr-CA' : 'fr_CA',
|
||
'fr-CH' : 'fr_CH',
|
||
'fr' : 'fr_FR',
|
||
...
|
||
'pl-PL' : 'pl_PL',
|
||
'pt-PT' : 'pt_PT'
|
||
..
|
||
'zh' : 'zh'
|
||
'zh_Hans' : 'zh'
|
||
'zh_Hant' : 'zh_TW'
|
||
}
|
||
|
||
.. hint::
|
||
|
||
The *SearXNG locale* string has to be known by babel!
|
||
|
||
If there is no direct 1:1 mapping, this functions tries to narrow down
|
||
engine's language (locale). If no value can be determined by these
|
||
approximation attempts the ``default`` value is returned.
|
||
|
||
Assumptions:
|
||
|
||
A. When user select a language the results should be optimized according to
|
||
the selected language.
|
||
|
||
B. When user select a language and a territory the results should be
|
||
optimized with first priority on territory and second on language.
|
||
|
||
First approximation rule (*by territory*):
|
||
|
||
When the user selects a locale with territory (and a language), the
|
||
territory has priority over the language. If any of the official languages
|
||
in the territory is supported by the engine (``engine_locales``) it will
|
||
be used.
|
||
|
||
Second approximation rule (*by language*):
|
||
|
||
If "First approximation rule" brings no result or the user selects only a
|
||
language without a territory. Check in which territories the language
|
||
has an official status and if one of these territories is supported by the
|
||
engine.
|
||
|
||
"""
|
||
# pylint: disable=too-many-branches, too-many-return-statements
|
||
|
||
engine_locale = engine_locales.get(searxng_locale)
|
||
|
||
if engine_locale is not None:
|
||
# There was a 1:1 mapping (e.g. a region "fr-BE --> fr_BE" or a language
|
||
# "zh --> zh"), no need to narrow language-script nor territory.
|
||
return engine_locale
|
||
|
||
try:
|
||
locale = babel.Locale.parse(searxng_locale, sep='-')
|
||
except babel.core.UnknownLocaleError:
|
||
try:
|
||
locale = babel.Locale.parse(searxng_locale.split('-')[0])
|
||
except babel.core.UnknownLocaleError:
|
||
return default
|
||
|
||
searxng_lang = language_tag(locale)
|
||
engine_locale = engine_locales.get(searxng_lang)
|
||
if engine_locale is not None:
|
||
# There was a 1:1 mapping (e.g. "zh-HK --> zh_Hant" or "zh-CN --> zh_Hans")
|
||
return engine_locale
|
||
|
||
# SearXNG's selected locale is not supported by the engine ..
|
||
|
||
if locale.territory:
|
||
# Try to narrow by *official* languages in the territory (??-XX).
|
||
|
||
for official_language in babel.languages.get_official_languages(locale.territory, de_facto=True):
|
||
searxng_locale = official_language + '-' + locale.territory
|
||
engine_locale = engine_locales.get(searxng_locale)
|
||
if engine_locale is not None:
|
||
return engine_locale
|
||
|
||
# Engine does not support one of the official languages in the territory or
|
||
# there is only a language selected without a territory.
|
||
|
||
# Now lets have a look if the searxng_lang (the language selected by the
|
||
# user) is a official language in other territories. If so, check if
|
||
# engine does support the searxng_lang in this other territory.
|
||
|
||
if locale.language:
|
||
|
||
terr_lang_dict = {}
|
||
for territory, langs in babel.core.get_global("territory_languages").items():
|
||
if not langs.get(searxng_lang, {}).get('official_status'):
|
||
continue
|
||
terr_lang_dict[territory] = langs.get(searxng_lang)
|
||
|
||
# first: check fr-FR, de-DE .. is supported by the engine
|
||
# exception: 'en' --> 'en-US'
|
||
|
||
territory = locale.language.upper()
|
||
if territory == 'EN':
|
||
territory = 'US'
|
||
|
||
if terr_lang_dict.get(territory):
|
||
searxng_locale = locale.language + '-' + territory
|
||
engine_locale = engine_locales.get(searxng_locale)
|
||
if engine_locale is not None:
|
||
return engine_locale
|
||
|
||
# second: sort by population_percent and take first match
|
||
|
||
# drawback of "population percent": if there is a territory with a
|
||
# small number of people (e.g 100) but the majority speaks the
|
||
# language, then the percentage might be 100% (--> 100 people) but in
|
||
# a different territory with more people (e.g. 10.000) where only 10%
|
||
# speak the language the total amount of speaker is higher (--> 200
|
||
# people).
|
||
#
|
||
# By example: The population of Saint-Martin is 33.000, of which 100%
|
||
# speak French, but this is less than the 30% of the approximately 2.5
|
||
# million Belgian citizens
|
||
#
|
||
# - 'fr-MF', 'population_percent': 100.0, 'official_status': 'official'
|
||
# - 'fr-BE', 'population_percent': 38.0, 'official_status': 'official'
|
||
|
||
terr_lang_list = []
|
||
for k, v in terr_lang_dict.items():
|
||
terr_lang_list.append((k, v))
|
||
|
||
for territory, _lang in sorted(terr_lang_list, key=lambda item: item[1]['population_percent'], reverse=True):
|
||
searxng_locale = locale.language + '-' + territory
|
||
engine_locale = engine_locales.get(searxng_locale)
|
||
if engine_locale is not None:
|
||
return engine_locale
|
||
|
||
# No luck: narrow by "language from territory" and "territory from language"
|
||
# does not fit to a locale supported by the engine.
|
||
|
||
if engine_locale is None:
|
||
engine_locale = default
|
||
|
||
return default
|
||
|
||
|
||
def match_locale(searxng_locale: str, locale_tag_list: list[str], fallback: str | None = None) -> str | None:
|
||
"""Return tag from ``locale_tag_list`` that best fits to ``searxng_locale``.
|
||
|
||
:param str searxng_locale: SearXNG's internal representation of locale (de,
|
||
de-DE, fr-BE, zh, zh-CN, zh-TW ..).
|
||
|
||
:param list locale_tag_list: The list of locale tags to select from
|
||
|
||
:param str fallback: fallback locale tag (if unset --> ``None``)
|
||
|
||
The rules to find a match are implemented in :py:obj:`get_engine_locale`,
|
||
the ``engine_locales`` is build up by :py:obj:`build_engine_locales`.
|
||
|
||
.. hint::
|
||
|
||
The *SearXNG locale* string and the members of ``locale_tag_list`` has to
|
||
be known by babel! The :py:obj:`ADDITIONAL_TRANSLATIONS` are used in the
|
||
UI and are not known by babel --> will be ignored.
|
||
"""
|
||
|
||
# searxng_locale = 'es'
|
||
# locale_tag_list = ['es-AR', 'es-ES', 'es-MX']
|
||
|
||
if not searxng_locale:
|
||
return fallback
|
||
|
||
locale = get_locale(searxng_locale)
|
||
if locale is None:
|
||
return fallback
|
||
|
||
# normalize to a SearXNG locale that can be passed to get_engine_locale
|
||
|
||
searxng_locale = language_tag(locale)
|
||
if locale.territory:
|
||
searxng_locale = region_tag(locale)
|
||
|
||
# clean up locale_tag_list
|
||
|
||
tag_list = []
|
||
for tag in locale_tag_list:
|
||
if tag in ('all', 'auto') or tag in ADDITIONAL_TRANSLATIONS:
|
||
continue
|
||
tag_list.append(tag)
|
||
|
||
# emulate fetch_traits
|
||
engine_locales = build_engine_locales(tag_list)
|
||
return get_engine_locale(searxng_locale, engine_locales, default=fallback)
|
||
|
||
|
||
def build_engine_locales(tag_list: list[str]):
|
||
"""From a list of locale tags a dictionary is build that can be passed by
|
||
argument ``engine_locales`` to :py:obj:`get_engine_locale`. This function
|
||
is mainly used by :py:obj:`match_locale` and is similar to what the
|
||
``fetch_traits(..)`` function of engines do.
|
||
|
||
If there are territory codes in the ``tag_list`` that have a *script code*
|
||
additional keys are added to the returned dictionary.
|
||
|
||
.. code:: python
|
||
|
||
>>> import locales
|
||
>>> engine_locales = locales.build_engine_locales(['en', 'en-US', 'zh', 'zh-CN', 'zh-TW'])
|
||
>>> engine_locales
|
||
{
|
||
'en': 'en', 'en-US': 'en-US',
|
||
'zh': 'zh', 'zh-CN': 'zh-CN', 'zh_Hans': 'zh-CN',
|
||
'zh-TW': 'zh-TW', 'zh_Hant': 'zh-TW'
|
||
}
|
||
>>> get_engine_locale('zh-Hans', engine_locales)
|
||
'zh-CN'
|
||
|
||
This function is a good example to understand the language/region model
|
||
of SearXNG:
|
||
|
||
SearXNG only distinguishes between **search languages** and **search
|
||
regions**, by adding the *script-tags*, languages with *script-tags* can
|
||
be assigned to the **regions** that SearXNG supports.
|
||
|
||
"""
|
||
engine_locales = {}
|
||
|
||
for tag in tag_list:
|
||
locale = get_locale(tag)
|
||
if locale is None:
|
||
logger.warning("build_engine_locales: skip locale tag %s / unknown by babel", tag)
|
||
continue
|
||
if locale.territory:
|
||
engine_locales[region_tag(locale)] = tag
|
||
if locale.script:
|
||
engine_locales[language_tag(locale)] = tag
|
||
else:
|
||
engine_locales[language_tag(locale)] = tag
|
||
return engine_locales
|