searxng/searx/locales.py
Alexandre Flament ed66ed758d [mod] reduce memory footprint by not calling babel.Locale.parse at runtime
babel.Locale.parse loads more than 60MB in RAM.  The only purpose is to get:

    LOCALE_NAMES   - searx.data.LOCALES["LOCALE_NAMES"]
    RTL_LOCALES    - searx.data.LOCALES["RTL_LOCALES"]

This commit calls babel.Locale.parse when the translations are update from
weblate and stored in::

    searx/data/locales.json

This file can be build by::

    ./manage data.locales

By store these variables in searx.data when the translations are updated we save
round about 65MB (usually 4 worker = 260MB of RAM saved.

Suggested-by: https://github.com/searxng/searxng/discussions/2633#discussioncomment-8490494
Co-authored-by: Markus Heiser <markus.heiser@darmarit.de>
2024-02-20 10:43:20 +01:00

462 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# -*- coding: utf-8 -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""
SearXNGs locale data
=====================
The variables :py:obj:`RTL_LOCALES` and :py:obj:`LOCALE_NAMES` are loaded from
:origin:`searx/data/locales.json` / see :py:obj:`locales_initialize` and
:ref:`update_locales.py`.
.. hint::
Whenever the value of :py:obj:`ADDITIONAL_TRANSLATIONS` or
:py:obj:`LOCALE_BEST_MATCH` is modified, the
:origin:`searx/data/locales.json` needs to be rebuild::
./manage data.locales
SearXNG's locale codes
======================
.. automodule:: searx.sxng_locales
:members:
SearXNGs locale implementations
================================
"""
from __future__ import annotations
from pathlib import Path
import babel
from babel.support import Translations
import babel.languages
import babel.core
import flask_babel
import flask
from flask.ctx import has_request_context
from searx import (
data,
logger,
searx_dir,
)
logger = logger.getChild('locales')
# safe before monkey patching flask_babel.get_translations
_flask_babel_get_translations = flask_babel.get_translations
LOCALE_NAMES = {}
"""Mapping of locales and their description. Locales e.g. 'fr' or 'pt-BR' (see
:py:obj:`locales_initialize`).
:meta hide-value:
"""
RTL_LOCALES: set[str] = set()
"""List of *Right-To-Left* locales e.g. 'he' or 'fa-IR' (see
:py:obj:`locales_initialize`)."""
ADDITIONAL_TRANSLATIONS = {
"dv": "ދިވެހި (Dhivehi)",
"oc": "Occitan",
"szl": "Ślōnski (Silesian)",
"pap": "Papiamento",
}
"""Additional languages SearXNG has translations for but not supported by
python-babel (see :py:obj:`locales_initialize`)."""
LOCALE_BEST_MATCH = {
"dv": "si",
"oc": 'fr-FR',
"szl": "pl",
"nl-BE": "nl",
"zh-HK": "zh-Hant-TW",
"pap": "pt-BR",
}
"""Map a locale we do not have a translations for to a locale we have a
translation for. By example: use Taiwan version of the translation for Hong
Kong."""
def localeselector():
locale = 'en'
if has_request_context():
value = flask.request.preferences.get_value('locale')
if value:
locale = value
# first, set the language that is not supported by babel
if locale in ADDITIONAL_TRANSLATIONS:
flask.request.form['use-translation'] = locale
# second, map locale to a value python-babel supports
locale = LOCALE_BEST_MATCH.get(locale, locale)
if locale == '':
# if there is an error loading the preferences
# the locale is going to be ''
locale = 'en'
# babel uses underscore instead of hyphen.
locale = locale.replace('-', '_')
return locale
def get_translations():
"""Monkey patch of :py:obj:`flask_babel.get_translations`"""
if has_request_context():
use_translation = flask.request.form.get('use-translation')
if use_translation in ADDITIONAL_TRANSLATIONS:
babel_ext = flask_babel.current_app.extensions['babel']
return Translations.load(babel_ext.translation_directories[0], use_translation)
return _flask_babel_get_translations()
_TR_LOCALES: list[str] = []
def get_translation_locales() -> list[str]:
"""Returns the list of transaltion locales (*underscore*). The list is
generated from the translation folders in :origin:`searx/translations`"""
global _TR_LOCALES # pylint:disable=global-statement
if _TR_LOCALES:
return _TR_LOCALES
tr_locales = []
for folder in (Path(searx_dir) / 'translations').iterdir():
if not folder.is_dir():
continue
if not (folder / 'LC_MESSAGES').is_dir():
continue
tr_locales.append(folder.name)
_TR_LOCALES = sorted(tr_locales)
return _TR_LOCALES
def locales_initialize():
"""Initialize locales environment of the SearXNG session.
- monkey patch :py:obj:`flask_babel.get_translations` by :py:obj:`get_translations`
- init global names :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`
"""
flask_babel.get_translations = get_translations
LOCALE_NAMES.update(data.LOCALES["LOCALE_NAMES"])
RTL_LOCALES.update(data.LOCALES["RTL_LOCALES"])
def region_tag(locale: babel.Locale) -> str:
"""Returns SearXNG's region tag from the locale (e.g. zh-TW , en-US)."""
if not locale.territory:
raise ValueError('%s missed a territory')
return locale.language + '-' + locale.territory
def language_tag(locale: babel.Locale) -> str:
"""Returns SearXNG's language tag from the locale and if exits, the tag
includes the script name (e.g. en, zh_Hant).
"""
sxng_lang = locale.language
if locale.script:
sxng_lang += '_' + locale.script
return sxng_lang
def get_locale(locale_tag: str) -> babel.Locale | None:
"""Returns a :py:obj:`babel.Locale` object parsed from argument
``locale_tag``"""
try:
locale = babel.Locale.parse(locale_tag, sep='-')
return locale
except babel.core.UnknownLocaleError:
return None
def get_official_locales(
territory: str, languages=None, regional: bool = False, de_facto: bool = True
) -> set[babel.Locale]:
"""Returns a list of :py:obj:`babel.Locale` with languages from
:py:obj:`babel.languages.get_official_languages`.
:param territory: The territory (country or region) code.
:param languages: A list of language codes the languages from
:py:obj:`babel.languages.get_official_languages` should be in
(intersection). If this argument is ``None``, all official languages in
this territory are used.
:param regional: If the regional flag is set, then languages which are
regionally official are also returned.
:param de_facto: If the de_facto flag is set to `False`, then languages
which are “de facto” official are not returned.
"""
ret_val = set()
o_languages = babel.languages.get_official_languages(territory, regional=regional, de_facto=de_facto)
if languages:
languages = [l.lower() for l in languages]
o_languages = set(l for l in o_languages if l.lower() in languages)
for lang in o_languages:
try:
locale = babel.Locale.parse(lang + '_' + territory)
ret_val.add(locale)
except babel.UnknownLocaleError:
continue
return ret_val
def get_engine_locale(searxng_locale, engine_locales, default=None):
"""Return engine's language (aka locale) string that best fits to argument
``searxng_locale``.
Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to
corresponding *engine locales*::
<engine>: {
# SearXNG string : engine-string
'ca-ES' : 'ca_ES',
'fr-BE' : 'fr_BE',
'fr-CA' : 'fr_CA',
'fr-CH' : 'fr_CH',
'fr' : 'fr_FR',
...
'pl-PL' : 'pl_PL',
'pt-PT' : 'pt_PT'
..
'zh' : 'zh'
'zh_Hans' : 'zh'
'zh_Hant' : 'zh_TW'
}
.. hint::
The *SearXNG locale* string has to be known by babel!
If there is no direct 1:1 mapping, this functions tries to narrow down
engine's language (locale). If no value can be determined by these
approximation attempts the ``default`` value is returned.
Assumptions:
A. When user select a language the results should be optimized according to
the selected language.
B. When user select a language and a territory the results should be
optimized with first priority on territory and second on language.
First approximation rule (*by territory*):
When the user selects a locale with territory (and a language), the
territory has priority over the language. If any of the official languages
in the territory is supported by the engine (``engine_locales``) it will
be used.
Second approximation rule (*by language*):
If "First approximation rule" brings no result or the user selects only a
language without a territory. Check in which territories the language
has an official status and if one of these territories is supported by the
engine.
"""
# pylint: disable=too-many-branches, too-many-return-statements
engine_locale = engine_locales.get(searxng_locale)
if engine_locale is not None:
# There was a 1:1 mapping (e.g. a region "fr-BE --> fr_BE" or a language
# "zh --> zh"), no need to narrow language-script nor territory.
return engine_locale
try:
locale = babel.Locale.parse(searxng_locale, sep='-')
except babel.core.UnknownLocaleError:
try:
locale = babel.Locale.parse(searxng_locale.split('-')[0])
except babel.core.UnknownLocaleError:
return default
searxng_lang = language_tag(locale)
engine_locale = engine_locales.get(searxng_lang)
if engine_locale is not None:
# There was a 1:1 mapping (e.g. "zh-HK --> zh_Hant" or "zh-CN --> zh_Hans")
return engine_locale
# SearXNG's selected locale is not supported by the engine ..
if locale.territory:
# Try to narrow by *official* languages in the territory (??-XX).
for official_language in babel.languages.get_official_languages(locale.territory, de_facto=True):
searxng_locale = official_language + '-' + locale.territory
engine_locale = engine_locales.get(searxng_locale)
if engine_locale is not None:
return engine_locale
# Engine does not support one of the official languages in the territory or
# there is only a language selected without a territory.
# Now lets have a look if the searxng_lang (the language selected by the
# user) is a official language in other territories. If so, check if
# engine does support the searxng_lang in this other territory.
if locale.language:
terr_lang_dict = {}
for territory, langs in babel.core.get_global("territory_languages").items():
if not langs.get(searxng_lang, {}).get('official_status'):
continue
terr_lang_dict[territory] = langs.get(searxng_lang)
# first: check fr-FR, de-DE .. is supported by the engine
# exception: 'en' --> 'en-US'
territory = locale.language.upper()
if territory == 'EN':
territory = 'US'
if terr_lang_dict.get(territory):
searxng_locale = locale.language + '-' + territory
engine_locale = engine_locales.get(searxng_locale)
if engine_locale is not None:
return engine_locale
# second: sort by population_percent and take first match
# drawback of "population percent": if there is a territory with a
# small number of people (e.g 100) but the majority speaks the
# language, then the percentage might be 100% (--> 100 people) but in
# a different territory with more people (e.g. 10.000) where only 10%
# speak the language the total amount of speaker is higher (--> 200
# people).
#
# By example: The population of Saint-Martin is 33.000, of which 100%
# speak French, but this is less than the 30% of the approximately 2.5
# million Belgian citizens
#
# - 'fr-MF', 'population_percent': 100.0, 'official_status': 'official'
# - 'fr-BE', 'population_percent': 38.0, 'official_status': 'official'
terr_lang_list = []
for k, v in terr_lang_dict.items():
terr_lang_list.append((k, v))
for territory, _lang in sorted(terr_lang_list, key=lambda item: item[1]['population_percent'], reverse=True):
searxng_locale = locale.language + '-' + territory
engine_locale = engine_locales.get(searxng_locale)
if engine_locale is not None:
return engine_locale
# No luck: narrow by "language from territory" and "territory from language"
# does not fit to a locale supported by the engine.
if engine_locale is None:
engine_locale = default
return default
def match_locale(searxng_locale: str, locale_tag_list: list[str], fallback: str | None = None) -> str | None:
"""Return tag from ``locale_tag_list`` that best fits to ``searxng_locale``.
:param str searxng_locale: SearXNG's internal representation of locale (de,
de-DE, fr-BE, zh, zh-CN, zh-TW ..).
:param list locale_tag_list: The list of locale tags to select from
:param str fallback: fallback locale tag (if unset --> ``None``)
The rules to find a match are implemented in :py:obj:`get_engine_locale`,
the ``engine_locales`` is build up by :py:obj:`build_engine_locales`.
.. hint::
The *SearXNG locale* string and the members of ``locale_tag_list`` has to
be known by babel! The :py:obj:`ADDITIONAL_TRANSLATIONS` are used in the
UI and are not known by babel --> will be ignored.
"""
# searxng_locale = 'es'
# locale_tag_list = ['es-AR', 'es-ES', 'es-MX']
if not searxng_locale:
return fallback
locale = get_locale(searxng_locale)
if locale is None:
return fallback
# normalize to a SearXNG locale that can be passed to get_engine_locale
searxng_locale = language_tag(locale)
if locale.territory:
searxng_locale = region_tag(locale)
# clean up locale_tag_list
tag_list = []
for tag in locale_tag_list:
if tag in ('all', 'auto') or tag in ADDITIONAL_TRANSLATIONS:
continue
tag_list.append(tag)
# emulate fetch_traits
engine_locales = build_engine_locales(tag_list)
return get_engine_locale(searxng_locale, engine_locales, default=fallback)
def build_engine_locales(tag_list: list[str]):
"""From a list of locale tags a dictionary is build that can be passed by
argument ``engine_locales`` to :py:obj:`get_engine_locale`. This function
is mainly used by :py:obj:`match_locale` and is similar to what the
``fetch_traits(..)`` function of engines do.
If there are territory codes in the ``tag_list`` that have a *script code*
additional keys are added to the returned dictionary.
.. code:: python
>>> import locales
>>> engine_locales = locales.build_engine_locales(['en', 'en-US', 'zh', 'zh-CN', 'zh-TW'])
>>> engine_locales
{
'en': 'en', 'en-US': 'en-US',
'zh': 'zh', 'zh-CN': 'zh-CN', 'zh_Hans': 'zh-CN',
'zh-TW': 'zh-TW', 'zh_Hant': 'zh-TW'
}
>>> get_engine_locale('zh-Hans', engine_locales)
'zh-CN'
This function is a good example to understand the language/region model
of SearXNG:
SearXNG only distinguishes between **search languages** and **search
regions**, by adding the *script-tags*, languages with *script-tags* can
be assigned to the **regions** that SearXNG supports.
"""
engine_locales = {}
for tag in tag_list:
locale = get_locale(tag)
if locale is None:
logger.warning("build_engine_locales: skip locale tag %s / unknown by babel", tag)
continue
if locale.territory:
engine_locales[region_tag(locale)] = tag
if locale.script:
engine_locales[language_tag(locale)] = tag
else:
engine_locales[language_tag(locale)] = tag
return engine_locales