From 61383edb27c50c3316226f88c60837d4a3c4f540 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 3 Oct 2022 17:21:13 +0200 Subject: [PATCH] [mod] Startpage: fetch engine traits (data_type: supported_languages) Implements a fetch_traits function for the Startpage engine. .. note:: Does not include migration of the request methode from 'supported_languages' to 'traits' (EngineTraits) object! Signed-off-by: Markus Heiser --- docs/src/searx.engines.startpage.rst | 18 +++ searx/data/engine_traits.json | 136 +++++++++++++++++++- searx/engines/startpage.py | 183 ++++++++++++++++++++++++++- 3 files changed, 328 insertions(+), 9 deletions(-) create mode 100644 docs/src/searx.engines.startpage.rst diff --git a/docs/src/searx.engines.startpage.rst b/docs/src/searx.engines.startpage.rst new file mode 100644 index 000000000..23ec63837 --- /dev/null +++ b/docs/src/searx.engines.startpage.rst @@ -0,0 +1,18 @@ +.. _startpage engines: + +================= +Startpage engines +================= + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.startpage + +Functions +========= + +.. autofunction:: searx.engines.startpage.fetch_traits +.. autofunction:: searx.engines.startpage.get_sc_code diff --git a/searx/data/engine_traits.json b/searx/data/engine_traits.json index b4fdd8b28..db7c428aa 100644 --- a/searx/data/engine_traits.json +++ b/searx/data/engine_traits.json @@ -3236,8 +3236,140 @@ "all_locale": null, "custom": {}, "data_type": "supported_languages", - "languages": {}, - "regions": {}, + "languages": { + "af": "afrikaans", + "am": "amharic", + "ar": "arabic", + "az": "azerbaijani", + "be": "belarusian", + "bg": "bulgarian", + "bn": "bengali", + "bs": "bosnian", + "ca": "catalan", + "cs": "czech", + "cy": "welsh", + "da": "dansk", + "de": "deutsch", + "el": "greek", + "en": "english", + "eo": "esperanto", + "es": "espanol", + "et": "estonian", + "eu": "basque", + "fa": "persian", + "fi": "suomi", + "fo": "faroese", + "fr": "francais", + "fy": "frisian", + "ga": "irish", + "gd": "gaelic", + "gl": "galician", + "gu": "gujarati", + "he": "hebrew", + "hi": "hindi", + "hr": "croatian", + "hu": "hungarian", + "ia": "interlingua", + "id": "indonesian", + "is": "icelandic", + "it": "italiano", + "ja": "nihongo", + "jv": "javanese", + "ka": "georgian", + "kn": "kannada", + "ko": "hangul", + "la": "latin", + "lt": "lithuanian", + "lv": "latvian", + "mai": "bihari", + "mk": "macedonian", + "ml": "malayalam", + "mr": "marathi", + "ms": "malay", + "mt": "maltese", + "nb": "norsk", + "ne": "nepali", + "nl": "nederlands", + "oc": "occitan", + "pa": "punjabi", + "pl": "polski", + "pt": "portugues", + "ro": "romanian", + "ru": "russian", + "si": "sinhalese", + "sk": "slovak", + "sl": "slovenian", + "sq": "albanian", + "sr": "serbian", + "su": "sudanese", + "sv": "svenska", + "sw": "swahili", + "ta": "tamil", + "te": "telugu", + "th": "thai", + "ti": "tigrinya", + "tl": "tagalog", + "tr": "turkce", + "uk": "ukrainian", + "ur": "urdu", + "uz": "uzbek", + "vi": "vietnamese", + "xh": "xhosa", + "zh": "jiantizhongwen", + "zh_Hant": "fantizhengwen", + "zu": "zulu" + }, + "regions": { + "ar-EG": "ar_EG", + "bg-BG": "bg_BG", + "ca-ES": "ca_ES", + "cs-CZ": "cs_CZ", + "da-DK": "da_DK", + "de-AT": "de_AT", + "de-CH": "de_CH", + "de-DE": "de_DE", + "el-GR": "el_GR", + "en-AU": "en_AU", + "en-CA": "en_CA", + "en-GB": "en-GB_GB", + "en-IE": "en_IE", + "en-MY": "en_MY", + "en-NZ": "en_NZ", + "en-US": "en_US", + "en-ZA": "en_ZA", + "es-AR": "es_AR", + "es-CL": "es_CL", + "es-ES": "es_ES", + "es-US": "es_US", + "es-UY": "es_UY", + "fi-FI": "fi_FI", + "fil-PH": "fil_PH", + "fr-BE": "fr_BE", + "fr-CA": "fr_CA", + "fr-CH": "fr_CH", + "fr-FR": "fr_FR", + "hi-IN": "hi_IN", + "it-CH": "it_CH", + "it-IT": "it_IT", + "ja-JP": "ja_JP", + "ko-KR": "ko_KR", + "ms-MY": "ms_MY", + "nb-NO": "no_NO", + "nl-BE": "nl_BE", + "nl-NL": "nl_NL", + "pl-PL": "pl_PL", + "pt-BR": "pt-BR_BR", + "pt-PT": "pt_PT", + "ro-RO": "ro_RO", + "ru-BY": "ru_BY", + "ru-RU": "ru_RU", + "sv-SE": "sv_SE", + "tr-TR": "tr_TR", + "uk-UA": "uk_UA", + "zh-CN": "zh-CN_CN", + "zh-HK": "zh-TW_HK", + "zh-TW": "zh-TW_TW" + }, "supported_languages": { "af": { "alias": "afrikaans" diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index f857f7b6d..75435c3fa 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -1,6 +1,63 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # lint: pylint -"""Startpage (Web) +"""Startpage's language & region selectors are a mess .. + +.. _startpage regions: + +Startpage regions +================= + +In the list of regions there are tags we need to map to common region tags:: + + pt-BR_BR --> pt_BR + zh-CN_CN --> zh_Hans_CN + zh-TW_TW --> zh_Hant_TW + zh-TW_HK --> zh_Hant_HK + en-GB_GB --> en_GB + +and there is at least one tag with a three letter language tag (ISO 639-2):: + + fil_PH --> fil_PH + +The locale code ``no_NO`` from Startpage does not exists and is mapped to +``nb-NO``:: + + babel.core.UnknownLocaleError: unknown locale 'no_NO' + +For reference see languages-subtag at iana; ``no`` is the macrolanguage [1]_ and +W3C recommends subtag over macrolanguage [2]_. + +.. [1] `iana: language-subtag-registry + `_ :: + + type: language + Subtag: nb + Description: Norwegian Bokmål + Added: 2005-10-16 + Suppress-Script: Latn + Macrolanguage: no + +.. [2] + Use macrolanguages with care. Some language subtags have a Scope field set to + macrolanguage, i.e. this primary language subtag encompasses a number of more + specific primary language subtags in the registry. ... As we recommended for + the collection subtags mentioned above, in most cases you should try to use + the more specific subtags ... `W3: The primary language subtag + `_ + +.. _startpage languages: + +Startpage languages +=================== + +The displayed name in Startpage's settings page depend on the location of the IP +when the 'Accept-Language' HTTP header is unset (in the language update script +we use "en-US,en;q=0.5" to get uniform names independent from the IP). + +Each option has a displayed name and a value, either of which may represent the +language name in the native script, the language name in English, an English +transliteration of the native name, the English name of the writing script used +by the language, or occasionally something else entirely. """ @@ -16,13 +73,16 @@ from lxml import html from babel import Locale from babel.localedata import locale_identifiers -from searx.network import get +from searx import network from searx.utils import extract_text, eval_xpath, match_language from searx.exceptions import ( SearxEngineResponseException, SearxEngineCaptchaException, ) +from searx.enginelib.traits import EngineTraits + +traits: EngineTraits # about about = { @@ -66,11 +126,11 @@ def raise_captcha(resp): def get_sc_code(headers): - """Get an actual `sc` argument from startpage's home page. + """Get an actual ``sc`` argument from Startpage's home page. - Startpage puts a `sc` argument on every link. Without this argument - startpage considers the request is from a bot. We do not know what is - encoded in the value of the `sc` argument, but it seems to be a kind of a + Startpage puts a ``sc`` argument on every link. Without this argument + Startpage considers the request is from a bot. We do not know what is + encoded in the value of the ``sc`` argument, but it seems to be a kind of a *time-stamp*. This *time-stamp* is valid for a few hours. This function scrap a new *time-stamp* from startpage's home page every hour @@ -83,7 +143,7 @@ def get_sc_code(headers): if time() > (sc_code_ts + 3000): logger.debug("query new sc time-stamp ...") - resp = get(base_url, headers=headers) + resp = network.get(base_url, headers=headers) raise_captcha(resp) dom = html.fromstring(resp.text) @@ -258,3 +318,112 @@ def _fetch_supported_languages(resp): print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text)) return supported_languages + + +def fetch_traits(engine_traits: EngineTraits): + """Fetch :ref:`languages ` and :ref:`regions ` from Startpage.""" + # pylint: disable=import-outside-toplevel, too-many-locals, too-many-branches + # pylint: disable=too-many-statements + + engine_traits.data_type = 'supported_languages' # deprecated + + import babel + from searx.utils import gen_useragent + from searx.locales import region_tag + + headers = { + 'User-Agent': gen_useragent(), + 'Accept-Language': "en-US,en;q=0.5", # bing needs to set the English language + } + resp = network.get('https://www.startpage.com/do/settings', headers=headers) + + if not resp.ok: + print("ERROR: response from Startpage is not OK.") + + dom = html.fromstring(resp.text) + + # regions + + sp_region_names = [] + for option in dom.xpath('//form[@name="settings"]//select[@name="search_results_region"]/option'): + sp_region_names.append(option.get('value')) + + for eng_tag in sp_region_names: + if eng_tag == 'all': + continue + babel_region_tag = {'no_NO': 'nb_NO'}.get(eng_tag, eng_tag) # norway + + if '-' in babel_region_tag: + l, r = babel_region_tag.split('-') + r = r.split('_')[-1] + sxng_tag = region_tag(babel.Locale.parse(l + '_' + r, sep='_')) + + else: + try: + sxng_tag = region_tag(babel.Locale.parse(babel_region_tag, sep='_')) + + except babel.UnknownLocaleError: + print("ERROR: can't determine babel locale of startpage's locale %s" % eng_tag) + continue + + conflict = engine_traits.regions.get(sxng_tag) + if conflict: + if conflict != eng_tag: + print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag)) + continue + engine_traits.regions[sxng_tag] = eng_tag + + # languages + + catalog_engine2code = {name.lower(): lang_code for lang_code, name in babel.Locale('en').languages.items()} + + # get the native name of every language known by babel + + for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, babel.localedata.locale_identifiers()): + native_name = babel.Locale(lang_code).get_language_name().lower() + # add native name exactly as it is + catalog_engine2code[native_name] = lang_code + + # add "normalized" language name (i.e. français becomes francais and español becomes espanol) + unaccented_name = ''.join(filter(lambda c: not combining(c), normalize('NFKD', native_name))) + if len(unaccented_name) == len(unaccented_name.encode()): + # add only if result is ascii (otherwise "normalization" didn't work) + catalog_engine2code[unaccented_name] = lang_code + + # values that can't be determined by babel's languages names + + catalog_engine2code.update( + { + # traditional chinese used in .. + 'fantizhengwen': 'zh_Hant', + # Korean alphabet + 'hangul': 'ko', + # Malayalam is one of 22 scheduled languages of India. + 'malayam': 'ml', + 'norsk': 'nb', + 'sinhalese': 'si', + } + ) + + skip_eng_tags = { + 'english_uk', # SearXNG lang 'en' already maps to 'english' + } + + for option in dom.xpath('//form[@name="settings"]//select[@name="language"]/option'): + + eng_tag = option.get('value') + if eng_tag in skip_eng_tags: + continue + name = extract_text(option).lower() + + sxng_tag = catalog_engine2code.get(eng_tag) + if sxng_tag is None: + sxng_tag = catalog_engine2code[name] + + conflict = engine_traits.languages.get(sxng_tag) + if conflict: + if conflict != eng_tag: + print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag)) + continue + engine_traits.languages[sxng_tag] = eng_tag