Merge pull request #1652 from return42/mod-qwant

Improve qwant engine
This commit is contained in:
Markus Heiser 2022-08-14 10:58:52 +02:00 committed by GitHub
commit 2bd3c2079e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 359 additions and 204 deletions

View file

@ -1396,170 +1396,155 @@
"sv",
"zh"
],
"qwant": [
"bg-BG",
"ca-ES",
"cs-CZ",
"da-DK",
"de-AT",
"de-CH",
"de-DE",
"el-GR",
"en-AU",
"en-CA",
"en-GB",
"en-IE",
"en-MY",
"en-NZ",
"en-US",
"es-AR",
"es-CL",
"es-ES",
"es-MX",
"et-EE",
"fi-FI",
"fr-BE",
"fr-CA",
"fr-CH",
"fr-FR",
"hu-HU",
"it-CH",
"it-IT",
"ko-KR",
"nb-NO",
"nl-BE",
"nl-NL",
"pl-PL",
"pt-PT",
"ro-RO",
"sv-SE",
"th-TH",
"zh-CN",
"zh-HK"
],
"qwant images": [
"bg-BG",
"ca-ES",
"cs-CZ",
"da-DK",
"de-AT",
"de-CH",
"de-DE",
"el-GR",
"en-AU",
"en-CA",
"en-GB",
"en-IE",
"en-MY",
"en-NZ",
"en-US",
"es-AR",
"es-CL",
"es-ES",
"es-MX",
"et-EE",
"fi-FI",
"fr-BE",
"fr-CA",
"fr-CH",
"fr-FR",
"hu-HU",
"it-CH",
"it-IT",
"ko-KR",
"nb-NO",
"nl-BE",
"nl-NL",
"pl-PL",
"pt-PT",
"ro-RO",
"sv-SE",
"th-TH",
"zh-CN",
"zh-HK"
],
"qwant news": [
"bg-BG",
"ca-ES",
"cs-CZ",
"da-DK",
"de-AT",
"de-CH",
"de-DE",
"el-GR",
"en-AU",
"en-CA",
"en-GB",
"en-IE",
"en-MY",
"en-NZ",
"en-US",
"es-AR",
"es-CL",
"es-ES",
"es-MX",
"et-EE",
"fi-FI",
"fr-BE",
"fr-CA",
"fr-CH",
"fr-FR",
"hu-HU",
"it-CH",
"it-IT",
"ko-KR",
"nb-NO",
"nl-BE",
"nl-NL",
"pl-PL",
"pt-PT",
"ro-RO",
"sv-SE",
"th-TH",
"zh-CN",
"zh-HK"
],
"qwant videos": [
"bg-BG",
"ca-ES",
"cs-CZ",
"da-DK",
"de-AT",
"de-CH",
"de-DE",
"el-GR",
"en-AU",
"en-CA",
"en-GB",
"en-IE",
"en-MY",
"en-NZ",
"en-US",
"es-AR",
"es-CL",
"es-ES",
"es-MX",
"et-EE",
"fi-FI",
"fr-BE",
"fr-CA",
"fr-CH",
"fr-FR",
"hu-HU",
"it-CH",
"it-IT",
"ko-KR",
"nb-NO",
"nl-BE",
"nl-NL",
"pl-PL",
"pt-PT",
"ro-RO",
"sv-SE",
"th-TH",
"zh-CN",
"zh-HK"
],
"qwant": {
"bg-BG": "bg_BG",
"ca-ES": "ca_ES",
"cs-CZ": "cs_CZ",
"da-DK": "da_DK",
"de-AT": "de_AT",
"de-CH": "de_CH",
"de-DE": "de_DE",
"el-GR": "el_GR",
"en-AU": "en_AU",
"en-CA": "en_CA",
"en-GB": "en_GB",
"en-IE": "en_IE",
"en-MY": "en_MY",
"en-NZ": "en_NZ",
"en-US": "en_US",
"es-AR": "es_AR",
"es-CL": "es_CL",
"es-ES": "es_ES",
"es-MX": "es_MX",
"et-EE": "et_EE",
"fi-FI": "fi_FI",
"fr-BE": "fr_BE",
"fr-CA": "fr_CA",
"fr-CH": "fr_CH",
"fr-FR": "fr_FR",
"hu-HU": "hu_HU",
"it-CH": "it_CH",
"it-IT": "it_IT",
"ko-KR": "ko_KR",
"nb-NO": "nb_NO",
"nl-BE": "nl_BE",
"nl-NL": "nl_NL",
"pl-PL": "pl_PL",
"pt-PT": "pt_PT",
"ro-RO": "ro_RO",
"sv-SE": "sv_SE",
"th-TH": "th_TH",
"zh-CN": "zh_CN",
"zh-HK": "zh_HK"
},
"qwant images": {
"bg-BG": "bg_BG",
"ca-ES": "ca_ES",
"cs-CZ": "cs_CZ",
"da-DK": "da_DK",
"de-AT": "de_AT",
"de-CH": "de_CH",
"de-DE": "de_DE",
"el-GR": "el_GR",
"en-AU": "en_AU",
"en-CA": "en_CA",
"en-GB": "en_GB",
"en-IE": "en_IE",
"en-MY": "en_MY",
"en-NZ": "en_NZ",
"en-US": "en_US",
"es-AR": "es_AR",
"es-CL": "es_CL",
"es-ES": "es_ES",
"es-MX": "es_MX",
"et-EE": "et_EE",
"fi-FI": "fi_FI",
"fr-BE": "fr_BE",
"fr-CA": "fr_CA",
"fr-CH": "fr_CH",
"fr-FR": "fr_FR",
"hu-HU": "hu_HU",
"it-CH": "it_CH",
"it-IT": "it_IT",
"ko-KR": "ko_KR",
"nb-NO": "nb_NO",
"nl-BE": "nl_BE",
"nl-NL": "nl_NL",
"pl-PL": "pl_PL",
"pt-PT": "pt_PT",
"ro-RO": "ro_RO",
"sv-SE": "sv_SE",
"th-TH": "th_TH",
"zh-CN": "zh_CN",
"zh-HK": "zh_HK"
},
"qwant news": {
"ca-ES": "ca_ES",
"de-AT": "de_AT",
"de-CH": "de_CH",
"de-DE": "de_DE",
"en-AU": "en_AU",
"en-CA": "en_CA",
"en-GB": "en_GB",
"en-IE": "en_IE",
"en-MY": "en_MY",
"en-NZ": "en_NZ",
"en-US": "en_US",
"es-AR": "es_AR",
"es-CL": "es_CL",
"es-ES": "es_ES",
"es-MX": "es_MX",
"fr-BE": "fr_BE",
"fr-CA": "fr_CA",
"fr-CH": "fr_CH",
"fr-FR": "fr_FR",
"it-CH": "it_CH",
"it-IT": "it_IT",
"nl-BE": "nl_BE",
"nl-NL": "nl_NL",
"pt-PT": "pt_PT"
},
"qwant videos": {
"bg-BG": "bg_BG",
"ca-ES": "ca_ES",
"cs-CZ": "cs_CZ",
"da-DK": "da_DK",
"de-AT": "de_AT",
"de-CH": "de_CH",
"de-DE": "de_DE",
"el-GR": "el_GR",
"en-AU": "en_AU",
"en-CA": "en_CA",
"en-GB": "en_GB",
"en-IE": "en_IE",
"en-MY": "en_MY",
"en-NZ": "en_NZ",
"en-US": "en_US",
"es-AR": "es_AR",
"es-CL": "es_CL",
"es-ES": "es_ES",
"es-MX": "es_MX",
"et-EE": "et_EE",
"fi-FI": "fi_FI",
"fr-BE": "fr_BE",
"fr-CA": "fr_CA",
"fr-CH": "fr_CH",
"fr-FR": "fr_FR",
"hu-HU": "hu_HU",
"it-CH": "it_CH",
"it-IT": "it_IT",
"ko-KR": "ko_KR",
"nb-NO": "nb_NO",
"nl-BE": "nl_BE",
"nl-NL": "nl_NL",
"pl-PL": "pl_PL",
"pt-PT": "pt_PT",
"ro-RO": "ro_RO",
"sv-SE": "sv_SE",
"th-TH": "th_TH",
"zh-CN": "zh_CN",
"zh-HK": "zh_HK"
},
"startpage": {
"af": {
"alias": "afrikaans"

View file

@ -9,16 +9,16 @@ https://www.qwant.com/ queries.
This implementation is used by different qwant engines in the settings.yml::
- name: qwant
categories: general
qwant_categ: web
...
- name: qwant news
categories: news
qwant_categ: news
...
- name: qwant images
categories: images
qwant_categ: images
...
- name: qwant videos
categories: videos
qwant_categ: videos
...
"""
@ -30,11 +30,11 @@ from datetime import (
from json import loads
from urllib.parse import urlencode
from flask_babel import gettext
import babel
from searx.utils import match_language
from searx.exceptions import SearxEngineAPIException
from searx.network import raise_for_httperror
from searx.locales import get_engine_locale
# about
about = {
@ -50,13 +50,20 @@ about = {
categories = []
paging = True
supported_languages_url = about['website']
qwant_categ = None # web|news|inages|videos
category_to_keyword = {
'general': 'web',
'news': 'news',
'images': 'images',
'videos': 'videos',
}
safesearch = True
safe_search_map = {0: '&safesearch=0', 1: '&safesearch=1', 2: '&safesearch=2'}
# fmt: off
qwant_news_locales = [
'ca_ad', 'ca_es', 'ca_fr', 'co_fr', 'de_at', 'de_ch', 'de_de', 'en_au',
'en_ca', 'en_gb', 'en_ie', 'en_my', 'en_nz', 'en_us', 'es_ad', 'es_ar',
'es_cl', 'es_co', 'es_es', 'es_mx', 'es_pe', 'eu_es', 'eu_fr', 'fc_ca',
'fr_ad', 'fr_be', 'fr_ca', 'fr_ch', 'fr_fr', 'it_ch', 'it_it', 'nl_be',
'nl_nl', 'pt_ad', 'pt_pt',
]
# fmt: on
# search-url
url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}'
@ -64,10 +71,13 @@ url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={o
def request(query, params):
"""Qwant search request"""
keyword = category_to_keyword[categories[0]]
if not query:
return None
count = 10 # web: count must be equal to 10
if keyword == 'images':
if qwant_categ == 'images':
count = 50
offset = (params['pageno'] - 1) * count
# count + offset must be lower than 250
@ -78,22 +88,18 @@ def request(query, params):
offset = min(offset, 40)
params['url'] = url.format(
keyword=keyword,
keyword=qwant_categ,
query=urlencode({'q': query}),
offset=offset,
count=count,
)
# add language tag
if params['language'] == 'all':
params['url'] += '&locale=en_US'
else:
language = match_language(
params['language'],
supported_languages,
language_aliases,
)
params['url'] += '&locale=' + language.replace('-', '_')
# add quant's locale
q_locale = get_engine_locale(params['language'], supported_languages, default='en_US')
params['url'] += '&locale=' + q_locale
# add safesearch option
params['url'] += safe_search_map.get(params['safesearch'], '')
params['raise_for_httperror'] = False
return params
@ -103,7 +109,6 @@ def response(resp):
"""Get response from Qwant's search request"""
# pylint: disable=too-many-locals, too-many-branches, too-many-statements
keyword = category_to_keyword[categories[0]]
results = []
# load JSON result
@ -125,7 +130,7 @@ def response(resp):
# raise for other errors
raise_for_httperror(resp)
if keyword == 'web':
if qwant_categ == 'web':
# The WEB query contains a list named 'mainline'. This list can contain
# different result types (e.g. mainline[0]['type'] returns type of the
# result items in mainline[0]['items']
@ -136,7 +141,7 @@ def response(resp):
# result['items'].
mainline = data.get('result', {}).get('items', [])
mainline = [
{'type': keyword, 'items': mainline},
{'type': qwant_categ, 'items': mainline},
]
# return empty array if there are no results
@ -146,7 +151,7 @@ def response(resp):
for row in mainline:
mainline_type = row.get('type', 'web')
if mainline_type != keyword:
if mainline_type != qwant_categ:
continue
if mainline_type == 'ads':
@ -238,19 +243,43 @@ def response(resp):
return results
# get supported languages from their site
def _fetch_supported_languages(resp):
# list of regions is embedded in page as a js object
response_text = resp.text
response_text = response_text[response_text.find('INITIAL_PROPS') :]
response_text = response_text[response_text.find('{') : response_text.find('</script>')]
regions_json = loads(response_text)
text = resp.text
text = text[text.find('INITIAL_PROPS') :]
text = text[text.find('{') : text.find('</script>')]
supported_languages = []
for country, langs in regions_json['locales'].items():
for lang in langs['langs']:
lang_code = "{lang}-{country}".format(lang=lang, country=country)
supported_languages.append(lang_code)
q_initial_props = loads(text)
q_locales = q_initial_props.get('locales')
q_valid_locales = []
for country, v in q_locales.items():
for lang in v['langs']:
_locale = "{lang}_{country}".format(lang=lang, country=country)
if qwant_categ == 'news' and _locale.lower() not in qwant_news_locales:
# qwant-news does not support all locales from qwant-web:
continue
q_valid_locales.append(_locale)
supported_languages = {}
for q_locale in q_valid_locales:
try:
locale = babel.Locale.parse(q_locale, sep='_')
except babel.core.UnknownLocaleError:
print("ERROR: can't determine babel locale of quant's locale %s" % q_locale)
continue
# note: supported_languages (dict)
#
# dict's key is a string build up from a babel.Locale object / the
# notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and
# language) notation and dict's values are the locale strings used by
# the engine.
searxng_locale = locale.language + '-' + locale.territory # --> params['language']
supported_languages[searxng_locale] = q_locale
return supported_languages

View file

@ -10,6 +10,8 @@ import pathlib
from babel import Locale
from babel.support import Translations
import babel.languages
import babel.core
import flask_babel
import flask
from flask.ctx import has_request_context
@ -150,3 +152,138 @@ def locales_initialize(directory=None):
LOCALE_NAMES[tag] = get_locale_descr(locale, dirname)
if locale.text_direction == 'rtl':
RTL_LOCALES.add(tag)
def get_engine_locale(searxng_locale, engine_locales, default=None):
"""Return engine's language (aka locale) string that best fits to argument
``searxng_locale``.
Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to
corresponding *engine locales*:
<engine>: {
# SearXNG string : engine-string
'ca-ES' : 'ca_ES',
'fr-BE' : 'fr_BE',
'fr-CA' : 'fr_CA',
'fr-CH' : 'fr_CH',
'fr' : 'fr_FR',
...
'pl-PL' : 'pl_PL',
'pt-PT' : 'pt_PT'
}
.. hint::
The *SearXNG locale* string has to be known by babel!
If there is no direct 1:1 mapping, this functions tries to narrow down
engine's language (locale). If no value can be determined by these
approximation attempts the ``default`` value is returned.
Assumptions:
A. When user select a language the results should be optimized according to
the selected language.
B. When user select a language and a territory the results should be
optimized with first priority on terrirtory and second on language.
First approximation rule (*by territory*):
When the user selects a locale with terrirtory (and a language), the
territory has priority over the language. If any of the offical languages
in the terrirtory is supported by the engine (``engine_locales``) it will
be used.
Second approximation rule (*by language*):
If "First approximation rule" brings no result or the user selects only a
language without a terrirtory. Check in which territories the language
has an offical status and if one of these territories is supported by the
engine.
"""
# pylint: disable=too-many-branches
engine_locale = engine_locales.get(searxng_locale)
if engine_locale is not None:
# There was a 1:1 mapping (e.g. "fr-BE --> fr_BE" or "fr --> fr_FR"), no
# need to narrow language nor territory.
return engine_locale
locale = babel.Locale.parse(searxng_locale, sep='-')
# SearXNG's selected locale is not supported by the engine ..
if locale.territory:
# Try to narrow by *offical* languages in the territory (??-XX).
for official_language in babel.languages.get_official_languages(locale.territory, de_facto=True):
searxng_locale = official_language + '-' + locale.territory
engine_locale = engine_locales.get(searxng_locale)
if engine_locale is not None:
return engine_locale
# Engine does not support one of the offical languages in the territory or
# there is only a language selected without a territory.
# Now lets have a look if the searxng_lang (the language selected by the
# user) is a offical language in other territories. If so, check if
# engine does support the searxng_lang in this other territory.
if locale.language:
searxng_lang = locale.language
if locale.script:
searxng_lang += '_' + locale.script
terr_lang_dict = {}
for territory, langs in babel.core.get_global("territory_languages").items():
if not langs.get(searxng_lang, {}).get('official_status'):
continue
terr_lang_dict[territory] = langs.get(searxng_lang)
# first: check fr-FR, de-DE .. is supported by the engine
territory = locale.language.upper()
if terr_lang_dict.get(territory):
searxng_locale = locale.language + '-' + territory
engine_locale = engine_locales.get(searxng_locale)
if engine_locale is not None:
return engine_locale
# second: sort by population_percent and take first match
# drawback of "population percent": if there is a terrirtory with a
# small number of people (e.g 100) but the majority speaks the
# language, then the percentage migth be 100% (--> 100 people) but in
# a different terrirtory with more people (e.g. 10.000) where only 10%
# speak the language the total amount of speaker is higher (--> 200
# people).
#
# By example: The population of Saint-Martin is 33.000, of which 100%
# speak French, but this is less than the 30% of the approximately 2.5
# million Belgian citizens
#
# - 'fr-MF', 'population_percent': 100.0, 'official_status': 'official'
# - 'fr-BE', 'population_percent': 38.0, 'official_status': 'official'
terr_lang_list = []
for k, v in terr_lang_dict.items():
terr_lang_list.append((k, v))
for territory, _lang in sorted(terr_lang_list, key=lambda item: item[1]['population_percent'], reverse=True):
searxng_locale = locale.language + '-' + territory
engine_locale = engine_locales.get(searxng_locale)
if engine_locale is not None:
return engine_locale
# No luck: narrow by "language from territory" and "territory from language"
# does not fit to a locale supported by the engine.
if engine_locale is None:
engine_locale = default
return default

View file

@ -1198,6 +1198,7 @@ engines:
results: HTML
- name: qwant
qwant_categ: web
engine: qwant
shortcut: qw
categories: [general, web]
@ -1206,6 +1207,7 @@ engines:
rosebud: *test_rosebud
- name: qwant news
qwant_categ: news
engine: qwant
shortcut: qwn
categories: news
@ -1213,6 +1215,7 @@ engines:
network: qwant
- name: qwant images
qwant_categ: images
engine: qwant
shortcut: qwi
categories: [images, web]
@ -1220,6 +1223,7 @@ engines:
network: qwant
- name: qwant videos
qwant_categ: videos
engine: qwant
shortcut: qwv
categories: [videos, web]