mirrors/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-26 21:01:01 +00:00
Code Issues Projects Releases Wiki Activity

[mod] bing: add time_range support & upgrade to data_type: traits_v1

Browse source 
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2022-10-15 21:17:39 +02:00
parent c9cd376186
commit d0f465e6fa
6 changed files with 366 additions and 1968 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1966
searx/data/engine_traits.json
View file

File diff suppressed because it is too large Load diff

280
searx/engines/bing.py
View file

@ -4,16 +4,28 @@
- https://github.com/searx/searx/issues/2019#issuecomment-648227442
"""
# pylint: disable=too-many-branches
# pylint: disable=too-many-branches, invalid-name
from typing import TYPE_CHECKING
import datetime
import re
from urllib.parse import urlencode, urlparse, parse_qs
import uuid
from urllib.parse import urlencode
from lxml import html
from searx.utils import eval_xpath, extract_text, eval_xpath_list, match_language, eval_xpath_getindex
from searx.network import multi_requests, Request
import babel
import babel.languages
from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_getindex
from searx import network
from searx.locales import language_tag, region_tag
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
@ -25,56 +37,124 @@ about = {
"results": 'HTML',
}
send_accept_language_header = True
"""Bing tries to guess user's language and territory from the HTTP
Accept-Language. Optional the user can select a search-language (can be
different to the UI language) and a region (market code)."""
# engine dependent config
categories = ['general', 'web']
paging = True
time_range_support = False
safesearch = False
send_accept_language_header = True
supported_languages_url = 'https://www.bing.com/account/general'
language_aliases = {}
time_range_support = True
safesearch = True
safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'} # cookie: ADLT=STRICT
# search-url
base_url = 'https://www.bing.com/'
base_url = 'https://www.bing.com/search'
"""Bing (Web) search URL"""
# initial query: https://www.bing.com/search?q=foo&search=&form=QBLH
inital_query = 'search?{query}&search=&form=QBLH'
# following queries: https://www.bing.com/search?q=foo&search=&first=11&FORM=PERE
page_query = 'search?{query}&search=&first={offset}&FORM=PERE'
bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/market-codes'
"""Bing (Web) search API description"""
def _get_offset_from_pageno(pageno):
return (pageno - 1) * 10 + 1
def set_bing_cookies(params, engine_language, engine_region, SID):
# set cookies
# -----------
params['cookies']['_EDGE_V'] = '1'
# _EDGE_S: F=1&SID=3A5253BD6BCA609509B741876AF961CA&mkt=zh-tw
_EDGE_S = [
'F=1',
'SID=%s' % SID,
'mkt=%s' % engine_region.lower(),
'ui=%s' % engine_language.lower(),
]
params['cookies']['_EDGE_S'] = '&'.join(_EDGE_S)
logger.debug("cookie _EDGE_S=%s", params['cookies']['_EDGE_S'])
# "_EDGE_CD": "m=zh-tw",
_EDGE_CD = [ # pylint: disable=invalid-name
'm=%s' % engine_region.lower(), # search region: zh-cn
'u=%s' % engine_language.lower(), # UI: en-us
]
params['cookies']['_EDGE_CD'] = '&'.join(_EDGE_CD) + ';'
logger.debug("cookie _EDGE_CD=%s", params['cookies']['_EDGE_CD'])
SRCHHPGUSR = [ # pylint: disable=invalid-name
'SRCHLANG=%s' % engine_language,
# Trying to set ADLT cookie here seems not to have any effect, I assume
# there is some age verification by a cookie (and/or session ID) needed,
# to disable the SafeSearch.
'ADLT=%s' % safesearch_types.get(params['safesearch'], 'DEMOTE'),
]
params['cookies']['SRCHHPGUSR'] = '&'.join(SRCHHPGUSR)
logger.debug("cookie SRCHHPGUSR=%s", params['cookies']['SRCHHPGUSR'])
def request(query, params):
"""Assemble a Bing-Web request."""
offset = _get_offset_from_pageno(params.get('pageno', 1))
engine_region = traits.get_region(params['searxng_locale'], 'en-US')
engine_language = traits.get_language(params['searxng_locale'], 'en')
# logger.debug("params['pageno'] --> %s", params.get('pageno'))
# logger.debug(" offset --> %s", offset)
SID = uuid.uuid1().hex.upper()
CVID = uuid.uuid1().hex.upper()
search_string = page_query
if offset == 1:
search_string = inital_query
set_bing_cookies(params, engine_language, engine_region, SID)
if params['language'] == 'all':
lang = 'EN'
else:
lang = match_language(params['language'], supported_languages, language_aliases)
# build URL query
# ---------------
query = 'language:{} {}'.format(lang.split('-')[0].upper(), query)
# query term
page = int(params.get('pageno', 1))
query_params = {
# fmt: off
'q': query,
'pq': query,
'cvid': CVID,
'qs': 'n',
'sp': '-1'
# fmt: on
}
search_path = search_string.format(query=urlencode({'q': query}), offset=offset)
if offset > 1:
referer = base_url + inital_query.format(query=urlencode({'q': query}))
# page
if page > 1:
referer = base_url + '?' + urlencode(query_params)
params['headers']['Referer'] = referer
logger.debug("headers.Referer --> %s", referer)
params['url'] = base_url + search_path
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
query_params['first'] = _get_offset_from_pageno(page)
if page == 2:
query_params['FORM'] = 'PERE'
elif page > 2:
query_params['FORM'] = 'PERE%s' % (page - 2)
filters = ''
if params['time_range']:
query_params['filt'] = 'custom'
if params['time_range'] == 'day':
filters = 'ex1:"ez1"'
elif params['time_range'] == 'week':
filters = 'ex1:"ez2"'
elif params['time_range'] == 'month':
filters = 'ex1:"ez3"'
elif params['time_range'] == 'year':
epoch_1970 = datetime.date(1970, 1, 1)
today_no = (datetime.date.today() - epoch_1970).days
filters = 'ex1:"ez5_%s_%s"' % (today_no - 365, today_no)
params['url'] = base_url + '?' + urlencode(query_params)
if filters:
params['url'] = params['url'] + '&filters=' + filters
return params
@ -111,7 +191,8 @@ def response(resp):
url_cite = extract_text(eval_xpath(result, './/div[@class="b_attribution"]/cite'))
# Bing can shorten the URL either at the end or in the middle of the string
if (
url_cite.startswith('https://')
url_cite
and url_cite.startswith('https://')
and '…' not in url_cite
and '...' not in url_cite
and '›' not in url_cite
@ -131,9 +212,9 @@ def response(resp):
# resolve all Bing redirections in parallel
request_list = [
Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve
network.Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve
]
response_list = multi_requests(request_list)
response_list = network.multi_requests(request_list)
for i, redirect_response in enumerate(response_list):
if not isinstance(redirect_response, Exception):
results[url_to_resolve_index[i]]['url'] = redirect_response.headers['location']
@ -161,94 +242,43 @@ def response(resp):
return results
# get supported languages from their site
def _fetch_supported_languages(resp):
lang_tags = set()
dom = html.fromstring(resp.text)
lang_links = eval_xpath(dom, '//div[@id="language-section"]//li')
for _li in lang_links:
href = eval_xpath(_li, './/@href')[0]
(_scheme, _netloc, _path, _params, query, _fragment) = urlparse(href)
query = parse_qs(query, keep_blank_values=True)
# fmt: off
setlang = query.get('setlang', [None, ])[0]
# example: 'mn-Cyrl-MN' --> '['mn', 'Cyrl-MN']
lang, nation = (setlang.split('-', maxsplit=1) + [None,])[:2] # fmt: skip
# fmt: on
tag = lang + '-' + nation if nation else lang
lang_tags.add(tag)
return list(lang_tags)
def fetch_traits(engine_traits: EngineTraits):
"""Fetch languages and regions from bing."""
"""Fetch languages and regions from Bing-Web."""
# pylint: disable=import-outside-toplevel, disable=too-many-branches,
# pylint: disable=too-many-locals, too-many-statements
xpath_market_codes = '//table[1]/tbody/tr/td[3]'
# xpath_country_codes = '//table[2]/tbody/tr/td[2]'
xpath_language_codes = '//table[3]/tbody/tr/td[2]'
engine_traits.data_type = 'supported_languages' # deprecated
_fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes)
import babel
import babel.languages
from searx import network
from searx.locales import get_offical_locales, language_tag, region_tag
from searx.utils import gen_useragent
headers = {
'User-Agent': gen_useragent(),
'Accept-Language': "en-US,en;q=0.5", # bing needs to set the English language
}
resp = network.get('https://www.bing.com/account/general', headers=headers)
def _fetch_traits(engine_traits: EngineTraits, url: str, xpath_language_codes: str, xpath_market_codes: str):
# insert alias to map from a language (zh) to a language + script (zh_Hans)
engine_traits.languages['zh'] = 'zh-hans'
resp = network.get(url)
if not resp.ok:
print("ERROR: response from peertube is not OK.")
dom = html.fromstring(resp.text)
# Selector to get items from "Display language"
map_lang = {'jp': 'ja'}
for td in eval_xpath(dom, xpath_language_codes):
eng_lang = td.text
lang_map = {
'prs': 'fa', # Persian
'pt_BR': 'pt', # Portuguese (Brasil)
'pt_PT': 'pt', # Portuguese (Portugal)
'ca-ES-VALENCIA': 'ca', # Catalan (Spain, Valencian)
}
unknow_langs = [
'quc', # K'iche'
'nso', # Sesotho sa Leboa
'tn', # Setswana
]
for div in eval_xpath(dom, '//div[@id="limit-languages"]//input/..'):
eng_lang = eval_xpath(div, './/input/@value')[0]
if eng_lang in unknow_langs:
if eng_lang in ('en-gb', 'pt-br'):
# language 'en' is already in the list and a language 'en-gb' can't
# be handled in SearXNG, same with pt-br which is covered by pt-pt.
continue
eng_lang = lang_map.get(eng_lang, eng_lang)
label = extract_text(eval_xpath(div, './/label'))
# The 'language:xx' query string in the request function (above) does
# only support the language codes from the "Display languages" list.
# Examples of items from the "Display languages" not sopported in the
# query string: zh_Hans --> zh / sr_latn --> sr
#
# eng_lang = eng_lang.split('_')[0]
babel_lang = map_lang.get(eng_lang, eng_lang).replace('-', '_')
try:
sxng_tag = language_tag(babel.Locale.parse(eng_lang.replace('-', '_'), sep='_'))
sxng_tag = language_tag(babel.Locale.parse(babel_lang))
except babel.UnknownLocaleError:
print("ERROR: %s (%s) is unknown by babel" % (label, eng_lang))
print("ERROR: language (%s) is unknown by babel" % (eng_lang))
continue
conflict = engine_traits.languages.get(sxng_tag)
if conflict:
if conflict != eng_lang:
@ -256,25 +286,27 @@ def fetch_traits(engine_traits: EngineTraits):
continue
engine_traits.languages[sxng_tag] = eng_lang
engine_traits.languages['zh'] = 'zh_Hans'
map_region = {
'en-ID': 'id_ID',
'no-NO': 'nb_NO',
}
# regiones
for td in eval_xpath(dom, xpath_market_codes):
eng_region = td.text
babel_region = map_region.get(eng_region, eng_region).replace('-', '_')
for a in eval_xpath(dom, '//div[@id="region-section-content"]//li/a'):
href = eval_xpath(a, './/@href')[0]
# lang_name = extract_text(a)
query = urlparse(href)[4]
query = parse_qs(query, keep_blank_values=True)
cc = query.get('cc')[0] # pylint:disable=invalid-name
if cc == 'clear':
if eng_region == 'en-WW':
engine_traits.all_locale = eng_region
continue
# Assert babel supports this locales
sxng_locales = get_offical_locales(cc.upper(), engine_traits.languages.keys())
if not sxng_locales:
# print("ERROR: can't map from bing country %s (%s) to a babel region." % (a.text_content().strip(), cc))
try:
sxng_tag = region_tag(babel.Locale.parse(babel_region))
except babel.UnknownLocaleError:
print("ERROR: region (%s) is unknown by babel" % (eng_region))
continue
for sxng_locale in sxng_locales:
engine_traits.regions[region_tag(sxng_locale)] = cc
conflict = engine_traits.regions.get(sxng_tag)
if conflict:
if conflict != eng_region:
print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_region))
continue
engine_traits.regions[sxng_tag] = eng_region

23
searx/engines/bing_images.py
View file

@ -10,12 +10,7 @@ from urllib.parse import urlencode
from lxml import html
from searx.utils import match_language
from searx.engines.bing import language_aliases
from searx.engines.bing import ( # pylint: disable=unused-import
_fetch_supported_languages,
fetch_traits,
supported_languages_url,
)
from searx.enginelib.traits import EngineTraits
# about
about = {
@ -106,3 +101,19 @@ def response(resp):
)
return results
def fetch_traits(engine_traits: EngineTraits):
"""Fetch languages and regions from Bing-News."""
# pylint: disable=import-outside-toplevel
from searx.engines.bing import _fetch_traits
url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-news-search/reference/market-codes'
xpath_market_codes = '//table[1]/tbody/tr/td[3]'
# xpath_country_codes = '//table[2]/tbody/tr/td[2]'
xpath_language_codes = '//table[3]/tbody/tr/td[2]'
_fetch_traits(engine_traits, url, xpath_language_codes, xpath_market_codes)

25
searx/engines/bing_news.py
View file

@ -14,12 +14,7 @@ from dateutil import parser
from lxml import etree
from lxml.etree import XPath
from searx.utils import match_language, eval_xpath_getindex
from searx.engines.bing import ( # pylint: disable=unused-import
language_aliases,
_fetch_supported_languages,
fetch_traits,
supported_languages_url,
)
from searx.enginelib.traits import EngineTraits
# about
about = {
@ -140,3 +135,21 @@ def response(resp):
results.append({'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content})
return results
def fetch_traits(engine_traits: EngineTraits):
"""Fetch languages and regions from Bing-News."""
# pylint: disable=import-outside-toplevel
from searx.engines.bing import _fetch_traits
url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-news-search/reference/market-codes'
# The description of the first table says "query parameter when calling the
# Video Search API." .. thats why I use the 4. table "News Category API markets"
xpath_market_codes = '//table[4]/tbody/tr/td[3]'
# xpath_country_codes = '//table[2]/tbody/tr/td[2]'
xpath_language_codes = '//table[3]/tbody/tr/td[2]'
_fetch_traits(engine_traits, url, xpath_language_codes, xpath_market_codes)

22
searx/engines/bing_videos.py
View file

@ -10,13 +10,7 @@ from urllib.parse import urlencode
from lxml import html
from searx.utils import match_language
from searx.engines.bing import language_aliases
from searx.engines.bing import ( # pylint: disable=unused-import
_fetch_supported_languages,
fetch_traits,
supported_languages_url,
)
from searx.enginelib.traits import EngineTraits
about = {
"website": 'https://www.bing.com/videos',
@ -97,3 +91,17 @@ def response(resp):
)
return results
def fetch_traits(engine_traits: EngineTraits):
"""Fetch languages and regions from Bing-Videos."""
# pylint: disable=import-outside-toplevel
from searx.engines.bing import _fetch_traits
url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-video-search/reference/market-codes'
xpath_market_codes = '//table[1]/tbody/tr/td[3]'
# xpath_country_codes = '//table[2]/tbody/tr/td[2]'
xpath_language_codes = '//table[3]/tbody/tr/td[2]'
_fetch_traits(engine_traits, url, xpath_language_codes, xpath_market_codes)

18
searx/sxng_locales.py
View file

@ -8,8 +8,6 @@ This file is generated automatically by::
sxng_locales = (
('ar', 'العربية', '', 'Arabic', '\U0001f310'),
('ar-EG', 'العربية', 'مصر', 'Arabic', '\U0001f1ea\U0001f1ec'),
('ar-SA', 'العربية', 'المملكة العربية السعودية', 'Arabic', '\U0001f1f8\U0001f1e6'),
('bg', 'Български', '', 'Bulgarian', '\U0001f310'),
('bg-BG', 'Български', 'България', 'Bulgarian', '\U0001f1e7\U0001f1ec'),
('ca', 'Català', '', 'Catalan', '\U0001f310'),
@ -30,19 +28,17 @@ sxng_locales = (
('en-GB', 'English', 'United Kingdom', 'English', '\U0001f1ec\U0001f1e7'),
('en-IE', 'English', 'Ireland', 'English', '\U0001f1ee\U0001f1ea'),
('en-IN', 'English', 'India', 'English', '\U0001f1ee\U0001f1f3'),
('en-MY', 'English', 'Malaysia', 'English', '\U0001f1f2\U0001f1fe'),
('en-NZ', 'English', 'New Zealand', 'English', '\U0001f1f3\U0001f1ff'),
('en-PH', 'English', 'Philippines', 'English', '\U0001f1f5\U0001f1ed'),
('en-PK', 'English', 'Pakistan', 'English', '\U0001f1f5\U0001f1f0'),
('en-SG', 'English', 'Singapore', 'English', '\U0001f1f8\U0001f1ec'),
('en-US', 'English', 'United States', 'English', '\U0001f1fa\U0001f1f8'),
('en-ZA', 'English', 'South Africa', 'English', '\U0001f1ff\U0001f1e6'),
('es', 'Español', '', 'Spanish', '\U0001f310'),
('es-AR', 'Español', 'Argentina', 'Spanish', '\U0001f1e6\U0001f1f7'),
('es-CL', 'Español', 'Chile', 'Spanish', '\U0001f1e8\U0001f1f1'),
('es-CO', 'Español', 'Colombia', 'Spanish', '\U0001f1e8\U0001f1f4'),
('es-ES', 'Español', 'España', 'Spanish', '\U0001f1ea\U0001f1f8'),
('es-MX', 'Español', 'México', 'Spanish', '\U0001f1f2\U0001f1fd'),
('es-PE', 'Español', 'Perú', 'Spanish', '\U0001f1f5\U0001f1ea'),
('es-US', 'Español', 'Estados Unidos', 'Spanish', '\U0001f1fa\U0001f1f8'),
('et', 'Eesti', '', 'Estonian', '\U0001f310'),
('et-EE', 'Eesti', 'Eesti', 'Estonian', '\U0001f1ea\U0001f1ea'),
('fi', 'Suomi', '', 'Finnish', '\U0001f310'),
@ -53,8 +49,6 @@ sxng_locales = (
('fr-CH', 'Français', 'Suisse', 'French', '\U0001f1e8\U0001f1ed'),
('fr-FR', 'Français', 'France', 'French', '\U0001f1eb\U0001f1f7'),
('he', 'עברית', '', 'Hebrew', '\U0001f1ee\U0001f1f7'),
('hr', 'Hrvatski', '', 'Croatian', '\U0001f310'),
('hr-HR', 'Hrvatski', 'Hrvatska', 'Croatian', '\U0001f1ed\U0001f1f7'),
('hu', 'Magyar', '', 'Hungarian', '\U0001f310'),
('hu-HU', 'Magyar', 'Magyarország', 'Hungarian', '\U0001f1ed\U0001f1fa'),
('id', 'Indonesia', '', 'Indonesian', '\U0001f310'),
@ -67,9 +61,7 @@ sxng_locales = (
('ko', '한국어', '', 'Korean', '\U0001f310'),
('ko-KR', '한국어', '대한민국', 'Korean', '\U0001f1f0\U0001f1f7'),
('lt', 'Lietuvių', '', 'Lithuanian', '\U0001f310'),
('lt-LT', 'Lietuvių', 'Lietuva', 'Lithuanian', '\U0001f1f1\U0001f1f9'),
('lv', 'Latviešu', '', 'Latvian', '\U0001f310'),
('lv-LV', 'Latviešu', 'Latvija', 'Latvian', '\U0001f1f1\U0001f1fb'),
('nb', 'Norsk Bokmål', '', 'Norwegian Bokmål', '\U0001f310'),
('nb-NO', 'Norsk Bokmål', 'Norge', 'Norwegian Bokmål', '\U0001f1f3\U0001f1f4'),
('nl', 'Nederlands', '', 'Dutch', '\U0001f310'),
@ -85,19 +77,13 @@ sxng_locales = (
('ru', 'Русский', '', 'Russian', '\U0001f310'),
('ru-RU', 'Русский', 'Россия', 'Russian', '\U0001f1f7\U0001f1fa'),
('sk', 'Slovenčina', '', 'Slovak', '\U0001f310'),
('sk-SK', 'Slovenčina', 'Slovensko', 'Slovak', '\U0001f1f8\U0001f1f0'),
('sl', 'Slovenščina', '', 'Slovenian', '\U0001f310'),
('sl-SI', 'Slovenščina', 'Slovenija', 'Slovenian', '\U0001f1f8\U0001f1ee'),
('sv', 'Svenska', '', 'Swedish', '\U0001f310'),
('sv-SE', 'Svenska', 'Sverige', 'Swedish', '\U0001f1f8\U0001f1ea'),
('th', 'ไทย', '', 'Thai', '\U0001f310'),
('th-TH', 'ไทย', 'ไทย', 'Thai', '\U0001f1f9\U0001f1ed'),
('tr', 'Türkçe', '', 'Turkish', '\U0001f310'),
('tr-TR', 'Türkçe', 'Türkiye', 'Turkish', '\U0001f1f9\U0001f1f7'),
('uk', 'Українська', '', 'Ukrainian', '\U0001f310'),
('uk-UA', 'Українська', 'Україна', 'Ukrainian', '\U0001f1fa\U0001f1e6'),
('vi', 'Tiếng Việt', '', 'Vietnamese', '\U0001f310'),
('vi-VN', 'Tiếng Việt', 'Việt Nam', 'Vietnamese', '\U0001f1fb\U0001f1f3'),
('zh', '中文', '', 'Chinese', '\U0001f310'),
('zh-CN', '中文', '中国', 'Chinese', '\U0001f1e8\U0001f1f3'),
('zh-HK', '中文', '中國香港', 'Chinese', '\U0001f1ed\U0001f1f0'),