mirror of
https://github.com/searxng/searxng.git
synced 2024-11-26 21:01:01 +00:00
[mod] DuckDuckGo: reversed engineered & upgrade to data_type: traits_v1
Partial reverse engineering of the DuckDuckGo (DDG) engines including a
improved language and region handling based on the enigne.traits_v1 data.
- DDG Lite
- DDG Instant Answer API
- DDG Images
- DDG Weather
docs/src/searx.engine.duckduckgo.rst:
Online documentation of the DDG engines (make docs.live)
searx/data/engine_traits.json
Add data type "traits_v1" generated by the fetch_traits() functions from:
- "duckduckgo" (WEB),
- "duckduckgo images" and
- "duckduckgo weather"
and remove data from obsolete data type "supported_languages".
searx/autocomplete.py:
Reversed engineered Autocomplete from DDG. Supports DDG's languages.
searx/engines/duckduckgo.py:
- fetch_traits(): Fetch languages & regions from DDG.
- get_ddg_lang(): Get DDG's language identifier from SearXNG's locale. DDG
defines its languages by region codes. DDG-Lite does not offer a language
selection to the user, only a region can be selected by the user.
- Cache ``vqd`` value: The vqd value depends on the query string and is needed
for the follow up pages or the images loaded by a XMLHttpRequest (DDG
images). The ``vqd`` value of a search term is stored for 10min in the
redis DB.
- DDG Lite engine: reversed engineered request method with improved Language
and region support and better ``vqd`` handling.
searx/engines/duckduckgo_definitions.py: DDG Instant Answer API
The *instant answers* API does not support languages, or at least we could not
find out how language support should work. It seems that most of the features
are based on English terms.
searx/engines/duckduckgo_images.py: DDG Images
Reversed engineered request method. Improved language and region handling
based on cookies and the enigne.traits_v1 data. Response: add image format to
the result list
searx/engines/duckduckgo_weather.py: DDG Weather
Improved language and region handling based on cookies and the
enigne.traits_v1 data.
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
parent
e9afc4f8ce
commit
c80e82a855
8 changed files with 775 additions and 442 deletions
22
docs/src/searx.engine.duckduckgo.rst
Normal file
22
docs/src/searx.engine.duckduckgo.rst
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
.. _duckduckgo engines:
|
||||
|
||||
=================
|
||||
DukcDukGo engines
|
||||
=================
|
||||
|
||||
.. contents:: Contents
|
||||
:depth: 2
|
||||
:local:
|
||||
:backlinks: entry
|
||||
|
||||
.. automodule:: searx.engines.duckduckgo
|
||||
:members:
|
||||
|
||||
.. automodule:: searx.engines.duckduckgo_images
|
||||
:members:
|
||||
|
||||
.. automodule:: searx.engines.duckduckgo_definitions
|
||||
:members:
|
||||
|
||||
.. automodule:: searx.engines.duckduckgo_weather
|
||||
:members:
|
||||
|
|
@ -61,14 +61,24 @@ def dbpedia(query, _lang):
|
|||
return results
|
||||
|
||||
|
||||
def duckduckgo(query, _lang):
|
||||
# duckduckgo autocompleter
|
||||
url = 'https://ac.duckduckgo.com/ac/?{0}&type=list'
|
||||
def duckduckgo(query, sxng_locale):
|
||||
"""Autocomplete from DuckDuckGo. Supports DuckDuckGo's languages"""
|
||||
|
||||
resp = loads(get(url.format(urlencode(dict(q=query)))).text)
|
||||
if len(resp) > 1:
|
||||
return resp[1]
|
||||
return []
|
||||
traits = engines['duckduckgo'].traits
|
||||
args = {
|
||||
'q': query,
|
||||
'kl': traits.get_region(sxng_locale, traits.all_locale),
|
||||
}
|
||||
|
||||
url = 'https://duckduckgo.com/ac/?type=list&' + urlencode(args)
|
||||
resp = get(url)
|
||||
|
||||
ret_val = []
|
||||
if resp.ok:
|
||||
j = resp.json()
|
||||
if len(j) > 1:
|
||||
ret_val = j[1]
|
||||
return ret_val
|
||||
|
||||
|
||||
def google(query, lang):
|
||||
|
|
|
|||
|
|
@ -471,145 +471,106 @@
|
|||
"zh_TW"
|
||||
]
|
||||
},
|
||||
"ddg definitions": {
|
||||
"all_locale": "wt-wt",
|
||||
"custom": {},
|
||||
"data_type": "supported_languages",
|
||||
"languages": {},
|
||||
"regions": {
|
||||
"ar-SA": "xa-ar",
|
||||
"bg-BG": "bg-bg",
|
||||
"ca-ES": "es-ca",
|
||||
"cs-CZ": "cz-cs",
|
||||
"da-DK": "dk-da",
|
||||
"de-AT": "at-de",
|
||||
"de-CH": "ch-de",
|
||||
"de-DE": "de-de",
|
||||
"el-GR": "gr-el",
|
||||
"en-AU": "au-en",
|
||||
"en-CA": "ca-en",
|
||||
"en-GB": "uk-en",
|
||||
"en-IE": "ie-en",
|
||||
"en-IL": "il-en",
|
||||
"en-IN": "in-en",
|
||||
"en-MY": "my-en",
|
||||
"en-NZ": "nz-en",
|
||||
"en-PH": "ph-en",
|
||||
"en-PK": "pk-en",
|
||||
"en-SG": "sg-en",
|
||||
"en-US": "us-en",
|
||||
"en-ZA": "za-en",
|
||||
"es-AR": "ar-es",
|
||||
"es-CL": "cl-es",
|
||||
"es-CO": "co-es",
|
||||
"es-ES": "es-es",
|
||||
"es-MX": "mx-es",
|
||||
"es-PE": "pe-es",
|
||||
"es-US": "us-es",
|
||||
"et-EE": "ee-et",
|
||||
"fi-FI": "fi-fi",
|
||||
"fr-BE": "be-fr",
|
||||
"fr-CA": "ca-fr",
|
||||
"fr-CH": "ch-fr",
|
||||
"fr-FR": "fr-fr",
|
||||
"hr-HR": "hr-hr",
|
||||
"hu-HU": "hu-hu",
|
||||
"id-ID": "id-en",
|
||||
"it-IT": "it-it",
|
||||
"ja-JP": "jp-jp",
|
||||
"ko-KR": "kr-kr",
|
||||
"lt-LT": "lt-lt",
|
||||
"lv-LV": "lv-lv",
|
||||
"nb-NO": "no-no",
|
||||
"nl-BE": "be-nl",
|
||||
"nl-NL": "nl-nl",
|
||||
"pl-PL": "pl-pl",
|
||||
"pt-BR": "br-pt",
|
||||
"pt-PT": "pt-pt",
|
||||
"ro-RO": "ro-ro",
|
||||
"ru-RU": "ru-ru",
|
||||
"sk-SK": "sk-sk",
|
||||
"sl-SI": "sl-sl",
|
||||
"sv-SE": "se-sv",
|
||||
"th-TH": "th-en",
|
||||
"tr-TR": "tr-tr",
|
||||
"uk-UA": "ua-uk",
|
||||
"vi-VN": "vn-en",
|
||||
"zh-CN": "cn-zh",
|
||||
"zh-HK": "hk-tzh",
|
||||
"zh-TW": "tw-tzh"
|
||||
},
|
||||
"supported_languages": [
|
||||
"ar-XA",
|
||||
"bg-BG",
|
||||
"ca-CT",
|
||||
"ca-ES",
|
||||
"cs-CZ",
|
||||
"da-DK",
|
||||
"de-AT",
|
||||
"de-CH",
|
||||
"de-DE",
|
||||
"el-GR",
|
||||
"en-AU",
|
||||
"en-CA",
|
||||
"en-ID",
|
||||
"en-IE",
|
||||
"en-IL",
|
||||
"en-IN",
|
||||
"en-MY",
|
||||
"en-NZ",
|
||||
"en-PH",
|
||||
"en-PK",
|
||||
"en-SG",
|
||||
"en-TH",
|
||||
"en-UK",
|
||||
"en-US",
|
||||
"en-VN",
|
||||
"en-ZA",
|
||||
"es-AR",
|
||||
"es-CL",
|
||||
"es-CO",
|
||||
"es-ES",
|
||||
"es-MX",
|
||||
"es-PE",
|
||||
"es-US",
|
||||
"et-EE",
|
||||
"fi-FI",
|
||||
"fr-BE",
|
||||
"fr-CA",
|
||||
"fr-CH",
|
||||
"fr-FR",
|
||||
"hr-HR",
|
||||
"hu-HU",
|
||||
"it-IT",
|
||||
"jp-JP",
|
||||
"kr-KR",
|
||||
"lt-LT",
|
||||
"lv-LV",
|
||||
"nl-BE",
|
||||
"nl-NL",
|
||||
"no-NO",
|
||||
"pl-PL",
|
||||
"pt-BR",
|
||||
"pt-PT",
|
||||
"ro-RO",
|
||||
"ru-RU",
|
||||
"sk-SK",
|
||||
"sl-SL",
|
||||
"sv-SE",
|
||||
"tr-TR",
|
||||
"tzh-HK",
|
||||
"tzh-TW",
|
||||
"uk-UA",
|
||||
"wt-WT",
|
||||
"zh-CN"
|
||||
]
|
||||
},
|
||||
"duckduckgo": {
|
||||
"all_locale": "wt-wt",
|
||||
"custom": {},
|
||||
"data_type": "supported_languages",
|
||||
"languages": {},
|
||||
"custom": {
|
||||
"lang_region": {
|
||||
"ar-DZ": "ar_DZ",
|
||||
"ar-JO": "ar_JO",
|
||||
"ar-SA": "ar_SA",
|
||||
"bn-IN": "bn_IN",
|
||||
"de-CH": "de_CH",
|
||||
"en-AU": "en_AU",
|
||||
"en-CA": "en_CA",
|
||||
"en-GB": "en_GB",
|
||||
"es-AR": "es_AR",
|
||||
"es-CL": "es_CL",
|
||||
"es-CO": "es_CO",
|
||||
"es-CR": "es_CR",
|
||||
"es-EC": "es_EC",
|
||||
"es-MX": "es_MX",
|
||||
"es-PE": "es_PE",
|
||||
"es-UY": "es_UY",
|
||||
"es-VE": "es_VE",
|
||||
"fr-BE": "fr_BE",
|
||||
"fr-CA": "fr_CA",
|
||||
"fr-CH": "fr_CH",
|
||||
"nl-BE": "nl_BE",
|
||||
"pt-BR": "pt_BR"
|
||||
}
|
||||
},
|
||||
"data_type": "traits_v1",
|
||||
"languages": {
|
||||
"af": "af_ZA",
|
||||
"ar": "ar_EG",
|
||||
"ast": "ast_ES",
|
||||
"az_Latn": "az_AZ",
|
||||
"be": "be_BY",
|
||||
"bg": "bg_BG",
|
||||
"bn": "bn_BD",
|
||||
"br": "br_FR",
|
||||
"bs_Latn": "bs_BA",
|
||||
"ca": "ca_ES",
|
||||
"cs": "cs_CZ",
|
||||
"cy": "cy_GB",
|
||||
"da": "da_DK",
|
||||
"de": "de_DE",
|
||||
"el": "el_GR",
|
||||
"en": "en_US",
|
||||
"eo": "eo_XX",
|
||||
"es": "es_ES",
|
||||
"et": "et_EE",
|
||||
"eu": "eu_ES",
|
||||
"fa": "fa_IR",
|
||||
"fi": "fi_FI",
|
||||
"fil": "tl_PH",
|
||||
"fr": "fr_FR",
|
||||
"ga": "ga_IE",
|
||||
"gd": "gd_GB",
|
||||
"gl": "gl_ES",
|
||||
"he": "he_IL",
|
||||
"hi": "hi_IN",
|
||||
"hr": "hr_HR",
|
||||
"hu": "hu_HU",
|
||||
"hy": "hy_AM",
|
||||
"id": "id_ID",
|
||||
"is": "is_IS",
|
||||
"it": "it_IT",
|
||||
"ja": "ja_JP",
|
||||
"kab": "kab_DZ",
|
||||
"kn": "kn_IN",
|
||||
"ko": "ko_KR",
|
||||
"ku": "ku",
|
||||
"kw": "kw_GB",
|
||||
"lt": "lt_LT",
|
||||
"lv": "lv_LV",
|
||||
"ml": "ml_IN",
|
||||
"mr": "mr_IN",
|
||||
"ms": "ms_MY",
|
||||
"nb": "nb_NO",
|
||||
"nl": "nl_NL",
|
||||
"nn": "nn_NO",
|
||||
"pl": "pl_PL",
|
||||
"pt": "pt_PT",
|
||||
"ro": "ro_RO",
|
||||
"ru": "ru_RU",
|
||||
"sc": "sc_IT",
|
||||
"si": "si_LK",
|
||||
"sk": "sk_SK",
|
||||
"sl": "sl_SI",
|
||||
"sq": "sq_AL",
|
||||
"sr_Cyrl": "sr_RS",
|
||||
"sv": "sv_SE",
|
||||
"ta": "ta_IN",
|
||||
"te": "te_IN",
|
||||
"th": "th_TH",
|
||||
"tr": "tr_TR",
|
||||
"uk": "uk_UA",
|
||||
"ur": "ur_PK",
|
||||
"vi": "vi_VN",
|
||||
"zh_Hans": "zh_CN",
|
||||
"zh_Hant": "zh_TW"
|
||||
},
|
||||
"regions": {
|
||||
"ar-SA": "xa-ar",
|
||||
"bg-BG": "bg-bg",
|
||||
|
|
@ -673,77 +634,108 @@
|
|||
"zh-HK": "hk-tzh",
|
||||
"zh-TW": "tw-tzh"
|
||||
},
|
||||
"supported_languages": [
|
||||
"ar-XA",
|
||||
"bg-BG",
|
||||
"ca-CT",
|
||||
"ca-ES",
|
||||
"cs-CZ",
|
||||
"da-DK",
|
||||
"de-AT",
|
||||
"de-CH",
|
||||
"de-DE",
|
||||
"el-GR",
|
||||
"en-AU",
|
||||
"en-CA",
|
||||
"en-ID",
|
||||
"en-IE",
|
||||
"en-IL",
|
||||
"en-IN",
|
||||
"en-MY",
|
||||
"en-NZ",
|
||||
"en-PH",
|
||||
"en-PK",
|
||||
"en-SG",
|
||||
"en-TH",
|
||||
"en-UK",
|
||||
"en-US",
|
||||
"en-VN",
|
||||
"en-ZA",
|
||||
"es-AR",
|
||||
"es-CL",
|
||||
"es-CO",
|
||||
"es-ES",
|
||||
"es-MX",
|
||||
"es-PE",
|
||||
"es-US",
|
||||
"et-EE",
|
||||
"fi-FI",
|
||||
"fr-BE",
|
||||
"fr-CA",
|
||||
"fr-CH",
|
||||
"fr-FR",
|
||||
"hr-HR",
|
||||
"hu-HU",
|
||||
"it-IT",
|
||||
"jp-JP",
|
||||
"kr-KR",
|
||||
"lt-LT",
|
||||
"lv-LV",
|
||||
"nl-BE",
|
||||
"nl-NL",
|
||||
"no-NO",
|
||||
"pl-PL",
|
||||
"pt-BR",
|
||||
"pt-PT",
|
||||
"ro-RO",
|
||||
"ru-RU",
|
||||
"sk-SK",
|
||||
"sl-SL",
|
||||
"sv-SE",
|
||||
"tr-TR",
|
||||
"tzh-HK",
|
||||
"tzh-TW",
|
||||
"uk-UA",
|
||||
"wt-WT",
|
||||
"zh-CN"
|
||||
]
|
||||
"supported_languages": {}
|
||||
},
|
||||
"duckduckgo images": {
|
||||
"all_locale": "wt-wt",
|
||||
"custom": {},
|
||||
"data_type": "supported_languages",
|
||||
"languages": {},
|
||||
"custom": {
|
||||
"lang_region": {
|
||||
"ar-DZ": "ar_DZ",
|
||||
"ar-JO": "ar_JO",
|
||||
"ar-SA": "ar_SA",
|
||||
"bn-IN": "bn_IN",
|
||||
"de-CH": "de_CH",
|
||||
"en-AU": "en_AU",
|
||||
"en-CA": "en_CA",
|
||||
"en-GB": "en_GB",
|
||||
"es-AR": "es_AR",
|
||||
"es-CL": "es_CL",
|
||||
"es-CO": "es_CO",
|
||||
"es-CR": "es_CR",
|
||||
"es-EC": "es_EC",
|
||||
"es-MX": "es_MX",
|
||||
"es-PE": "es_PE",
|
||||
"es-UY": "es_UY",
|
||||
"es-VE": "es_VE",
|
||||
"fr-BE": "fr_BE",
|
||||
"fr-CA": "fr_CA",
|
||||
"fr-CH": "fr_CH",
|
||||
"nl-BE": "nl_BE",
|
||||
"pt-BR": "pt_BR"
|
||||
}
|
||||
},
|
||||
"data_type": "traits_v1",
|
||||
"languages": {
|
||||
"af": "af_ZA",
|
||||
"ar": "ar_EG",
|
||||
"ast": "ast_ES",
|
||||
"az_Latn": "az_AZ",
|
||||
"be": "be_BY",
|
||||
"bg": "bg_BG",
|
||||
"bn": "bn_BD",
|
||||
"br": "br_FR",
|
||||
"bs_Latn": "bs_BA",
|
||||
"ca": "ca_ES",
|
||||
"cs": "cs_CZ",
|
||||
"cy": "cy_GB",
|
||||
"da": "da_DK",
|
||||
"de": "de_DE",
|
||||
"el": "el_GR",
|
||||
"en": "en_US",
|
||||
"eo": "eo_XX",
|
||||
"es": "es_ES",
|
||||
"et": "et_EE",
|
||||
"eu": "eu_ES",
|
||||
"fa": "fa_IR",
|
||||
"fi": "fi_FI",
|
||||
"fil": "tl_PH",
|
||||
"fr": "fr_FR",
|
||||
"ga": "ga_IE",
|
||||
"gd": "gd_GB",
|
||||
"gl": "gl_ES",
|
||||
"he": "he_IL",
|
||||
"hi": "hi_IN",
|
||||
"hr": "hr_HR",
|
||||
"hu": "hu_HU",
|
||||
"hy": "hy_AM",
|
||||
"id": "id_ID",
|
||||
"is": "is_IS",
|
||||
"it": "it_IT",
|
||||
"ja": "ja_JP",
|
||||
"kab": "kab_DZ",
|
||||
"kn": "kn_IN",
|
||||
"ko": "ko_KR",
|
||||
"ku": "ku",
|
||||
"kw": "kw_GB",
|
||||
"lt": "lt_LT",
|
||||
"lv": "lv_LV",
|
||||
"ml": "ml_IN",
|
||||
"mr": "mr_IN",
|
||||
"ms": "ms_MY",
|
||||
"nb": "nb_NO",
|
||||
"nl": "nl_NL",
|
||||
"nn": "nn_NO",
|
||||
"pl": "pl_PL",
|
||||
"pt": "pt_PT",
|
||||
"ro": "ro_RO",
|
||||
"ru": "ru_RU",
|
||||
"sc": "sc_IT",
|
||||
"si": "si_LK",
|
||||
"sk": "sk_SK",
|
||||
"sl": "sl_SI",
|
||||
"sq": "sq_AL",
|
||||
"sr_Cyrl": "sr_RS",
|
||||
"sv": "sv_SE",
|
||||
"ta": "ta_IN",
|
||||
"te": "te_IN",
|
||||
"th": "th_TH",
|
||||
"tr": "tr_TR",
|
||||
"uk": "uk_UA",
|
||||
"ur": "ur_PK",
|
||||
"vi": "vi_VN",
|
||||
"zh_Hans": "zh_CN",
|
||||
"zh_Hant": "zh_TW"
|
||||
},
|
||||
"regions": {
|
||||
"ar-SA": "xa-ar",
|
||||
"bg-BG": "bg-bg",
|
||||
|
|
@ -807,71 +799,172 @@
|
|||
"zh-HK": "hk-tzh",
|
||||
"zh-TW": "tw-tzh"
|
||||
},
|
||||
"supported_languages": [
|
||||
"ar-XA",
|
||||
"bg-BG",
|
||||
"ca-CT",
|
||||
"ca-ES",
|
||||
"cs-CZ",
|
||||
"da-DK",
|
||||
"de-AT",
|
||||
"de-CH",
|
||||
"de-DE",
|
||||
"el-GR",
|
||||
"en-AU",
|
||||
"en-CA",
|
||||
"en-ID",
|
||||
"en-IE",
|
||||
"en-IL",
|
||||
"en-IN",
|
||||
"en-MY",
|
||||
"en-NZ",
|
||||
"en-PH",
|
||||
"en-PK",
|
||||
"en-SG",
|
||||
"en-TH",
|
||||
"en-UK",
|
||||
"en-US",
|
||||
"en-VN",
|
||||
"en-ZA",
|
||||
"es-AR",
|
||||
"es-CL",
|
||||
"es-CO",
|
||||
"es-ES",
|
||||
"es-MX",
|
||||
"es-PE",
|
||||
"es-US",
|
||||
"et-EE",
|
||||
"fi-FI",
|
||||
"fr-BE",
|
||||
"fr-CA",
|
||||
"fr-CH",
|
||||
"fr-FR",
|
||||
"hr-HR",
|
||||
"hu-HU",
|
||||
"it-IT",
|
||||
"jp-JP",
|
||||
"kr-KR",
|
||||
"lt-LT",
|
||||
"lv-LV",
|
||||
"nl-BE",
|
||||
"nl-NL",
|
||||
"no-NO",
|
||||
"pl-PL",
|
||||
"pt-BR",
|
||||
"pt-PT",
|
||||
"ro-RO",
|
||||
"ru-RU",
|
||||
"sk-SK",
|
||||
"sl-SL",
|
||||
"sv-SE",
|
||||
"tr-TR",
|
||||
"tzh-HK",
|
||||
"tzh-TW",
|
||||
"uk-UA",
|
||||
"wt-WT",
|
||||
"zh-CN"
|
||||
]
|
||||
"supported_languages": {}
|
||||
},
|
||||
"duckduckgo weather": {
|
||||
"all_locale": "wt-wt",
|
||||
"custom": {
|
||||
"lang_region": {
|
||||
"ar-DZ": "ar_DZ",
|
||||
"ar-JO": "ar_JO",
|
||||
"ar-SA": "ar_SA",
|
||||
"bn-IN": "bn_IN",
|
||||
"de-CH": "de_CH",
|
||||
"en-AU": "en_AU",
|
||||
"en-CA": "en_CA",
|
||||
"en-GB": "en_GB",
|
||||
"es-AR": "es_AR",
|
||||
"es-CL": "es_CL",
|
||||
"es-CO": "es_CO",
|
||||
"es-CR": "es_CR",
|
||||
"es-EC": "es_EC",
|
||||
"es-MX": "es_MX",
|
||||
"es-PE": "es_PE",
|
||||
"es-UY": "es_UY",
|
||||
"es-VE": "es_VE",
|
||||
"fr-BE": "fr_BE",
|
||||
"fr-CA": "fr_CA",
|
||||
"fr-CH": "fr_CH",
|
||||
"nl-BE": "nl_BE",
|
||||
"pt-BR": "pt_BR"
|
||||
}
|
||||
},
|
||||
"data_type": "traits_v1",
|
||||
"languages": {
|
||||
"af": "af_ZA",
|
||||
"ar": "ar_EG",
|
||||
"ast": "ast_ES",
|
||||
"az_Latn": "az_AZ",
|
||||
"be": "be_BY",
|
||||
"bg": "bg_BG",
|
||||
"bn": "bn_BD",
|
||||
"br": "br_FR",
|
||||
"bs_Latn": "bs_BA",
|
||||
"ca": "ca_ES",
|
||||
"cs": "cs_CZ",
|
||||
"cy": "cy_GB",
|
||||
"da": "da_DK",
|
||||
"de": "de_DE",
|
||||
"el": "el_GR",
|
||||
"en": "en_US",
|
||||
"eo": "eo_XX",
|
||||
"es": "es_ES",
|
||||
"et": "et_EE",
|
||||
"eu": "eu_ES",
|
||||
"fa": "fa_IR",
|
||||
"fi": "fi_FI",
|
||||
"fil": "tl_PH",
|
||||
"fr": "fr_FR",
|
||||
"ga": "ga_IE",
|
||||
"gd": "gd_GB",
|
||||
"gl": "gl_ES",
|
||||
"he": "he_IL",
|
||||
"hi": "hi_IN",
|
||||
"hr": "hr_HR",
|
||||
"hu": "hu_HU",
|
||||
"hy": "hy_AM",
|
||||
"id": "id_ID",
|
||||
"is": "is_IS",
|
||||
"it": "it_IT",
|
||||
"ja": "ja_JP",
|
||||
"kab": "kab_DZ",
|
||||
"kn": "kn_IN",
|
||||
"ko": "ko_KR",
|
||||
"ku": "ku",
|
||||
"kw": "kw_GB",
|
||||
"lt": "lt_LT",
|
||||
"lv": "lv_LV",
|
||||
"ml": "ml_IN",
|
||||
"mr": "mr_IN",
|
||||
"ms": "ms_MY",
|
||||
"nb": "nb_NO",
|
||||
"nl": "nl_NL",
|
||||
"nn": "nn_NO",
|
||||
"pl": "pl_PL",
|
||||
"pt": "pt_PT",
|
||||
"ro": "ro_RO",
|
||||
"ru": "ru_RU",
|
||||
"sc": "sc_IT",
|
||||
"si": "si_LK",
|
||||
"sk": "sk_SK",
|
||||
"sl": "sl_SI",
|
||||
"sq": "sq_AL",
|
||||
"sr_Cyrl": "sr_RS",
|
||||
"sv": "sv_SE",
|
||||
"ta": "ta_IN",
|
||||
"te": "te_IN",
|
||||
"th": "th_TH",
|
||||
"tr": "tr_TR",
|
||||
"uk": "uk_UA",
|
||||
"ur": "ur_PK",
|
||||
"vi": "vi_VN",
|
||||
"zh_Hans": "zh_CN",
|
||||
"zh_Hant": "zh_TW"
|
||||
},
|
||||
"regions": {
|
||||
"ar-SA": "xa-ar",
|
||||
"bg-BG": "bg-bg",
|
||||
"ca-ES": "es-ca",
|
||||
"cs-CZ": "cz-cs",
|
||||
"da-DK": "dk-da",
|
||||
"de-AT": "at-de",
|
||||
"de-CH": "ch-de",
|
||||
"de-DE": "de-de",
|
||||
"el-GR": "gr-el",
|
||||
"en-AU": "au-en",
|
||||
"en-CA": "ca-en",
|
||||
"en-GB": "uk-en",
|
||||
"en-IE": "ie-en",
|
||||
"en-IL": "il-en",
|
||||
"en-IN": "in-en",
|
||||
"en-MY": "my-en",
|
||||
"en-NZ": "nz-en",
|
||||
"en-PH": "ph-en",
|
||||
"en-PK": "pk-en",
|
||||
"en-SG": "sg-en",
|
||||
"en-US": "us-en",
|
||||
"en-ZA": "za-en",
|
||||
"es-AR": "ar-es",
|
||||
"es-CL": "cl-es",
|
||||
"es-CO": "co-es",
|
||||
"es-ES": "es-es",
|
||||
"es-MX": "mx-es",
|
||||
"es-PE": "pe-es",
|
||||
"es-US": "us-es",
|
||||
"et-EE": "ee-et",
|
||||
"fi-FI": "fi-fi",
|
||||
"fr-BE": "be-fr",
|
||||
"fr-CA": "ca-fr",
|
||||
"fr-CH": "ch-fr",
|
||||
"fr-FR": "fr-fr",
|
||||
"hr-HR": "hr-hr",
|
||||
"hu-HU": "hu-hu",
|
||||
"id-ID": "id-en",
|
||||
"it-IT": "it-it",
|
||||
"ja-JP": "jp-jp",
|
||||
"ko-KR": "kr-kr",
|
||||
"lt-LT": "lt-lt",
|
||||
"lv-LV": "lv-lv",
|
||||
"nb-NO": "no-no",
|
||||
"nl-BE": "be-nl",
|
||||
"nl-NL": "nl-nl",
|
||||
"pl-PL": "pl-pl",
|
||||
"pt-BR": "br-pt",
|
||||
"pt-PT": "pt-pt",
|
||||
"ro-RO": "ro-ro",
|
||||
"ru-RU": "ru-ru",
|
||||
"sk-SK": "sk-sk",
|
||||
"sl-SI": "sl-sl",
|
||||
"sv-SE": "se-sv",
|
||||
"th-TH": "th-en",
|
||||
"tr-TR": "tr-tr",
|
||||
"uk-UA": "ua-uk",
|
||||
"vi-VN": "vn-en",
|
||||
"zh-CN": "cn-zh",
|
||||
"zh-HK": "hk-tzh",
|
||||
"zh-TW": "tw-tzh"
|
||||
},
|
||||
"supported_languages": {}
|
||||
},
|
||||
"google": {
|
||||
"all_locale": "ZZ",
|
||||
|
|
|
|||
|
|
@ -1,73 +1,207 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""DuckDuckGo Lite
|
||||
"""
|
||||
DuckDuckGo Lite
|
||||
~~~~~~~~~~~~~~~
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlencode
|
||||
import json
|
||||
from lxml import html
|
||||
import babel
|
||||
import lxml.html
|
||||
|
||||
from searx import (
|
||||
network,
|
||||
locales,
|
||||
redislib,
|
||||
)
|
||||
from searx import redisdb
|
||||
from searx.utils import (
|
||||
dict_subset,
|
||||
eval_xpath,
|
||||
eval_xpath_getindex,
|
||||
extract_text,
|
||||
match_language,
|
||||
)
|
||||
from searx import network
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import logging
|
||||
|
||||
logger: logging.Logger
|
||||
|
||||
traits: EngineTraits
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": 'https://lite.duckduckgo.com/lite/',
|
||||
"wikidata_id": 'Q12805',
|
||||
"official_api_documentation": 'https://duckduckgo.com/api',
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'HTML',
|
||||
}
|
||||
|
||||
send_accept_language_header = True
|
||||
"""DuckDuckGo-Lite tries to guess user's prefered language from the HTTP
|
||||
``Accept-Language``. Optional the user can select a region filter (but not a
|
||||
language).
|
||||
"""
|
||||
|
||||
# engine dependent config
|
||||
categories = ['general', 'web']
|
||||
paging = True
|
||||
supported_languages_url = 'https://duckduckgo.com/util/u588.js'
|
||||
time_range_support = True
|
||||
send_accept_language_header = True
|
||||
safesearch = True # user can't select but the results are filtered
|
||||
|
||||
language_aliases = {
|
||||
'ar-SA': 'ar-XA',
|
||||
'es-419': 'es-XL',
|
||||
'ja': 'jp-JP',
|
||||
'ko': 'kr-KR',
|
||||
'sl-SI': 'sl-SL',
|
||||
'zh-TW': 'tzh-TW',
|
||||
'zh-HK': 'tzh-HK',
|
||||
}
|
||||
url = 'https://lite.duckduckgo.com/lite/'
|
||||
# url_ping = 'https://duckduckgo.com/t/sl_l'
|
||||
|
||||
time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
|
||||
form_data = {'v': 'l', 'api': 'd.js', 'o': 'json'}
|
||||
|
||||
# search-url
|
||||
url = 'https://lite.duckduckgo.com/lite/'
|
||||
url_ping = 'https://duckduckgo.com/t/sl_l'
|
||||
|
||||
# match query's language to a region code that duckduckgo will accept
|
||||
def get_region_code(lang, lang_list=None):
|
||||
if lang == 'all':
|
||||
return None
|
||||
def cache_vqd(query, value):
|
||||
"""Caches a ``vqd`` value from a query.
|
||||
|
||||
lang_code = match_language(lang, lang_list or [], language_aliases, 'wt-WT')
|
||||
lang_parts = lang_code.split('-')
|
||||
The vqd value depends on the query string and is needed for the follow up
|
||||
pages or the images loaded by a XMLHttpRequest:
|
||||
|
||||
# country code goes first
|
||||
return lang_parts[1].lower() + '-' + lang_parts[0].lower()
|
||||
- DuckDuckGo Web: `https://links.duckduckgo.com/d.js?q=...&vqd=...`
|
||||
- DuckDuckGo Images: `https://duckduckgo.com/i.js??q=...&vqd=...`
|
||||
|
||||
"""
|
||||
c = redisdb.client()
|
||||
if c:
|
||||
logger.debug("cache vqd value: %s", value)
|
||||
key = 'SearXNG_ddg_vqd' + redislib.secret_hash(query)
|
||||
c.set(key, value, ex=600)
|
||||
|
||||
|
||||
def get_vqd(query, headers):
|
||||
"""Returns the ``vqd`` that fits to the *query*. If there is no ``vqd`` cached
|
||||
(:py:obj:`cache_vqd`) the query is sent to DDG to get a vqd value from the
|
||||
response.
|
||||
|
||||
"""
|
||||
value = None
|
||||
c = redisdb.client()
|
||||
if c:
|
||||
key = 'SearXNG_ddg_vqd' + redislib.secret_hash(query)
|
||||
value = c.get(key)
|
||||
if value:
|
||||
value = value.decode('utf-8')
|
||||
logger.debug("re-use cached vqd value: %s", value)
|
||||
return value
|
||||
|
||||
query_url = 'https://duckduckgo.com/?{query}&iar=images'.format(query=urlencode({'q': query}))
|
||||
res = network.get(query_url, headers=headers)
|
||||
content = res.text
|
||||
if content.find('vqd=\'') == -1:
|
||||
raise SearxEngineAPIException('Request failed')
|
||||
value = content[content.find('vqd=\'') + 5 :]
|
||||
value = value[: value.find('\'')]
|
||||
logger.debug("new vqd value: %s", value)
|
||||
cache_vqd(query, value)
|
||||
return value
|
||||
|
||||
|
||||
def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
|
||||
"""Get DuckDuckGo's language identifier from SearXNG's locale.
|
||||
|
||||
DuckDuckGo defines its lanaguages by region codes (see
|
||||
:py:obj:`fetch_traits`).
|
||||
|
||||
To get region and language of a DDG service use:
|
||||
|
||||
.. code: python
|
||||
|
||||
eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
|
||||
eng_lang = get_ddg_lang(traits, params['searxng_locale'])
|
||||
|
||||
It might confuse, but the ``l`` value of the cookie is what SearXNG calls
|
||||
the *region*:
|
||||
|
||||
.. code:: python
|
||||
|
||||
# !ddi paris :es-AR --> {'ad': 'es_AR', 'ah': 'ar-es', 'l': 'ar-es'}
|
||||
params['cookies']['ad'] = eng_lang
|
||||
params['cookies']['ah'] = eng_region
|
||||
params['cookies']['l'] = eng_region
|
||||
|
||||
.. hint::
|
||||
|
||||
`DDG-lite <https://lite.duckduckgo.com/lite>`__ does not offer a language
|
||||
selection to the user, only a region can be selected by the user
|
||||
(``eng_region`` from the example above). DDG-lite stores the selected
|
||||
region in a cookie::
|
||||
|
||||
params['cookies']['kl'] = eng_region # 'ar-es'
|
||||
|
||||
"""
|
||||
return eng_traits.custom['lang_region'].get(sxng_locale, eng_traits.get_language(sxng_locale, default))
|
||||
|
||||
|
||||
ddg_reg_map = {
|
||||
'tw-tzh': 'zh_TW',
|
||||
'hk-tzh': 'zh_HK',
|
||||
'ct-ca': 'skip', # ct-ca and es-ca both map to ca_ES
|
||||
'es-ca': 'ca_ES',
|
||||
'id-en': 'id_ID',
|
||||
'no-no': 'nb_NO',
|
||||
'jp-jp': 'ja_JP',
|
||||
'kr-kr': 'ko_KR',
|
||||
'xa-ar': 'ar_SA',
|
||||
'sl-sl': 'sl_SI',
|
||||
'th-en': 'th_TH',
|
||||
'vn-en': 'vi_VN',
|
||||
}
|
||||
|
||||
ddg_lang_map = {
|
||||
# use ar --> ar_EG (Egypt's arabic)
|
||||
"ar_DZ": 'lang_region',
|
||||
"ar_JO": 'lang_region',
|
||||
"ar_SA": 'lang_region',
|
||||
# use bn --> bn_BD
|
||||
'bn_IN': 'lang_region',
|
||||
# use de --> de_DE
|
||||
'de_CH': 'lang_region',
|
||||
# use en --> en_US,
|
||||
'en_AU': 'lang_region',
|
||||
'en_CA': 'lang_region',
|
||||
'en_GB': 'lang_region',
|
||||
# Esperanto
|
||||
'eo_XX': 'eo',
|
||||
# use es --> es_ES,
|
||||
'es_AR': 'lang_region',
|
||||
'es_CL': 'lang_region',
|
||||
'es_CO': 'lang_region',
|
||||
'es_CR': 'lang_region',
|
||||
'es_EC': 'lang_region',
|
||||
'es_MX': 'lang_region',
|
||||
'es_PE': 'lang_region',
|
||||
'es_UY': 'lang_region',
|
||||
'es_VE': 'lang_region',
|
||||
# use fr --> rf_FR
|
||||
'fr_CA': 'lang_region',
|
||||
'fr_CH': 'lang_region',
|
||||
'fr_BE': 'lang_region',
|
||||
# use nl --> nl_NL
|
||||
'nl_BE': 'lang_region',
|
||||
# use pt --> pt_PT
|
||||
'pt_BR': 'lang_region',
|
||||
# skip these languages
|
||||
'od_IN': 'skip',
|
||||
'io_XX': 'skip',
|
||||
'tokipona_XX': 'skip',
|
||||
}
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
|
||||
# eng_lang = get_ddg_lang(traits, params['searxng_locale'])
|
||||
|
||||
params['url'] = url
|
||||
params['method'] = 'POST'
|
||||
|
||||
params['data']['q'] = query
|
||||
|
||||
# The API is not documented, so we do some reverse engineering and emulate
|
||||
|
|
@ -90,23 +224,19 @@ def request(query, params):
|
|||
params['data']['s'] = offset
|
||||
params['data']['dc'] = offset + 1
|
||||
|
||||
# request needs a vqd argument
|
||||
params['data']['vqd'] = get_vqd(query, params["headers"])
|
||||
|
||||
# initial page does not have additional data in the input form
|
||||
if params['pageno'] > 1:
|
||||
# request the second page (and more pages) needs 'o' and 'api' arguments
|
||||
params['data']['o'] = 'json'
|
||||
params['data']['api'] = 'd.js'
|
||||
|
||||
# initial page does not have additional data in the input form
|
||||
if params['pageno'] > 2:
|
||||
# request the third page (and more pages) some more arguments
|
||||
params['data']['nextParams'] = ''
|
||||
params['data']['v'] = ''
|
||||
params['data']['vqd'] = ''
|
||||
params['data']['o'] = form_data.get('o', 'json')
|
||||
params['data']['api'] = form_data.get('api', 'd.js')
|
||||
params['data']['nextParams'] = form_data.get('nextParams', '')
|
||||
params['data']['v'] = form_data.get('v', 'l')
|
||||
|
||||
region_code = get_region_code(params['language'], supported_languages)
|
||||
if region_code:
|
||||
params['data']['kl'] = region_code
|
||||
params['cookies']['kl'] = region_code
|
||||
params['data']['kl'] = eng_region
|
||||
params['cookies']['kl'] = eng_region
|
||||
|
||||
params['data']['df'] = ''
|
||||
if params['time_range'] in time_range_dict:
|
||||
|
|
@ -118,26 +248,40 @@ def request(query, params):
|
|||
return params
|
||||
|
||||
|
||||
# get response from search-request
|
||||
def response(resp):
|
||||
|
||||
headers_ping = dict_subset(resp.request.headers, ['User-Agent', 'Accept-Encoding', 'Accept', 'Cookie'])
|
||||
network.get(url_ping, headers=headers_ping)
|
||||
|
||||
if resp.status_code == 303:
|
||||
return []
|
||||
|
||||
results = []
|
||||
doc = html.fromstring(resp.text)
|
||||
doc = lxml.html.fromstring(resp.text)
|
||||
|
||||
result_table = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table')
|
||||
if not len(result_table) >= 3:
|
||||
|
||||
if len(result_table) == 2:
|
||||
# some locales (at least China) does not have a "next page" button and
|
||||
# the layout of the HTML tables is different.
|
||||
result_table = result_table[1]
|
||||
elif not len(result_table) >= 3:
|
||||
# no more results
|
||||
return []
|
||||
else:
|
||||
result_table = result_table[2]
|
||||
# update form data from response
|
||||
form = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table//input/..')
|
||||
if len(form):
|
||||
|
||||
form = form[0]
|
||||
form_data['v'] = eval_xpath(form, '//input[@name="v"]/@value')[0]
|
||||
form_data['api'] = eval_xpath(form, '//input[@name="api"]/@value')[0]
|
||||
form_data['o'] = eval_xpath(form, '//input[@name="o"]/@value')[0]
|
||||
logger.debug('form_data: %s', form_data)
|
||||
|
||||
value = eval_xpath(form, '//input[@name="vqd"]/@value')[0]
|
||||
query = resp.search_params['data']['q']
|
||||
cache_vqd(query, value)
|
||||
|
||||
tr_rows = eval_xpath(result_table, './/tr')
|
||||
|
||||
# In the last <tr> is the form of the 'previous/next page' links
|
||||
tr_rows = tr_rows[:-1]
|
||||
|
||||
|
|
@ -174,32 +318,35 @@ def response(resp):
|
|||
return results
|
||||
|
||||
|
||||
# get supported languages from their site
|
||||
def _fetch_supported_languages(resp):
|
||||
|
||||
# response is a js file with regions as an embedded object
|
||||
response_page = resp.text
|
||||
response_page = response_page[response_page.find('regions:{') + 8 :]
|
||||
response_page = response_page[: response_page.find('}') + 1]
|
||||
|
||||
regions_json = json.loads(response_page)
|
||||
supported_languages = map((lambda x: x[3:] + '-' + x[:2].upper()), regions_json.keys())
|
||||
|
||||
return list(supported_languages)
|
||||
|
||||
|
||||
def fetch_traits(engine_traits: EngineTraits):
|
||||
"""Fetch regions from DuckDuckGo."""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
"""Fetch languages & regions from DuckDuckGo.
|
||||
|
||||
engine_traits.data_type = 'supported_languages' # deprecated
|
||||
SearXNG's ``all`` locale maps DuckDuckGo's "Alle regions" (``wt-wt``).
|
||||
DuckDuckGo's language "Browsers prefered language" (``wt_WT``) makes no
|
||||
sense in a SearXNG request since SearXNG's ``all`` will not add a
|
||||
``Accept-Language`` HTTP header. The value in ``engine_traits.all_locale``
|
||||
is ``wt-wt`` (the region).
|
||||
|
||||
import babel
|
||||
from searx.locales import region_tag
|
||||
Beside regions DuckDuckGo also defines its lanaguages by region codes. By
|
||||
example these are the english languages in DuckDuckGo:
|
||||
|
||||
- en_US
|
||||
- en_AU
|
||||
- en_CA
|
||||
- en_GB
|
||||
|
||||
The function :py:obj:`get_ddg_lang` evaluates DuckDuckGo's language from
|
||||
SearXNG's locale.
|
||||
|
||||
"""
|
||||
# pylint: disable=too-many-branches, too-many-statements
|
||||
# fetch regions
|
||||
|
||||
engine_traits.all_locale = 'wt-wt'
|
||||
|
||||
resp = network.get('https://duckduckgo.com/util/u588.js')
|
||||
# updated from u588 to u661 / should be updated automatically?
|
||||
resp = network.get('https://duckduckgo.com/util/u661.js')
|
||||
|
||||
if not resp.ok:
|
||||
print("ERROR: response from DuckDuckGo is not OK.")
|
||||
|
||||
|
|
@ -208,28 +355,13 @@ def fetch_traits(engine_traits: EngineTraits):
|
|||
pos = js_code.find('}') + 1
|
||||
regions = json.loads(js_code[:pos])
|
||||
|
||||
reg_map = {
|
||||
'tw-tzh': 'zh_TW',
|
||||
'hk-tzh': 'zh_HK',
|
||||
'ct-ca': 'skip', # ct-ca and es-ca both map to ca_ES
|
||||
'es-ca': 'ca_ES',
|
||||
'id-en': 'id_ID',
|
||||
'no-no': 'nb_NO',
|
||||
'jp-jp': 'ja_JP',
|
||||
'kr-kr': 'ko_KR',
|
||||
'xa-ar': 'ar_SA',
|
||||
'sl-sl': 'sl_SI',
|
||||
'th-en': 'th_TH',
|
||||
'vn-en': 'vi_VN',
|
||||
}
|
||||
|
||||
for eng_tag, name in regions.items():
|
||||
|
||||
if eng_tag == 'wt-wt':
|
||||
engine_traits.all_locale = 'wt-wt'
|
||||
continue
|
||||
|
||||
region = reg_map.get(eng_tag)
|
||||
region = ddg_reg_map.get(eng_tag)
|
||||
if region == 'skip':
|
||||
continue
|
||||
|
||||
|
|
@ -238,7 +370,7 @@ def fetch_traits(engine_traits: EngineTraits):
|
|||
region = eng_lang + '_' + eng_territory.upper()
|
||||
|
||||
try:
|
||||
sxng_tag = region_tag(babel.Locale.parse(region))
|
||||
sxng_tag = locales.region_tag(babel.Locale.parse(region))
|
||||
except babel.UnknownLocaleError:
|
||||
print("ERROR: %s (%s) -> %s is unknown by babel" % (name, eng_tag, region))
|
||||
continue
|
||||
|
|
@ -249,3 +381,42 @@ def fetch_traits(engine_traits: EngineTraits):
|
|||
print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
|
||||
continue
|
||||
engine_traits.regions[sxng_tag] = eng_tag
|
||||
|
||||
# fetch languages
|
||||
|
||||
engine_traits.custom['lang_region'] = {}
|
||||
|
||||
pos = resp.text.find('languages:{') + 10
|
||||
js_code = resp.text[pos:]
|
||||
pos = js_code.find('}') + 1
|
||||
js_code = '{"' + js_code[1:pos].replace(':', '":').replace(',', ',"')
|
||||
languages = json.loads(js_code)
|
||||
|
||||
for eng_lang, name in languages.items():
|
||||
|
||||
if eng_lang == 'wt_WT':
|
||||
continue
|
||||
|
||||
babel_tag = ddg_lang_map.get(eng_lang, eng_lang)
|
||||
if babel_tag == 'skip':
|
||||
continue
|
||||
|
||||
try:
|
||||
|
||||
if babel_tag == 'lang_region':
|
||||
sxng_tag = locales.region_tag(babel.Locale.parse(eng_lang))
|
||||
engine_traits.custom['lang_region'][sxng_tag] = eng_lang
|
||||
continue
|
||||
|
||||
sxng_tag = locales.language_tag(babel.Locale.parse(babel_tag))
|
||||
|
||||
except babel.UnknownLocaleError:
|
||||
print("ERROR: language %s (%s) is unknown by babel" % (name, eng_lang))
|
||||
continue
|
||||
|
||||
conflict = engine_traits.languages.get(sxng_tag)
|
||||
if conflict:
|
||||
if conflict != eng_lang:
|
||||
print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_lang))
|
||||
continue
|
||||
engine_traits.languages[sxng_tag] = eng_lang
|
||||
|
|
|
|||
|
|
@ -1,23 +1,33 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""DuckDuckGo (Instant Answer API)
|
||||
"""
|
||||
DuckDuckGo Instant Answer API
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The `DDG-API <https://duckduckgo.com/api>`__ is no longer documented but from
|
||||
reverse engineering we can see that some services (e.g. instant answers) still
|
||||
in use from the DDG search engine.
|
||||
|
||||
As far we can say the *instant answers* API does not support languages, or at
|
||||
least we could not find out how language support should work. It seems that
|
||||
most of the features are based on English terms.
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from urllib.parse import urlencode, urlparse, urljoin
|
||||
from lxml import html
|
||||
|
||||
from searx.data import WIKIDATA_UNITS
|
||||
from searx.engines.duckduckgo import language_aliases
|
||||
from searx.engines.duckduckgo import ( # pylint: disable=unused-import
|
||||
fetch_traits,
|
||||
_fetch_supported_languages,
|
||||
supported_languages_url,
|
||||
)
|
||||
from searx.utils import extract_text, html_to_text, match_language, get_string_replaces_function
|
||||
from searx.utils import extract_text, html_to_text, get_string_replaces_function
|
||||
from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import logging
|
||||
|
||||
logger: logging.Logger
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": 'https://duckduckgo.com/',
|
||||
|
|
@ -38,7 +48,7 @@ replace_http_by_https = get_string_replaces_function({'http:': 'https:'})
|
|||
|
||||
|
||||
def is_broken_text(text):
|
||||
"""duckduckgo may return something like "<a href="xxxx">http://somewhere Related website<a/>"
|
||||
"""duckduckgo may return something like ``<a href="xxxx">http://somewhere Related website<a/>``
|
||||
|
||||
The href URL is broken, the "Related website" may contains some HTML.
|
||||
|
||||
|
|
@ -63,8 +73,6 @@ def result_to_text(text, htmlResult):
|
|||
|
||||
def request(query, params):
|
||||
params['url'] = URL.format(query=urlencode({'q': query}))
|
||||
language = match_language(params['language'], supported_languages, language_aliases)
|
||||
language = language.split('-')[0]
|
||||
return params
|
||||
|
||||
|
||||
|
|
@ -72,7 +80,7 @@ def response(resp):
|
|||
# pylint: disable=too-many-locals, too-many-branches, too-many-statements
|
||||
results = []
|
||||
|
||||
search_res = json.loads(resp.text)
|
||||
search_res = resp.json()
|
||||
|
||||
# search_res.get('Entity') possible values (not exhaustive) :
|
||||
# * continent / country / department / location / waterfall
|
||||
|
|
@ -236,7 +244,7 @@ def unit_to_str(unit):
|
|||
|
||||
|
||||
def area_to_str(area):
|
||||
"""parse {'unit': 'http://www.wikidata.org/entity/Q712226', 'amount': '+20.99'}"""
|
||||
"""parse ``{'unit': 'https://www.wikidata.org/entity/Q712226', 'amount': '+20.99'}``"""
|
||||
unit = unit_to_str(area.get('unit'))
|
||||
if unit is not None:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,27 +1,30 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""
|
||||
DuckDuckGo (Images)
|
||||
DuckDuckGo Images
|
||||
~~~~~~~~~~~~~~~~~
|
||||
"""
|
||||
|
||||
from json import loads
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlencode
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.engines.duckduckgo import get_region_code
|
||||
from searx.engines.duckduckgo import ( # pylint: disable=unused-import
|
||||
fetch_traits,
|
||||
_fetch_supported_languages,
|
||||
supported_languages_url,
|
||||
|
||||
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.duckduckgo import (
|
||||
get_ddg_lang,
|
||||
get_vqd,
|
||||
)
|
||||
from searx.network import get
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import logging
|
||||
|
||||
logger: logging.Logger
|
||||
|
||||
traits: EngineTraits
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": 'https://duckduckgo.com/',
|
||||
"wikidata_id": 'Q12805',
|
||||
"official_api_documentation": {
|
||||
'url': 'https://duckduckgo.com/api',
|
||||
'comment': 'but images are not supported',
|
||||
},
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'JSON (site requires js to get images)',
|
||||
|
|
@ -33,70 +36,64 @@ paging = True
|
|||
safesearch = True
|
||||
send_accept_language_header = True
|
||||
|
||||
# search-url
|
||||
images_url = 'https://duckduckgo.com/i.js?{query}&s={offset}&p={safesearch}&o=json&vqd={vqd}'
|
||||
site_url = 'https://duckduckgo.com/?{query}&iar=images&iax=1&ia=images'
|
||||
safesearch_cookies = {0: '-2', 1: None, 2: '1'}
|
||||
safesearch_args = {0: '1', 1: None, 2: '1'}
|
||||
|
||||
|
||||
# run query in site to get vqd number needed for requesting images
|
||||
# TODO: find a way to get this number without an extra request (is it a hash of the query?)
|
||||
def get_vqd(query, headers):
|
||||
query_url = site_url.format(query=urlencode({'q': query}))
|
||||
res = get(query_url, headers=headers)
|
||||
content = res.text
|
||||
if content.find('vqd=\'') == -1:
|
||||
raise SearxEngineAPIException('Request failed')
|
||||
vqd = content[content.find('vqd=\'') + 5 :]
|
||||
vqd = vqd[: vqd.find('\'')]
|
||||
return vqd
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
# to avoid running actual external requests when testing
|
||||
if 'is_test' not in params:
|
||||
vqd = get_vqd(query, params['headers'])
|
||||
else:
|
||||
vqd = '12345'
|
||||
|
||||
offset = (params['pageno'] - 1) * 50
|
||||
eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
|
||||
eng_lang = get_ddg_lang(traits, params['searxng_locale'])
|
||||
|
||||
safesearch = params['safesearch'] - 1
|
||||
args = {
|
||||
'q': query,
|
||||
'o': 'json',
|
||||
# 'u': 'bing',
|
||||
'l': eng_region,
|
||||
'vqd': get_vqd(query, params["headers"]),
|
||||
}
|
||||
|
||||
region_code = get_region_code(params['language'], lang_list=supported_languages)
|
||||
if region_code:
|
||||
params['url'] = images_url.format(
|
||||
query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd
|
||||
)
|
||||
else:
|
||||
params['url'] = images_url.format(query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
|
||||
if params['pageno'] > 1:
|
||||
args['s'] = (params['pageno'] - 1) * 100
|
||||
|
||||
params['cookies']['ad'] = eng_lang # zh_CN
|
||||
params['cookies']['ah'] = eng_region # "us-en,de-de"
|
||||
params['cookies']['l'] = eng_region # "hk-tzh"
|
||||
logger.debug("cookies: %s", params['cookies'])
|
||||
|
||||
safe_search = safesearch_cookies.get(params['safesearch'])
|
||||
if safe_search is not None:
|
||||
params['cookies']['p'] = safe_search # "-2", "1"
|
||||
safe_search = safesearch_args.get(params['safesearch'])
|
||||
if safe_search is not None:
|
||||
args['p'] = safe_search # "-1", "1"
|
||||
|
||||
args = urlencode(args)
|
||||
params['url'] = 'https://duckduckgo.com/i.js?{args}&f={f}'.format(args=args, f=',,,,,')
|
||||
|
||||
params['headers']['Accept'] = 'application/json, text/javascript, */*; q=0.01'
|
||||
params['headers']['Referer'] = 'https://duckduckgo.com/'
|
||||
params['headers']['X-Requested-With'] = 'XMLHttpRequest'
|
||||
logger.debug("headers: %s", params['headers'])
|
||||
|
||||
return params
|
||||
|
||||
|
||||
# get response from search-request
|
||||
def response(resp):
|
||||
results = []
|
||||
res_json = resp.json()
|
||||
|
||||
content = resp.text
|
||||
res_json = loads(content)
|
||||
|
||||
# parse results
|
||||
for result in res_json['results']:
|
||||
title = result['title']
|
||||
url = result['url']
|
||||
thumbnail = result['thumbnail']
|
||||
image = result['image']
|
||||
|
||||
# append result
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'title': title,
|
||||
'title': result['title'],
|
||||
'content': '',
|
||||
'thumbnail_src': thumbnail,
|
||||
'img_src': image,
|
||||
'url': url,
|
||||
'thumbnail_src': result['thumbnail'],
|
||||
'img_src': result['image'],
|
||||
'url': result['url'],
|
||||
'img_format': '%s x %s' % (result['width'], result['height']),
|
||||
'source': result['source'],
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,29 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""DuckDuckGo Weather"""
|
||||
"""
|
||||
DuckDuckGo Weather
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from json import loads
|
||||
from urllib.parse import quote
|
||||
|
||||
from datetime import datetime
|
||||
from flask_babel import gettext
|
||||
|
||||
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.duckduckgo import get_ddg_lang
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import logging
|
||||
|
||||
logger: logging.Logger
|
||||
|
||||
traits: EngineTraits
|
||||
|
||||
|
||||
about = {
|
||||
"website": 'https://duckduckgo.com/',
|
||||
"wikidata_id": 'Q12805',
|
||||
|
|
@ -17,9 +33,11 @@ about = {
|
|||
"results": "JSON",
|
||||
}
|
||||
|
||||
categories = ["others"]
|
||||
send_accept_language_header = True
|
||||
|
||||
url = "https://duckduckgo.com/js/spice/forecast/{query}/{lang}"
|
||||
# engine dependent config
|
||||
categories = ["others"]
|
||||
URL = "https://duckduckgo.com/js/spice/forecast/{query}/{lang}"
|
||||
|
||||
|
||||
def generate_condition_table(condition):
|
||||
|
|
@ -72,8 +90,17 @@ def generate_day_table(day):
|
|||
|
||||
|
||||
def request(query, params):
|
||||
params["url"] = url.format(query=quote(query), lang=params['language'].split('-')[0])
|
||||
|
||||
eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
|
||||
eng_lang = get_ddg_lang(traits, params['searxng_locale'])
|
||||
|
||||
# !ddw paris :es-AR --> {'ad': 'es_AR', 'ah': 'ar-es', 'l': 'ar-es'}
|
||||
params['cookies']['ad'] = eng_lang
|
||||
params['cookies']['ah'] = eng_region
|
||||
params['cookies']['l'] = eng_region
|
||||
logger.debug("cookies: %s", params['cookies'])
|
||||
|
||||
params["url"] = URL.format(query=quote(query), lang=eng_lang.split('_')[0])
|
||||
return params
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -49,10 +49,13 @@ sxng_locales = (
|
|||
('fr-CH', 'Français', 'Suisse', 'French', '\U0001f1e8\U0001f1ed'),
|
||||
('fr-FR', 'Français', 'France', 'French', '\U0001f1eb\U0001f1f7'),
|
||||
('he', 'עברית', '', 'Hebrew', '\U0001f1ee\U0001f1f7'),
|
||||
('hi', 'हिन्दी', '', 'Hindi', '\U0001f310'),
|
||||
('hr', 'Hrvatski', '', 'Croatian', '\U0001f310'),
|
||||
('hu', 'Magyar', '', 'Hungarian', '\U0001f310'),
|
||||
('hu-HU', 'Magyar', 'Magyarország', 'Hungarian', '\U0001f1ed\U0001f1fa'),
|
||||
('id', 'Indonesia', '', 'Indonesian', '\U0001f310'),
|
||||
('id-ID', 'Indonesia', 'Indonesia', 'Indonesian', '\U0001f1ee\U0001f1e9'),
|
||||
('is', 'Íslenska', '', 'Icelandic', '\U0001f310'),
|
||||
('it', 'Italiano', '', 'Italian', '\U0001f310'),
|
||||
('it-CH', 'Italiano', 'Svizzera', 'Italian', '\U0001f1e8\U0001f1ed'),
|
||||
('it-IT', 'Italiano', 'Italia', 'Italian', '\U0001f1ee\U0001f1f9'),
|
||||
|
|
@ -84,6 +87,8 @@ sxng_locales = (
|
|||
('th-TH', 'ไทย', 'ไทย', 'Thai', '\U0001f1f9\U0001f1ed'),
|
||||
('tr', 'Türkçe', '', 'Turkish', '\U0001f310'),
|
||||
('tr-TR', 'Türkçe', 'Türkiye', 'Turkish', '\U0001f1f9\U0001f1f7'),
|
||||
('uk', 'Українська', '', 'Ukrainian', '\U0001f310'),
|
||||
('vi', 'Tiếng Việt', '', 'Vietnamese', '\U0001f310'),
|
||||
('zh', '中文', '', 'Chinese', '\U0001f310'),
|
||||
('zh-CN', '中文', '中国', 'Chinese', '\U0001f1e8\U0001f1f3'),
|
||||
('zh-HK', '中文', '中國香港', 'Chinese', '\U0001f1ed\U0001f1f0'),
|
||||
|
|
|
|||
Loading…
Reference in a new issue