Merge branch 'master' of https://github.com/asciimoo/searx into filtron

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2020-02-23 13:50:22 +01:00
commit 4d67164918
7 changed files with 84 additions and 17 deletions

View file

@ -12,7 +12,7 @@ DOCS_URL = os.environ.get("DOCS_URL", "https://asciimoo.github.io/searx/")
# Project -------------------------------------------------------------- # Project --------------------------------------------------------------
project = u'searx' project = u'searx'
copyright = u'2015-2019, Adam Tauber, Noémi Ványi' copyright = u'2015-2020, Adam Tauber, Noémi Ványi'
author = u'Adam Tauber' author = u'Adam Tauber'
release, version = VERSION_STRING, VERSION_STRING release, version = VERSION_STRING, VERSION_STRING
highlight_language = 'none' highlight_language = 'none'

View file

@ -285,7 +285,7 @@ content becomes smart.
:rst:role:`pep` :pep:`8` ``:pep:`8``` :rst:role:`pep` :pep:`8` ``:pep:`8```
sphinx.ext.extlinks_ sphinx.ext.extlinks_
-------------------------------------------------------------------------------------------------- --------------------------------------------------------------------------------------------------
project's wiki article :wiki:`Searx-instances` ``:wiki:`Searx-instances``` project's wiki article :wiki:`Offline-engines` ``:wiki:`Offline-engines```
to docs public URL :docs:`dev/reST.html` ``:docs:`dev/reST.html``` to docs public URL :docs:`dev/reST.html` ``:docs:`dev/reST.html```
files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst``` files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst```
pull request :pull:`1756` ``:pull:`1756``` pull request :pull:`1756` ``:pull:`1756```

View file

@ -8,8 +8,8 @@ Searx is a free internet metasearch engine which aggregates results from more
than 70 search services. Users are neither tracked nor profiled. Additionally, than 70 search services. Users are neither tracked nor profiled. Additionally,
searx can be used over Tor for online anonymity. searx can be used over Tor for online anonymity.
Get started with searx by using one of the :wiki:`Searx-instances`. If you Get started with searx by using one of the Searx-instances_. If you don't trust
don't trust anyone, you can set up your own, see :ref:`installation`. anyone, you can set up your own, see :ref:`installation`.
.. sidebar:: Features .. sidebar:: Features
@ -32,3 +32,5 @@ don't trust anyone, you can set up your own, see :ref:`installation`.
dev/index dev/index
utils/index utils/index
blog/index blog/index
.. _Searx-instances: https://searx.space

54
searx/engines/etools.py Normal file
View file

@ -0,0 +1,54 @@
"""
eTools (Web)
@website https://www.etools.ch
@provide-api no
@using-api no
@results HTML
@stable no (HTML can change)
@parse url, title, content
"""
from lxml import html
from searx.engines.xpath import extract_text
from searx.url_utils import quote
from searx.utils import eval_xpath
categories = ['general']
paging = False
language_support = False
safesearch = True
base_url = 'https://www.etools.ch'
search_path = '/searchAdvancedSubmit.do'\
'?query={search_term}'\
'&pageResults=20'\
'&safeSearch={safesearch}'
def request(query, params):
if params['safesearch']:
safesearch = 'true'
else:
safesearch = 'false'
params['url'] = base_url + search_path.format(search_term=quote(query), safesearch=safesearch)
return params
def response(resp):
results = []
dom = html.fromstring(resp.text)
for result in eval_xpath(dom, '//table[@class="result"]//td[@class="record"]'):
url = eval_xpath(result, './a/@href')[0]
title = extract_text(eval_xpath(result, './a//text()'))
content = extract_text(eval_xpath(result, './/div[@class="text"]//text()'))
results.append({'url': url,
'title': title,
'content': content})
return results

View file

@ -4,6 +4,7 @@ from sys import version
from searx import settings, autocomplete from searx import settings, autocomplete
from searx.languages import language_codes as languages from searx.languages import language_codes as languages
from searx.utils import match_language
from searx.url_utils import parse_qs, urlencode from searx.url_utils import parse_qs, urlencode
if version[0] == '3': if version[0] == '3':
@ -11,7 +12,7 @@ if version[0] == '3':
COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years
LANGUAGE_CODES = [l[0].split('-')[0] for l in languages] LANGUAGE_CODES = [l[0] for l in languages]
LANGUAGE_CODES.append('all') LANGUAGE_CODES.append('all')
DISABLED = 0 DISABLED = 0
ENABLED = 1 ENABLED = 1
@ -132,6 +133,10 @@ class SetSetting(Setting):
class SearchLanguageSetting(EnumStringSetting): class SearchLanguageSetting(EnumStringSetting):
"""Available choices may change, so user's value may not be in choices anymore""" """Available choices may change, so user's value may not be in choices anymore"""
def _validate_selection(self, selection):
if not match_language(selection, self.choices, fallback=None) and selection != "":
raise ValidationException('Invalid language code: "{0}"'.format(selection))
def parse(self, data): def parse(self, data):
if data not in self.choices and data != self.value: if data not in self.choices and data != self.value:
# hack to give some backwards compatibility with old language cookies # hack to give some backwards compatibility with old language cookies
@ -268,7 +273,7 @@ class Preferences(object):
super(Preferences, self).__init__() super(Preferences, self).__init__()
self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories + ['none']), self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories + ['none']),
'language': SearchLanguageSetting(settings['ui']['default_locale'], 'language': SearchLanguageSetting(settings['search']['default_lang'],
choices=list(LANGUAGE_CODES) + ['']), choices=list(LANGUAGE_CODES) + ['']),
'locale': EnumStringSetting(settings['ui']['default_locale'], 'locale': EnumStringSetting(settings['ui']['default_locale'],
choices=list(settings['locales'].keys()) + ['']), choices=list(settings['locales'].keys()) + ['']),

View file

@ -5,6 +5,7 @@ general:
search: search:
safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
default_lang : "" # Default search language - leave blank to detect from browser information or use codes from 'languages.py'
ban_time_on_fail : 5 # ban time in seconds after engine errors ban_time_on_fail : 5 # ban time in seconds after engine errors
max_ban_time_on_fail : 120 # max ban time in seconds after engine errors max_ban_time_on_fail : 120 # max ban time in seconds after engine errors
@ -202,6 +203,11 @@ engines:
timeout: 3.0 timeout: 3.0
disabled : True disabled : True
- name : etools
engine : etools
shortcut : eto
disabled : True
- name : etymonline - name : etymonline
engine : xpath engine : xpath
paging : True paging : True

View file

@ -157,14 +157,16 @@ _category_names = (gettext('files'),
outgoing_proxies = settings['outgoing'].get('proxies') or None outgoing_proxies = settings['outgoing'].get('proxies') or None
def _get_browser_language(request, lang_list):
for lang in request.headers.get("Accept-Language", "en").split(","):
locale = match_language(lang, lang_list, fallback=None)
if locale is not None:
return locale
@babel.localeselector @babel.localeselector
def get_locale(): def get_locale():
locale = "en-US" locale = _get_browser_language(request, settings['locales'].keys())
for lang in request.headers.get("Accept-Language", locale).split(","):
locale = match_language(lang, settings['locales'].keys(), fallback=None)
if locale is not None:
break
logger.debug("default locale from browser info is `%s`", locale) logger.debug("default locale from browser info is `%s`", locale)
@ -372,8 +374,7 @@ def render(template_name, override_theme=None, **kwargs):
kwargs['language_codes'] = languages kwargs['language_codes'] = languages
if 'current_language' not in kwargs: if 'current_language' not in kwargs:
kwargs['current_language'] = match_language(request.preferences.get_value('language'), kwargs['current_language'] = match_language(request.preferences.get_value('language'),
LANGUAGE_CODES, LANGUAGE_CODES)
fallback=locale)
# override url_for function in templates # override url_for function in templates
kwargs['url_for'] = url_for_theme kwargs['url_for'] = url_for_theme
@ -444,11 +445,10 @@ def pre_request():
request.errors.append(gettext('Invalid settings')) request.errors.append(gettext('Invalid settings'))
# init search language and locale # init search language and locale
locale = get_locale()
if not preferences.get_value("language"): if not preferences.get_value("language"):
preferences.parse_dict({"language": locale}) preferences.parse_dict({"language": _get_browser_language(request, LANGUAGE_CODES)})
if not preferences.get_value("locale"): if not preferences.get_value("locale"):
preferences.parse_dict({"locale": locale}) preferences.parse_dict({"locale": get_locale()})
# request.user_plugins # request.user_plugins
request.user_plugins = [] request.user_plugins = []