From f0684a5bb5860c2b9caffefb47dc55781092819e Mon Sep 17 00:00:00 2001 From: piplongrun Date: Wed, 12 Feb 2020 23:58:50 +0100 Subject: [PATCH 1/4] Add eTools engine --- searx/engines/etools.py | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 searx/engines/etools.py diff --git a/searx/engines/etools.py b/searx/engines/etools.py new file mode 100644 index 000000000..a9eb0980d --- /dev/null +++ b/searx/engines/etools.py @@ -0,0 +1,54 @@ +""" + eTools (Web) + + @website https://www.etools.ch + @provide-api no + @using-api no + @results HTML + @stable no (HTML can change) + @parse url, title, content +""" + +from lxml import html +from searx.engines.xpath import extract_text +from searx.url_utils import quote +from searx.utils import eval_xpath + +categories = ['general'] +paging = False +language_support = False +safesearch = True + +base_url = 'https://www.etools.ch' +search_path = '/searchAdvancedSubmit.do'\ + '?query={search_term}'\ + '&pageResults=20'\ + '&safeSearch={safesearch}' + + +def request(query, params): + if params['safesearch']: + safesearch = 'true' + else: + safesearch = 'false' + + params['url'] = base_url + search_path.format(search_term=quote(query), safesearch=safesearch) + + return params + + +def response(resp): + results = [] + + dom = html.fromstring(resp.text) + + for result in eval_xpath(dom, '//table[@class="result"]//td[@class="record"]'): + url = eval_xpath(result, './a/@href')[0] + title = extract_text(eval_xpath(result, './a//text()')) + content = extract_text(eval_xpath(result, './/div[@class="text"]//text()')) + + results.append({'url': url, + 'title': title, + 'content': content}) + + return results From e1b60106b7dab37fe29cb68c9b625c5fe52c1385 Mon Sep 17 00:00:00 2001 From: piplongrun Date: Wed, 12 Feb 2020 23:59:18 +0100 Subject: [PATCH 2/4] Add etools engine to the list --- searx/settings.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/searx/settings.yml b/searx/settings.yml index ab3de3951..08ff8dc62 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -202,6 +202,11 @@ engines: timeout: 3.0 disabled : True + - name : etools + engine : etools + shortcut : eto + disabled : True + - name : etymonline engine : xpath paging : True From a9d184473f8da473918ad02ca1b3346420b296fc Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 20 Feb 2020 18:11:34 +0100 Subject: [PATCH 3/4] docs: fix dead Searx-instance links to https://searx.space Related to #1853 / BTW update year of copyright note Signed-off-by: Markus Heiser --- docs/conf.py | 2 +- docs/dev/reST.rst | 2 +- docs/index.rst | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index cd61b697e..ebcd102b2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,7 +11,7 @@ DOCS_URL = os.environ.get("DOCS_URL", "https://asciimoo.github.io/searx/") # Project -------------------------------------------------------------- project = u'searx' -copyright = u'2015-2019, Adam Tauber, Noémi Ványi' +copyright = u'2015-2020, Adam Tauber, Noémi Ványi' author = u'Adam Tauber' release, version = VERSION_STRING, VERSION_STRING highlight_language = 'none' diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index 4dc1279f0..4835cbbcf 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -285,7 +285,7 @@ content becomes smart. :rst:role:`pep` :pep:`8` ``:pep:`8``` sphinx.ext.extlinks_ -------------------------------------------------------------------------------------------------- - project's wiki article :wiki:`Searx-instances` ``:wiki:`Searx-instances``` + project's wiki article :wiki:`Offline-engines` ``:wiki:`Offline-engines``` to docs public URL :docs:`dev/reST.html` ``:docs:`dev/reST.html``` files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst``` pull request :pull:`1756` ``:pull:`1756``` diff --git a/docs/index.rst b/docs/index.rst index d9503fef6..b62f4dc5d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,8 +20,8 @@ Searx is a free internet metasearch engine which aggregates results from more than 70 search services. Users are neither tracked nor profiled. Additionally, searx can be used over Tor for online anonymity. -Get started with searx by using one of the :wiki:`Searx-instances`. If you -don't trust anyone, you can set up your own, see :ref:`installation`. +Get started with searx by using one of the Searx-instances_. If you don't trust +anyone, you can set up your own, see :ref:`installation`. .. toctree:: :maxdepth: 2 @@ -30,3 +30,5 @@ don't trust anyone, you can set up your own, see :ref:`installation`. admin/index dev/index blog/index + +.. _Searx-instances: https://searx.space From c0006cadf70f0ee457d881bbd4ec40e30b0acd8f Mon Sep 17 00:00:00 2001 From: Marc Abonce Seguin Date: Sun, 23 Feb 2020 02:03:42 -0700 Subject: [PATCH 4/4] fix default locale and language issues --- searx/preferences.py | 9 +++++++-- searx/settings.yml | 1 + searx/webapp.py | 22 +++++++++++----------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/searx/preferences.py b/searx/preferences.py index 37d4f13a8..6e31aa187 100644 --- a/searx/preferences.py +++ b/searx/preferences.py @@ -4,6 +4,7 @@ from sys import version from searx import settings, autocomplete from searx.languages import language_codes as languages +from searx.utils import match_language from searx.url_utils import parse_qs, urlencode if version[0] == '3': @@ -11,7 +12,7 @@ if version[0] == '3': COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years -LANGUAGE_CODES = [l[0].split('-')[0] for l in languages] +LANGUAGE_CODES = [l[0] for l in languages] LANGUAGE_CODES.append('all') DISABLED = 0 ENABLED = 1 @@ -132,6 +133,10 @@ class SetSetting(Setting): class SearchLanguageSetting(EnumStringSetting): """Available choices may change, so user's value may not be in choices anymore""" + def _validate_selection(self, selection): + if not match_language(selection, self.choices, fallback=None) and selection != "": + raise ValidationException('Invalid language code: "{0}"'.format(selection)) + def parse(self, data): if data not in self.choices and data != self.value: # hack to give some backwards compatibility with old language cookies @@ -268,7 +273,7 @@ class Preferences(object): super(Preferences, self).__init__() self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories + ['none']), - 'language': SearchLanguageSetting(settings['ui']['default_locale'], + 'language': SearchLanguageSetting(settings['search']['default_lang'], choices=list(LANGUAGE_CODES) + ['']), 'locale': EnumStringSetting(settings['ui']['default_locale'], choices=list(settings['locales'].keys()) + ['']), diff --git a/searx/settings.yml b/searx/settings.yml index ab3de3951..65f42af8c 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -5,6 +5,7 @@ general: search: safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default + default_lang : "" # Default search language - leave blank to detect from browser information or use codes from 'languages.py' ban_time_on_fail : 5 # ban time in seconds after engine errors max_ban_time_on_fail : 120 # max ban time in seconds after engine errors diff --git a/searx/webapp.py b/searx/webapp.py index a856c07dd..8712cc3c1 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -157,14 +157,16 @@ _category_names = (gettext('files'), outgoing_proxies = settings['outgoing'].get('proxies') or None +def _get_browser_language(request, lang_list): + for lang in request.headers.get("Accept-Language", "en").split(","): + locale = match_language(lang, lang_list, fallback=None) + if locale is not None: + return locale + + @babel.localeselector def get_locale(): - locale = "en-US" - - for lang in request.headers.get("Accept-Language", locale).split(","): - locale = match_language(lang, settings['locales'].keys(), fallback=None) - if locale is not None: - break + locale = _get_browser_language(request, settings['locales'].keys()) logger.debug("default locale from browser info is `%s`", locale) @@ -372,8 +374,7 @@ def render(template_name, override_theme=None, **kwargs): kwargs['language_codes'] = languages if 'current_language' not in kwargs: kwargs['current_language'] = match_language(request.preferences.get_value('language'), - LANGUAGE_CODES, - fallback=locale) + LANGUAGE_CODES) # override url_for function in templates kwargs['url_for'] = url_for_theme @@ -444,11 +445,10 @@ def pre_request(): request.errors.append(gettext('Invalid settings')) # init search language and locale - locale = get_locale() if not preferences.get_value("language"): - preferences.parse_dict({"language": locale}) + preferences.parse_dict({"language": _get_browser_language(request, LANGUAGE_CODES)}) if not preferences.get_value("locale"): - preferences.parse_dict({"locale": locale}) + preferences.parse_dict({"locale": get_locale()}) # request.user_plugins request.user_plugins = []