From 3a63dfbdd7ff53c2e083915b0dea481a6ea0aaaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 9 Dec 2020 17:33:18 +0100 Subject: [PATCH] display if an engine does not support https Closes #302 --- searx/engines/__init__.py | 32 +++++++++++++++++++++++++ searx/engines/acgsou.py | 4 ++-- searx/engines/arxiv.py | 2 +- searx/engines/currency_convert.py | 1 + searx/engines/dictzone.py | 1 + searx/engines/translated.py | 1 + searx/templates/oscar/macros.html | 4 ++-- searx/templates/oscar/preferences.html | 4 ++-- searx/templates/simple/macros.html | 4 ++-- searx/templates/simple/preferences.html | 4 ++-- 10 files changed, 46 insertions(+), 11 deletions(-) diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index ddd6a7feb..0b77f2a95 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -20,6 +20,7 @@ import sys import threading from os.path import realpath, dirname from babel.localedata import locale_identifiers +from urllib.parse import urlparse from flask_babel import gettext from operator import itemgetter from searx import settings @@ -289,3 +290,34 @@ def initialize_engines(engine_list): if init_fn: logger.debug('%s engine: Starting background initialization', engine_name) threading.Thread(target=engine_init, args=(engine_name, init_fn)).start() + + _set_https_support_for_engine(engine) + + +def _set_https_support_for_engine(engine): + # check HTTPS support if it is not disabled + if not engine.offline and not hasattr(engine, 'https_support'): + params = engine.request('http_test', { + 'method': 'GET', + 'headers': {}, + 'data': {}, + 'url': '', + 'cookies': {}, + 'verify': True, + 'auth': None, + 'pageno': 1, + 'time_range': None, + 'language': '', + 'safesearch': False, + 'is_test': True, + 'category': 'files', + 'raise_for_status': True, + }) + + if 'url' not in params: + return + + parsed_url = urlparse(params['url']) + https_support = parsed_url.scheme == 'https' + + setattr(engine, 'https_support', https_support) diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py index b8b367c24..637443edc 100644 --- a/searx/engines/acgsou.py +++ b/searx/engines/acgsou.py @@ -18,7 +18,7 @@ categories = ['files', 'images', 'videos', 'music'] paging = True # search-url -base_url = 'http://www.acgsou.com/' +base_url = 'https://www.acgsou.com/' search_url = base_url + 'search.php?{query}&page={offset}' # xpath queries xpath_results = '//table[contains(@class, "list_style table_fixed")]//tr[not(th)]' @@ -40,7 +40,7 @@ def response(resp): for result in eval_xpath_list(dom, xpath_results): # defaults filesize = 0 - magnet_link = "magnet:?xt=urn:btih:{}&tr=http://tracker.acgsou.com:2710/announce" + magnet_link = "magnet:?xt=urn:btih:{}&tr=https://tracker.acgsou.com:2710/announce" category = extract_text(eval_xpath_getindex(result, xpath_category, 0, default=[])) page_a = eval_xpath_getindex(result, xpath_title, 0) diff --git a/searx/engines/arxiv.py b/searx/engines/arxiv.py index c702c5987..1190de363 100644 --- a/searx/engines/arxiv.py +++ b/searx/engines/arxiv.py @@ -19,7 +19,7 @@ from searx.utils import eval_xpath_list, eval_xpath_getindex categories = ['science'] paging = True -base_url = 'http://export.arxiv.org/api/query?search_query=all:'\ +base_url = 'https://export.arxiv.org/api/query?search_query=all:'\ + '{query}&start={offset}&max_results={number_of_results}' # engine dependent config diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py index f41c135b9..87e21d0af 100644 --- a/searx/engines/currency_convert.py +++ b/searx/engines/currency_convert.py @@ -9,6 +9,7 @@ url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}' weight = 100 parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I) +https_support = True def normalize_name(name): diff --git a/searx/engines/dictzone.py b/searx/engines/dictzone.py index 5a1fea3cf..727eb6598 100644 --- a/searx/engines/dictzone.py +++ b/searx/engines/dictzone.py @@ -20,6 +20,7 @@ weight = 100 parser_re = re.compile('.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I) results_xpath = './/table[@id="r"]/tr' +https_support = True def request(query, params): diff --git a/searx/engines/translated.py b/searx/engines/translated.py index a50e7c830..75b8b5f42 100644 --- a/searx/engines/translated.py +++ b/searx/engines/translated.py @@ -15,6 +15,7 @@ categories = ['general'] url = 'https://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}' web_url = 'https://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}' weight = 100 +https_support = True parser_re = re.compile('.*?([a-z]+)-([a-z]+) (.{2,})$', re.I) api_key = '' diff --git a/searx/templates/oscar/macros.html b/searx/templates/oscar/macros.html index 2bc1e7805..f40eebd37 100644 --- a/searx/templates/oscar/macros.html +++ b/searx/templates/oscar/macros.html @@ -1,6 +1,6 @@ -{% macro icon(action) -%} - +{% macro icon(action, alt) -%} + {%- endmacro %} diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html index bc688dade..fc20b8ca5 100644 --- a/searx/templates/oscar/preferences.html +++ b/searx/templates/oscar/preferences.html @@ -230,8 +230,8 @@ {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in disabled_engines) }} - {{ search_engine.name }} - {{ shortcuts[search_engine.name] }} + {% if not search_engine.https_support %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }} + {{ shortcuts[search_engine.name] }} {{ support_toggle(stats[search_engine.name].supports_selected_language) }} {{ support_toggle(search_engine.safesearch==True) }} {{ support_toggle(search_engine.time_range_support==True) }} diff --git a/searx/templates/simple/macros.html b/searx/templates/simple/macros.html index cacbbec9f..1eb42667a 100644 --- a/searx/templates/simple/macros.html +++ b/searx/templates/simple/macros.html @@ -1,6 +1,6 @@ -{% macro icon(action) -%} - +{% macro icon(action, alt) -%} + {%- endmacro %} {% macro icon_small(action) -%} diff --git a/searx/templates/simple/preferences.html b/searx/templates/simple/preferences.html index d68e4be5f..f091a97cf 100644 --- a/searx/templates/simple/preferences.html +++ b/searx/templates/simple/preferences.html @@ -1,4 +1,4 @@ -{% from 'simple/macros.html' import tabs_open, tabs_close, tab_header, tab_footer, checkbox_onoff, checkbox %} +{% from 'simple/macros.html' import icon, tabs_open, tabs_close, tab_header, tab_footer, checkbox_onoff, checkbox %} {% extends "simple/base.html" %} @@ -121,7 +121,7 @@ {% set engine_id = 'engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_') %} {{ checkbox_onoff(engine_id, (search_engine.name, categ) in disabled_engines) }} - {{ search_engine.name }} + {% if not search_engine.https_support %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }} {{ shortcuts[search_engine.name] }} {{ checkbox(engine_id + '_supported_languages', current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages, true, true) }} {{ checkbox(engine_id + '_safesearch', search_engine.safesearch==True, true, true) }}