From 1ed618222f17e205b684f6ef63cdaa470beb0dcb Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Mon, 17 Jan 2022 07:54:37 +0100 Subject: [PATCH 1/7] [typing] add ExtendedRequest webapp.py monkey-patches the Flask request global. This commit adds a type cast so that e.g. Pyright[1] doesn't show "Cannot access member" errors everywhere. [1]: https://github.com/microsoft/pyright --- searx/webapp.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/searx/webapp.py b/searx/webapp.py index 3df9f2876..905f53d1f 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -14,6 +14,8 @@ from datetime import datetime, timedelta from timeit import default_timer from html import escape from io import StringIO +import typing +from typing import List, Dict import urllib from urllib.parse import urlencode @@ -28,7 +30,6 @@ import flask from flask import ( Flask, - request, render_template, url_for, Response, @@ -89,7 +90,7 @@ from searx.utils import ( ) from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH from searx.query import RawTextQuery -from searx.plugins import plugins, initialize as plugin_initialize +from searx.plugins import Plugin, plugins, initialize as plugin_initialize from searx.plugins.oa_doi_rewrite import get_doi_resolver from searx.preferences import ( Preferences, @@ -224,6 +225,21 @@ exception_classname_to_text = { _flask_babel_get_translations = flask_babel.get_translations +class ExtendedRequest(flask.Request): + """This class is never initialized and only used for type checking.""" + + preferences: Preferences + errors: List[str] + user_plugins: List[Plugin] + form: Dict[str, str] + start_time: float + render_time: float + timings: List[dict] + + +request = typing.cast(ExtendedRequest, flask.request) + + def _get_translations(): if has_request_context() and request.form.get('use-translation') == 'oc': babel_ext = flask_babel.current_app.extensions['babel'] From fdf562bc32028cc0a2d3da1bad02a84da1e6a1d6 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Mon, 17 Jan 2022 08:06:31 +0100 Subject: [PATCH 2/7] [typing] add results.Timing --- searx/results.py | 19 +++++++++++-------- searx/webapp.py | 14 +++++++------- tests/unit/test_webapp.py | 6 +++++- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/searx/results.py b/searx/results.py index 6ab751c56..39e6e95d4 100644 --- a/searx/results.py +++ b/searx/results.py @@ -2,7 +2,9 @@ import re from collections import defaultdict from operator import itemgetter from threading import RLock +from typing import List, NamedTuple from urllib.parse import urlparse, unquote + from searx import logger from searx.engines import engines from searx.metrics import histogram_observe, counter_add, count_error @@ -137,6 +139,12 @@ def result_score(result): return sum((occurences * weight) / position for position in result['positions']) +class Timing(NamedTuple): + engine: str + total: float + load: float + + class ResultContainer: """docstring for ResultContainer""" @@ -169,7 +177,7 @@ class ResultContainer: self._closed = False self.paging = False self.unresponsive_engines = set() - self.timings = [] + self.timings: List[Timing] = [] self.redirect_url = None self.on_result = lambda _: True self._lock = RLock() @@ -405,13 +413,8 @@ class ResultContainer: if engines[engine_name].display_error_messages: self.unresponsive_engines.add((engine_name, error_type, error_message, suspended)) - def add_timing(self, engine_name, engine_time, page_load_time): - timing = { - 'engine': engines[engine_name].shortcut, - 'total': engine_time, - 'load': page_load_time, - } - self.timings.append(timing) + def add_timing(self, engine_name: str, engine_time: float, page_load_time: float): + self.timings.append(Timing(engine_name, total=engine_time, load=page_load_time)) def get_timings(self): return self.timings diff --git a/searx/webapp.py b/searx/webapp.py index 905f53d1f..fc851d44e 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -56,6 +56,7 @@ from searx import ( searx_debug, ) from searx.data import ENGINE_DESCRIPTIONS +from searx.results import Timing from searx.settings_defaults import OUTPUT_FORMATS from searx.settings_loader import get_default_settings_path from searx.exceptions import SearxParameterException @@ -234,7 +235,7 @@ class ExtendedRequest(flask.Request): form: Dict[str, str] start_time: float render_time: float - timings: List[dict] + timings: List[Timing] request = typing.cast(ExtendedRequest, flask.request) @@ -585,15 +586,14 @@ def post_request(response): 'render;dur=' + str(round(request.render_time * 1000, 3)), ] if len(request.timings) > 0: - timings = sorted(request.timings, key=lambda v: v['total']) + timings = sorted(request.timings, key=lambda t: t.total) timings_total = [ - 'total_' + str(i) + '_' + v['engine'] + ';dur=' + str(round(v['total'] * 1000, 3)) - for i, v in enumerate(timings) + 'total_' + str(i) + '_' + t.engine + ';dur=' + str(round(t.total * 1000, 3)) for i, t in enumerate(timings) ] timings_load = [ - 'load_' + str(i) + '_' + v['engine'] + ';dur=' + str(round(v['load'] * 1000, 3)) - for i, v in enumerate(timings) - if v.get('load') + 'load_' + str(i) + '_' + t.engine + ';dur=' + str(round(t.load * 1000, 3)) + for i, t in enumerate(timings) + if t.load ] timings_all = timings_all + timings_total + timings_load response.headers.add('Server-Timing', ', '.join(timings_all)) diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index 920a346a5..fd7c72e60 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -3,6 +3,7 @@ import json from urllib.parse import ParseResult from mock import Mock +from searx.results import Timing import searx.search.processors from searx.search import Search @@ -46,7 +47,10 @@ class ViewsTestCase(SearxTestCase): }, ] - timings = [{'engine': 'startpage', 'total': 0.8, 'load': 0.7}, {'engine': 'youtube', 'total': 0.9, 'load': 0.6}] + timings = [ + Timing(engine='startpage', total=0.8, load=0.7), + Timing(engine='youtube', total=0.9, load=0.6), + ] def search_mock(search_self, *args): search_self.result_container = Mock( From 6d43cf7952cb22dbf8c7d1851eeee1931f8604c3 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Mon, 17 Jan 2022 08:11:06 +0100 Subject: [PATCH 3/7] [typing] add optional attrs to Plugin --- searx/plugins/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 6c1bea8d0..3d431f329 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -10,7 +10,7 @@ from os.path import abspath, basename, dirname, exists, join from shutil import copyfile from pkgutil import iter_modules from logging import getLogger -from typing import List +from typing import List, Tuple from searx import logger, settings @@ -22,6 +22,9 @@ class Plugin: # pylint: disable=too-few-public-methods name: str description: str default_on: bool + js_dependencies: Tuple[str] + css_dependencies: Tuple[str] + preference_section: str logger = logger.getChild("plugins") From def62c3a47384b6672cb584cd586ea104b86754f Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Mon, 17 Jan 2022 09:04:51 +0100 Subject: [PATCH 4/7] [typing] add type hints for dictionaries --- searx/network/client.py | 4 +++- searx/network/network.py | 3 ++- searx/search/processors/__init__.py | 3 ++- searx/search/processors/abstract.py | 5 +++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/searx/network/client.py b/searx/network/client.py index 6858ac05b..11086dd33 100644 --- a/searx/network/client.py +++ b/searx/network/client.py @@ -4,7 +4,9 @@ import asyncio import logging +from ssl import SSLContext import threading +from typing import Any, Dict import httpx from httpx_socks import AsyncProxyTransport @@ -23,7 +25,7 @@ else: logger = logger.getChild('searx.network.client') LOOP = None -SSLCONTEXTS = {} +SSLCONTEXTS: Dict[Any, SSLContext] = {} TRANSPORT_KWARGS = { 'trust_env': False, } diff --git a/searx/network/network.py b/searx/network/network.py index 43140b44d..c9af97649 100644 --- a/searx/network/network.py +++ b/searx/network/network.py @@ -7,6 +7,7 @@ import atexit import asyncio import ipaddress from itertools import cycle +from typing import Dict import httpx @@ -16,7 +17,7 @@ from .client import new_client, get_loop, AsyncHTTPTransportNoHttp logger = logger.getChild('network') DEFAULT_NAME = '__DEFAULT__' -NETWORKS = {} +NETWORKS: Dict[str, 'Network'] = {} # requests compatibility when reading proxy settings from settings.yml PROXY_PATTERN_MAPPING = { 'http': 'http://', diff --git a/searx/search/processors/__init__.py b/searx/search/processors/__init__.py index 966b990ec..4e85527ba 100644 --- a/searx/search/processors/__init__.py +++ b/searx/search/processors/__init__.py @@ -15,6 +15,7 @@ __all__ = [ ] import threading +from typing import Dict from searx import logger from searx import engines @@ -26,7 +27,7 @@ from .online_currency import OnlineCurrencyProcessor from .abstract import EngineProcessor logger = logger.getChild('search.processors') -PROCESSORS = {} +PROCESSORS: Dict[str, EngineProcessor] = {} """Cache request processores, stored by *engine-name* (:py:func:`initialize`)""" diff --git a/searx/search/processors/abstract.py b/searx/search/processors/abstract.py index 732b55d52..b7703496b 100644 --- a/searx/search/processors/abstract.py +++ b/searx/search/processors/abstract.py @@ -8,6 +8,7 @@ import threading from abc import abstractmethod, ABC from timeit import default_timer +from typing import Dict, Union from searx import settings, logger from searx.engines import engines @@ -17,7 +18,7 @@ from searx.exceptions import SearxEngineAccessDeniedException, SearxEngineRespon from searx.utils import get_engine_from_settings logger = logger.getChild('searx.search.processor') -SUSPENDED_STATUS = {} +SUSPENDED_STATUS: Dict[Union[int, str], 'SuspendedStatus'] = {} class SuspendedStatus: @@ -61,7 +62,7 @@ class EngineProcessor(ABC): __slots__ = 'engine', 'engine_name', 'lock', 'suspended_status', 'logger' - def __init__(self, engine, engine_name): + def __init__(self, engine, engine_name: str): self.engine = engine self.engine_name = engine_name self.logger = engines[engine_name].logger From 0c6a09cae34997b2b5aba83bac50fabacafde59f Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Mon, 17 Jan 2022 10:56:14 +0100 Subject: [PATCH 5/7] [refactor] remove never used parameter --- searx/results.py | 4 ++-- searx/webapp.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/searx/results.py b/searx/results.py index 39e6e95d4..5e1b46ed5 100644 --- a/searx/results.py +++ b/searx/results.py @@ -409,9 +409,9 @@ class ResultContainer: return 0 return resultnum_sum / len(self._number_of_results) - def add_unresponsive_engine(self, engine_name, error_type, error_message=None, suspended=False): + def add_unresponsive_engine(self, engine_name, error_type, suspended=False): if engines[engine_name].display_error_messages: - self.unresponsive_engines.add((engine_name, error_type, error_message, suspended)) + self.unresponsive_engines.add((engine_name, error_type, suspended)) def add_timing(self, engine_name: str, engine_time: float, page_load_time: float): self.timings.append(Timing(engine_name, total=engine_time, load=page_load_time)) diff --git a/searx/webapp.py b/searx/webapp.py index fc851d44e..bb6e74b25 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -857,8 +857,6 @@ def __get_translated_errors(unresponsive_engines): error_user_text = exception_classname_to_text[None] error_msg = gettext(error_user_text) if unresponsive_engine[2]: - error_msg = "{} {}".format(error_msg, unresponsive_engine[2]) - if unresponsive_engine[3]: error_msg = gettext('Suspended') + ': ' + error_msg translated_errors.append((unresponsive_engine[0], error_msg)) From 193b0efd12a78172fec1fef03bfc2f7929c705a3 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Mon, 17 Jan 2022 11:11:39 +0100 Subject: [PATCH 6/7] [typing] add results.UnresponsiveEngine --- searx/results.py | 14 ++++++++++---- searx/webapp.py | 14 +++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/searx/results.py b/searx/results.py index 5e1b46ed5..1ac6a5ab5 100644 --- a/searx/results.py +++ b/searx/results.py @@ -2,7 +2,7 @@ import re from collections import defaultdict from operator import itemgetter from threading import RLock -from typing import List, NamedTuple +from typing import List, NamedTuple, Set from urllib.parse import urlparse, unquote from searx import logger @@ -145,6 +145,12 @@ class Timing(NamedTuple): load: float +class UnresponsiveEngine(NamedTuple): + engine: str + error_type: str + suspended: bool + + class ResultContainer: """docstring for ResultContainer""" @@ -176,7 +182,7 @@ class ResultContainer: self.engine_data = defaultdict(dict) self._closed = False self.paging = False - self.unresponsive_engines = set() + self.unresponsive_engines: Set[UnresponsiveEngine] = set() self.timings: List[Timing] = [] self.redirect_url = None self.on_result = lambda _: True @@ -409,9 +415,9 @@ class ResultContainer: return 0 return resultnum_sum / len(self._number_of_results) - def add_unresponsive_engine(self, engine_name, error_type, suspended=False): + def add_unresponsive_engine(self, engine_name: str, error_type: str, suspended: bool = False): if engines[engine_name].display_error_messages: - self.unresponsive_engines.add((engine_name, error_type, suspended)) + self.unresponsive_engines.add(UnresponsiveEngine(engine_name, error_type, suspended)) def add_timing(self, engine_name: str, engine_time: float, page_load_time: float): self.timings.append(Timing(engine_name, total=engine_time, load=page_load_time)) diff --git a/searx/webapp.py b/searx/webapp.py index bb6e74b25..196bfb0d2 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -15,7 +15,7 @@ from timeit import default_timer from html import escape from io import StringIO import typing -from typing import List, Dict +from typing import List, Dict, Iterable import urllib from urllib.parse import urlencode @@ -56,7 +56,7 @@ from searx import ( searx_debug, ) from searx.data import ENGINE_DESCRIPTIONS -from searx.results import Timing +from searx.results import Timing, UnresponsiveEngine from searx.settings_defaults import OUTPUT_FORMATS from searx.settings_loader import get_default_settings_path from searx.exceptions import SearxParameterException @@ -844,21 +844,21 @@ def search(): ) -def __get_translated_errors(unresponsive_engines): +def __get_translated_errors(unresponsive_engines: Iterable[UnresponsiveEngine]): translated_errors = [] # make a copy unresponsive_engines to avoid "RuntimeError: Set changed size # during iteration" it happens when an engine modifies the ResultContainer # after the search_multiple_requests method has stopped waiting - for unresponsive_engine in list(unresponsive_engines): - error_user_text = exception_classname_to_text.get(unresponsive_engine[1]) + for unresponsive_engine in unresponsive_engines: + error_user_text = exception_classname_to_text.get(unresponsive_engine.error_type) if not error_user_text: error_user_text = exception_classname_to_text[None] error_msg = gettext(error_user_text) - if unresponsive_engine[2]: + if unresponsive_engine.suspended: error_msg = gettext('Suspended') + ': ' + error_msg - translated_errors.append((unresponsive_engine[0], error_msg)) + translated_errors.append((unresponsive_engine.engine, error_msg)) return sorted(translated_errors, key=lambda e: e[0]) From 96655cbd4ecc061eaf592f948ae2cac4192850fc Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Mon, 17 Jan 2022 11:16:11 +0100 Subject: [PATCH 7/7] [typing] add type hints to webapp.py --- searx/webapp.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/searx/webapp.py b/searx/webapp.py index 196bfb0d2..f509fea24 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -18,6 +18,7 @@ import typing from typing import List, Dict, Iterable import urllib +import urllib.parse from urllib.parse import urlencode import httpx @@ -338,7 +339,7 @@ def code_highlighter(codelines, language=None): return html_code -def get_current_theme_name(override=None): +def get_current_theme_name(override: str = None) -> str: """Returns theme name. Checks in this order: @@ -354,14 +355,14 @@ def get_current_theme_name(override=None): return theme_name -def get_result_template(theme_name, template_name): +def get_result_template(theme_name: str, template_name: str): themed_path = theme_name + '/result_templates/' + template_name if themed_path in result_templates: return themed_path return 'result_templates/' + template_name -def url_for_theme(endpoint, override_theme=None, **values): +def url_for_theme(endpoint: str, override_theme: str = None, **values): if endpoint == 'static' and values.get('filename'): theme_name = get_current_theme_name(override=override_theme) filename_with_theme = "themes/{}/{}".format(theme_name, values['filename']) @@ -371,7 +372,7 @@ def url_for_theme(endpoint, override_theme=None, **values): return url -def proxify(url): +def proxify(url: str): if url.startswith('//'): url = 'https:' + url @@ -386,7 +387,7 @@ def proxify(url): return '{0}?{1}'.format(settings['result_proxy']['url'], urlencode(url_params)) -def image_proxify(url): +def image_proxify(url: str): if url.startswith('//'): url = 'https:' + url @@ -422,7 +423,7 @@ def get_translations(): } -def _get_enable_categories(all_categories): +def _get_enable_categories(all_categories: Iterable[str]): disabled_engines = request.preferences.engines.get_disabled() enabled_categories = set( # pylint: disable=consider-using-dict-items @@ -434,14 +435,14 @@ def _get_enable_categories(all_categories): return [x for x in all_categories if x in enabled_categories] -def get_pretty_url(parsed_url): +def get_pretty_url(parsed_url: urllib.parse.ParseResult): path = parsed_url.path path = path[:-1] if len(path) > 0 and path[-1] == '/' else path path = path.replace("/", " › ") return [parsed_url.scheme + "://" + parsed_url.netloc, path] -def render(template_name, override_theme=None, **kwargs): +def render(template_name: str, override_theme: str = None, **kwargs): # values from the HTTP requests kwargs['endpoint'] = 'results' if 'q' in kwargs else request.endpoint kwargs['cookies'] = request.cookies @@ -569,7 +570,7 @@ def pre_request(): @app.after_request -def add_default_headers(response): +def add_default_headers(response: flask.Response): # set default http headers for header, value in settings['server']['default_http_headers'].items(): if header in response.headers: @@ -579,7 +580,7 @@ def add_default_headers(response): @app.after_request -def post_request(response): +def post_request(response: flask.Response): total_time = default_timer() - request.start_time timings_all = [ 'total;dur=' + str(round(total_time * 1000, 3)), @@ -600,7 +601,7 @@ def post_request(response): return response -def index_error(output_format, error_message): +def index_error(output_format: str, error_message: str): if output_format == 'json': return Response(json.dumps({'error': error_message}), mimetype='application/json') if output_format == 'csv': @@ -1074,7 +1075,7 @@ def preferences(): ) -def _is_selected_language_supported(engine, preferences): # pylint: disable=redefined-outer-name +def _is_selected_language_supported(engine, preferences: Preferences): # pylint: disable=redefined-outer-name language = preferences.get_value('language') if language == 'all': return True