From 89089370463b8d4c8b766d9ac9b59757e9328296 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 1 Jun 2021 13:34:41 +0200 Subject: [PATCH] [mod] searx.engines.load_engine return None instead of sys.exit(1) Loading an engine should not exit the application (*). Instead of exit, return None. (*) RuntimeError still exit the application: syntax error, etc... BTW: add documentation and normalize indentation (no functional change) Suggested-by: @dalf https://github.com/searxng/searxng/pull/116#issuecomment-851865627 Signed-off-by: Markus Heiser --- searx/engines/__init__.py | 87 ++++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 20 deletions(-) diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 1ce90d5ad..70a8ab025 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -1,10 +1,15 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # lint: pylint # pylint: disable=missing-function-docstring -""" -Engine loader: -call load_engines(settings['engines]) -to initialiaze categories, engines, engine_shortcuts +"""This module implements the engine loader. + +Load and initialize the ``engines``, see :py:func:`load_engines` and register +:py:obj:`engine_shortcuts`. + +usage:: + + load_engines( settings['engines'] ) + """ import sys @@ -20,8 +25,10 @@ from searx.utils import load_module, match_language, gen_useragent logger = logger.getChild('engines') ENGINE_DIR = dirname(realpath(__file__)) -BABEL_LANGS = [lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0] - for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())] +BABEL_LANGS = [ + lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0] + for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers()) +] ENGINE_DEFAULT_ARGS = { "engine_type": "online", "inactive": False, @@ -38,16 +45,45 @@ ENGINE_DEFAULT_ARGS = { "display_error_messages": True, "tokens": [], } +"""Defaults for the namespace of an engine module, see :py:func:`load_engine`""" + categories = {'general': []} engines = {} engine_shortcuts = {} +"""Simple map of registered *shortcuts* to name of the engine (or ``None``). +:: + + engine_shortcuts[engine.shortcut] = engine.name + +""" def load_engine(engine_data): + """Load engine from ``engine_data``. + + :param dict engine_data: Attributes from YAML ``settings:engines/`` + :return: initialized namespace of the ````. + + 1. create a namespace and load module of the ```` + 2. update namespace with the defaults from :py:obj:`ENGINE_DEFAULT_ARGS` + 3. update namespace with values from ``engine_data`` + + If engine *is active*, return namespace of the engine, otherwise return + ``None``. + + This function also returns ``None`` if initialization of the namespace fails + for one of the following reasons: + + - engine name contains underscore + - engine name is not lowercase + - required attribute is not set :py:func:`is_missing_required_attributes` + + """ + engine_name = engine_data['name'] if '_' in engine_name: logger.error('Engine name contains underscore: "{}"'.format(engine_name)) - sys.exit(1) + return None if engine_name.lower() != engine_name: logger.warn('Engine name is not lowercase: "{}", converting to lowercase'.format(engine_name)) @@ -69,10 +105,12 @@ def load_engine(engine_data): set_language_attributes(engine) update_attributes_for_tor(engine) - if is_missing_required_attributes(engine): - sys.exit(1) if not is_engine_active(engine): return None + + if is_missing_required_attributes(engine): + return None + return engine @@ -101,8 +139,11 @@ def set_language_attributes(engine): # find custom aliases for non standard language codes for engine_lang in engine.supported_languages: iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None) - if iso_lang and iso_lang != engine_lang and not engine_lang.startswith(iso_lang) and \ - iso_lang not in engine.supported_languages: + if (iso_lang + and iso_lang != engine_lang + and not engine_lang.startswith(iso_lang) + and iso_lang not in engine.supported_languages + ): engine.language_aliases[iso_lang] = engine_lang # language_support @@ -114,25 +155,30 @@ def set_language_attributes(engine): 'User-Agent': gen_useragent(), 'Accept-Language': 'ja-JP,ja;q=0.8,en-US;q=0.5,en;q=0.3', # bing needs a non-English language } - engine.fetch_supported_languages =\ - lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers)) + engine.fetch_supported_languages = ( + lambda: engine._fetch_supported_languages( + get(engine.supported_languages_url, headers=headers)) + ) def update_attributes_for_tor(engine): - if settings['outgoing'].get('using_tor_proxy') and hasattr(engine, 'onion_url'): + if (settings['outgoing'].get('using_tor_proxy') + and hasattr(engine, 'onion_url') ): engine.search_url = engine.onion_url + getattr(engine, 'search_path', '') engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0) def is_missing_required_attributes(engine): - """an attribute is required when its name doesn't start with '_'. - Required attributes must not be None + """An attribute is required when its name doesn't start with ``_`` (underline). + Required attributes must not be ``None``. + """ missing = False for engine_attr in dir(engine): if not engine_attr.startswith('_') and getattr(engine, engine_attr) is None: - logger.error('Missing engine config attribute: "{0}.{1}"' - .format(engine.name, engine_attr)) + logger.error( + 'Missing engine config attribute: "{0}.{1}"' + .format(engine.name, engine_attr)) missing = True return missing @@ -143,7 +189,8 @@ def is_engine_active(engine): return False # exclude onion engines if not using tor - if 'onions' in engine.categories and not settings['outgoing'].get('using_tor_proxy'): + if ('onions' in engine.categories + and not settings['outgoing'].get('using_tor_proxy') ): return False return True @@ -160,7 +207,7 @@ def register_engine(engine): def load_engines(engine_list): - """Use case: engine_list = settings['engines'] + """usage: ``engine_list = settings['engines']`` """ engines.clear() engine_shortcuts.clear()