Merge pull request #2291 from dalf/settings2

[enh] user settings can relied on the default settings
This commit is contained in:
Alexandre Flament 2020-12-01 14:57:12 +01:00 committed by GitHub
commit a1e6bc4cee
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 499 additions and 28 deletions

View file

@ -206,3 +206,80 @@ Engine settings
A few more options are possible, but they are pretty specific to some
engines, and so won't be described here.
.. _settings location:
settings.yml location
=====================
First, searx will try to load settings.yml from these locations:
1. the full path specified in the ``SEARX_SETTINGS_PATH`` environment variable.
2. ``/etc/searx/settings.yml``
If these files don't exist (or are empty or can't be read), searx uses the :origin:`searx/settings.yml` file.
.. _ settings use_default_settings:
use_default_settings
====================
.. note::
If searx is cloned from a git repository, most probably there is no need to have an user settings.
The user defined settings.yml can relied on the default configuration :origin:`searx/settings.yml` using ``use_default_settings: True``.
In the following example, the actual settings are the default settings defined in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and the ``bind_address``:
.. code-block:: yaml
use_default_settings: True
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
bind_address: "0.0.0.0"
With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.
In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:
.. code-block:: yaml
use_default_settings: True
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
engines:
- name: arch linux wiki
tokens: ['$ecretValue']
It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:
.. code-block:: yaml
use_default_settings:
engines:
remove:
- google
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
engines:
- name: arch linux wiki
tokens: ['$ecretValue']
As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
.. code-block:: yaml
use_default_settings:
engines:
keep_only:
- google
- duckduckgo
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
engines:
- name: google
tokens: ['$ecretValue']
- name: duckduckgo
tokens: ['$ecretValue']

View file

@ -7,6 +7,8 @@ enabled engines on their instances. It might be because they do not want to
expose some private information through an offline engine. Or they
would rather share engines only with their trusted friends or colleagues.
.. _private engines:
Private engines
===============

View file

@ -16,39 +16,15 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
'''
import logging
import searx.settings_loader
from os import environ
from os.path import realpath, dirname, join, abspath, isfile
from io import open
from yaml import safe_load
searx_dir = abspath(dirname(__file__))
engine_dir = dirname(realpath(__file__))
static_path = abspath(join(dirname(__file__), 'static'))
def check_settings_yml(file_name):
if isfile(file_name):
return file_name
else:
return None
# find location of settings.yml
if 'SEARX_SETTINGS_PATH' in environ:
# if possible set path to settings using the
# enviroment variable SEARX_SETTINGS_PATH
settings_path = check_settings_yml(environ['SEARX_SETTINGS_PATH'])
else:
# if not, get it from searx code base or last solution from /etc/searx
settings_path = check_settings_yml(join(searx_dir, 'settings.yml')) or check_settings_yml('/etc/searx/settings.yml')
if not settings_path:
raise Exception('settings.yml not found')
# load settings
with open(settings_path, 'r', encoding='utf-8') as settings_yaml:
settings = safe_load(settings_yaml)
settings, settings_load_message = searx.settings_loader.load_settings()
if settings['ui']['static_path']:
static_path = settings['ui']['static_path']
@ -58,7 +34,6 @@ enable debug if
the environnement variable SEARX_DEBUG is 1 or true
(whatever the value in settings.yml)
or general.debug=True in settings.yml
disable debug if
the environnement variable SEARX_DEBUG is 0 or false
(whatever the value in settings.yml)
@ -78,7 +53,7 @@ else:
logging.basicConfig(level=logging.WARNING)
logger = logging.getLogger('searx')
logger.debug('read configuration from %s', settings_path)
logger.info(settings_load_message)
logger.info('Initialisation done')
if 'SEARX_SECRET' in environ:

View file

@ -31,3 +31,11 @@ class SearxParameterException(SearxException):
self.message = message
self.parameter_name = name
self.parameter_value = value
class SearxSettingsException(SearxException):
def __init__(self, message, filename):
super().__init__(message)
self.message = message
self.filename = filename

129
searx/settings_loader.py Normal file
View file

@ -0,0 +1,129 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
from os import environ
from os.path import dirname, join, abspath, isfile
from collections.abc import Mapping
from itertools import filterfalse
import yaml
from searx.exceptions import SearxSettingsException
searx_dir = abspath(dirname(__file__))
def check_settings_yml(file_name):
if isfile(file_name):
return file_name
return None
def load_yaml(file_name):
try:
with open(file_name, 'r', encoding='utf-8') as settings_yaml:
return yaml.safe_load(settings_yaml)
except IOError as e:
raise SearxSettingsException(e, file_name)
except yaml.YAMLError as e:
raise SearxSettingsException(e, file_name)
def get_default_settings_path():
return check_settings_yml(join(searx_dir, 'settings.yml'))
def get_user_settings_path():
# find location of settings.yml
if 'SEARX_SETTINGS_PATH' in environ:
# if possible set path to settings using the
# enviroment variable SEARX_SETTINGS_PATH
return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
# if not, get it from searx code base or last solution from /etc/searx
return check_settings_yml('/etc/searx/settings.yml')
def update_dict(default_dict, user_dict):
for k, v in user_dict.items():
if isinstance(v, Mapping):
default_dict[k] = update_dict(default_dict.get(k, {}), v)
else:
default_dict[k] = v
return default_dict
def update_settings(default_settings, user_settings):
# merge everything except the engines
for k, v in user_settings.items():
if k not in ('use_default_settings', 'engines'):
update_dict(default_settings[k], v)
# parse the engines
remove_engines = None
keep_only_engines = None
use_default_settings = user_settings.get('use_default_settings')
if isinstance(use_default_settings, dict):
remove_engines = use_default_settings.get('engines', {}).get('remove')
keep_only_engines = use_default_settings.get('engines', {}).get('keep_only')
if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None:
engines = default_settings['engines']
# parse "use_default_settings.engines.remove"
if remove_engines is not None:
engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines))
# parse "use_default_settings.engines.keep_only"
if keep_only_engines is not None:
engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines))
# parse "engines"
user_engines = user_settings.get('engines')
if user_engines:
engines_dict = dict((definition['name'], definition) for definition in engines)
for user_engine in user_engines:
default_engine = engines_dict.get(user_engine['name'])
if default_engine:
update_dict(default_engine, user_engine)
else:
engines.append(user_engine)
# store the result
default_settings['engines'] = engines
return default_settings
def is_use_default_settings(user_settings):
use_default_settings = user_settings.get('use_default_settings')
if use_default_settings is True:
return True
if isinstance(use_default_settings, dict):
return True
if use_default_settings is False or use_default_settings is None:
return False
raise ValueError('Invalid value for use_default_settings')
def load_settings(load_user_setttings=True):
default_settings_path = get_default_settings_path()
user_settings_path = get_user_settings_path()
if user_settings_path is None or not load_user_setttings:
# no user settings
return (load_yaml(default_settings_path),
'load the default settings from {}'.format(default_settings_path))
# user settings
user_settings = load_yaml(user_settings_path)
if is_use_default_settings(user_settings):
# the user settings are merged with the default configuration
default_settings = load_yaml(default_settings_path)
update_settings(default_settings, user_settings)
return (default_settings,
'merge the default settings ( {} ) and the user setttings ( {} )'
.format(default_settings_path, user_settings_path))
# the user settings, fully replace the default configuration
return (user_settings,
'load the user settings from {}'.format(user_settings_path))

View file

View file

@ -0,0 +1,2 @@
Test:
**********

View file

@ -0,0 +1,111 @@
general:
debug : False
instance_name : "searx"
search:
safe_search : 0
autocomplete : ""
default_lang : ""
ban_time_on_fail : 5
max_ban_time_on_fail : 120
server:
port : 9000
bind_address : "0.0.0.0"
secret_key : "user_settings_secret"
base_url : False
image_proxy : False
http_protocol_version : "1.0"
method: "POST"
default_http_headers:
X-Content-Type-Options : nosniff
X-XSS-Protection : 1; mode=block
X-Download-Options : noopen
X-Robots-Tag : noindex, nofollow
Referrer-Policy : no-referrer
ui:
static_path : ""
templates_path : ""
default_theme : oscar
default_locale : ""
theme_args :
oscar_style : logicodev
engines:
- name : wikidata
engine : wikidata
shortcut : wd
timeout : 3.0
weight : 2
- name : wikibooks
engine : mediawiki
shortcut : wb
categories : general
base_url : "https://{language}.wikibooks.org/"
number_of_results : 5
search_type : text
- name : wikinews
engine : mediawiki
shortcut : wn
categories : news
base_url : "https://{language}.wikinews.org/"
number_of_results : 5
search_type : text
- name : wikiquote
engine : mediawiki
shortcut : wq
categories : general
base_url : "https://{language}.wikiquote.org/"
number_of_results : 5
search_type : text
locales:
en : English
ar : العَرَبِيَّة (Arabic)
bg : Български (Bulgarian)
bo : བོད་སྐད་ (Tibetian)
ca : Català (Catalan)
cs : Čeština (Czech)
cy : Cymraeg (Welsh)
da : Dansk (Danish)
de : Deutsch (German)
el_GR : Ελληνικά (Greek_Greece)
eo : Esperanto (Esperanto)
es : Español (Spanish)
et : Eesti (Estonian)
eu : Euskara (Basque)
fa_IR : (fārsī) فارسى (Persian)
fi : Suomi (Finnish)
fil : Wikang Filipino (Filipino)
fr : Français (French)
gl : Galego (Galician)
he : עברית (Hebrew)
hr : Hrvatski (Croatian)
hu : Magyar (Hungarian)
ia : Interlingua (Interlingua)
it : Italiano (Italian)
ja : 日本語 (Japanese)
lt : Lietuvių (Lithuanian)
nl : Nederlands (Dutch)
nl_BE : Vlaams (Dutch_Belgium)
oc : Lenga D'òc (Occitan)
pl : Polski (Polish)
pt : Português (Portuguese)
pt_BR : Português (Portuguese_Brazil)
ro : Română (Romanian)
ru : Русский (Russian)
sk : Slovenčina (Slovak)
sl : Slovenski (Slovene)
sr : српски (Serbian)
sv : Svenska (Swedish)
te : తెలుగు (telugu)
ta : தமிழ் (Tamil)
tr : Türkçe (Turkish)
uk : українська мова (Ukrainian)
vi : tiếng việt (Vietnamese)
zh : 中文 (Chinese)
zh_TW : 國語 (Taiwanese Mandarin)

View file

@ -0,0 +1,14 @@
use_default_settings:
engines:
keep_only:
- wikibooks
- wikinews
server:
secret_key: "user_secret_key"
bind_address: "0.0.0.0"
default_http_headers:
Custom-Header: Custom-Value
engines:
- name: wikipedia
- name: newengine
engine: dummy

View file

@ -0,0 +1,10 @@
use_default_settings:
engines:
remove:
- wikibooks
- wikinews
server:
secret_key: "user_secret_key"
bind_address: "0.0.0.0"
default_http_headers:
Custom-Header: Custom-Value

View file

@ -0,0 +1,15 @@
use_default_settings:
engines:
remove:
- wikibooks
- wikinews
server:
secret_key: "user_secret_key"
bind_address: "0.0.0.0"
default_http_headers:
Custom-Header: Custom-Value
engines:
- name: wikipedia
tokens: ['secret_token']
- name: newengine
engine: dummy

View file

@ -0,0 +1,6 @@
use_default_settings: True
server:
secret_key: "user_secret_key"
bind_address: "0.0.0.0"
default_http_headers:
Custom-Header: Custom-Value

View file

@ -0,0 +1,122 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
from os.path import dirname, join, abspath
from unittest.mock import patch
from searx.testing import SearxTestCase
from searx.exceptions import SearxSettingsException
from searx import settings_loader
test_dir = abspath(dirname(__file__))
class TestLoad(SearxTestCase):
def test_load_zero(self):
with self.assertRaises(SearxSettingsException):
settings_loader.load_yaml('/dev/zero')
with self.assertRaises(SearxSettingsException):
settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml'))
with self.assertRaises(SearxSettingsException):
settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml'))
def test_check_settings_yml(self):
self.assertIsNone(settings_loader.check_settings_yml('/dev/zero'))
bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml')
self.assertEqual(settings_loader.check_settings_yml(bad_settings_path), bad_settings_path)
class TestDefaultSettings(SearxTestCase):
def test_load(self):
settings, msg = settings_loader.load_settings(load_user_setttings=False)
self.assertTrue(msg.startswith('load the default settings from'))
self.assertFalse(settings['general']['debug'])
self.assertTrue(isinstance(settings['general']['instance_name'], str))
self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
self.assertTrue(isinstance(settings['server']['port'], int))
self.assertTrue(isinstance(settings['server']['bind_address'], str))
self.assertTrue(isinstance(settings['engines'], list))
self.assertTrue(isinstance(settings['locales'], dict))
self.assertTrue(isinstance(settings['doi_resolvers'], dict))
self.assertTrue(isinstance(settings['default_doi_resolver'], str))
class TestUserSettings(SearxTestCase):
def test_is_use_default_settings(self):
self.assertFalse(settings_loader.is_use_default_settings({}))
self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': True}))
self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': {}}))
with self.assertRaises(ValueError):
self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 1}))
with self.assertRaises(ValueError):
self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0}))
def test_user_settings_not_found(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': '/dev/null'}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('load the default settings from'))
self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
def test_user_settings(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('merge the default settings'))
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
def test_user_settings_remove(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('merge the default settings'))
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
engine_names = [engine['name'] for engine in settings['engines']]
self.assertNotIn('wikinews', engine_names)
self.assertNotIn('wikibooks', engine_names)
self.assertIn('wikipedia', engine_names)
def test_user_settings_remove2(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('merge the default settings'))
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
engine_names = [engine['name'] for engine in settings['engines']]
self.assertNotIn('wikinews', engine_names)
self.assertNotIn('wikibooks', engine_names)
self.assertIn('wikipedia', engine_names)
wikipedia = list(filter(lambda engine: (engine.get('name')) == 'wikipedia', settings['engines']))
self.assertEqual(wikipedia[0]['engine'], 'wikipedia')
self.assertEqual(wikipedia[0]['tokens'], ['secret_token'])
newengine = list(filter(lambda engine: (engine.get('name')) == 'newengine', settings['engines']))
self.assertEqual(newengine[0]['engine'], 'dummy')
def test_user_settings_keep_only(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('merge the default settings'))
engine_names = [engine['name'] for engine in settings['engines']]
self.assertEqual(engine_names, ['wikibooks', 'wikinews', 'wikipedia', 'newengine'])
# wikipedia has been removed, then added again with the "engine" section of user_settings_keep_only.yml
self.assertEqual(len(settings['engines'][2]), 1)
def test_custom_settings(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('load the user settings from'))
self.assertEqual(settings['server']['port'], 9000)
self.assertEqual(settings['server']['secret_key'], "user_settings_secret")
engine_names = [engine['name'] for engine in settings['engines']]
self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote'])