mirrors/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-29 22:21:02 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1861 from return42/fix-prefs

Browse source 
fix serious bugs of the test procedure
This commit is contained in:
Markus Heiser 2020-02-25 08:56:34 +01:00 committed by GitHub
commit 6a3ef5561b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 175 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Makefile
View file

@ -72,9 +72,12 @@ $(GH_PAGES)::
PHONY += test test.pylint test.pep8 test.unit test.robot PHONY += test test.pylint test.pep8 test.unit test.robot
test: test.pylint test.pep8 test.unit test.robot
# TODO: balance linting with pylint # TODO: balance linting with pylint
test: test.pep8 test.unit test.robot test.pylint: pyenvinstall
- make pylint $(call cmd,pylint,searx/preferences.py)
$(call cmd,pylint,searx/testing.py)
test.pep8: pyenvinstall test.pep8: pyenvinstall
$(PY_ENV_ACT); ./manage.sh pep8_check $(PY_ENV_ACT); ./manage.sh pep8_check

194
searx/preferences.py
View file

@ -1,3 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Searx preferences implementation.
"""
# pylint: disable=useless-object-inheritance
from base64 import urlsafe_b64encode, urlsafe_b64decode from base64 import urlsafe_b64encode, urlsafe_b64decode
from zlib import compress, decompress from zlib import compress, decompress
from sys import version from sys import version
@ -8,6 +14,7 @@ from searx.utils import match_language
from searx.url_utils import parse_qs, urlencode from searx.url_utils import parse_qs, urlencode
if version[0] == '3': if version[0] == '3':
# pylint: disable=invalid-name
unicode = str unicode = str
@ -20,11 +27,14 @@ DOI_RESOLVERS = list(settings['doi_resolvers'])
class MissingArgumentException(Exception): class MissingArgumentException(Exception):
pass """Exption from ``cls._post_init`` when a argument is missed.
"""
class ValidationException(Exception): class ValidationException(Exception):
pass
"""Exption from ``cls._post_init`` when configuration value is invalid.
"""
class Setting(object): class Setting(object):
@ -42,33 +52,45 @@ class Setting(object):
pass pass
def parse(self, data): def parse(self, data):
"""Parse ``data`` and store the result at ``self.value``
If needed, its overwritten in the inheritance.
"""
self.value = data self.value = data
def get_value(self): def get_value(self):
"""Returns the value of the setting
If needed, its overwritten in the inheritance.
"""
return self.value return self.value
def save(self, name, resp): def save(self, name, resp):
"""Save cookie ``name`` in the HTTP reponse obect
If needed, its overwritten in the inheritance."""
resp.set_cookie(name, self.value, max_age=COOKIE_MAX_AGE) resp.set_cookie(name, self.value, max_age=COOKIE_MAX_AGE)
class StringSetting(Setting): class StringSetting(Setting):
"""Setting of plain string values""" """Setting of plain string values"""
pass
class EnumStringSetting(Setting): class EnumStringSetting(Setting):
"""Setting of a value which can only come from the given choices""" """Setting of a value which can only come from the given choices"""
def _validate_selection(self, selection):
if selection not in self.choices:
raise ValidationException('Invalid value: "{0}"'.format(selection))
def _post_init(self): def _post_init(self):
if not hasattr(self, 'choices'): if not hasattr(self, 'choices'):
raise MissingArgumentException('Missing argument: choices') raise MissingArgumentException('Missing argument: choices')
self._validate_selection(self.value) self._validate_selection(self.value)
def _validate_selection(self, selection):
if selection not in self.choices: # pylint: disable=no-member
raise ValidationException('Invalid value: "{0}"'.format(selection))
def parse(self, data): def parse(self, data):
"""Parse and validate ``data`` and store the result at ``self.value``
"""
self._validate_selection(data) self._validate_selection(data)
self.value = data self.value = data
@ -78,7 +100,7 @@ class MultipleChoiceSetting(EnumStringSetting):
def _validate_selections(self, selections): def _validate_selections(self, selections):
for item in selections: for item in selections:
if item not in self.choices: if item not in self.choices: # pylint: disable=no-member
raise ValidationException('Invalid value: "{0}"'.format(selections)) raise ValidationException('Invalid value: "{0}"'.format(selections))
def _post_init(self): def _post_init(self):
@ -87,6 +109,8 @@ class MultipleChoiceSetting(EnumStringSetting):
self._validate_selections(self.value) self._validate_selections(self.value)
def parse(self, data): def parse(self, data):
"""Parse and validate ``data`` and store the result at ``self.value``
"""
if data == '': if data == '':
self.value = [] self.value = []
return return
@ -95,38 +119,47 @@ class MultipleChoiceSetting(EnumStringSetting):
self._validate_selections(elements) self._validate_selections(elements)
self.value = elements self.value = elements
def parse_form(self, data): def parse_form(self, data): # pylint: disable=missing-function-docstring
self.value = [] self.value = []
for choice in data: for choice in data:
if choice in self.choices and choice not in self.value: if choice in self.choices and choice not in self.value: # pylint: disable=no-member
self.value.append(choice) self.value.append(choice)
def save(self, name, resp): def save(self, name, resp):
"""Save cookie ``name`` in the HTTP reponse obect
"""
resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE) resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE)
class SetSetting(Setting): class SetSetting(Setting):
"""Setting of values of type ``set`` (comma separated string) """
def _post_init(self): def _post_init(self):
if not hasattr(self, 'values'): if not hasattr(self, 'values'):
self.values = set() self.values = set()
def get_value(self): def get_value(self):
"""Returns a string with comma separated values.
"""
return ','.join(self.values) return ','.join(self.values)
def parse(self, data): def parse(self, data):
"""Parse and validate ``data`` and store the result at ``self.value``
"""
if data == '': if data == '':
self.values = set() self.values = set() # pylint: disable=attribute-defined-outside-init
return return
elements = data.split(',') elements = data.split(',')
for element in elements: for element in elements:
self.values.add(element) self.values.add(element)
def parse_form(self, data): def parse_form(self, data): # pylint: disable=missing-function-docstring
elements = data.split(',') elements = data.split(',')
self.values = set(elements) self.values = set(elements) # pylint: disable=attribute-defined-outside-init
def save(self, name, resp): def save(self, name, resp):
"""Save cookie ``name`` in the HTTP reponse obect
"""
resp.set_cookie(name, ','.join(self.values), max_age=COOKIE_MAX_AGE) resp.set_cookie(name, ','.join(self.values), max_age=COOKIE_MAX_AGE)
@ -134,14 +167,19 @@ class SearchLanguageSetting(EnumStringSetting):
"""Available choices may change, so user's value may not be in choices anymore""" """Available choices may change, so user's value may not be in choices anymore"""
def _validate_selection(self, selection): def _validate_selection(self, selection):
if not match_language(selection, self.choices, fallback=None) and selection != "": if selection != "" and not match_language(
# pylint: disable=no-member
selection, self.choices, fallback=None):
raise ValidationException('Invalid language code: "{0}"'.format(selection)) raise ValidationException('Invalid language code: "{0}"'.format(selection))
def parse(self, data): def parse(self, data):
if data not in self.choices and data != self.value: """Parse and validate ``data`` and store the result at ``self.value``
"""
if data not in self.choices and data != self.value: # pylint: disable=no-member
# hack to give some backwards compatibility with old language cookies # hack to give some backwards compatibility with old language cookies
data = str(data).replace('_', '-') data = str(data).replace('_', '-')
lang = data.split('-')[0] lang = data.split('-')[0]
# pylint: disable=no-member
if data in self.choices: if data in self.choices:
pass pass
elif lang in self.choices: elif lang in self.choices:
@ -157,16 +195,21 @@ class MapSetting(Setting):
def _post_init(self): def _post_init(self):
if not hasattr(self, 'map'): if not hasattr(self, 'map'):
raise MissingArgumentException('missing argument: map') raise MissingArgumentException('missing argument: map')
if self.value not in self.map.values(): if self.value not in self.map.values(): # pylint: disable=no-member
raise ValidationException('Invalid default value') raise ValidationException('Invalid default value')
def parse(self, data): def parse(self, data):
"""Parse and validate ``data`` and store the result at ``self.value``
"""
# pylint: disable=no-member
if data not in self.map: if data not in self.map:
raise ValidationException('Invalid choice: {0}'.format(data)) raise ValidationException('Invalid choice: {0}'.format(data))
self.value = self.map[data] self.value = self.map[data]
self.key = data self.key = data # pylint: disable=attribute-defined-outside-init
def save(self, name, resp): def save(self, name, resp):
"""Save cookie ``name`` in the HTTP reponse obect
"""
if hasattr(self, 'key'): if hasattr(self, 'key'):
resp.set_cookie(name, self.key, max_age=COOKIE_MAX_AGE) resp.set_cookie(name, self.key, max_age=COOKIE_MAX_AGE)
@ -180,24 +223,26 @@ class SwitchableSetting(Setting):
if not hasattr(self, 'choices'): if not hasattr(self, 'choices'):
raise MissingArgumentException('missing argument: choices') raise MissingArgumentException('missing argument: choices')
def transform_form_items(self, items): def transform_form_items(self, items): # pylint: disable=missing-function-docstring
# pylint: disable=no-self-use
return items return items
def transform_values(self, values): def transform_values(self, values): # pylint: disable=missing-function-docstring
# pylint: disable=no-self-use
return values return values
def parse_cookie(self, data): def parse_cookie(self, data): # pylint: disable=missing-function-docstring
# pylint: disable=attribute-defined-outside-init
if data[DISABLED] != '': if data[DISABLED] != '':
self.disabled = set(data[DISABLED].split(',')) self.disabled = set(data[DISABLED].split(','))
if data[ENABLED] != '': if data[ENABLED] != '':
self.enabled = set(data[ENABLED].split(',')) self.enabled = set(data[ENABLED].split(','))
def parse_form(self, items): def parse_form(self, items): # pylint: disable=missing-function-docstring
items = self.transform_form_items(items) items = self.transform_form_items(items)
self.disabled = set() # pylint: disable=attribute-defined-outside-init
self.disabled = set() self.enabled = set() # pylint: disable=attribute-defined-outside-init
self.enabled = set() for choice in self.choices: # pylint: disable=no-member
for choice in self.choices:
if choice['default_on']: if choice['default_on']:
if choice['id'] in items: if choice['id'] in items:
self.disabled.add(choice['id']) self.disabled.add(choice['id'])
@ -205,31 +250,34 @@ class SwitchableSetting(Setting):
if choice['id'] not in items: if choice['id'] not in items:
self.enabled.add(choice['id']) self.enabled.add(choice['id'])
def save(self, resp): def save(self, resp): # pylint: disable=arguments-differ
"""Save cookie in the HTTP reponse obect
"""
resp.set_cookie('disabled_{0}'.format(self.value), ','.join(self.disabled), max_age=COOKIE_MAX_AGE) resp.set_cookie('disabled_{0}'.format(self.value), ','.join(self.disabled), max_age=COOKIE_MAX_AGE)
resp.set_cookie('enabled_{0}'.format(self.value), ','.join(self.enabled), max_age=COOKIE_MAX_AGE) resp.set_cookie('enabled_{0}'.format(self.value), ','.join(self.enabled), max_age=COOKIE_MAX_AGE)
def get_disabled(self): def get_disabled(self): # pylint: disable=missing-function-docstring
disabled = self.disabled disabled = self.disabled
for choice in self.choices: for choice in self.choices: # pylint: disable=no-member
if not choice['default_on'] and choice['id'] not in self.enabled: if not choice['default_on'] and choice['id'] not in self.enabled:
disabled.add(choice['id']) disabled.add(choice['id'])
return self.transform_values(disabled) return self.transform_values(disabled)
def get_enabled(self): def get_enabled(self): # pylint: disable=missing-function-docstring
enabled = self.enabled enabled = self.enabled
for choice in self.choices: for choice in self.choices: # pylint: disable=no-member
if choice['default_on'] and choice['id'] not in self.disabled: if choice['default_on'] and choice['id'] not in self.disabled:
enabled.add(choice['id']) enabled.add(choice['id'])
return self.transform_values(enabled) return self.transform_values(enabled)
class EnginesSetting(SwitchableSetting): class EnginesSetting(SwitchableSetting):
"""Engine settings"""
def _post_init(self): def _post_init(self):
super(EnginesSetting, self)._post_init() super(EnginesSetting, self)._post_init()
transformed_choices = [] transformed_choices = []
for engine_name, engine in self.choices.items(): for engine_name, engine in self.choices.items(): # pylint: disable=no-member,access-member-before-definition
for category in engine.categories: for category in engine.categories:
transformed_choice = dict() transformed_choice = dict()
transformed_choice['default_on'] = not engine.disabled transformed_choice['default_on'] = not engine.disabled
@ -251,11 +299,12 @@ class EnginesSetting(SwitchableSetting):
class PluginsSetting(SwitchableSetting): class PluginsSetting(SwitchableSetting):
"""Plugin settings"""
def _post_init(self): def _post_init(self):
super(PluginsSetting, self)._post_init() super(PluginsSetting, self)._post_init()
transformed_choices = [] transformed_choices = []
for plugin in self.choices: for plugin in self.choices: # pylint: disable=access-member-before-definition
transformed_choice = dict() transformed_choice = dict()
transformed_choice['default_on'] = plugin.default_on transformed_choice['default_on'] = plugin.default_on
transformed_choice['id'] = plugin.id transformed_choice['id'] = plugin.id
@ -272,29 +321,60 @@ class Preferences(object):
def __init__(self, themes, categories, engines, plugins): def __init__(self, themes, categories, engines, plugins):
super(Preferences, self).__init__() super(Preferences, self).__init__()
self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories + ['none']), self.key_value_settings = {
'language': SearchLanguageSetting(settings['search']['default_lang'], 'categories': MultipleChoiceSetting(
choices=list(LANGUAGE_CODES) + ['']), ['general'], choices=categories + ['none']
'locale': EnumStringSetting(settings['ui']['default_locale'], ),
choices=list(settings['locales'].keys()) + ['']), 'language': SearchLanguageSetting(
'autocomplete': EnumStringSetting(settings['search']['autocomplete'], settings['search'].get('default_lang', ''),
choices=list(autocomplete.backends.keys()) + ['']), choices=list(LANGUAGE_CODES) + ['']
'image_proxy': MapSetting(settings['server']['image_proxy'], ),
map={'': settings['server']['image_proxy'], 'locale': EnumStringSetting(
settings['ui'].get('default_locale', ''),
choices=list(settings['locales'].keys()) + ['']
),
'autocomplete': EnumStringSetting(
settings['search'].get('autocomplete', ''),
choices=list(autocomplete.backends.keys()) + ['']
),
'image_proxy': MapSetting(
settings['server'].get('image_proxy', False),
map={
'': settings['server'].get('image_proxy', 0),
'0': False, '0': False,
'1': True, '1': True,
'True': True, 'True': True,
'False': False}), 'False': False
'method': EnumStringSetting('POST', choices=('GET', 'POST')), }
'safesearch': MapSetting(settings['search']['safe_search'], map={'0': 0, ),
'method': EnumStringSetting(
'POST',
choices=('GET', 'POST')
),
'safesearch': MapSetting(
settings['search'].get('safe_search', 0),
map={
'0': 0,
'1': 1, '1': 1,
'2': 2}), '2': 2
'theme': EnumStringSetting(settings['ui']['default_theme'], choices=themes), }
'results_on_new_tab': MapSetting(False, map={'0': False, ),
'theme': EnumStringSetting(
settings['ui'].get('default_theme', 'oscar'),
choices=themes
),
'results_on_new_tab': MapSetting(
False,
map={
'0': False,
'1': True, '1': True,
'False': False, 'False': False,
'True': True}), 'True': True
'doi_resolver': MultipleChoiceSetting(['oadoi.org'], choices=DOI_RESOLVERS), }
),
'doi_resolver': MultipleChoiceSetting(
['oadoi.org'], choices=DOI_RESOLVERS
),
'oscar-style': EnumStringSetting( 'oscar-style': EnumStringSetting(
settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'), settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'),
choices=['', 'logicodev', 'logicodev-dark', 'pointhi']), choices=['', 'logicodev', 'logicodev-dark', 'pointhi']),
@ -306,6 +386,7 @@ class Preferences(object):
self.unknown_params = {} self.unknown_params = {}
def get_as_url_params(self): def get_as_url_params(self):
"""Return preferences as URL parameters"""
settings_kv = {} settings_kv = {}
for k, v in self.key_value_settings.items(): for k, v in self.key_value_settings.items():
if isinstance(v, MultipleChoiceSetting): if isinstance(v, MultipleChoiceSetting):
@ -324,6 +405,7 @@ class Preferences(object):
return urlsafe_b64encode(compress(urlencode(settings_kv).encode('utf-8'))).decode('utf-8') return urlsafe_b64encode(compress(urlencode(settings_kv).encode('utf-8'))).decode('utf-8')
def parse_encoded_data(self, input_data): def parse_encoded_data(self, input_data):
"""parse (base64) preferences from request (``flask.request.form['preferences']``)"""
decoded_data = decompress(urlsafe_b64decode(input_data.encode('utf-8'))) decoded_data = decompress(urlsafe_b64decode(input_data.encode('utf-8')))
dict_data = {} dict_data = {}
for x, y in parse_qs(decoded_data).items(): for x, y in parse_qs(decoded_data).items():
@ -331,6 +413,7 @@ class Preferences(object):
self.parse_dict(dict_data) self.parse_dict(dict_data)
def parse_dict(self, input_data): def parse_dict(self, input_data):
"""parse preferences from request (``flask.request.form``)"""
for user_setting_name, user_setting in input_data.items(): for user_setting_name, user_setting in input_data.items():
if user_setting_name in self.key_value_settings: if user_setting_name in self.key_value_settings:
self.key_value_settings[user_setting_name].parse(user_setting) self.key_value_settings[user_setting_name].parse(user_setting)
@ -351,6 +434,7 @@ class Preferences(object):
self.unknown_params[user_setting_name] = user_setting self.unknown_params[user_setting_name] = user_setting
def parse_form(self, input_data): def parse_form(self, input_data):
"""Parse formular (``<input>``) data from a ``flask.request.form``"""
disabled_engines = [] disabled_engines = []
enabled_categories = [] enabled_categories = []
disabled_plugins = [] disabled_plugins = []
@ -373,12 +457,18 @@ class Preferences(object):
# cannot be used in case of engines or plugins # cannot be used in case of engines or plugins
def get_value(self, user_setting_name): def get_value(self, user_setting_name):
"""Returns the value for ``user_setting_name``
"""
ret_val = None
if user_setting_name in self.key_value_settings: if user_setting_name in self.key_value_settings:
return self.key_value_settings[user_setting_name].get_value() ret_val = self.key_value_settings[user_setting_name].get_value()
if user_setting_name in self.unknown_params: if user_setting_name in self.unknown_params:
return self.unknown_params[user_setting_name] ret_val = self.unknown_params[user_setting_name]
return ret_val
def save(self, resp): def save(self, resp):
"""Save cookie in the HTTP reponse obect
"""
for user_setting_name, user_setting in self.key_value_settings.items(): for user_setting_name, user_setting in self.key_value_settings.items():
user_setting.save(user_setting_name, resp) user_setting.save(user_setting_name, resp)
self.engines.save(resp) self.engines.save(resp)
@ -388,7 +478,7 @@ class Preferences(object):
resp.set_cookie(k, v, max_age=COOKIE_MAX_AGE) resp.set_cookie(k, v, max_age=COOKIE_MAX_AGE)
return resp return resp
def validate_token(self, engine): def validate_token(self, engine): # pylint: disable=missing-function-docstring
valid = True valid = True
if hasattr(engine, 'tokens') and engine.tokens: if hasattr(engine, 'tokens') and engine.tokens:
valid = False valid = False

4
searx/settings_robot.yml
View file

@ -3,8 +3,6 @@ general:
instance_name : "searx_test" instance_name : "searx_test"
search: search:
safe_search : 0
autocomplete : ""
language: "all" language: "all"
server: server:
@ -12,14 +10,12 @@ server:
bind_address : 127.0.0.1 bind_address : 127.0.0.1
secret_key : "ultrasecretkey" # change this! secret_key : "ultrasecretkey" # change this!
base_url : False base_url : False
image_proxy : False
http_protocol_version : "1.0" http_protocol_version : "1.0"
ui: ui:
static_path : "" static_path : ""
templates_path : "" templates_path : ""
default_theme : oscar default_theme : oscar
default_locale : ""
outgoing: outgoing:
request_timeout : 1.0 # seconds request_timeout : 1.0 # seconds

29
searx/testing.py
View file

@ -1,12 +1,14 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Shared testing code.""" """Shared testing code."""
# pylint: disable=missing-function-docstring
import os import os
import subprocess import subprocess
import traceback import traceback
from os.path import dirname, join, abspath from os.path import dirname, join, abspath, realpath
from splinter import Browser from splinter import Browser
from unittest2 import TestCase from unittest2 import TestCase
@ -17,21 +19,21 @@ class SearxTestLayer:
__name__ = u'SearxTestLayer' __name__ = u'SearxTestLayer'
@classmethod
def setUp(cls): def setUp(cls):
pass pass
setUp = classmethod(setUp)
@classmethod
def tearDown(cls): def tearDown(cls):
pass pass
tearDown = classmethod(tearDown)
@classmethod
def testSetUp(cls): def testSetUp(cls):
pass pass
testSetUp = classmethod(testSetUp)
@classmethod
def testTearDown(cls): def testTearDown(cls):
pass pass
testTearDown = classmethod(testTearDown)
class SearxRobotLayer(): class SearxRobotLayer():
@ -41,14 +43,19 @@ class SearxRobotLayer():
os.setpgrp() # create new process group, become its leader os.setpgrp() # create new process group, become its leader
# get program paths # get program paths
webapp = os.path.join( webapp = join(abspath(dirname(realpath(__file__))), 'webapp.py')
os.path.abspath(os.path.dirname(os.path.realpath(__file__))),
'webapp.py'
)
exe = 'python' exe = 'python'
# The Flask app is started by Flask.run(...), don't enable Flask's debug
# mode, the debugger from Flask will cause wired process model, where
# the server never dies. Further read:
#
# - debug mode: https://flask.palletsprojects.com/quickstart/#debug-mode
# - Flask.run(..): https://flask.palletsprojects.com/api/#flask.Flask.run
os.environ['SEARX_DEBUG'] = '0'
# set robot settings path # set robot settings path
os.environ['SEARX_DEBUG'] = '1'
os.environ['SEARX_SETTINGS_PATH'] = abspath( os.environ['SEARX_SETTINGS_PATH'] = abspath(
dirname(__file__) + '/settings_robot.yml') dirname(__file__) + '/settings_robot.yml')
@ -105,7 +112,7 @@ if __name__ == '__main__':
try: try:
test_layer.setUp() test_layer.setUp()
run_robot_tests([getattr(robot, x) for x in dir(robot) if x.startswith('test_')]) run_robot_tests([getattr(robot, x) for x in dir(robot) if x.startswith('test_')])
except Exception: except Exception: # pylint: disable=broad-except
errors = True errors = True
print('Error occured: {0}'.format(traceback.format_exc())) print('Error occured: {0}'.format(traceback.format_exc()))
test_layer.tearDown() test_layer.tearDown()