From fe691a09888f6e6c6cc647de06f4eca0feb3451f Mon Sep 17 00:00:00 2001 From: Noemi Vanyi Date: Fri, 8 Apr 2016 16:38:05 +0200 Subject: [PATCH] new preferences handling Preferences class was introduced in order to handle user preferences. Right now it parses cookies and the form in preferences. Also it can retrieve settings based on the name of the setting. ATTENTION Please note that engine preferences are handled differently from now on. So it introduces incompatible changes. Every user who has saved preferences should reset and save his/her settings again. This change was needed, because everytime a default disabled engine was added saved user preferences would broke. Now engine setting tracking is fixed. --- searx/preferences.py | 269 ++++++++++++++++++++++++++++++++++++++ searx/search.py | 15 +-- searx/settings_robot.yml | 2 +- searx/utils.py | 23 ---- searx/webapp.py | 167 +++++------------------ tests/unit/test_webapp.py | 6 +- 6 files changed, 315 insertions(+), 167 deletions(-) create mode 100644 searx/preferences.py diff --git a/searx/preferences.py b/searx/preferences.py new file mode 100644 index 000000000..4cb83ef0c --- /dev/null +++ b/searx/preferences.py @@ -0,0 +1,269 @@ +from searx import settings, autocomplete +from searx.languages import language_codes as languages + + +COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years +LANGUAGE_CODES = [l[0] for l in languages] +LANGUAGE_CODES.append('all') +DISABLED = 0 +ENABLED = 1 + + +class MissingArgumentException(Exception): + pass + + +class ValidationException(Exception): + pass + + +class Setting(object): + """Base class of user settings""" + + def __init__(self, default_value, **kwargs): + super(Setting, self).__init__() + self.value = default_value + for key, value in kwargs.iteritems(): + setattr(self, key, value) + + self._post_init() + + def _post_init(self): + pass + + def parse(self, data): + self.value = data + + def get_value(self): + return self.value + + def save(self, name, resp): + resp.set_cookie(name, bytes(self.value), max_age=COOKIE_MAX_AGE) + + +class StringSetting(Setting): + """Setting of plain string values""" + pass + + +class EnumStringSetting(Setting): + """Setting of a value which can only come from the given choices""" + + def _post_init(self): + if not hasattr(self, 'choices'): + raise MissingArgumentException('Missing argument: choices') + + if self.value != '' and self.value not in self.choices: + raise ValidationException('Invalid default value: {0}'.format(self.value)) + + def parse(self, data): + if data not in self.choices and data != self.value: + raise ValidationException('Invalid choice: {0}'.format(data)) + self.value = data + + +class MultipleChoiceSetting(EnumStringSetting): + """Setting of values which can only come from the given choices""" + + def _post_init(self): + if not hasattr(self, 'choices'): + raise MissingArgumentException('Missing argument: choices') + for item in self.value: + if item not in self.choices: + raise ValidationException('Invalid default value: {0}'.format(self.value)) + + def parse(self, data): + if data == '': + self.value = [] + return + + elements = data.split(',') + for item in elements: + if item not in self.choices: + raise ValidationException('Invalid choice: {0}'.format(item)) + self.value = elements + + def parse_form(self, data): + self.value = [] + for choice in data: + if choice in self.choices and choice not in self.value: + self.value.append(choice) + + def save(self, name, resp): + resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE) + + +class MapSetting(Setting): + """Setting of a value that has to be translated in order to be storable""" + + def _post_init(self): + if not hasattr(self, 'map'): + raise MissingArgumentException('missing argument: map') + if self.value not in self.map.values(): + raise ValidationException('Invalid default value') + + def parse(self, data): + if data not in self.map: + raise ValidationException('Invalid choice: {0}'.format(data)) + self.value = self.map[data] + self.key = data + + def save(self, name, resp): + resp.set_cookie(name, bytes(self.key), max_age=COOKIE_MAX_AGE) + + +class SwitchableSetting(Setting): + """ Base class for settings that can be turned on && off""" + + def _post_init(self): + self.disabled = set() + self.enabled = set() + if not hasattr(self, 'choices'): + raise MissingArgumentException('missing argument: choices') + + def transform_form_items(self, items): + return items + + def transform_values(self, values): + return values + + def parse_cookie(self, data): + if data[DISABLED] != '': + self.disabled = set(data[DISABLED].split(',')) + if data[ENABLED] != '': + self.enabled = set(data[ENABLED].split(',')) + + def parse_form(self, items): + items = self.transform_form_items(items) + + self.disabled = set() + self.enabled = set() + for choice in self.choices: + if choice['default_on']: + if choice['id'] in items: + self.disabled.add(choice['id']) + else: + if choice['id'] not in items: + self.enabled.add(choice['id']) + + def save(self, resp): + resp.set_cookie('disabled_{0}'.format(self.value), ','.join(self.disabled), max_age=COOKIE_MAX_AGE) + resp.set_cookie('enabled_{0}'.format(self.value), ','.join(self.enabled), max_age=COOKIE_MAX_AGE) + + def get_disabled(self): + disabled = self.disabled + for choice in self.choices: + if not choice['default_on'] and choice['id'] not in self.enabled: + disabled.add(choice['id']) + return self.transform_values(disabled) + + def get_enabled(self): + enabled = self.enabled + for choice in self.choices: + if choice['default_on'] and choice['id'] not in self.disabled: + enabled.add(choice['id']) + return self.transform_values(enabled) + + +class EnginesSetting(SwitchableSetting): + def _post_init(self): + super(EnginesSetting, self)._post_init() + transformed_choices = [] + for engine_name, engine in self.choices.iteritems(): + for category in engine.categories: + transformed_choice = dict() + transformed_choice['default_on'] = not engine.disabled + transformed_choice['id'] = '{}__{}'.format(engine_name, category) + transformed_choices.append(transformed_choice) + self.choices = transformed_choices + + def transform_form_items(self, items): + return [item[len('engine_'):].replace('_', ' ').replace(' ', '__') for item in items] + + def transform_values(self, values): + if len(values) == 1 and values[0] == '': + return list() + transformed_values = [] + for value in values: + engine, category = value.split('__') + transformed_values.append((engine, category)) + return transformed_values + + +class PluginsSetting(SwitchableSetting): + def _post_init(self): + super(PluginsSetting, self)._post_init() + transformed_choices = [] + for plugin in self.choices: + transformed_choice = dict() + transformed_choice['default_on'] = plugin.default_on + transformed_choice['id'] = plugin.id + transformed_choices.append(transformed_choice) + self.choices = transformed_choices + + def transform_form_items(self, items): + return [item[len('plugin_'):] for item in items] + + +class Preferences(object): + """Stores, validates and saves preferences to cookies""" + + def __init__(self, themes, categories, engines, plugins): + super(Preferences, self).__init__() + + self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories), + 'language': EnumStringSetting('all', choices=LANGUAGE_CODES), + 'locale': EnumStringSetting(settings['ui']['default_locale'], + choices=settings['locales'].keys()), + 'autocomplete': EnumStringSetting(settings['search']['autocomplete'], + choices=autocomplete.backends.keys()), + 'image_proxy': MapSetting(settings['server']['image_proxy'], + map={'': settings['server']['image_proxy'], + '0': False, + '1': True}), + 'method': EnumStringSetting('POST', choices=('GET', 'POST')), + 'safesearch': MapSetting(settings['search']['safe_search'], map={'0': 0, + '1': 1, + '2': 2}), + 'theme': EnumStringSetting(settings['ui']['default_theme'], choices=themes)} + + self.engines = EnginesSetting('engines', choices=engines) + self.plugins = PluginsSetting('plugins', choices=plugins) + + def parse_cookies(self, input_data): + for user_setting_name, user_setting in input_data.iteritems(): + if user_setting_name in self.key_value_settings: + self.key_value_settings[user_setting_name].parse(user_setting) + elif user_setting_name == 'disabled_engines': + self.engines.parse_cookie([input_data['disabled_engines'], input_data['enabled_engines']]) + elif user_setting_name == 'disabled_plugins': + self.plugins.parse_cookie([input_data['disabled_plugins'], input_data['enabled_plugins']]) + + def parse_form(self, input_data): + disabled_engines = [] + enabled_categories = [] + disabled_plugins = [] + for user_setting_name, user_setting in input_data.iteritems(): + if user_setting_name in self.key_value_settings: + self.key_value_settings[user_setting_name].parse(user_setting) + elif user_setting_name.startswith('engine_'): + disabled_engines.append(user_setting_name) + elif user_setting_name.startswith('category_'): + enabled_categories.append(user_setting_name[len('category_'):]) + elif user_setting_name.startswith('plugin_'): + disabled_plugins.append(user_setting_name) + self.key_value_settings['categories'].parse_form(enabled_categories) + self.engines.parse_form(disabled_engines) + self.plugins.parse_form(disabled_plugins) + + # cannot be used in case of engines or plugins + def get_value(self, user_setting_name): + if user_setting_name in self.key_value_settings: + return self.key_value_settings[user_setting_name].get_value() + + def save(self, resp): + for user_setting_name, user_setting in self.key_value_settings.iteritems(): + user_setting.save(user_setting_name, resp) + self.engines.save(resp) + self.plugins.save(resp) + return resp diff --git a/searx/search.py b/searx/search.py index ce41b231b..f38d40106 100644 --- a/searx/search.py +++ b/searx/search.py @@ -23,7 +23,7 @@ from searx.engines import ( categories, engines ) from searx.languages import language_codes -from searx.utils import gen_useragent, get_blocked_engines +from searx.utils import gen_useragent from searx.query import Query from searx.results import ResultContainer from searx import logger @@ -140,15 +140,13 @@ class Search(object): self.lang = 'all' # set blocked engines - self.blocked_engines = get_blocked_engines(engines, request.cookies) + self.blocked_engines = request.preferences.engines.get_disabled() self.result_container = ResultContainer() self.request_data = {} # set specific language if set - if request.cookies.get('language')\ - and request.cookies['language'] in (x[0] for x in language_codes): - self.lang = request.cookies['language'] + self.lang = request.preferences.get_value('language') # set request method if request.method == 'POST': @@ -294,11 +292,8 @@ class Search(object): else: request_params['language'] = self.lang - try: - # 0 = None, 1 = Moderate, 2 = Strict - request_params['safesearch'] = int(request.cookies.get('safesearch')) - except Exception: - request_params['safesearch'] = settings['search']['safe_search'] + # 0 = None, 1 = Moderate, 2 = Strict + request_params['safesearch'] = request.preferences.get_value('safesearch') # update request parameters dependent on # search-engine (contained in engines folder) diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml index fb193e43d..7c7c4eec2 100644 --- a/searx/settings_robot.yml +++ b/searx/settings_robot.yml @@ -4,7 +4,7 @@ general: search: safe_search : 0 - autocomplete : 0 + autocomplete : "" server: port : 11111 diff --git a/searx/utils.py b/searx/utils.py index 506228465..b297582ef 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -230,26 +230,3 @@ def list_get(a_list, index, default=None): return a_list[index] else: return default - - -def get_blocked_engines(engines, cookies): - if 'blocked_engines' not in cookies: - return [(engine_name, category) for engine_name in engines - for category in engines[engine_name].categories if engines[engine_name].disabled] - - blocked_engine_strings = cookies.get('blocked_engines', '').split(',') - blocked_engines = [] - - if not blocked_engine_strings: - return blocked_engines - - for engine_string in blocked_engine_strings: - if engine_string.find('__') > -1: - engine, category = engine_string.split('__', 1) - if engine in engines and category in engines[engine].categories: - blocked_engines.append((engine, category)) - elif engine_string in engines: - for category in engines[engine_string].categories: - blocked_engines.append((engine_string, category)) - - return blocked_engines diff --git a/searx/webapp.py b/searx/webapp.py index d10f04e21..7f0b56101 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -56,7 +56,7 @@ from searx.engines import ( from searx.utils import ( UnicodeWriter, highlight_content, html_to_text, get_themes, get_static_files, get_result_templates, gen_useragent, dict_subset, - prettify_url, get_blocked_engines + prettify_url ) from searx.version import VERSION_STRING from searx.languages import language_codes @@ -64,6 +64,7 @@ from searx.search import Search from searx.query import Query from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx.plugins import plugins +from searx.preferences import Preferences # check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed. # They are needed for SSL connection without trouble, see #298 @@ -109,8 +110,6 @@ for indice, theme in enumerate(themes): for (dirpath, dirnames, filenames) in os.walk(theme_img_path): global_favicons[indice].extend(filenames) -cookie_max_age = 60 * 60 * 24 * 365 * 5 # 5 years - _category_names = (gettext('files'), gettext('general'), gettext('music'), @@ -222,9 +221,7 @@ def get_current_theme_name(override=None): if override and override in themes: return override - theme_name = request.args.get('theme', - request.cookies.get('theme', - default_theme)) + theme_name = request.args.get('theme', request.preferences.get_value('theme')) if theme_name not in themes: theme_name = default_theme return theme_name @@ -262,12 +259,8 @@ def image_proxify(url): def render(template_name, override_theme=None, **kwargs): - blocked_engines = get_blocked_engines(engines, request.cookies) - - autocomplete = request.cookies.get('autocomplete', settings['search']['autocomplete']) - - if autocomplete not in autocomplete_backends: - autocomplete = None + blocked_engines = request.preferences.engines.get_disabled() + autocomplete = request.preferences.get_value('autocomplete') nonblocked_categories = set(category for engine_name in engines for category in engines[engine_name].categories @@ -295,7 +288,7 @@ def render(template_name, override_theme=None, **kwargs): kwargs['selected_categories'].append(c) if not kwargs['selected_categories']: - cookie_categories = request.cookies.get('categories', '').split(',') + cookie_categories = request.preferences.get_value('categories') for ccateg in cookie_categories: if ccateg in categories: kwargs['selected_categories'].append(ccateg) @@ -311,9 +304,9 @@ def render(template_name, override_theme=None, **kwargs): kwargs['searx_version'] = VERSION_STRING - kwargs['method'] = request.cookies.get('method', 'POST') + kwargs['method'] = request.preferences.get_value('method') - kwargs['safesearch'] = request.cookies.get('safesearch', str(settings['search']['safe_search'])) + kwargs['safesearch'] = str(request.preferences.get_value('safesearch')) # override url_for function in templates kwargs['url_for'] = url_for_theme @@ -347,14 +340,18 @@ def render(template_name, override_theme=None, **kwargs): @app.before_request def pre_request(): # merge GET, POST vars + preferences = Preferences(themes, categories.keys(), engines, plugins) + preferences.parse_cookies(request.cookies) + request.preferences = preferences + request.form = dict(request.form.items()) for k, v in request.args.items(): if k not in request.form: request.form[k] = v request.user_plugins = [] - allowed_plugins = request.cookies.get('allowed_plugins', '').split(',') - disabled_plugins = request.cookies.get('disabled_plugins', '').split(',') + allowed_plugins = preferences.plugins.get_enabled() + disabled_plugins = preferences.plugins.get_disabled() for plugin in plugins: if ((plugin.default_on and plugin.id not in disabled_plugins) or plugin.id in allowed_plugins): @@ -486,7 +483,7 @@ def autocompleter(): request_data = request.args # set blocked engines - blocked_engines = get_blocked_engines(engines, request.cookies) + blocked_engines = request.preferences.engines.get_disabled() # parse query query = Query(request_data.get('q', '').encode('utf-8'), blocked_engines) @@ -496,8 +493,8 @@ def autocompleter(): if not query.getSearchQuery(): return '', 400 - # get autocompleter - completer = autocomplete_backends.get(request.cookies.get('autocomplete', settings['search']['autocomplete'])) + # run autocompleter + completer = autocomplete_backends.get(request.preferences.get_value('autocomplete')) # parse searx specific autocompleter results like !bang raw_results = searx_bang(query) @@ -532,117 +529,23 @@ def autocompleter(): @app.route('/preferences', methods=['GET', 'POST']) def preferences(): - """Render preferences page. + """Render preferences page && save user preferences""" - Settings that are going to be saved as cookies.""" - lang = None - image_proxy = request.cookies.get('image_proxy', settings['server'].get('image_proxy')) + # save preferences + if request.method == 'POST': + resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index')))) + try: + request.preferences.parse_form(request.form) + except ValidationException: + # TODO use flash feature of flask + return resp + return request.preferences.save(resp) - if request.cookies.get('language')\ - and request.cookies['language'] in (x[0] for x in language_codes): - lang = request.cookies['language'] - - blocked_engines = [] - - resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index')))) - - if request.method == 'GET': - blocked_engines = get_blocked_engines(engines, request.cookies) - else: # on save - selected_categories = [] - post_disabled_plugins = [] - locale = None - autocomplete = '' - method = 'POST' - safesearch = settings['search']['safe_search'] - for pd_name, pd in request.form.items(): - if pd_name.startswith('category_'): - category = pd_name[9:] - if category not in categories: - continue - selected_categories.append(category) - elif pd_name == 'locale' and pd in settings['locales']: - locale = pd - elif pd_name == 'image_proxy': - image_proxy = pd - elif pd_name == 'autocomplete': - autocomplete = pd - elif pd_name == 'language' and (pd == 'all' or - pd in (x[0] for - x in language_codes)): - lang = pd - elif pd_name == 'method': - method = pd - elif pd_name == 'safesearch': - safesearch = pd - elif pd_name.startswith('engine_'): - if pd_name.find('__') > -1: - # TODO fix underscore vs space - engine_name, category = [x.replace('_', ' ') for x in - pd_name.replace('engine_', '', 1).split('__', 1)] - if engine_name in engines and category in engines[engine_name].categories: - blocked_engines.append((engine_name, category)) - elif pd_name == 'theme': - theme = pd if pd in themes else default_theme - elif pd_name.startswith('plugin_'): - plugin_id = pd_name.replace('plugin_', '', 1) - if not any(plugin.id == plugin_id for plugin in plugins): - continue - post_disabled_plugins.append(plugin_id) - else: - resp.set_cookie(pd_name, pd, max_age=cookie_max_age) - - disabled_plugins = [] - allowed_plugins = [] - for plugin in plugins: - if plugin.default_on: - if plugin.id in post_disabled_plugins: - disabled_plugins.append(plugin.id) - elif plugin.id not in post_disabled_plugins: - allowed_plugins.append(plugin.id) - - resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age) - - resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age) - - resp.set_cookie( - 'blocked_engines', ','.join('__'.join(e) for e in blocked_engines), - max_age=cookie_max_age - ) - - if locale: - resp.set_cookie( - 'locale', locale, - max_age=cookie_max_age - ) - - if lang: - resp.set_cookie( - 'language', lang, - max_age=cookie_max_age - ) - - if selected_categories: - # cookie max age: 4 weeks - resp.set_cookie( - 'categories', ','.join(selected_categories), - max_age=cookie_max_age - ) - - resp.set_cookie( - 'autocomplete', autocomplete, - max_age=cookie_max_age - ) - - resp.set_cookie('method', method, max_age=cookie_max_age) - - resp.set_cookie('safesearch', str(safesearch), max_age=cookie_max_age) - - resp.set_cookie('image_proxy', image_proxy, max_age=cookie_max_age) - - resp.set_cookie('theme', theme, max_age=cookie_max_age) - - return resp + # render preferences + image_proxy = request.preferences.get_value('image_proxy') + lang = request.preferences.get_value('language') + blocked_engines = request.preferences.engines.get_disabled() + allowed_plugins = request.preferences.plugins.get_enabled() # stats for preferences page stats = {} @@ -664,7 +567,7 @@ def preferences(): return render('preferences.html', locales=settings['locales'], current_locale=get_locale(), - current_language=lang or 'all', + current_language=lang, image_proxy=image_proxy, language_codes=language_codes, engines_by_category=categories, @@ -674,7 +577,7 @@ def preferences(): shortcuts={y: x for x, y in engine_shortcuts.items()}, themes=themes, plugins=plugins, - allowed_plugins=[plugin.id for plugin in request.user_plugins], + allowed_plugins=allowed_plugins, theme=get_current_theme_name()) @@ -750,7 +653,7 @@ Disallow: /preferences def opensearch(): method = 'post' - if request.cookies.get('method', 'POST') == 'GET': + if request.preferences.get_value('method') == 'GET': method = 'get' # chrome/chromium only supports HTTP GET.... diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index 071c01df3..5697017d9 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -12,7 +12,6 @@ class ViewsTestCase(SearxTestCase): def setUp(self): webapp.app.config['TESTING'] = True # to get better error messages self.app = webapp.app.test_client() - webapp.default_theme = 'default' # set some defaults self.test_results = [ @@ -43,6 +42,11 @@ class ViewsTestCase(SearxTestCase): webapp.Search.search = search_mock + def get_current_theme_name_mock(override=None): + return 'default' + + webapp.get_current_theme_name = get_current_theme_name_mock + self.maxDiff = None # to see full diffs def test_index_empty(self):