From 2dbc0de0cd000459ebfdb3d015be8684e737e95c Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 22 Sep 2020 13:59:27 +0200 Subject: [PATCH 1/8] [mod] add searx/webadapter.py * move searx.search.get_search_query_from_webapp to searx.webadapter * move searx.query.SearchQuery to searx.search --- searx/query.py | 20 ---- searx/search.py | 186 +++++--------------------------------- searx/webadapter.py | 162 +++++++++++++++++++++++++++++++++ searx/webapp.py | 3 +- tests/unit/test_search.py | 67 +++++++------- utils/standalone_searx.py | 3 +- 6 files changed, 221 insertions(+), 220 deletions(-) create mode 100644 searx/webadapter.py diff --git a/searx/query.py b/searx/query.py index 7c2b0a249..9e2af0c45 100644 --- a/searx/query.py +++ b/searx/query.py @@ -178,23 +178,3 @@ class RawTextQuery: def getFullQuery(self): # get full querry including whitespaces return ''.join(self.query_parts) - - -class SearchQuery: - """container for all the search parameters (query, language, etc...)""" - - def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range, - timeout_limit=None, preferences=None, external_bang=None): - self.query = query - self.engines = engines - self.categories = categories - self.lang = lang - self.safesearch = safesearch - self.pageno = pageno - self.time_range = None if time_range in ('', 'None', None) else time_range - self.timeout_limit = timeout_limit - self.preferences = preferences - self.external_bang = external_bang - - def __str__(self): - return self.query + ";" + str(self.engines) diff --git a/searx/search.py b/searx/search.py index 96fffd569..d3b131d44 100644 --- a/searx/search.py +++ b/searx/search.py @@ -16,26 +16,20 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. ''' import gc -import sys import threading from time import time from uuid import uuid4 from _thread import start_new_thread -from flask_babel import gettext import requests.exceptions import searx.poolrequests as requests_lib -from searx.engines import ( - categories, engines, settings -) +from searx.engines import engines, settings from searx.answerers import ask from searx.external_bang import get_bang_url from searx.utils import gen_useragent -from searx.query import RawTextQuery, SearchQuery, VALID_LANGUAGE_CODE from searx.results import ResultContainer from searx import logger from searx.plugins import plugins -from searx.exceptions import SearxParameterException logger = logger.getChild('search') @@ -53,6 +47,26 @@ else: exit(1) +class SearchQuery: + """container for all the search parameters (query, language, etc...)""" + + def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range, + timeout_limit=None, preferences=None, external_bang=None): + self.query = query + self.engines = engines + self.categories = categories + self.lang = lang + self.safesearch = safesearch + self.pageno = pageno + self.time_range = None if time_range in ('', 'None', None) else time_range + self.timeout_limit = timeout_limit + self.preferences = preferences + self.external_bang = external_bang + + def __str__(self): + return self.query + ";" + str(self.engines) + + def send_http_request(engine, request_params): # create dictionary which contain all # informations about the request @@ -247,164 +261,6 @@ def default_request_params(): } -# remove duplicate queries. -# FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music' -def deduplicate_query_engines(query_engines): - uniq_query_engines = {q["category"] + '|' + q["name"]: q for q in query_engines} - return uniq_query_engines.values() - - -def get_search_query_from_webapp(preferences, form): - # no text for the query ? - if not form.get('q'): - raise SearxParameterException('q', '') - - # set blocked engines - disabled_engines = preferences.engines.get_disabled() - - # parse query, if tags are set, which change - # the serch engine or search-language - raw_text_query = RawTextQuery(form['q'], disabled_engines) - - # set query - query = raw_text_query.getQuery() - - # get and check page number - pageno_param = form.get('pageno', '1') - if not pageno_param.isdigit() or int(pageno_param) < 1: - raise SearxParameterException('pageno', pageno_param) - query_pageno = int(pageno_param) - - # get language - # set specific language if set on request, query or preferences - # TODO support search with multible languages - if len(raw_text_query.languages): - query_lang = raw_text_query.languages[-1] - elif 'language' in form: - query_lang = form.get('language') - else: - query_lang = preferences.get_value('language') - - # check language - if not VALID_LANGUAGE_CODE.match(query_lang): - raise SearxParameterException('language', query_lang) - - # get safesearch - if 'safesearch' in form: - query_safesearch = form.get('safesearch') - # first check safesearch - if not query_safesearch.isdigit(): - raise SearxParameterException('safesearch', query_safesearch) - query_safesearch = int(query_safesearch) - else: - query_safesearch = preferences.get_value('safesearch') - - # safesearch : second check - if query_safesearch < 0 or query_safesearch > 2: - raise SearxParameterException('safesearch', query_safesearch) - - # get time_range - query_time_range = form.get('time_range') - - # check time_range - if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'): - raise SearxParameterException('time_range', query_time_range) - - # query_engines - query_engines = raw_text_query.engines - - # timeout_limit - query_timeout = raw_text_query.timeout_limit - if query_timeout is None and 'timeout_limit' in form: - raw_time_limit = form.get('timeout_limit') - if raw_time_limit in ['None', '']: - raw_time_limit = None - else: - try: - query_timeout = float(raw_time_limit) - except ValueError: - raise SearxParameterException('timeout_limit', raw_time_limit) - - # query_categories - query_categories = [] - - # if engines are calculated from query, - # set categories by using that informations - if query_engines and raw_text_query.specific: - additional_categories = set() - for engine in query_engines: - if 'from_bang' in engine and engine['from_bang']: - additional_categories.add('none') - else: - additional_categories.add(engine['category']) - query_categories = list(additional_categories) - - # otherwise, using defined categories to - # calculate which engines should be used - else: - # set categories/engines - load_default_categories = True - for pd_name, pd in form.items(): - if pd_name == 'categories': - query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories) - elif pd_name == 'engines': - pd_engines = [{'category': engines[engine].categories[0], - 'name': engine} - for engine in map(str.strip, pd.split(',')) if engine in engines] - if pd_engines: - query_engines.extend(pd_engines) - load_default_categories = False - elif pd_name.startswith('category_'): - category = pd_name[9:] - - # if category is not found in list, skip - if category not in categories: - continue - - if pd != 'off': - # add category to list - query_categories.append(category) - elif category in query_categories: - # remove category from list if property is set to 'off' - query_categories.remove(category) - - if not load_default_categories: - if not query_categories: - query_categories = list(set(engine['category'] - for engine in query_engines)) - else: - # if no category is specified for this search, - # using user-defined default-configuration which - # (is stored in cookie) - if not query_categories: - cookie_categories = preferences.get_value('categories') - for ccateg in cookie_categories: - if ccateg in categories: - query_categories.append(ccateg) - - # if still no category is specified, using general - # as default-category - if not query_categories: - query_categories = ['general'] - - # using all engines for that search, which are - # declared under the specific categories - for categ in query_categories: - query_engines.extend({'category': categ, - 'name': engine.name} - for engine in categories[categ] - if (engine.name, categ) not in disabled_engines) - - query_engines = deduplicate_query_engines(query_engines) - external_bang = raw_text_query.external_bang - - return (SearchQuery(query, query_engines, query_categories, - query_lang, query_safesearch, query_pageno, - query_time_range, query_timeout, preferences, - external_bang=external_bang), - raw_text_query) - - class Search: """Search information container""" diff --git a/searx/webadapter.py b/searx/webadapter.py new file mode 100644 index 000000000..cad834bba --- /dev/null +++ b/searx/webadapter.py @@ -0,0 +1,162 @@ +from searx.exceptions import SearxParameterException +from searx.query import RawTextQuery, VALID_LANGUAGE_CODE +from searx.engines import categories, engines +from searx.search import SearchQuery + + +# remove duplicate queries. +# FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music' +def deduplicate_query_engines(query_engines): + uniq_query_engines = {q["category"] + '|' + q["name"]: q for q in query_engines} + return uniq_query_engines.values() + + +def get_search_query_from_webapp(preferences, form): + # no text for the query ? + if not form.get('q'): + raise SearxParameterException('q', '') + + # set blocked engines + disabled_engines = preferences.engines.get_disabled() + + # parse query, if tags are set, which change + # the serch engine or search-language + raw_text_query = RawTextQuery(form['q'], disabled_engines) + + # set query + query = raw_text_query.getQuery() + + # get and check page number + pageno_param = form.get('pageno', '1') + if not pageno_param.isdigit() or int(pageno_param) < 1: + raise SearxParameterException('pageno', pageno_param) + query_pageno = int(pageno_param) + + # get language + # set specific language if set on request, query or preferences + # TODO support search with multible languages + if len(raw_text_query.languages): + query_lang = raw_text_query.languages[-1] + elif 'language' in form: + query_lang = form.get('language') + else: + query_lang = preferences.get_value('language') + + # check language + if not VALID_LANGUAGE_CODE.match(query_lang): + raise SearxParameterException('language', query_lang) + + # get safesearch + if 'safesearch' in form: + query_safesearch = form.get('safesearch') + # first check safesearch + if not query_safesearch.isdigit(): + raise SearxParameterException('safesearch', query_safesearch) + query_safesearch = int(query_safesearch) + else: + query_safesearch = preferences.get_value('safesearch') + + # safesearch : second check + if query_safesearch < 0 or query_safesearch > 2: + raise SearxParameterException('safesearch', query_safesearch) + + # get time_range + query_time_range = form.get('time_range') + + # check time_range + if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'): + raise SearxParameterException('time_range', query_time_range) + + # query_engines + query_engines = raw_text_query.engines + + # timeout_limit + query_timeout = raw_text_query.timeout_limit + if query_timeout is None and 'timeout_limit' in form: + raw_time_limit = form.get('timeout_limit') + if raw_time_limit in ['None', '']: + raw_time_limit = None + else: + try: + query_timeout = float(raw_time_limit) + except ValueError: + raise SearxParameterException('timeout_limit', raw_time_limit) + + # query_categories + query_categories = [] + + # if engines are calculated from query, + # set categories by using that informations + if query_engines and raw_text_query.specific: + additional_categories = set() + for engine in query_engines: + if 'from_bang' in engine and engine['from_bang']: + additional_categories.add('none') + else: + additional_categories.add(engine['category']) + query_categories = list(additional_categories) + + # otherwise, using defined categories to + # calculate which engines should be used + else: + # set categories/engines + load_default_categories = True + for pd_name, pd in form.items(): + if pd_name == 'categories': + query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories) + elif pd_name == 'engines': + pd_engines = [{'category': engines[engine].categories[0], + 'name': engine} + for engine in map(str.strip, pd.split(',')) if engine in engines] + if pd_engines: + query_engines.extend(pd_engines) + load_default_categories = False + elif pd_name.startswith('category_'): + category = pd_name[9:] + + # if category is not found in list, skip + if category not in categories: + continue + + if pd != 'off': + # add category to list + query_categories.append(category) + elif category in query_categories: + # remove category from list if property is set to 'off' + query_categories.remove(category) + + if not load_default_categories: + if not query_categories: + query_categories = list(set(engine['category'] + for engine in query_engines)) + else: + # if no category is specified for this search, + # using user-defined default-configuration which + # (is stored in cookie) + if not query_categories: + cookie_categories = preferences.get_value('categories') + for ccateg in cookie_categories: + if ccateg in categories: + query_categories.append(ccateg) + + # if still no category is specified, using general + # as default-category + if not query_categories: + query_categories = ['general'] + + # using all engines for that search, which are + # declared under the specific categories + for categ in query_categories: + query_engines.extend({'category': categ, + 'name': engine.name} + for engine in categories[categ] + if (engine.name, categ) not in disabled_engines) + + query_engines = deduplicate_query_engines(query_engines) + external_bang = raw_text_query.external_bang + + return (SearchQuery(query, query_engines, query_categories, + query_lang, query_safesearch, query_pageno, + query_time_range, query_timeout, preferences, + external_bang=external_bang), + raw_text_query) diff --git a/searx/webapp.py b/searx/webapp.py index bba37cce1..ad9372915 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -67,10 +67,11 @@ from searx.webutils import ( get_static_files, get_result_templates, get_themes, prettify_url, new_hmac ) +from searx.webadapter import get_search_query_from_webapp from searx.utils import html_to_text, gen_useragent, dict_subset, match_language from searx.version import VERSION_STRING from searx.languages import language_codes as languages -from searx.search import SearchWithPlugins, get_search_query_from_webapp +from searx.search import SearchWithPlugins from searx.query import RawTextQuery from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx.plugins import plugins diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py index a15d2c899..d2322b20c 100644 --- a/tests/unit/test_search.py +++ b/tests/unit/test_search.py @@ -5,6 +5,7 @@ from searx.preferences import Preferences from searx.engines import engines import searx.search +from searx.search import SearchQuery SAFESEARCH = 0 @@ -40,53 +41,53 @@ class SearchTestCase(SearxTestCase): def test_timeout_simple(self): searx.search.max_request_timeout = None - search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, - preferences=Preferences(['oscar'], ['general'], engines, [])) + search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, + preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 3.0) def test_timeout_query_above_default_nomax(self): searx.search.max_request_timeout = None - search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0, + preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 3.0) def test_timeout_query_below_default_nomax(self): searx.search.max_request_timeout = None - search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0, + preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 1.0) def test_timeout_query_below_max(self): searx.search.max_request_timeout = 10.0 - search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0, + preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 5.0) def test_timeout_query_above_max(self): searx.search.max_request_timeout = 10.0 - search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0, + preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 10.0) def test_query_private_engine_without_token(self): - search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, + preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) results = search.search() self.assertEqual(results.results_length(), 0) @@ -94,9 +95,9 @@ class SearchTestCase(SearxTestCase): def test_query_private_engine_with_incorrect_token(self): preferences_with_tokens = Preferences(['oscar'], ['general'], engines, []) preferences_with_tokens.parse_dict({'tokens': 'bad-token'}) - search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, - preferences=preferences_with_tokens) + search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, + preferences=preferences_with_tokens) search = searx.search.Search(search_query) results = search.search() self.assertEqual(results.results_length(), 0) @@ -104,28 +105,28 @@ class SearchTestCase(SearxTestCase): def test_query_private_engine_with_correct_token(self): preferences_with_tokens = Preferences(['oscar'], ['general'], engines, []) preferences_with_tokens.parse_dict({'tokens': 'my-token'}) - search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, - preferences=preferences_with_tokens) + search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, + preferences=preferences_with_tokens) search = searx.search.Search(search_query) results = search.search() self.assertEqual(results.results_length(), 1) def test_external_bang(self): - search_query = searx.query.SearchQuery('yes yes', - [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, - preferences=Preferences(['oscar'], ['general'], engines, [],), - external_bang="yt") + search_query = SearchQuery('yes yes', + [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, + preferences=Preferences(['oscar'], ['general'], engines, [],), + external_bang="yt") search = searx.search.Search(search_query) results = search.search() # For checking if the user redirected with the youtube external bang self.assertTrue(results.redirect_url is not None) - search_query = searx.query.SearchQuery('youtube never gonna give you up', - [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, - preferences=Preferences(['oscar'], ['general'], engines, []),) + search_query = SearchQuery('youtube never gonna give you up', + [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, + preferences=Preferences(['oscar'], ['general'], engines, []),) search = searx.search.Search(search_query) results = search.search() diff --git a/utils/standalone_searx.py b/utils/standalone_searx.py index d43b474d7..2ca3fea56 100755 --- a/utils/standalone_searx.py +++ b/utils/standalone_searx.py @@ -31,6 +31,7 @@ import searx.query import searx.search import searx.engines import searx.preferences +import searx.webadapter import argparse searx.engines.initialize_engines(settings['engines']) @@ -64,7 +65,7 @@ form = { preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, []) preferences.key_value_settings['safesearch'].parse(args.safesearch) -search_query, raw_text_query = searx.search.get_search_query_from_webapp(preferences, form) +search_query, raw_text_query = searx.webadapter.get_search_query_from_webapp(preferences, form) search = searx.search.Search(search_query) result_container = search.search() From 2929495112e9869af24986b27066163572bc558a Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 22 Sep 2020 16:22:22 +0200 Subject: [PATCH 2/8] [mod] add searx.search.EngineRef was previously a Dict with two or three keys: name, category, from_bang make clear that this is a engine reference (see tests/unit/test_search.py for example) all variables using this class are renamed accordingly. --- searx/query.py | 22 ++++++++-------------- searx/search.py | 35 ++++++++++++++++++++++------------- searx/webadapter.py | 36 +++++++++++++++++------------------- tests/unit/test_search.py | 22 +++++++++++----------- 4 files changed, 58 insertions(+), 57 deletions(-) diff --git a/searx/query.py b/searx/query.py index 9e2af0c45..2d5a72bcb 100644 --- a/searx/query.py +++ b/searx/query.py @@ -20,9 +20,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. import re from searx.languages import language_codes -from searx.engines import ( - categories, engines, engine_shortcuts -) +from searx.engines import categories, engines, engine_shortcuts +from searx.search import EngineRef VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$') @@ -40,7 +39,7 @@ class RawTextQuery: self.disabled_engines = disabled_engines self.query_parts = [] - self.engines = [] + self.enginerefs = [] self.languages = [] self.timeout_limit = None self.external_bang = None @@ -135,26 +134,21 @@ class RawTextQuery: parse_next = True engine_name = engine_shortcuts[prefix] if engine_name in engines: - self.engines.append({'category': 'none', - 'name': engine_name, - 'from_bang': True}) + self.enginerefs.append(EngineRef(engine_name, 'none', True)) # check if prefix is equal with engine name elif prefix in engines: parse_next = True - self.engines.append({'category': 'none', - 'name': prefix, - 'from_bang': True}) + self.enginerefs.append(EngineRef(prefix, 'none', True)) # check if prefix is equal with categorie name elif prefix in categories: # using all engines for that search, which # are declared under that categorie name parse_next = True - self.engines.extend({'category': prefix, - 'name': engine.name} - for engine in categories[prefix] - if (engine.name, prefix) not in self.disabled_engines) + self.enginerefs.extend(EngineRef(engine.name, prefix) + for engine in categories[prefix] + if (engine.name, prefix) not in self.disabled_engines) if query_part[0] == '!': self.specific = True diff --git a/searx/search.py b/searx/search.py index d3b131d44..b81112c9d 100644 --- a/searx/search.py +++ b/searx/search.py @@ -47,13 +47,24 @@ else: exit(1) +class EngineRef: + + def __init__(self, name, category, from_bang=False): + self.name = name + self.category = category + self.from_bang = from_bang + + def __str__(self): + return "(" + self.name + "," + self.category + "," + str(self.from_bang) + ")" + + class SearchQuery: """container for all the search parameters (query, language, etc...)""" - def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range, + def __init__(self, query, engineref_list, categories, lang, safesearch, pageno, time_range, timeout_limit=None, preferences=None, external_bang=None): self.query = query - self.engines = engines + self.engineref_list = engineref_list self.categories = categories self.lang = lang self.safesearch = safesearch @@ -64,7 +75,7 @@ class SearchQuery: self.external_bang = external_bang def __str__(self): - return self.query + ";" + str(self.engines) + return self.query + ";" + str(self.engineref_list) def send_http_request(engine, request_params): @@ -318,13 +329,13 @@ class Search: return True - def _get_params(self, selected_engine, user_agent): - if selected_engine['name'] not in engines: + def _get_params(self, engineref, user_agent): + if engineref.name not in engines: return None, None - engine = engines[selected_engine['name']] + engine = engines[engineref.name] - if not self._is_accepted(selected_engine['name'], engine): + if not self._is_accepted(engineref.name, engine): return None, None # set default request parameters @@ -341,15 +352,13 @@ class Search: request_params['safesearch'] = self.search_query.safesearch request_params['time_range'] = self.search_query.time_range - request_params['category'] = selected_engine['category'] + request_params['category'] = engineref.category request_params['pageno'] = self.search_query.pageno return request_params, engine.timeout # do search-request def _get_requests(self): - global number_of_searches - # init vars requests = [] @@ -361,14 +370,14 @@ class Search: default_timeout = 0 # start search-reqest for all selected engines - for selected_engine in self.search_query.engines: + for engineref in self.search_query.engineref_list: # set default request parameters - request_params, engine_timeout = self._get_params(selected_engine, user_agent) + request_params, engine_timeout = self._get_params(engineref, user_agent) if request_params is None: continue # append request to list - requests.append((selected_engine['name'], self.search_query.query, request_params)) + requests.append((engineref.name, self.search_query.query, request_params)) # update default_timeout default_timeout = max(default_timeout, engine_timeout) diff --git a/searx/webadapter.py b/searx/webadapter.py index cad834bba..97379b17e 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -1,14 +1,14 @@ from searx.exceptions import SearxParameterException from searx.query import RawTextQuery, VALID_LANGUAGE_CODE from searx.engines import categories, engines -from searx.search import SearchQuery +from searx.search import SearchQuery, EngineRef # remove duplicate queries. # FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music' -def deduplicate_query_engines(query_engines): - uniq_query_engines = {q["category"] + '|' + q["name"]: q for q in query_engines} - return uniq_query_engines.values() +def deduplicate_engineref_list(engineref_list): + engineref_dict = {q.category + '|' + q.name: q for q in engineref_list} + return engineref_dict.values() def get_search_query_from_webapp(preferences, form): @@ -68,7 +68,7 @@ def get_search_query_from_webapp(preferences, form): raise SearxParameterException('time_range', query_time_range) # query_engines - query_engines = raw_text_query.engines + query_engineref_list = raw_text_query.enginerefs # timeout_limit query_timeout = raw_text_query.timeout_limit @@ -87,13 +87,13 @@ def get_search_query_from_webapp(preferences, form): # if engines are calculated from query, # set categories by using that informations - if query_engines and raw_text_query.specific: + if query_engineref_list and raw_text_query.specific: additional_categories = set() - for engine in query_engines: - if 'from_bang' in engine and engine['from_bang']: + for engineref in query_engineref_list: + if engineref.from_bang: additional_categories.add('none') else: - additional_categories.add(engine['category']) + additional_categories.add(engineref.category) query_categories = list(additional_categories) # otherwise, using defined categories to @@ -105,11 +105,10 @@ def get_search_query_from_webapp(preferences, form): if pd_name == 'categories': query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories) elif pd_name == 'engines': - pd_engines = [{'category': engines[engine].categories[0], - 'name': engine} + pd_engines = [EngineRef(engineref, engines[engineref].categories[0]) for engine in map(str.strip, pd.split(',')) if engine in engines] if pd_engines: - query_engines.extend(pd_engines) + query_engineref_list.extend(pd_engines) load_default_categories = False elif pd_name.startswith('category_'): category = pd_name[9:] @@ -128,7 +127,7 @@ def get_search_query_from_webapp(preferences, form): if not load_default_categories: if not query_categories: query_categories = list(set(engine['category'] - for engine in query_engines)) + for engine in query_engineref_list)) else: # if no category is specified for this search, # using user-defined default-configuration which @@ -147,15 +146,14 @@ def get_search_query_from_webapp(preferences, form): # using all engines for that search, which are # declared under the specific categories for categ in query_categories: - query_engines.extend({'category': categ, - 'name': engine.name} - for engine in categories[categ] - if (engine.name, categ) not in disabled_engines) + query_engineref_list.extend(EngineRef(engine.name, categ) + for engine in categories[categ] + if (engine.name, categ) not in disabled_engines) - query_engines = deduplicate_query_engines(query_engines) + query_engineref_list = deduplicate_engineref_list(query_engineref_list) external_bang = raw_text_query.external_bang - return (SearchQuery(query, query_engines, query_categories, + return (SearchQuery(query, query_engineref_list, query_categories, query_lang, query_safesearch, query_pageno, query_time_range, query_timeout, preferences, external_bang=external_bang), diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py index d2322b20c..9da28bc68 100644 --- a/tests/unit/test_search.py +++ b/tests/unit/test_search.py @@ -5,7 +5,7 @@ from searx.preferences import Preferences from searx.engines import engines import searx.search -from searx.search import SearchQuery +from searx.search import SearchQuery, EngineRef SAFESEARCH = 0 @@ -41,7 +41,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_simple(self): searx.search.max_request_timeout = None - search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) @@ -50,7 +50,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_query_above_default_nomax(self): searx.search.max_request_timeout = None - search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0, preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) @@ -59,7 +59,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_query_below_default_nomax(self): searx.search.max_request_timeout = None - search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0, preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) @@ -68,7 +68,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_query_below_max(self): searx.search.max_request_timeout = 10.0 - search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0, preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) @@ -77,7 +77,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_query_above_max(self): searx.search.max_request_timeout = 10.0 - search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0, preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) @@ -85,7 +85,7 @@ class SearchTestCase(SearxTestCase): self.assertEqual(search.actual_timeout, 10.0) def test_query_private_engine_without_token(self): - search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], + search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, preferences=Preferences(['oscar'], ['general'], engines, [])) search = searx.search.Search(search_query) @@ -95,7 +95,7 @@ class SearchTestCase(SearxTestCase): def test_query_private_engine_with_incorrect_token(self): preferences_with_tokens = Preferences(['oscar'], ['general'], engines, []) preferences_with_tokens.parse_dict({'tokens': 'bad-token'}) - search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], + search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, preferences=preferences_with_tokens) search = searx.search.Search(search_query) @@ -105,7 +105,7 @@ class SearchTestCase(SearxTestCase): def test_query_private_engine_with_correct_token(self): preferences_with_tokens = Preferences(['oscar'], ['general'], engines, []) preferences_with_tokens.parse_dict({'tokens': 'my-token'}) - search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}], + search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, preferences=preferences_with_tokens) search = searx.search.Search(search_query) @@ -114,7 +114,7 @@ class SearchTestCase(SearxTestCase): def test_external_bang(self): search_query = SearchQuery('yes yes', - [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + [EngineRef(PUBLIC_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, preferences=Preferences(['oscar'], ['general'], engines, [],), external_bang="yt") @@ -124,7 +124,7 @@ class SearchTestCase(SearxTestCase): self.assertTrue(results.redirect_url is not None) search_query = SearchQuery('youtube never gonna give you up', - [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}], + [EngineRef(PUBLIC_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, preferences=Preferences(['oscar'], ['general'], engines, []),) From eecfff268913045f957c0fceb2a1caf56f236a1f Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 22 Sep 2020 16:31:17 +0200 Subject: [PATCH 3/8] [mod] check time_range parameter in searx/webadapter.py --- searx/search.py | 2 +- searx/webadapter.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/searx/search.py b/searx/search.py index b81112c9d..3619287b9 100644 --- a/searx/search.py +++ b/searx/search.py @@ -69,7 +69,7 @@ class SearchQuery: self.lang = lang self.safesearch = safesearch self.pageno = pageno - self.time_range = None if time_range in ('', 'None', None) else time_range + self.time_range = time_range self.timeout_limit = timeout_limit self.preferences = preferences self.external_bang = external_bang diff --git a/searx/webadapter.py b/searx/webadapter.py index 97379b17e..0567c8d34 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -64,7 +64,8 @@ def get_search_query_from_webapp(preferences, form): query_time_range = form.get('time_range') # check time_range - if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'): + query_time_range = None if query_time_range in ('', 'None') else query_time_range + if query_time_range not in (None, 'day', 'week', 'month', 'year'): raise SearxParameterException('time_range', query_time_range) # query_engines From 691d12726be1411d8c80895e927293bdcc3e9572 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 22 Sep 2020 16:55:59 +0200 Subject: [PATCH 4/8] [mod] check the engine tokens in searx/webadapter.py instead of searx/search.py --- searx/search.py | 6 +--- searx/webadapter.py | 33 +++++++++++++++++-- searx/webapp.py | 2 +- tests/unit/test_search.py | 62 ++++------------------------------- tests/unit/test_webadapter.py | 54 ++++++++++++++++++++++++++++++ utils/standalone_searx.py | 2 +- 6 files changed, 95 insertions(+), 64 deletions(-) create mode 100644 tests/unit/test_webadapter.py diff --git a/searx/search.py b/searx/search.py index 3619287b9..a99ca6b16 100644 --- a/searx/search.py +++ b/searx/search.py @@ -62,7 +62,7 @@ class SearchQuery: """container for all the search parameters (query, language, etc...)""" def __init__(self, query, engineref_list, categories, lang, safesearch, pageno, time_range, - timeout_limit=None, preferences=None, external_bang=None): + timeout_limit=None, external_bang=None): self.query = query self.engineref_list = engineref_list self.categories = categories @@ -71,7 +71,6 @@ class SearchQuery: self.pageno = pageno self.time_range = time_range self.timeout_limit = timeout_limit - self.preferences = preferences self.external_bang = external_bang def __str__(self): @@ -311,9 +310,6 @@ class Search: return False def _is_accepted(self, engine_name, engine): - if not self.search_query.preferences.validate_token(engine): - return False - # skip suspended engines if engine.suspend_end_time >= time(): logger.debug('Engine currently suspended: %s', engine_name) diff --git a/searx/webadapter.py b/searx/webadapter.py index 0567c8d34..667d44c86 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -11,6 +11,31 @@ def deduplicate_engineref_list(engineref_list): return engineref_dict.values() +def validate_engineref_list(engineref_list, preferences): + """ + Validate query_engines according to the preferences + Returns: + list of existing engines with a validated token + list of unknown engine + list of engine with invalid token according to the preferences + """ + valid = [] + unknown = [] + no_token = [] + for engineref in engineref_list: + if engineref.name not in engines: + unknown.append(engineref) + continue + + engine = engines[engineref.name] + if not preferences.validate_token(engine): + no_token.append(engineref) + continue + + valid.append(engineref) + return valid, unknown, no_token + + def get_search_query_from_webapp(preferences, form): # no text for the query ? if not form.get('q'): @@ -152,10 +177,14 @@ def get_search_query_from_webapp(preferences, form): if (engine.name, categ) not in disabled_engines) query_engineref_list = deduplicate_engineref_list(query_engineref_list) + query_engineref_list, query_engineref_list_unknown, query_engineref_list_notoken =\ + validate_engineref_list(query_engineref_list, preferences) external_bang = raw_text_query.external_bang return (SearchQuery(query, query_engineref_list, query_categories, query_lang, query_safesearch, query_pageno, - query_time_range, query_timeout, preferences, + query_time_range, query_timeout, external_bang=external_bang), - raw_text_query) + raw_text_query, + query_engineref_list_unknown, + query_engineref_list_notoken) diff --git a/searx/webapp.py b/searx/webapp.py index ad9372915..2a763292d 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -562,7 +562,7 @@ def index(): raw_text_query = None result_container = None try: - search_query, raw_text_query = get_search_query_from_webapp(request.preferences, request.form) + search_query, raw_text_query, _, _ = get_search_query_from_webapp(request.preferences, request.form) # search = Search(search_query) # without plugins search = SearchWithPlugins(search_query, request.user_plugins, request) diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py index 9da28bc68..bd302c741 100644 --- a/tests/unit/test_search.py +++ b/tests/unit/test_search.py @@ -1,17 +1,13 @@ # -*- coding: utf-8 -*- from searx.testing import SearxTestCase -from searx.preferences import Preferences -from searx.engines import engines - -import searx.search from searx.search import SearchQuery, EngineRef +import searx.search SAFESEARCH = 0 PAGENO = 1 PUBLIC_ENGINE_NAME = 'general dummy' -PRIVATE_ENGINE_NAME = 'general private offline' TEST_ENGINES = [ { 'name': PUBLIC_ENGINE_NAME, @@ -21,15 +17,6 @@ TEST_ENGINES = [ 'timeout': 3.0, 'tokens': [], }, - { - 'name': PRIVATE_ENGINE_NAME, - 'engine': 'dummy-offline', - 'categories': 'general', - 'shortcut': 'do', - 'timeout': 3.0, - 'offline': True, - 'tokens': ['my-token'], - }, ] @@ -42,8 +29,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_simple(self): searx.search.max_request_timeout = None search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, - preferences=Preferences(['oscar'], ['general'], engines, [])) + ['general'], 'en-US', SAFESEARCH, PAGENO, None, None) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 3.0) @@ -51,8 +37,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_query_above_default_nomax(self): searx.search.max_request_timeout = None search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 3.0) @@ -60,8 +45,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_query_below_default_nomax(self): searx.search.max_request_timeout = None search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 1.0) @@ -69,8 +53,7 @@ class SearchTestCase(SearxTestCase): def test_timeout_query_below_max(self): searx.search.max_request_timeout = 10.0 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 5.0) @@ -78,45 +61,15 @@ class SearchTestCase(SearxTestCase): def test_timeout_query_above_max(self): searx.search.max_request_timeout = 10.0 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) + ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0) search = searx.search.Search(search_query) search.search() self.assertEqual(search.actual_timeout, 10.0) - def test_query_private_engine_without_token(self): - search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, - preferences=Preferences(['oscar'], ['general'], engines, [])) - search = searx.search.Search(search_query) - results = search.search() - self.assertEqual(results.results_length(), 0) - - def test_query_private_engine_with_incorrect_token(self): - preferences_with_tokens = Preferences(['oscar'], ['general'], engines, []) - preferences_with_tokens.parse_dict({'tokens': 'bad-token'}) - search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, - preferences=preferences_with_tokens) - search = searx.search.Search(search_query) - results = search.search() - self.assertEqual(results.results_length(), 0) - - def test_query_private_engine_with_correct_token(self): - preferences_with_tokens = Preferences(['oscar'], ['general'], engines, []) - preferences_with_tokens.parse_dict({'tokens': 'my-token'}) - search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0, - preferences=preferences_with_tokens) - search = searx.search.Search(search_query) - results = search.search() - self.assertEqual(results.results_length(), 1) - def test_external_bang(self): search_query = SearchQuery('yes yes', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, - preferences=Preferences(['oscar'], ['general'], engines, [],), external_bang="yt") search = searx.search.Search(search_query) results = search.search() @@ -125,8 +78,7 @@ class SearchTestCase(SearxTestCase): search_query = SearchQuery('youtube never gonna give you up', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], - ['general'], 'en-US', SAFESEARCH, PAGENO, None, None, - preferences=Preferences(['oscar'], ['general'], engines, []),) + ['general'], 'en-US', SAFESEARCH, PAGENO, None, None) search = searx.search.Search(search_query) results = search.search() diff --git a/tests/unit/test_webadapter.py b/tests/unit/test_webadapter.py new file mode 100644 index 000000000..7806353d3 --- /dev/null +++ b/tests/unit/test_webadapter.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +from searx.testing import SearxTestCase +from searx.preferences import Preferences +from searx.engines import engines + +import searx.search +from searx.search import EngineRef, SearchQuery +from searx.webadapter import validate_engineref_list + + +PRIVATE_ENGINE_NAME = 'general private offline' +TEST_ENGINES = [ + { + 'name': PRIVATE_ENGINE_NAME, + 'engine': 'dummy-offline', + 'categories': 'general', + 'shortcut': 'do', + 'timeout': 3.0, + 'offline': True, + 'tokens': ['my-token'], + }, +] +SEARCHQUERY = [EngineRef(PRIVATE_ENGINE_NAME, 'general')] + + +class ValidateQueryCase(SearxTestCase): + + @classmethod + def setUpClass(cls): + searx.engines.initialize_engines(TEST_ENGINES) + + def test_query_private_engine_without_token(self): + preferences = Preferences(['oscar'], ['general'], engines, []) + valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences) + self.assertEqual(len(valid), 0) + self.assertEqual(len(unknown), 0) + self.assertEqual(len(invalid_token), 1) + + def test_query_private_engine_with_incorrect_token(self): + preferences_with_tokens = Preferences(['oscar'], ['general'], engines, []) + preferences_with_tokens.parse_dict({'tokens': 'bad-token'}) + valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences_with_tokens) + self.assertEqual(len(valid), 0) + self.assertEqual(len(unknown), 0) + self.assertEqual(len(invalid_token), 1) + + def test_query_private_engine_with_correct_token(self): + preferences_with_tokens = Preferences(['oscar'], ['general'], engines, []) + preferences_with_tokens.parse_dict({'tokens': 'my-token'}) + valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences_with_tokens) + self.assertEqual(len(valid), 1) + self.assertEqual(len(unknown), 0) + self.assertEqual(len(invalid_token), 0) diff --git a/utils/standalone_searx.py b/utils/standalone_searx.py index 2ca3fea56..d969c398d 100755 --- a/utils/standalone_searx.py +++ b/utils/standalone_searx.py @@ -65,7 +65,7 @@ form = { preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, []) preferences.key_value_settings['safesearch'].parse(args.safesearch) -search_query, raw_text_query = searx.webadapter.get_search_query_from_webapp(preferences, form) +search_query, raw_text_query, _, _ = searx.webadapter.get_search_query_from_webapp(preferences, form) search = searx.search.Search(search_query) result_container = search.search() From d5b5e48f04e2e156cb975c136ec4b179b170509a Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 22 Sep 2020 17:45:32 +0200 Subject: [PATCH 5/8] [mod] searx/webadapter.py: each web parameter has a dedicated function (parse_pageno, page_lang,....). --- searx/webadapter.py | 64 +++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/searx/webadapter.py b/searx/webadapter.py index 667d44c86..eed320a22 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -36,27 +36,14 @@ def validate_engineref_list(engineref_list, preferences): return valid, unknown, no_token -def get_search_query_from_webapp(preferences, form): - # no text for the query ? - if not form.get('q'): - raise SearxParameterException('q', '') - - # set blocked engines - disabled_engines = preferences.engines.get_disabled() - - # parse query, if tags are set, which change - # the serch engine or search-language - raw_text_query = RawTextQuery(form['q'], disabled_engines) - - # set query - query = raw_text_query.getQuery() - - # get and check page number +def parse_pageno(form): pageno_param = form.get('pageno', '1') if not pageno_param.isdigit() or int(pageno_param) < 1: raise SearxParameterException('pageno', pageno_param) - query_pageno = int(pageno_param) + return int(pageno_param) + +def parse_lang(raw_text_query, form, preferences): # get language # set specific language if set on request, query or preferences # TODO support search with multible languages @@ -71,7 +58,10 @@ def get_search_query_from_webapp(preferences, form): if not VALID_LANGUAGE_CODE.match(query_lang): raise SearxParameterException('language', query_lang) - # get safesearch + return query_lang + + +def parse_safesearch(form, preferences): if 'safesearch' in form: query_safesearch = form.get('safesearch') # first check safesearch @@ -85,30 +75,54 @@ def get_search_query_from_webapp(preferences, form): if query_safesearch < 0 or query_safesearch > 2: raise SearxParameterException('safesearch', query_safesearch) - # get time_range - query_time_range = form.get('time_range') + return query_safesearch + +def parse_time_range(form): + query_time_range = form.get('time_range') # check time_range query_time_range = None if query_time_range in ('', 'None') else query_time_range if query_time_range not in (None, 'day', 'week', 'month', 'year'): raise SearxParameterException('time_range', query_time_range) + return query_time_range - # query_engines - query_engineref_list = raw_text_query.enginerefs - # timeout_limit +def parse_timeout(raw_text_query, form): query_timeout = raw_text_query.timeout_limit if query_timeout is None and 'timeout_limit' in form: raw_time_limit = form.get('timeout_limit') if raw_time_limit in ['None', '']: - raw_time_limit = None + return None else: try: - query_timeout = float(raw_time_limit) + return float(raw_time_limit) except ValueError: raise SearxParameterException('timeout_limit', raw_time_limit) + +def get_search_query_from_webapp(preferences, form): + # no text for the query ? + if not form.get('q'): + raise SearxParameterException('q', '') + + # set blocked engines + disabled_engines = preferences.engines.get_disabled() + + # parse query, if tags are set, which change + # the serch engine or search-language + raw_text_query = RawTextQuery(form['q'], disabled_engines) + + # set query + query = raw_text_query.getQuery() + query_pageno = parse_pageno(form) + query_lang = parse_lang(raw_text_query, form, preferences) + query_safesearch = parse_safesearch(form, preferences) + query_time_range = parse_time_range(form) + query_timeout = parse_timeout(raw_text_query, form) + external_bang = raw_text_query.external_bang + # query_categories + query_engineref_list = raw_text_query.enginerefs query_categories = [] # if engines are calculated from query, From 678699beaf97184a9059b0c6514b1a51c7e6dfd0 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 22 Sep 2020 18:03:42 +0200 Subject: [PATCH 6/8] [mod] searx/webadapter.py: add get_selected_categories share common code with get_search_query_from_webapp Update searx/webapp.py to use get_selected_categories Close #2142 --- searx/webadapter.py | 162 +++++++++++++++++++++++++------------------- searx/webapp.py | 25 ++----- 2 files changed, 99 insertions(+), 88 deletions(-) diff --git a/searx/webadapter.py b/searx/webadapter.py index eed320a22..982517eaa 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -100,6 +100,93 @@ def parse_timeout(raw_text_query, form): raise SearxParameterException('timeout_limit', raw_time_limit) +def parse_specific(raw_text_query: RawTextQuery): + query_engineref_list = raw_text_query.enginerefs + additional_categories = set() + for engineref in raw_text_query.enginerefs: + if engineref.from_bang: + additional_categories.add('none') + else: + additional_categories.add(engineref.category) + query_categories = list(additional_categories) + return query_engineref_list, query_categories + + +def parse_category_form(query_categories, name, value): + if name == 'categories': + query_categories.extend(categ for categ in map(str.strip, value.split(',')) if categ in categories) + elif name.startswith('category_'): + category = name[9:] + + # if category is not found in list, skip + if category not in categories: + return + + if value != 'off': + # add category to list + query_categories.append(category) + elif category in query_categories: + # remove category from list if property is set to 'off' + query_categories.remove(category) + + +def get_selected_categories(form, preferences): + selected_categories = [] + + if form is not None: + for name, value in form.items(): + parse_category_form(selected_categories, name, value) + + # if no category is specified for this search, + # using user-defined default-configuration which + # (is stored in cookie) + if not selected_categories: + cookie_categories = preferences.get_value('categories') + for ccateg in cookie_categories: + selected_categories.append(ccateg) + + # if still no category is specified, using general + # as default-category + if not selected_categories: + selected_categories = ['general'] + + return selected_categories + + +def parse_generic(form, preferences, disabled_engines): + query_engineref_list = [] + query_categories = [] + + # set categories/engines + load_default_categories = True + for pd_name, pd in form.items(): + if pd_name == 'engines': + pd_engines = [EngineRef(engine_name, engines[engine_name].categories[0]) + for engine_name in map(str.strip, pd.split(',')) if engine_name in engines] + if pd_engines: + query_engineref_list.extend(pd_engines) + load_default_categories = False + else: + parse_category_form(query_categories, pd_name, pd) + + if not load_default_categories: + if not query_categories: + query_categories = list(set(engine['category'] + for engine in query_engineref_list)) + else: + if not query_categories: + query_categories = get_selected_categories(None, preferences) + + # using all engines for that search, which are + # declared under the specific categories + for categ in query_categories: + query_engineref_list.extend(EngineRef(engine.name, categ) + for engine in categories[categ] + if (engine.name, categ) not in disabled_engines) + + return query_engineref_list, query_categories + + def get_search_query_from_webapp(preferences, form): # no text for the query ? if not form.get('q'): @@ -121,79 +208,18 @@ def get_search_query_from_webapp(preferences, form): query_timeout = parse_timeout(raw_text_query, form) external_bang = raw_text_query.external_bang - # query_categories - query_engineref_list = raw_text_query.enginerefs - query_categories = [] - - # if engines are calculated from query, - # set categories by using that informations - if query_engineref_list and raw_text_query.specific: - additional_categories = set() - for engineref in query_engineref_list: - if engineref.from_bang: - additional_categories.add('none') - else: - additional_categories.add(engineref.category) - query_categories = list(additional_categories) - - # otherwise, using defined categories to - # calculate which engines should be used + if raw_text_query.enginerefs and raw_text_query.specific: + # if engines are calculated from query, + # set categories by using that informations + query_engineref_list, query_categories = parse_specific(raw_text_query) else: - # set categories/engines - load_default_categories = True - for pd_name, pd in form.items(): - if pd_name == 'categories': - query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories) - elif pd_name == 'engines': - pd_engines = [EngineRef(engineref, engines[engineref].categories[0]) - for engine in map(str.strip, pd.split(',')) if engine in engines] - if pd_engines: - query_engineref_list.extend(pd_engines) - load_default_categories = False - elif pd_name.startswith('category_'): - category = pd_name[9:] - - # if category is not found in list, skip - if category not in categories: - continue - - if pd != 'off': - # add category to list - query_categories.append(category) - elif category in query_categories: - # remove category from list if property is set to 'off' - query_categories.remove(category) - - if not load_default_categories: - if not query_categories: - query_categories = list(set(engine['category'] - for engine in query_engineref_list)) - else: - # if no category is specified for this search, - # using user-defined default-configuration which - # (is stored in cookie) - if not query_categories: - cookie_categories = preferences.get_value('categories') - for ccateg in cookie_categories: - if ccateg in categories: - query_categories.append(ccateg) - - # if still no category is specified, using general - # as default-category - if not query_categories: - query_categories = ['general'] - - # using all engines for that search, which are - # declared under the specific categories - for categ in query_categories: - query_engineref_list.extend(EngineRef(engine.name, categ) - for engine in categories[categ] - if (engine.name, categ) not in disabled_engines) + # otherwise, using defined categories to + # calculate which engines should be used + query_engineref_list, query_categories = parse_generic(form, preferences, disabled_engines) query_engineref_list = deduplicate_engineref_list(query_engineref_list) query_engineref_list, query_engineref_list_unknown, query_engineref_list_notoken =\ validate_engineref_list(query_engineref_list, preferences) - external_bang = raw_text_query.external_bang return (SearchQuery(query, query_engineref_list, query_categories, query_lang, query_safesearch, query_pageno, diff --git a/searx/webapp.py b/searx/webapp.py index 2a763292d..119f9f3b7 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -67,7 +67,7 @@ from searx.webutils import ( get_static_files, get_result_templates, get_themes, prettify_url, new_hmac ) -from searx.webadapter import get_search_query_from_webapp +from searx.webadapter import get_search_query_from_webapp, get_selected_categories from searx.utils import html_to_text, gen_useragent, dict_subset, match_language from searx.version import VERSION_STRING from searx.languages import language_codes as languages @@ -355,25 +355,6 @@ def render(template_name, override_theme=None, **kwargs): _get_ordered_categories() if x in enabled_categories] - if 'all_categories' not in kwargs: - kwargs['all_categories'] = _get_ordered_categories() - - if 'selected_categories' not in kwargs: - kwargs['selected_categories'] = [] - for arg in request.args: - if arg.startswith('category_'): - c = arg.split('_', 1)[1] - if c in categories: - kwargs['selected_categories'].append(c) - - if not kwargs['selected_categories']: - cookie_categories = request.preferences.get_value('categories') - for ccateg in cookie_categories: - kwargs['selected_categories'].append(ccateg) - - if not kwargs['selected_categories']: - kwargs['selected_categories'] = ['general'] - if 'autocomplete' not in kwargs: kwargs['autocomplete'] = request.preferences.get_value('autocomplete') @@ -532,6 +513,7 @@ def index_error(output_format, error_message): request.errors.append(gettext('search error')) return render( 'index.html', + selected_categories=get_selected_categories(request.form, request.preferences), ) @@ -553,6 +535,7 @@ def index(): if output_format == 'html': return render( 'index.html', + selected_categories=get_selected_categories(request.form, request.preferences), ) else: return index_error(output_format, 'No query'), 400 @@ -833,6 +816,8 @@ def preferences(): # end of stats return render('preferences.html', + selected_categories=get_selected_categories(request.form, request.preferences), + all_categories=_get_ordered_categories(), locales=settings['locales'], current_locale=request.preferences.get_value("locale"), image_proxy=image_proxy, From 485a502b886bc7fb8a806ffc576b0eec99990a5c Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 10 Sep 2020 18:08:14 +0200 Subject: [PATCH 7/8] [mod] add typing and __slots__ --- searx/search.py | 26 ++++++++++++++++++++++---- searx/webadapter.py | 20 ++++++++++++-------- tests/unit/test_search.py | 1 + utils/standalone_searx.py | 1 + 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/searx/search.py b/searx/search.py index a99ca6b16..3dfdd7213 100644 --- a/searx/search.py +++ b/searx/search.py @@ -15,6 +15,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, ''' +import typing import gc import threading from time import time @@ -49,7 +50,9 @@ else: class EngineRef: - def __init__(self, name, category, from_bang=False): + __slots__ = 'name', 'category', 'from_bang' + + def __init__(self, name: str, category: str, from_bang: bool=False): self.name = name self.category = category self.from_bang = from_bang @@ -61,8 +64,19 @@ class EngineRef: class SearchQuery: """container for all the search parameters (query, language, etc...)""" - def __init__(self, query, engineref_list, categories, lang, safesearch, pageno, time_range, - timeout_limit=None, external_bang=None): + __slots__ = 'query', 'engineref_list', 'categories', 'lang', 'safesearch', 'pageno', 'time_range',\ + 'timeout_limit', 'external_bang' + + def __init__(self, + query: str, + engineref_list: typing.List[EngineRef], + categories: typing.List[str], + lang: str, + safesearch: bool, + pageno: int, + time_range: typing.Optional[str], + timeout_limit: typing.Optional[float]=None, + external_bang: typing.Optional[str]=False): self.query = query self.engineref_list = engineref_list self.categories = categories @@ -274,6 +288,8 @@ def default_request_params(): class Search: """Search information container""" + __slots__ = "search_query", "result_container", "start_time", "actual_timeout" + def __init__(self, search_query): # init vars super().__init__() @@ -396,7 +412,7 @@ class Search: actual_timeout = min(query_timeout, max_request_timeout) logger.debug("actual_timeout={0} (default_timeout={1}, ?timeout_limit={2}, max_request_timeout={3})" - .format(self.actual_timeout, default_timeout, query_timeout, max_request_timeout)) + .format(actual_timeout, default_timeout, query_timeout, max_request_timeout)) return requests, actual_timeout @@ -428,6 +444,8 @@ class Search: class SearchWithPlugins(Search): """Similar to the Search class but call the plugins.""" + __slots__ = 'ordered_plugin_list', 'request' + def __init__(self, search_query, ordered_plugin_list, request): super().__init__(search_query) self.ordered_plugin_list = ordered_plugin_list diff --git a/searx/webadapter.py b/searx/webadapter.py index 982517eaa..1ec84bbdc 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -1,19 +1,22 @@ +import typing from searx.exceptions import SearxParameterException from searx.query import RawTextQuery, VALID_LANGUAGE_CODE from searx.engines import categories, engines from searx.search import SearchQuery, EngineRef +from searx.preferences import Preferences # remove duplicate queries. # FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music' -def deduplicate_engineref_list(engineref_list): +def deduplicate_engineref_list(engineref_list: typing.List[EngineRef]) -> typing.List[EngineRef]: engineref_dict = {q.category + '|' + q.name: q for q in engineref_list} return engineref_dict.values() -def validate_engineref_list(engineref_list, preferences): +def validate_engineref_list(engineref_list: typing.List[EngineRef], preferences: Preferences): """ Validate query_engines according to the preferences + Returns: list of existing engines with a validated token list of unknown engine @@ -36,14 +39,14 @@ def validate_engineref_list(engineref_list, preferences): return valid, unknown, no_token -def parse_pageno(form): +def parse_pageno(form: typing.Dict[str, str]) -> int: pageno_param = form.get('pageno', '1') if not pageno_param.isdigit() or int(pageno_param) < 1: raise SearxParameterException('pageno', pageno_param) return int(pageno_param) -def parse_lang(raw_text_query, form, preferences): +def parse_lang(raw_text_query: RawTextQuery, form: typing.Dict[str, str], preferences: Preferences) -> str: # get language # set specific language if set on request, query or preferences # TODO support search with multible languages @@ -61,7 +64,7 @@ def parse_lang(raw_text_query, form, preferences): return query_lang -def parse_safesearch(form, preferences): +def parse_safesearch(form: typing.Dict[str, str], preferences: Preferences) -> int: if 'safesearch' in form: query_safesearch = form.get('safesearch') # first check safesearch @@ -78,7 +81,7 @@ def parse_safesearch(form, preferences): return query_safesearch -def parse_time_range(form): +def parse_time_range(form: typing.Dict[str, str]) -> str: query_time_range = form.get('time_range') # check time_range query_time_range = None if query_time_range in ('', 'None') else query_time_range @@ -87,7 +90,7 @@ def parse_time_range(form): return query_time_range -def parse_timeout(raw_text_query, form): +def parse_timeout(raw_text_query: RawTextQuery, form: typing.Dict[str, str]) -> typing.Optional[float]: query_timeout = raw_text_query.timeout_limit if query_timeout is None and 'timeout_limit' in form: raw_time_limit = form.get('timeout_limit') @@ -187,7 +190,8 @@ def parse_generic(form, preferences, disabled_engines): return query_engineref_list, query_categories -def get_search_query_from_webapp(preferences, form): +def get_search_query_from_webapp(preferences: Preferences, form: typing.Dict[str, str])\ + -> typing.Tuple[SearchQuery, RawTextQuery, typing.List[EngineRef], typing.List[EngineRef]]: # no text for the query ? if not form.get('q'): raise SearxParameterException('q', '') diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py index bd302c741..36135913c 100644 --- a/tests/unit/test_search.py +++ b/tests/unit/test_search.py @@ -3,6 +3,7 @@ from searx.testing import SearxTestCase from searx.search import SearchQuery, EngineRef import searx.search +import searx.engines SAFESEARCH = 0 diff --git a/utils/standalone_searx.py b/utils/standalone_searx.py index d969c398d..3aab7a6cc 100755 --- a/utils/standalone_searx.py +++ b/utils/standalone_searx.py @@ -30,6 +30,7 @@ import codecs import searx.query import searx.search import searx.engines +import searx.webapdater import searx.preferences import searx.webadapter import argparse From f2f3300bde0cc304f80809ff766fc557b5486098 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 24 Sep 2020 16:26:00 +0200 Subject: [PATCH 8/8] [mod] more typing --- searx/webadapter.py | 51 +++++++++++++++++++++++---------------------- searx/webapp.py | 6 +++--- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/searx/webadapter.py b/searx/webadapter.py index 1ec84bbdc..835cf2276 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -1,4 +1,4 @@ -import typing +from typing import Dict, List, Optional, Tuple from searx.exceptions import SearxParameterException from searx.query import RawTextQuery, VALID_LANGUAGE_CODE from searx.engines import categories, engines @@ -8,19 +8,19 @@ from searx.preferences import Preferences # remove duplicate queries. # FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music' -def deduplicate_engineref_list(engineref_list: typing.List[EngineRef]) -> typing.List[EngineRef]: +def deduplicate_engineref_list(engineref_list: List[EngineRef]) -> List[EngineRef]: engineref_dict = {q.category + '|' + q.name: q for q in engineref_list} return engineref_dict.values() -def validate_engineref_list(engineref_list: typing.List[EngineRef], preferences: Preferences): - """ - Validate query_engines according to the preferences +def validate_engineref_list(engineref_list: List[EngineRef], preferences: Preferences)\ + -> Tuple[List[EngineRef], List[EngineRef], List[EngineRef]]: + """Validate query_engines according to the preferences - Returns: - list of existing engines with a validated token - list of unknown engine - list of engine with invalid token according to the preferences + Returns: + List[EngineRef]: list of existing engines with a validated token + List[EngineRef]: list of unknown engine + List[EngineRef]: list of engine with invalid token according to the preferences """ valid = [] unknown = [] @@ -39,14 +39,14 @@ def validate_engineref_list(engineref_list: typing.List[EngineRef], preferences: return valid, unknown, no_token -def parse_pageno(form: typing.Dict[str, str]) -> int: +def parse_pageno(form: Dict[str, str]) -> int: pageno_param = form.get('pageno', '1') if not pageno_param.isdigit() or int(pageno_param) < 1: raise SearxParameterException('pageno', pageno_param) return int(pageno_param) -def parse_lang(raw_text_query: RawTextQuery, form: typing.Dict[str, str], preferences: Preferences) -> str: +def parse_lang(preferences: Preferences, form: Dict[str, str], raw_text_query: RawTextQuery) -> str: # get language # set specific language if set on request, query or preferences # TODO support search with multible languages @@ -64,7 +64,7 @@ def parse_lang(raw_text_query: RawTextQuery, form: typing.Dict[str, str], prefer return query_lang -def parse_safesearch(form: typing.Dict[str, str], preferences: Preferences) -> int: +def parse_safesearch(preferences: Preferences, form: Dict[str, str]) -> int: if 'safesearch' in form: query_safesearch = form.get('safesearch') # first check safesearch @@ -81,7 +81,7 @@ def parse_safesearch(form: typing.Dict[str, str], preferences: Preferences) -> i return query_safesearch -def parse_time_range(form: typing.Dict[str, str]) -> str: +def parse_time_range(form: Dict[str, str]) -> str: query_time_range = form.get('time_range') # check time_range query_time_range = None if query_time_range in ('', 'None') else query_time_range @@ -90,7 +90,7 @@ def parse_time_range(form: typing.Dict[str, str]) -> str: return query_time_range -def parse_timeout(raw_text_query: RawTextQuery, form: typing.Dict[str, str]) -> typing.Optional[float]: +def parse_timeout(form: Dict[str, str], raw_text_query: RawTextQuery) -> Optional[float]: query_timeout = raw_text_query.timeout_limit if query_timeout is None and 'timeout_limit' in form: raw_time_limit = form.get('timeout_limit') @@ -103,7 +103,7 @@ def parse_timeout(raw_text_query: RawTextQuery, form: typing.Dict[str, str]) -> raise SearxParameterException('timeout_limit', raw_time_limit) -def parse_specific(raw_text_query: RawTextQuery): +def parse_specific(raw_text_query: RawTextQuery) -> Tuple[List[EngineRef], List[str]]: query_engineref_list = raw_text_query.enginerefs additional_categories = set() for engineref in raw_text_query.enginerefs: @@ -115,7 +115,7 @@ def parse_specific(raw_text_query: RawTextQuery): return query_engineref_list, query_categories -def parse_category_form(query_categories, name, value): +def parse_category_form(query_categories: List[str], name: str, value: str) -> None: if name == 'categories': query_categories.extend(categ for categ in map(str.strip, value.split(',')) if categ in categories) elif name.startswith('category_'): @@ -133,7 +133,7 @@ def parse_category_form(query_categories, name, value): query_categories.remove(category) -def get_selected_categories(form, preferences): +def get_selected_categories(preferences: Preferences, form: Dict[str, str]) -> List[str]: selected_categories = [] if form is not None: @@ -156,7 +156,8 @@ def get_selected_categories(form, preferences): return selected_categories -def parse_generic(form, preferences, disabled_engines): +def parse_generic(preferences: Preferences, form: Dict[str, str], disabled_engines: List[str])\ + -> Tuple[List[EngineRef], List[str]]: query_engineref_list = [] query_categories = [] @@ -178,7 +179,7 @@ def parse_generic(form, preferences, disabled_engines): for engine in query_engineref_list)) else: if not query_categories: - query_categories = get_selected_categories(None, preferences) + query_categories = get_selected_categories(preferences, None) # using all engines for that search, which are # declared under the specific categories @@ -190,8 +191,8 @@ def parse_generic(form, preferences, disabled_engines): return query_engineref_list, query_categories -def get_search_query_from_webapp(preferences: Preferences, form: typing.Dict[str, str])\ - -> typing.Tuple[SearchQuery, RawTextQuery, typing.List[EngineRef], typing.List[EngineRef]]: +def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])\ + -> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef]]: # no text for the query ? if not form.get('q'): raise SearxParameterException('q', '') @@ -206,10 +207,10 @@ def get_search_query_from_webapp(preferences: Preferences, form: typing.Dict[str # set query query = raw_text_query.getQuery() query_pageno = parse_pageno(form) - query_lang = parse_lang(raw_text_query, form, preferences) - query_safesearch = parse_safesearch(form, preferences) + query_lang = parse_lang(preferences, form, raw_text_query) + query_safesearch = parse_safesearch(preferences, form) query_time_range = parse_time_range(form) - query_timeout = parse_timeout(raw_text_query, form) + query_timeout = parse_timeout(form, raw_text_query) external_bang = raw_text_query.external_bang if raw_text_query.enginerefs and raw_text_query.specific: @@ -219,7 +220,7 @@ def get_search_query_from_webapp(preferences: Preferences, form: typing.Dict[str else: # otherwise, using defined categories to # calculate which engines should be used - query_engineref_list, query_categories = parse_generic(form, preferences, disabled_engines) + query_engineref_list, query_categories = parse_generic(preferences, form, disabled_engines) query_engineref_list = deduplicate_engineref_list(query_engineref_list) query_engineref_list, query_engineref_list_unknown, query_engineref_list_notoken =\ diff --git a/searx/webapp.py b/searx/webapp.py index 119f9f3b7..f3bbbc2b8 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -513,7 +513,7 @@ def index_error(output_format, error_message): request.errors.append(gettext('search error')) return render( 'index.html', - selected_categories=get_selected_categories(request.form, request.preferences), + selected_categories=get_selected_categories(request.preferences, request.form), ) @@ -535,7 +535,7 @@ def index(): if output_format == 'html': return render( 'index.html', - selected_categories=get_selected_categories(request.form, request.preferences), + selected_categories=get_selected_categories(request.preferences, request.form), ) else: return index_error(output_format, 'No query'), 400 @@ -816,7 +816,7 @@ def preferences(): # end of stats return render('preferences.html', - selected_categories=get_selected_categories(request.form, request.preferences), + selected_categories=get_selected_categories(request.preferences, request.form), all_categories=_get_ordered_categories(), locales=settings['locales'], current_locale=request.preferences.get_value("locale"),