From 44f4a9d49a0d78668b37fca8558ae63c5fdef271 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Tue, 2 Mar 2021 14:24:55 +0100 Subject: [PATCH] [enh] add ability to send engine data to subsequent requests --- searx/engines/__init__.py | 1 + searx/results.py | 6 +++++- searx/search/__init__.py | 2 ++ searx/search/models.py | 8 ++++++-- searx/templates/oscar/results.html | 11 +++++++++++ searx/templates/simple/results.html | 9 +++++++++ searx/webadapter.py | 14 +++++++++++++- searx/webapp.py | 1 + tests/unit/test_webapp.py | 3 ++- 9 files changed, 50 insertions(+), 5 deletions(-) diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 9b9e043c8..7270724b6 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -327,6 +327,7 @@ def _set_https_support_for_engine(engine): 'is_test': True, 'category': 'files', 'raise_for_status': True, + 'engine_data': {}, }) if 'url' not in params: diff --git a/searx/results.py b/searx/results.py index fb7e816eb..b3b874118 100644 --- a/searx/results.py +++ b/searx/results.py @@ -1,4 +1,5 @@ import re +from collections import defaultdict from operator import itemgetter from threading import RLock from urllib.parse import urlparse, unquote @@ -144,7 +145,7 @@ class ResultContainer: """docstring for ResultContainer""" __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\ - '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url' + '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url', 'engine_data' def __init__(self): super().__init__() @@ -154,6 +155,7 @@ class ResultContainer: self.answers = {} self.corrections = set() self._number_of_results = [] + self.engine_data = defaultdict(dict) self._ordered = False self.paging = False self.unresponsive_engines = set() @@ -175,6 +177,8 @@ class ResultContainer: self._merge_infobox(result) elif 'number_of_results' in result: self._number_of_results.append(result['number_of_results']) + elif 'engine_data' in result: + self.engine_data[engine_name][result['key']] = result['engine_data'] else: # standard result (url, title, content) if 'url' in result and not isinstance(result['url'], str): diff --git a/searx/search/__init__.py b/searx/search/__init__.py index f777e8595..27c6c3a88 100644 --- a/searx/search/__init__.py +++ b/searx/search/__init__.py @@ -111,6 +111,8 @@ class Search: if request_params is None: continue + request_params['engine_data'] = self.search_query.engine_data.get(engineref.name, {}) + with threading.RLock(): processor.engine.stats['sent_search_count'] += 1 diff --git a/searx/search/models.py b/searx/search/models.py index 80ceaa223..21c0fe590 100644 --- a/searx/search/models.py +++ b/searx/search/models.py @@ -25,7 +25,7 @@ class SearchQuery: """container for all the search parameters (query, language, etc...)""" __slots__ = 'query', 'engineref_list', 'lang', 'safesearch', 'pageno', 'time_range',\ - 'timeout_limit', 'external_bang' + 'timeout_limit', 'external_bang', 'engine_data' def __init__(self, query: str, @@ -35,7 +35,8 @@ class SearchQuery: pageno: int=1, time_range: typing.Optional[str]=None, timeout_limit: typing.Optional[float]=None, - external_bang: typing.Optional[str]=None): + external_bang: typing.Optional[str]=None, + engine_data: typing.Optional[dict]=None): self.query = query self.engineref_list = engineref_list self.lang = lang @@ -44,6 +45,9 @@ class SearchQuery: self.time_range = time_range self.timeout_limit = timeout_limit self.external_bang = external_bang + self.engine_data = engine_data + if engine_data is None: + self.engine_data = {} @property def categories(self): diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html index eada8cd26..df2d42430 100644 --- a/searx/templates/oscar/results.html +++ b/searx/templates/oscar/results.html @@ -7,6 +7,13 @@ {{- "" -}} {% if timeout_limit %}{% endif -%} {%- endmacro %} +{% macro engine_data_form(engine_data) -%} + {% for engine_name, kv_data in engine_data.items() %} + {% for k, v in kv_data.items() %} + + {% endfor %} + {% endfor %} +{%- endmacro %} {%- macro search_url() %}{{ url_for('search', _external=True) }}?q={{ q|urlencode }}{% if selected_categories %}&categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if pageno > 1 %}&pageno={{ pageno }}{% endif %}{% if time_range %}&time_range={{ time_range }}{% endif %}{% if current_language != 'all' %}&language={{ current_language }}{% endif %}{% endmacro -%} {% block title %}{{ q|e }} - {% endblock %} @@ -142,12 +149,14 @@
{{- "" -}}
{{- search_form_attrs(pageno+1) -}} + {{- engine_data_form(engine_data) -}} {{- "" -}}
{{- "" -}}
{{- "" -}}
{{- search_form_attrs(pageno-1) -}} + {{- engine_data_form(engine_data) -}} {{- "" -}}
{{- "" -}}
@@ -158,12 +167,14 @@
{{- "" -}}
{{- search_form_attrs(pageno-1) -}} + {{- engine_data_form(engine_data) -}} {{- "" -}}
{{- "" -}}
{{- "" -}}
{{- search_form_attrs(pageno+1) -}} + {{- engine_data_form(engine_data) -}} {{- "" -}}
{{- "" -}}
diff --git a/searx/templates/simple/results.html b/searx/templates/simple/results.html index 936de8831..7ece6c7ad 100644 --- a/searx/templates/simple/results.html +++ b/searx/templates/simple/results.html @@ -1,5 +1,12 @@ {% extends "simple/base.html" %} {% from 'simple/macros.html' import icon, icon_small %} +{% macro engine_data_form(engine_data) -%} + {% for engine_name, kv_data in engine_data.items() %} + {% for k, v in kv_data.items() %} + + {% endfor %} + {% endfor %} +{%- endmacro %} {% block title %}{% if method == 'GET' %}{{- q|e -}} -{% endif %}{% endblock %} {% block meta %}{% endblock %} {% block content %} @@ -136,6 +143,7 @@
+ {{- engine_data_form(engine_data) -}} {% for category in selected_categories %} {% endfor %} @@ -152,6 +160,7 @@
+ {{- engine_data_form(engine_data) -}} {% for category in selected_categories %} {% endfor %} diff --git a/searx/webadapter.py b/searx/webadapter.py index ca2604c3e..59d1ffbe3 100644 --- a/searx/webadapter.py +++ b/searx/webadapter.py @@ -1,3 +1,4 @@ +from collections import defaultdict from typing import Dict, List, Optional, Tuple from searx.exceptions import SearxParameterException from searx.webutils import VALID_LANGUAGE_CODE @@ -196,6 +197,15 @@ def parse_generic(preferences: Preferences, form: Dict[str, str], disabled_engin return query_engineref_list +def parse_engine_data(form): + engine_data = defaultdict(dict) + for k, v in form.items(): + if k.startswith("engine_data"): + _, engine, key = k.split('-') + engine_data[engine][key] = v + return engine_data + + def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])\ -> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef]]: # no text for the query ? @@ -217,6 +227,7 @@ def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str]) query_time_range = parse_time_range(form) query_timeout = parse_timeout(form, raw_text_query) external_bang = raw_text_query.external_bang + engine_data = parse_engine_data(form) if not is_locked('categories') and raw_text_query.enginerefs and raw_text_query.specific: # if engines are calculated from query, @@ -232,7 +243,8 @@ def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str]) validate_engineref_list(query_engineref_list, preferences) return (SearchQuery(query, query_engineref_list, query_lang, query_safesearch, query_pageno, - query_time_range, query_timeout, external_bang=external_bang), + query_time_range, query_timeout, external_bang=external_bang, + engine_data=engine_data), raw_text_query, query_engineref_list_unknown, query_engineref_list_notoken) diff --git a/searx/webapp.py b/searx/webapp.py index f1034c270..ed1e9ce49 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -730,6 +730,7 @@ def search(): answers=result_container.answers, corrections=correction_urls, infoboxes=result_container.infoboxes, + engine_data=result_container.engine_data, paging=result_container.paging, unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines), current_language=match_language(search_query.lang, diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index a488faf58..7d7a04fdc 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -57,7 +57,8 @@ class ViewsTestCase(SearxTestCase): results_number=lambda: 3, results_length=lambda: len(test_results), get_timings=lambda: timings, - redirect_url=None) + redirect_url=None, + engine_data={}) self.setattr4test(Search, 'search', search_mock)