2013-10-14 21:09:13 +00:00
|
|
|
|
#!/usr/bin/env python
|
2021-05-26 08:16:34 +00:00
|
|
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
2021-05-26 11:32:04 +00:00
|
|
|
|
"""WebbApp
|
2021-03-18 18:59:01 +00:00
|
|
|
|
|
2021-05-26 11:32:04 +00:00
|
|
|
|
"""
|
2023-02-10 12:40:12 +00:00
|
|
|
|
# pylint: disable=use-dict-literal
|
|
|
|
|
|
2015-01-17 20:54:40 +00:00
|
|
|
|
import hashlib
|
2016-10-16 21:40:56 +00:00
|
|
|
|
import hmac
|
|
|
|
|
import json
|
|
|
|
|
import os
|
2021-05-26 08:16:34 +00:00
|
|
|
|
import sys
|
2022-04-13 15:08:19 +00:00
|
|
|
|
import base64
|
2015-03-10 18:55:22 +00:00
|
|
|
|
|
2021-04-14 15:23:15 +00:00
|
|
|
|
from timeit import default_timer
|
2020-08-06 15:42:46 +00:00
|
|
|
|
from html import escape
|
|
|
|
|
from io import StringIO
|
2022-01-17 06:54:37 +00:00
|
|
|
|
import typing
|
2022-05-07 13:11:05 +00:00
|
|
|
|
from typing import List, Dict, Iterable
|
2021-05-26 08:16:34 +00:00
|
|
|
|
|
[fix] debug log: UnicodeEncodeError: 'ascii' codec can't encode
The issue exists only in the debug log::
--- Logging error ---
Traceback (most recent call last):
File "/usr/lib/python3.9/logging/__init__.py", line 1086, in emit
stream.write(msg + self.terminator)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 79-89: ordinal not in range(128)
Call stack:
File "/usr/local/searx/searx-pyenv/lib/python3.9/site-packages/flask/app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/searx/searx-src/searx/webapp.py", line 1316, in __call__
return self.app(environ, start_response)
File "/usr/local/searx/searx-pyenv/lib/python3.9/site-packages/werkzeug/middleware/proxy_fix.py", line 169, in __call__
return self.app(environ, start_response)
File "/usr/local/searx/searx-pyenv/lib/python3.9/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/searx/searx-pyenv/lib/python3.9/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/searx/searx-pyenv/lib/python3.9/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/searx/searx-src/searx/webapp.py", line 766, in search
number_of_results=format_decimal(number_of_results),
File "/usr/local/searx/searx-pyenv/lib/python3.9/site-packages/flask_babel/__init__.py", line 458, in format_decimal
locale = get_locale()
File "/usr/local/searx/searx-pyenv/lib/python3.9/site-packages/flask_babel/__init__.py", line 226, in get_locale
rv = babel.locale_selector_func()
File "/usr/local/searx/searx-src/searx/webapp.py", line 249, in get_locale
logger.debug("%s uses locale `%s` from %s", request.url, locale, locale_source)
Unable to print the message and arguments - possible formatting error.
Use the traceback above to help find the error.
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2021-04-26 09:36:35 +00:00
|
|
|
|
import urllib
|
2022-01-17 10:16:11 +00:00
|
|
|
|
import urllib.parse
|
2023-08-08 16:12:07 +00:00
|
|
|
|
from urllib.parse import urlencode, urlparse, unquote
|
2021-05-26 08:16:34 +00:00
|
|
|
|
|
|
|
|
|
import httpx
|
2020-08-06 15:42:46 +00:00
|
|
|
|
|
|
|
|
|
from pygments import highlight
|
|
|
|
|
from pygments.lexers import get_lexer_by_name
|
2020-11-03 10:35:53 +00:00
|
|
|
|
from pygments.formatters import HtmlFormatter # pylint: disable=no-name-in-module
|
2020-08-06 15:42:46 +00:00
|
|
|
|
|
2021-05-26 17:43:27 +00:00
|
|
|
|
import flask
|
|
|
|
|
|
2014-02-05 19:24:31 +00:00
|
|
|
|
from flask import (
|
2021-05-26 08:16:34 +00:00
|
|
|
|
Flask,
|
|
|
|
|
render_template,
|
|
|
|
|
url_for,
|
|
|
|
|
make_response,
|
|
|
|
|
redirect,
|
|
|
|
|
send_from_directory,
|
2014-02-05 19:24:31 +00:00
|
|
|
|
)
|
2022-01-24 10:44:18 +00:00
|
|
|
|
from flask.wrappers import Response
|
2016-06-07 21:08:48 +00:00
|
|
|
|
from flask.json import jsonify
|
2021-05-26 08:16:34 +00:00
|
|
|
|
|
|
|
|
|
from flask_babel import (
|
|
|
|
|
Babel,
|
|
|
|
|
gettext,
|
|
|
|
|
format_decimal,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
from searx import (
|
2021-09-02 14:15:22 +00:00
|
|
|
|
logger,
|
|
|
|
|
get_setting,
|
2021-05-26 08:16:34 +00:00
|
|
|
|
settings,
|
|
|
|
|
searx_debug,
|
|
|
|
|
)
|
2022-03-12 09:18:08 +00:00
|
|
|
|
|
|
|
|
|
from searx import infopage
|
2023-10-02 14:36:07 +00:00
|
|
|
|
from searx import limiter
|
|
|
|
|
from searx.botdetection import link_token
|
|
|
|
|
|
2021-09-18 08:59:56 +00:00
|
|
|
|
from searx.data import ENGINE_DESCRIPTIONS
|
2023-06-18 14:43:48 +00:00
|
|
|
|
from searx.results import Timing
|
2021-05-28 16:45:22 +00:00
|
|
|
|
from searx.settings_defaults import OUTPUT_FORMATS
|
2024-06-12 16:01:18 +00:00
|
|
|
|
from searx.settings_loader import DEFAULT_SETTINGS_FILE
|
2016-11-30 17:43:03 +00:00
|
|
|
|
from searx.exceptions import SearxParameterException
|
2021-05-26 08:16:34 +00:00
|
|
|
|
from searx.engines import (
|
2022-07-24 07:32:05 +00:00
|
|
|
|
DEFAULT_CATEGORY,
|
2021-05-26 08:16:34 +00:00
|
|
|
|
categories,
|
|
|
|
|
engines,
|
|
|
|
|
engine_shortcuts,
|
|
|
|
|
)
|
2023-06-18 14:43:48 +00:00
|
|
|
|
|
|
|
|
|
from searx import webutils
|
2020-09-19 16:25:24 +00:00
|
|
|
|
from searx.webutils import (
|
2021-05-26 08:16:34 +00:00
|
|
|
|
highlight_content,
|
|
|
|
|
get_static_files,
|
|
|
|
|
get_result_templates,
|
|
|
|
|
get_themes,
|
2023-06-18 14:43:48 +00:00
|
|
|
|
exception_classname_to_text,
|
2021-05-26 08:16:34 +00:00
|
|
|
|
new_hmac,
|
2021-12-28 12:44:28 +00:00
|
|
|
|
is_hmac_of,
|
2021-05-26 08:16:34 +00:00
|
|
|
|
is_flask_run_cmdline,
|
2022-01-03 06:24:20 +00:00
|
|
|
|
group_engines_in_tab,
|
2021-05-26 08:16:34 +00:00
|
|
|
|
)
|
|
|
|
|
from searx.webadapter import (
|
|
|
|
|
get_search_query_from_webapp,
|
|
|
|
|
get_selected_categories,
|
2023-01-31 11:40:23 +00:00
|
|
|
|
parse_lang,
|
2021-05-26 08:16:34 +00:00
|
|
|
|
)
|
2024-09-17 14:43:48 +00:00
|
|
|
|
from searx.utils import gen_useragent, dict_subset
|
2021-08-18 17:51:07 +00:00
|
|
|
|
from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
|
2016-11-14 21:15:03 +00:00
|
|
|
|
from searx.query import RawTextQuery
|
2023-05-23 16:16:37 +00:00
|
|
|
|
from searx.plugins import Plugin, plugins, initialize as plugin_initialize
|
2017-11-01 12:58:48 +00:00
|
|
|
|
from searx.plugins.oa_doi_rewrite import get_doi_resolver
|
2021-05-26 08:16:34 +00:00
|
|
|
|
from searx.preferences import (
|
|
|
|
|
Preferences,
|
2023-01-30 17:40:22 +00:00
|
|
|
|
ClientPref,
|
2021-05-26 08:16:34 +00:00
|
|
|
|
ValidationException,
|
|
|
|
|
)
|
2021-09-02 14:15:22 +00:00
|
|
|
|
from searx.answerers import (
|
|
|
|
|
answerers,
|
|
|
|
|
ask,
|
|
|
|
|
)
|
2024-09-17 14:43:48 +00:00
|
|
|
|
from searx.metrics import get_engines_stats, get_engine_errors, get_reliabilities, histogram, counter, openmetrics
|
2021-06-08 08:08:41 +00:00
|
|
|
|
from searx.flaskfix import patch_application
|
2021-05-26 08:16:34 +00:00
|
|
|
|
|
2022-06-10 15:01:12 +00:00
|
|
|
|
from searx.locales import (
|
|
|
|
|
LOCALE_NAMES,
|
|
|
|
|
RTL_LOCALES,
|
|
|
|
|
localeselector,
|
|
|
|
|
locales_initialize,
|
2023-02-07 13:11:58 +00:00
|
|
|
|
match_locale,
|
2022-06-10 15:01:12 +00:00
|
|
|
|
)
|
2021-05-26 08:16:34 +00:00
|
|
|
|
|
2022-06-10 15:01:12 +00:00
|
|
|
|
# renaming names from searx imports ...
|
2021-05-26 08:16:34 +00:00
|
|
|
|
from searx.autocomplete import search_autocomplete, backends as autocomplete_backends
|
2024-08-19 15:47:54 +00:00
|
|
|
|
from searx import favicons
|
|
|
|
|
|
2022-11-11 20:58:32 +00:00
|
|
|
|
from searx.redisdb import initialize as redis_initialize
|
2022-10-10 17:31:22 +00:00
|
|
|
|
from searx.sxng_locales import sxng_locales
|
2021-05-26 08:16:34 +00:00
|
|
|
|
from searx.search import SearchWithPlugins, initialize as search_initialize
|
2021-08-14 18:12:11 +00:00
|
|
|
|
from searx.network import stream as http_stream, set_context_network_name
|
2021-05-26 08:16:34 +00:00
|
|
|
|
from searx.search.checker import get_result as checker_get_result
|
|
|
|
|
|
|
|
|
|
logger = logger.getChild('webapp')
|
2013-12-01 22:52:49 +00:00
|
|
|
|
|
2020-12-21 09:21:10 +00:00
|
|
|
|
# check secret_key
|
|
|
|
|
if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
|
|
|
|
|
logger.error('server.secret_key is not changed. Please use something else instead of ultrasecretkey.')
|
2021-05-26 11:32:04 +00:00
|
|
|
|
sys.exit(1)
|
2020-12-21 09:21:10 +00:00
|
|
|
|
|
2017-01-06 12:23:30 +00:00
|
|
|
|
# about static
|
2021-05-28 16:45:22 +00:00
|
|
|
|
logger.debug('static directory is %s', settings['ui']['static_path'])
|
|
|
|
|
static_files = get_static_files(settings['ui']['static_path'])
|
2014-10-09 17:26:02 +00:00
|
|
|
|
|
2017-01-06 12:23:30 +00:00
|
|
|
|
# about templates
|
2021-05-28 16:45:22 +00:00
|
|
|
|
logger.debug('templates directory is %s', settings['ui']['templates_path'])
|
2015-08-02 17:38:27 +00:00
|
|
|
|
default_theme = settings['ui']['default_theme']
|
2021-05-28 16:45:22 +00:00
|
|
|
|
templates_path = settings['ui']['templates_path']
|
2017-01-06 12:52:59 +00:00
|
|
|
|
themes = get_themes(templates_path)
|
2017-01-06 12:23:30 +00:00
|
|
|
|
result_templates = get_result_templates(templates_path)
|
2014-04-24 23:46:40 +00:00
|
|
|
|
|
2021-05-26 11:32:04 +00:00
|
|
|
|
STATS_SORT_PARAMETERS = {
|
|
|
|
|
'name': (False, 'name', ''),
|
2022-09-02 08:30:38 +00:00
|
|
|
|
'score': (True, 'score_per_result', 0),
|
2021-05-26 11:32:04 +00:00
|
|
|
|
'result_count': (True, 'result_count', 0),
|
|
|
|
|
'time': (False, 'total', 0),
|
|
|
|
|
'reliability': (False, 'reliability', 100),
|
|
|
|
|
}
|
|
|
|
|
|
2017-01-06 12:23:30 +00:00
|
|
|
|
# Flask app
|
2021-12-27 08:26:22 +00:00
|
|
|
|
app = Flask(__name__, static_folder=settings['ui']['static_path'], template_folder=templates_path)
|
2014-01-19 21:59:01 +00:00
|
|
|
|
|
2015-02-14 00:42:06 +00:00
|
|
|
|
app.jinja_env.trim_blocks = True
|
|
|
|
|
app.jinja_env.lstrip_blocks = True
|
2020-11-03 10:35:53 +00:00
|
|
|
|
app.jinja_env.add_extension('jinja2.ext.loopcontrols') # pylint: disable=no-member
|
2021-12-28 13:51:21 +00:00
|
|
|
|
app.jinja_env.filters['group_engines_in_tab'] = group_engines_in_tab # pylint: disable=no-member
|
2014-01-19 21:59:01 +00:00
|
|
|
|
app.secret_key = settings['server']['secret_key']
|
2014-01-14 17:17:19 +00:00
|
|
|
|
|
2020-02-24 04:46:26 +00:00
|
|
|
|
|
2022-01-17 06:54:37 +00:00
|
|
|
|
class ExtendedRequest(flask.Request):
|
|
|
|
|
"""This class is never initialized and only used for type checking."""
|
|
|
|
|
|
|
|
|
|
preferences: Preferences
|
|
|
|
|
errors: List[str]
|
|
|
|
|
user_plugins: List[Plugin]
|
|
|
|
|
form: Dict[str, str]
|
|
|
|
|
start_time: float
|
|
|
|
|
render_time: float
|
2022-01-17 07:06:31 +00:00
|
|
|
|
timings: List[Timing]
|
2022-01-17 06:54:37 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
request = typing.cast(ExtendedRequest, flask.request)
|
|
|
|
|
|
|
|
|
|
|
2021-08-17 06:18:30 +00:00
|
|
|
|
def get_locale():
|
2022-06-10 15:01:12 +00:00
|
|
|
|
locale = localeselector()
|
2021-08-17 06:18:30 +00:00
|
|
|
|
logger.debug("%s uses locale `%s`", urllib.parse.quote(request.url), locale)
|
|
|
|
|
return locale
|
2020-02-24 04:46:26 +00:00
|
|
|
|
|
2014-01-19 22:04:09 +00:00
|
|
|
|
|
2023-01-20 09:19:18 +00:00
|
|
|
|
babel = Babel(app, locale_selector=get_locale)
|
|
|
|
|
|
|
|
|
|
|
2021-08-17 06:18:30 +00:00
|
|
|
|
def _get_browser_language(req, lang_list):
|
2023-01-30 17:40:22 +00:00
|
|
|
|
client = ClientPref.from_http_request(req)
|
|
|
|
|
locale = match_locale(client.locale_tag, lang_list, fallback='en')
|
|
|
|
|
return locale
|
2014-01-21 23:15:23 +00:00
|
|
|
|
|
|
|
|
|
|
2021-10-06 08:26:40 +00:00
|
|
|
|
def _get_locale_rfc5646(locale):
|
|
|
|
|
"""Get locale name for <html lang="...">
|
|
|
|
|
Chrom* browsers don't detect the language when there is a subtag (ie a territory).
|
|
|
|
|
For example "zh-TW" is detected but not "zh-Hant-TW".
|
|
|
|
|
This function returns a locale without the subtag.
|
|
|
|
|
"""
|
|
|
|
|
parts = locale.split('-')
|
|
|
|
|
return parts[0].lower() + '-' + parts[-1].upper()
|
|
|
|
|
|
|
|
|
|
|
2014-12-20 22:33:03 +00:00
|
|
|
|
# code-highlighter
|
|
|
|
|
@app.template_filter('code_highlighter')
|
|
|
|
|
def code_highlighter(codelines, language=None):
|
|
|
|
|
if not language:
|
|
|
|
|
language = 'text'
|
|
|
|
|
|
2014-12-22 15:26:45 +00:00
|
|
|
|
try:
|
2022-09-27 15:01:00 +00:00
|
|
|
|
# find lexer by programming language
|
2014-12-22 15:26:45 +00:00
|
|
|
|
lexer = get_lexer_by_name(language, stripall=True)
|
2021-05-26 11:32:04 +00:00
|
|
|
|
|
|
|
|
|
except Exception as e: # pylint: disable=broad-except
|
2023-09-06 17:12:27 +00:00
|
|
|
|
logger.warning("pygments lexer: %s " % e)
|
2014-12-22 15:26:45 +00:00
|
|
|
|
# if lexer is not found, using default one
|
|
|
|
|
lexer = get_lexer_by_name('text', stripall=True)
|
|
|
|
|
|
2014-12-20 22:33:03 +00:00
|
|
|
|
html_code = ''
|
|
|
|
|
tmp_code = ''
|
|
|
|
|
last_line = None
|
2022-01-24 10:44:18 +00:00
|
|
|
|
line_code_start = None
|
2014-12-20 22:33:03 +00:00
|
|
|
|
|
|
|
|
|
# parse lines
|
|
|
|
|
for line, code in codelines:
|
|
|
|
|
if not last_line:
|
|
|
|
|
line_code_start = line
|
|
|
|
|
|
|
|
|
|
# new codeblock is detected
|
2021-12-27 08:26:22 +00:00
|
|
|
|
if last_line is not None and last_line + 1 != line:
|
2014-12-20 22:33:03 +00:00
|
|
|
|
|
|
|
|
|
# highlight last codepart
|
2021-12-27 08:26:22 +00:00
|
|
|
|
formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start, cssclass="code-highlight")
|
2014-12-20 22:33:03 +00:00
|
|
|
|
html_code = html_code + highlight(tmp_code, lexer, formatter)
|
2014-12-22 15:26:45 +00:00
|
|
|
|
|
2014-12-20 22:33:03 +00:00
|
|
|
|
# reset conditions for next codepart
|
|
|
|
|
tmp_code = ''
|
|
|
|
|
line_code_start = line
|
|
|
|
|
|
|
|
|
|
# add codepart
|
|
|
|
|
tmp_code += code + '\n'
|
2014-12-22 15:26:45 +00:00
|
|
|
|
|
2014-12-20 22:33:03 +00:00
|
|
|
|
# update line
|
|
|
|
|
last_line = line
|
|
|
|
|
|
|
|
|
|
# highlight last codepart
|
2020-09-16 06:53:05 +00:00
|
|
|
|
formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start, cssclass="code-highlight")
|
2014-12-20 22:33:03 +00:00
|
|
|
|
html_code = html_code + highlight(tmp_code, lexer, formatter)
|
|
|
|
|
|
|
|
|
|
return html_code
|
|
|
|
|
|
|
|
|
|
|
2022-01-17 10:16:11 +00:00
|
|
|
|
def get_result_template(theme_name: str, template_name: str):
|
2021-05-26 11:32:04 +00:00
|
|
|
|
themed_path = theme_name + '/result_templates/' + template_name
|
2015-01-01 17:59:53 +00:00
|
|
|
|
if themed_path in result_templates:
|
|
|
|
|
return themed_path
|
|
|
|
|
return 'result_templates/' + template_name
|
|
|
|
|
|
|
|
|
|
|
2022-05-07 13:11:05 +00:00
|
|
|
|
def custom_url_for(endpoint: str, **values):
|
2020-10-03 13:02:44 +00:00
|
|
|
|
suffix = ""
|
2015-01-01 16:48:12 +00:00
|
|
|
|
if endpoint == 'static' and values.get('filename'):
|
2022-03-29 17:02:21 +00:00
|
|
|
|
file_hash = static_files.get(values['filename'])
|
|
|
|
|
if not file_hash:
|
|
|
|
|
# try file in the current theme
|
2022-05-07 13:11:05 +00:00
|
|
|
|
theme_name = request.preferences.get_value('theme')
|
2022-03-29 17:02:21 +00:00
|
|
|
|
filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
|
2022-03-20 15:28:14 +00:00
|
|
|
|
file_hash = static_files.get(filename_with_theme)
|
2022-03-29 19:17:36 +00:00
|
|
|
|
if file_hash:
|
|
|
|
|
values['filename'] = filename_with_theme
|
2022-03-29 17:02:21 +00:00
|
|
|
|
if get_setting('ui.static_use_hash') and file_hash:
|
|
|
|
|
suffix = "?" + file_hash
|
2022-03-13 21:15:27 +00:00
|
|
|
|
if endpoint == 'info' and 'locale' not in values:
|
|
|
|
|
locale = request.preferences.get_value('locale')
|
2024-05-28 15:56:54 +00:00
|
|
|
|
if infopage.INFO_PAGES.get_page(values['pagename'], locale) is None:
|
|
|
|
|
locale = infopage.INFO_PAGES.locale_default
|
2022-03-13 21:15:27 +00:00
|
|
|
|
values['locale'] = locale
|
2020-10-03 13:02:44 +00:00
|
|
|
|
return url_for(endpoint, **values) + suffix
|
2014-04-24 23:46:40 +00:00
|
|
|
|
|
|
|
|
|
|
2022-08-27 10:01:04 +00:00
|
|
|
|
def morty_proxify(url: str):
|
2016-10-16 22:22:41 +00:00
|
|
|
|
if url.startswith('//'):
|
|
|
|
|
url = 'https:' + url
|
|
|
|
|
|
2022-08-27 10:01:04 +00:00
|
|
|
|
if not settings['result_proxy']['url']:
|
2016-10-16 22:22:41 +00:00
|
|
|
|
return url
|
|
|
|
|
|
2022-01-24 10:44:18 +00:00
|
|
|
|
url_params = dict(mortyurl=url)
|
2016-10-29 21:12:24 +00:00
|
|
|
|
|
2022-08-27 10:01:04 +00:00
|
|
|
|
if settings['result_proxy']['key']:
|
2021-12-27 08:26:22 +00:00
|
|
|
|
url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'], url.encode(), hashlib.sha256).hexdigest()
|
|
|
|
|
|
|
|
|
|
return '{0}?{1}'.format(settings['result_proxy']['url'], urlencode(url_params))
|
2016-10-16 22:22:41 +00:00
|
|
|
|
|
|
|
|
|
|
2022-01-17 10:16:11 +00:00
|
|
|
|
def image_proxify(url: str):
|
2015-01-16 15:26:15 +00:00
|
|
|
|
|
|
|
|
|
if url.startswith('//'):
|
|
|
|
|
url = 'https:' + url
|
|
|
|
|
|
2016-04-09 16:32:07 +00:00
|
|
|
|
if not request.preferences.get_value('image_proxy'):
|
2015-01-16 15:26:15 +00:00
|
|
|
|
return url
|
|
|
|
|
|
2020-06-22 11:57:33 +00:00
|
|
|
|
if url.startswith('data:image/'):
|
|
|
|
|
# 50 is an arbitrary number to get only the beginning of the image.
|
2021-12-27 08:26:22 +00:00
|
|
|
|
partial_base64 = url[len('data:image/') : 50].split(';')
|
|
|
|
|
if (
|
|
|
|
|
len(partial_base64) == 2
|
|
|
|
|
and partial_base64[0] in ['gif', 'png', 'jpeg', 'pjpeg', 'webp', 'tiff', 'bmp']
|
|
|
|
|
and partial_base64[1].startswith('base64,')
|
|
|
|
|
):
|
2020-06-22 11:57:33 +00:00
|
|
|
|
return url
|
2021-05-26 11:32:04 +00:00
|
|
|
|
return None
|
2019-01-18 08:04:40 +00:00
|
|
|
|
|
2022-08-27 10:01:04 +00:00
|
|
|
|
if settings['result_proxy']['url']:
|
|
|
|
|
return morty_proxify(url)
|
2016-10-30 20:15:18 +00:00
|
|
|
|
|
2020-08-06 15:42:46 +00:00
|
|
|
|
h = new_hmac(settings['server']['secret_key'], url.encode())
|
2015-01-18 08:54:24 +00:00
|
|
|
|
|
2021-12-27 08:26:22 +00:00
|
|
|
|
return '{0}?{1}'.format(url_for('image_proxy'), urlencode(dict(url=url.encode(), h=h)))
|
2015-01-16 15:26:15 +00:00
|
|
|
|
|
|
|
|
|
|
2021-03-16 10:07:04 +00:00
|
|
|
|
def get_translations():
|
|
|
|
|
return {
|
|
|
|
|
# when there is autocompletion
|
2021-09-18 09:01:39 +00:00
|
|
|
|
'no_item_found': gettext('No item found'),
|
|
|
|
|
# /preferences: the source of the engine description (wikipedata, wikidata, website)
|
|
|
|
|
'Source': gettext('Source'),
|
2022-01-23 10:37:57 +00:00
|
|
|
|
# infinite scroll
|
|
|
|
|
'error_loading_next_page': gettext('Error loading the next page'),
|
2021-03-16 10:07:04 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2022-07-23 16:26:43 +00:00
|
|
|
|
def get_enabled_categories(category_names: Iterable[str]):
|
|
|
|
|
"""The categories in ``category_names```for which there is no active engine
|
|
|
|
|
are filtered out and a reduced list is returned."""
|
|
|
|
|
|
|
|
|
|
enabled_engines = [item[0] for item in request.preferences.engines.get_enabled()]
|
|
|
|
|
enabled_categories = set()
|
|
|
|
|
for engine_name in enabled_engines:
|
|
|
|
|
enabled_categories.update(engines[engine_name].categories)
|
|
|
|
|
return [x for x in category_names if x in enabled_categories]
|
2021-06-08 09:56:30 +00:00
|
|
|
|
|
|
|
|
|
|
2022-01-17 10:16:11 +00:00
|
|
|
|
def get_pretty_url(parsed_url: urllib.parse.ParseResult):
|
2024-11-27 12:42:08 +00:00
|
|
|
|
url_formatting_pref = request.preferences.get_value('url_formatting')
|
|
|
|
|
|
|
|
|
|
if url_formatting_pref == 'full':
|
|
|
|
|
return [parsed_url.geturl()]
|
|
|
|
|
|
|
|
|
|
if url_formatting_pref == 'host':
|
|
|
|
|
return [parsed_url.netloc]
|
|
|
|
|
|
2021-11-21 20:38:00 +00:00
|
|
|
|
path = parsed_url.path
|
|
|
|
|
path = path[:-1] if len(path) > 0 and path[-1] == '/' else path
|
2022-04-07 15:15:58 +00:00
|
|
|
|
path = unquote(path.replace("/", " › "))
|
2021-12-27 08:26:22 +00:00
|
|
|
|
return [parsed_url.scheme + "://" + parsed_url.netloc, path]
|
2021-11-21 20:38:00 +00:00
|
|
|
|
|
|
|
|
|
|
2022-04-13 15:08:19 +00:00
|
|
|
|
def get_client_settings():
|
|
|
|
|
req_pref = request.preferences
|
|
|
|
|
return {
|
|
|
|
|
'autocomplete_provider': req_pref.get_value('autocomplete'),
|
2022-05-07 16:23:10 +00:00
|
|
|
|
'autocomplete_min': get_setting('search.autocomplete_min'),
|
|
|
|
|
'http_method': req_pref.get_value('method'),
|
2022-04-13 15:08:19 +00:00
|
|
|
|
'infinite_scroll': req_pref.get_value('infinite_scroll'),
|
|
|
|
|
'translations': get_translations(),
|
2023-09-19 14:45:11 +00:00
|
|
|
|
'search_on_category_select': req_pref.get_value('search_on_category_select'),
|
2023-09-16 15:14:18 +00:00
|
|
|
|
'hotkeys': req_pref.get_value('hotkeys'),
|
2024-11-27 12:42:08 +00:00
|
|
|
|
'url_formatting': req_pref.get_value('url_formatting'),
|
2022-04-13 15:08:19 +00:00
|
|
|
|
'theme_static_path': custom_url_for('static', filename='themes/simple'),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2022-05-07 13:11:05 +00:00
|
|
|
|
def render(template_name: str, **kwargs):
|
2023-02-07 13:11:58 +00:00
|
|
|
|
# pylint: disable=too-many-statements
|
2022-04-13 15:08:19 +00:00
|
|
|
|
kwargs['client_settings'] = str(
|
|
|
|
|
base64.b64encode(
|
|
|
|
|
bytes(
|
|
|
|
|
json.dumps(get_client_settings()),
|
|
|
|
|
encoding='utf-8',
|
|
|
|
|
)
|
2022-05-07 16:23:10 +00:00
|
|
|
|
),
|
|
|
|
|
encoding='utf-8',
|
2022-04-13 15:08:19 +00:00
|
|
|
|
)
|
|
|
|
|
|
2021-06-08 09:17:55 +00:00
|
|
|
|
# values from the HTTP requests
|
|
|
|
|
kwargs['endpoint'] = 'results' if 'q' in kwargs else request.endpoint
|
|
|
|
|
kwargs['cookies'] = request.cookies
|
|
|
|
|
kwargs['errors'] = request.errors
|
2023-05-23 16:16:37 +00:00
|
|
|
|
kwargs['link_token'] = link_token.get_token()
|
2014-03-29 15:29:19 +00:00
|
|
|
|
|
2021-06-08 09:17:55 +00:00
|
|
|
|
# values from the preferences
|
|
|
|
|
kwargs['preferences'] = request.preferences
|
2022-05-10 20:25:42 +00:00
|
|
|
|
kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
|
2024-08-11 08:39:46 +00:00
|
|
|
|
kwargs['favicon_resolver'] = request.preferences.get_value('favicon_resolver')
|
2022-05-10 20:25:42 +00:00
|
|
|
|
kwargs['infinite_scroll'] = request.preferences.get_value('infinite_scroll')
|
2023-09-09 14:49:14 +00:00
|
|
|
|
kwargs['search_on_category_select'] = request.preferences.get_value('search_on_category_select')
|
2023-09-16 15:14:18 +00:00
|
|
|
|
kwargs['hotkeys'] = request.preferences.get_value('hotkeys')
|
2024-11-27 12:42:08 +00:00
|
|
|
|
kwargs['url_formatting'] = request.preferences.get_value('url_formatting')
|
2021-06-08 09:17:55 +00:00
|
|
|
|
kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')
|
2021-06-08 09:56:30 +00:00
|
|
|
|
kwargs['advanced_search'] = request.preferences.get_value('advanced_search')
|
2021-11-06 11:26:48 +00:00
|
|
|
|
kwargs['query_in_title'] = request.preferences.get_value('query_in_title')
|
2021-06-08 09:17:55 +00:00
|
|
|
|
kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
|
2022-05-07 13:11:05 +00:00
|
|
|
|
kwargs['theme'] = request.preferences.get_value('theme')
|
2022-05-10 15:39:00 +00:00
|
|
|
|
kwargs['method'] = request.preferences.get_value('method')
|
2022-01-04 17:00:45 +00:00
|
|
|
|
kwargs['categories_as_tabs'] = list(settings['categories_as_tabs'].keys())
|
2022-07-23 16:26:43 +00:00
|
|
|
|
kwargs['categories'] = get_enabled_categories(settings['categories_as_tabs'].keys())
|
2022-07-24 07:32:05 +00:00
|
|
|
|
kwargs['DEFAULT_CATEGORY'] = DEFAULT_CATEGORY
|
2014-03-29 15:29:19 +00:00
|
|
|
|
|
2021-06-08 09:17:55 +00:00
|
|
|
|
# i18n
|
2022-10-10 17:31:22 +00:00
|
|
|
|
kwargs['sxng_locales'] = [l for l in sxng_locales if l[0] in settings['search']['languages']]
|
2014-03-29 15:29:19 +00:00
|
|
|
|
|
2019-03-28 17:07:03 +00:00
|
|
|
|
locale = request.preferences.get_value('locale')
|
2021-10-06 08:26:40 +00:00
|
|
|
|
kwargs['locale_rfc5646'] = _get_locale_rfc5646(locale)
|
2021-10-05 13:03:28 +00:00
|
|
|
|
|
2021-08-03 13:13:00 +00:00
|
|
|
|
if locale in RTL_LOCALES and 'rtl' not in kwargs:
|
2015-02-10 14:23:56 +00:00
|
|
|
|
kwargs['rtl'] = True
|
2023-02-07 13:11:58 +00:00
|
|
|
|
|
2016-08-06 04:34:56 +00:00
|
|
|
|
if 'current_language' not in kwargs:
|
2023-01-31 11:40:23 +00:00
|
|
|
|
kwargs['current_language'] = parse_lang(request.preferences, {}, RawTextQuery('', []))
|
2016-08-06 04:34:56 +00:00
|
|
|
|
|
2021-06-08 09:17:55 +00:00
|
|
|
|
# values from settings
|
2021-12-27 08:26:22 +00:00
|
|
|
|
kwargs['search_formats'] = [x for x in settings['search']['formats'] if x != 'html']
|
2021-07-18 13:38:52 +00:00
|
|
|
|
kwargs['instance_name'] = get_setting('general.instance_name')
|
2021-06-08 09:17:55 +00:00
|
|
|
|
kwargs['searx_version'] = VERSION_STRING
|
2021-07-27 16:37:46 +00:00
|
|
|
|
kwargs['searx_git_url'] = GIT_URL
|
2022-09-02 08:52:23 +00:00
|
|
|
|
kwargs['enable_metrics'] = get_setting('general.enable_metrics')
|
2021-07-18 13:38:52 +00:00
|
|
|
|
kwargs['get_setting'] = get_setting
|
2021-11-21 20:38:00 +00:00
|
|
|
|
kwargs['get_pretty_url'] = get_pretty_url
|
2020-03-25 10:49:33 +00:00
|
|
|
|
|
2022-07-02 09:29:21 +00:00
|
|
|
|
# values from settings: donation_url
|
|
|
|
|
donation_url = get_setting('general.donation_url')
|
|
|
|
|
if donation_url is True:
|
|
|
|
|
donation_url = custom_url_for('info', pagename='donate')
|
|
|
|
|
kwargs['donation_url'] = donation_url
|
|
|
|
|
|
2021-06-08 09:17:55 +00:00
|
|
|
|
# helpers to create links to other pages
|
2022-03-16 21:24:35 +00:00
|
|
|
|
kwargs['url_for'] = custom_url_for # override url_for function in templates
|
2021-06-08 09:17:55 +00:00
|
|
|
|
kwargs['image_proxify'] = image_proxify
|
2024-08-19 15:47:54 +00:00
|
|
|
|
kwargs['favicon_url'] = favicons.favicon_url
|
2022-08-27 10:01:04 +00:00
|
|
|
|
kwargs['proxify'] = morty_proxify if settings['result_proxy']['url'] is not None else None
|
|
|
|
|
kwargs['proxify_results'] = settings['result_proxy']['proxify_results']
|
2022-09-04 07:36:01 +00:00
|
|
|
|
kwargs['cache_url'] = settings['ui']['cache_url']
|
2021-06-08 09:17:55 +00:00
|
|
|
|
kwargs['get_result_template'] = get_result_template
|
2022-09-24 12:19:51 +00:00
|
|
|
|
kwargs['doi_resolver'] = get_doi_resolver(request.preferences)
|
2024-02-25 17:15:34 +00:00
|
|
|
|
kwargs['opensearch_url'] = (
|
|
|
|
|
url_for('opensearch')
|
|
|
|
|
+ '?'
|
|
|
|
|
+ urlencode(
|
|
|
|
|
{
|
|
|
|
|
'method': request.preferences.get_value('method'),
|
|
|
|
|
'autocomplete': request.preferences.get_value('autocomplete'),
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
)
|
2023-08-08 16:12:07 +00:00
|
|
|
|
kwargs['urlparse'] = urlparse
|
2021-03-16 10:07:04 +00:00
|
|
|
|
|
2021-06-08 09:17:55 +00:00
|
|
|
|
# scripts from plugins
|
2015-04-12 17:24:01 +00:00
|
|
|
|
kwargs['scripts'] = set()
|
|
|
|
|
for plugin in request.user_plugins:
|
|
|
|
|
for script in plugin.js_dependencies:
|
|
|
|
|
kwargs['scripts'].add(script)
|
|
|
|
|
|
2021-06-08 09:17:55 +00:00
|
|
|
|
# styles from plugins
|
2015-04-12 17:24:01 +00:00
|
|
|
|
kwargs['styles'] = set()
|
|
|
|
|
for plugin in request.user_plugins:
|
|
|
|
|
for css in plugin.css_dependencies:
|
|
|
|
|
kwargs['styles'].add(css)
|
|
|
|
|
|
2021-05-06 07:39:52 +00:00
|
|
|
|
start_time = default_timer()
|
2021-12-27 08:26:22 +00:00
|
|
|
|
result = render_template('{}/{}'.format(kwargs['theme'], template_name), **kwargs)
|
2021-05-14 09:15:35 +00:00
|
|
|
|
request.render_time += default_timer() - start_time # pylint: disable=assigning-non-slot
|
|
|
|
|
|
2021-05-06 07:39:52 +00:00
|
|
|
|
return result
|
2013-10-15 18:50:12 +00:00
|
|
|
|
|
2014-01-19 21:59:01 +00:00
|
|
|
|
|
2015-03-10 19:44:02 +00:00
|
|
|
|
@app.before_request
|
|
|
|
|
def pre_request():
|
2021-05-14 09:15:35 +00:00
|
|
|
|
request.start_time = default_timer() # pylint: disable=assigning-non-slot
|
|
|
|
|
request.render_time = 0 # pylint: disable=assigning-non-slot
|
|
|
|
|
request.timings = [] # pylint: disable=assigning-non-slot
|
|
|
|
|
request.errors = [] # pylint: disable=assigning-non-slot
|
2016-11-14 21:07:23 +00:00
|
|
|
|
|
2023-01-30 17:40:22 +00:00
|
|
|
|
client_pref = ClientPref.from_http_request(request)
|
|
|
|
|
# pylint: disable=redefined-outer-name
|
|
|
|
|
preferences = Preferences(themes, list(categories.keys()), engines, plugins, client_pref)
|
|
|
|
|
|
2020-08-18 16:29:58 +00:00
|
|
|
|
user_agent = request.headers.get('User-Agent', '').lower()
|
|
|
|
|
if 'webkit' in user_agent and 'android' in user_agent:
|
|
|
|
|
preferences.key_value_settings['method'].value = 'GET'
|
2021-05-14 09:15:35 +00:00
|
|
|
|
request.preferences = preferences # pylint: disable=assigning-non-slot
|
2021-05-26 11:32:04 +00:00
|
|
|
|
|
2016-09-04 13:56:46 +00:00
|
|
|
|
try:
|
2017-07-10 10:47:25 +00:00
|
|
|
|
preferences.parse_dict(request.cookies)
|
2021-05-26 11:32:04 +00:00
|
|
|
|
|
|
|
|
|
except Exception as e: # pylint: disable=broad-except
|
|
|
|
|
logger.exception(e, exc_info=True)
|
2016-11-14 21:07:23 +00:00
|
|
|
|
request.errors.append(gettext('Invalid settings, please edit your preferences'))
|
2016-04-08 14:38:05 +00:00
|
|
|
|
|
2016-11-14 21:07:23 +00:00
|
|
|
|
# merge GET, POST vars
|
2016-10-22 11:10:31 +00:00
|
|
|
|
# request.form
|
2021-05-14 09:15:35 +00:00
|
|
|
|
request.form = dict(request.form.items()) # pylint: disable=assigning-non-slot
|
2015-03-15 11:13:24 +00:00
|
|
|
|
for k, v in request.args.items():
|
2015-03-10 21:45:59 +00:00
|
|
|
|
if k not in request.form:
|
|
|
|
|
request.form[k] = v
|
2017-10-25 21:56:37 +00:00
|
|
|
|
|
|
|
|
|
if request.form.get('preferences'):
|
|
|
|
|
preferences.parse_encoded_data(request.form['preferences'])
|
|
|
|
|
else:
|
|
|
|
|
try:
|
|
|
|
|
preferences.parse_dict(request.form)
|
2021-05-26 11:32:04 +00:00
|
|
|
|
except Exception as e: # pylint: disable=broad-except
|
|
|
|
|
logger.exception(e, exc_info=True)
|
2017-10-25 21:56:37 +00:00
|
|
|
|
request.errors.append(gettext('Invalid settings'))
|
2015-03-10 19:44:02 +00:00
|
|
|
|
|
2021-08-17 06:18:30 +00:00
|
|
|
|
# language is defined neither in settings nor in preferences
|
|
|
|
|
# use browser headers
|
2019-03-28 17:07:03 +00:00
|
|
|
|
if not preferences.get_value("language"):
|
2021-12-27 08:26:22 +00:00
|
|
|
|
language = _get_browser_language(request, settings['search']['languages'])
|
2021-08-17 06:18:30 +00:00
|
|
|
|
preferences.parse_dict({"language": language})
|
2022-06-10 15:01:12 +00:00
|
|
|
|
logger.debug('set language %s (from browser)', preferences.get_value("language"))
|
2021-08-17 06:18:30 +00:00
|
|
|
|
|
2024-10-27 09:25:42 +00:00
|
|
|
|
# UI locale is defined neither in settings nor in preferences
|
2021-08-17 06:18:30 +00:00
|
|
|
|
# use browser headers
|
2019-03-28 17:07:03 +00:00
|
|
|
|
if not preferences.get_value("locale"):
|
2021-10-06 08:26:40 +00:00
|
|
|
|
locale = _get_browser_language(request, LOCALE_NAMES.keys())
|
2021-08-17 06:18:30 +00:00
|
|
|
|
preferences.parse_dict({"locale": locale})
|
2022-06-10 15:01:12 +00:00
|
|
|
|
logger.debug('set locale %s (from browser)', preferences.get_value("locale"))
|
2019-03-28 17:07:03 +00:00
|
|
|
|
|
2016-10-22 11:10:31 +00:00
|
|
|
|
# request.user_plugins
|
2021-05-14 09:15:35 +00:00
|
|
|
|
request.user_plugins = [] # pylint: disable=assigning-non-slot
|
2016-04-08 14:38:05 +00:00
|
|
|
|
allowed_plugins = preferences.plugins.get_enabled()
|
|
|
|
|
disabled_plugins = preferences.plugins.get_disabled()
|
2015-03-10 19:44:02 +00:00
|
|
|
|
for plugin in plugins:
|
2021-12-27 08:26:22 +00:00
|
|
|
|
if (plugin.default_on and plugin.id not in disabled_plugins) or plugin.id in allowed_plugins:
|
2015-03-10 19:44:02 +00:00
|
|
|
|
request.user_plugins.append(plugin)
|
|
|
|
|
|
|
|
|
|
|
2020-11-04 16:32:51 +00:00
|
|
|
|
@app.after_request
|
2022-01-17 10:16:11 +00:00
|
|
|
|
def add_default_headers(response: flask.Response):
|
2020-11-04 16:32:51 +00:00
|
|
|
|
# set default http headers
|
2021-05-28 16:45:22 +00:00
|
|
|
|
for header, value in settings['server']['default_http_headers'].items():
|
2020-11-04 16:32:51 +00:00
|
|
|
|
if header in response.headers:
|
|
|
|
|
continue
|
|
|
|
|
response.headers[header] = value
|
|
|
|
|
return response
|
|
|
|
|
|
|
|
|
|
|
2019-07-17 08:38:45 +00:00
|
|
|
|
@app.after_request
|
2022-01-17 10:16:11 +00:00
|
|
|
|
def post_request(response: flask.Response):
|
2021-04-14 15:23:15 +00:00
|
|
|
|
total_time = default_timer() - request.start_time
|
2021-12-27 08:26:22 +00:00
|
|
|
|
timings_all = [
|
|
|
|
|
'total;dur=' + str(round(total_time * 1000, 3)),
|
|
|
|
|
'render;dur=' + str(round(request.render_time * 1000, 3)),
|
|
|
|
|
]
|
2019-07-17 08:38:45 +00:00
|
|
|
|
if len(request.timings) > 0:
|
2022-01-17 07:06:31 +00:00
|
|
|
|
timings = sorted(request.timings, key=lambda t: t.total)
|
2021-06-08 16:19:30 +00:00
|
|
|
|
timings_total = [
|
2022-01-17 07:06:31 +00:00
|
|
|
|
'total_' + str(i) + '_' + t.engine + ';dur=' + str(round(t.total * 1000, 3)) for i, t in enumerate(timings)
|
2021-06-08 16:19:30 +00:00
|
|
|
|
]
|
|
|
|
|
timings_load = [
|
2022-01-17 07:06:31 +00:00
|
|
|
|
'load_' + str(i) + '_' + t.engine + ';dur=' + str(round(t.load * 1000, 3))
|
|
|
|
|
for i, t in enumerate(timings)
|
|
|
|
|
if t.load
|
2021-06-08 16:19:30 +00:00
|
|
|
|
]
|
2019-07-17 08:38:45 +00:00
|
|
|
|
timings_all = timings_all + timings_total + timings_load
|
|
|
|
|
response.headers.add('Server-Timing', ', '.join(timings_all))
|
|
|
|
|
return response
|
|
|
|
|
|
|
|
|
|
|
2022-01-17 10:16:11 +00:00
|
|
|
|
def index_error(output_format: str, error_message: str):
|
2017-01-20 17:52:47 +00:00
|
|
|
|
if output_format == 'json':
|
2021-12-27 08:26:22 +00:00
|
|
|
|
return Response(json.dumps({'error': error_message}), mimetype='application/json')
|
2021-05-26 11:32:04 +00:00
|
|
|
|
if output_format == 'csv':
|
2017-01-20 17:52:47 +00:00
|
|
|
|
response = Response('', mimetype='application/csv')
|
|
|
|
|
cont_disp = 'attachment;Filename=searx.csv'
|
|
|
|
|
response.headers.add('Content-Disposition', cont_disp)
|
|
|
|
|
return response
|
2021-05-26 11:32:04 +00:00
|
|
|
|
|
|
|
|
|
if output_format == 'rss':
|
2017-01-20 17:52:47 +00:00
|
|
|
|
response_rss = render(
|
|
|
|
|
'opensearch_response_rss.xml',
|
|
|
|
|
results=[],
|
|
|
|
|
q=request.form['q'] if 'q' in request.form else '',
|
|
|
|
|
number_of_results=0,
|
2017-02-13 20:36:45 +00:00
|
|
|
|
error_message=error_message,
|
2017-01-20 17:52:47 +00:00
|
|
|
|
)
|
|
|
|
|
return Response(response_rss, mimetype='text/xml')
|
2021-05-26 11:32:04 +00:00
|
|
|
|
|
|
|
|
|
# html
|
|
|
|
|
request.errors.append(gettext('search error'))
|
|
|
|
|
return render(
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: off
|
2021-05-26 11:32:04 +00:00
|
|
|
|
'index.html',
|
|
|
|
|
selected_categories=get_selected_categories(request.preferences, request.form),
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: on
|
2021-05-26 11:32:04 +00:00
|
|
|
|
)
|
2017-01-20 17:52:47 +00:00
|
|
|
|
|
|
|
|
|
|
2014-01-14 17:19:21 +00:00
|
|
|
|
@app.route('/', methods=['GET', 'POST'])
|
2013-10-14 21:09:13 +00:00
|
|
|
|
def index():
|
2019-07-30 04:25:05 +00:00
|
|
|
|
"""Render index page."""
|
|
|
|
|
|
|
|
|
|
# redirect to search if there's a query in the request
|
|
|
|
|
if request.form.get('q'):
|
2020-11-06 11:11:52 +00:00
|
|
|
|
query = ('?' + request.query_string.decode()) if request.query_string else ''
|
|
|
|
|
return redirect(url_for('search') + query, 308)
|
2019-07-30 04:25:05 +00:00
|
|
|
|
|
|
|
|
|
return render(
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: off
|
2019-07-30 04:25:05 +00:00
|
|
|
|
'index.html',
|
2020-11-06 11:11:52 +00:00
|
|
|
|
selected_categories=get_selected_categories(request.preferences, request.form),
|
2022-03-12 09:18:08 +00:00
|
|
|
|
current_locale = request.preferences.get_value("locale"),
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: on
|
2019-07-30 04:25:05 +00:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
2021-10-02 15:00:37 +00:00
|
|
|
|
@app.route('/healthz', methods=['GET'])
|
|
|
|
|
def health():
|
|
|
|
|
return Response('OK', mimetype='text/plain')
|
|
|
|
|
|
|
|
|
|
|
2023-05-23 16:16:37 +00:00
|
|
|
|
@app.route('/client<token>.css', methods=['GET', 'POST'])
|
|
|
|
|
def client_token(token=None):
|
|
|
|
|
link_token.ping(request, token)
|
2023-04-19 15:20:03 +00:00
|
|
|
|
return Response('', mimetype='text/css')
|
|
|
|
|
|
|
|
|
|
|
2024-11-24 17:19:11 +00:00
|
|
|
|
@app.route('/rss.xsl', methods=['GET', 'POST'])
|
|
|
|
|
def rss_xsl():
|
|
|
|
|
return render_template(
|
|
|
|
|
f"{request.preferences.get_value('theme')}/rss.xsl",
|
|
|
|
|
url_for=custom_url_for,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
2019-07-30 04:25:05 +00:00
|
|
|
|
@app.route('/search', methods=['GET', 'POST'])
|
|
|
|
|
def search():
|
|
|
|
|
"""Search query in q and return results.
|
2014-01-31 06:08:24 +00:00
|
|
|
|
|
|
|
|
|
Supported outputs: html, json, csv, rss.
|
|
|
|
|
"""
|
2021-05-26 11:32:04 +00:00
|
|
|
|
# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches
|
|
|
|
|
# pylint: disable=too-many-statements
|
2013-11-03 23:18:07 +00:00
|
|
|
|
|
2017-01-20 17:52:47 +00:00
|
|
|
|
# output_format
|
|
|
|
|
output_format = request.form.get('format', 'html')
|
2021-05-26 17:43:27 +00:00
|
|
|
|
if output_format not in OUTPUT_FORMATS:
|
2017-01-20 17:52:47 +00:00
|
|
|
|
output_format = 'html'
|
|
|
|
|
|
2021-05-28 16:45:22 +00:00
|
|
|
|
if output_format not in settings['search']['formats']:
|
2021-05-26 17:43:27 +00:00
|
|
|
|
flask.abort(403)
|
|
|
|
|
|
2020-11-06 11:11:52 +00:00
|
|
|
|
# check if there is query (not None and not an empty string)
|
|
|
|
|
if not request.form.get('q'):
|
2017-01-20 17:52:47 +00:00
|
|
|
|
if output_format == 'html':
|
|
|
|
|
return render(
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: off
|
2017-01-20 17:52:47 +00:00
|
|
|
|
'index.html',
|
2020-09-24 14:26:00 +00:00
|
|
|
|
selected_categories=get_selected_categories(request.preferences, request.form),
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: on
|
2017-01-20 17:52:47 +00:00
|
|
|
|
)
|
2021-05-26 11:32:04 +00:00
|
|
|
|
return index_error(output_format, 'No query'), 400
|
2014-02-07 02:15:34 +00:00
|
|
|
|
|
2016-10-22 11:10:31 +00:00
|
|
|
|
# search
|
|
|
|
|
search_query = None
|
2019-07-16 14:27:29 +00:00
|
|
|
|
raw_text_query = None
|
2016-10-22 11:10:31 +00:00
|
|
|
|
result_container = None
|
2014-02-07 00:19:07 +00:00
|
|
|
|
try:
|
2023-01-31 11:40:23 +00:00
|
|
|
|
search_query, raw_text_query, _, _, selected_locale = get_search_query_from_webapp(
|
|
|
|
|
request.preferences, request.form
|
|
|
|
|
)
|
2021-05-26 11:32:04 +00:00
|
|
|
|
search = SearchWithPlugins(search_query, request.user_plugins, request) # pylint: disable=redefined-outer-name
|
2016-10-22 11:10:31 +00:00
|
|
|
|
result_container = search.search()
|
2020-07-03 13:25:04 +00:00
|
|
|
|
|
2020-11-03 10:35:53 +00:00
|
|
|
|
except SearxParameterException as e:
|
|
|
|
|
logger.exception('search error: SearxParameterException')
|
|
|
|
|
return index_error(output_format, e.message), 400
|
2021-05-26 11:32:04 +00:00
|
|
|
|
except Exception as e: # pylint: disable=broad-except
|
|
|
|
|
logger.exception(e, exc_info=True)
|
2020-11-03 10:35:53 +00:00
|
|
|
|
return index_error(output_format, gettext('search error')), 500
|
2017-01-20 17:52:47 +00:00
|
|
|
|
|
2023-06-18 14:43:48 +00:00
|
|
|
|
# 1. check if the result is a redirect for an external bang
|
2020-07-03 13:25:04 +00:00
|
|
|
|
if result_container.redirect_url:
|
|
|
|
|
return redirect(result_container.redirect_url)
|
|
|
|
|
|
2023-06-18 14:43:48 +00:00
|
|
|
|
# 2. add Server-Timing header for measuring performance characteristics of
|
|
|
|
|
# web applications
|
2021-05-14 09:15:35 +00:00
|
|
|
|
request.timings = result_container.get_timings() # pylint: disable=assigning-non-slot
|
2019-07-17 08:38:45 +00:00
|
|
|
|
|
2023-06-18 14:43:48 +00:00
|
|
|
|
# 3. formats without a template
|
|
|
|
|
|
|
|
|
|
if output_format == 'json':
|
|
|
|
|
|
|
|
|
|
response = webutils.get_json_response(search_query, result_container)
|
|
|
|
|
return Response(response, mimetype='application/json')
|
|
|
|
|
|
|
|
|
|
if output_format == 'csv':
|
|
|
|
|
|
|
|
|
|
csv = webutils.CSVWriter(StringIO())
|
|
|
|
|
webutils.write_csv_response(csv, result_container)
|
|
|
|
|
csv.stream.seek(0)
|
|
|
|
|
|
|
|
|
|
response = Response(csv.stream.read(), mimetype='application/csv')
|
|
|
|
|
cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)
|
|
|
|
|
response.headers.add('Content-Disposition', cont_disp)
|
|
|
|
|
return response
|
|
|
|
|
|
|
|
|
|
# 4. formats rendered by a template / RSS & HTML
|
|
|
|
|
|
2022-02-21 21:18:48 +00:00
|
|
|
|
current_template = None
|
|
|
|
|
previous_result = None
|
|
|
|
|
|
2023-06-18 14:43:48 +00:00
|
|
|
|
results = result_container.get_ordered_results()
|
2023-09-10 16:44:16 +00:00
|
|
|
|
|
|
|
|
|
if search_query.redirect_to_first_result and results:
|
|
|
|
|
return redirect(results[0]['url'], 302)
|
|
|
|
|
|
2016-07-16 19:41:02 +00:00
|
|
|
|
for result in results:
|
2016-10-22 11:10:31 +00:00
|
|
|
|
if output_format == 'html':
|
2016-09-05 20:22:25 +00:00
|
|
|
|
if 'content' in result and result['content']:
|
2016-11-30 17:43:03 +00:00
|
|
|
|
result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query)
|
2019-09-23 15:14:32 +00:00
|
|
|
|
if 'title' in result and result['title']:
|
2020-08-06 15:42:46 +00:00
|
|
|
|
result['title'] = highlight_content(escape(result['title'] or ''), search_query.query)
|
2014-06-24 14:30:04 +00:00
|
|
|
|
|
2019-09-23 15:14:32 +00:00
|
|
|
|
if 'url' in result:
|
2023-06-18 14:43:48 +00:00
|
|
|
|
result['pretty_url'] = webutils.prettify_url(result['url'])
|
2021-04-04 10:43:55 +00:00
|
|
|
|
if result.get('publishedDate'): # do not try to get a date from an empty string or a None type
|
2016-03-29 09:59:16 +00:00
|
|
|
|
try: # test if publishedDate >= 1900 (datetime module bug)
|
|
|
|
|
result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
|
|
|
|
|
except ValueError:
|
|
|
|
|
result['publishedDate'] = None
|
2014-03-14 08:55:04 +00:00
|
|
|
|
else:
|
2023-06-18 14:43:48 +00:00
|
|
|
|
result['publishedDate'] = webutils.searxng_l10n_timespan(result['publishedDate'])
|
2014-03-14 08:55:04 +00:00
|
|
|
|
|
2022-02-21 21:18:48 +00:00
|
|
|
|
# set result['open_group'] = True when the template changes from the previous result
|
|
|
|
|
# set result['close_group'] = True when the template changes on the next result
|
|
|
|
|
if current_template != result.get('template'):
|
|
|
|
|
result['open_group'] = True
|
|
|
|
|
if previous_result:
|
|
|
|
|
previous_result['close_group'] = True # pylint: disable=unsupported-assignment-operation
|
|
|
|
|
current_template = result.get('template')
|
|
|
|
|
previous_result = result
|
|
|
|
|
|
|
|
|
|
if previous_result:
|
|
|
|
|
previous_result['close_group'] = True
|
|
|
|
|
|
2023-06-18 14:43:48 +00:00
|
|
|
|
# 4.a RSS
|
2020-07-03 13:25:04 +00:00
|
|
|
|
|
2021-05-26 11:32:04 +00:00
|
|
|
|
if output_format == 'rss':
|
2014-01-19 21:59:01 +00:00
|
|
|
|
response_rss = render(
|
|
|
|
|
'opensearch_response_rss.xml',
|
2016-07-16 19:41:02 +00:00
|
|
|
|
results=results,
|
2016-11-02 13:52:22 +00:00
|
|
|
|
q=request.form['q'],
|
2023-06-18 14:43:48 +00:00
|
|
|
|
number_of_results=result_container.number_of_results,
|
2014-01-19 21:59:01 +00:00
|
|
|
|
)
|
2014-01-14 21:18:21 +00:00
|
|
|
|
return Response(response_rss, mimetype='text/xml')
|
2014-01-14 17:17:19 +00:00
|
|
|
|
|
2023-06-18 14:43:48 +00:00
|
|
|
|
# 4.b HTML
|
2019-07-16 14:27:29 +00:00
|
|
|
|
|
|
|
|
|
# suggestions: use RawTextQuery to get the suggestion URLs with the same bang
|
2021-06-08 16:19:30 +00:00
|
|
|
|
suggestion_urls = list(
|
|
|
|
|
map(
|
2021-12-27 08:26:22 +00:00
|
|
|
|
lambda suggestion: {'url': raw_text_query.changeQuery(suggestion).getFullQuery(), 'title': suggestion},
|
|
|
|
|
result_container.suggestions,
|
|
|
|
|
)
|
|
|
|
|
)
|
2021-06-08 16:19:30 +00:00
|
|
|
|
|
|
|
|
|
correction_urls = list(
|
|
|
|
|
map(
|
2021-12-27 08:26:22 +00:00
|
|
|
|
lambda correction: {'url': raw_text_query.changeQuery(correction).getFullQuery(), 'title': correction},
|
|
|
|
|
result_container.corrections,
|
|
|
|
|
)
|
|
|
|
|
)
|
2021-06-08 16:19:30 +00:00
|
|
|
|
|
2024-08-22 17:14:24 +00:00
|
|
|
|
# engine_timings: get engine response times sorted from slowest to fastest
|
|
|
|
|
engine_timings = sorted(result_container.get_timings(), reverse=True, key=lambda e: e.total)
|
|
|
|
|
max_response_time = engine_timings[0].total if engine_timings else None
|
|
|
|
|
engine_timings_pairs = [(timing.engine, timing.total) for timing in engine_timings]
|
|
|
|
|
|
2022-12-16 20:28:57 +00:00
|
|
|
|
# search_query.lang contains the user choice (all, auto, en, ...)
|
|
|
|
|
# when the user choice is "auto", search.search_query.lang contains the detected language
|
|
|
|
|
# otherwise it is equals to search_query.lang
|
2014-01-19 21:59:01 +00:00
|
|
|
|
return render(
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: off
|
2014-01-19 21:59:01 +00:00
|
|
|
|
'results.html',
|
2021-06-08 16:19:30 +00:00
|
|
|
|
results = results,
|
2016-11-02 13:52:22 +00:00
|
|
|
|
q=request.form['q'],
|
2021-06-08 16:19:30 +00:00
|
|
|
|
selected_categories = search_query.categories,
|
|
|
|
|
pageno = search_query.pageno,
|
2023-02-08 05:54:19 +00:00
|
|
|
|
time_range = search_query.time_range or '',
|
2023-06-18 14:43:48 +00:00
|
|
|
|
number_of_results = format_decimal(result_container.number_of_results),
|
2021-06-08 16:19:30 +00:00
|
|
|
|
suggestions = suggestion_urls,
|
|
|
|
|
answers = result_container.answers,
|
|
|
|
|
corrections = correction_urls,
|
|
|
|
|
infoboxes = result_container.infoboxes,
|
|
|
|
|
engine_data = result_container.engine_data,
|
|
|
|
|
paging = result_container.paging,
|
2023-06-18 14:43:48 +00:00
|
|
|
|
unresponsive_engines = webutils.get_translated_errors(
|
2021-06-08 16:19:30 +00:00
|
|
|
|
result_container.unresponsive_engines
|
|
|
|
|
),
|
2022-03-12 09:18:08 +00:00
|
|
|
|
current_locale = request.preferences.get_value("locale"),
|
2023-01-31 11:40:23 +00:00
|
|
|
|
current_language = selected_locale,
|
2023-02-07 13:11:58 +00:00
|
|
|
|
search_language = match_locale(
|
2022-12-16 20:28:57 +00:00
|
|
|
|
search.search_query.lang,
|
|
|
|
|
settings['search']['languages'],
|
|
|
|
|
fallback=request.preferences.get_value("language")
|
|
|
|
|
),
|
2024-08-22 17:14:24 +00:00
|
|
|
|
timeout_limit = request.form.get('timeout_limit', None),
|
|
|
|
|
timings = engine_timings_pairs,
|
|
|
|
|
max_response_time = max_response_time
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: on
|
2014-01-19 21:59:01 +00:00
|
|
|
|
)
|
2014-01-01 21:16:53 +00:00
|
|
|
|
|
2013-10-14 21:09:13 +00:00
|
|
|
|
|
2013-10-20 22:28:48 +00:00
|
|
|
|
@app.route('/about', methods=['GET'])
|
|
|
|
|
def about():
|
2022-01-31 10:24:45 +00:00
|
|
|
|
"""Redirect to about page"""
|
2022-03-16 21:24:35 +00:00
|
|
|
|
# custom_url_for is going to add the locale
|
|
|
|
|
return redirect(custom_url_for('info', pagename='about'))
|
2022-01-31 10:24:45 +00:00
|
|
|
|
|
2022-03-12 09:18:08 +00:00
|
|
|
|
|
|
|
|
|
@app.route('/info/<locale>/<pagename>', methods=['GET'])
|
|
|
|
|
def info(pagename, locale):
|
|
|
|
|
"""Render page of online user documentation"""
|
2024-05-28 15:56:54 +00:00
|
|
|
|
page = infopage.INFO_PAGES.get_page(pagename, locale)
|
2022-01-31 10:24:45 +00:00
|
|
|
|
if page is None:
|
|
|
|
|
flask.abort(404)
|
|
|
|
|
|
2022-03-16 21:24:35 +00:00
|
|
|
|
user_locale = request.preferences.get_value('locale')
|
2022-01-31 11:42:04 +00:00
|
|
|
|
return render(
|
2022-03-12 09:18:08 +00:00
|
|
|
|
'info.html',
|
2024-05-28 15:56:54 +00:00
|
|
|
|
all_pages=infopage.INFO_PAGES.iter_pages(user_locale, fallback_to_default=True),
|
2022-03-12 09:18:08 +00:00
|
|
|
|
active_page=page,
|
|
|
|
|
active_pagename=pagename,
|
2022-01-31 11:42:04 +00:00
|
|
|
|
)
|
2014-01-17 15:23:23 +00:00
|
|
|
|
|
|
|
|
|
|
2014-03-20 09:28:24 +00:00
|
|
|
|
@app.route('/autocompleter', methods=['GET', 'POST'])
|
|
|
|
|
def autocompleter():
|
|
|
|
|
"""Return autocompleter results"""
|
2014-03-29 15:29:19 +00:00
|
|
|
|
|
2021-02-22 17:13:50 +00:00
|
|
|
|
# run autocompleter
|
|
|
|
|
results = []
|
|
|
|
|
|
2014-10-01 15:18:18 +00:00
|
|
|
|
# set blocked engines
|
2016-04-09 16:26:29 +00:00
|
|
|
|
disabled_engines = request.preferences.engines.get_disabled()
|
2014-10-01 15:18:18 +00:00
|
|
|
|
|
|
|
|
|
# parse query
|
2020-09-21 17:14:24 +00:00
|
|
|
|
raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
|
[fix] url bar autocomplete (opensearch suggestions)
Since #2593 is merged the OpenSearch-Format is buggy. The loop in [1] will
change raw_text_query object and this will change also the value of
`raw_text_query.query` on every `raw_text_query.changeQuery(result)`.
This patch fixes this issue by storing the initial query value in `sug_prefix`.
[1] https://github.com/searx/searx/blob/ac0fdc3b9670168de8061ed0e656dd66a567d741/searx/webapp.py#L804-L806
OpenSearch-Format::
[ "<query>",
[ "<term 1>", "<term 2>", ... "<term n>" ],
[ "<content 1>", "<content 2>", ..., "<content n>" ],
[ "<url 1>", "<url 2>", ..., "<url n>" ]
]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1080002
- https://developer.mozilla.org/en-US/docs/Archive/Add-ons/Supporting_search_suggestions_in_search_plugins#implementing_search_suggestion_support_on_the_server
Legacy-Format::
[ "<term 1>", "<term 2>", ..., "<term n>" ]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1081079
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2021-04-03 16:18:50 +00:00
|
|
|
|
sug_prefix = raw_text_query.getQuery()
|
2014-03-29 15:29:19 +00:00
|
|
|
|
|
2018-01-19 02:51:27 +00:00
|
|
|
|
# normal autocompletion results only appear if no inner results returned
|
2021-02-22 17:13:50 +00:00
|
|
|
|
# and there is a query part
|
[fix] url bar autocomplete (opensearch suggestions)
Since #2593 is merged the OpenSearch-Format is buggy. The loop in [1] will
change raw_text_query object and this will change also the value of
`raw_text_query.query` on every `raw_text_query.changeQuery(result)`.
This patch fixes this issue by storing the initial query value in `sug_prefix`.
[1] https://github.com/searx/searx/blob/ac0fdc3b9670168de8061ed0e656dd66a567d741/searx/webapp.py#L804-L806
OpenSearch-Format::
[ "<query>",
[ "<term 1>", "<term 2>", ... "<term n>" ],
[ "<content 1>", "<content 2>", ..., "<content n>" ],
[ "<url 1>", "<url 2>", ..., "<url n>" ]
]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1080002
- https://developer.mozilla.org/en-US/docs/Archive/Add-ons/Supporting_search_suggestions_in_search_plugins#implementing_search_suggestion_support_on_the_server
Legacy-Format::
[ "<term 1>", "<term 2>", ..., "<term n>" ]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1081079
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2021-04-03 16:18:50 +00:00
|
|
|
|
if len(raw_text_query.autocomplete_list) == 0 and len(sug_prefix) > 0:
|
|
|
|
|
|
2022-09-29 18:54:46 +00:00
|
|
|
|
# get SearXNG's locale and autocomplete backend from cookie
|
|
|
|
|
sxng_locale = request.preferences.get_value('language')
|
|
|
|
|
backend_name = request.preferences.get_value('autocomplete')
|
[fix] url bar autocomplete (opensearch suggestions)
Since #2593 is merged the OpenSearch-Format is buggy. The loop in [1] will
change raw_text_query object and this will change also the value of
`raw_text_query.query` on every `raw_text_query.changeQuery(result)`.
This patch fixes this issue by storing the initial query value in `sug_prefix`.
[1] https://github.com/searx/searx/blob/ac0fdc3b9670168de8061ed0e656dd66a567d741/searx/webapp.py#L804-L806
OpenSearch-Format::
[ "<query>",
[ "<term 1>", "<term 2>", ... "<term n>" ],
[ "<content 1>", "<content 2>", ..., "<content n>" ],
[ "<url 1>", "<url 2>", ..., "<url n>" ]
]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1080002
- https://developer.mozilla.org/en-US/docs/Archive/Add-ons/Supporting_search_suggestions_in_search_plugins#implementing_search_suggestion_support_on_the_server
Legacy-Format::
[ "<term 1>", "<term 2>", ..., "<term n>" ]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1081079
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2021-04-03 16:18:50 +00:00
|
|
|
|
|
2022-09-29 18:54:46 +00:00
|
|
|
|
for result in search_autocomplete(backend_name, sug_prefix, sxng_locale):
|
[fix] url bar autocomplete (opensearch suggestions)
Since #2593 is merged the OpenSearch-Format is buggy. The loop in [1] will
change raw_text_query object and this will change also the value of
`raw_text_query.query` on every `raw_text_query.changeQuery(result)`.
This patch fixes this issue by storing the initial query value in `sug_prefix`.
[1] https://github.com/searx/searx/blob/ac0fdc3b9670168de8061ed0e656dd66a567d741/searx/webapp.py#L804-L806
OpenSearch-Format::
[ "<query>",
[ "<term 1>", "<term 2>", ... "<term n>" ],
[ "<content 1>", "<content 2>", ..., "<content n>" ],
[ "<url 1>", "<url 2>", ..., "<url n>" ]
]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1080002
- https://developer.mozilla.org/en-US/docs/Archive/Add-ons/Supporting_search_suggestions_in_search_plugins#implementing_search_suggestion_support_on_the_server
Legacy-Format::
[ "<term 1>", "<term 2>", ..., "<term n>" ]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1081079
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2021-04-03 16:18:50 +00:00
|
|
|
|
# attention: this loop will change raw_text_query object and this is
|
|
|
|
|
# the reason why the sug_prefix was stored before (see above)
|
2022-06-12 09:21:24 +00:00
|
|
|
|
if result != sug_prefix:
|
|
|
|
|
results.append(raw_text_query.changeQuery(result).getFullQuery())
|
2021-02-22 17:13:50 +00:00
|
|
|
|
|
|
|
|
|
if len(raw_text_query.autocomplete_list) > 0:
|
|
|
|
|
for autocomplete_text in raw_text_query.autocomplete_list:
|
|
|
|
|
results.append(raw_text_query.get_autocomplete_full_query(autocomplete_text))
|
|
|
|
|
|
|
|
|
|
for answers in ask(raw_text_query):
|
|
|
|
|
for answer in answers:
|
|
|
|
|
results.append(str(answer['answer']))
|
2014-03-29 15:29:19 +00:00
|
|
|
|
|
2020-08-09 11:59:49 +00:00
|
|
|
|
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
|
[fix] url bar autocomplete (opensearch suggestions)
Since #2593 is merged the OpenSearch-Format is buggy. The loop in [1] will
change raw_text_query object and this will change also the value of
`raw_text_query.query` on every `raw_text_query.changeQuery(result)`.
This patch fixes this issue by storing the initial query value in `sug_prefix`.
[1] https://github.com/searx/searx/blob/ac0fdc3b9670168de8061ed0e656dd66a567d741/searx/webapp.py#L804-L806
OpenSearch-Format::
[ "<query>",
[ "<term 1>", "<term 2>", ... "<term n>" ],
[ "<content 1>", "<content 2>", ..., "<content n>" ],
[ "<url 1>", "<url 2>", ..., "<url n>" ]
]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1080002
- https://developer.mozilla.org/en-US/docs/Archive/Add-ons/Supporting_search_suggestions_in_search_plugins#implementing_search_suggestion_support_on_the_server
Legacy-Format::
[ "<term 1>", "<term 2>", ..., "<term n>" ]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1081079
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2021-04-03 16:18:50 +00:00
|
|
|
|
# the suggestion request comes from the searx search form
|
|
|
|
|
suggestions = json.dumps(results)
|
|
|
|
|
mimetype = 'application/json'
|
|
|
|
|
else:
|
|
|
|
|
# the suggestion request comes from browser's URL bar
|
|
|
|
|
suggestions = json.dumps([sug_prefix, results])
|
|
|
|
|
mimetype = 'application/x-suggestions+json'
|
2015-01-25 21:52:48 +00:00
|
|
|
|
|
2022-05-07 13:11:05 +00:00
|
|
|
|
suggestions = escape(suggestions, False)
|
[fix] url bar autocomplete (opensearch suggestions)
Since #2593 is merged the OpenSearch-Format is buggy. The loop in [1] will
change raw_text_query object and this will change also the value of
`raw_text_query.query` on every `raw_text_query.changeQuery(result)`.
This patch fixes this issue by storing the initial query value in `sug_prefix`.
[1] https://github.com/searx/searx/blob/ac0fdc3b9670168de8061ed0e656dd66a567d741/searx/webapp.py#L804-L806
OpenSearch-Format::
[ "<query>",
[ "<term 1>", "<term 2>", ... "<term n>" ],
[ "<content 1>", "<content 2>", ..., "<content n>" ],
[ "<url 1>", "<url 2>", ..., "<url n>" ]
]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1080002
- https://developer.mozilla.org/en-US/docs/Archive/Add-ons/Supporting_search_suggestions_in_search_plugins#implementing_search_suggestion_support_on_the_server
Legacy-Format::
[ "<term 1>", "<term 2>", ..., "<term n>" ]
- https://www.google.com/support/enterprise/static/gsa/docs/admin/current/gsa_doc_set/xml_reference/query_suggestion.html#1081079
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2021-04-03 16:18:50 +00:00
|
|
|
|
return Response(suggestions, mimetype=mimetype)
|
2014-03-20 09:28:24 +00:00
|
|
|
|
|
|
|
|
|
|
2014-01-01 21:16:53 +00:00
|
|
|
|
@app.route('/preferences', methods=['GET', 'POST'])
|
|
|
|
|
def preferences():
|
2016-04-08 14:38:05 +00:00
|
|
|
|
"""Render preferences page && save user preferences"""
|
2014-02-06 23:35:15 +00:00
|
|
|
|
|
2021-05-26 11:32:04 +00:00
|
|
|
|
# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches
|
|
|
|
|
# pylint: disable=too-many-statements
|
|
|
|
|
|
2024-01-21 11:35:52 +00:00
|
|
|
|
# save preferences using the link the /preferences?preferences=...
|
|
|
|
|
if request.args.get('preferences') or request.form.get('preferences'):
|
2022-05-14 12:12:55 +00:00
|
|
|
|
resp = make_response(redirect(url_for('index', _external=True)))
|
|
|
|
|
return request.preferences.save(resp)
|
|
|
|
|
|
2016-04-08 14:38:05 +00:00
|
|
|
|
# save preferences
|
|
|
|
|
if request.method == 'POST':
|
2021-04-12 13:29:08 +00:00
|
|
|
|
resp = make_response(redirect(url_for('index', _external=True)))
|
2016-04-08 14:38:05 +00:00
|
|
|
|
try:
|
|
|
|
|
request.preferences.parse_form(request.form)
|
|
|
|
|
except ValidationException:
|
2016-11-14 21:07:23 +00:00
|
|
|
|
request.errors.append(gettext('Invalid settings, please edit your preferences'))
|
2016-04-08 14:38:05 +00:00
|
|
|
|
return resp
|
|
|
|
|
return request.preferences.save(resp)
|
|
|
|
|
|
|
|
|
|
# render preferences
|
2021-05-26 11:32:04 +00:00
|
|
|
|
image_proxy = request.preferences.get_value('image_proxy') # pylint: disable=redefined-outer-name
|
2016-04-09 16:26:29 +00:00
|
|
|
|
disabled_engines = request.preferences.engines.get_disabled()
|
2016-04-08 14:38:05 +00:00
|
|
|
|
allowed_plugins = request.preferences.plugins.get_enabled()
|
2015-05-30 10:15:23 +00:00
|
|
|
|
|
|
|
|
|
# stats for preferences page
|
2022-01-09 14:53:58 +00:00
|
|
|
|
filtered_engines = dict(filter(lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
|
2015-05-30 10:15:23 +00:00
|
|
|
|
|
2020-02-01 10:01:17 +00:00
|
|
|
|
engines_by_category = {}
|
2021-07-03 15:51:39 +00:00
|
|
|
|
|
2021-12-27 08:26:22 +00:00
|
|
|
|
for c in categories: # pylint: disable=consider-using-dict-items
|
2021-04-14 15:23:15 +00:00
|
|
|
|
engines_by_category[c] = [e for e in categories[c] if e.name in filtered_engines]
|
2021-04-14 16:11:35 +00:00
|
|
|
|
# sort the engines alphabetically since the order in settings.yml is meaningless.
|
|
|
|
|
list.sort(engines_by_category[c], key=lambda e: e.name)
|
2020-02-01 10:01:17 +00:00
|
|
|
|
|
2016-11-05 12:45:20 +00:00
|
|
|
|
# get first element [0], the engine time,
|
|
|
|
|
# and then the second element [1] : the time (the first one is the label)
|
2021-05-26 11:32:04 +00:00
|
|
|
|
stats = {} # pylint: disable=redefined-outer-name
|
2021-04-14 16:11:35 +00:00
|
|
|
|
max_rate95 = 0
|
2021-04-14 15:23:15 +00:00
|
|
|
|
for _, e in filtered_engines.items():
|
|
|
|
|
h = histogram('engine', e.name, 'time', 'total')
|
|
|
|
|
median = round(h.percentage(50), 1) if h.count > 0 else None
|
2021-04-14 16:11:35 +00:00
|
|
|
|
rate80 = round(h.percentage(80), 1) if h.count > 0 else None
|
|
|
|
|
rate95 = round(h.percentage(95), 1) if h.count > 0 else None
|
|
|
|
|
|
|
|
|
|
max_rate95 = max(max_rate95, rate95 or 0)
|
|
|
|
|
|
|
|
|
|
result_count_sum = histogram('engine', e.name, 'result', 'count').sum
|
|
|
|
|
successful_count = counter('engine', e.name, 'search', 'count', 'successful')
|
|
|
|
|
result_count = int(result_count_sum / float(successful_count)) if successful_count else 0
|
2021-04-14 15:23:15 +00:00
|
|
|
|
|
|
|
|
|
stats[e.name] = {
|
2021-05-19 09:24:18 +00:00
|
|
|
|
'time': median,
|
|
|
|
|
'rate80': rate80,
|
|
|
|
|
'rate95': rate95,
|
2021-04-14 15:23:15 +00:00
|
|
|
|
'warn_timeout': e.timeout > settings['outgoing']['request_timeout'],
|
2022-09-29 18:54:46 +00:00
|
|
|
|
'supports_selected_language': e.traits.is_locale_supported(
|
|
|
|
|
str(request.preferences.get_value('language') or 'all')
|
|
|
|
|
),
|
2021-04-14 16:11:35 +00:00
|
|
|
|
'result_count': result_count,
|
2021-04-14 15:23:15 +00:00
|
|
|
|
}
|
2015-05-30 10:15:23 +00:00
|
|
|
|
# end of stats
|
|
|
|
|
|
2021-04-14 16:11:35 +00:00
|
|
|
|
# reliabilities
|
|
|
|
|
reliabilities = {}
|
|
|
|
|
engine_errors = get_engine_errors(filtered_engines)
|
|
|
|
|
checker_results = checker_get_result()
|
2021-12-27 08:26:22 +00:00
|
|
|
|
checker_results = (
|
|
|
|
|
checker_results['engines'] if checker_results['status'] == 'ok' and 'engines' in checker_results else {}
|
|
|
|
|
)
|
2021-04-14 16:11:35 +00:00
|
|
|
|
for _, e in filtered_engines.items():
|
|
|
|
|
checker_result = checker_results.get(e.name, {})
|
|
|
|
|
checker_success = checker_result.get('success', True)
|
|
|
|
|
errors = engine_errors.get(e.name) or []
|
|
|
|
|
if counter('engine', e.name, 'search', 'count', 'sent') == 0:
|
|
|
|
|
# no request
|
2023-09-15 07:53:03 +00:00
|
|
|
|
reliability = None
|
2021-04-14 16:11:35 +00:00
|
|
|
|
elif checker_success and not errors:
|
2023-09-15 07:53:03 +00:00
|
|
|
|
reliability = 100
|
2021-04-14 16:11:35 +00:00
|
|
|
|
elif 'simple' in checker_result.get('errors', {}):
|
2023-09-15 07:53:03 +00:00
|
|
|
|
# the basic (simple) test doesn't work: the engine is broken according to the checker
|
2021-04-14 16:11:35 +00:00
|
|
|
|
# even if there is no exception
|
2023-09-15 07:53:03 +00:00
|
|
|
|
reliability = 0
|
2021-04-14 16:11:35 +00:00
|
|
|
|
else:
|
2022-06-03 13:41:52 +00:00
|
|
|
|
# pylint: disable=consider-using-generator
|
2023-09-15 07:53:03 +00:00
|
|
|
|
reliability = 100 - sum([error['percentage'] for error in errors if not error.get('secondary')])
|
2021-04-14 16:11:35 +00:00
|
|
|
|
|
|
|
|
|
reliabilities[e.name] = {
|
2023-09-15 07:53:03 +00:00
|
|
|
|
'reliability': reliability,
|
2021-04-14 16:11:35 +00:00
|
|
|
|
'errors': [],
|
|
|
|
|
'checker': checker_results.get(e.name, {}).get('errors', {}).keys(),
|
|
|
|
|
}
|
|
|
|
|
# keep the order of the list checker_results[e.name]['errors'] and deduplicate.
|
|
|
|
|
# the first element has the highest percentage rate.
|
|
|
|
|
reliabilities_errors = []
|
|
|
|
|
for error in errors:
|
2021-04-26 09:12:02 +00:00
|
|
|
|
error_user_text = None
|
2021-04-14 16:11:35 +00:00
|
|
|
|
if error.get('secondary') or 'exception_classname' not in error:
|
|
|
|
|
continue
|
2021-04-26 09:12:02 +00:00
|
|
|
|
error_user_text = exception_classname_to_text.get(error.get('exception_classname'))
|
2021-04-14 16:11:35 +00:00
|
|
|
|
if not error:
|
2021-04-26 09:12:02 +00:00
|
|
|
|
error_user_text = exception_classname_to_text[None]
|
|
|
|
|
if error_user_text not in reliabilities_errors:
|
|
|
|
|
reliabilities_errors.append(error_user_text)
|
2021-04-14 16:11:35 +00:00
|
|
|
|
reliabilities[e.name]['errors'] = reliabilities_errors
|
|
|
|
|
|
|
|
|
|
# supports
|
|
|
|
|
supports = {}
|
|
|
|
|
for _, e in filtered_engines.items():
|
2022-09-29 18:54:46 +00:00
|
|
|
|
supports_selected_language = e.traits.is_locale_supported(
|
|
|
|
|
str(request.preferences.get_value('language') or 'all')
|
|
|
|
|
)
|
2021-04-14 16:11:35 +00:00
|
|
|
|
safesearch = e.safesearch
|
|
|
|
|
time_range_support = e.time_range_support
|
|
|
|
|
for checker_test_name in checker_results.get(e.name, {}).get('errors', {}):
|
|
|
|
|
if supports_selected_language and checker_test_name.startswith('lang_'):
|
|
|
|
|
supports_selected_language = '?'
|
|
|
|
|
elif safesearch and checker_test_name == 'safesearch':
|
|
|
|
|
safesearch = '?'
|
|
|
|
|
elif time_range_support and checker_test_name == 'time_range':
|
|
|
|
|
time_range_support = '?'
|
|
|
|
|
supports[e.name] = {
|
|
|
|
|
'supports_selected_language': supports_selected_language,
|
|
|
|
|
'safesearch': safesearch,
|
|
|
|
|
'time_range_support': time_range_support,
|
|
|
|
|
}
|
|
|
|
|
|
2021-06-08 16:19:30 +00:00
|
|
|
|
return render(
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: off
|
2021-06-08 16:19:30 +00:00
|
|
|
|
'preferences.html',
|
|
|
|
|
selected_categories = get_selected_categories(request.preferences, request.form),
|
2021-08-03 13:13:00 +00:00
|
|
|
|
locales = LOCALE_NAMES,
|
2021-06-08 16:19:30 +00:00
|
|
|
|
current_locale = request.preferences.get_value("locale"),
|
|
|
|
|
image_proxy = image_proxy,
|
|
|
|
|
engines_by_category = engines_by_category,
|
|
|
|
|
stats = stats,
|
|
|
|
|
max_rate95 = max_rate95,
|
|
|
|
|
reliabilities = reliabilities,
|
|
|
|
|
supports = supports,
|
|
|
|
|
answerers = [
|
|
|
|
|
{'info': a.self_info(), 'keywords': a.keywords}
|
|
|
|
|
for a in answerers
|
|
|
|
|
],
|
|
|
|
|
disabled_engines = disabled_engines,
|
|
|
|
|
autocomplete_backends = autocomplete_backends,
|
2024-08-19 15:47:54 +00:00
|
|
|
|
favicon_resolver_names = favicons.proxy.CFG.resolver_map.keys(),
|
2021-06-08 16:19:30 +00:00
|
|
|
|
shortcuts = {y: x for x, y in engine_shortcuts.items()},
|
|
|
|
|
themes = themes,
|
|
|
|
|
plugins = plugins,
|
|
|
|
|
doi_resolvers = settings['doi_resolvers'],
|
2021-09-06 06:47:11 +00:00
|
|
|
|
current_doi_resolver = get_doi_resolver(request.preferences),
|
2021-06-08 16:19:30 +00:00
|
|
|
|
allowed_plugins = allowed_plugins,
|
|
|
|
|
preferences_url_params = request.preferences.get_as_url_params(),
|
|
|
|
|
locked_preferences = settings['preferences']['lock'],
|
|
|
|
|
preferences = True
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: on
|
2021-06-08 16:19:30 +00:00
|
|
|
|
)
|
2014-01-01 21:16:53 +00:00
|
|
|
|
|
|
|
|
|
|
2024-08-19 15:47:54 +00:00
|
|
|
|
app.add_url_rule('/favicon_proxy', methods=['GET'], endpoint="favicon_proxy", view_func=favicons.favicon_proxy)
|
|
|
|
|
|
|
|
|
|
|
2015-01-16 15:02:21 +00:00
|
|
|
|
@app.route('/image_proxy', methods=['GET'])
|
|
|
|
|
def image_proxy():
|
2021-08-14 17:36:30 +00:00
|
|
|
|
# pylint: disable=too-many-return-statements, too-many-branches
|
2021-05-26 11:32:04 +00:00
|
|
|
|
|
2021-03-18 18:59:01 +00:00
|
|
|
|
url = request.args.get('url')
|
2015-01-16 15:02:21 +00:00
|
|
|
|
if not url:
|
|
|
|
|
return '', 400
|
|
|
|
|
|
2021-12-28 12:44:28 +00:00
|
|
|
|
if not is_hmac_of(settings['server']['secret_key'], url.encode(), request.args.get('h', '')):
|
2015-01-17 20:54:40 +00:00
|
|
|
|
return '', 400
|
|
|
|
|
|
2021-03-18 18:59:01 +00:00
|
|
|
|
maximum_size = 5 * 1024 * 1024
|
2021-08-14 17:36:30 +00:00
|
|
|
|
forward_resp = False
|
|
|
|
|
resp = None
|
2021-03-18 18:59:01 +00:00
|
|
|
|
try:
|
2021-08-14 17:36:30 +00:00
|
|
|
|
request_headers = {
|
|
|
|
|
'User-Agent': gen_useragent(),
|
|
|
|
|
'Accept': 'image/webp,*/*',
|
|
|
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
|
|
|
'Sec-GPC': '1',
|
|
|
|
|
'DNT': '1',
|
|
|
|
|
}
|
2021-08-14 18:12:11 +00:00
|
|
|
|
set_context_network_name('image_proxy')
|
2022-02-19 13:26:58 +00:00
|
|
|
|
resp, stream = http_stream(method='GET', url=url, headers=request_headers, allow_redirects=True)
|
2021-03-18 18:59:01 +00:00
|
|
|
|
content_length = resp.headers.get('Content-Length')
|
2021-12-27 08:26:22 +00:00
|
|
|
|
if content_length and content_length.isdigit() and int(content_length) > maximum_size:
|
2021-03-18 18:59:01 +00:00
|
|
|
|
return 'Max size', 400
|
|
|
|
|
|
|
|
|
|
if resp.status_code != 200:
|
2021-08-14 17:36:30 +00:00
|
|
|
|
logger.debug('image-proxy: wrong response code: %i', resp.status_code)
|
2021-03-18 18:59:01 +00:00
|
|
|
|
if resp.status_code >= 400:
|
|
|
|
|
return '', resp.status_code
|
|
|
|
|
return '', 400
|
|
|
|
|
|
2022-05-31 19:07:30 +00:00
|
|
|
|
if not resp.headers.get('Content-Type', '').startswith('image/') and not resp.headers.get(
|
|
|
|
|
'Content-Type', ''
|
|
|
|
|
).startswith('binary/octet-stream'):
|
2021-08-14 17:36:30 +00:00
|
|
|
|
logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', ''))
|
2021-03-18 18:59:01 +00:00
|
|
|
|
return '', 400
|
2015-01-16 15:02:21 +00:00
|
|
|
|
|
2021-08-14 17:36:30 +00:00
|
|
|
|
forward_resp = True
|
|
|
|
|
except httpx.HTTPError:
|
|
|
|
|
logger.exception('HTTP error')
|
|
|
|
|
return '', 400
|
|
|
|
|
finally:
|
|
|
|
|
if resp and not forward_resp:
|
|
|
|
|
# the code is about to return an HTTP 400 error to the browser
|
|
|
|
|
# we make sure to close the response between searxng and the HTTP server
|
|
|
|
|
try:
|
|
|
|
|
resp.close()
|
|
|
|
|
except httpx.HTTPError:
|
|
|
|
|
logger.exception('HTTP error on closing')
|
|
|
|
|
|
2021-09-28 15:44:00 +00:00
|
|
|
|
def close_stream():
|
|
|
|
|
nonlocal resp, stream
|
|
|
|
|
try:
|
2022-01-24 10:44:18 +00:00
|
|
|
|
if resp:
|
|
|
|
|
resp.close()
|
2021-09-28 15:44:00 +00:00
|
|
|
|
del resp
|
|
|
|
|
del stream
|
|
|
|
|
except httpx.HTTPError as e:
|
|
|
|
|
logger.debug('Exception while closing response', e)
|
|
|
|
|
|
2021-08-14 17:36:30 +00:00
|
|
|
|
try:
|
2021-12-27 08:26:22 +00:00
|
|
|
|
headers = dict_subset(resp.headers, {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'})
|
|
|
|
|
response = Response(stream, mimetype=resp.headers['Content-Type'], headers=headers, direct_passthrough=True)
|
2021-09-28 15:44:00 +00:00
|
|
|
|
response.call_on_close(close_stream)
|
|
|
|
|
return response
|
2021-03-18 18:59:01 +00:00
|
|
|
|
except httpx.HTTPError:
|
2021-09-28 15:44:00 +00:00
|
|
|
|
close_stream()
|
2021-03-18 18:59:01 +00:00
|
|
|
|
return '', 400
|
2021-09-18 08:59:56 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@app.route('/engine_descriptions.json', methods=['GET'])
|
|
|
|
|
def engine_descriptions():
|
|
|
|
|
locale = get_locale().split('_')[0]
|
|
|
|
|
result = ENGINE_DESCRIPTIONS['en'].copy()
|
|
|
|
|
if locale != 'en':
|
|
|
|
|
for engine, description in ENGINE_DESCRIPTIONS.get(locale, {}).items():
|
|
|
|
|
result[engine] = description
|
|
|
|
|
for engine, description in result.items():
|
2021-12-27 08:26:22 +00:00
|
|
|
|
if len(description) == 2 and description[1] == 'ref':
|
2021-09-18 08:59:56 +00:00
|
|
|
|
ref_engine, ref_lang = description[0].split(':')
|
|
|
|
|
description = ENGINE_DESCRIPTIONS[ref_lang][ref_engine]
|
|
|
|
|
if isinstance(description, str):
|
2021-12-27 08:26:22 +00:00
|
|
|
|
description = [description, 'wikipedia']
|
2021-09-18 08:59:56 +00:00
|
|
|
|
result[engine] = description
|
2022-01-23 19:55:14 +00:00
|
|
|
|
|
|
|
|
|
# overwrite by about:description (from settings)
|
|
|
|
|
for engine_name, engine_mod in engines.items():
|
|
|
|
|
descr = getattr(engine_mod, 'about', {}).get('description', None)
|
|
|
|
|
if descr is not None:
|
|
|
|
|
result[engine_name] = [descr, "SearXNG config"]
|
|
|
|
|
|
2021-09-18 08:59:56 +00:00
|
|
|
|
return jsonify(result)
|
2015-01-16 15:02:21 +00:00
|
|
|
|
|
|
|
|
|
|
2013-10-26 23:03:05 +00:00
|
|
|
|
@app.route('/stats', methods=['GET'])
|
|
|
|
|
def stats():
|
2014-01-31 06:08:24 +00:00
|
|
|
|
"""Render engine statistics page."""
|
2021-04-23 19:08:48 +00:00
|
|
|
|
sort_order = request.args.get('sort', default='name', type=str)
|
|
|
|
|
selected_engine_name = request.args.get('engine', default=None, type=str)
|
|
|
|
|
|
2022-01-09 14:53:58 +00:00
|
|
|
|
filtered_engines = dict(filter(lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
|
2021-04-23 19:08:48 +00:00
|
|
|
|
if selected_engine_name:
|
|
|
|
|
if selected_engine_name not in filtered_engines:
|
|
|
|
|
selected_engine_name = None
|
|
|
|
|
else:
|
|
|
|
|
filtered_engines = [selected_engine_name]
|
|
|
|
|
|
2021-04-22 15:47:53 +00:00
|
|
|
|
checker_results = checker_get_result()
|
2021-06-08 16:19:30 +00:00
|
|
|
|
checker_results = (
|
2021-12-27 08:26:22 +00:00
|
|
|
|
checker_results['engines'] if checker_results['status'] == 'ok' and 'engines' in checker_results else {}
|
2021-06-08 16:19:30 +00:00
|
|
|
|
)
|
2021-04-22 15:47:53 +00:00
|
|
|
|
|
2021-04-14 15:23:15 +00:00
|
|
|
|
engine_stats = get_engines_stats(filtered_engines)
|
2021-04-22 15:47:53 +00:00
|
|
|
|
engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
|
|
|
|
|
|
2021-05-26 11:32:04 +00:00
|
|
|
|
if sort_order not in STATS_SORT_PARAMETERS:
|
2021-04-22 15:47:53 +00:00
|
|
|
|
sort_order = 'name'
|
|
|
|
|
|
2021-05-26 11:32:04 +00:00
|
|
|
|
reverse, key_name, default_value = STATS_SORT_PARAMETERS[sort_order]
|
2021-04-22 15:47:53 +00:00
|
|
|
|
|
|
|
|
|
def get_key(engine_stat):
|
2023-09-15 07:53:03 +00:00
|
|
|
|
reliability = engine_reliabilities.get(engine_stat['name'], {}).get('reliability', 0)
|
2021-04-22 15:47:53 +00:00
|
|
|
|
reliability_order = 0 if reliability else 1
|
|
|
|
|
if key_name == 'reliability':
|
|
|
|
|
key = reliability
|
|
|
|
|
reliability_order = 0
|
|
|
|
|
else:
|
|
|
|
|
key = engine_stat.get(key_name) or default_value
|
|
|
|
|
if reverse:
|
|
|
|
|
reliability_order = 1 - reliability_order
|
|
|
|
|
return (reliability_order, key, engine_stat['name'])
|
|
|
|
|
|
2023-10-05 14:42:14 +00:00
|
|
|
|
technical_report = []
|
|
|
|
|
for error in engine_reliabilities.get(selected_engine_name, {}).get('errors', []):
|
|
|
|
|
technical_report.append(
|
|
|
|
|
f"\
|
|
|
|
|
Error: {error['exception_classname'] or error['log_message']} \
|
|
|
|
|
Parameters: {error['log_parameters']} \
|
|
|
|
|
File name: {error['filename'] }:{ error['line_no'] } \
|
|
|
|
|
Error Function: {error['function']} \
|
|
|
|
|
Code: {error['code']} \
|
|
|
|
|
".replace(
|
|
|
|
|
' ' * 12, ''
|
|
|
|
|
).strip()
|
|
|
|
|
)
|
|
|
|
|
technical_report = ' '.join(technical_report)
|
|
|
|
|
|
2021-04-22 15:47:53 +00:00
|
|
|
|
engine_stats['time'] = sorted(engine_stats['time'], reverse=reverse, key=get_key)
|
2014-03-21 11:19:48 +00:00
|
|
|
|
return render(
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: off
|
2014-03-21 11:19:48 +00:00
|
|
|
|
'stats.html',
|
2021-06-08 16:19:30 +00:00
|
|
|
|
sort_order = sort_order,
|
|
|
|
|
engine_stats = engine_stats,
|
|
|
|
|
engine_reliabilities = engine_reliabilities,
|
|
|
|
|
selected_engine_name = selected_engine_name,
|
2021-08-18 08:36:53 +00:00
|
|
|
|
searx_git_branch = GIT_BRANCH,
|
2023-10-05 14:42:14 +00:00
|
|
|
|
technical_report = technical_report,
|
2021-12-27 08:16:03 +00:00
|
|
|
|
# fmt: on
|
2014-03-21 11:19:48 +00:00
|
|
|
|
)
|
2013-10-26 23:03:05 +00:00
|
|
|
|
|
2014-01-01 21:16:53 +00:00
|
|
|
|
|
2020-11-26 14:12:11 +00:00
|
|
|
|
@app.route('/stats/errors', methods=['GET'])
|
|
|
|
|
def stats_errors():
|
2022-01-09 14:53:58 +00:00
|
|
|
|
filtered_engines = dict(filter(lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
|
2021-04-14 15:23:15 +00:00
|
|
|
|
result = get_engine_errors(filtered_engines)
|
2020-11-26 14:12:11 +00:00
|
|
|
|
return jsonify(result)
|
|
|
|
|
|
|
|
|
|
|
2021-01-05 10:24:39 +00:00
|
|
|
|
@app.route('/stats/checker', methods=['GET'])
|
|
|
|
|
def stats_checker():
|
|
|
|
|
result = checker_get_result()
|
|
|
|
|
return jsonify(result)
|
|
|
|
|
|
|
|
|
|
|
2024-09-17 14:43:48 +00:00
|
|
|
|
@app.route('/metrics')
|
|
|
|
|
def stats_open_metrics():
|
|
|
|
|
password = settings['general'].get("open_metrics")
|
|
|
|
|
|
|
|
|
|
if not (settings['general'].get("enable_metrics") and password):
|
|
|
|
|
return Response('open metrics is disabled', status=404, mimetype='text/plain')
|
|
|
|
|
|
|
|
|
|
if not request.authorization or request.authorization.password != password:
|
|
|
|
|
return Response('access forbidden', status=401, mimetype='text/plain')
|
|
|
|
|
|
|
|
|
|
filtered_engines = dict(filter(lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
|
|
|
|
|
|
|
|
|
|
checker_results = checker_get_result()
|
|
|
|
|
checker_results = (
|
|
|
|
|
checker_results['engines'] if checker_results['status'] == 'ok' and 'engines' in checker_results else {}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
engine_stats = get_engines_stats(filtered_engines)
|
|
|
|
|
engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
|
|
|
|
|
metrics_text = openmetrics(engine_stats, engine_reliabilities)
|
|
|
|
|
|
|
|
|
|
return Response(metrics_text, mimetype='text/plain')
|
|
|
|
|
|
|
|
|
|
|
2013-12-01 15:10:38 +00:00
|
|
|
|
@app.route('/robots.txt', methods=['GET'])
|
|
|
|
|
def robots():
|
2021-12-27 08:26:22 +00:00
|
|
|
|
return Response(
|
|
|
|
|
"""User-agent: *
|
2022-03-16 21:24:35 +00:00
|
|
|
|
Allow: /info/en/about
|
2013-12-01 15:10:38 +00:00
|
|
|
|
Disallow: /stats
|
2022-03-16 21:24:35 +00:00
|
|
|
|
Disallow: /image_proxy
|
2014-02-07 17:43:05 +00:00
|
|
|
|
Disallow: /preferences
|
2016-10-01 18:22:52 +00:00
|
|
|
|
Disallow: /*?*q=*
|
2021-12-27 08:26:22 +00:00
|
|
|
|
""",
|
|
|
|
|
mimetype='text/plain',
|
|
|
|
|
)
|
2013-12-01 15:10:38 +00:00
|
|
|
|
|
2014-01-01 21:16:53 +00:00
|
|
|
|
|
2013-10-15 22:01:08 +00:00
|
|
|
|
@app.route('/opensearch.xml', methods=['GET'])
|
|
|
|
|
def opensearch():
|
2022-08-11 16:37:12 +00:00
|
|
|
|
method = request.preferences.get_value('method')
|
|
|
|
|
autocomplete = request.preferences.get_value('autocomplete')
|
2015-03-15 19:07:50 +00:00
|
|
|
|
|
2013-10-20 22:28:48 +00:00
|
|
|
|
# chrome/chromium only supports HTTP GET....
|
2013-10-20 20:37:55 +00:00
|
|
|
|
if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
|
2022-08-11 16:37:12 +00:00
|
|
|
|
method = 'GET'
|
2014-03-29 15:29:19 +00:00
|
|
|
|
|
2022-08-11 16:37:12 +00:00
|
|
|
|
if method not in ('POST', 'GET'):
|
|
|
|
|
method = 'POST'
|
2022-05-07 13:11:05 +00:00
|
|
|
|
|
|
|
|
|
ret = render('opensearch.xml', opensearch_method=method, autocomplete=autocomplete)
|
2021-12-27 08:26:22 +00:00
|
|
|
|
resp = Response(response=ret, status=200, mimetype="application/opensearchdescription+xml")
|
2013-10-15 22:01:08 +00:00
|
|
|
|
return resp
|
|
|
|
|
|
2014-01-19 21:59:01 +00:00
|
|
|
|
|
2013-12-01 22:52:49 +00:00
|
|
|
|
@app.route('/favicon.ico')
|
|
|
|
|
def favicon():
|
2022-05-07 13:11:05 +00:00
|
|
|
|
theme = request.preferences.get_value("theme")
|
2021-05-26 11:32:04 +00:00
|
|
|
|
return send_from_directory(
|
2022-07-28 11:02:56 +00:00
|
|
|
|
os.path.join(app.root_path, settings['ui']['static_path'], 'themes', theme, 'img'), # pyright: ignore
|
2021-05-26 11:32:04 +00:00
|
|
|
|
'favicon.png',
|
2021-12-27 08:26:22 +00:00
|
|
|
|
mimetype='image/vnd.microsoft.icon',
|
2021-05-26 11:32:04 +00:00
|
|
|
|
)
|
2013-12-01 22:52:49 +00:00
|
|
|
|
|
2021-12-27 08:26:22 +00:00
|
|
|
|
|
2015-04-07 09:07:48 +00:00
|
|
|
|
@app.route('/clear_cookies')
|
|
|
|
|
def clear_cookies():
|
2021-04-03 11:56:47 +00:00
|
|
|
|
resp = make_response(redirect(url_for('index', _external=True)))
|
2015-04-07 09:07:48 +00:00
|
|
|
|
for cookie_name in request.cookies:
|
|
|
|
|
resp.delete_cookie(cookie_name)
|
|
|
|
|
return resp
|
|
|
|
|
|
|
|
|
|
|
2016-06-07 21:08:48 +00:00
|
|
|
|
@app.route('/config')
|
|
|
|
|
def config():
|
2020-03-25 16:12:02 +00:00
|
|
|
|
"""Return configuration in JSON format."""
|
|
|
|
|
_engines = []
|
|
|
|
|
for name, engine in engines.items():
|
|
|
|
|
if not request.preferences.validate_token(engine):
|
|
|
|
|
continue
|
|
|
|
|
|
2022-09-29 18:54:46 +00:00
|
|
|
|
_languages = engine.traits.languages.keys()
|
2021-12-27 08:26:22 +00:00
|
|
|
|
_engines.append(
|
|
|
|
|
{
|
|
|
|
|
'name': name,
|
|
|
|
|
'categories': engine.categories,
|
|
|
|
|
'shortcut': engine.shortcut,
|
|
|
|
|
'enabled': not engine.disabled,
|
|
|
|
|
'paging': engine.paging,
|
|
|
|
|
'language_support': engine.language_support,
|
2022-09-29 18:54:46 +00:00
|
|
|
|
'languages': list(_languages),
|
|
|
|
|
'regions': list(engine.traits.regions.keys()),
|
2021-12-27 08:26:22 +00:00
|
|
|
|
'safesearch': engine.safesearch,
|
|
|
|
|
'time_range_support': engine.time_range_support,
|
|
|
|
|
'timeout': engine.timeout,
|
|
|
|
|
}
|
|
|
|
|
)
|
2020-03-25 16:12:02 +00:00
|
|
|
|
|
|
|
|
|
_plugins = []
|
|
|
|
|
for _ in plugins:
|
|
|
|
|
_plugins.append({'name': _.name, 'enabled': _.default_on})
|
|
|
|
|
|
2023-09-23 21:22:59 +00:00
|
|
|
|
_limiter_cfg = limiter.get_cfg()
|
|
|
|
|
|
2021-12-27 08:26:22 +00:00
|
|
|
|
return jsonify(
|
|
|
|
|
{
|
|
|
|
|
'categories': list(categories.keys()),
|
|
|
|
|
'engines': _engines,
|
|
|
|
|
'plugins': _plugins,
|
|
|
|
|
'instance_name': settings['general']['instance_name'],
|
|
|
|
|
'locales': LOCALE_NAMES,
|
|
|
|
|
'default_locale': settings['ui']['default_locale'],
|
|
|
|
|
'autocomplete': settings['search']['autocomplete'],
|
|
|
|
|
'safe_search': settings['search']['safe_search'],
|
|
|
|
|
'default_theme': settings['ui']['default_theme'],
|
|
|
|
|
'version': VERSION_STRING,
|
|
|
|
|
'brand': {
|
2022-06-15 20:06:52 +00:00
|
|
|
|
'PRIVACYPOLICY_URL': get_setting('general.privacypolicy_url'),
|
2021-12-27 08:26:22 +00:00
|
|
|
|
'CONTACT_URL': get_setting('general.contact_url'),
|
|
|
|
|
'GIT_URL': GIT_URL,
|
|
|
|
|
'GIT_BRANCH': GIT_BRANCH,
|
|
|
|
|
'DOCS_URL': get_setting('brand.docs_url'),
|
|
|
|
|
},
|
2023-09-23 21:22:59 +00:00
|
|
|
|
'limiter': {
|
2023-10-02 14:36:07 +00:00
|
|
|
|
'enabled': limiter.is_installed(),
|
2023-09-23 21:22:59 +00:00
|
|
|
|
'botdetection.ip_limit.link_token': _limiter_cfg.get('botdetection.ip_limit.link_token'),
|
|
|
|
|
'botdetection.ip_lists.pass_searxng_org': _limiter_cfg.get('botdetection.ip_lists.pass_searxng_org'),
|
|
|
|
|
},
|
2021-12-27 08:26:22 +00:00
|
|
|
|
'doi_resolvers': list(settings['doi_resolvers'].keys()),
|
|
|
|
|
'default_doi_resolver': settings['default_doi_resolver'],
|
2023-09-24 09:02:29 +00:00
|
|
|
|
'public_instance': settings['server']['public_instance'],
|
2021-12-27 08:26:22 +00:00
|
|
|
|
}
|
|
|
|
|
)
|
2016-06-07 21:08:48 +00:00
|
|
|
|
|
|
|
|
|
|
2016-08-24 17:53:09 +00:00
|
|
|
|
@app.errorhandler(404)
|
2021-05-26 11:32:04 +00:00
|
|
|
|
def page_not_found(_e):
|
2016-09-07 06:32:01 +00:00
|
|
|
|
return render('404.html'), 404
|
2016-08-24 17:53:09 +00:00
|
|
|
|
|
|
|
|
|
|
2021-09-13 17:37:51 +00:00
|
|
|
|
# see https://flask.palletsprojects.com/en/1.1.x/cli/
|
|
|
|
|
# True if "FLASK_APP=searx/webapp.py FLASK_ENV=development flask run"
|
|
|
|
|
flask_run_development = (
|
2021-12-27 08:26:22 +00:00
|
|
|
|
os.environ.get("FLASK_APP") is not None and os.environ.get("FLASK_ENV") == 'development' and is_flask_run_cmdline()
|
2021-09-13 17:37:51 +00:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# True if reload feature is activated of werkzeug, False otherwise (including uwsgi, etc..)
|
|
|
|
|
# __name__ != "__main__" if searx.webapp is imported (make test, make docs, uwsgi...)
|
|
|
|
|
# see run() at the end of this file : searx_debug activates the reload feature.
|
|
|
|
|
werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__main__")
|
|
|
|
|
|
|
|
|
|
# initialize the engines except on the first run of the werkzeug server.
|
2021-12-27 08:26:22 +00:00
|
|
|
|
if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
|
2022-06-10 15:01:12 +00:00
|
|
|
|
locales_initialize()
|
2022-07-15 16:38:32 +00:00
|
|
|
|
redis_initialize()
|
2021-09-13 17:37:51 +00:00
|
|
|
|
plugin_initialize(app)
|
2021-12-26 21:44:46 +00:00
|
|
|
|
search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics'])
|
2023-10-02 14:36:07 +00:00
|
|
|
|
limiter.initialize(app, settings)
|
2024-08-19 15:47:54 +00:00
|
|
|
|
favicons.init()
|
2021-09-13 17:37:51 +00:00
|
|
|
|
|
|
|
|
|
|
2014-01-12 11:40:27 +00:00
|
|
|
|
def run():
|
2021-12-27 08:26:22 +00:00
|
|
|
|
logger.debug('starting webserver on %s:%s', settings['server']['bind_address'], settings['server']['port'])
|
2014-01-19 21:59:01 +00:00
|
|
|
|
app.run(
|
2021-12-27 08:26:22 +00:00
|
|
|
|
debug=searx_debug,
|
|
|
|
|
use_debugger=searx_debug,
|
|
|
|
|
port=settings['server']['port'],
|
|
|
|
|
host=settings['server']['bind_address'],
|
|
|
|
|
threaded=True,
|
2024-06-12 16:01:18 +00:00
|
|
|
|
extra_files=[DEFAULT_SETTINGS_FILE],
|
2014-01-19 21:59:01 +00:00
|
|
|
|
)
|
2014-01-12 11:40:27 +00:00
|
|
|
|
|
2021-12-27 08:26:22 +00:00
|
|
|
|
|
2015-06-16 17:55:31 +00:00
|
|
|
|
application = app
|
2021-06-08 08:08:41 +00:00
|
|
|
|
patch_application(app)
|
2014-07-03 20:02:53 +00:00
|
|
|
|
|
2014-01-12 11:40:27 +00:00
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
run()
|