searxng/searx/webapp.py

1164 lines
42 KiB
Python
Raw Normal View History

2013-10-14 21:09:13 +00:00
#!/usr/bin/env python
2013-10-14 22:33:18 +00:00
'''
searx is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
searx is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
import sys
if sys.version_info[0] < 3:
print('\033[1;31m Python2 is no longer supported\033[0m')
exit(1)
2014-02-14 15:16:20 +00:00
if __name__ == '__main__':
from os.path import realpath, dirname
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
2014-02-14 15:16:20 +00:00
import hashlib
import hmac
import json
import os
2016-11-30 17:43:03 +00:00
import requests
2014-01-21 20:28:54 +00:00
from searx import logger
logger = logger.getChild('webapp')
2014-03-14 08:55:04 +00:00
from datetime import datetime, timedelta
from time import time
from html import escape
from io import StringIO
from urllib.parse import urlencode, urlparse
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter # pylint: disable=no-name-in-module
from werkzeug.middleware.proxy_fix import ProxyFix
2014-02-05 19:24:31 +00:00
from flask import (
Flask, request, render_template, url_for, Response, make_response,
redirect, send_from_directory
)
from babel.support import Translations
import flask_babel
from flask_babel import Babel, gettext, format_date, format_decimal
from flask.ctx import has_request_context
from flask.json import jsonify
2020-07-14 16:56:57 +00:00
from searx import brand, static_path
from searx import settings, searx_dir, searx_debug
2016-11-30 17:43:03 +00:00
from searx.exceptions import SearxParameterException
2014-02-05 19:24:31 +00:00
from searx.engines import (
categories, engines, engine_shortcuts, get_engines_stats
2014-02-05 19:24:31 +00:00
)
from searx.webutils import (
UnicodeWriter, highlight_content, get_resources_directory,
get_static_files, get_result_templates, get_themes,
prettify_url, new_hmac, is_flask_run_cmdline
2014-04-24 23:46:40 +00:00
)
from searx.webadapter import get_search_query_from_webapp, get_selected_categories
from searx.utils import html_to_text, gen_useragent, dict_subset, match_language
2014-11-18 10:37:42 +00:00
from searx.version import VERSION_STRING
from searx.languages import language_codes as languages
from searx.search import SearchWithPlugins, initialize as search_initialize
from searx.search.checker import get_result as checker_get_result
2016-11-14 21:15:03 +00:00
from searx.query import RawTextQuery
from searx.autocomplete import search_autocomplete, backends as autocomplete_backends
from searx.plugins import plugins
from searx.plugins.oa_doi_rewrite import get_doi_resolver
from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
from searx.answerers import answerers
from searx.poolrequests import get_global_proxies
from searx.answerers import ask
from searx.metrology.error_recorder import errors_per_engines
2013-12-01 22:52:49 +00:00
2016-09-15 11:47:09 +00:00
# serve pages with HTTP/1.1
from werkzeug.serving import WSGIRequestHandler
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
2013-10-14 21:09:13 +00:00
# check secret_key
if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
logger.error('server.secret_key is not changed. Please use something else instead of ultrasecretkey.')
exit(1)
# about static
static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
logger.debug('static directory is %s', static_path)
static_files = get_static_files(static_path)
2014-10-09 17:26:02 +00:00
# about templates
default_theme = settings['ui']['default_theme']
templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
logger.debug('templates directory is %s', templates_path)
themes = get_themes(templates_path)
result_templates = get_result_templates(templates_path)
global_favicons = []
for indice, theme in enumerate(themes):
global_favicons.append([])
theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons')
for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
global_favicons[indice].extend(filenames)
2014-04-24 23:46:40 +00:00
# Flask app
app = Flask(
__name__,
2014-04-24 23:46:40 +00:00
static_folder=static_path,
template_folder=templates_path
)
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
app.jinja_env.add_extension('jinja2.ext.loopcontrols') # pylint: disable=no-member
app.secret_key = settings['server']['secret_key']
2014-01-14 17:17:19 +00:00
# see https://flask.palletsprojects.com/en/1.1.x/cli/
# True if "FLASK_APP=searx/webapp.py FLASK_ENV=development flask run"
flask_run_development = \
os.environ.get("FLASK_APP") is not None\
and os.environ.get("FLASK_ENV") == 'development'\
and is_flask_run_cmdline()
# True if reload feature is activated of werkzeug, False otherwise (including uwsgi, etc..)
# __name__ != "__main__" if searx.webapp is imported (make test, make docs, uwsgi...)
# see run() at the end of this file : searx_debug activates the reload feature.
werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__main__")
# initialize the engines except on the first run of the werkzeug server.
if not werkzeug_reloader\
or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
search_initialize(enable_checker=True)
babel = Babel(app)
rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'fa_IR', 'glk', 'he',
'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi']
ui_locale_codes = [l.replace('_', '-') for l in settings['locales'].keys()]
# used when translating category names
2015-02-10 22:14:37 +00:00
_category_names = (gettext('files'),
gettext('general'),
gettext('music'),
gettext('social media'),
gettext('images'),
gettext('videos'),
gettext('it'),
gettext('news'),
gettext('map'),
gettext('onions'),
2016-01-21 14:57:02 +00:00
gettext('science'))
2015-02-10 22:14:37 +00:00
_flask_babel_get_translations = flask_babel.get_translations
# monkey patch for flask_babel.get_translations
def _get_translations():
if has_request_context() and request.form.get('use-translation') == 'oc':
babel_ext = flask_babel.current_app.extensions['babel']
return Translations.load(next(babel_ext.translation_directories), 'oc')
return _flask_babel_get_translations()
flask_babel.get_translations = _get_translations
2014-01-19 22:04:09 +00:00
def _get_browser_or_settings_language(request, lang_list):
2020-02-23 09:03:42 +00:00
for lang in request.headers.get("Accept-Language", "en").split(","):
if ';' in lang:
lang = lang.split(';')[0]
if '-' in lang:
lang_parts = lang.split('-')
lang = "{}-{}".format(lang_parts[0], lang_parts[-1].upper())
2020-02-23 09:03:42 +00:00
locale = match_language(lang, lang_list, fallback=None)
if locale is not None:
return locale
return settings['search']['default_lang'] or 'en'
2020-02-23 09:03:42 +00:00
@babel.localeselector
def get_locale():
if 'locale' in request.form\
and request.form['locale'] in settings['locales']:
# use locale from the form
2014-01-21 23:59:18 +00:00
locale = request.form['locale']
locale_source = 'form'
elif request.preferences.get_value('locale') != '':
# use locale from the preferences
locale = request.preferences.get_value('locale')
locale_source = 'preferences'
else:
# use local from the browser
locale = _get_browser_or_settings_language(request, ui_locale_codes)
locale = locale.replace('-', '_')
locale_source = 'browser'
2014-01-21 23:59:18 +00:00
# see _get_translations function
# and https://github.com/searx/searx/pull/1863
if locale == 'oc':
request.form['use-translation'] = 'oc'
locale = 'fr_FR'
logger.debug("%s uses locale `%s` from %s", request.url, locale, locale_source)
2014-01-21 23:59:18 +00:00
return locale
2014-12-20 22:33:03 +00:00
# code-highlighter
@app.template_filter('code_highlighter')
def code_highlighter(codelines, language=None):
if not language:
language = 'text'
try:
# find lexer by programing language
lexer = get_lexer_by_name(language, stripall=True)
except:
# if lexer is not found, using default one
2015-01-15 17:39:40 +00:00
logger.debug('highlighter cannot find lexer for {0}'.format(language))
lexer = get_lexer_by_name('text', stripall=True)
2014-12-20 22:33:03 +00:00
html_code = ''
tmp_code = ''
last_line = None
# parse lines
for line, code in codelines:
if not last_line:
line_code_start = line
# new codeblock is detected
if last_line is not None and\
last_line + 1 != line:
2014-12-20 22:33:03 +00:00
# highlight last codepart
formatter = HtmlFormatter(linenos='inline',
linenostart=line_code_start,
cssclass="code-highlight")
2014-12-20 22:33:03 +00:00
html_code = html_code + highlight(tmp_code, lexer, formatter)
2014-12-20 22:33:03 +00:00
# reset conditions for next codepart
tmp_code = ''
line_code_start = line
# add codepart
tmp_code += code + '\n'
2014-12-20 22:33:03 +00:00
# update line
last_line = line
# highlight last codepart
formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start, cssclass="code-highlight")
2014-12-20 22:33:03 +00:00
html_code = html_code + highlight(tmp_code, lexer, formatter)
return html_code
2015-02-15 18:09:17 +00:00
# Extract domain from url
@app.template_filter('extract_domain')
def extract_domain(url):
return urlparse(url)[1]
2014-01-14 17:17:19 +00:00
def get_base_url():
return url_for('index', _external=True)
2014-01-14 17:17:19 +00:00
2014-04-24 23:46:40 +00:00
def get_current_theme_name(override=None):
"""Returns theme name.
Checks in this order:
1. override
2. cookies
3. settings"""
if override and (override in themes or override == '__common__'):
2014-04-24 23:46:40 +00:00
return override
theme_name = request.args.get('theme', request.preferences.get_value('theme'))
2014-04-24 23:46:40 +00:00
if theme_name not in themes:
theme_name = default_theme
return theme_name
2015-01-01 17:59:53 +00:00
def get_result_template(theme, template_name):
themed_path = theme + '/result_templates/' + template_name
if themed_path in result_templates:
return themed_path
return 'result_templates/' + template_name
2014-04-24 23:46:40 +00:00
def url_for_theme(endpoint, override_theme=None, **values):
2015-01-01 16:48:12 +00:00
if endpoint == 'static' and values.get('filename'):
2014-04-24 23:46:40 +00:00
theme_name = get_current_theme_name(override=override_theme)
2015-01-01 16:48:12 +00:00
filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
if filename_with_theme in static_files:
values['filename'] = filename_with_theme
url = url_for(endpoint, **values)
return url
2014-04-24 23:46:40 +00:00
2016-10-16 22:22:41 +00:00
def proxify(url):
if url.startswith('//'):
url = 'https:' + url
if not settings.get('result_proxy'):
return url
url_params = dict(mortyurl=url.encode())
if settings['result_proxy'].get('key'):
url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'],
url.encode(),
hashlib.sha256).hexdigest()
2016-10-16 22:22:41 +00:00
return '{0}?{1}'.format(settings['result_proxy']['url'],
urlencode(url_params))
2016-10-16 22:22:41 +00:00
2015-01-16 15:26:15 +00:00
def image_proxify(url):
if url.startswith('//'):
url = 'https:' + url
2016-04-09 16:32:07 +00:00
if not request.preferences.get_value('image_proxy'):
2015-01-16 15:26:15 +00:00
return url
if url.startswith('data:image/'):
# 50 is an arbitrary number to get only the beginning of the image.
partial_base64 = url[len('data:image/'):50].split(';')
if len(partial_base64) == 2 \
and partial_base64[0] in ['gif', 'png', 'jpeg', 'pjpeg', 'webp', 'tiff', 'bmp']\
and partial_base64[1].startswith('base64,'):
return url
else:
return None
2019-01-18 08:04:40 +00:00
if settings.get('result_proxy'):
return proxify(url)
h = new_hmac(settings['server']['secret_key'], url.encode())
2015-01-16 15:26:15 +00:00
return '{0}?{1}'.format(url_for('image_proxy'),
urlencode(dict(url=url.encode(), h=h)))
2015-01-16 15:26:15 +00:00
def get_translations():
return {
# when overpass AJAX request fails (on a map result)
'could_not_load': gettext('could not load data'),
# when there is autocompletion
'no_item_found': gettext('No item found')
}
2014-04-24 23:46:40 +00:00
def render(template_name, override_theme=None, **kwargs):
disabled_engines = request.preferences.engines.get_disabled()
2014-03-29 15:29:19 +00:00
enabled_categories = set(category for engine_name in engines
for category in engines[engine_name].categories
if (engine_name, category) not in disabled_engines)
2014-03-29 15:29:19 +00:00
2014-10-19 10:41:04 +00:00
if 'categories' not in kwargs:
kwargs['categories'] = [x for x in
_get_ordered_categories()
if x in enabled_categories]
2014-03-29 15:29:19 +00:00
2014-10-19 10:41:04 +00:00
if 'autocomplete' not in kwargs:
2016-04-09 16:32:07 +00:00
kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
2014-03-29 15:29:19 +00:00
locale = request.preferences.get_value('locale')
if locale in rtl_locales and 'rtl' not in kwargs:
kwargs['rtl'] = True
2014-11-18 10:37:42 +00:00
kwargs['searx_version'] = VERSION_STRING
kwargs['method'] = request.preferences.get_value('method')
kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
kwargs['language_codes'] = languages
if 'current_language' not in kwargs:
kwargs['current_language'] = match_language(request.preferences.get_value('language'),
2020-02-23 09:03:42 +00:00
LANGUAGE_CODES)
2014-04-24 23:46:40 +00:00
# override url_for function in templates
kwargs['url_for'] = url_for_theme
2015-01-16 15:26:15 +00:00
kwargs['image_proxify'] = image_proxify
kwargs['proxify'] = proxify if settings.get('result_proxy', {}).get('url') else None
2016-10-16 22:22:41 +00:00
kwargs['opensearch_url'] = url_for('opensearch') + '?' \
+ urlencode({'method': kwargs['method'], 'autocomplete': kwargs['autocomplete']})
2015-01-01 17:59:53 +00:00
kwargs['get_result_template'] = get_result_template
2014-04-24 23:46:40 +00:00
kwargs['theme'] = get_current_theme_name(override=override_theme)
2014-11-18 18:55:39 +00:00
kwargs['template_name'] = template_name
2014-04-24 23:46:40 +00:00
2015-01-20 15:29:54 +00:00
kwargs['cookies'] = request.cookies
kwargs['errors'] = request.errors
kwargs['instance_name'] = settings['general']['instance_name']
kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')
kwargs['preferences'] = request.preferences
kwargs['brand'] = brand
kwargs['translations'] = json.dumps(get_translations(), separators=(',', ':'))
kwargs['scripts'] = set()
kwargs['endpoint'] = 'results' if 'q' in kwargs else request.endpoint
for plugin in request.user_plugins:
for script in plugin.js_dependencies:
kwargs['scripts'].add(script)
kwargs['styles'] = set()
for plugin in request.user_plugins:
for css in plugin.css_dependencies:
kwargs['styles'].add(css)
2014-04-24 23:46:40 +00:00
return render_template(
'{}/{}'.format(kwargs['theme'], template_name), **kwargs)
2013-10-15 18:50:12 +00:00
def _get_ordered_categories():
ordered_categories = []
if 'categories_order' not in settings['ui']:
ordered_categories = ['general']
ordered_categories.extend(x for x in sorted(categories.keys()) if x != 'general')
return ordered_categories
ordered_categories = settings['ui']['categories_order']
ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
return ordered_categories
2015-03-10 19:44:02 +00:00
@app.before_request
def pre_request():
request.start_time = time()
request.timings = []
request.errors = []
2016-11-30 17:43:03 +00:00
preferences = Preferences(themes, list(categories.keys()), engines, plugins)
user_agent = request.headers.get('User-Agent', '').lower()
if 'webkit' in user_agent and 'android' in user_agent:
preferences.key_value_settings['method'].value = 'GET'
request.preferences = preferences
try:
preferences.parse_dict(request.cookies)
except:
request.errors.append(gettext('Invalid settings, please edit your preferences'))
# merge GET, POST vars
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
# request.form
2015-03-10 21:45:59 +00:00
request.form = dict(request.form.items())
2015-03-15 11:13:24 +00:00
for k, v in request.args.items():
2015-03-10 21:45:59 +00:00
if k not in request.form:
request.form[k] = v
if request.form.get('preferences'):
preferences.parse_encoded_data(request.form['preferences'])
else:
try:
preferences.parse_dict(request.form)
except Exception:
logger.exception('invalid settings')
request.errors.append(gettext('Invalid settings'))
2015-03-10 19:44:02 +00:00
# init search language and locale
if not preferences.get_value("language"):
preferences.parse_dict({"language": _get_browser_or_settings_language(request, LANGUAGE_CODES)})
if not preferences.get_value("locale"):
2020-02-23 09:03:42 +00:00
preferences.parse_dict({"locale": get_locale()})
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
# request.user_plugins
2015-03-10 19:44:02 +00:00
request.user_plugins = []
allowed_plugins = preferences.plugins.get_enabled()
disabled_plugins = preferences.plugins.get_disabled()
2015-03-10 19:44:02 +00:00
for plugin in plugins:
if ((plugin.default_on and plugin.id not in disabled_plugins)
or plugin.id in allowed_plugins):
2015-03-10 19:44:02 +00:00
request.user_plugins.append(plugin)
@app.after_request
def add_default_headers(response):
# set default http headers
for header, value in settings['server'].get('default_http_headers', {}).items():
if header in response.headers:
continue
response.headers[header] = value
return response
@app.after_request
def post_request(response):
total_time = time() - request.start_time
timings_all = ['total;dur=' + str(round(total_time * 1000, 3))]
if len(request.timings) > 0:
timings = sorted(request.timings, key=lambda v: v['total'])
timings_total = ['total_' + str(i) + '_' + v['engine'] +
';dur=' + str(round(v['total'] * 1000, 3)) for i, v in enumerate(timings)]
timings_load = ['load_' + str(i) + '_' + v['engine'] +
';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings)]
timings_all = timings_all + timings_total + timings_load
response.headers.add('Server-Timing', ', '.join(timings_all))
return response
def index_error(output_format, error_message):
if output_format == 'json':
return Response(json.dumps({'error': error_message}),
mimetype='application/json')
elif output_format == 'csv':
response = Response('', mimetype='application/csv')
cont_disp = 'attachment;Filename=searx.csv'
response.headers.add('Content-Disposition', cont_disp)
return response
elif output_format == 'rss':
response_rss = render(
'opensearch_response_rss.xml',
results=[],
q=request.form['q'] if 'q' in request.form else '',
number_of_results=0,
base_url=get_base_url(),
error_message=error_message,
override_theme='__common__',
)
return Response(response_rss, mimetype='text/xml')
else:
# html
request.errors.append(gettext('search error'))
return render(
'index.html',
2020-09-24 14:26:00 +00:00
selected_categories=get_selected_categories(request.preferences, request.form),
)
2014-01-14 17:19:21 +00:00
@app.route('/', methods=['GET', 'POST'])
2013-10-14 21:09:13 +00:00
def index():
"""Render index page."""
# UI
2020-11-23 18:13:29 +00:00
advanced_search = request.preferences.get_value('advanced_search')
# redirect to search if there's a query in the request
if request.form.get('q'):
query = ('?' + request.query_string.decode()) if request.query_string else ''
return redirect(url_for('search') + query, 308)
return render(
'index.html',
selected_categories=get_selected_categories(request.preferences, request.form),
advanced_search=advanced_search,
)
@app.route('/search', methods=['GET', 'POST'])
def search():
"""Search query in q and return results.
2014-01-31 06:08:24 +00:00
Supported outputs: html, json, csv, rss.
"""
2013-11-03 23:18:07 +00:00
# output_format
output_format = request.form.get('format', 'html')
if output_format not in ['html', 'csv', 'json', 'rss']:
output_format = 'html'
# check if there is query (not None and not an empty string)
if not request.form.get('q'):
if output_format == 'html':
return render(
'index.html',
2020-11-26 15:27:46 +00:00
advanced_search=request.preferences.get_value('advanced_search'),
2020-09-24 14:26:00 +00:00
selected_categories=get_selected_categories(request.preferences, request.form),
)
else:
return index_error(output_format, 'No query'), 400
2014-02-07 02:15:34 +00:00
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
# search
search_query = None
raw_text_query = None
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
result_container = None
try:
search_query, raw_text_query, _, _ = get_search_query_from_webapp(request.preferences, request.form)
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
# search = Search(search_query) # without plugins
search = SearchWithPlugins(search_query, request.user_plugins, request)
Created new plugin type custom_results. Added new plugin bang_redirect (#2027) * Made first attempt at the bangs redirects plugin. * It redirects. But in a messy way via javascript. * First version with custom plugin * Added a help page and a operator to see all the bangs available. * Changed to .format because of support * Changed to .format because of support * Removed : in params * Fixed path to json file and changed bang operator * Changed bang operator back to & * Made first attempt at the bangs redirects plugin. * It redirects. But in a messy way via javascript. * First version with custom plugin * Added a help page and a operator to see all the bangs available. * Changed to .format because of support * Changed to .format because of support * Removed : in params * Fixed path to json file and changed bang operator * Changed bang operator back to & * Refactored getting search query. Also changed bang operator to ! and is now working. * Removed prints * Removed temporary bangs_redirect.js file. Updated plugin documentation * Added unit test for the bangs plugin * Fixed a unit test and added 2 more for bangs plugin * Changed back to default settings.yml * Added myself to AUTHORS.rst * Refacored working of custom plugin. * Refactored _get_bangs_data from list to dict to improve search speed. * Decoupled bangs plugin from webserver with redirect_url * Refactored bangs unit tests * Fixed unit test bangs. Removed dubbel parsing in bangs.py * Removed a dumb print statement * Refactored bangs plugin to core engine. * Removed bangs plugin. * Refactored external bangs unit tests from plugin to core. * Removed custom_results/bangs documentation from plugins.rst * Added newline in settings.yml so the PR stays clean. * Changed searx/plugins/__init__.py back to the old file * Removed newline search.py * Refactored get_external_bang_operator from utils to external_bang.py * Removed unnecessary import form test_plugins.py * Removed _parseExternalBang and _isExternalBang from query.py * Removed get_external_bang_operator since it was not necessary * Simplified external_bang.py * Simplified external_bang.py * Moved external_bangs unit tests to test_webapp.py. Fixed return in search with external_bang * Refactored query parsing to unicode to support python2 * Refactored query parsing to unicode to support python2 * Refactored bangs plugin to core engine. * Refactored search parameter to search_query in external_bang.py
2020-07-03 13:25:04 +00:00
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
result_container = search.search()
Created new plugin type custom_results. Added new plugin bang_redirect (#2027) * Made first attempt at the bangs redirects plugin. * It redirects. But in a messy way via javascript. * First version with custom plugin * Added a help page and a operator to see all the bangs available. * Changed to .format because of support * Changed to .format because of support * Removed : in params * Fixed path to json file and changed bang operator * Changed bang operator back to & * Made first attempt at the bangs redirects plugin. * It redirects. But in a messy way via javascript. * First version with custom plugin * Added a help page and a operator to see all the bangs available. * Changed to .format because of support * Changed to .format because of support * Removed : in params * Fixed path to json file and changed bang operator * Changed bang operator back to & * Refactored getting search query. Also changed bang operator to ! and is now working. * Removed prints * Removed temporary bangs_redirect.js file. Updated plugin documentation * Added unit test for the bangs plugin * Fixed a unit test and added 2 more for bangs plugin * Changed back to default settings.yml * Added myself to AUTHORS.rst * Refacored working of custom plugin. * Refactored _get_bangs_data from list to dict to improve search speed. * Decoupled bangs plugin from webserver with redirect_url * Refactored bangs unit tests * Fixed unit test bangs. Removed dubbel parsing in bangs.py * Removed a dumb print statement * Refactored bangs plugin to core engine. * Removed bangs plugin. * Refactored external bangs unit tests from plugin to core. * Removed custom_results/bangs documentation from plugins.rst * Added newline in settings.yml so the PR stays clean. * Changed searx/plugins/__init__.py back to the old file * Removed newline search.py * Refactored get_external_bang_operator from utils to external_bang.py * Removed unnecessary import form test_plugins.py * Removed _parseExternalBang and _isExternalBang from query.py * Removed get_external_bang_operator since it was not necessary * Simplified external_bang.py * Simplified external_bang.py * Moved external_bangs unit tests to test_webapp.py. Fixed return in search with external_bang * Refactored query parsing to unicode to support python2 * Refactored query parsing to unicode to support python2 * Refactored bangs plugin to core engine. * Refactored search parameter to search_query in external_bang.py
2020-07-03 13:25:04 +00:00
except SearxParameterException as e:
logger.exception('search error: SearxParameterException')
return index_error(output_format, e.message), 400
except Exception as e:
2016-11-14 14:49:06 +00:00
logger.exception('search error')
return index_error(output_format, gettext('search error')), 500
# results
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
results = result_container.get_ordered_results()
number_of_results = result_container.results_number()
if number_of_results < result_container.results_length():
number_of_results = 0
Created new plugin type custom_results. Added new plugin bang_redirect (#2027) * Made first attempt at the bangs redirects plugin. * It redirects. But in a messy way via javascript. * First version with custom plugin * Added a help page and a operator to see all the bangs available. * Changed to .format because of support * Changed to .format because of support * Removed : in params * Fixed path to json file and changed bang operator * Changed bang operator back to & * Made first attempt at the bangs redirects plugin. * It redirects. But in a messy way via javascript. * First version with custom plugin * Added a help page and a operator to see all the bangs available. * Changed to .format because of support * Changed to .format because of support * Removed : in params * Fixed path to json file and changed bang operator * Changed bang operator back to & * Refactored getting search query. Also changed bang operator to ! and is now working. * Removed prints * Removed temporary bangs_redirect.js file. Updated plugin documentation * Added unit test for the bangs plugin * Fixed a unit test and added 2 more for bangs plugin * Changed back to default settings.yml * Added myself to AUTHORS.rst * Refacored working of custom plugin. * Refactored _get_bangs_data from list to dict to improve search speed. * Decoupled bangs plugin from webserver with redirect_url * Refactored bangs unit tests * Fixed unit test bangs. Removed dubbel parsing in bangs.py * Removed a dumb print statement * Refactored bangs plugin to core engine. * Removed bangs plugin. * Refactored external bangs unit tests from plugin to core. * Removed custom_results/bangs documentation from plugins.rst * Added newline in settings.yml so the PR stays clean. * Changed searx/plugins/__init__.py back to the old file * Removed newline search.py * Refactored get_external_bang_operator from utils to external_bang.py * Removed unnecessary import form test_plugins.py * Removed _parseExternalBang and _isExternalBang from query.py * Removed get_external_bang_operator since it was not necessary * Simplified external_bang.py * Simplified external_bang.py * Moved external_bangs unit tests to test_webapp.py. Fixed return in search with external_bang * Refactored query parsing to unicode to support python2 * Refactored query parsing to unicode to support python2 * Refactored bangs plugin to core engine. * Refactored search parameter to search_query in external_bang.py
2020-07-03 13:25:04 +00:00
# checkin for a external bang
if result_container.redirect_url:
return redirect(result_container.redirect_url)
# Server-Timing header
request.timings = result_container.get_timings()
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
# output
for result in results:
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
if output_format == 'html':
if 'content' in result and result['content']:
2016-11-30 17:43:03 +00:00
result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query)
if 'title' in result and result['title']:
result['title'] = highlight_content(escape(result['title'] or ''), search_query.query)
2014-01-10 22:38:08 +00:00
else:
if result.get('content'):
2014-01-10 22:38:08 +00:00
result['content'] = html_to_text(result['content']).strip()
# removing html content and whitespace duplications
result['title'] = ' '.join(html_to_text(result['title']).strip().split())
2014-06-24 14:30:04 +00:00
if 'url' in result:
result['pretty_url'] = prettify_url(result['url'])
2013-11-15 18:28:30 +00:00
2014-03-14 08:55:04 +00:00
# TODO, check if timezone is calculated right
if result.get('publishedDate'): # do not try to get a date from an empty string or a None type
try: # test if publishedDate >= 1900 (datetime module bug)
result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
except ValueError:
result['publishedDate'] = None
2014-03-14 08:55:04 +00:00
else:
if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):
timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)
minutes = int((timedifference.seconds / 60) % 60)
hours = int(timedifference.seconds / 60 / 60)
if hours == 0:
result['publishedDate'] = gettext('{minutes} minute(s) ago').format(minutes=minutes)
else:
result['publishedDate'] = gettext('{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes) # noqa
else:
result['publishedDate'] = format_date(result['publishedDate'])
2014-03-14 08:55:04 +00:00
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
if output_format == 'json':
return Response(json.dumps({'query': search_query.query,
'number_of_results': number_of_results,
'results': results,
'answers': list(result_container.answers),
2017-01-14 08:40:37 +00:00
'corrections': list(result_container.corrections),
'infoboxes': result_container.infoboxes,
'suggestions': list(result_container.suggestions),
'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)}, # noqa
default=lambda item: list(item) if isinstance(item, set) else item),
2014-01-20 01:31:20 +00:00
mimetype='application/json')
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
elif output_format == 'csv':
2016-11-30 17:43:03 +00:00
csv = UnicodeWriter(StringIO())
keys = ('title', 'url', 'content', 'host', 'engine', 'score', 'type')
csv.writerow(keys)
for row in results:
row['host'] = row['parsed_url'].netloc
row['type'] = 'result'
csv.writerow([row.get(key, '') for key in keys])
for a in result_container.answers:
row = {'title': a, 'type': 'answer'}
csv.writerow([row.get(key, '') for key in keys])
for a in result_container.suggestions:
row = {'title': a, 'type': 'suggestion'}
csv.writerow([row.get(key, '') for key in keys])
for a in result_container.corrections:
row = {'title': a, 'type': 'correction'}
csv.writerow([row.get(key, '') for key in keys])
csv.stream.seek(0)
2013-11-15 18:28:30 +00:00
response = Response(csv.stream.read(), mimetype='application/csv')
cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)
2014-02-07 01:45:12 +00:00
response.headers.add('Content-Disposition', cont_disp)
2013-11-15 17:55:18 +00:00
return response
Created new plugin type custom_results. Added new plugin bang_redirect (#2027) * Made first attempt at the bangs redirects plugin. * It redirects. But in a messy way via javascript. * First version with custom plugin * Added a help page and a operator to see all the bangs available. * Changed to .format because of support * Changed to .format because of support * Removed : in params * Fixed path to json file and changed bang operator * Changed bang operator back to & * Made first attempt at the bangs redirects plugin. * It redirects. But in a messy way via javascript. * First version with custom plugin * Added a help page and a operator to see all the bangs available. * Changed to .format because of support * Changed to .format because of support * Removed : in params * Fixed path to json file and changed bang operator * Changed bang operator back to & * Refactored getting search query. Also changed bang operator to ! and is now working. * Removed prints * Removed temporary bangs_redirect.js file. Updated plugin documentation * Added unit test for the bangs plugin * Fixed a unit test and added 2 more for bangs plugin * Changed back to default settings.yml * Added myself to AUTHORS.rst * Refacored working of custom plugin. * Refactored _get_bangs_data from list to dict to improve search speed. * Decoupled bangs plugin from webserver with redirect_url * Refactored bangs unit tests * Fixed unit test bangs. Removed dubbel parsing in bangs.py * Removed a dumb print statement * Refactored bangs plugin to core engine. * Removed bangs plugin. * Refactored external bangs unit tests from plugin to core. * Removed custom_results/bangs documentation from plugins.rst * Added newline in settings.yml so the PR stays clean. * Changed searx/plugins/__init__.py back to the old file * Removed newline search.py * Refactored get_external_bang_operator from utils to external_bang.py * Removed unnecessary import form test_plugins.py * Removed _parseExternalBang and _isExternalBang from query.py * Removed get_external_bang_operator since it was not necessary * Simplified external_bang.py * Simplified external_bang.py * Moved external_bangs unit tests to test_webapp.py. Fixed return in search with external_bang * Refactored query parsing to unicode to support python2 * Refactored query parsing to unicode to support python2 * Refactored bangs plugin to core engine. * Refactored search parameter to search_query in external_bang.py
2020-07-03 13:25:04 +00:00
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
elif output_format == 'rss':
response_rss = render(
'opensearch_response_rss.xml',
results=results,
answers=result_container.answers,
corrections=result_container.corrections,
suggestions=result_container.suggestions,
q=request.form['q'],
number_of_results=number_of_results,
base_url=get_base_url(),
override_theme='__common__',
)
2014-01-14 21:18:21 +00:00
return Response(response_rss, mimetype='text/xml')
2014-01-14 17:17:19 +00:00
# HTML output format
# suggestions: use RawTextQuery to get the suggestion URLs with the same bang
suggestion_urls = list(map(lambda suggestion: {
'url': raw_text_query.changeQuery(suggestion).getFullQuery(),
'title': suggestion
},
result_container.suggestions))
2019-10-23 04:38:21 +00:00
correction_urls = list(map(lambda correction: {
'url': raw_text_query.changeQuery(correction).getFullQuery(),
2019-10-23 04:38:21 +00:00
'title': correction
},
result_container.corrections))
#
return render(
'results.html',
results=results,
q=request.form['q'],
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
selected_categories=search_query.categories,
pageno=search_query.pageno,
time_range=search_query.time_range,
number_of_results=format_decimal(number_of_results),
suggestions=suggestion_urls,
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
answers=result_container.answers,
2019-10-23 04:38:21 +00:00
corrections=correction_urls,
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
infoboxes=result_container.infoboxes,
engine_data=result_container.engine_data,
Clean up the architecture Purposes : - isolate the plugins calls - distinction between parsing the web request and running the search (Search class). To be able to test code easily, to run searx code outside a web server, to filter the search query parameters with plugins more easily, etc... Details : - request.request_data contains request.form or request.args (initialize inside pre_request() function) - Query class is renamed RawTextQuery - SearchQuery class defines all search parameters - get_search_query_from_webapp create a SearchQuery instance (basically the previous Search.__init__ code) - Search class and SearchWithPlugins class takes a SearchQuery instance as class constructor parameter - SearchWithPlugins class inherites from Search class, and run plugins - A dedicated function search_with_plugins executes plugins to have a well define locals() (which is used by the plugins code). - All plugins code is executed inside the try...except block (webapp.py, index function) - advanced_search HTTP parameter value stays in webapp.py (it is only part of UI) - multiple calls to result_container.get_ordered_results() doesn't compute the order multiple time (note : this method was call only once before) - paging value is stored in the result_container class (compute in the extend method) - test about engine.suspend_end_time is done during search method call (instead of __init__) - check that the format parameter value is one of these : html, rss, json, rss (before the html value was assumed but some text formatting wasn't not done)
2016-10-22 11:10:31 +00:00
paging=result_container.paging,
unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),
current_language=match_language(search_query.lang,
LANGUAGE_CODES,
fallback=request.preferences.get_value("language")),
2014-03-04 17:53:56 +00:00
base_url=get_base_url(),
theme=get_current_theme_name(),
favicons=global_favicons[themes.index(get_current_theme_name())],
timeout_limit=request.form.get('timeout_limit', None)
)
2014-01-01 21:16:53 +00:00
2013-10-14 21:09:13 +00:00
def __get_translated_errors(unresponsive_engines):
translated_errors = set()
for unresponsive_engine in unresponsive_engines:
error_msg = gettext(unresponsive_engine[1])
if unresponsive_engine[2]:
error_msg = "{} {}".format(error_msg, unresponsive_engine[2])
translated_errors.add((unresponsive_engine[0], error_msg))
return translated_errors
2013-10-20 22:28:48 +00:00
@app.route('/about', methods=['GET'])
def about():
2014-01-31 06:08:24 +00:00
"""Render about page"""
2014-03-21 11:19:48 +00:00
return render(
'about.html',
)
2014-01-17 15:23:23 +00:00
@app.route('/autocompleter', methods=['GET', 'POST'])
def autocompleter():
"""Return autocompleter results"""
2014-03-29 15:29:19 +00:00
# run autocompleter
results = []
# set blocked engines
disabled_engines = request.preferences.engines.get_disabled()
# parse query
raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
2014-03-29 15:29:19 +00:00
# normal autocompletion results only appear if no inner results returned
# and there is a query part
if len(raw_text_query.autocomplete_list) == 0 and len(raw_text_query.getQuery()) > 0:
# get language from cookie
2016-04-09 16:32:07 +00:00
language = request.preferences.get_value('language')
if not language or language == 'all':
language = 'en'
else:
language = language.split('-')[0]
2015-01-10 18:55:21 +00:00
# run autocompletion
raw_results = search_autocomplete(request.preferences.get_value('autocomplete'),
raw_text_query.getQuery(), language)
for result in raw_results:
results.append(raw_text_query.changeQuery(result).getFullQuery())
if len(raw_text_query.autocomplete_list) > 0:
for autocomplete_text in raw_text_query.autocomplete_list:
results.append(raw_text_query.get_autocomplete_full_query(autocomplete_text))
for answers in ask(raw_text_query):
for answer in answers:
results.append(str(answer['answer']))
2014-03-29 15:29:19 +00:00
# return autocompleter results
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return Response(json.dumps(results),
2014-03-29 15:29:19 +00:00
mimetype='application/json')
return Response(json.dumps([raw_text_query.query, results]),
mimetype='application/x-suggestions+json')
2014-01-01 21:16:53 +00:00
@app.route('/preferences', methods=['GET', 'POST'])
def preferences():
"""Render preferences page && save user preferences"""
2014-02-06 23:35:15 +00:00
# save preferences
if request.method == 'POST':
resp = make_response(url_for('index', _external=True))
try:
request.preferences.parse_form(request.form)
except ValidationException:
request.errors.append(gettext('Invalid settings, please edit your preferences'))
return resp
return request.preferences.save(resp)
# render preferences
image_proxy = request.preferences.get_value('image_proxy')
disabled_engines = request.preferences.engines.get_disabled()
allowed_plugins = request.preferences.plugins.get_enabled()
# stats for preferences page
stats = {}
engines_by_category = {}
for c in categories:
engines_by_category[c] = []
for e in categories[c]:
if not request.preferences.validate_token(e):
continue
stats[e.name] = {'time': None,
'warn_timeout': False,
'warn_time': False}
if e.timeout > settings['outgoing']['request_timeout']:
stats[e.name]['warn_timeout'] = True
stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)
engines_by_category[c].append(e)
# get first element [0], the engine time,
# and then the second element [1] : the time (the first one is the label)
for engine_stat in get_engines_stats(request.preferences)[0][1]:
stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
stats[engine_stat.get('name')]['warn_time'] = True
# end of stats
2020-10-23 14:22:55 +00:00
locked_preferences = list()
if 'preferences' in settings and 'lock' in settings['preferences']:
locked_preferences = settings['preferences']['lock']
2014-01-22 00:20:38 +00:00
return render('preferences.html',
2020-09-24 14:26:00 +00:00
selected_categories=get_selected_categories(request.preferences, request.form),
all_categories=_get_ordered_categories(),
2014-01-22 00:20:38 +00:00
locales=settings['locales'],
current_locale=request.preferences.get_value("locale"),
2015-01-16 16:37:34 +00:00
image_proxy=image_proxy,
engines_by_category=engines_by_category,
stats=stats,
answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
disabled_engines=disabled_engines,
2014-03-29 15:29:19 +00:00
autocomplete_backends=autocomplete_backends,
2014-04-24 23:46:40 +00:00
shortcuts={y: x for x, y in engine_shortcuts.items()},
themes=themes,
plugins=plugins,
doi_resolvers=settings['doi_resolvers'],
current_doi_resolver=get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')),
allowed_plugins=allowed_plugins,
theme=get_current_theme_name(),
preferences_url_params=request.preferences.get_as_url_params(),
base_url=get_base_url(),
2020-10-23 14:22:55 +00:00
locked_preferences=locked_preferences,
preferences=True)
2014-01-01 21:16:53 +00:00
def _is_selected_language_supported(engine, preferences):
language = preferences.get_value('language')
return (language == 'all'
or match_language(language,
getattr(engine, 'supported_languages', []),
getattr(engine, 'language_aliases', {}), None))
2015-01-16 15:02:21 +00:00
@app.route('/image_proxy', methods=['GET'])
def image_proxy():
url = request.args.get('url').encode()
2015-01-16 15:02:21 +00:00
if not url:
return '', 400
2017-07-20 13:44:02 +00:00
h = new_hmac(settings['server']['secret_key'], url)
if h != request.args.get('h'):
return '', 400
headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})
headers['User-Agent'] = gen_useragent()
resp = requests.get(url,
stream=True,
timeout=settings['outgoing']['request_timeout'],
headers=headers,
proxies=get_global_proxies())
if resp.status_code == 304:
return '', resp.status_code
2015-01-16 15:02:21 +00:00
if resp.status_code != 200:
logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))
if resp.status_code >= 400:
return '', resp.status_code
return '', 400
if not resp.headers.get('content-type', '').startswith('image/'):
2015-06-21 14:50:42 +00:00
logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
2015-01-16 15:02:21 +00:00
return '', 400
2017-07-20 13:44:02 +00:00
img = b''
2015-01-16 15:02:21 +00:00
chunk_counter = 0
for chunk in resp.iter_content(1024 * 1024):
2015-01-16 15:02:21 +00:00
chunk_counter += 1
if chunk_counter > 5:
return '', 502 # Bad gateway - file is too big (>5M)
img += chunk
headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})
return Response(img, mimetype=resp.headers['content-type'], headers=headers)
2015-01-16 15:02:21 +00:00
2013-10-26 23:03:05 +00:00
@app.route('/stats', methods=['GET'])
def stats():
2014-01-31 06:08:24 +00:00
"""Render engine statistics page."""
stats = get_engines_stats(request.preferences)
2014-03-21 11:19:48 +00:00
return render(
'stats.html',
stats=stats,
)
2013-10-26 23:03:05 +00:00
2014-01-01 21:16:53 +00:00
@app.route('/stats/errors', methods=['GET'])
def stats_errors():
result = {}
engine_names = list(errors_per_engines.keys())
engine_names.sort()
for engine_name in engine_names:
error_stats = errors_per_engines[engine_name]
sent_search_count = max(engines[engine_name].stats['sent_search_count'], 1)
sorted_context_count_list = sorted(error_stats.items(), key=lambda context_count: context_count[1])
r = []
percentage_sum = 0
for context, count in sorted_context_count_list:
percentage = round(20 * count / sent_search_count) * 5
percentage_sum += percentage
r.append({
'filename': context.filename,
'function': context.function,
'line_no': context.line_no,
'code': context.code,
'exception_classname': context.exception_classname,
'log_message': context.log_message,
'log_parameters': context.log_parameters,
'percentage': percentage,
})
result[engine_name] = sorted(r, reverse=True, key=lambda d: d['percentage'])
return jsonify(result)
@app.route('/stats/checker', methods=['GET'])
def stats_checker():
result = checker_get_result()
return jsonify(result)
2013-12-01 15:10:38 +00:00
@app.route('/robots.txt', methods=['GET'])
def robots():
return Response("""User-agent: *
Allow: /
Allow: /about
Disallow: /stats
2014-02-07 17:43:05 +00:00
Disallow: /preferences
Disallow: /*?*q=*
2013-12-01 15:10:38 +00:00
""", mimetype='text/plain')
2014-01-01 21:16:53 +00:00
2013-10-15 22:01:08 +00:00
@app.route('/opensearch.xml', methods=['GET'])
def opensearch():
2013-10-20 20:37:55 +00:00
method = 'post'
if request.preferences.get_value('method') == 'GET':
method = 'get'
2013-10-20 22:28:48 +00:00
# chrome/chromium only supports HTTP GET....
2013-10-20 20:37:55 +00:00
if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
method = 'get'
2014-03-29 15:29:19 +00:00
ret = render(
'opensearch.xml',
opensearch_method=method,
override_theme='__common__'
)
2014-03-29 15:29:19 +00:00
2013-10-15 22:01:08 +00:00
resp = Response(response=ret,
2014-01-20 01:31:20 +00:00
status=200,
mimetype="application/opensearchdescription+xml")
2013-10-15 22:01:08 +00:00
return resp
2013-12-01 22:52:49 +00:00
@app.route('/favicon.ico')
def favicon():
2014-04-24 23:46:40 +00:00
return send_from_directory(os.path.join(app.root_path,
static_path,
'themes',
2014-04-24 23:46:40 +00:00
get_current_theme_name(),
'img'),
2014-01-20 01:31:20 +00:00
'favicon.png',
mimetype='image/vnd.microsoft.icon')
2013-12-01 22:52:49 +00:00
2015-04-07 09:07:48 +00:00
@app.route('/clear_cookies')
def clear_cookies():
resp = make_response(redirect(url_for('index', _external=True)))
2015-04-07 09:07:48 +00:00
for cookie_name in request.cookies:
resp.delete_cookie(cookie_name)
return resp
@app.route('/config')
def config():
"""Return configuration in JSON format."""
_engines = []
for name, engine in engines.items():
if not request.preferences.validate_token(engine):
continue
supported_languages = engine.supported_languages
if isinstance(engine.supported_languages, dict):
supported_languages = list(engine.supported_languages.keys())
_engines.append({
'name': name,
'categories': engine.categories,
'shortcut': engine.shortcut,
'enabled': not engine.disabled,
'paging': engine.paging,
'language_support': engine.language_support,
'supported_languages': supported_languages,
'safesearch': engine.safesearch,
'time_range_support': engine.time_range_support,
'timeout': engine.timeout
})
_plugins = []
for _ in plugins:
_plugins.append({'name': _.name, 'enabled': _.default_on})
return jsonify({
'categories': list(categories.keys()),
'engines': _engines,
'plugins': _plugins,
'instance_name': settings['general']['instance_name'],
'locales': settings['locales'],
'default_locale': settings['ui']['default_locale'],
'autocomplete': settings['search']['autocomplete'],
'safe_search': settings['search']['safe_search'],
'default_theme': settings['ui']['default_theme'],
'version': VERSION_STRING,
'brand': {
'CONTACT_URL': brand.CONTACT_URL,
'GIT_URL': brand.GIT_URL,
'DOCS_URL': brand.DOCS_URL
},
'doi_resolvers': [r for r in settings['doi_resolvers']],
'default_doi_resolver': settings['default_doi_resolver'],
})
2016-08-24 17:53:09 +00:00
@app.errorhandler(404)
def page_not_found(e):
return render('404.html'), 404
2016-08-24 17:53:09 +00:00
def run():
logger.debug('starting webserver on %s:%s', settings['server']['bind_address'], settings['server']['port'])
app.run(
debug=searx_debug,
use_debugger=searx_debug,
port=settings['server']['port'],
host=settings['server']['bind_address'],
threaded=True
)
2020-08-12 07:42:27 +00:00
class ReverseProxyPathFix:
2015-06-16 17:55:31 +00:00
'''Wrap the application in this middleware and configure the
front-end server to add these headers, to let you quietly bind
this to a URL other than / and to an HTTP scheme that is
different than what is used locally.
http://flask.pocoo.org/snippets/35/
In nginx:
location /myprefix {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /myprefix;
}
:param app: the WSGI application
'''
2015-06-16 17:55:31 +00:00
def __init__(self, app):
2015-06-16 17:55:31 +00:00
self.app = app
self.script_name = None
self.scheme = None
self.server = None
if settings['server']['base_url']:
# If base_url is specified, then these values from are given
# preference over any Flask's generics.
base_url = urlparse(settings['server']['base_url'])
self.script_name = base_url.path
self.scheme = base_url.scheme
self.server = base_url.netloc
2015-06-16 17:55:31 +00:00
def __call__(self, environ, start_response):
script_name = self.script_name or environ.get('HTTP_X_SCRIPT_NAME', '')
2015-06-16 17:55:31 +00:00
if script_name:
environ['SCRIPT_NAME'] = script_name
path_info = environ['PATH_INFO']
if path_info.startswith(script_name):
environ['PATH_INFO'] = path_info[len(script_name):]
scheme = self.scheme or environ.get('HTTP_X_SCHEME', '')
2015-06-16 17:55:31 +00:00
if scheme:
environ['wsgi.url_scheme'] = scheme
server = self.server or environ.get('HTTP_X_FORWARDED_HOST', '')
if server:
environ['HTTP_HOST'] = server
2015-06-16 17:55:31 +00:00
return self.app(environ, start_response)
2014-07-03 20:02:53 +00:00
2015-06-16 17:55:31 +00:00
application = app
# patch app to handle non root url-s behind proxy & wsgi
app.wsgi_app = ReverseProxyPathFix(ProxyFix(application.wsgi_app))
2014-07-03 20:02:53 +00:00
if __name__ == "__main__":
run()