Merge branch 'plugins'

This commit is contained in:
Adam Tauber 2015-03-15 11:38:07 +01:00
commit bd92b43449
10 changed files with 238 additions and 74 deletions

48
searx/plugins/__init__.py Normal file
View file

@ -0,0 +1,48 @@
from searx.plugins import self_ip
from searx import logger
from sys import exit
logger = logger.getChild('plugins')
required_attrs = (('name', str),
('description', str),
('default_on', bool))
class Plugin():
default_on = False
name = 'Default plugin'
description = 'Default plugin description'
class PluginStore():
def __init__(self):
self.plugins = []
def __iter__(self):
for plugin in self.plugins:
yield plugin
def register(self, *plugins):
for plugin in plugins:
for plugin_attr, plugin_attr_type in required_attrs:
if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))
exit(3)
plugin.id = plugin.name.replace(' ', '_')
self.plugins.append(plugin)
def call(self, plugin_type, request, *args, **kwargs):
ret = True
for plugin in request.user_plugins:
if hasattr(plugin, plugin_type):
ret = getattr(plugin, plugin_type)(request, *args, **kwargs)
if not ret:
break
return ret
plugins = PluginStore()
plugins.register(self_ip)

21
searx/plugins/self_ip.py Normal file
View file

@ -0,0 +1,21 @@
from flask.ext.babel import gettext
name = "Self IP"
description = gettext('Display your source IP address if the query expression is "ip"')
default_on = True
# attach callback to the pre search hook
# request: flask request object
# ctx: the whole local context of the pre search hook
def pre_search(request, ctx):
if ctx['search'].query == 'ip':
x_forwarded_for = request.headers.getlist("X-Forwarded-For")
if x_forwarded_for:
ip = x_forwarded_for[0]
else:
ip = request.remote_addr
ctx['search'].answers.clear()
ctx['search'].answers.add(ip)
# return False prevents exeecution of the original block
return False
return True

View file

@ -329,8 +329,8 @@ class Search(object):
self.blocked_engines = get_blocked_engines(engines, request.cookies) self.blocked_engines = get_blocked_engines(engines, request.cookies)
self.results = [] self.results = []
self.suggestions = [] self.suggestions = set()
self.answers = [] self.answers = set()
self.infoboxes = [] self.infoboxes = []
self.request_data = {} self.request_data = {}
@ -429,9 +429,6 @@ class Search(object):
requests = [] requests = []
results_queue = Queue() results_queue = Queue()
results = {} results = {}
suggestions = set()
answers = set()
infoboxes = []
# increase number of searches # increase number of searches
number_of_searches += 1 number_of_searches += 1
@ -511,7 +508,7 @@ class Search(object):
selected_engine['name'])) selected_engine['name']))
if not requests: if not requests:
return results, suggestions, answers, infoboxes return self
# send all search-request # send all search-request
threaded_requests(requests) threaded_requests(requests)
@ -519,19 +516,19 @@ class Search(object):
engine_name, engine_results = results_queue.get_nowait() engine_name, engine_results = results_queue.get_nowait()
# TODO type checks # TODO type checks
[suggestions.add(x['suggestion']) [self.suggestions.add(x['suggestion'])
for x in list(engine_results) for x in list(engine_results)
if 'suggestion' in x if 'suggestion' in x
and engine_results.remove(x) is None] and engine_results.remove(x) is None]
[answers.add(x['answer']) [self.answers.add(x['answer'])
for x in list(engine_results) for x in list(engine_results)
if 'answer' in x if 'answer' in x
and engine_results.remove(x) is None] and engine_results.remove(x) is None]
infoboxes.extend(x for x in list(engine_results) self.infoboxes.extend(x for x in list(engine_results)
if 'infobox' in x if 'infobox' in x
and engine_results.remove(x) is None) and engine_results.remove(x) is None)
results[engine_name] = engine_results results[engine_name] = engine_results
@ -541,16 +538,16 @@ class Search(object):
engines[engine_name].stats['result_count'] += len(engine_results) engines[engine_name].stats['result_count'] += len(engine_results)
# score results and remove duplications # score results and remove duplications
results = score_results(results) self.results = score_results(results)
# merge infoboxes according to their ids # merge infoboxes according to their ids
infoboxes = merge_infoboxes(infoboxes) self.infoboxes = merge_infoboxes(self.infoboxes)
# update engine stats, using calculated score # update engine stats, using calculated score
for result in results: for result in self.results:
for res_engine in result['engines']: for res_engine in result['engines']:
engines[result['engine']]\ engines[result['engine']]\
.stats['score_count'] += result['score'] .stats['score_count'] += result['score']
# return results, suggestions, answers and infoboxes # return results, suggestions, answers and infoboxes
return results, suggestions, answers, infoboxes return self

View file

@ -106,6 +106,7 @@ engines:
- name : gigablast - name : gigablast
engine : gigablast engine : gigablast
shortcut : gb shortcut : gb
disabled: True
- name : github - name : github
engine : github engine : github

View file

@ -59,3 +59,11 @@
</div> </div>
{% endif %} {% endif %}
{%- endmacro %} {%- endmacro %}
{% macro checkbox_toggle(id, blocked) -%}
<div class="checkbox">
<input class="hidden" type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} />
<label class="btn btn-success label_hide_if_checked" for="{{ id }}">{{ _('Block') }}</label>
<label class="btn btn-danger label_hide_if_not_checked" for="{{ id }}">{{ _('Allow') }}</label>
</div>
{%- endmacro %}

View file

@ -1,4 +1,4 @@
{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl %} {% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl, checkbox_toggle %}
{% extends "oscar/base.html" %} {% extends "oscar/base.html" %}
{% block title %}{{ _('preferences') }} - {% endblock %} {% block title %}{{ _('preferences') }} - {% endblock %}
{% block site_alert_warning_nojs %} {% block site_alert_warning_nojs %}
@ -16,6 +16,7 @@
<ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;"> <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;">
<li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li> <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>
<li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li> <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>
<li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>
</ul> </ul>
<!-- Tab panes --> <!-- Tab panes -->
@ -139,11 +140,7 @@
<div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div> <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div>
{% endif %} {% endif %}
<div class="col-xs-6 col-sm-4 col-md-4"> <div class="col-xs-6 col-sm-4 col-md-4">
<div class="checkbox"> {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }}
<input class="hidden" type="checkbox" id="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}" name="engine_{{ search_engine.name }}__{{ categ }}"{% if (search_engine.name, categ) in blocked_engines %} checked="checked"{% endif %} />
<label class="btn btn-success label_hide_if_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Block') }}</label>
<label class="btn btn-danger label_hide_if_not_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Allow') }}</label>
</div>
</div> </div>
{% if rtl %} {% if rtl %}
<div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})&lrm;</div> <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})&lrm;</div>
@ -157,6 +154,28 @@
{% endfor %} {% endfor %}
</div> </div>
</div> </div>
<div class="tab-pane active_if_nojs" id="tab_plugins">
<noscript>
<h3>{{ _('Plugins') }}</h3>
</noscript>
<fieldset>
<div class="container-fluid">
{% for plugin in plugins %}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">{{ plugin.name }}</h3>
</div>
<div class="panel-body">
<div class="col-xs-6 col-sm-4 col-md-6">{{ plugin.description }}</div>
<div class="col-xs-6 col-sm-4 col-md-6">
{{ checkbox_toggle('plugin_' + plugin.id, plugin.id not in allowed_plugins) }}
</div>
</div>
</div>
{% endfor %}
</div>
</fieldset>
</div>
</div> </div>
<p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }} <p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }}
<br /> <br />

View file

@ -26,7 +26,7 @@
</div> </div>
{% endfor %} {% endfor %}
{% if not results %} {% if not results and not answers %}
{% include 'oscar/messages/no_results.html' %} {% include 'oscar/messages/no_results.html' %}
{% endif %} {% endif %}

View file

@ -0,0 +1,51 @@
# -*- coding: utf-8 -*-
from searx.testing import SearxTestCase
from searx import plugins
from mock import Mock
class PluginStoreTest(SearxTestCase):
def test_PluginStore_init(self):
store = plugins.PluginStore()
self.assertTrue(isinstance(store.plugins, list) and len(store.plugins) == 0)
def test_PluginStore_register(self):
store = plugins.PluginStore()
testplugin = plugins.Plugin()
store.register(testplugin)
self.assertTrue(len(store.plugins) == 1)
def test_PluginStore_call(self):
store = plugins.PluginStore()
testplugin = plugins.Plugin()
store.register(testplugin)
setattr(testplugin, 'asdf', Mock())
request = Mock(user_plugins=[])
store.call('asdf', request, Mock())
self.assertFalse(testplugin.asdf.called)
request.user_plugins.append(testplugin)
store.call('asdf', request, Mock())
self.assertTrue(testplugin.asdf.called)
class SelfIPTest(SearxTestCase):
def test_PluginStore_init(self):
store = plugins.PluginStore()
store.register(plugins.self_ip)
self.assertTrue(len(store.plugins) == 1)
request = Mock(user_plugins=store.plugins,
remote_addr='127.0.0.1')
request.headers.getlist.return_value = []
ctx = {'search': Mock(answers=set(),
query='ip')}
store.call('pre_search', request, ctx)
self.assertTrue('127.0.0.1' in ctx['search'].answers)

View file

@ -2,7 +2,6 @@
import json import json
from urlparse import ParseResult from urlparse import ParseResult
from mock import patch
from searx import webapp from searx import webapp
from searx.testing import SearxTestCase from searx.testing import SearxTestCase
@ -33,6 +32,11 @@ class ViewsTestCase(SearxTestCase):
}, },
] ]
def search_mock(search_self, *args):
search_self.results = self.test_results
webapp.Search.search = search_mock
self.maxDiff = None # to see full diffs self.maxDiff = None # to see full diffs
def test_index_empty(self): def test_index_empty(self):
@ -40,14 +44,7 @@ class ViewsTestCase(SearxTestCase):
self.assertEqual(result.status_code, 200) self.assertEqual(result.status_code, 200)
self.assertIn('<div class="title"><h1>searx</h1></div>', result.data) self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)
@patch('searx.search.Search.search') def test_index_html(self):
def test_index_html(self, search):
search.return_value = (
self.test_results,
set(),
set(),
set()
)
result = self.app.post('/', data={'q': 'test'}) result = self.app.post('/', data={'q': 'test'})
self.assertIn( self.assertIn(
'<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa
@ -58,14 +55,7 @@ class ViewsTestCase(SearxTestCase):
result.data result.data
) )
@patch('searx.search.Search.search') def test_index_json(self):
def test_index_json(self, search):
search.return_value = (
self.test_results,
set(),
set(),
set()
)
result = self.app.post('/', data={'q': 'test', 'format': 'json'}) result = self.app.post('/', data={'q': 'test', 'format': 'json'})
result_dict = json.loads(result.data) result_dict = json.loads(result.data)
@ -76,14 +66,7 @@ class ViewsTestCase(SearxTestCase):
self.assertEqual( self.assertEqual(
result_dict['results'][0]['url'], 'http://first.test.xyz') result_dict['results'][0]['url'], 'http://first.test.xyz')
@patch('searx.search.Search.search') def test_index_csv(self):
def test_index_csv(self, search):
search.return_value = (
self.test_results,
set(),
set(),
set()
)
result = self.app.post('/', data={'q': 'test', 'format': 'csv'}) result = self.app.post('/', data={'q': 'test', 'format': 'csv'})
self.assertEqual( self.assertEqual(
@ -93,14 +76,7 @@ class ViewsTestCase(SearxTestCase):
result.data result.data
) )
@patch('searx.search.Search.search') def test_index_rss(self):
def test_index_rss(self, search):
search.return_value = (
self.test_results,
set(),
set(),
set()
)
result = self.app.post('/', data={'q': 'test', 'format': 'rss'}) result = self.app.post('/', data={'q': 'test', 'format': 'rss'})
self.assertIn( self.assertIn(

View file

@ -27,6 +27,18 @@ import cStringIO
import os import os
import hashlib import hashlib
from searx import logger
logger = logger.getChild('webapp')
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
from datetime import datetime, timedelta from datetime import datetime, timedelta
from urllib import urlencode from urllib import urlencode
from werkzeug.contrib.fixers import ProxyFix from werkzeug.contrib.fixers import ProxyFix
@ -51,19 +63,9 @@ from searx.https_rewrite import https_url_rewrite
from searx.search import Search from searx.search import Search
from searx.query import Query from searx.query import Query
from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx import logger from searx.plugins import plugins
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
logger = logger.getChild('webapp')
static_path, templates_path, themes =\ static_path, templates_path, themes =\
get_themes(settings['themes_path'] get_themes(settings['themes_path']
if settings.get('themes_path') if settings.get('themes_path')
@ -303,6 +305,23 @@ def render(template_name, override_theme=None, **kwargs):
'{}/{}'.format(kwargs['theme'], template_name), **kwargs) '{}/{}'.format(kwargs['theme'], template_name), **kwargs)
@app.before_request
def pre_request():
# merge GET, POST vars
request.form = dict(request.form.items())
for k, v in request.args:
if k not in request.form:
request.form[k] = v
request.user_plugins = []
allowed_plugins = request.cookies.get('allowed_plugins', '').split(',')
disabled_plugins = request.cookies.get('disabled_plugins', '').split(',')
for plugin in plugins:
if ((plugin.default_on and plugin.id not in disabled_plugins)
or plugin.id in allowed_plugins):
request.user_plugins.append(plugin)
@app.route('/search', methods=['GET', 'POST']) @app.route('/search', methods=['GET', 'POST'])
@app.route('/', methods=['GET', 'POST']) @app.route('/', methods=['GET', 'POST'])
def index(): def index():
@ -323,8 +342,10 @@ def index():
'index.html', 'index.html',
) )
search.results, search.suggestions,\ if plugins.call('pre_search', request, locals()):
search.answers, search.infoboxes = search.search(request) search.search(request)
plugins.call('post_search', request, locals())
for result in search.results: for result in search.results:
@ -487,11 +508,11 @@ def preferences():
blocked_engines = get_blocked_engines(engines, request.cookies) blocked_engines = get_blocked_engines(engines, request.cookies)
else: # on save else: # on save
selected_categories = [] selected_categories = []
post_disabled_plugins = []
locale = None locale = None
autocomplete = '' autocomplete = ''
method = 'POST' method = 'POST'
safesearch = '1' safesearch = '1'
for pd_name, pd in request.form.items(): for pd_name, pd in request.form.items():
if pd_name.startswith('category_'): if pd_name.startswith('category_'):
category = pd_name[9:] category = pd_name[9:]
@ -514,14 +535,34 @@ def preferences():
safesearch = pd safesearch = pd
elif pd_name.startswith('engine_'): elif pd_name.startswith('engine_'):
if pd_name.find('__') > -1: if pd_name.find('__') > -1:
engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1) # TODO fix underscore vs space
engine_name, category = [x.replace('_', ' ') for x in
pd_name.replace('engine_', '', 1).split('__', 1)]
if engine_name in engines and category in engines[engine_name].categories: if engine_name in engines and category in engines[engine_name].categories:
blocked_engines.append((engine_name, category)) blocked_engines.append((engine_name, category))
elif pd_name == 'theme': elif pd_name == 'theme':
theme = pd if pd in themes else default_theme theme = pd if pd in themes else default_theme
elif pd_name.startswith('plugin_'):
plugin_id = pd_name.replace('plugin_', '', 1)
if not any(plugin.id == plugin_id for plugin in plugins):
continue
post_disabled_plugins.append(plugin_id)
else: else:
resp.set_cookie(pd_name, pd, max_age=cookie_max_age) resp.set_cookie(pd_name, pd, max_age=cookie_max_age)
disabled_plugins = []
allowed_plugins = []
for plugin in plugins:
if plugin.default_on:
if plugin.id in post_disabled_plugins:
disabled_plugins.append(plugin.id)
elif plugin.id not in post_disabled_plugins:
allowed_plugins.append(plugin.id)
resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age)
resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age)
resp.set_cookie( resp.set_cookie(
'blocked_engines', ','.join('__'.join(e) for e in blocked_engines), 'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),
max_age=cookie_max_age max_age=cookie_max_age
@ -571,6 +612,8 @@ def preferences():
autocomplete_backends=autocomplete_backends, autocomplete_backends=autocomplete_backends,
shortcuts={y: x for x, y in engine_shortcuts.items()}, shortcuts={y: x for x, y in engine_shortcuts.items()},
themes=themes, themes=themes,
plugins=plugins,
allowed_plugins=[plugin.id for plugin in request.user_plugins],
theme=get_current_theme_name()) theme=get_current_theme_name())