From d5ec0f43e465e1bd016f1c841122fc3c774fc718 Mon Sep 17 00:00:00 2001 From: asciimoo Date: Tue, 15 Oct 2013 18:19:06 +0200 Subject: [PATCH] [mod] search refactor --- searx/engines/__init__.py | 39 ++++++++++++++++++++++++++++++++++++++- searx/webapp.py | 35 ++--------------------------------- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 34d88b749..ced673bde 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -2,6 +2,7 @@ from os.path import realpath, dirname, splitext, join from os import listdir from imp import load_source +import grequests engine_dir = dirname(realpath(__file__)) @@ -12,4 +13,40 @@ for filename in listdir(engine_dir): if filename.startswith('_') or not filename.endswith('.py'): continue filepath = join(engine_dir, filename) - engines.append(load_source(modname, filepath)) + engine = load_source(modname, filepath) + if not hasattr(engine, 'request') or not hasattr(engine, 'response'): + continue + engines.append(engine) + +def default_request_params(): + return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''} + +def make_callback(results, callback): + def process_callback(response, **kwargs): + results.extend(callback(response)) + return process_callback + +def search(query, request): + global engines + requests = [] + results = [] + user_agent = request.headers.get('User-Agent', '') + for engine in engines: + headers = default_request_params() + headers['User-Agent'] = user_agent + request_params = engine.request(query, headers) + callback = make_callback(results, engine.response) + if request_params['method'] == 'GET': + req = grequests.get(request_params['url'] + ,headers=headers + ,hooks=dict(response=callback) + ) + else: + req = grequests.post(request_params['url'] + ,data=request_params['data'] + ,headers=headers + ,hooks=dict(response=callback) + ) + requests.append(req) + grequests.map(requests) + return results diff --git a/searx/webapp.py b/searx/webapp.py index cfb30ba67..2dd9e28d7 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -25,8 +25,7 @@ if __name__ == "__main__": from flask import Flask, request, flash, render_template import ConfigParser from os import getenv -from searx.engines import engines -import grequests +from searx.engines import search cfg = ConfigParser.SafeConfigParser() cfg.read('/etc/searx.conf') @@ -38,14 +37,6 @@ cfg.read('searx.conf') app = Flask(__name__) app.secret_key = cfg.get('app', 'secret_key') -def default_request_params(): - return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''} - -def make_callback(results, callback): - def process_callback(response, **kwargs): - results.extend(callback(response)) - return process_callback - @app.route('/', methods=['GET', 'POST']) def index(): if request.method=='POST': @@ -53,30 +44,8 @@ def index(): flash('Wrong post data') return render_template('index.html') query = request.form['q'] - requests = [] - results = [] - user_agent = request.headers.get('User-Agent', '') - for engine in engines: - headers = default_request_params() - headers['User-Agent'] = user_agent - request_params = engine.request(query, headers) - callback = make_callback(results, engine.response) - if request_params['method'] == 'GET': - req = grequests.get(request_params['url'] - ,headers=headers - ,hooks=dict(response=callback) - ) - else: - req = grequests.post(request_params['url'] - ,data=request_params['data'] - ,headers=headers - ,hooks=dict(response=callback) - ) - requests.append(req) - grequests.map(requests) + results = search(query, request) return render_template('results.html', results=results, q=query) - - return render_template('index.html') if __name__ == "__main__":