searxng/searx/engines/__init__.py

60 lines
2.1 KiB
Python
Raw Normal View History

2013-10-14 21:09:13 +00:00
from os.path import realpath, dirname, splitext, join
from os import listdir
from imp import load_source
2013-10-15 16:19:06 +00:00
import grequests
2013-10-15 21:20:26 +00:00
from itertools import izip_longest, chain
2013-10-14 21:09:13 +00:00
engine_dir = dirname(realpath(__file__))
2013-10-15 17:11:43 +00:00
engines = {}
2013-10-14 21:09:13 +00:00
for filename in listdir(engine_dir):
modname = splitext(filename)[0]
if filename.startswith('_') or not filename.endswith('.py'):
continue
filepath = join(engine_dir, filename)
2013-10-15 16:19:06 +00:00
engine = load_source(modname, filepath)
if not hasattr(engine, 'request') or not hasattr(engine, 'response'):
continue
2013-10-15 17:11:43 +00:00
engines[modname] = engine
2013-10-15 16:19:06 +00:00
def default_request_params():
return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}
2013-10-15 17:11:43 +00:00
def make_callback(engine_name, results, callback):
2013-10-15 16:19:06 +00:00
def process_callback(response, **kwargs):
2013-10-15 21:20:26 +00:00
cb_res = []
2013-10-15 17:11:43 +00:00
for result in callback(response):
result['engine'] = engine_name
2013-10-15 21:20:26 +00:00
cb_res.append(result)
results[engine_name] = cb_res
2013-10-15 16:19:06 +00:00
return process_callback
2013-10-15 20:18:08 +00:00
def search(query, request, selected_engines):
2013-10-15 16:19:06 +00:00
global engines
requests = []
2013-10-15 21:20:26 +00:00
results = {}
2013-10-15 16:19:06 +00:00
user_agent = request.headers.get('User-Agent', '')
2013-10-15 17:11:43 +00:00
for ename, engine in engines.items():
2013-10-15 20:18:08 +00:00
if ename not in selected_engines:
continue
2013-10-15 16:19:06 +00:00
headers = default_request_params()
headers['User-Agent'] = user_agent
request_params = engine.request(query, headers)
2013-10-15 17:11:43 +00:00
callback = make_callback(ename, results, engine.response)
2013-10-15 16:19:06 +00:00
if request_params['method'] == 'GET':
req = grequests.get(request_params['url']
,headers=headers
,hooks=dict(response=callback)
)
else:
req = grequests.post(request_params['url']
,data=request_params['data']
,headers=headers
,hooks=dict(response=callback)
)
requests.append(req)
grequests.map(requests)
2013-10-15 21:20:26 +00:00
return list(filter(None, chain(*izip_longest(*results.values()))))