mirror of
https://github.com/searxng/searxng.git
synced 2024-11-29 14:11:02 +00:00
[enh] gevent/grequests changed to the built-in threading lib
This commit is contained in:
parent
b1b0b82a2a
commit
d959cb1c05
5 changed files with 20 additions and 23 deletions
|
@ -1,6 +1,6 @@
|
||||||
flask
|
flask
|
||||||
flask-babel
|
flask-babel
|
||||||
grequests
|
requests
|
||||||
lxml
|
lxml
|
||||||
pyyaml
|
pyyaml
|
||||||
python-dateutil
|
python-dateutil
|
||||||
|
|
|
@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
||||||
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
||||||
'''
|
'''
|
||||||
|
|
||||||
import grequests
|
import requests as requests_lib
|
||||||
|
import threading
|
||||||
import re
|
import re
|
||||||
from itertools import izip_longest, chain
|
from itertools import izip_longest, chain
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
@ -32,6 +33,18 @@ from searx.query import Query
|
||||||
number_of_searches = 0
|
number_of_searches = 0
|
||||||
|
|
||||||
|
|
||||||
|
def threaded_requests(requests):
|
||||||
|
for fn, url, request_args in requests:
|
||||||
|
th = threading.Thread(
|
||||||
|
target=fn, args=(url,), kwargs=request_args, name=url,
|
||||||
|
)
|
||||||
|
th.start()
|
||||||
|
|
||||||
|
for th in threading.enumerate():
|
||||||
|
if th.name.startswith('http'):
|
||||||
|
th.join()
|
||||||
|
|
||||||
|
|
||||||
# get default reqest parameter
|
# get default reqest parameter
|
||||||
def default_request_params():
|
def default_request_params():
|
||||||
return {
|
return {
|
||||||
|
@ -471,9 +484,9 @@ class Search(object):
|
||||||
|
|
||||||
# specific type of request (GET or POST)
|
# specific type of request (GET or POST)
|
||||||
if request_params['method'] == 'GET':
|
if request_params['method'] == 'GET':
|
||||||
req = grequests.get
|
req = requests_lib.get
|
||||||
else:
|
else:
|
||||||
req = grequests.post
|
req = requests_lib.post
|
||||||
request_args['data'] = request_params['data']
|
request_args['data'] = request_params['data']
|
||||||
|
|
||||||
# ignoring empty urls
|
# ignoring empty urls
|
||||||
|
@ -481,10 +494,10 @@ class Search(object):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# append request to list
|
# append request to list
|
||||||
requests.append(req(request_params['url'], **request_args))
|
requests.append((req, request_params['url'], request_args))
|
||||||
|
|
||||||
# send all search-request
|
# send all search-request
|
||||||
grequests.map(requests)
|
threaded_requests(requests)
|
||||||
|
|
||||||
# update engine-specific stats
|
# update engine-specific stats
|
||||||
for engine_name, engine_results in results.items():
|
for engine_name, engine_results in results.items():
|
||||||
|
|
|
@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
||||||
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
||||||
'''
|
'''
|
||||||
|
|
||||||
from gevent import monkey
|
|
||||||
monkey.patch_all()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
from sys import path
|
from sys import path
|
||||||
from os.path import realpath, dirname
|
from os.path import realpath, dirname
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -41,7 +41,7 @@ setup(
|
||||||
install_requires=[
|
install_requires=[
|
||||||
'flask',
|
'flask',
|
||||||
'flask-babel',
|
'flask-babel',
|
||||||
'grequests',
|
'requests',
|
||||||
'lxml',
|
'lxml',
|
||||||
'pyyaml',
|
'pyyaml',
|
||||||
'setuptools',
|
'setuptools',
|
||||||
|
|
12
versions.cfg
12
versions.cfg
|
@ -36,18 +36,6 @@ zc.recipe.testrunner = 2.0.0
|
||||||
# WebTest==2.0.11
|
# WebTest==2.0.11
|
||||||
beautifulsoup4 = 4.3.2
|
beautifulsoup4 = 4.3.2
|
||||||
|
|
||||||
# Required by:
|
|
||||||
# grequests==0.2.0
|
|
||||||
gevent = 1.0
|
|
||||||
|
|
||||||
# Required by:
|
|
||||||
# gevent==1.0
|
|
||||||
greenlet = 0.4.2
|
|
||||||
|
|
||||||
# Required by:
|
|
||||||
# searx==0.1
|
|
||||||
grequests = 0.2.0
|
|
||||||
|
|
||||||
# Required by:
|
# Required by:
|
||||||
# robotframework-httplibrary==0.4.2
|
# robotframework-httplibrary==0.4.2
|
||||||
jsonpatch = 1.3
|
jsonpatch = 1.3
|
||||||
|
|
Loading…
Reference in a new issue