[enh] use one single http connection pool : improve response time. close #100

This commit is contained in:
dalf 2015-01-21 11:33:16 +01:00
parent a865e6672f
commit d07cfd9089
5 changed files with 65 additions and 5 deletions

View file

@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
from lxml import etree
from requests import get
from json import loads
from urllib import urlencode
from searx.languages import language_codes
from searx.engines import (
categories, engines, engine_shortcuts
)
from searx.poolrequests import get
def searx_bang(full_query):

View file

@ -1,6 +1,6 @@
import json
from requests import get
from urllib import urlencode
from searx.poolrequests import get
from searx.utils import format_date_by_locale
result_count = 1

61
searx/poolrequests.py Normal file
View file

@ -0,0 +1,61 @@
import requests
the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
class SessionSinglePool(requests.Session):
def __init__(self):
global the_https_adapter, the_http_adapter
super(SessionSinglePool, self).__init__()
# reuse the same adapters
self.adapters.clear()
self.mount('https://', the_https_adapter)
self.mount('http://', the_http_adapter)
def close(self):
"""Call super, but clear adapters since there are managed globaly"""
self.adapters.clear()
super(SessionSinglePool, self).close()
def request(method, url, **kwargs):
"""same as requests/requests/api.py request(...) except it use SessionSinglePool"""
session = SessionSinglePool()
response = session.request(method=method, url=url, **kwargs)
session.close()
return response
def get(url, **kwargs):
kwargs.setdefault('allow_redirects', True)
return request('get', url, **kwargs)
def options(url, **kwargs):
kwargs.setdefault('allow_redirects', True)
return request('options', url, **kwargs)
def head(url, **kwargs):
kwargs.setdefault('allow_redirects', False)
return request('head', url, **kwargs)
def post(url, data=None, json=None, **kwargs):
return request('post', url, data=data, json=json, **kwargs)
def put(url, data=None, **kwargs):
return request('put', url, data=data, **kwargs)
def patch(url, data=None, **kwargs):
return request('patch', url, data=data, **kwargs)
def delete(url, **kwargs):
return request('delete', url, **kwargs)

View file

@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
import requests as requests_lib
import threading
import re
import searx.poolrequests as requests_lib
from itertools import izip_longest, chain
from operator import itemgetter
from Queue import Queue
@ -31,7 +31,6 @@ from searx.utils import gen_useragent
from searx.query import Query
from searx import logger
logger = logger.getChild('search')
number_of_searches = 0

View file

@ -27,7 +27,6 @@ import cStringIO
import os
from datetime import datetime, timedelta
from requests import get as http_get
from itertools import chain
from urllib import urlencode
from flask import (
@ -36,6 +35,7 @@ from flask import (
)
from flask.ext.babel import Babel, gettext, format_date
from searx import settings, searx_dir
from searx.poolrequests import get as http_get
from searx.engines import (
categories, engines, get_engines_stats, engine_shortcuts
)