Merge pull request #1180 from bookwyrm-social/search-timeout

Adds timeouts to get requests
This commit is contained in:
Mouse Reeve 2021-06-17 14:51:05 -07:00 committed by GitHub
commit 1f071d3c18
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 4 deletions

View file

@ -37,7 +37,7 @@ class AbstractMinimalConnector(ABC):
for field in self_fields:
setattr(self, field, getattr(info, field))
def search(self, query, min_confidence=None):
def search(self, query, min_confidence=None, timeout=5):
"""free text search"""
params = {}
if min_confidence:
@ -46,6 +46,7 @@ class AbstractMinimalConnector(ABC):
data = self.get_search_data(
"%s%s" % (self.search_url, query),
params=params,
timeout=timeout,
)
results = []
@ -218,7 +219,7 @@ def dict_from_mappings(data, mappings):
return result
def get_data(url, params=None):
def get_data(url, params=None, timeout=10):
"""wrapper for request.get"""
# check if the url is blocked
if models.FederatedServer.is_blocked(url):
@ -234,6 +235,7 @@ def get_data(url, params=None):
"Accept": "application/json; charset=utf-8",
"User-Agent": settings.USER_AGENT,
},
timeout=timeout,
)
except (RequestError, SSLError, ConnectionError) as e:
logger.exception(e)
@ -250,7 +252,7 @@ def get_data(url, params=None):
return data
def get_image(url):
def get_image(url, timeout=10):
"""wrapper for requesting an image"""
try:
resp = requests.get(
@ -258,6 +260,7 @@ def get_image(url):
headers={
"User-Agent": settings.USER_AGENT,
},
timeout=timeout,
)
except (RequestError, SSLError) as e:
logger.exception(e)

View file

@ -1,4 +1,5 @@
""" interface with whatever connectors the app has """
from datetime import datetime
import importlib
import logging
import re
@ -29,9 +30,11 @@ def search(query, min_confidence=0.1, return_first=False):
isbn = re.sub(r"[\W_]", "", query)
maybe_isbn = len(isbn) in [10, 13] # ISBN10 or ISBN13
timeout = 15
start_time = datetime.now()
for connector in get_connectors():
result_set = None
if maybe_isbn and connector.isbn_search_url and connector.isbn_search_url == "":
if maybe_isbn and connector.isbn_search_url and connector.isbn_search_url != "":
# Search on ISBN
try:
result_set = connector.isbn_search(isbn)
@ -59,6 +62,8 @@ def search(query, min_confidence=0.1, return_first=False):
"results": result_set,
}
)
if (datetime.now() - start_time).seconds >= timeout:
break
if return_first:
return None