From b18c69e186b4fea32bbdec6a05d8ab5c210c7d12 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 7 Jan 2022 07:42:05 -0800 Subject: [PATCH] Make search timeouts configurable --- .env.dev.example | 7 +++++++ .env.prod.example | 7 +++++++ bookwyrm/connectors/abstract_connector.py | 5 +++-- bookwyrm/connectors/connector_manager.py | 4 ++-- bookwyrm/settings.py | 6 ++++++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.env.dev.example b/.env.dev.example index b65c1b023..9aac77a54 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -16,6 +16,7 @@ DEFAULT_LANGUAGE="English" MEDIA_ROOT=images/ +# Database configuration PGPORT=5432 POSTGRES_PASSWORD=securedbypassword123 POSTGRES_USER=fedireads @@ -32,10 +33,12 @@ REDIS_ACTIVITY_PASSWORD=redispassword345 REDIS_BROKER_PORT=6379 REDIS_BROKER_PASSWORD=redispassword123 +# Monitoring for celery FLOWER_PORT=8888 FLOWER_USER=mouse FLOWER_PASSWORD=changeme +# Email config EMAIL_HOST=smtp.mailgun.org EMAIL_PORT=587 EMAIL_HOST_USER=mail@your.domain.here @@ -43,6 +46,10 @@ EMAIL_HOST_PASSWORD=emailpassword123 EMAIL_USE_TLS=true EMAIL_USE_SSL=false +# Query timeouts +SEARCH_TIMEOUT=15 +QUERY_TIMEOUT=5 + # Thumbnails Generation ENABLE_THUMBNAIL_GENERATION=false diff --git a/.env.prod.example b/.env.prod.example index 2e0ced5e8..603fb2f20 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -16,6 +16,7 @@ DEFAULT_LANGUAGE="English" MEDIA_ROOT=images/ +# Database configuration PGPORT=5432 POSTGRES_PASSWORD=securedbypassword123 POSTGRES_USER=fedireads @@ -32,10 +33,12 @@ REDIS_ACTIVITY_PASSWORD=redispassword345 REDIS_BROKER_PORT=6379 REDIS_BROKER_PASSWORD=redispassword123 +# Monitoring for celery FLOWER_PORT=8888 FLOWER_USER=mouse FLOWER_PASSWORD=changeme +# Email config EMAIL_HOST=smtp.mailgun.org EMAIL_PORT=587 EMAIL_HOST_USER=mail@your.domain.here @@ -43,6 +46,10 @@ EMAIL_HOST_PASSWORD=emailpassword123 EMAIL_USE_TLS=true EMAIL_USE_SSL=false +# Query timeouts +SEARCH_TIMEOUT=15 +QUERY_TIMEOUT=5 + # Thumbnails Generation ENABLE_THUMBNAIL_GENERATION=false diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 20a175264..5ed57df1f 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -35,7 +35,7 @@ class AbstractMinimalConnector(ABC): for field in self_fields: setattr(self, field, getattr(info, field)) - def search(self, query, min_confidence=None, timeout=5): + def search(self, query, min_confidence=None, timeout=settings.QUERY_TIMEOUT): """free text search""" params = {} if min_confidence: @@ -52,12 +52,13 @@ class AbstractMinimalConnector(ABC): results.append(self.format_search_result(doc)) return results - def isbn_search(self, query): + def isbn_search(self, query, timeout=settings.QUERY_TIMEOUT): """isbn search""" params = {} data = self.get_search_data( f"{self.isbn_search_url}{query}", params=params, + timeout=timeout, ) results = [] diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 45530cd60..3bdd5cb41 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -11,6 +11,7 @@ from django.db.models import signals from requests import HTTPError from bookwyrm import book_search, models +from bookwyrm.settings import SEARCH_TIMEOUT from bookwyrm.tasks import app logger = logging.getLogger(__name__) @@ -30,7 +31,6 @@ def search(query, min_confidence=0.1, return_first=False): isbn = re.sub(r"[\W_]", "", query) maybe_isbn = len(isbn) in [10, 13] # ISBN10 or ISBN13 - timeout = 15 start_time = datetime.now() for connector in get_connectors(): result_set = None @@ -62,7 +62,7 @@ def search(query, min_confidence=0.1, return_first=False): "results": result_set, } ) - if (datetime.now() - start_time).seconds >= timeout: + if (datetime.now() - start_time).seconds >= SEARCH_TIMEOUT: break if return_first: diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 5b69e3782..cacccf783 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -119,6 +119,12 @@ STREAMS = [ {"key": "books", "name": _("Books Timeline"), "shortname": _("Books")}, ] +# Search configuration +# total time in seconds that the instance will spend searching connectors +SEARCH_TIMEOUT = int(env("SEARCH_TIMEOUT", 15)) +# timeout for a query to an individual connector +QUERY_TIMEOUT = int(env("QUERY_TIMEOUT", 5)) + # Redis cache backend if env("USE_DUMMY_CACHE", False): CACHES = {