Make search timeouts configurable

This commit is contained in:
Mouse Reeve 2022-01-07 07:42:05 -08:00
parent 0ddecd7049
commit b18c69e186
5 changed files with 25 additions and 4 deletions

View file

@ -16,6 +16,7 @@ DEFAULT_LANGUAGE="English"
MEDIA_ROOT=images/ MEDIA_ROOT=images/
# Database configuration
PGPORT=5432 PGPORT=5432
POSTGRES_PASSWORD=securedbypassword123 POSTGRES_PASSWORD=securedbypassword123
POSTGRES_USER=fedireads POSTGRES_USER=fedireads
@ -32,10 +33,12 @@ REDIS_ACTIVITY_PASSWORD=redispassword345
REDIS_BROKER_PORT=6379 REDIS_BROKER_PORT=6379
REDIS_BROKER_PASSWORD=redispassword123 REDIS_BROKER_PASSWORD=redispassword123
# Monitoring for celery
FLOWER_PORT=8888 FLOWER_PORT=8888
FLOWER_USER=mouse FLOWER_USER=mouse
FLOWER_PASSWORD=changeme FLOWER_PASSWORD=changeme
# Email config
EMAIL_HOST=smtp.mailgun.org EMAIL_HOST=smtp.mailgun.org
EMAIL_PORT=587 EMAIL_PORT=587
EMAIL_HOST_USER=mail@your.domain.here EMAIL_HOST_USER=mail@your.domain.here
@ -43,6 +46,10 @@ EMAIL_HOST_PASSWORD=emailpassword123
EMAIL_USE_TLS=true EMAIL_USE_TLS=true
EMAIL_USE_SSL=false EMAIL_USE_SSL=false
# Query timeouts
SEARCH_TIMEOUT=15
QUERY_TIMEOUT=5
# Thumbnails Generation # Thumbnails Generation
ENABLE_THUMBNAIL_GENERATION=false ENABLE_THUMBNAIL_GENERATION=false

View file

@ -16,6 +16,7 @@ DEFAULT_LANGUAGE="English"
MEDIA_ROOT=images/ MEDIA_ROOT=images/
# Database configuration
PGPORT=5432 PGPORT=5432
POSTGRES_PASSWORD=securedbypassword123 POSTGRES_PASSWORD=securedbypassword123
POSTGRES_USER=fedireads POSTGRES_USER=fedireads
@ -32,10 +33,12 @@ REDIS_ACTIVITY_PASSWORD=redispassword345
REDIS_BROKER_PORT=6379 REDIS_BROKER_PORT=6379
REDIS_BROKER_PASSWORD=redispassword123 REDIS_BROKER_PASSWORD=redispassword123
# Monitoring for celery
FLOWER_PORT=8888 FLOWER_PORT=8888
FLOWER_USER=mouse FLOWER_USER=mouse
FLOWER_PASSWORD=changeme FLOWER_PASSWORD=changeme
# Email config
EMAIL_HOST=smtp.mailgun.org EMAIL_HOST=smtp.mailgun.org
EMAIL_PORT=587 EMAIL_PORT=587
EMAIL_HOST_USER=mail@your.domain.here EMAIL_HOST_USER=mail@your.domain.here
@ -43,6 +46,10 @@ EMAIL_HOST_PASSWORD=emailpassword123
EMAIL_USE_TLS=true EMAIL_USE_TLS=true
EMAIL_USE_SSL=false EMAIL_USE_SSL=false
# Query timeouts
SEARCH_TIMEOUT=15
QUERY_TIMEOUT=5
# Thumbnails Generation # Thumbnails Generation
ENABLE_THUMBNAIL_GENERATION=false ENABLE_THUMBNAIL_GENERATION=false

View file

@ -35,7 +35,7 @@ class AbstractMinimalConnector(ABC):
for field in self_fields: for field in self_fields:
setattr(self, field, getattr(info, field)) setattr(self, field, getattr(info, field))
def search(self, query, min_confidence=None, timeout=5): def search(self, query, min_confidence=None, timeout=settings.QUERY_TIMEOUT):
"""free text search""" """free text search"""
params = {} params = {}
if min_confidence: if min_confidence:
@ -52,12 +52,13 @@ class AbstractMinimalConnector(ABC):
results.append(self.format_search_result(doc)) results.append(self.format_search_result(doc))
return results return results
def isbn_search(self, query): def isbn_search(self, query, timeout=settings.QUERY_TIMEOUT):
"""isbn search""" """isbn search"""
params = {} params = {}
data = self.get_search_data( data = self.get_search_data(
f"{self.isbn_search_url}{query}", f"{self.isbn_search_url}{query}",
params=params, params=params,
timeout=timeout,
) )
results = [] results = []

View file

@ -11,6 +11,7 @@ from django.db.models import signals
from requests import HTTPError from requests import HTTPError
from bookwyrm import book_search, models from bookwyrm import book_search, models
from bookwyrm.settings import SEARCH_TIMEOUT
from bookwyrm.tasks import app from bookwyrm.tasks import app
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -30,7 +31,6 @@ def search(query, min_confidence=0.1, return_first=False):
isbn = re.sub(r"[\W_]", "", query) isbn = re.sub(r"[\W_]", "", query)
maybe_isbn = len(isbn) in [10, 13] # ISBN10 or ISBN13 maybe_isbn = len(isbn) in [10, 13] # ISBN10 or ISBN13
timeout = 15
start_time = datetime.now() start_time = datetime.now()
for connector in get_connectors(): for connector in get_connectors():
result_set = None result_set = None
@ -62,7 +62,7 @@ def search(query, min_confidence=0.1, return_first=False):
"results": result_set, "results": result_set,
} }
) )
if (datetime.now() - start_time).seconds >= timeout: if (datetime.now() - start_time).seconds >= SEARCH_TIMEOUT:
break break
if return_first: if return_first:

View file

@ -119,6 +119,12 @@ STREAMS = [
{"key": "books", "name": _("Books Timeline"), "shortname": _("Books")}, {"key": "books", "name": _("Books Timeline"), "shortname": _("Books")},
] ]
# Search configuration
# total time in seconds that the instance will spend searching connectors
SEARCH_TIMEOUT = int(env("SEARCH_TIMEOUT", 15))
# timeout for a query to an individual connector
QUERY_TIMEOUT = int(env("QUERY_TIMEOUT", 5))
# Redis cache backend # Redis cache backend
if env("USE_DUMMY_CACHE", False): if env("USE_DUMMY_CACHE", False):
CACHES = { CACHES = {