moviewyrm/bookwyrm/connectors/connector_manager.py

136 lines
4.3 KiB
Python
Raw Normal View History

2021-03-08 16:49:10 +00:00
""" interface with whatever connectors the app has """
2021-06-17 19:34:54 +00:00
from datetime import datetime
2021-01-02 16:14:28 +00:00
import importlib
2021-04-07 15:09:47 +00:00
import logging
2021-03-01 20:09:21 +00:00
import re
from urllib.parse import urlparse
from django.dispatch import receiver
from django.db.models import signals
from requests import HTTPError
from bookwyrm import book_search, models
2021-01-02 16:14:28 +00:00
from bookwyrm.tasks import app
2021-04-07 15:09:47 +00:00
logger = logging.getLogger(__name__)
2020-03-07 20:22:28 +00:00
2021-01-02 16:14:28 +00:00
class ConnectorException(HTTPError):
2021-04-26 16:15:42 +00:00
"""when the connector can't do what was asked"""
def search(query, min_confidence=0.1, return_first=False):
2021-04-26 16:15:42 +00:00
"""find books based on arbitary keywords"""
2021-03-31 19:03:58 +00:00
if not query:
return []
results = []
2021-03-01 20:09:21 +00:00
# Have we got a ISBN ?
isbn = re.sub(r"[\W_]", "", query)
2021-03-08 16:49:10 +00:00
maybe_isbn = len(isbn) in [10, 13] # ISBN10 or ISBN13
2021-03-01 20:09:21 +00:00
2021-06-17 19:34:54 +00:00
timeout = 15
start_time = datetime.now()
for connector in get_connectors():
2021-03-01 20:09:21 +00:00
result_set = None
2021-06-17 19:34:54 +00:00
if maybe_isbn and connector.isbn_search_url and connector.isbn_search_url != "":
2021-03-01 20:09:21 +00:00
# Search on ISBN
try:
result_set = connector.isbn_search(isbn)
2021-06-18 21:29:24 +00:00
except Exception as err: # pylint: disable=broad-except
logger.exception(err)
# if this fails, we can still try regular search
# if no isbn search results, we fallback to generic search
if not result_set:
2021-03-01 20:09:21 +00:00
try:
result_set = connector.search(query, min_confidence=min_confidence)
2021-06-18 21:29:24 +00:00
except Exception as err: # pylint: disable=broad-except
2021-04-07 15:09:47 +00:00
# we don't want *any* error to crash the whole search page
2021-06-18 21:29:24 +00:00
logger.exception(err)
2021-03-01 20:09:21 +00:00
continue
2020-05-03 20:32:23 +00:00
if return_first and result_set:
# if we found anything, return it
return result_set[0]
2021-09-14 22:26:18 +00:00
if result_set:
results.append(
{
"connector": connector,
"results": result_set,
}
)
2021-06-17 19:34:54 +00:00
if (datetime.now() - start_time).seconds >= timeout:
break
2020-04-29 17:57:20 +00:00
if return_first:
return None
2020-04-29 17:57:20 +00:00
return results
2020-10-29 22:29:23 +00:00
def first_search_result(query, min_confidence=0.1):
2021-04-26 16:15:42 +00:00
"""search until you find a result that fits"""
# try local search first
result = book_search.search(query, min_confidence=min_confidence, return_first=True)
if result:
return result
# otherwise, try remote endpoints
return search(query, min_confidence=min_confidence, return_first=True) or None
2020-05-03 22:26:47 +00:00
def get_connectors():
2021-04-26 16:15:42 +00:00
"""load all connectors"""
2021-05-11 18:34:58 +00:00
for info in models.Connector.objects.filter(active=True).order_by("priority").all():
2020-05-12 17:01:36 +00:00
yield load_connector(info)
2021-01-02 16:14:28 +00:00
def get_or_create_connector(remote_id):
2021-04-26 16:15:42 +00:00
"""get the connector related to the object's server"""
2021-01-02 16:14:28 +00:00
url = urlparse(remote_id)
identifier = url.netloc
if not identifier:
2021-03-08 16:49:10 +00:00
raise ValueError("Invalid remote id")
2021-01-02 16:14:28 +00:00
try:
connector_info = models.Connector.objects.get(identifier=identifier)
except models.Connector.DoesNotExist:
connector_info = models.Connector.objects.create(
identifier=identifier,
2021-03-08 16:49:10 +00:00
connector_file="bookwyrm_connector",
2021-09-18 18:32:00 +00:00
base_url=f"https://{identifier}",
books_url=f"https://{identifier}/book",
covers_url=f"https://{identifier}/images/covers",
search_url=f"https://{identifier}/search?q=",
2021-03-08 16:49:10 +00:00
priority=2,
2021-01-02 16:14:28 +00:00
)
return load_connector(connector_info)
2021-09-08 00:09:44 +00:00
@app.task(queue="low_priority")
2021-01-02 16:14:28 +00:00
def load_more_data(connector_id, book_id):
2021-04-26 16:15:42 +00:00
"""background the work of getting all 10,000 editions of LoTR"""
2021-01-02 16:14:28 +00:00
connector_info = models.Connector.objects.get(id=connector_id)
connector = load_connector(connector_info)
book = models.Book.objects.select_subclasses().get(id=book_id)
connector.expand_book_data(book)
def load_connector(connector_info):
2021-04-26 16:15:42 +00:00
"""instantiate the connector class"""
2021-01-02 16:14:28 +00:00
connector = importlib.import_module(
2021-09-18 18:32:00 +00:00
f"bookwyrm.connectors.{connector_info.connector_file}"
2021-01-02 16:14:28 +00:00
)
return connector.Connector(connector_info.identifier)
@receiver(signals.post_save, sender="bookwyrm.FederatedServer")
# pylint: disable=unused-argument
def create_connector(sender, instance, created, *args, **kwargs):
2021-04-26 16:15:42 +00:00
"""create a connector to an external bookwyrm server"""
if instance.application_type == "bookwyrm":
2021-09-18 18:32:00 +00:00
get_or_create_connector(f"https://{instance.server_name}")