2021-03-08 16:49:10 +00:00
|
|
|
""" interface with whatever connectors the app has """
|
2021-06-17 19:34:54 +00:00
|
|
|
from datetime import datetime
|
2021-01-02 16:14:28 +00:00
|
|
|
import importlib
|
2021-04-07 15:09:47 +00:00
|
|
|
import logging
|
2021-03-01 20:09:21 +00:00
|
|
|
import re
|
2020-05-04 19:36:55 +00:00
|
|
|
from urllib.parse import urlparse
|
2020-03-27 22:25:08 +00:00
|
|
|
|
2021-04-02 00:02:45 +00:00
|
|
|
from django.dispatch import receiver
|
|
|
|
from django.db.models import signals
|
|
|
|
|
2020-09-17 20:02:52 +00:00
|
|
|
from requests import HTTPError
|
|
|
|
|
2020-09-21 15:10:37 +00:00
|
|
|
from bookwyrm import models
|
2021-01-02 16:14:28 +00:00
|
|
|
from bookwyrm.tasks import app
|
2020-03-27 22:25:08 +00:00
|
|
|
|
2021-04-07 15:09:47 +00:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
2020-03-07 20:22:28 +00:00
|
|
|
|
2021-01-02 16:14:28 +00:00
|
|
|
class ConnectorException(HTTPError):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""when the connector can't do what was asked"""
|
2020-05-04 19:36:55 +00:00
|
|
|
|
|
|
|
|
2021-05-10 16:57:53 +00:00
|
|
|
def search(query, min_confidence=0.1, return_first=False):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""find books based on arbitary keywords"""
|
2021-03-31 19:03:58 +00:00
|
|
|
if not query:
|
|
|
|
return []
|
2020-05-03 19:59:06 +00:00
|
|
|
results = []
|
2021-03-01 20:09:21 +00:00
|
|
|
|
|
|
|
# Have we got a ISBN ?
|
2021-03-13 18:01:17 +00:00
|
|
|
isbn = re.sub(r"[\W_]", "", query)
|
2021-03-08 16:49:10 +00:00
|
|
|
maybe_isbn = len(isbn) in [10, 13] # ISBN10 or ISBN13
|
2021-03-01 20:09:21 +00:00
|
|
|
|
2021-06-17 19:34:54 +00:00
|
|
|
timeout = 15
|
|
|
|
start_time = datetime.now()
|
2020-05-03 19:59:06 +00:00
|
|
|
for connector in get_connectors():
|
2021-03-01 20:09:21 +00:00
|
|
|
result_set = None
|
2021-06-17 19:34:54 +00:00
|
|
|
if maybe_isbn and connector.isbn_search_url and connector.isbn_search_url != "":
|
2021-03-01 20:09:21 +00:00
|
|
|
# Search on ISBN
|
2021-05-10 19:53:36 +00:00
|
|
|
try:
|
|
|
|
result_set = connector.isbn_search(isbn)
|
2021-06-18 21:29:24 +00:00
|
|
|
except Exception as err: # pylint: disable=broad-except
|
|
|
|
logger.exception(err)
|
2021-05-10 19:53:36 +00:00
|
|
|
# if this fails, we can still try regular search
|
|
|
|
|
|
|
|
# if no isbn search results, we fallback to generic search
|
2021-05-10 20:01:11 +00:00
|
|
|
if not result_set:
|
2021-03-01 20:09:21 +00:00
|
|
|
try:
|
|
|
|
result_set = connector.search(query, min_confidence=min_confidence)
|
2021-06-18 21:29:24 +00:00
|
|
|
except Exception as err: # pylint: disable=broad-except
|
2021-04-07 15:09:47 +00:00
|
|
|
# we don't want *any* error to crash the whole search page
|
2021-06-18 21:29:24 +00:00
|
|
|
logger.exception(err)
|
2021-03-01 20:09:21 +00:00
|
|
|
continue
|
2020-05-03 20:32:23 +00:00
|
|
|
|
2021-05-10 16:57:53 +00:00
|
|
|
if return_first and result_set:
|
2021-05-10 19:53:36 +00:00
|
|
|
# if we found anything, return it
|
2021-05-10 16:57:53 +00:00
|
|
|
return result_set[0]
|
|
|
|
|
2021-05-10 20:01:11 +00:00
|
|
|
if result_set or connector.local:
|
|
|
|
results.append(
|
|
|
|
{
|
|
|
|
"connector": connector,
|
|
|
|
"results": result_set,
|
|
|
|
}
|
|
|
|
)
|
2021-06-17 19:34:54 +00:00
|
|
|
if (datetime.now() - start_time).seconds >= timeout:
|
|
|
|
break
|
2020-04-29 17:57:20 +00:00
|
|
|
|
2021-05-10 16:57:53 +00:00
|
|
|
if return_first:
|
|
|
|
return None
|
|
|
|
|
2020-04-29 17:57:20 +00:00
|
|
|
return results
|
2020-03-27 22:25:08 +00:00
|
|
|
|
2020-05-03 19:59:06 +00:00
|
|
|
|
2021-04-26 15:02:30 +00:00
|
|
|
def local_search(query, min_confidence=0.1, raw=False, filters=None):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""only look at local search results"""
|
2020-05-04 17:15:41 +00:00
|
|
|
connector = load_connector(models.Connector.objects.get(local=True))
|
2021-04-26 15:02:30 +00:00
|
|
|
return connector.search(
|
|
|
|
query, min_confidence=min_confidence, raw=raw, filters=filters
|
|
|
|
)
|
2020-05-04 17:15:41 +00:00
|
|
|
|
|
|
|
|
2021-03-01 20:09:21 +00:00
|
|
|
def isbn_local_search(query, raw=False):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""only look at local search results"""
|
2021-03-01 20:09:21 +00:00
|
|
|
connector = load_connector(models.Connector.objects.get(local=True))
|
|
|
|
return connector.isbn_search(query, raw=raw)
|
|
|
|
|
|
|
|
|
2020-10-29 22:29:23 +00:00
|
|
|
def first_search_result(query, min_confidence=0.1):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""search until you find a result that fits"""
|
2021-05-10 19:53:36 +00:00
|
|
|
return search(query, min_confidence=min_confidence, return_first=True) or None
|
2020-05-03 22:26:47 +00:00
|
|
|
|
|
|
|
|
2020-05-03 19:59:06 +00:00
|
|
|
def get_connectors():
|
2021-04-26 16:15:42 +00:00
|
|
|
"""load all connectors"""
|
2021-05-11 18:34:58 +00:00
|
|
|
for info in models.Connector.objects.filter(active=True).order_by("priority").all():
|
2020-05-12 17:01:36 +00:00
|
|
|
yield load_connector(info)
|
2021-01-02 16:14:28 +00:00
|
|
|
|
|
|
|
|
|
|
|
def get_or_create_connector(remote_id):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""get the connector related to the object's server"""
|
2021-01-02 16:14:28 +00:00
|
|
|
url = urlparse(remote_id)
|
|
|
|
identifier = url.netloc
|
|
|
|
if not identifier:
|
2021-03-08 16:49:10 +00:00
|
|
|
raise ValueError("Invalid remote id")
|
2021-01-02 16:14:28 +00:00
|
|
|
|
|
|
|
try:
|
|
|
|
connector_info = models.Connector.objects.get(identifier=identifier)
|
|
|
|
except models.Connector.DoesNotExist:
|
|
|
|
connector_info = models.Connector.objects.create(
|
|
|
|
identifier=identifier,
|
2021-03-08 16:49:10 +00:00
|
|
|
connector_file="bookwyrm_connector",
|
2021-09-18 18:32:00 +00:00
|
|
|
base_url=f"https://{identifier}",
|
|
|
|
books_url=f"https://{identifier}/book",
|
|
|
|
covers_url=f"https://{identifier}/images/covers",
|
|
|
|
search_url=f"https://{identifier}/search?q=",
|
2021-03-08 16:49:10 +00:00
|
|
|
priority=2,
|
2021-01-02 16:14:28 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
return load_connector(connector_info)
|
|
|
|
|
|
|
|
|
2021-09-08 00:09:44 +00:00
|
|
|
@app.task(queue="low_priority")
|
2021-01-02 16:14:28 +00:00
|
|
|
def load_more_data(connector_id, book_id):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""background the work of getting all 10,000 editions of LoTR"""
|
2021-01-02 16:14:28 +00:00
|
|
|
connector_info = models.Connector.objects.get(id=connector_id)
|
|
|
|
connector = load_connector(connector_info)
|
|
|
|
book = models.Book.objects.select_subclasses().get(id=book_id)
|
|
|
|
connector.expand_book_data(book)
|
|
|
|
|
|
|
|
|
|
|
|
def load_connector(connector_info):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""instantiate the connector class"""
|
2021-01-02 16:14:28 +00:00
|
|
|
connector = importlib.import_module(
|
2021-09-18 18:32:00 +00:00
|
|
|
f"bookwyrm.connectors.{connector_info.connector_file}"
|
2021-01-02 16:14:28 +00:00
|
|
|
)
|
|
|
|
return connector.Connector(connector_info.identifier)
|
2021-04-02 00:02:45 +00:00
|
|
|
|
|
|
|
|
|
|
|
@receiver(signals.post_save, sender="bookwyrm.FederatedServer")
|
|
|
|
# pylint: disable=unused-argument
|
|
|
|
def create_connector(sender, instance, created, *args, **kwargs):
|
2021-04-26 16:15:42 +00:00
|
|
|
"""create a connector to an external bookwyrm server"""
|
2021-04-02 00:02:45 +00:00
|
|
|
if instance.application_type == "bookwyrm":
|
2021-09-18 18:32:00 +00:00
|
|
|
get_or_create_connector(f"https://{instance.server_name}")
|