moviewyrm/bookwyrm/connectors/abstract_connector.py

314 lines
9.6 KiB
Python
Raw Normal View History

2021-03-08 16:49:10 +00:00
""" functionality outline for a book data connector """
2020-03-07 20:22:28 +00:00
from abc import ABC, abstractmethod
2020-12-30 19:37:26 +00:00
from dataclasses import asdict, dataclass
2020-12-30 17:14:07 +00:00
import logging
from urllib3.exceptions import RequestError
2020-03-07 20:22:28 +00:00
2020-05-10 19:56:59 +00:00
from django.db import transaction
import requests
2020-12-03 20:35:57 +00:00
from requests.exceptions import SSLError
2020-05-10 19:56:59 +00:00
from bookwyrm import activitypub, models, settings
2021-01-02 16:14:28 +00:00
from .connector_manager import load_more_data, ConnectorException
2020-03-07 20:22:28 +00:00
2020-12-30 17:14:07 +00:00
logger = logging.getLogger(__name__)
2021-03-08 16:49:10 +00:00
2020-11-29 02:56:28 +00:00
class AbstractMinimalConnector(ABC):
2021-04-26 16:15:42 +00:00
"""just the bare bones, for other bookwyrm instances"""
2021-03-08 16:49:10 +00:00
def __init__(self, identifier):
2020-03-07 20:22:28 +00:00
# load connector settings
info = models.Connector.objects.get(identifier=identifier)
2020-03-28 23:01:02 +00:00
self.connector = info
2020-03-07 20:22:28 +00:00
2020-05-10 23:41:24 +00:00
# the things in the connector model to copy over
self_fields = [
2021-03-08 16:49:10 +00:00
"base_url",
"books_url",
"covers_url",
"search_url",
"isbn_search_url",
"name",
"identifier",
"local",
2020-05-10 18:29:10 +00:00
]
2020-05-10 23:41:24 +00:00
for field in self_fields:
2020-05-10 18:29:10 +00:00
setattr(self, field, getattr(info, field))
def search(self, query, min_confidence=None):
2021-04-26 16:15:42 +00:00
"""free text search"""
params = {}
if min_confidence:
2021-03-08 16:49:10 +00:00
params["min_confidence"] = min_confidence
data = self.get_search_data(
2021-03-08 16:49:10 +00:00
"%s%s" % (self.search_url, query),
params=params,
)
results = []
for doc in self.parse_search_data(data)[:10]:
results.append(self.format_search_result(doc))
return results
2021-03-01 20:09:21 +00:00
def isbn_search(self, query):
2021-04-26 16:15:42 +00:00
"""isbn search"""
2021-03-01 20:09:21 +00:00
params = {}
data = self.get_search_data(
2021-03-08 16:49:10 +00:00
"%s%s" % (self.isbn_search_url, query),
2021-03-01 20:09:21 +00:00
params=params,
)
results = []
# this shouldn't be returning mutliple results, but just in case
for doc in self.parse_isbn_search_data(data)[:10]:
2021-03-01 20:09:21 +00:00
results.append(self.format_isbn_search_result(doc))
return results
def get_search_data(self, remote_id, **kwargs): # pylint: disable=no-self-use
"""this allows connectors to override the default behavior"""
return get_data(remote_id, **kwargs)
2020-11-29 02:56:28 +00:00
@abstractmethod
def get_or_create_book(self, remote_id):
2021-04-26 16:15:42 +00:00
"""pull up a book record by whatever means possible"""
2020-11-29 02:56:28 +00:00
@abstractmethod
def parse_search_data(self, data):
2021-04-26 16:15:42 +00:00
"""turn the result json from a search into a list"""
2020-11-29 02:56:28 +00:00
@abstractmethod
def format_search_result(self, search_result):
2021-04-26 16:15:42 +00:00
"""create a SearchResult obj from json"""
2020-11-29 02:56:28 +00:00
2021-03-01 20:09:21 +00:00
@abstractmethod
def parse_isbn_search_data(self, data):
2021-04-26 16:15:42 +00:00
"""turn the result json from a search into a list"""
2021-03-01 20:09:21 +00:00
@abstractmethod
def format_isbn_search_result(self, search_result):
2021-04-26 16:15:42 +00:00
"""create a SearchResult obj from json"""
2021-03-01 20:09:21 +00:00
2020-11-29 02:56:28 +00:00
class AbstractConnector(AbstractMinimalConnector):
2021-04-26 16:15:42 +00:00
"""generic book data connector"""
2021-03-08 16:49:10 +00:00
2020-11-29 02:56:28 +00:00
def __init__(self, identifier):
super().__init__(identifier)
# fields we want to look for in book data to copy over
# title we handle separately.
self.book_mappings = []
def get_or_create_book(self, remote_id):
2021-04-26 16:15:42 +00:00
"""translate arbitrary json into an Activitypub dataclass"""
# first, check if we have the origin_id saved
2021-03-08 16:49:10 +00:00
existing = models.Edition.find_existing_by_remote_id(
remote_id
) or models.Work.find_existing_by_remote_id(remote_id)
if existing:
if hasattr(existing, "default_edition"):
return existing.default_edition
return existing
# load the json
data = self.get_book_data(remote_id)
2020-05-10 19:56:59 +00:00
if self.is_work_data(data):
try:
edition_data = self.get_edition_from_work_data(data)
except (KeyError, ConnectorException):
2020-05-10 19:56:59 +00:00
# hack: re-use the work data as the edition data
# this is why remote ids aren't necessarily unique
2020-12-20 00:14:05 +00:00
edition_data = data
2021-04-30 22:48:52 +00:00
work_data = data
2020-05-10 19:56:59 +00:00
else:
2020-12-20 00:14:05 +00:00
edition_data = data
try:
work_data = self.get_work_from_edition_data(data)
2021-04-30 22:48:52 +00:00
except (KeyError, ConnectorException) as e:
logger.exception(e)
work_data = data
if not work_data or not edition_data:
2021-03-08 16:49:10 +00:00
raise ConnectorException("Unable to load book data: %s" % remote_id)
with transaction.atomic():
# create activitypub object
2021-04-30 22:48:52 +00:00
work_activity = activitypub.Work(
**dict_from_mappings(work_data, self.book_mappings)
)
# this will dedupe automatically
work = work_activity.to_model(model=models.Work)
2021-04-30 22:48:52 +00:00
for author in self.get_authors_from_data(work_data):
work.authors.add(author)
edition = self.create_edition_from_data(work, edition_data)
load_more_data.delay(self.connector.id, work.id)
return edition
2021-04-07 00:46:06 +00:00
def get_book_data(self, remote_id): # pylint: disable=no-self-use
"""this allows connectors to override the default behavior"""
return get_data(remote_id)
2020-12-20 00:14:05 +00:00
def create_edition_from_data(self, work, edition_data):
2021-04-26 16:15:42 +00:00
"""if we already have the work, we're ready"""
2020-12-20 00:14:05 +00:00
mapped_data = dict_from_mappings(edition_data, self.book_mappings)
2021-03-08 16:49:10 +00:00
mapped_data["work"] = work.remote_id
2020-12-20 00:14:05 +00:00
edition_activity = activitypub.Edition(**mapped_data)
edition = edition_activity.to_model(model=models.Edition)
2020-12-20 00:14:05 +00:00
edition.connector = self.connector
edition.save()
2020-12-20 00:14:05 +00:00
for author in self.get_authors_from_data(edition_data):
edition.authors.add(author)
2020-12-20 00:14:05 +00:00
if not edition.authors.exists() and work.authors.exists():
edition.authors.set(work.authors.all())
return edition
2020-05-10 19:56:59 +00:00
2020-12-19 23:20:31 +00:00
def get_or_create_author(self, remote_id):
2021-04-26 16:15:42 +00:00
"""load that author"""
2020-12-20 00:14:05 +00:00
existing = models.Author.find_existing_by_remote_id(remote_id)
2020-12-19 23:20:31 +00:00
if existing:
return existing
data = self.get_book_data(remote_id)
2020-12-19 23:20:31 +00:00
2020-12-20 00:14:05 +00:00
mapped_data = dict_from_mappings(data, self.author_mappings)
try:
activity = activitypub.Author(**mapped_data)
except activitypub.ActivitySerializerError:
return None
2020-12-19 23:20:31 +00:00
# this will dedupe
return activity.to_model(model=models.Author)
2020-05-10 19:56:59 +00:00
@abstractmethod
def is_work_data(self, data):
2021-04-26 16:15:42 +00:00
"""differentiate works and editions"""
2020-05-10 19:56:59 +00:00
@abstractmethod
def get_edition_from_work_data(self, data):
2021-04-26 16:15:42 +00:00
"""every work needs at least one edition"""
2020-05-10 19:56:59 +00:00
@abstractmethod
def get_work_from_edition_data(self, data):
2021-04-26 16:15:42 +00:00
"""every edition needs a work"""
2020-05-09 19:53:55 +00:00
@abstractmethod
def get_authors_from_data(self, data):
2021-04-26 16:15:42 +00:00
"""load author data"""
2020-04-02 05:11:31 +00:00
@abstractmethod
def expand_book_data(self, book):
2021-04-26 16:15:42 +00:00
"""get more info on a book"""
2020-04-02 05:11:31 +00:00
2020-12-20 00:14:05 +00:00
def dict_from_mappings(data, mappings):
2021-03-08 16:49:10 +00:00
"""create a dict in Activitypub format, using mappings supplies by
the subclass"""
2020-12-19 23:20:31 +00:00
result = {}
for mapping in mappings:
# sometimes there are multiple mappings for one field, don't
# overwrite earlier writes in that case
if mapping.local_field in result and result[mapping.local_field]:
continue
2020-12-19 23:20:31 +00:00
result[mapping.local_field] = mapping.get_value(data)
return result
2020-03-30 00:40:51 +00:00
def get_data(url, params=None):
2021-04-26 16:15:42 +00:00
"""wrapper for request.get"""
# check if the url is blocked
if models.FederatedServer.is_blocked(url):
raise ConnectorException(
"Attempting to load data from blocked url: {:s}".format(url)
)
try:
resp = requests.get(
url,
params=params,
headers={
2021-03-08 16:49:10 +00:00
"Accept": "application/json; charset=utf-8",
"User-Agent": settings.USER_AGENT,
},
)
except (RequestError, SSLError, ConnectionError) as e:
logger.exception(e)
raise ConnectorException()
if not resp.ok:
raise ConnectorException()
try:
data = resp.json()
except ValueError as e:
logger.exception(e)
raise ConnectorException()
return data
2020-11-29 17:40:15 +00:00
def get_image(url):
2021-04-26 16:15:42 +00:00
"""wrapper for requesting an image"""
2020-11-29 17:40:15 +00:00
try:
resp = requests.get(
url,
headers={
2021-03-08 16:49:10 +00:00
"User-Agent": settings.USER_AGENT,
},
)
2021-02-11 01:54:49 +00:00
except (RequestError, SSLError) as e:
logger.exception(e)
2020-11-29 17:40:15 +00:00
return None
if not resp.ok:
return None
return resp
2020-10-29 22:29:23 +00:00
@dataclass
2020-09-21 17:25:26 +00:00
class SearchResult:
2021-04-26 16:15:42 +00:00
"""standardized search result object"""
2021-03-08 16:49:10 +00:00
2020-10-29 22:29:23 +00:00
title: str
key: str
connector: object
view_link: str = None
2021-03-13 21:55:20 +00:00
author: str = None
year: str = None
cover: str = None
2020-10-29 22:29:23 +00:00
confidence: int = 1
def __repr__(self):
return "<SearchResult key={!r} title={!r} author={!r}>".format(
2021-03-08 16:49:10 +00:00
self.key, self.title, self.author
)
2020-05-10 23:41:24 +00:00
2020-12-30 19:37:26 +00:00
def json(self):
2021-04-26 16:15:42 +00:00
"""serialize a connector for json response"""
2020-12-30 19:37:26 +00:00
serialized = asdict(self)
2021-03-08 16:49:10 +00:00
del serialized["connector"]
2020-12-30 19:37:26 +00:00
return serialized
2020-05-10 23:41:24 +00:00
2020-09-21 17:25:26 +00:00
class Mapping:
2021-04-26 16:15:42 +00:00
"""associate a local database field with a field in an external dataset"""
2021-03-08 16:49:10 +00:00
def __init__(self, local_field, remote_field=None, formatter=None):
2021-04-07 00:46:06 +00:00
noop = lambda x: x
2020-05-10 23:41:24 +00:00
self.local_field = local_field
self.remote_field = remote_field or local_field
self.formatter = formatter or noop
def get_value(self, data):
2021-04-26 16:15:42 +00:00
"""pull a field from incoming json and return the formatted version"""
value = data.get(self.remote_field)
2020-12-20 00:14:05 +00:00
if not value:
return None
try:
2021-04-07 00:46:06 +00:00
return self.formatter(value)
2021-03-08 16:49:10 +00:00
except: # pylint: disable=bare-except
2020-12-20 00:14:05 +00:00
return None