rafaelcaricio/moviewyrm
1
0
Fork 0
forked from mirrors/bookwyrm
Code Issues 4 Pull requests Projects Releases Wiki Activity

Use activitypub to_model de-serializer in openlibrary connector

Browse source
This commit is contained in:
Mouse Reeve 2020-12-19 14:56:03 -08:00
parent e190a3e53c
commit 2128219b05
2 changed files with 66 additions and 185 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

194
bookwyrm/connectors/abstract_connector.py
View file

@ -10,7 +10,7 @@ import requests
from requests import HTTPError from requests import HTTPError
from requests.exceptions import SSLError from requests.exceptions import SSLError
from bookwyrm import models from bookwyrm import activitypub, models
class ConnectorException(HTTPError): class ConnectorException(HTTPError):
@ -38,7 +38,7 @@ class AbstractMinimalConnector(ABC):
for field in self_fields: for field in self_fields:
setattr(self, field, getattr(info, field)) setattr(self, field, getattr(info, field))
def search(self, query, min_confidence=None): def search(self, query, min_confidence=None):# pylint: disable=unused-argument
''' free text search ''' ''' free text search '''
resp = requests.get( resp = requests.get(
'%s%s' % (self.search_url, query), '%s%s' % (self.search_url, query),
@ -72,9 +72,6 @@ class AbstractConnector(AbstractMinimalConnector):
''' generic book data connector ''' ''' generic book data connector '''
def __init__(self, identifier): def __init__(self, identifier):
super().__init__(identifier) super().__init__(identifier)
self.key_mappings = []
# fields we want to look for in book data to copy over # fields we want to look for in book data to copy over
# title we handle separately. # title we handle separately.
self.book_mappings = [] self.book_mappings = []
@ -88,153 +85,68 @@ class AbstractConnector(AbstractMinimalConnector):
return True return True
@transaction.atomic
def get_or_create_book(self, remote_id): def get_or_create_book(self, remote_id):
# try to load the book ''' translate arbitrary json into an Activitypub dataclass '''
book = models.Book.objects.select_subclasses().filter( # first, check if we have the origin_id saved
origin_id=remote_id existing = models.Edition.find_existing_by_remote_id(remote_id) or \
).first() models.Work.find_existing_by_remote_id(remote_id)
if book: if existing:
if isinstance(book, models.Work): if hasattr(existing, 'get_default_editon'):
return book.default_edition return existing.get_default_editon()
return book return existing
# no book was found, so we start creating a new one # load the json
data = get_data(remote_id) data = get_data(remote_id)
mapped_data = self.dict_from_mappings(data)
work = None
edition = None
if self.is_work_data(data): if self.is_work_data(data):
work_data = data
# if we requested a work and there's already an edition, we're set
work = self.match_from_mappings(work_data, models.Work)
if work and work.default_edition:
return work.default_edition
# no such luck, we need more information.
try: try:
edition_data = self.get_edition_from_work_data(work_data) edition_data = self.get_edition_from_work_data(data)
edition_data = self.dict_from_mappings(edition_data)
except KeyError: except KeyError:
# hack: re-use the work data as the edition data # hack: re-use the work data as the edition data
# this is why remote ids aren't necessarily unique # this is why remote ids aren't necessarily unique
edition_data = data edition_data = mapped_data
work_data = mapped_data
else: else:
edition_data = data
edition = self.match_from_mappings(edition_data, models.Edition)
# no need to figure out about the work if we already know about it
if edition and edition.parent_work:
return edition
# no such luck, we need more information.
try: try:
work_data = self.get_work_from_edition_date(edition_data) work_data = self.get_work_from_edition_data(data)
work_data = self.dict_from_mappings(work_data)
except KeyError: except KeyError:
# remember this hack: re-use the work data as the edition data work_data = mapped_data
work_data = data edition_data = mapped_data
if not work_data or not edition_data: if not work_data or not edition_data:
raise ConnectorException('Unable to load book data: %s' % remote_id) raise ConnectorException('Unable to load book data: %s' % remote_id)
# at this point, we need to figure out the work, edition, or both # create activitypub object
# atomic so that we don't save a work with no edition for vice versa work_activity = activitypub.Work(**work_data)
with transaction.atomic(): edition_activity = activitypub.Edition(**edition_data)
if not work:
work_key = self.get_remote_id_from_data(work_data) # this will dedupe automatically
work = self.create_book(work_key, work_data, models.Work) work = work_activity.to_model(models.Work, save=False)
edition = edition_activity.to_model(models.Edition, save=False)
if not edition:
ed_key = self.get_remote_id_from_data(edition_data)
edition = self.create_book(ed_key, edition_data, models.Edition)
edition.parent_work = work edition.parent_work = work
edition.save()
work.default_edition = edition work.default_edition = edition
work.save()
# now's our change to fill in author gaps work.save()
if not edition.authors.exists() and work.authors.exists():
edition.authors.set(work.authors.all())
edition.author_text = work.author_text
edition.save() edition.save()
if not edition: for author in self.get_authors_from_data(data):
raise ConnectorException('Unable to create book: %s' % remote_id) work.authors.add(author)
edition.authors.add(author)
return edition return edition
def create_book(self, remote_id, data, model): def dict_from_mappings(self, data):
''' create a work or edition from data ''' ''' create a dict in Activitypub format, using mappings supplies by
book = model.objects.create( the subclass '''
origin_id=remote_id, result = {}
title=data['title'], for mapping in self.book_mapping:
connector=self.connector, result[mapping.local_field] = mapping.get_value(data)
) return result
return self.update_book_from_data(book, data)
def update_book_from_data(self, book, data, update_cover=True):
''' for creating a new book or syncing with data '''
book = update_from_mappings(book, data, self.book_mappings)
author_text = []
for author in self.get_authors_from_data(data):
book.authors.add(author)
author_text.append(author.name)
book.author_text = ', '.join(author_text)
book.save()
if not update_cover:
return book
cover = self.get_cover_from_data(data)
if cover:
book.cover.save(*cover, save=True)
return book
def update_book(self, book, data=None):
''' load new data '''
if not book.sync and not book.sync_cover:
return book
if not data:
key = getattr(book, self.key_name)
data = self.load_book_data(key)
if book.sync:
book = self.update_book_from_data(
book, data, update_cover=book.sync_cover)
else:
cover = self.get_cover_from_data(data)
if cover:
book.cover.save(*cover, save=True)
return book
def match_from_mappings(self, data, model):
''' try to find existing copies of this book using various keys '''
relevent_mappings = [m for m in self.key_mappings if \
not m.model or model == m.model]
for mapping in relevent_mappings:
# check if this field is present in the data
value = data.get(mapping.remote_field)
if not value:
continue
# extract the value in the right format
value = mapping.formatter(value)
# search our database for a matching book
kwargs = {mapping.local_field: value}
match = model.objects.filter(**kwargs).first()
if match:
return match
return None
@abstractmethod
def get_remote_id_from_data(self, data):
''' otherwise we won't properly set the remote_id in the db '''
@abstractmethod @abstractmethod
@ -266,25 +178,6 @@ class AbstractConnector(AbstractMinimalConnector):
''' get more info on a book ''' ''' get more info on a book '''
def update_from_mappings(obj, data, mappings):
''' assign data to model with mappings '''
for mapping in mappings:
# check if this field is present in the data
value = data.get(mapping.remote_field)
if not value:
continue
# extract the value in the right format
try:
value = mapping.formatter(value)
except:
continue
# assign the formatted value to the model
obj.__setattr__(mapping.local_field, value)
return obj
def get_date(date_string): def get_date(date_string):
''' helper function to try to interpret dates ''' ''' helper function to try to interpret dates '''
if not date_string: if not date_string:
@ -349,11 +242,14 @@ class SearchResult:
class Mapping: class Mapping:
''' associate a local database field with a field in an external dataset ''' ''' associate a local database field with a field in an external dataset '''
def __init__( def __init__(self, local_field, remote_field=None, formatter=None):
self, local_field, remote_field=None, formatter=None, model=None):
noop = lambda x: x noop = lambda x: x
self.local_field = local_field self.local_field = local_field
self.remote_field = remote_field or local_field self.remote_field = remote_field or local_field
self.formatter = formatter or noop self.formatter = formatter or noop
self.model = model
def get_value(self, data):
''' pull a field from incoming json and return the formatted version '''
value = data.get(self.remote_field)
return self.formatter(value)

51
bookwyrm/connectors/openlibrary.py
View file

@ -17,50 +17,35 @@ class Connector(AbstractConnector):
super().__init__(identifier) super().__init__(identifier)
get_first = lambda a: a[0] get_first = lambda a: a[0]
self.key_mappings = [ self.book_mappings = [
Mapping('isbn_13', model=models.Edition, formatter=get_first), Mapping('title'),
Mapping('isbn_10', model=models.Edition, formatter=get_first), Mapping('sortTitle', remote_field='sort_title'),
Mapping('lccn', model=models.Work, formatter=get_first),
Mapping(
'oclc_number',
remote_field='oclc_numbers',
model=models.Edition,
formatter=get_first
),
Mapping(
'openlibrary_key',
remote_field='key',
formatter=get_openlibrary_key
),
Mapping('goodreads_key'),
Mapping('asin'),
]
self.book_mappings = self.key_mappings + [
Mapping('sort_title'),
Mapping('subtitle'), Mapping('subtitle'),
Mapping('description', formatter=get_description), Mapping('description', formatter=get_description),
Mapping('languages', formatter=get_languages), Mapping('languages', formatter=get_languages),
Mapping('series', formatter=get_first), Mapping('series', formatter=get_first),
Mapping('series_number'), Mapping('seriesNumber', remote_field='series_number'),
Mapping('subjects'), Mapping('subjects'),
Mapping('subject_places'), Mapping('subjectPlaces'),
Mapping('isbn13', formatter=get_first),
Mapping('isbn10', formatter=get_first),
Mapping('lccn', formatter=get_first),
Mapping( Mapping(
'first_published_date', 'oclcNumber', remote_field='oclc_numbers',
remote_field='first_publish_date', formatter=get_first
formatter=get_date
), ),
Mapping( Mapping(
'published_date', 'openlibraryKey', remote_field='key',
remote_field='publish_date', formatter=get_openlibrary_key
formatter=get_date
), ),
Mapping('goodreadsKey', remote_field='goodreads_key'),
Mapping('asin'),
Mapping( Mapping(
'pages', 'firstPublishedDate', remote_field='first_publish_date',
model=models.Edition,
remote_field='number_of_pages'
), ),
Mapping('physical_format', model=models.Edition), Mapping('publishedDate', remote_field='publish_date'),
Mapping('pages', remote_field='number_of_pages'),
Mapping('physicalFormat', remote_field='physical_format'),
Mapping('publishers'), Mapping('publishers'),
] ]