From 203e526a8385811f6684fc93150cd9b91757b07d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 30 Oct 2020 17:04:10 -0700 Subject: [PATCH 1/4] Fixes loading remote books - saves remote_id correctly - loads remote books for incoming statuses --- bookwyrm/activitypub/__init__.py | 2 +- bookwyrm/activitypub/note.py | 8 ++++++++ bookwyrm/connectors/abstract_connector.py | 14 +++++++++++--- bookwyrm/connectors/bookwyrm_connector.py | 6 +++++- bookwyrm/connectors/openlibrary.py | 8 ++++++++ bookwyrm/connectors/self_connector.py | 3 +++ bookwyrm/incoming.py | 9 +++++++++ bookwyrm/models/status.py | 12 ++++++++++++ 8 files changed, 57 insertions(+), 5 deletions(-) diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py index 446455fa..c10d1ca1 100644 --- a/bookwyrm/activitypub/__init__.py +++ b/bookwyrm/activitypub/__init__.py @@ -4,7 +4,7 @@ import sys from .base_activity import ActivityEncoder, Image, PublicKey, Signature from .note import Note, GeneratedNote, Article, Comment, Review, Quotation -from .note import Tombstone +from .note import Tombstone, Link from .interaction import Boost, Like from .ordered_collection import OrderedCollection, OrderedCollectionPage from .person import Person diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py index 4e36b01f..d187acc6 100644 --- a/bookwyrm/activitypub/note.py +++ b/bookwyrm/activitypub/note.py @@ -36,9 +36,17 @@ class Article(Note): type: str = 'Article' +@dataclass +class Link(): + ''' for tagging a book in a status ''' + href: str + name: str + type: str = 'Link' + @dataclass(init=False) class GeneratedNote(Note): ''' just a re-typed note ''' + tag: List[Link] type: str = 'GeneratedNote' diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 3172ea79..c6e2f19e 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -122,11 +122,11 @@ class AbstractConnector(ABC): # atomic so that we don't save a work with no edition for vice versa with transaction.atomic(): if not work: - work_key = work_data.get('url') + work_key = self.get_remote_id_from_data(work_data) work = self.create_book(work_key, work_data, models.Work) if not edition: - ed_key = edition_data.get('url') + ed_key = self.get_remote_id_from_data(edition_data) edition = self.create_book(ed_key, edition_data, models.Edition) edition.default = True edition.parent_work = work @@ -146,11 +146,13 @@ class AbstractConnector(ABC): def create_book(self, remote_id, data, model): ''' create a work or edition from data ''' + print(remote_id) book = model.objects.create( remote_id=remote_id, title=data['title'], connector=self.connector, ) + print(book.remote_id) return self.update_book_from_data(book, data) @@ -161,7 +163,8 @@ class AbstractConnector(ABC): author_text = [] for author in self.get_authors_from_data(data): book.authors.add(author) - author_text.append(author.display_name) + if author.display_name: + author_text.append(author.display_name) book.author_text = ', '.join(author_text) book.save() @@ -215,6 +218,11 @@ class AbstractConnector(ABC): return None + @abstractmethod + def get_remote_id_from_data(self, data): + ''' otherwise we won't properly set the remote_id in the db ''' + + @abstractmethod def is_work_data(self, data): ''' differentiate works and editions ''' diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index a9517ca0..072910b6 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -47,8 +47,12 @@ class Connector(AbstractConnector): ] + def get_remote_id_from_data(self, data): + return data.get('id') + + def is_work_data(self, data): - return data['book_type'] == 'Work' + return data['type'] == 'Work' def get_edition_from_work_data(self, data): diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 0ae3ce35..00b76f41 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -73,6 +73,14 @@ class Connector(AbstractConnector): + def get_remote_id_from_data(self, data): + try: + key = data['key'] + except KeyError: + raise ConnectorException('Invalid book data') + return '%s/%s' % (self.books_url, key) + + def is_work_data(self, data): return bool(re.match(r'^[\/\w]+OL\d+W$', data['key'])) diff --git a/bookwyrm/connectors/self_connector.py b/bookwyrm/connectors/self_connector.py index 0e77ecf6..fb70978d 100644 --- a/bookwyrm/connectors/self_connector.py +++ b/bookwyrm/connectors/self_connector.py @@ -51,6 +51,9 @@ class Connector(AbstractConnector): ) + def get_remote_id_from_data(self, data): + pass + def is_work_data(self, data): pass diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 7507ee5b..882769bb 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -225,6 +225,15 @@ def handle_create(activity): if not reply: return + # look up books + book_urls = [] + if hasattr(activity, 'inReplyToBook'): + book_urls.append(activity.inReplyToBook) + if hasattr(activity, 'tag'): + book_urls += [t.href for t in activity.tag if t.type == 'Book'] + for remote_id in book_urls: + book = books_manager.get_or_create_book(remote_id) + model = models.activity_models[activity.type] status = activity.to_model(model) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 8798589f..36dbb06d 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -57,6 +57,17 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ''' structured replies block ''' return self.to_replies() + @property + def ap_tag(self): + tags = [] + for book in self.mention_books.all(): + tags.append(activitypub.Link( + href=book.local_id, + name=book.title, + type='Book' + )) + return tags + shared_mappings = [ ActivityMapping('id', 'remote_id'), ActivityMapping('url', 'remote_id'), @@ -66,6 +77,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ActivityMapping('to', 'ap_to'), ActivityMapping('cc', 'ap_cc'), ActivityMapping('replies', 'ap_replies'), + ActivityMapping('tag', 'ap_tag'), ] # serializing to bookwyrm expanded activitypub From 0393d8123073a3c130e0cc61a373152ebb4cd04f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 30 Oct 2020 17:18:25 -0700 Subject: [PATCH 2/4] Fixes loading covers and authors --- bookwyrm/activitypub/base_activity.py | 1 - bookwyrm/connectors/abstract_connector.py | 2 -- bookwyrm/connectors/bookwyrm_connector.py | 8 ++++++-- bookwyrm/models/book.py | 9 +++++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 1cffd0ed..fc5b1128 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -15,7 +15,6 @@ class ActivityEncoder(JSONEncoder): @dataclass class Image: ''' image block ''' - mediaType: str url: str type: str = 'Image' diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index c6e2f19e..9f9aed43 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -146,13 +146,11 @@ class AbstractConnector(ABC): def create_book(self, remote_id, data, model): ''' create a work or edition from data ''' - print(remote_id) book = model.objects.create( remote_id=remote_id, title=data['title'], connector=self.connector, ) - print(book.remote_id) return self.update_book_from_data(book, data) diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 072910b6..c7a0f2ec 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -41,9 +41,13 @@ class Connector(AbstractConnector): ] self.author_mappings = [ - Mapping('born', remote_field='birth_date', formatter=get_date), - Mapping('died', remote_field='death_date', formatter=get_date), + Mapping('name'), Mapping('bio'), + Mapping('openlibrary_key'), + Mapping('wikipedia_link'), + Mapping('aliases'), + Mapping('born', formatter=get_date), + Mapping('died', formatter=get_date), ] diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 03b2c1f8..0920a931 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -58,6 +58,14 @@ class Book(ActivitypubMixin, BookWyrmModel): ''' the activitypub serialization should be a list of author ids ''' return [a.remote_id for a in self.authors.all()] + @property + def ap_cover(self): + ''' an image attachment ''' + return [activitypub.Image( + url='https://%s%s' % (DOMAIN, self.cover.url), + )] + + activity_mappings = [ ActivityMapping('id', 'remote_id'), @@ -90,6 +98,7 @@ class Book(ActivitypubMixin, BookWyrmModel): ActivityMapping('lccn', 'lccn'), ActivityMapping('editions', 'editions_path'), + ActivityMapping('attachment', 'ap_cover'), ] def save(self, *args, **kwargs): From b8e9f901387437c6ee583120a6294eb36f74f7d3 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 30 Oct 2020 17:41:32 -0700 Subject: [PATCH 3/4] no need to assign book var in incoming --- bookwyrm/incoming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 882769bb..56bea456 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -232,7 +232,7 @@ def handle_create(activity): if hasattr(activity, 'tag'): book_urls += [t.href for t in activity.tag if t.type == 'Book'] for remote_id in book_urls: - book = books_manager.get_or_create_book(remote_id) + books_manager.get_or_create_book(remote_id) model = models.activity_models[activity.type] status = activity.to_model(model) From 2cdd281e98509d5e38134e330a45d50c085bbb55 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 31 Oct 2020 10:50:00 -0700 Subject: [PATCH 4/4] Prevent error on serializing book cover --- bookwyrm/models/book.py | 2 ++ bookwyrm/models/import_job.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 0920a931..b993ef5e 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -61,6 +61,8 @@ class Book(ActivitypubMixin, BookWyrmModel): @property def ap_cover(self): ''' an image attachment ''' + if not self.cover or not hasattr(self.cover, 'url'): + return [] return [activitypub.Image( url='https://%s%s' % (DOMAIN, self.cover.url), )] diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index f7b5e8a2..891bfd1b 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -72,7 +72,7 @@ class ImportItem(models.Model): def get_book_from_isbn(self): ''' search by isbn ''' search_result = books_manager.first_search_result( - self.isbn, min_confidence=0.992 + self.isbn, min_confidence=0.995 ) if search_result: try: @@ -90,7 +90,7 @@ class ImportItem(models.Model): self.data['Author'] ) search_result = books_manager.first_search_result( - search_term, min_confidence=0.992 + search_term, min_confidence=0.995 ) if search_result: try: