diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index 6401bb89..c344c120 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -63,7 +63,6 @@ class ActivityObject:
setattr(self, field.name, value)
- @transaction.atomic
def to_model(self, model, instance=None, save=True):
''' convert from an activity to a model instance '''
if not isinstance(self, model.activity_serializer):
@@ -74,6 +73,9 @@ class ActivityObject:
model.activity_serializer)
)
+ if hasattr(model, 'ignore_activity') and model.ignore_activity(self):
+ return instance
+
# check for an existing instance, if we're not updating a known obj
instance = instance or model.find_existing(self.serialize()) or model()
@@ -88,13 +90,14 @@ class ActivityObject:
if not save:
return instance
- # we can't set many to many and reverse fields on an unsaved object
- instance.save()
+ with transaction.atomic():
+ # we can't set many to many and reverse fields on an unsaved object
+ instance.save()
- # add many to many fields, which have to be set post-save
- for field in instance.many_to_many_fields:
- # mention books/users, for example
- field.set_field_from_activity(instance, self)
+ # add many to many fields, which have to be set post-save
+ for field in instance.many_to_many_fields:
+ # mention books/users, for example
+ field.set_field_from_activity(instance, self)
# reversed relationships in the models
for (model_field_name, activity_field_name) in \
@@ -103,20 +106,23 @@ class ActivityObject:
values = getattr(self, activity_field_name)
if values is None or values is MISSING:
continue
+
+ model_field = getattr(model, model_field_name)
try:
# this is for one to many
- related_model = getattr(model, model_field_name).field.model
+ related_model = model_field.field.model
+ related_field_name = model_field.field.name
except AttributeError:
# it's a one to one or foreign key
- related_model = getattr(model, model_field_name)\
- .related.related_model
+ related_model = model_field.related.related_model
+ related_field_name = model_field.related.related_name
values = [values]
for item in values:
set_related_field.delay(
related_model.__name__,
instance.__class__.__name__,
- instance.__class__.__name__.lower(),
+ related_field_name,
instance.remote_id,
item
)
@@ -142,23 +148,25 @@ def set_related_field(
require_ready=True
)
- if isinstance(data, str):
- item = resolve_remote_id(model, data, save=False)
- else:
- # look for a match based on all the available data
- item = model.find_existing(data)
- if not item:
- # create a new model instance
- item = model.activity_serializer(**data)
- item = item.to_model(model, save=False)
- # this must exist because it's the object that triggered this function
- instance = origin_model.find_existing_by_remote_id(related_remote_id)
- if not instance:
- raise ValueError('Invalid related remote id: %s' % related_remote_id)
+ with transaction.atomic():
+ if isinstance(data, str):
+ item = resolve_remote_id(model, data, save=False)
+ else:
+ # look for a match based on all the available data
+ item = model.find_existing(data)
+ if not item:
+ # create a new model instance
+ item = model.activity_serializer(**data)
+ item = item.to_model(model, save=False)
+ # this must exist because it's the object that triggered this function
+ instance = origin_model.find_existing_by_remote_id(related_remote_id)
+ if not instance:
+ raise ValueError(
+ 'Invalid related remote id: %s' % related_remote_id)
- # edition.parent_work = instance, for example
- setattr(item, related_field_name, instance)
- item.save()
+ # edition.parent_work = instance, for example
+ setattr(item, related_field_name, instance)
+ item.save()
def resolve_remote_id(model, remote_id, refresh=False, save=True):
diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py
index ae9c334d..ee4b8851 100644
--- a/bookwyrm/activitypub/book.py
+++ b/bookwyrm/activitypub/book.py
@@ -38,7 +38,7 @@ class Edition(Book):
isbn13: str = ''
oclcNumber: str = ''
asin: str = ''
- pages: str = ''
+ pages: int = None
physicalFormat: str = ''
publishers: List[str] = field(default_factory=lambda: [])
@@ -50,7 +50,7 @@ class Work(Book):
''' work instance of a book object '''
lccn: str = ''
defaultEdition: str = ''
- editions: List[str]
+ editions: List[str] = field(default_factory=lambda: [])
type: str = 'Work'
@@ -58,9 +58,9 @@ class Work(Book):
class Author(ActivityObject):
''' author of a book '''
name: str
- born: str = ''
- died: str = ''
- aliases: str = ''
+ born: str = None
+ died: str = None
+ aliases: List[str] = field(default_factory=lambda: [])
bio: str = ''
openlibraryKey: str = ''
wikipediaLink: str = ''
diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py
index df28bf8d..72fbe5fc 100644
--- a/bookwyrm/activitypub/note.py
+++ b/bookwyrm/activitypub/note.py
@@ -23,6 +23,7 @@ class Note(ActivityObject):
cc: List[str] = field(default_factory=lambda: [])
replies: Dict = field(default_factory=lambda: {})
inReplyTo: str = ''
+ summary: str = ''
tag: List[Link] = field(default_factory=lambda: [])
attachment: List[Image] = field(default_factory=lambda: [])
sensitive: bool = False
@@ -52,8 +53,8 @@ class Comment(Note):
@dataclass(init=False)
class Review(Comment):
''' a full book review '''
- name: str
- rating: int
+ name: str = None
+ rating: int = None
type: str = 'Review'
diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py
index 88349c02..7e7d027e 100644
--- a/bookwyrm/activitypub/person.py
+++ b/bookwyrm/activitypub/person.py
@@ -18,13 +18,13 @@ class PublicKey(ActivityObject):
class Person(ActivityObject):
''' actor activitypub json '''
preferredUsername: str
- name: str
inbox: str
outbox: str
followers: str
- summary: str
publicKey: PublicKey
endpoints: Dict
+ name: str = None
+ summary: str = None
icon: Image = field(default_factory=lambda: {})
bookwyrmUser: bool = False
manuallyApprovesFollowers: str = False
diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py
index e890d81f..7c627927 100644
--- a/bookwyrm/activitypub/verbs.py
+++ b/bookwyrm/activitypub/verbs.py
@@ -3,7 +3,7 @@ from dataclasses import dataclass
from typing import List
from .base_activity import ActivityObject, Signature
-from .book import Book
+from .book import Edition
@dataclass(init=False)
class Verb(ActivityObject):
@@ -73,7 +73,7 @@ class Add(Verb):
@dataclass(init=False)
class AddBook(Verb):
'''Add activity that's aware of the book obj '''
- target: Book
+ target: Edition
type: str = 'Add'
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index c9f1ad2e..86ac7435 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -1,16 +1,14 @@
''' functionality outline for a book data connector '''
from abc import ABC, abstractmethod
from dataclasses import dataclass
-import pytz
from urllib3.exceptions import RequestError
from django.db import transaction
-from dateutil import parser
import requests
from requests import HTTPError
from requests.exceptions import SSLError
-from bookwyrm import models
+from bookwyrm import activitypub, models
class ConnectorException(HTTPError):
@@ -38,7 +36,7 @@ class AbstractMinimalConnector(ABC):
for field in self_fields:
setattr(self, field, getattr(info, field))
- def search(self, query, min_confidence=None):
+ def search(self, query, min_confidence=None):# pylint: disable=unused-argument
''' free text search '''
resp = requests.get(
'%s%s' % (self.search_url, query),
@@ -72,9 +70,6 @@ class AbstractConnector(AbstractMinimalConnector):
''' generic book data connector '''
def __init__(self, identifier):
super().__init__(identifier)
-
- self.key_mappings = []
-
# fields we want to look for in book data to copy over
# title we handle separately.
self.book_mappings = []
@@ -88,217 +83,110 @@ class AbstractConnector(AbstractMinimalConnector):
return True
+ @transaction.atomic
def get_or_create_book(self, remote_id):
- # try to load the book
- book = models.Book.objects.select_subclasses().filter(
- origin_id=remote_id
- ).first()
- if book:
- if isinstance(book, models.Work):
- return book.default_edition
- return book
+ ''' translate arbitrary json into an Activitypub dataclass '''
+ # first, check if we have the origin_id saved
+ existing = models.Edition.find_existing_by_remote_id(remote_id) or \
+ models.Work.find_existing_by_remote_id(remote_id)
+ if existing:
+ if hasattr(existing, 'get_default_editon'):
+ return existing.get_default_editon()
+ return existing
- # no book was found, so we start creating a new one
+ # load the json
data = get_data(remote_id)
-
- work = None
- edition = None
+ mapped_data = dict_from_mappings(data, self.book_mappings)
if self.is_work_data(data):
- work_data = data
- # if we requested a work and there's already an edition, we're set
- work = self.match_from_mappings(work_data, models.Work)
- if work and work.default_edition:
- return work.default_edition
-
- # no such luck, we need more information.
try:
- edition_data = self.get_edition_from_work_data(work_data)
+ edition_data = self.get_edition_from_work_data(data)
except KeyError:
# hack: re-use the work data as the edition data
# this is why remote ids aren't necessarily unique
edition_data = data
+ work_data = mapped_data
else:
- edition_data = data
- edition = self.match_from_mappings(edition_data, models.Edition)
- # no need to figure out about the work if we already know about it
- if edition and edition.parent_work:
- return edition
-
- # no such luck, we need more information.
try:
- work_data = self.get_work_from_edition_date(edition_data)
+ work_data = self.get_work_from_edition_data(data)
+ work_data = dict_from_mappings(work_data, self.book_mappings)
except KeyError:
- # remember this hack: re-use the work data as the edition data
- work_data = data
+ work_data = mapped_data
+ edition_data = data
if not work_data or not edition_data:
raise ConnectorException('Unable to load book data: %s' % remote_id)
- # at this point, we need to figure out the work, edition, or both
- # atomic so that we don't save a work with no edition for vice versa
- with transaction.atomic():
- if not work:
- work_key = self.get_remote_id_from_data(work_data)
- work = self.create_book(work_key, work_data, models.Work)
+ # create activitypub object
+ work_activity = activitypub.Work(**work_data)
+ # this will dedupe automatically
+ work = work_activity.to_model(models.Work)
+ for author in self.get_authors_from_data(data):
+ work.authors.add(author)
+ return self.create_edition_from_data(work, edition_data)
- if not edition:
- ed_key = self.get_remote_id_from_data(edition_data)
- edition = self.create_book(ed_key, edition_data, models.Edition)
- edition.parent_work = work
- edition.save()
- work.default_edition = edition
- work.save()
- # now's our change to fill in author gaps
+ def create_edition_from_data(self, work, edition_data):
+ ''' if we already have the work, we're ready '''
+ mapped_data = dict_from_mappings(edition_data, self.book_mappings)
+ mapped_data['work'] = work.remote_id
+ edition_activity = activitypub.Edition(**mapped_data)
+ edition = edition_activity.to_model(models.Edition)
+ edition.connector = self.connector
+ edition.save()
+
+ work.default_edition = edition
+ work.save()
+
+ for author in self.get_authors_from_data(edition_data):
+ edition.authors.add(author)
if not edition.authors.exists() and work.authors.exists():
edition.authors.set(work.authors.all())
- edition.author_text = work.author_text
- edition.save()
-
- if not edition:
- raise ConnectorException('Unable to create book: %s' % remote_id)
return edition
- def create_book(self, remote_id, data, model):
- ''' create a work or edition from data '''
- book = model.objects.create(
- origin_id=remote_id,
- title=data['title'],
- connector=self.connector,
- )
- return self.update_book_from_data(book, data)
+ def get_or_create_author(self, remote_id):
+ ''' load that author '''
+ existing = models.Author.find_existing_by_remote_id(remote_id)
+ if existing:
+ return existing
+ data = get_data(remote_id)
- def update_book_from_data(self, book, data, update_cover=True):
- ''' for creating a new book or syncing with data '''
- book = update_from_mappings(book, data, self.book_mappings)
-
- author_text = []
- for author in self.get_authors_from_data(data):
- book.authors.add(author)
- author_text.append(author.name)
- book.author_text = ', '.join(author_text)
- book.save()
-
- if not update_cover:
- return book
-
- cover = self.get_cover_from_data(data)
- if cover:
- book.cover.save(*cover, save=True)
- return book
-
-
- def update_book(self, book, data=None):
- ''' load new data '''
- if not book.sync and not book.sync_cover:
- return book
-
- if not data:
- key = getattr(book, self.key_name)
- data = self.load_book_data(key)
-
- if book.sync:
- book = self.update_book_from_data(
- book, data, update_cover=book.sync_cover)
- else:
- cover = self.get_cover_from_data(data)
- if cover:
- book.cover.save(*cover, save=True)
-
- return book
-
-
- def match_from_mappings(self, data, model):
- ''' try to find existing copies of this book using various keys '''
- relevent_mappings = [m for m in self.key_mappings if \
- not m.model or model == m.model]
- for mapping in relevent_mappings:
- # check if this field is present in the data
- value = data.get(mapping.remote_field)
- if not value:
- continue
-
- # extract the value in the right format
- value = mapping.formatter(value)
-
- # search our database for a matching book
- kwargs = {mapping.local_field: value}
- match = model.objects.filter(**kwargs).first()
- if match:
- return match
- return None
-
-
- @abstractmethod
- def get_remote_id_from_data(self, data):
- ''' otherwise we won't properly set the remote_id in the db '''
+ mapped_data = dict_from_mappings(data, self.author_mappings)
+ activity = activitypub.Author(**mapped_data)
+ # this will dedupe
+ return activity.to_model(models.Author)
@abstractmethod
def is_work_data(self, data):
''' differentiate works and editions '''
-
@abstractmethod
def get_edition_from_work_data(self, data):
''' every work needs at least one edition '''
-
@abstractmethod
def get_work_from_edition_date(self, data):
''' every edition needs a work '''
-
@abstractmethod
def get_authors_from_data(self, data):
''' load author data '''
-
- @abstractmethod
- def get_cover_from_data(self, data):
- ''' load cover '''
-
@abstractmethod
def expand_book_data(self, book):
''' get more info on a book '''
-def update_from_mappings(obj, data, mappings):
- ''' assign data to model with mappings '''
+def dict_from_mappings(data, mappings):
+ ''' create a dict in Activitypub format, using mappings supplies by
+ the subclass '''
+ result = {}
for mapping in mappings:
- # check if this field is present in the data
- value = data.get(mapping.remote_field)
- if not value:
- continue
-
- # extract the value in the right format
- try:
- value = mapping.formatter(value)
- except:
- continue
-
- # assign the formatted value to the model
- obj.__setattr__(mapping.local_field, value)
- return obj
-
-
-def get_date(date_string):
- ''' helper function to try to interpret dates '''
- if not date_string:
- return None
-
- try:
- return pytz.utc.localize(parser.parse(date_string))
- except ValueError:
- pass
-
- try:
- return parser.parse(date_string)
- except ValueError:
- return None
+ result[mapping.local_field] = mapping.get_value(data)
+ return result
def get_data(url):
@@ -349,11 +237,19 @@ class SearchResult:
class Mapping:
''' associate a local database field with a field in an external dataset '''
- def __init__(
- self, local_field, remote_field=None, formatter=None, model=None):
+ def __init__(self, local_field, remote_field=None, formatter=None):
noop = lambda x: x
self.local_field = local_field
self.remote_field = remote_field or local_field
self.formatter = formatter or noop
- self.model = model
+
+ def get_value(self, data):
+ ''' pull a field from incoming json and return the formatted version '''
+ value = data.get(self.remote_field)
+ if not value:
+ return None
+ try:
+ return self.formatter(value)
+ except:# pylint: disable=bare-except
+ return None
diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py
index 28eb1ea0..3b60c307 100644
--- a/bookwyrm/connectors/openlibrary.py
+++ b/bookwyrm/connectors/openlibrary.py
@@ -1,13 +1,9 @@
''' openlibrary data connector '''
import re
-import requests
-
-from django.core.files.base import ContentFile
from bookwyrm import models
from .abstract_connector import AbstractConnector, SearchResult, Mapping
-from .abstract_connector import ConnectorException
-from .abstract_connector import get_date, get_data, update_from_mappings
+from .abstract_connector import ConnectorException, get_data
from .openlibrary_languages import languages
@@ -17,62 +13,57 @@ class Connector(AbstractConnector):
super().__init__(identifier)
get_first = lambda a: a[0]
- self.key_mappings = [
- Mapping('isbn_13', model=models.Edition, formatter=get_first),
- Mapping('isbn_10', model=models.Edition, formatter=get_first),
- Mapping('lccn', model=models.Work, formatter=get_first),
+ get_remote_id = lambda a: self.base_url + a
+ self.book_mappings = [
+ Mapping('title'),
+ Mapping('id', remote_field='key', formatter=get_remote_id),
Mapping(
- 'oclc_number',
- remote_field='oclc_numbers',
- model=models.Edition,
- formatter=get_first
- ),
- Mapping(
- 'openlibrary_key',
- remote_field='key',
- formatter=get_openlibrary_key
- ),
- Mapping('goodreads_key'),
- Mapping('asin'),
- ]
-
- self.book_mappings = self.key_mappings + [
- Mapping('sort_title'),
+ 'cover', remote_field='covers', formatter=self.get_cover_url),
+ Mapping('sortTitle', remote_field='sort_title'),
Mapping('subtitle'),
Mapping('description', formatter=get_description),
Mapping('languages', formatter=get_languages),
Mapping('series', formatter=get_first),
- Mapping('series_number'),
+ Mapping('seriesNumber', remote_field='series_number'),
Mapping('subjects'),
- Mapping('subject_places'),
+ Mapping('subjectPlaces'),
+ Mapping('isbn13', formatter=get_first),
+ Mapping('isbn10', formatter=get_first),
+ Mapping('lccn', formatter=get_first),
Mapping(
- 'first_published_date',
- remote_field='first_publish_date',
- formatter=get_date
+ 'oclcNumber', remote_field='oclc_numbers',
+ formatter=get_first
),
Mapping(
- 'published_date',
- remote_field='publish_date',
- formatter=get_date
+ 'openlibraryKey', remote_field='key',
+ formatter=get_openlibrary_key
),
+ Mapping('goodreadsKey', remote_field='goodreads_key'),
+ Mapping('asin'),
Mapping(
- 'pages',
- model=models.Edition,
- remote_field='number_of_pages'
+ 'firstPublishedDate', remote_field='first_publish_date',
),
- Mapping('physical_format', model=models.Edition),
+ Mapping('publishedDate', remote_field='publish_date'),
+ Mapping('pages', remote_field='number_of_pages'),
+ Mapping('physicalFormat', remote_field='physical_format'),
Mapping('publishers'),
]
self.author_mappings = [
+ Mapping('id', remote_field='key', formatter=get_remote_id),
Mapping('name'),
- Mapping('born', remote_field='birth_date', formatter=get_date),
- Mapping('died', remote_field='death_date', formatter=get_date),
+ Mapping(
+ 'openlibraryKey', remote_field='key',
+ formatter=get_openlibrary_key
+ ),
+ Mapping('born', remote_field='birth_date'),
+ Mapping('died', remote_field='death_date'),
Mapping('bio', formatter=get_description),
]
def get_remote_id_from_data(self, data):
+ ''' format a url from an openlibrary id field '''
try:
key = data['key']
except KeyError:
@@ -107,24 +98,17 @@ class Connector(AbstractConnector):
''' parse author json and load or create authors '''
for author_blob in data.get('authors', []):
author_blob = author_blob.get('author', author_blob)
- # this id is "/authors/OL1234567A" and we want just "OL1234567A"
- author_id = author_blob['key'].split('/')[-1]
- yield self.get_or_create_author(author_id)
+ # this id is "/authors/OL1234567A"
+ author_id = author_blob['key']
+ url = '%s/%s.json' % (self.base_url, author_id)
+ yield self.get_or_create_author(url)
- def get_cover_from_data(self, data):
+ def get_cover_url(self, cover_blob):
''' ask openlibrary for the cover '''
- if not data.get('covers'):
- return None
-
- cover_id = data.get('covers')[0]
+ cover_id = cover_blob[0]
image_name = '%s-M.jpg' % cover_id
- url = '%s/b/id/%s' % (self.covers_url, image_name)
- response = requests.get(url)
- if not response.ok:
- response.raise_for_status()
- image_content = ContentFile(response.content)
- return [image_name, image_content]
+ return '%s/b/id/%s' % (self.covers_url, image_name)
def parse_search_data(self, data):
@@ -158,37 +142,7 @@ class Connector(AbstractConnector):
# we can mass download edition data from OL to avoid repeatedly querying
edition_options = self.load_edition_data(work.openlibrary_key)
for edition_data in edition_options.get('entries'):
- olkey = edition_data.get('key').split('/')[-1]
- # make sure the edition isn't already in the database
- if models.Edition.objects.filter(openlibrary_key=olkey).count():
- continue
-
- # creates and populates the book from the data
- edition = self.create_book(olkey, edition_data, models.Edition)
- # ensures that the edition is associated with the work
- edition.parent_work = work
- edition.save()
- # get author data from the work if it's missing from the edition
- if not edition.authors and work.authors:
- edition.authors.set(work.authors.all())
-
-
- def get_or_create_author(self, olkey):
- ''' load that author '''
- if not re.match(r'^OL\d+A$', olkey):
- raise ValueError('Invalid OpenLibrary author ID')
- author = models.Author.objects.filter(openlibrary_key=olkey).first()
- if author:
- return author
-
- url = '%s/authors/%s.json' % (self.base_url, olkey)
- data = get_data(url)
-
- author = models.Author(openlibrary_key=olkey)
- author = update_from_mappings(author, data, self.author_mappings)
- author.save()
-
- return author
+ self.create_edition_from_data(work, edition_data)
def get_description(description_blob):
@@ -220,7 +174,7 @@ def pick_default_edition(options):
if len(options) == 1:
return options[0]
- options = [e for e in options if e.get('cover')] or options
+ options = [e for e in options if e.get('covers')] or options
options = [e for e in options if \
'/languages/eng' in str(e.get('languages'))] or options
formats = ['paperback', 'hardcover', 'mass market paperback']
diff --git a/bookwyrm/connectors/self_connector.py b/bookwyrm/connectors/self_connector.py
index 80d3a67d..8d31c8a1 100644
--- a/bookwyrm/connectors/self_connector.py
+++ b/bookwyrm/connectors/self_connector.py
@@ -13,7 +13,7 @@ class Connector(AbstractConnector):
that gets implemented it will totally rule '''
vector = SearchVector('title', weight='A') +\
SearchVector('subtitle', weight='B') +\
- SearchVector('author_text', weight='C') +\
+ SearchVector('authors__name', weight='C') +\
SearchVector('isbn_13', weight='A') +\
SearchVector('isbn_10', weight='A') +\
SearchVector('openlibrary_key', weight='C') +\
diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py
index a2c3e24b..1422b4b9 100644
--- a/bookwyrm/forms.py
+++ b/bookwyrm/forms.py
@@ -60,25 +60,36 @@ class RatingForm(CustomForm):
class ReviewForm(CustomForm):
class Meta:
model = models.Review
- fields = ['user', 'book', 'name', 'content', 'rating', 'privacy']
+ fields = [
+ 'user', 'book',
+ 'name', 'content', 'rating',
+ 'content_warning', 'sensitive',
+ 'privacy']
class CommentForm(CustomForm):
class Meta:
model = models.Comment
- fields = ['user', 'book', 'content', 'privacy']
+ fields = [
+ 'user', 'book', 'content',
+ 'content_warning', 'sensitive',
+ 'privacy']
class QuotationForm(CustomForm):
class Meta:
model = models.Quotation
- fields = ['user', 'book', 'quote', 'content', 'privacy']
+ fields = [
+ 'user', 'book', 'quote', 'content',
+ 'content_warning', 'sensitive', 'privacy']
class ReplyForm(CustomForm):
class Meta:
model = models.Status
- fields = ['user', 'content', 'reply_parent', 'privacy']
+ fields = [
+ 'user', 'content', 'content_warning', 'sensitive',
+ 'reply_parent', 'privacy']
class EditUserForm(CustomForm):
diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py
index 9c8c2887..ddf99f97 100644
--- a/bookwyrm/incoming.py
+++ b/bookwyrm/incoming.py
@@ -57,7 +57,6 @@ def shared_inbox(request):
'Announce': handle_boost,
'Add': {
'Edition': handle_add,
- 'Work': handle_add,
},
'Undo': {
'Follow': handle_unfollow,
@@ -198,28 +197,15 @@ def handle_create(activity):
# not a type of status we are prepared to deserialize
return
- if activity.type == 'Note':
- # keep notes if they are replies to existing statuses
- reply = models.Status.objects.filter(
- remote_id=activity.inReplyTo
- ).first()
-
- if not reply:
- discard = True
- # keep notes if they mention local users
- tags = [l['href'] for l in activity.tag if l['type'] == 'Mention']
- for tag in tags:
- if models.User.objects.filter(
- remote_id=tag, local=True).exists():
- # we found a mention of a known use boost
- discard = False
- break
- if discard:
- return
-
status = activity.to_model(model)
+ if not status:
+ # it was discarded because it's not a bookwyrm type
+ return
+
# create a notification if this is a reply
+ notified = []
if status.reply_parent and status.reply_parent.user.local:
+ notified.append(status.reply_parent.user)
status_builder.create_notification(
status.reply_parent.user,
'REPLY',
@@ -228,7 +214,7 @@ def handle_create(activity):
)
if status.mention_users.exists():
for mentioned_user in status.mention_users.all():
- if not mentioned_user.local:
+ if not mentioned_user.local or mentioned_user in notified:
continue
status_builder.create_notification(
mentioned_user,
diff --git a/bookwyrm/migrations/0017_auto_20201212_0059.py b/bookwyrm/migrations/0017_auto_20201212_0059.py
new file mode 100644
index 00000000..c9e3fcf4
--- /dev/null
+++ b/bookwyrm/migrations/0017_auto_20201212_0059.py
@@ -0,0 +1,19 @@
+# Generated by Django 3.0.7 on 2020-12-12 00:59
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bookwyrm', '0016_auto_20201211_2026'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='readthrough',
+ name='book',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0023_merge_20201216_0112.py b/bookwyrm/migrations/0023_merge_20201216_0112.py
new file mode 100644
index 00000000..e3af4849
--- /dev/null
+++ b/bookwyrm/migrations/0023_merge_20201216_0112.py
@@ -0,0 +1,14 @@
+# Generated by Django 3.0.7 on 2020-12-16 01:12
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bookwyrm', '0017_auto_20201212_0059'),
+ ('bookwyrm', '0022_auto_20201212_1744'),
+ ]
+
+ operations = [
+ ]
diff --git a/bookwyrm/migrations/0024_merge_20201216_1721.py b/bookwyrm/migrations/0024_merge_20201216_1721.py
new file mode 100644
index 00000000..41f81335
--- /dev/null
+++ b/bookwyrm/migrations/0024_merge_20201216_1721.py
@@ -0,0 +1,14 @@
+# Generated by Django 3.0.7 on 2020-12-16 17:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bookwyrm', '0023_auto_20201214_0511'),
+ ('bookwyrm', '0023_merge_20201216_0112'),
+ ]
+
+ operations = [
+ ]
diff --git a/bookwyrm/migrations/0025_auto_20201217_0046.py b/bookwyrm/migrations/0025_auto_20201217_0046.py
new file mode 100644
index 00000000..a3ffe8c1
--- /dev/null
+++ b/bookwyrm/migrations/0025_auto_20201217_0046.py
@@ -0,0 +1,39 @@
+# Generated by Django 3.0.7 on 2020-12-17 00:46
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bookwyrm', '0024_merge_20201216_1721'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='author',
+ name='bio',
+ field=bookwyrm.models.fields.HtmlField(blank=True, null=True),
+ ),
+ migrations.AlterField(
+ model_name='book',
+ name='description',
+ field=bookwyrm.models.fields.HtmlField(blank=True, null=True),
+ ),
+ migrations.AlterField(
+ model_name='quotation',
+ name='quote',
+ field=bookwyrm.models.fields.HtmlField(),
+ ),
+ migrations.AlterField(
+ model_name='status',
+ name='content',
+ field=bookwyrm.models.fields.HtmlField(blank=True, null=True),
+ ),
+ migrations.AlterField(
+ model_name='user',
+ name='summary',
+ field=bookwyrm.models.fields.HtmlField(default=''),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0026_status_content_warning.py b/bookwyrm/migrations/0026_status_content_warning.py
new file mode 100644
index 00000000..f4e494db
--- /dev/null
+++ b/bookwyrm/migrations/0026_status_content_warning.py
@@ -0,0 +1,19 @@
+# Generated by Django 3.0.7 on 2020-12-17 03:17
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bookwyrm', '0025_auto_20201217_0046'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='status',
+ name='content_warning',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=500, null=True),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0027_auto_20201220_2007.py b/bookwyrm/migrations/0027_auto_20201220_2007.py
new file mode 100644
index 00000000..a3ad4dda
--- /dev/null
+++ b/bookwyrm/migrations/0027_auto_20201220_2007.py
@@ -0,0 +1,24 @@
+# Generated by Django 3.0.7 on 2020-12-20 20:07
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bookwyrm', '0026_status_content_warning'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='user',
+ name='name',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True),
+ ),
+ migrations.AlterField(
+ model_name='user',
+ name='summary',
+ field=bookwyrm.models.fields.HtmlField(blank=True, null=True),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0028_remove_book_author_text.py b/bookwyrm/migrations/0028_remove_book_author_text.py
new file mode 100644
index 00000000..8743c910
--- /dev/null
+++ b/bookwyrm/migrations/0028_remove_book_author_text.py
@@ -0,0 +1,17 @@
+# Generated by Django 3.0.7 on 2020-12-21 19:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bookwyrm', '0027_auto_20201220_2007'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='book',
+ name='author_text',
+ ),
+ ]
diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index 331d2dd6..a2eac507 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -21,18 +21,17 @@ class Author(ActivitypubMixin, BookWyrmModel):
# idk probably other keys would be useful here?
born = fields.DateTimeField(blank=True, null=True)
died = fields.DateTimeField(blank=True, null=True)
- name = fields.CharField(max_length=255)
+ name = fields.CharField(max_length=255, deduplication_field=True)
aliases = fields.ArrayField(
models.CharField(max_length=255), blank=True, default=list
)
- bio = fields.TextField(null=True, blank=True)
+ bio = fields.HtmlField(null=True, blank=True)
def save(self, *args, **kwargs):
- ''' can't be abstract for query reasons, but you shouldn't USE it '''
- if self.id and not self.remote_id:
+ ''' handle remote vs origin ids '''
+ if self.id:
self.remote_id = self.get_remote_id()
-
- if not self.id:
+ else:
self.origin_id = self.remote_id
self.remote_id = None
return super().save(*args, **kwargs)
diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py
index 08cc6052..0de61fd1 100644
--- a/bookwyrm/models/base_model.py
+++ b/bookwyrm/models/base_model.py
@@ -151,9 +151,9 @@ class ActivitypubMixin:
return self.activity_serializer(**activity).serialize()
- def to_create_activity(self, user):
+ def to_create_activity(self, user, **kwargs):
''' returns the object wrapped in a Create activity '''
- activity_object = self.to_activity()
+ activity_object = self.to_activity(**kwargs)
signer = pkcs1_15.new(RSA.import_key(user.key_pair.private_key))
content = activity_object['content']
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index bcd4bc04..21311d6c 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -36,7 +36,7 @@ class Book(ActivitypubMixin, BookWyrmModel):
title = fields.CharField(max_length=255)
sort_title = fields.CharField(max_length=255, blank=True, null=True)
subtitle = fields.CharField(max_length=255, blank=True, null=True)
- description = fields.TextField(blank=True, null=True)
+ description = fields.HtmlField(blank=True, null=True)
languages = fields.ArrayField(
models.CharField(max_length=255), blank=True, default=list
)
@@ -51,22 +51,45 @@ class Book(ActivitypubMixin, BookWyrmModel):
# TODO: include an annotation about the type of authorship (ie, translator)
authors = fields.ManyToManyField('Author')
# preformatted authorship string for search and easier display
- author_text = models.CharField(max_length=255, blank=True, null=True)
- cover = fields.ImageField(upload_to='covers/', blank=True, null=True)
+ cover = fields.ImageField(
+ upload_to='covers/', blank=True, null=True, alt_field='alt_text')
first_published_date = fields.DateTimeField(blank=True, null=True)
published_date = fields.DateTimeField(blank=True, null=True)
objects = InheritanceManager()
+ @property
+ def author_text(self):
+ ''' format a list of authors '''
+ return ', '.join(a.name for a in self.authors.all())
+
+ @property
+ def edition_info(self):
+ ''' properties of this edition, as a string '''
+ items = [
+ self.physical_format if hasattr(self, 'physical_format') else None,
+ self.languages[0] + ' language' if self.languages and \
+ self.languages[0] != 'English' else None,
+ str(self.published_date.year) if self.published_date else None,
+ ]
+ return ', '.join(i for i in items if i)
+
+ @property
+ def alt_text(self):
+ ''' image alt test '''
+ text = '%s cover' % self.title
+ if self.edition_info:
+ text += ' (%s)' % self.edition_info
+ return text
+
def save(self, *args, **kwargs):
''' can't be abstract for query reasons, but you shouldn't USE it '''
if not isinstance(self, Edition) and not isinstance(self, Work):
raise ValueError('Books should be added as Editions or Works')
- if self.id and not self.remote_id:
+ if self.id:
self.remote_id = self.get_remote_id()
-
- if not self.id:
+ else:
self.origin_id = self.remote_id
self.remote_id = None
return super().save(*args, **kwargs)
@@ -92,7 +115,8 @@ class Work(OrderedCollectionPageMixin, Book):
default_edition = fields.ForeignKey(
'Edition',
on_delete=models.PROTECT,
- null=True
+ null=True,
+ load_remote=False
)
def get_default_edition(self):
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index f6142e37..6dd3b496 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -5,7 +5,6 @@ from uuid import uuid4
import dateutil.parser
from dateutil.parser import ParserError
-from django.contrib.auth.models import AbstractUser
from django.contrib.postgres.fields import ArrayField as DjangoArrayField
from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
@@ -13,8 +12,9 @@ from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
from bookwyrm.connectors import get_image
+from bookwyrm.sanitize_html import InputHtmlParser
+from bookwyrm.settings import DOMAIN
def validate_remote_id(value):
@@ -26,6 +26,15 @@ def validate_remote_id(value):
)
+def validate_username(value):
+ ''' make sure usernames look okay '''
+ if not re.match(r'^[A-Za-z\-_\.]+$', value):
+ raise ValidationError(
+ _('%(value)s is not a valid remote_id'),
+ params={'value': value},
+ )
+
+
class ActivitypubFieldMixin:
''' make a database field serializable '''
def __init__(self, *args, \
@@ -63,6 +72,9 @@ class ActivitypubFieldMixin:
return
key = self.get_activitypub_field()
+ # TODO: surely there's a better way
+ if instance.__class__.__name__ == 'Boost' and key == 'attributedTo':
+ key = 'actor'
if isinstance(activity.get(key), list):
activity[key] += formatted
else:
@@ -92,12 +104,19 @@ class ActivitypubFieldMixin:
class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
''' default (de)serialization for foreign key and one to one '''
+ def __init__(self, *args, load_remote=True, **kwargs):
+ self.load_remote = load_remote
+ super().__init__(*args, **kwargs)
+
def field_from_activity(self, value):
if not value:
return None
related_model = self.related_model
if isinstance(value, dict) and value.get('id'):
+ if not self.load_remote:
+ # only look in the local database
+ return related_model.find_existing(value)
# this is an activitypub object, which we can deserialize
activity_serializer = related_model.activity_serializer
return activity_serializer(**value).to_model(related_model)
@@ -108,6 +127,9 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
# we don't know what this is, ignore it
return None
# gets or creates the model field from the remote id
+ if not self.load_remote:
+ # only look in the local database
+ return related_model.find_existing_by_remote_id(value)
return activitypub.resolve_remote_id(related_model, value)
@@ -134,7 +156,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField):
_('username'),
max_length=150,
unique=True,
- validators=[AbstractUser.username_validator],
+ validators=[validate_username],
error_messages={
'unique': _('A user with that username already exists.'),
},
@@ -276,6 +298,8 @@ class TagField(ManyToManyField):
for link_json in value:
link = activitypub.Link(**link_json)
tag_type = link.type if link.type != 'Mention' else 'Person'
+ if tag_type == 'Book':
+ tag_type = 'Edition'
if tag_type != self.related_model.activity_serializer.type:
# tags can contain multiple types
continue
@@ -285,18 +309,22 @@ class TagField(ManyToManyField):
return items
-def image_serializer(value):
+def image_serializer(value, alt):
''' helper for serializing images '''
if value and hasattr(value, 'url'):
url = value.url
else:
return None
url = 'https://%s%s' % (DOMAIN, url)
- return activitypub.Image(url=url)
+ return activitypub.Image(url=url, name=alt)
class ImageField(ActivitypubFieldMixin, models.ImageField):
''' activitypub-aware image field '''
+ def __init__(self, *args, alt_field=None, **kwargs):
+ self.alt_field = alt_field
+ super().__init__(*args, **kwargs)
+
# pylint: disable=arguments-differ
def set_field_from_activity(self, instance, data, save=True):
''' helper function for assinging a value to the field '''
@@ -306,9 +334,19 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
return
getattr(instance, self.name).save(*formatted, save=save)
+ def set_activity_from_field(self, activity, instance):
+ value = getattr(instance, self.name)
+ if value is None:
+ return
+ alt_text = getattr(instance, self.alt_field)
+ formatted = self.field_to_activity(value, alt_text)
- def field_to_activity(self, value):
- return image_serializer(value)
+ key = self.get_activitypub_field()
+ activity[key] = formatted
+
+
+ def field_to_activity(self, value, alt=None):
+ return image_serializer(value, alt)
def field_from_activity(self, value):
@@ -353,6 +391,15 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
except (ParserError, TypeError):
return None
+class HtmlField(ActivitypubFieldMixin, models.TextField):
+ ''' a text field for storing html '''
+ def field_from_activity(self, value):
+ if not value or value == MISSING:
+ return None
+ sanitizer = InputHtmlParser()
+ sanitizer.feed(value)
+ return sanitizer.get_output()
+
class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
''' activitypub-aware array field '''
def field_to_activity(self, value):
diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py
index 68f3614f..69df43b4 100644
--- a/bookwyrm/models/shelf.py
+++ b/bookwyrm/models/shelf.py
@@ -3,7 +3,7 @@ import re
from django.db import models
from bookwyrm import activitypub
-from .base_model import BookWyrmModel
+from .base_model import ActivitypubMixin, BookWyrmModel
from .base_model import OrderedCollectionMixin
from . import fields
@@ -51,7 +51,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
unique_together = ('user', 'identifier')
-class ShelfBook(BookWyrmModel):
+class ShelfBook(ActivitypubMixin, BookWyrmModel):
''' many to many join table for books and shelves '''
book = fields.ForeignKey(
'Edition', on_delete=models.PROTECT, activitypub_field='object')
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index f0cd3c1d..3654e554 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -1,7 +1,11 @@
''' models for storing different kinds of Activities '''
-from django.utils import timezone
+from dataclasses import MISSING
+import re
+
+from django.apps import apps
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models
+from django.utils import timezone
from model_utils.managers import InheritanceManager
from bookwyrm import activitypub
@@ -14,10 +18,12 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
''' any post, like a reply to a review, etc '''
user = fields.ForeignKey(
'User', on_delete=models.PROTECT, activitypub_field='attributedTo')
- content = fields.TextField(blank=True, null=True)
+ content = fields.HtmlField(blank=True, null=True)
mention_users = fields.TagField('User', related_name='mention_user')
mention_books = fields.TagField('Edition', related_name='mention_book')
local = models.BooleanField(default=True)
+ content_warning = fields.CharField(
+ max_length=500, blank=True, null=True, activitypub_field='summary')
privacy = fields.PrivacyField(max_length=255)
sensitive = fields.BooleanField(default=False)
# created date is different than publish date because of federated posts
@@ -44,6 +50,27 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
serialize_reverse_fields = [('attachments', 'attachment')]
deserialize_reverse_fields = [('attachments', 'attachment')]
+ @classmethod
+ def ignore_activity(cls, activity):
+ ''' keep notes if they are replies to existing statuses '''
+ if activity.type != 'Note':
+ return False
+ if cls.objects.filter(
+ remote_id=activity.inReplyTo).exists():
+ return False
+
+ # keep notes if they mention local users
+ if activity.tag == MISSING or activity.tag is None:
+ return True
+ tags = [l['href'] for l in activity.tag if l['type'] == 'Mention']
+ for tag in tags:
+ user_model = apps.get_model('bookwyrm.User', require_ready=True)
+ if user_model.objects.filter(
+ remote_id=tag, local=True).exists():
+ # we found a mention of a known use boost
+ return False
+ return True
+
@classmethod
def replies(cls, status):
''' load all replies to a status. idk if there's a better way
@@ -57,6 +84,11 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
''' expose the type of status for the ui using activity type '''
return self.activity_serializer.__name__
+ @property
+ def boostable(self):
+ ''' you can't boost dms '''
+ return self.privacy in ['unlisted', 'public']
+
def to_replies(self, **kwargs):
''' helper function for loading AP serialized replies to a status '''
return self.to_ordered_collection(
@@ -78,17 +110,17 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
activity['replies'] = self.to_replies()
# "pure" serialization for non-bookwyrm instances
- if pure:
+ if pure and hasattr(self, 'pure_content'):
activity['content'] = self.pure_content
if 'name' in activity:
activity['name'] = self.pure_name
activity['type'] = self.pure_type
activity['attachment'] = [
- image_serializer(b.cover) for b in self.mention_books.all() \
- if b.cover]
+ image_serializer(b.cover, b.alt_text) \
+ for b in self.mention_books.all()[:4] if b.cover]
if hasattr(self, 'book'):
activity['attachment'].append(
- image_serializer(self.book.cover)
+ image_serializer(self.book.cover, self.book.alt_text)
)
return activity
@@ -125,8 +157,8 @@ class Comment(Status):
@property
def pure_content(self):
''' indicate the book in question for mastodon (or w/e) users '''
- return self.content + '
(comment on "%s")' % \
- (self.book.remote_id, self.book.title)
+ return '%s
(comment on "%s")
' % \ + (self.content, self.book.remote_id, self.book.title) activity_serializer = activitypub.Comment pure_type = 'Note' @@ -134,15 +166,17 @@ class Comment(Status): class Quotation(Status): ''' like a review but without a rating and transient ''' - quote = fields.TextField() + quote = fields.HtmlField() book = fields.ForeignKey( 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') @property def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' - return '"%s"', '
"', self.quote) + quote = re.sub(r'
$', '"', quote) + return '%s-- "%s"
%s' % ( + quote, self.book.remote_id, self.book.title, self.content, @@ -182,8 +216,7 @@ class Review(Status): @property def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' - return self.content + '+ This edition is on your {{ shelf.shelf.name }} shelf. + {% include 'snippets/shelf_selector.html' with current=shelf.shelf %} +
+ {% endfor %} + + {% for shelf in other_edition_shelves %} ++ A different edition of this book is on your {{ shelf.shelf.name }} shelf. + {% include 'snippets/switch_edition_button.html' with edition=book %} +
+ {% endfor %} + + {% for readthrough in readthroughs %} + {% include 'snippets/readthrough.html' with readthrough=readthrough %} + {% endfor %}{{ book.title }}
-({{ book|edition_info }})
+({{ book.edition_info }})