diff --git a/README.md b/README.md index 51e39541..2dfedebb 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,6 @@ cp .env.example .env For most testing, you'll want to use ngrok. Remember to set the DOMAIN in `.env` to your ngrok domain. - -#### With Docker You'll have to install the Docker and docker-compose. When you're ready, run: ```bash @@ -70,33 +68,7 @@ docker-compose run --rm web python manage.py migrate docker-compose run --rm web python manage.py initdb ``` -### Without Docker -You will need postgres installed and running on your computer. - -``` bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -createdb bookwyrm -``` - -Create the psql user in `psql bookwyrm`: -``` psql -CREATE ROLE bookwyrm WITH LOGIN PASSWORD 'bookwyrm'; -GRANT ALL PRIVILEGES ON DATABASE bookwyrm TO bookwyrm; -``` - -Initialize the database (or, more specifically, delete the existing database, run migrations, and start fresh): -``` bash -./rebuilddb.sh -``` -This creates two users, `mouse` with password `password123` and `rat` with password `ratword`. - -The application uses Celery and Redis for task management, which must also be installed and configured. - -And go to the app at `localhost:8000` - - +Once the build is complete, you can access the instance at `localhost:1333` ## Installing in Production diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py index 85245929..b5b124ec 100644 --- a/bookwyrm/activitypub/__init__.py +++ b/bookwyrm/activitypub/__init__.py @@ -2,20 +2,19 @@ import inspect import sys -from .base_activity import ActivityEncoder, PublicKey, Signature +from .base_activity import ActivityEncoder, Signature from .base_activity import Link, Mention -from .base_activity import ActivitySerializerError -from .base_activity import tag_formatter +from .base_activity import ActivitySerializerError, resolve_remote_id from .image import Image from .note import Note, GeneratedNote, Article, Comment, Review, Quotation from .note import Tombstone from .interaction import Boost, Like from .ordered_collection import OrderedCollection, OrderedCollectionPage -from .person import Person +from .person import Person, PublicKey from .book import Edition, Work, Author from .verbs import Create, Delete, Undo, Update from .verbs import Follow, Accept, Reject -from .verbs import Add, Remove +from .verbs import Add, AddBook, Remove # this creates a list of all the Activity types that we can serialize, # so when an Activity comes in from outside, we can check if it's known diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index caa4aeb8..6401bb89 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -1,18 +1,12 @@ ''' basics for an activitypub serializer ''' from dataclasses import dataclass, fields, MISSING from json import JSONEncoder -from uuid import uuid4 -from django.core.files.base import ContentFile +from django.apps import apps from django.db import transaction -from django.db.models.fields.related_descriptors \ - import ForwardManyToOneDescriptor, ManyToManyDescriptor, \ - ReverseManyToOneDescriptor -from django.db.models.fields.files import ImageFileDescriptor -import requests - -from bookwyrm import books_manager, models +from bookwyrm.connectors import ConnectorException, get_data +from bookwyrm.tasks import app class ActivitySerializerError(ValueError): ''' routine problems serializing activitypub json ''' @@ -25,26 +19,19 @@ class ActivityEncoder(JSONEncoder): @dataclass -class Link(): +class Link: ''' for tagging a book in a status ''' href: str name: str type: str = 'Link' + @dataclass class Mention(Link): ''' a subtype of Link for mentioning an actor ''' type: str = 'Mention' -@dataclass -class PublicKey: - ''' public key block ''' - id: str - owner: str - publicKeyPem: str - - @dataclass class Signature: ''' public key block ''' @@ -76,88 +63,63 @@ class ActivityObject: setattr(self, field.name, value) - def to_model(self, model, instance=None): + @transaction.atomic + def to_model(self, model, instance=None, save=True): ''' convert from an activity to a model instance ''' if not isinstance(self, model.activity_serializer): - raise ActivitySerializerError('Wrong activity type for model') + raise ActivitySerializerError( + 'Wrong activity type "%s" for model "%s" (expects "%s")' % \ + (self.__class__, + model.__name__, + model.activity_serializer) + ) # check for an existing instance, if we're not updating a known obj - if not instance: - try: - return model.objects.get(remote_id=self.id) - except model.DoesNotExist: - pass + instance = instance or model.find_existing(self.serialize()) or model() - model_fields = [m.name for m in model._meta.get_fields()] - mapped_fields = {} - many_to_many_fields = {} - one_to_many_fields = {} - image_fields = {} + for field in instance.simple_fields: + field.set_field_from_activity(instance, self) - for mapping in model.activity_mappings: - if mapping.model_key not in model_fields: + # image fields have to be set after other fields because they can save + # too early and jank up users + for field in instance.image_fields: + field.set_field_from_activity(instance, self, save=save) + + if not save: + return instance + + # we can't set many to many and reverse fields on an unsaved object + instance.save() + + # add many to many fields, which have to be set post-save + for field in instance.many_to_many_fields: + # mention books/users, for example + field.set_field_from_activity(instance, self) + + # reversed relationships in the models + for (model_field_name, activity_field_name) in \ + instance.deserialize_reverse_fields: + # attachments on Status, for example + values = getattr(self, activity_field_name) + if values is None or values is MISSING: continue - # value is None if there's a default that isn't supplied - # in the activity but is supplied in the formatter - value = None - if mapping.activity_key: - value = getattr(self, mapping.activity_key) - model_field = getattr(model, mapping.model_key) - - formatted_value = mapping.model_formatter(value) - if isinstance(model_field, ForwardManyToOneDescriptor) and \ - formatted_value: - # foreign key remote id reolver (work on Edition, for example) - fk_model = model_field.field.related_model - reference = resolve_foreign_key(fk_model, formatted_value) - mapped_fields[mapping.model_key] = reference - elif isinstance(model_field, ManyToManyDescriptor): - # status mentions book/users - many_to_many_fields[mapping.model_key] = formatted_value - elif isinstance(model_field, ReverseManyToOneDescriptor): - # attachments on Status, for example - one_to_many_fields[mapping.model_key] = formatted_value - elif isinstance(model_field, ImageFileDescriptor): - # image fields need custom handling - image_fields[mapping.model_key] = formatted_value - else: - mapped_fields[mapping.model_key] = formatted_value - - with transaction.atomic(): - if instance: - # updating an existing model isntance - for k, v in mapped_fields.items(): - setattr(instance, k, v) - instance.save() - else: - # creating a new model instance - instance = model.objects.create(**mapped_fields) - - # add images - for (model_key, value) in image_fields.items(): - formatted_value = image_formatter(value) - if not formatted_value: - continue - getattr(instance, model_key).save(*formatted_value, save=True) - - for (model_key, values) in many_to_many_fields.items(): - # mention books, mention users - getattr(instance, model_key).set(values) - - # add one to many fields - for (model_key, values) in one_to_many_fields.items(): - if values == MISSING: - continue - model_field = getattr(instance, model_key) - model = model_field.model - for item in values: - item = model.activity_serializer(**item) - field_name = instance.__class__.__name__.lower() - with transaction.atomic(): - item = item.to_model(model) - setattr(item, field_name, instance) - item.save() + try: + # this is for one to many + related_model = getattr(model, model_field_name).field.model + except AttributeError: + # it's a one to one or foreign key + related_model = getattr(model, model_field_name)\ + .related.related_model + values = [values] + for item in values: + set_related_field.delay( + related_model.__name__, + instance.__class__.__name__, + instance.__class__.__name__.lower(), + instance.remote_id, + item + ) return instance @@ -168,66 +130,57 @@ class ActivityObject: return data -def resolve_foreign_key(model, remote_id): - ''' look up the remote_id on an activity json field ''' - if model in [models.Edition, models.Work, models.Book]: - return books_manager.get_or_create_book(remote_id) +@app.task +@transaction.atomic +def set_related_field( + model_name, origin_model_name, + related_field_name, related_remote_id, data): + ''' load reverse related fields (editions, attachments) without blocking ''' + model = apps.get_model('bookwyrm.%s' % model_name, require_ready=True) + origin_model = apps.get_model( + 'bookwyrm.%s' % origin_model_name, + require_ready=True + ) - result = model.objects - if hasattr(model.objects, 'select_subclasses'): - result = result.select_subclasses() - - result = result.filter( - remote_id=remote_id - ).first() - - if not result: - raise ActivitySerializerError( - 'Could not resolve remote_id in %s model: %s' % \ - (model.__name__, remote_id)) - return result - - -def tag_formatter(tags, tag_type): - ''' helper function to extract foreign keys from tag activity json ''' - if not isinstance(tags, list): - return [] - items = [] - types = { - 'Book': models.Book, - 'Mention': models.User, - } - for tag in [t for t in tags if t.get('type') == tag_type]: - if not tag_type in types: - continue - remote_id = tag.get('href') - try: - item = resolve_foreign_key(types[tag_type], remote_id) - except ActivitySerializerError: - continue - items.append(item) - return items - - -def image_formatter(image_slug): - ''' helper function to load images and format them for a model ''' - # when it's an inline image (User avatar/icon, Book cover), it's a json - # blob, but when it's an attached image, it's just a url - if isinstance(image_slug, dict): - url = image_slug.get('url') - elif isinstance(image_slug, str): - url = image_slug + if isinstance(data, str): + item = resolve_remote_id(model, data, save=False) else: - return None - if not url: - return None - try: - response = requests.get(url) - except ConnectionError: - return None - if not response.ok: - return None + # look for a match based on all the available data + item = model.find_existing(data) + if not item: + # create a new model instance + item = model.activity_serializer(**data) + item = item.to_model(model, save=False) + # this must exist because it's the object that triggered this function + instance = origin_model.find_existing_by_remote_id(related_remote_id) + if not instance: + raise ValueError('Invalid related remote id: %s' % related_remote_id) - image_name = str(uuid4()) + '.' + url.split('.')[-1] - image_content = ContentFile(response.content) - return [image_name, image_content] + # edition.parent_work = instance, for example + setattr(item, related_field_name, instance) + item.save() + + +def resolve_remote_id(model, remote_id, refresh=False, save=True): + ''' take a remote_id and return an instance, creating if necessary ''' + result = model.find_existing_by_remote_id(remote_id) + if result and not refresh: + return result + + # load the data and create the object + try: + data = get_data(remote_id) + except (ConnectorException, ConnectionError): + raise ActivitySerializerError( + 'Could not connect to host for remote_id in %s model: %s' % \ + (model.__name__, remote_id)) + + # check for existing items with shared unique identifiers + if not result: + result = model.find_existing(data) + if result and not refresh: + return result + + item = model.activity_serializer(**data) + # if we're refreshing, "result" will be set and we'll update it + return item.to_model(model, instance=result, save=save) diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index 02cab281..ae9c334d 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -12,13 +12,13 @@ class Book(ActivityObject): sortTitle: str = '' subtitle: str = '' description: str = '' - languages: List[str] + languages: List[str] = field(default_factory=lambda: []) series: str = '' seriesNumber: str = '' - subjects: List[str] - subjectPlaces: List[str] + subjects: List[str] = field(default_factory=lambda: []) + subjectPlaces: List[str] = field(default_factory=lambda: []) - authors: List[str] + authors: List[str] = field(default_factory=lambda: []) firstPublishedDate: str = '' publishedDate: str = '' @@ -33,22 +33,23 @@ class Book(ActivityObject): @dataclass(init=False) class Edition(Book): ''' Edition instance of a book object ''' - isbn10: str - isbn13: str - oclcNumber: str - asin: str - pages: str - physicalFormat: str - publishers: List[str] - work: str + isbn10: str = '' + isbn13: str = '' + oclcNumber: str = '' + asin: str = '' + pages: str = '' + physicalFormat: str = '' + publishers: List[str] = field(default_factory=lambda: []) + type: str = 'Edition' @dataclass(init=False) class Work(Book): ''' work instance of a book object ''' - lccn: str + lccn: str = '' + defaultEdition: str = '' editions: List[str] type: str = 'Work' diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py index aeb078dc..df28bf8d 100644 --- a/bookwyrm/activitypub/note.py +++ b/bookwyrm/activitypub/note.py @@ -8,7 +8,6 @@ from .image import Image @dataclass(init=False) class Tombstone(ActivityObject): ''' the placeholder for a deleted status ''' - url: str published: str deleted: str type: str = 'Tombstone' @@ -17,14 +16,13 @@ class Tombstone(ActivityObject): @dataclass(init=False) class Note(ActivityObject): ''' Note activity ''' - url: str - inReplyTo: str published: str attributedTo: str - to: List[str] - cc: List[str] content: str - replies: Dict + to: List[str] = field(default_factory=lambda: []) + cc: List[str] = field(default_factory=lambda: []) + replies: Dict = field(default_factory=lambda: {}) + inReplyTo: str = '' tag: List[Link] = field(default_factory=lambda: []) attachment: List[Image] = field(default_factory=lambda: []) sensitive: bool = False diff --git a/bookwyrm/activitypub/ordered_collection.py b/bookwyrm/activitypub/ordered_collection.py index efd23d5a..9aeaf664 100644 --- a/bookwyrm/activitypub/ordered_collection.py +++ b/bookwyrm/activitypub/ordered_collection.py @@ -12,6 +12,7 @@ class OrderedCollection(ActivityObject): first: str last: str = '' name: str = '' + owner: str = '' type: str = 'OrderedCollection' diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py index e7d720ec..88349c02 100644 --- a/bookwyrm/activitypub/person.py +++ b/bookwyrm/activitypub/person.py @@ -2,9 +2,18 @@ from dataclasses import dataclass, field from typing import Dict -from .base_activity import ActivityObject, PublicKey +from .base_activity import ActivityObject from .image import Image + +@dataclass(init=False) +class PublicKey(ActivityObject): + ''' public key block ''' + owner: str + publicKeyPem: str + type: str = 'PublicKey' + + @dataclass(init=False) class Person(ActivityObject): ''' actor activitypub json ''' diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py index eb166260..e890d81f 100644 --- a/bookwyrm/activitypub/verbs.py +++ b/bookwyrm/activitypub/verbs.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from typing import List from .base_activity import ActivityObject, Signature +from .book import Book @dataclass(init=False) class Verb(ActivityObject): @@ -69,6 +70,13 @@ class Add(Verb): type: str = 'Add' +@dataclass(init=False) +class AddBook(Verb): + '''Add activity that's aware of the book obj ''' + target: Book + type: str = 'Add' + + @dataclass(init=False) class Remove(Verb): '''Remove activity ''' diff --git a/bookwyrm/books_manager.py b/bookwyrm/books_manager.py index 461017a0..3b865768 100644 --- a/bookwyrm/books_manager.py +++ b/bookwyrm/books_manager.py @@ -16,23 +16,6 @@ def get_edition(book_id): return book -def get_or_create_book(remote_id): - ''' pull up a book record by whatever means possible ''' - book = models.Book.objects.select_subclasses().filter( - remote_id=remote_id - ).first() - if book: - return book - - connector = get_or_create_connector(remote_id) - - # raises ConnectorException - book = connector.get_or_create_book(remote_id) - if book: - load_more_data.delay(book.id) - return book - - def get_or_create_connector(remote_id): ''' get the connector related to the author's server ''' url = urlparse(remote_id) @@ -102,12 +85,6 @@ def first_search_result(query, min_confidence=0.1): return None -def update_book(book, data=None): - ''' re-sync with the original data source ''' - connector = load_connector(book.connector) - connector.update_book(book, data=data) - - def get_connectors(): ''' load all connectors ''' for info in models.Connector.objects.order_by('priority').all(): diff --git a/bookwyrm/broadcast.py b/bookwyrm/broadcast.py index a1eebaee..a98b6774 100644 --- a/bookwyrm/broadcast.py +++ b/bookwyrm/broadcast.py @@ -65,7 +65,7 @@ def sign_and_send(sender, data, destination): ''' crpyto whatever and http junk ''' now = http_date() - if not sender.private_key: + if not sender.key_pair.private_key: # this shouldn't happen. it would be bad if it happened. raise ValueError('No private key found for sender') diff --git a/bookwyrm/connectors/__init__.py b/bookwyrm/connectors/__init__.py index b5d93b47..4eb91de4 100644 --- a/bookwyrm/connectors/__init__.py +++ b/bookwyrm/connectors/__init__.py @@ -1,3 +1,4 @@ ''' bring connectors into the namespace ''' from .settings import CONNECTORS from .abstract_connector import ConnectorException +from .abstract_connector import get_data, get_image diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index d709b075..c9f1ad2e 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -8,6 +8,7 @@ from django.db import transaction from dateutil import parser import requests from requests import HTTPError +from requests.exceptions import SSLError from bookwyrm import models @@ -16,20 +17,13 @@ class ConnectorException(HTTPError): ''' when the connector can't do what was asked ''' -class AbstractConnector(ABC): - ''' generic book data connector ''' - +class AbstractMinimalConnector(ABC): + ''' just the bare bones, for other bookwyrm instances ''' def __init__(self, identifier): # load connector settings info = models.Connector.objects.get(identifier=identifier) self.connector = info - self.key_mappings = [] - - # fields we want to look for in book data to copy over - # title we handle separately. - self.book_mappings = [] - # the things in the connector model to copy over self_fields = [ 'base_url', @@ -44,15 +38,6 @@ class AbstractConnector(ABC): for field in self_fields: setattr(self, field, getattr(info, field)) - - def is_available(self): - ''' check if you're allowed to use this connector ''' - if self.max_query_count is not None: - if self.connector.query_count >= self.max_query_count: - return False - return True - - def search(self, query, min_confidence=None): ''' free text search ''' resp = requests.get( @@ -70,9 +55,40 @@ class AbstractConnector(ABC): results.append(self.format_search_result(doc)) return results - + @abstractmethod def get_or_create_book(self, remote_id): ''' pull up a book record by whatever means possible ''' + + @abstractmethod + def parse_search_data(self, data): + ''' turn the result json from a search into a list ''' + + @abstractmethod + def format_search_result(self, search_result): + ''' create a SearchResult obj from json ''' + + +class AbstractConnector(AbstractMinimalConnector): + ''' generic book data connector ''' + def __init__(self, identifier): + super().__init__(identifier) + + self.key_mappings = [] + + # fields we want to look for in book data to copy over + # title we handle separately. + self.book_mappings = [] + + + def is_available(self): + ''' check if you're allowed to use this connector ''' + if self.max_query_count is not None: + if self.connector.query_count >= self.max_query_count: + return False + return True + + + def get_or_create_book(self, remote_id): # try to load the book book = models.Book.objects.select_subclasses().filter( origin_id=remote_id @@ -157,13 +173,12 @@ class AbstractConnector(ABC): def update_book_from_data(self, book, data, update_cover=True): ''' for creating a new book or syncing with data ''' - book = self.update_from_mappings(book, data, self.book_mappings) + book = update_from_mappings(book, data, self.book_mappings) author_text = [] for author in self.get_authors_from_data(data): book.authors.add(author) - if author.display_name: - author_text.append(author.display_name) + author_text.append(author.name) book.author_text = ', '.join(author_text) book.save() @@ -246,39 +261,28 @@ class AbstractConnector(ABC): def get_cover_from_data(self, data): ''' load cover ''' - - @abstractmethod - def parse_search_data(self, data): - ''' turn the result json from a search into a list ''' - - - @abstractmethod - def format_search_result(self, search_result): - ''' create a SearchResult obj from json ''' - - @abstractmethod def expand_book_data(self, book): ''' get more info on a book ''' - def update_from_mappings(self, obj, data, mappings): - ''' assign data to model with mappings ''' - for mapping in mappings: - # check if this field is present in the data - value = data.get(mapping.remote_field) - if not value: - continue +def update_from_mappings(obj, data, mappings): + ''' assign data to model with mappings ''' + for mapping in mappings: + # check if this field is present in the data + value = data.get(mapping.remote_field) + if not value: + continue - # extract the value in the right format - try: - value = mapping.formatter(value) - except: - continue + # extract the value in the right format + try: + value = mapping.formatter(value) + except: + continue - # assign the formatted value to the model - obj.__setattr__(mapping.local_field, value) - return obj + # assign the formatted value to the model + obj.__setattr__(mapping.local_field, value) + return obj def get_date(date_string): @@ -310,10 +314,25 @@ def get_data(url): raise ConnectorException() if not resp.ok: resp.raise_for_status() - data = resp.json() + try: + data = resp.json() + except ValueError: + raise ConnectorException() + return data +def get_image(url): + ''' wrapper for requesting an image ''' + try: + resp = requests.get(url) + except (RequestError, SSLError): + return None + if not resp.ok: + return None + return resp + + @dataclass class SearchResult: ''' standardized search result object ''' diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 1bc81450..e4d32fd3 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -1,83 +1,16 @@ ''' using another bookwyrm instance as a source of book data ''' -from django.db import transaction - from bookwyrm import activitypub, models -from .abstract_connector import AbstractConnector, SearchResult -from .abstract_connector import get_data +from .abstract_connector import AbstractMinimalConnector, SearchResult -class Connector(AbstractConnector): - ''' interact with other instances ''' - - def update_from_mappings(self, obj, data, mappings): - ''' serialize book data into a model ''' - if self.is_work_data(data): - work_data = activitypub.Work(**data) - return work_data.to_model(models.Work, instance=obj) - edition_data = activitypub.Edition(**data) - return edition_data.to_model(models.Edition, instance=obj) - - - def get_remote_id_from_data(self, data): - return data.get('id') - - - def is_work_data(self, data): - return data.get('type') == 'Work' - - - def get_edition_from_work_data(self, data): - ''' we're served a list of edition urls ''' - path = data['editions'][0] - return get_data(path) - - - def get_work_from_edition_date(self, data): - return get_data(data['work']) - - - def get_authors_from_data(self, data): - ''' load author data ''' - for author_id in data.get('authors', []): - try: - yield models.Author.objects.get(origin_id=author_id) - except models.Author.DoesNotExist: - pass - data = get_data(author_id) - author_data = activitypub.Author(**data) - author = author_data.to_model(models.Author) - yield author - - - def get_cover_from_data(self, data): - pass +class Connector(AbstractMinimalConnector): + ''' this is basically just for search ''' + def get_or_create_book(self, remote_id): + return activitypub.resolve_remote_id(models.Edition, remote_id) def parse_search_data(self, data): return data - def format_search_result(self, search_result): return SearchResult(**search_result) - - - def expand_book_data(self, book): - work = book - # go from the edition to the work, if necessary - if isinstance(book, models.Edition): - work = book.parent_work - - # it may be that we actually want to request this url - editions_url = '%s/editions?page=true' % work.remote_id - edition_options = get_data(editions_url) - for edition_data in edition_options['orderedItems']: - with transaction.atomic(): - edition = self.create_book( - edition_data['id'], - edition_data, - models.Edition - ) - edition.parent_work = work - edition.save() - if not edition.authors.exists() and work.authors.exists(): - edition.authors.set(work.authors.all()) diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 5e18616d..28eb1ea0 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -7,7 +7,7 @@ from django.core.files.base import ContentFile from bookwyrm import models from .abstract_connector import AbstractConnector, SearchResult, Mapping from .abstract_connector import ConnectorException -from .abstract_connector import get_date, get_data +from .abstract_connector import get_date, get_data, update_from_mappings from .openlibrary_languages import languages @@ -65,6 +65,7 @@ class Connector(AbstractConnector): ] self.author_mappings = [ + Mapping('name'), Mapping('born', remote_field='birth_date', formatter=get_date), Mapping('died', remote_field='death_date', formatter=get_date), Mapping('bio', formatter=get_description), @@ -184,12 +185,7 @@ class Connector(AbstractConnector): data = get_data(url) author = models.Author(openlibrary_key=olkey) - author = self.update_from_mappings(author, data, self.author_mappings) - name = data.get('name') - # TODO this is making some BOLD assumption - if name: - author.last_name = name.split(' ')[-1] - author.first_name = ' '.join(name.split(' ')[:-1]) + author = update_from_mappings(author, data, self.author_mappings) author.save() return author diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py index 72839dce..a1471ac4 100644 --- a/bookwyrm/context_processors.py +++ b/bookwyrm/context_processors.py @@ -1,7 +1,7 @@ ''' customize the info available in context for rendering templates ''' from bookwyrm import models -def site_settings(request): +def site_settings(request):# pylint: disable=unused-argument ''' include the custom info about the site ''' return { 'site': models.SiteSettings.objects.get() diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py index 784f1038..a2c3e24b 100644 --- a/bookwyrm/forms.py +++ b/bookwyrm/forms.py @@ -30,7 +30,7 @@ class CustomForm(ModelForm): visible.field.widget.attrs['rows'] = None visible.field.widget.attrs['class'] = css_classes[input_type] - +# pylint: disable=missing-class-docstring class LoginForm(CustomForm): class Meta: model = models.User @@ -131,6 +131,7 @@ class ImportForm(forms.Form): class ExpiryWidget(widgets.Select): def value_from_datadict(self, data, files, name): + ''' human-readable exiration time buckets ''' selected_string = super().value_from_datadict(data, files, name) if selected_string == 'day': diff --git a/bookwyrm/goodreads_import.py b/bookwyrm/goodreads_import.py index 3fd330ab..93fc1c48 100644 --- a/bookwyrm/goodreads_import.py +++ b/bookwyrm/goodreads_import.py @@ -53,7 +53,7 @@ def import_data(job_id): for item in job.items.all(): try: item.resolve() - except Exception as e: + except Exception as e:# pylint: disable=broad-except logger.exception(e) item.fail_reason = 'Error loading book' item.save() diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 0e7c1856..fe521772 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -1,6 +1,6 @@ ''' handles all of the activity coming in to the server ''' import json -from urllib.parse import urldefrag, unquote_plus +from urllib.parse import urldefrag import django.db.utils from django.http import HttpResponse @@ -8,9 +8,8 @@ from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.views.decorators.csrf import csrf_exempt import requests -from bookwyrm import activitypub, books_manager, models, outgoing +from bookwyrm import activitypub, models, outgoing from bookwyrm import status as status_builder -from bookwyrm.remote_user import get_or_create_remote_user, refresh_remote_user from bookwyrm.tasks import app from bookwyrm.signatures import Signature @@ -18,9 +17,6 @@ from bookwyrm.signatures import Signature @csrf_exempt def inbox(request, username): ''' incoming activitypub events ''' - # TODO: should do some kind of checking if the user accepts - # this action from the sender probably? idk - # but this will just throw a 404 if the user doesn't exist try: models.User.objects.get(localname=username) except models.User.DoesNotExist: @@ -60,9 +56,8 @@ def shared_inbox(request): 'Like': handle_favorite, 'Announce': handle_boost, 'Add': { - 'Tag': handle_tag, - 'Edition': handle_shelve, - 'Work': handle_shelve, + 'Edition': handle_add, + 'Work': handle_add, }, 'Undo': { 'Follow': handle_unfollow, @@ -97,16 +92,20 @@ def has_valid_signature(request, activity): if key_actor != activity.get('actor'): raise ValueError("Wrong actor created signature.") - remote_user = get_or_create_remote_user(key_actor) + remote_user = activitypub.resolve_remote_id(models.User, key_actor) + if not remote_user: + return False try: - signature.verify(remote_user.public_key, request) + signature.verify(remote_user.key_pair.public_key, request) except ValueError: - old_key = remote_user.public_key - refresh_remote_user(remote_user) - if remote_user.public_key == old_key: + old_key = remote_user.key_pair.public_key + remote_user = activitypub.resolve_remote_id( + models.User, remote_user.remote_id, refresh=True + ) + if remote_user.key_pair.public_key == old_key: raise # Key unchanged. - signature.verify(remote_user.public_key, request) + signature.verify(remote_user.key_pair.public_key, request) except (ValueError, requests.exceptions.HTTPError): return False return True @@ -115,26 +114,10 @@ def has_valid_signature(request, activity): @app.task def handle_follow(activity): ''' someone wants to follow a local user ''' - # figure out who they want to follow -- not using get_or_create because - # we only care if you want to follow local users try: - to_follow = models.User.objects.get(remote_id=activity['object']) - except models.User.DoesNotExist: - # some rando, who cares - return - if not to_follow.local: - # just ignore follow alerts about other servers. maybe they should be - # handled. maybe they shouldn't be sent at all. - return - - # figure out who the actor is - actor = get_or_create_remote_user(activity['actor']) - try: - relationship = models.UserFollowRequest.objects.create( - user_subject=actor, - user_object=to_follow, - remote_id=activity['id'] - ) + relationship = activitypub.Follow( + **activity + ).to_model(models.UserFollowRequest) except django.db.utils.IntegrityError as err: if err.__cause__.diag.constraint_name != 'userfollowrequest_unique': raise @@ -143,27 +126,22 @@ def handle_follow(activity): ) # send the accept normally for a duplicate request - if not to_follow.manually_approves_followers: - status_builder.create_notification( - to_follow, - 'FOLLOW', - related_user=actor - ) + manually_approves = relationship.user_object.manually_approves_followers + + status_builder.create_notification( + relationship.user_object, + 'FOLLOW_REQUEST' if manually_approves else 'FOLLOW', + related_user=relationship.user_subject + ) + if not manually_approves: outgoing.handle_accept(relationship) - else: - # Accept will be triggered manually - status_builder.create_notification( - to_follow, - 'FOLLOW_REQUEST', - related_user=actor - ) @app.task def handle_unfollow(activity): ''' unfollow a local user ''' obj = activity['object'] - requester = get_or_create_remote_user(obj['actor']) + requester = activitypub.resolve_remote_id(models.user, obj['actor']) to_unfollow = models.User.objects.get(remote_id=obj['object']) # raises models.User.DoesNotExist @@ -176,7 +154,7 @@ def handle_follow_accept(activity): # figure out who they want to follow requester = models.User.objects.get(remote_id=activity['object']['actor']) # figure out who they are - accepter = get_or_create_remote_user(activity['actor']) + accepter = activitypub.resolve_remote_id(models.User, activity['actor']) try: request = models.UserFollowRequest.objects.get( @@ -193,7 +171,7 @@ def handle_follow_accept(activity): def handle_follow_reject(activity): ''' someone is rejecting a follow request ''' requester = models.User.objects.get(remote_id=activity['object']['actor']) - rejecter = get_or_create_remote_user(activity['actor']) + rejecter = activitypub.resolve_remote_id(models.User, activity['actor']) request = models.UserFollowRequest.objects.get( user_subject=requester, @@ -206,25 +184,40 @@ def handle_follow_reject(activity): @app.task def handle_create(activity): ''' someone did something, good on them ''' - if activity['object'].get('type') not in \ - ['Note', 'Comment', 'Quotation', 'Review', 'GeneratedNote']: - # if it's an article or unknown type, ignore it - return - - user = get_or_create_remote_user(activity['actor']) - if user.local: - # we really oughtn't even be sending in this case - return - # deduplicate incoming activities - status_id = activity['object']['id'] + activity = activity['object'] + status_id = activity['id'] if models.Status.objects.filter(remote_id=status_id).count(): return - status = status_builder.create_status(activity['object']) - if not status: + serializer = activitypub.activity_objects[activity['type']] + activity = serializer(**activity) + try: + model = models.activity_models[activity.type] + except KeyError: + # not a type of status we are prepared to deserialize return + if activity.type == 'Note': + # keep notes if they are replies to existing statuses + reply = models.Status.objects.filter( + remote_id=activity.inReplyTo + ).first() + + if not reply: + discard = True + # keep notes if they mention local users + tags = [l['href'] for l in activity.tag if l['type'] == 'Mention'] + for tag in tags: + if models.User.objects.filter( + remote_id=tag, local=True).exists(): + # we found a mention of a known use boost + discard = False + break + if discard: + return + + status = activity.to_model(model) # create a notification if this is a reply if status.reply_parent and status.reply_parent.user.local: status_builder.create_notification( @@ -258,16 +251,14 @@ def handle_favorite(activity): ''' approval of your good good post ''' fav = activitypub.Like(**activity) - liker = get_or_create_remote_user(activity['actor']) - if liker.local: - return - fav = fav.to_model(models.Favorite) + if fav.user.local: + return status_builder.create_notification( fav.status.user, 'FAVORITE', - related_user=liker, + related_user=fav.user, related_status=fav.status, ) @@ -312,35 +303,13 @@ def handle_unboost(activity): @app.task -def handle_tag(activity): - ''' someone is tagging a book ''' - user = get_or_create_remote_user(activity['actor']) - if not user.local: - # ordered collection weirndess so we can't just to_model - book = books_manager.get_or_create_book(activity['object']['id']) - name = activity['object']['target'].split('/')[-1] - name = unquote_plus(name) - models.Tag.objects.get_or_create( - user=user, - book=book, - name=name - ) - - -@app.task -def handle_shelve(activity): +def handle_add(activity): ''' putting a book on a shelf ''' - user = get_or_create_remote_user(activity['actor']) - book = books_manager.get_or_create_book(activity['object']) + #this is janky as heck but I haven't thought of a better solution try: - shelf = models.Shelf.objects.get(remote_id=activity['target']) - except models.Shelf.DoesNotExist: - return - if shelf.user != user: - # this doesn't add up. - return - shelf.books.add(book) - shelf.save() + activitypub.AddBook(**activity).to_model(models.ShelfBook) + except activitypub.ActivitySerializerError: + activitypub.AddBook(**activity).to_model(models.Tag) @app.task @@ -360,13 +329,4 @@ def handle_update_user(activity): @app.task def handle_update_book(activity): ''' a remote instance changed a book (Document) ''' - document = activity['object'] - # check if we have their copy and care about their updates - book = models.Book.objects.select_subclasses().filter( - remote_id=document['id'], - sync=True, - ).first() - if not book: - return - - books_manager.update_book(book, data=document) + activitypub.Edition(**activity['object']).to_model(models.Edition) diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py index f29ed102..9fd11787 100644 --- a/bookwyrm/management/commands/initdb.py +++ b/bookwyrm/management/commands/initdb.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import Group, Permission from django.contrib.contenttypes.models import ContentType -from bookwyrm.models import Connector, User +from bookwyrm.models import Connector, SiteSettings, User from bookwyrm.settings import DOMAIN def init_groups(): @@ -73,7 +73,7 @@ def init_connectors(): identifier='bookwyrm.social', name='BookWyrm dot Social', connector_file='bookwyrm_connector', - base_url='https://bookwyrm.social' , + base_url='https://bookwyrm.social', books_url='https://bookwyrm.social/book', covers_url='https://bookwyrm.social/images/covers', search_url='https://bookwyrm.social/search?q=', @@ -91,10 +91,14 @@ def init_connectors(): priority=3, ) +def init_settings(): + SiteSettings.objects.create() + class Command(BaseCommand): help = 'Initializes the database with starter data' def handle(self, *args, **options): init_groups() init_permissions() - init_connectors() \ No newline at end of file + init_connectors() + init_settings() diff --git a/bookwyrm/migrations/0001_initial.py b/bookwyrm/migrations/0001_initial.py index b1aba7df..347057e1 100644 --- a/bookwyrm/migrations/0001_initial.py +++ b/bookwyrm/migrations/0001_initial.py @@ -7,7 +7,7 @@ import django.core.validators from django.db import migrations, models import django.db.models.deletion import django.utils.timezone -import bookwyrm.utils.fields +from django.contrib.postgres.fields import JSONField class Migration(migrations.Migration): @@ -62,7 +62,7 @@ class Migration(migrations.Migration): ('content', models.TextField(blank=True, null=True)), ('created_date', models.DateTimeField(auto_now_add=True)), ('openlibrary_key', models.CharField(max_length=255)), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ], options={ 'abstract': False, @@ -75,7 +75,7 @@ class Migration(migrations.Migration): ('content', models.TextField(blank=True, null=True)), ('created_date', models.DateTimeField(auto_now_add=True)), ('openlibrary_key', models.CharField(max_length=255, unique=True)), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ('cover', models.ImageField(blank=True, null=True, upload_to='covers/')), ('added_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), ('authors', models.ManyToManyField(to='bookwyrm.Author')), diff --git a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py index 13cb1406..6a149ab5 100644 --- a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py +++ b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py @@ -2,7 +2,6 @@ import bookwyrm.models.connector import bookwyrm.models.site -import bookwyrm.utils.fields from django.conf import settings import django.contrib.postgres.operations import django.core.validators @@ -10,6 +9,7 @@ from django.db import migrations, models import django.db.models.deletion import django.db.models.expressions import django.utils.timezone +from django.contrib.postgres.fields import JSONField, ArrayField import uuid @@ -148,7 +148,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='misc_identifiers', - field=bookwyrm.utils.fields.JSONField(null=True), + field=JSONField(null=True), ), migrations.AddField( model_name='book', @@ -226,7 +226,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='author', name='aliases', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='user', @@ -394,17 +394,17 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='subject_places', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='book', name='subjects', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='edition', name='publishers', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AlterField( model_name='connector', @@ -578,7 +578,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='languages', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='edition', @@ -676,7 +676,7 @@ class Migration(migrations.Migration): name='ImportItem', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ], ), migrations.CreateModel( diff --git a/bookwyrm/migrations/0016_auto_20201129_0304.py b/bookwyrm/migrations/0016_auto_20201129_0304.py new file mode 100644 index 00000000..1e715969 --- /dev/null +++ b/bookwyrm/migrations/0016_auto_20201129_0304.py @@ -0,0 +1,62 @@ +# Generated by Django 3.0.7 on 2020-11-29 03:04 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +from django.contrib.postgres.fields import ArrayField + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0015_auto_20201128_0349'), + ] + + operations = [ + migrations.AlterField( + model_name='book', + name='subject_places', + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subjects', + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='edition', + name='parent_work', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'), + ), + migrations.AlterField( + model_name='tag', + name='name', + field=models.CharField(max_length=100, unique=True), + ), + migrations.AlterUniqueTogether( + name='tag', + unique_together=set(), + ), + migrations.RemoveField( + model_name='tag', + name='book', + ), + migrations.RemoveField( + model_name='tag', + name='user', + ), + migrations.CreateModel( + name='UserTag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_date', models.DateTimeField(auto_now_add=True)), + ('updated_date', models.DateTimeField(auto_now=True)), + ('remote_id', models.CharField(max_length=255, null=True)), + ('book', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition')), + ('tag', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'book', 'tag')}, + }, + ), + ] diff --git a/bookwyrm/migrations/0017_auto_20201130_1819.py b/bookwyrm/migrations/0017_auto_20201130_1819.py new file mode 100644 index 00000000..ce9f1cc7 --- /dev/null +++ b/bookwyrm/migrations/0017_auto_20201130_1819.py @@ -0,0 +1,189 @@ +# Generated by Django 3.0.7 on 2020-11-30 18:19 + +import bookwyrm.models.base_model +import bookwyrm.models.fields +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + +def copy_rsa_keys(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + users = app_registry.get_model('bookwyrm', 'User') + keypair = app_registry.get_model('bookwyrm', 'KeyPair') + for user in users.objects.using(db_alias): + if user.public_key or user.private_key: + user.key_pair = keypair.objects.create( + remote_id='%s/#main-key' % user.remote_id, + private_key=user.private_key, + public_key=user.public_key + ) + user.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0016_auto_20201129_0304'), + ] + operations = [ + migrations.CreateModel( + name='KeyPair', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_date', models.DateTimeField(auto_now_add=True)), + ('updated_date', models.DateTimeField(auto_now=True)), + ('remote_id', bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id])), + ('private_key', models.TextField(blank=True, null=True)), + ('public_key', bookwyrm.models.fields.TextField(blank=True, null=True)), + ], + options={ + 'abstract': False, + }, + bases=(bookwyrm.models.base_model.ActivitypubMixin, models.Model), + ), + migrations.AddField( + model_name='user', + name='followers', + field=bookwyrm.models.fields.ManyToManyField(related_name='following', through='bookwyrm.UserFollows', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='author', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='book', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='connector', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='favorite', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='federatedserver', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='image', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='notification', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='readthrough', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='shelf', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='shelfbook', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='status', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='tag', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='avatar', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='avatars/'), + ), + migrations.AlterField( + model_name='user', + name='bookwyrm_user', + field=bookwyrm.models.fields.BooleanField(default=True), + ), + migrations.AlterField( + model_name='user', + name='inbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='local', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='user', + name='manually_approves_followers', + field=bookwyrm.models.fields.BooleanField(default=False), + ), + migrations.AlterField( + model_name='user', + name='name', + field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='user', + name='outbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='shared_inbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='user', + name='username', + field=bookwyrm.models.fields.UsernameField(), + ), + migrations.AlterField( + model_name='userblocks', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='userfollows', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='usertag', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AddField( + model_name='user', + name='key_pair', + field=bookwyrm.models.fields.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='owner', to='bookwyrm.KeyPair'), + ), + migrations.RunPython(copy_rsa_keys), + ] diff --git a/bookwyrm/migrations/0018_auto_20201130_1832.py b/bookwyrm/migrations/0018_auto_20201130_1832.py new file mode 100644 index 00000000..278446cf --- /dev/null +++ b/bookwyrm/migrations/0018_auto_20201130_1832.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.7 on 2020-11-30 18:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0017_auto_20201130_1819'), + ] + + operations = [ + migrations.RemoveField( + model_name='user', + name='following', + ), + migrations.RemoveField( + model_name='user', + name='private_key', + ), + migrations.RemoveField( + model_name='user', + name='public_key', + ), + ] diff --git a/bookwyrm/migrations/0019_auto_20201130_1939.py b/bookwyrm/migrations/0019_auto_20201130_1939.py new file mode 100644 index 00000000..11cf6a3b --- /dev/null +++ b/bookwyrm/migrations/0019_auto_20201130_1939.py @@ -0,0 +1,36 @@ +# Generated by Django 3.0.7 on 2020-11-30 19:39 + +import bookwyrm.models.fields +from django.db import migrations + +def update_notnull(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + users = app_registry.get_model('bookwyrm', 'User') + for user in users.objects.using(db_alias): + if user.name and user.summary: + continue + if not user.summary: + user.summary = '' + if not user.name: + user.name = '' + user.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0018_auto_20201130_1832'), + ] + + operations = [ + migrations.RunPython(update_notnull), + migrations.AlterField( + model_name='user', + name='name', + field=bookwyrm.models.fields.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.TextField(default=''), + ), + ] diff --git a/bookwyrm/migrations/0020_auto_20201208_0213.py b/bookwyrm/migrations/0020_auto_20201208_0213.py new file mode 100644 index 00000000..9c5345c7 --- /dev/null +++ b/bookwyrm/migrations/0020_auto_20201208_0213.py @@ -0,0 +1,353 @@ +# Generated by Django 3.0.7 on 2020-12-08 02:13 + +import bookwyrm.models.fields +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0019_auto_20201130_1939'), + ] + + operations = [ + migrations.AlterField( + model_name='author', + name='aliases', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='author', + name='bio', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='born', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='died', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='name', + field=bookwyrm.models.fields.CharField(max_length=255), + ), + migrations.AlterField( + model_name='author', + name='openlibrary_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='author', + name='wikipedia_link', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='authors', + field=bookwyrm.models.fields.ManyToManyField(to='bookwyrm.Author'), + ), + migrations.AlterField( + model_name='book', + name='cover', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='covers/'), + ), + migrations.AlterField( + model_name='book', + name='description', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='first_published_date', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='goodreads_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='languages', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='book', + name='librarything_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='openlibrary_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='published_date', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='series', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='series_number', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='sort_title', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='subject_places', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subjects', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subtitle', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='title', + field=bookwyrm.models.fields.CharField(max_length=255), + ), + migrations.AlterField( + model_name='boost', + name='boosted_status', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='boosters', to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='comment', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='edition', + name='asin', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='isbn_10', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='isbn_13', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='oclc_number', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='pages', + field=bookwyrm.models.fields.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='edition', + name='parent_work', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'), + ), + migrations.AlterField( + model_name='edition', + name='physical_format', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='publishers', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='favorite', + name='status', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='favorite', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='image', + name='caption', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='image', + name='image', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='status/'), + ), + migrations.AlterField( + model_name='quotation', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='quotation', + name='quote', + field=bookwyrm.models.fields.TextField(), + ), + migrations.AlterField( + model_name='review', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='review', + name='name', + field=bookwyrm.models.fields.CharField(max_length=255, null=True), + ), + migrations.AlterField( + model_name='review', + name='rating', + field=bookwyrm.models.fields.IntegerField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(5)]), + ), + migrations.AlterField( + model_name='shelf', + name='name', + field=bookwyrm.models.fields.CharField(max_length=100), + ), + migrations.AlterField( + model_name='shelf', + name='privacy', + field=bookwyrm.models.fields.CharField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255), + ), + migrations.AlterField( + model_name='shelf', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='shelfbook', + name='added_by', + field=bookwyrm.models.fields.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='shelfbook', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='shelfbook', + name='shelf', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Shelf'), + ), + migrations.AlterField( + model_name='status', + name='content', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='status', + name='mention_books', + field=bookwyrm.models.fields.TagField(related_name='mention_book', to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='status', + name='mention_users', + field=bookwyrm.models.fields.TagField(related_name='mention_user', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='status', + name='published_date', + field=bookwyrm.models.fields.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AlterField( + model_name='status', + name='reply_parent', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='status', + name='sensitive', + field=bookwyrm.models.fields.BooleanField(default=False), + ), + migrations.AlterField( + model_name='status', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='tag', + name='name', + field=bookwyrm.models.fields.CharField(max_length=100, unique=True), + ), + migrations.AlterField( + model_name='userblocks', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userblocks', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollows', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollows', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='usertag', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='usertag', + name='tag', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag'), + ), + migrations.AlterField( + model_name='usertag', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='work', + name='default_edition', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='work', + name='lccn', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/migrations/0021_merge_20201212_1737.py b/bookwyrm/migrations/0021_merge_20201212_1737.py new file mode 100644 index 00000000..4ccf8c8c --- /dev/null +++ b/bookwyrm/migrations/0021_merge_20201212_1737.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-12-12 17:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0020_auto_20201208_0213'), + ('bookwyrm', '0016_auto_20201211_2026'), + ] + + operations = [ + ] diff --git a/bookwyrm/migrations/0022_auto_20201212_1744.py b/bookwyrm/migrations/0022_auto_20201212_1744.py new file mode 100644 index 00000000..0a98597f --- /dev/null +++ b/bookwyrm/migrations/0022_auto_20201212_1744.py @@ -0,0 +1,30 @@ +# Generated by Django 3.0.7 on 2020-12-12 17:44 + +from django.db import migrations + + +def set_author_name(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + authors = app_registry.get_model('bookwyrm', 'Author') + for author in authors.objects.using(db_alias): + if not author.name: + author.name = '%s %s' % (author.first_name, author.last_name) + author.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0021_merge_20201212_1737'), + ] + + operations = [ + migrations.RunPython(set_author_name), + migrations.RemoveField( + model_name='author', + name='first_name', + ), + migrations.RemoveField( + model_name='author', + name='last_name', + ), + ] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index 3d854478..86bdf219 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -12,9 +12,9 @@ from .status import Status, GeneratedNote, Review, Comment, Quotation from .status import Favorite, Boost, Notification, ReadThrough from .attachment import Image -from .tag import Tag +from .tag import Tag, UserTag -from .user import User +from .user import User, KeyPair from .relationship import UserFollows, UserFollowRequest, UserBlocks from .federated_server import FederatedServer diff --git a/bookwyrm/models/attachment.py b/bookwyrm/models/attachment.py index 7329e65d..b3337e15 100644 --- a/bookwyrm/models/attachment.py +++ b/bookwyrm/models/attachment.py @@ -3,7 +3,8 @@ from django.db import models from bookwyrm import activitypub from .base_model import ActivitypubMixin -from .base_model import ActivityMapping, BookWyrmModel +from .base_model import BookWyrmModel +from . import fields class Attachment(ActivitypubMixin, BookWyrmModel): @@ -14,19 +15,16 @@ class Attachment(ActivitypubMixin, BookWyrmModel): related_name='attachments', null=True ) + reverse_unfurl = True class Meta: ''' one day we'll have other types of attachments besides images ''' abstract = True - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('url', 'image'), - ActivityMapping('name', 'caption'), - ] class Image(Attachment): ''' an image attachment ''' - image = models.ImageField(upload_to='status/', null=True, blank=True) - caption = models.TextField(null=True, blank=True) + image = fields.ImageField( + upload_to='status/', null=True, blank=True, activitypub_field='url') + caption = fields.TextField(null=True, blank=True, activitypub_field='name') activity_serializer = activitypub.Image diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 1d701797..331d2dd6 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -3,48 +3,42 @@ from django.db import models from django.utils import timezone from bookwyrm import activitypub -from bookwyrm.utils.fields import ArrayField +from bookwyrm.settings import DOMAIN -from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel +from .base_model import ActivitypubMixin, BookWyrmModel +from . import fields class Author(ActivitypubMixin, BookWyrmModel): ''' basic biographic info ''' origin_id = models.CharField(max_length=255, null=True) - ''' copy of an author from OL ''' - openlibrary_key = models.CharField(max_length=255, blank=True, null=True) + openlibrary_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) sync = models.BooleanField(default=True) last_sync_date = models.DateTimeField(default=timezone.now) - wikipedia_link = models.CharField(max_length=255, blank=True, null=True) + wikipedia_link = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) # idk probably other keys would be useful here? - born = models.DateTimeField(blank=True, null=True) - died = models.DateTimeField(blank=True, null=True) - name = models.CharField(max_length=255) - last_name = models.CharField(max_length=255, blank=True, null=True) - first_name = models.CharField(max_length=255, blank=True, null=True) - aliases = ArrayField( + born = fields.DateTimeField(blank=True, null=True) + died = fields.DateTimeField(blank=True, null=True) + name = fields.CharField(max_length=255) + aliases = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) - bio = models.TextField(null=True, blank=True) + bio = fields.TextField(null=True, blank=True) - @property - def display_name(self): - ''' Helper to return a displayable name''' - if self.name: - return self.name - # don't want to return a spurious space if all of these are None - if self.first_name and self.last_name: - return self.first_name + ' ' + self.last_name - return self.last_name or self.first_name + def save(self, *args, **kwargs): + ''' can't be abstract for query reasons, but you shouldn't USE it ''' + if self.id and not self.remote_id: + self.remote_id = self.get_remote_id() + + if not self.id: + self.origin_id = self.remote_id + self.remote_id = None + return super().save(*args, **kwargs) + + def get_remote_id(self): + ''' editions and works both use "book" instead of model_name ''' + return 'https://%s/author/%s' % (DOMAIN, self.id) - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('name', 'name'), - ActivityMapping('born', 'born'), - ActivityMapping('died', 'died'), - ActivityMapping('aliases', 'aliases'), - ActivityMapping('bio', 'bio'), - ActivityMapping('openlibraryKey', 'openlibrary_key'), - ActivityMapping('wikipediaLink', 'wikipedia_link'), - ] activity_serializer = activitypub.Author diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 4109a49b..08cc6052 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -1,34 +1,27 @@ ''' base model with default fields ''' -from datetime import datetime from base64 import b64encode -from dataclasses import dataclass -from typing import Callable +from functools import reduce +import operator from uuid import uuid4 -from urllib.parse import urlencode from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 from Crypto.Hash import SHA256 +from django.core.paginator import Paginator from django.db import models -from django.db.models.fields.files import ImageFieldFile +from django.db.models import Q from django.dispatch import receiver from bookwyrm import activitypub -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import DOMAIN, PAGE_LENGTH +from .fields import ImageField, ManyToManyField, RemoteIdField -PrivacyLevels = models.TextChoices('Privacy', [ - 'public', - 'unlisted', - 'followers', - 'direct' -]) - class BookWyrmModel(models.Model): ''' shared fields ''' created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) - remote_id = models.CharField(max_length=255, null=True) + remote_id = RemoteIdField(null=True, activitypub_field='id') def get_remote_id(self): ''' generate a url that resolves to the local object ''' @@ -44,6 +37,7 @@ class BookWyrmModel(models.Model): @receiver(models.signals.post_save) +#pylint: disable=unused-argument def execute_after_save(sender, instance, created, *args, **kwargs): ''' set the remote_id after save (when the id is available) ''' if not created or not hasattr(instance, 'get_remote_id'): @@ -53,58 +47,115 @@ def execute_after_save(sender, instance, created, *args, **kwargs): instance.save() +def unfurl_related_field(related_field): + ''' load reverse lookups (like public key owner or Status attachment ''' + if hasattr(related_field, 'all'): + return [unfurl_related_field(i) for i in related_field.all()] + if related_field.reverse_unfurl: + return related_field.field_to_activity() + return related_field.remote_id + + class ActivitypubMixin: ''' add this mixin for models that are AP serializable ''' activity_serializer = lambda: {} + reverse_unfurl = False - def to_activity(self, pure=False): - ''' convert from a model to an activity ''' - if pure: - # works around bookwyrm-specific fields for vanilla AP services - mappings = self.pure_activity_mappings - else: - # may include custom fields that bookwyrm instances will understand - mappings = self.activity_mappings - - fields = {} - for mapping in mappings: - if not hasattr(self, mapping.model_key) or not mapping.activity_key: - # this field on the model isn't serialized + def __init__(self, *args, **kwargs): + ''' collect some info on model fields ''' + self.image_fields = [] + self.many_to_many_fields = [] + self.simple_fields = [] # "simple" + for field in self._meta.get_fields(): + if not hasattr(field, 'field_to_activity'): continue - value = getattr(self, mapping.model_key) - if hasattr(value, 'remote_id'): - # this is probably a foreign key field, which we want to - # serialize as just the remote_id url reference - value = value.remote_id - elif isinstance(value, datetime): - value = value.isoformat() - elif isinstance(value, ImageFieldFile): - value = image_formatter(value) - # run the custom formatter function set in the model - formatted_value = mapping.activity_formatter(value) - if mapping.activity_key in fields and \ - isinstance(fields[mapping.activity_key], list): - # there can be two database fields that map to the same AP list - # this happens in status tags, which combines user and book tags - fields[mapping.activity_key] += formatted_value + if isinstance(field, ImageField): + self.image_fields.append(field) + elif isinstance(field, ManyToManyField): + self.many_to_many_fields.append(field) else: - fields[mapping.activity_key] = formatted_value + self.simple_fields.append(field) - if pure: - return self.pure_activity_serializer( - **fields - ).serialize() - return self.activity_serializer( - **fields - ).serialize() + self.activity_fields = self.image_fields + \ + self.many_to_many_fields + self.simple_fields + + self.deserialize_reverse_fields = self.deserialize_reverse_fields \ + if hasattr(self, 'deserialize_reverse_fields') else [] + self.serialize_reverse_fields = self.serialize_reverse_fields \ + if hasattr(self, 'serialize_reverse_fields') else [] + + super().__init__(*args, **kwargs) - def to_create_activity(self, user, pure=False): + @classmethod + def find_existing_by_remote_id(cls, remote_id): + ''' look up a remote id in the db ''' + return cls.find_existing({'id': remote_id}) + + @classmethod + def find_existing(cls, data): + ''' compare data to fields that can be used for deduplation. + This always includes remote_id, but can also be unique identifiers + like an isbn for an edition ''' + filters = [] + for field in cls._meta.get_fields(): + if not hasattr(field, 'deduplication_field') or \ + not field.deduplication_field: + continue + + value = data.get(field.activitypub_field) + if not value: + continue + filters.append({field.name: value}) + + if hasattr(cls, 'origin_id') and 'id' in data: + # kinda janky, but this handles special case for books + filters.append({'origin_id': data['id']}) + + if not filters: + # if there are no deduplication fields, it will match the first + # item no matter what. this shouldn't happen but just in case. + return None + + objects = cls.objects + if hasattr(objects, 'select_subclasses'): + objects = objects.select_subclasses() + + # an OR operation on all the match fields + match = objects.filter( + reduce( + operator.or_, (Q(**f) for f in filters) + ) + ) + # there OUGHT to be only one match + return match.first() + + + def to_activity(self): + ''' convert from a model to an activity ''' + activity = {} + for field in self.activity_fields: + field.set_activity_from_field(activity, self) + + if hasattr(self, 'serialize_reverse_fields'): + # for example, editions of a work + for model_field_name, activity_field_name in \ + self.serialize_reverse_fields: + related_field = getattr(self, model_field_name) + activity[activity_field_name] = \ + unfurl_related_field(related_field) + + if not activity.get('id'): + activity['id'] = self.get_remote_id() + return self.activity_serializer(**activity).serialize() + + + def to_create_activity(self, user): ''' returns the object wrapped in a Create activity ''' - activity_object = self.to_activity(pure=pure) + activity_object = self.to_activity() - signer = pkcs1_15.new(RSA.import_key(user.private_key)) + signer = pkcs1_15.new(RSA.import_key(user.key_pair.private_key)) content = activity_object['content'] signed_message = signer.sign(SHA256.new(content.encode('utf8'))) create_id = self.remote_id + '/activity' @@ -118,8 +169,8 @@ class ActivitypubMixin: return activitypub.Create( id=create_id, actor=user.remote_id, - to=['%s/followers' % user.remote_id], - cc=['https://www.w3.org/ns/activitystreams#Public'], + to=activity_object['to'], + cc=activity_object['cc'], object=activity_object, signature=signature, ).serialize() @@ -127,21 +178,18 @@ class ActivitypubMixin: def to_delete_activity(self, user): ''' notice of deletion ''' - # this should be a tombstone - activity_object = self.to_activity() - return activitypub.Delete( id=self.remote_id + '/activity', actor=user.remote_id, to=['%s/followers' % user.remote_id], cc=['https://www.w3.org/ns/activitystreams#Public'], - object=activity_object, + object=self.to_activity(), ).serialize() def to_update_activity(self, user): ''' wrapper for Updates to an activity ''' - activity_id = '%s#update/%s' % (user.remote_id, uuid4()) + activity_id = '%s#update/%s' % (self.remote_id, uuid4()) return activitypub.Update( id=activity_id, actor=user.remote_id, @@ -153,10 +201,10 @@ class ActivitypubMixin: def to_undo_activity(self, user): ''' undo an action ''' return activitypub.Undo( - id='%s#undo' % user.remote_id, + id='%s#undo' % self.remote_id, actor=user.remote_id, object=self.to_activity() - ) + ).serialize() class OrderedCollectionPageMixin(ActivitypubMixin): @@ -167,77 +215,53 @@ class OrderedCollectionPageMixin(ActivitypubMixin): ''' this can be overriden if there's a special remote id, ie outbox ''' return self.remote_id - def page(self, min_id=None, max_id=None): - ''' helper function to create the pagination url ''' - params = {'page': 'true'} - if min_id: - params['min_id'] = min_id - if max_id: - params['max_id'] = max_id - return '?%s' % urlencode(params) - - def next_page(self, items): - ''' use the max id of the last item ''' - if not items.count(): - return '' - return self.page(max_id=items[items.count() - 1].id) - - def prev_page(self, items): - ''' use the min id of the first item ''' - if not items.count(): - return '' - return self.page(min_id=items[0].id) - - def to_ordered_collection_page(self, queryset, remote_id, \ - id_only=False, min_id=None, max_id=None): - ''' serialize and pagiante a queryset ''' - # TODO: weird place to define this - limit = 20 - # filters for use in the django queryset min/max - filters = {} - if min_id is not None: - filters['id__gt'] = min_id - if max_id is not None: - filters['id__lte'] = max_id - page_id = self.page(min_id=min_id, max_id=max_id) - - items = queryset.filter( - **filters - ).all()[:limit] - - if id_only: - page = [s.remote_id for s in items] - else: - page = [s.to_activity() for s in items] - return activitypub.OrderedCollectionPage( - id='%s%s' % (remote_id, page_id), - partOf=remote_id, - orderedItems=page, - next='%s%s' % (remote_id, self.next_page(items)), - prev='%s%s' % (remote_id, self.prev_page(items)) - ).serialize() def to_ordered_collection(self, queryset, \ remote_id=None, page=False, **kwargs): ''' an ordered collection of whatevers ''' remote_id = remote_id or self.remote_id if page: - return self.to_ordered_collection_page( + return to_ordered_collection_page( queryset, remote_id, **kwargs) - name = '' - if hasattr(self, 'name'): - name = self.name + name = self.name if hasattr(self, 'name') else None + owner = self.user.remote_id if hasattr(self, 'user') else '' - size = queryset.count() + paginated = Paginator(queryset, PAGE_LENGTH) return activitypub.OrderedCollection( id=remote_id, - totalItems=size, + totalItems=paginated.count, name=name, - first='%s%s' % (remote_id, self.page()), - last='%s%s' % (remote_id, self.page(min_id=0)) + owner=owner, + first='%s?page=1' % remote_id, + last='%s?page=%d' % (remote_id, paginated.num_pages) ).serialize() +def to_ordered_collection_page(queryset, remote_id, id_only=False, page=1): + ''' serialize and pagiante a queryset ''' + paginated = Paginator(queryset, PAGE_LENGTH) + + activity_page = paginated.page(page) + if id_only: + items = [s.remote_id for s in activity_page.object_list] + else: + items = [s.to_activity() for s in activity_page.object_list] + + prev_page = next_page = None + if activity_page.has_next(): + next_page = '%s?page=%d' % (remote_id, activity_page.next_page_number()) + if activity_page.has_previous(): + prev_page = '%s?page=%d' % \ + (remote_id, activity_page.previous_page_number()) + return activitypub.OrderedCollectionPage( + id='%s?page=%s' % (remote_id, page), + partOf=remote_id, + orderedItems=items, + next=next_page, + prev=prev_page + ).serialize() + + class OrderedCollectionMixin(OrderedCollectionPageMixin): ''' extends activitypub models to work as ordered collections ''' @property @@ -250,39 +274,3 @@ class OrderedCollectionMixin(OrderedCollectionPageMixin): def to_activity(self, **kwargs): ''' an ordered collection of the specified model queryset ''' return self.to_ordered_collection(self.collection_queryset, **kwargs) - - -@dataclass(frozen=True) -class ActivityMapping: - ''' translate between an activitypub json field and a model field ''' - activity_key: str - model_key: str - activity_formatter: Callable = lambda x: x - model_formatter: Callable = lambda x: x - - -def tag_formatter(items, name_field, activity_type): - ''' helper function to format lists of foreign keys into Tags ''' - tags = [] - for item in items.all(): - tags.append(activitypub.Link( - href=item.remote_id, - name=getattr(item, name_field), - type=activity_type - )) - return tags - - -def image_formatter(image): - ''' convert images into activitypub json ''' - if image and hasattr(image, 'url'): - url = image.url - else: - return None - url = 'https://%s%s' % (DOMAIN, url) - return activitypub.Image(url=url) - - -def image_attachments_formatter(images): - ''' create a list of image attachments ''' - return [image_formatter(i) for i in images] diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 132b4c07..bcd4bc04 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -2,24 +2,26 @@ import re from django.db import models -from django.db.models import Q from django.utils import timezone from model_utils.managers import InheritanceManager from bookwyrm import activitypub from bookwyrm.settings import DOMAIN -from bookwyrm.utils.fields import ArrayField -from .base_model import ActivityMapping, BookWyrmModel +from .base_model import BookWyrmModel from .base_model import ActivitypubMixin, OrderedCollectionPageMixin +from . import fields class Book(ActivitypubMixin, BookWyrmModel): ''' a generic book, which can mean either an edition or a work ''' origin_id = models.CharField(max_length=255, null=True, blank=True) # these identifiers apply to both works and editions - openlibrary_key = models.CharField(max_length=255, blank=True, null=True) - librarything_key = models.CharField(max_length=255, blank=True, null=True) - goodreads_key = models.CharField(max_length=255, blank=True, null=True) + openlibrary_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + librarything_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + goodreads_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) # info about where the data comes from and where/if to sync sync = models.BooleanField(default=True) @@ -31,78 +33,43 @@ class Book(ActivitypubMixin, BookWyrmModel): # TODO: edit history # book/work metadata - title = models.CharField(max_length=255) - sort_title = models.CharField(max_length=255, blank=True, null=True) - subtitle = models.CharField(max_length=255, blank=True, null=True) - description = models.TextField(blank=True, null=True) - languages = ArrayField( + title = fields.CharField(max_length=255) + sort_title = fields.CharField(max_length=255, blank=True, null=True) + subtitle = fields.CharField(max_length=255, blank=True, null=True) + description = fields.TextField(blank=True, null=True) + languages = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) - series = models.CharField(max_length=255, blank=True, null=True) - series_number = models.CharField(max_length=255, blank=True, null=True) - subjects = ArrayField( - models.CharField(max_length=255), blank=True, default=list + series = fields.CharField(max_length=255, blank=True, null=True) + series_number = fields.CharField(max_length=255, blank=True, null=True) + subjects = fields.ArrayField( + models.CharField(max_length=255), blank=True, null=True, default=list ) - subject_places = ArrayField( - models.CharField(max_length=255), blank=True, default=list + subject_places = fields.ArrayField( + models.CharField(max_length=255), blank=True, null=True, default=list ) # TODO: include an annotation about the type of authorship (ie, translator) - authors = models.ManyToManyField('Author') + authors = fields.ManyToManyField('Author') # preformatted authorship string for search and easier display author_text = models.CharField(max_length=255, blank=True, null=True) - cover = models.ImageField(upload_to='covers/', blank=True, null=True) - first_published_date = models.DateTimeField(blank=True, null=True) - published_date = models.DateTimeField(blank=True, null=True) + cover = fields.ImageField(upload_to='covers/', blank=True, null=True) + first_published_date = fields.DateTimeField(blank=True, null=True) + published_date = fields.DateTimeField(blank=True, null=True) + objects = InheritanceManager() - @property - def ap_authors(self): - ''' the activitypub serialization should be a list of author ids ''' - return [a.remote_id for a in self.authors.all()] - - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - - ActivityMapping('authors', 'ap_authors'), - ActivityMapping('firstPublishedDate', 'firstpublished_date'), - ActivityMapping('publishedDate', 'published_date'), - - ActivityMapping('title', 'title'), - ActivityMapping('sortTitle', 'sort_title'), - ActivityMapping('subtitle', 'subtitle'), - ActivityMapping('description', 'description'), - ActivityMapping('languages', 'languages'), - ActivityMapping('series', 'series'), - ActivityMapping('seriesNumber', 'series_number'), - ActivityMapping('subjects', 'subjects'), - ActivityMapping('subjectPlaces', 'subject_places'), - - ActivityMapping('openlibraryKey', 'openlibrary_key'), - ActivityMapping('librarythingKey', 'librarything_key'), - ActivityMapping('goodreadsKey', 'goodreads_key'), - - ActivityMapping('work', 'parent_work'), - ActivityMapping('isbn10', 'isbn_10'), - ActivityMapping('isbn13', 'isbn_13'), - ActivityMapping('oclcNumber', 'oclc_number'), - ActivityMapping('asin', 'asin'), - ActivityMapping('pages', 'pages'), - ActivityMapping('physicalFormat', 'physical_format'), - ActivityMapping('publishers', 'publishers'), - - ActivityMapping('lccn', 'lccn'), - ActivityMapping('editions', 'editions_path'), - ActivityMapping('cover', 'cover'), - ] - def save(self, *args, **kwargs): ''' can't be abstract for query reasons, but you shouldn't USE it ''' if not isinstance(self, Edition) and not isinstance(self, Work): raise ValueError('Books should be added as Editions or Works') + if self.id and not self.remote_id: self.remote_id = self.get_remote_id() - super().save(*args, **kwargs) + if not self.id: + self.origin_id = self.remote_id + self.remote_id = None + return super().save(*args, **kwargs) def get_remote_id(self): ''' editions and works both use "book" instead of model_name ''' @@ -119,47 +86,38 @@ class Book(ActivitypubMixin, BookWyrmModel): class Work(OrderedCollectionPageMixin, Book): ''' a work (an abstract concept of a book that manifests in an edition) ''' # library of congress catalog control number - lccn = models.CharField(max_length=255, blank=True, null=True) + lccn = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) # this has to be nullable but should never be null - default_edition = models.ForeignKey( + default_edition = fields.ForeignKey( 'Edition', on_delete=models.PROTECT, null=True ) - @property - def editions_path(self): - ''' it'd be nice to serialize the edition instead but, recursion ''' - default = self.default_edition - ed_list = [ - e.remote_id for e in self.edition_set.filter(~Q(id=default.id)).all() - ] - return [default.remote_id] + ed_list - - - def to_edition_list(self, **kwargs): - ''' activitypub serialization for this work's editions ''' - remote_id = self.remote_id + '/editions' - return self.to_ordered_collection( - self.edition_set, - remote_id=remote_id, - **kwargs - ) - + def get_default_edition(self): + ''' in case the default edition is not set ''' + return self.default_edition or self.editions.first() activity_serializer = activitypub.Work + serialize_reverse_fields = [('editions', 'editions')] + deserialize_reverse_fields = [('editions', 'editions')] class Edition(Book): ''' an edition of a book ''' # these identifiers only apply to editions, not works - isbn_10 = models.CharField(max_length=255, blank=True, null=True) - isbn_13 = models.CharField(max_length=255, blank=True, null=True) - oclc_number = models.CharField(max_length=255, blank=True, null=True) - asin = models.CharField(max_length=255, blank=True, null=True) - pages = models.IntegerField(blank=True, null=True) - physical_format = models.CharField(max_length=255, blank=True, null=True) - publishers = ArrayField( + isbn_10 = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + isbn_13 = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + oclc_number = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + asin = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + pages = fields.IntegerField(blank=True, null=True) + physical_format = fields.CharField(max_length=255, blank=True, null=True) + publishers = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) shelves = models.ManyToManyField( @@ -168,9 +126,12 @@ class Edition(Book): through='ShelfBook', through_fields=('book', 'shelf') ) - parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True) + parent_work = fields.ForeignKey( + 'Work', on_delete=models.PROTECT, null=True, + related_name='editions', activitypub_field='work') activity_serializer = activitypub.Edition + name_field = 'title' def save(self, *args, **kwargs): ''' calculate isbn 10/13 ''' diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py new file mode 100644 index 00000000..34f90103 --- /dev/null +++ b/bookwyrm/models/fields.py @@ -0,0 +1,363 @@ +''' activitypub-aware django model fields ''' +from dataclasses import MISSING +import re +from uuid import uuid4 + +import dateutil.parser +from dateutil.parser import ParserError +from django.contrib.auth.models import AbstractUser +from django.contrib.postgres.fields import ArrayField as DjangoArrayField +from django.core.exceptions import ValidationError +from django.core.files.base import ContentFile +from django.db import models +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from bookwyrm import activitypub +from bookwyrm.settings import DOMAIN +from bookwyrm.connectors import get_image + + +def validate_remote_id(value): + ''' make sure the remote_id looks like a url ''' + if not value or not re.match(r'^http.?:\/\/[^\s]+$', value): + raise ValidationError( + _('%(value)s is not a valid remote_id'), + params={'value': value}, + ) + + +class ActivitypubFieldMixin: + ''' make a database field serializable ''' + def __init__(self, *args, \ + activitypub_field=None, activitypub_wrapper=None, + deduplication_field=False, **kwargs): + self.deduplication_field = deduplication_field + if activitypub_wrapper: + self.activitypub_wrapper = activitypub_field + self.activitypub_field = activitypub_wrapper + else: + self.activitypub_field = activitypub_field + super().__init__(*args, **kwargs) + + + def set_field_from_activity(self, instance, data): + ''' helper function for assinging a value to the field ''' + value = getattr(data, self.get_activitypub_field()) + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + setattr(instance, self.name, formatted) + + + def set_activity_from_field(self, activity, instance): + ''' update the json object ''' + value = getattr(instance, self.name) + formatted = self.field_to_activity(value) + if formatted is None: + return + + key = self.get_activitypub_field() + if isinstance(activity.get(key), list): + activity[key] += formatted + else: + activity[key] = formatted + + + def field_to_activity(self, value): + ''' formatter to convert a model value into activitypub ''' + if hasattr(self, 'activitypub_wrapper'): + return {self.activitypub_wrapper: value} + return value + + def field_from_activity(self, value): + ''' formatter to convert activitypub into a model value ''' + if hasattr(self, 'activitypub_wrapper'): + value = value.get(self.activitypub_wrapper) + return value + + def get_activitypub_field(self): + ''' model_field_name to activitypubFieldName ''' + if self.activitypub_field: + return self.activitypub_field + name = self.name.split('.')[-1] + components = name.split('_') + return components[0] + ''.join(x.title() for x in components[1:]) + + +class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin): + ''' default (de)serialization for foreign key and one to one ''' + def field_from_activity(self, value): + if not value: + return None + + related_model = self.related_model + if isinstance(value, dict) and value.get('id'): + # this is an activitypub object, which we can deserialize + activity_serializer = related_model.activity_serializer + return activity_serializer(**value).to_model(related_model) + try: + # make sure the value looks like a remote id + validate_remote_id(value) + except ValidationError: + # we don't know what this is, ignore it + return None + # gets or creates the model field from the remote id + return activitypub.resolve_remote_id(related_model, value) + + +class RemoteIdField(ActivitypubFieldMixin, models.CharField): + ''' a url that serves as a unique identifier ''' + def __init__(self, *args, max_length=255, validators=None, **kwargs): + validators = validators or [validate_remote_id] + super().__init__( + *args, max_length=max_length, validators=validators, + **kwargs + ) + # for this field, the default is true. false everywhere else. + self.deduplication_field = kwargs.get('deduplication_field', True) + + +class UsernameField(ActivitypubFieldMixin, models.CharField): + ''' activitypub-aware username field ''' + def __init__(self, activitypub_field='preferredUsername'): + self.activitypub_field = activitypub_field + # I don't totally know why pylint is mad at this, but it makes it work + super( #pylint: disable=bad-super-call + ActivitypubFieldMixin, self + ).__init__( + _('username'), + max_length=150, + unique=True, + validators=[AbstractUser.username_validator], + error_messages={ + 'unique': _('A user with that username already exists.'), + }, + ) + + def deconstruct(self): + ''' implementation of models.Field deconstruct ''' + name, path, args, kwargs = super().deconstruct() + del kwargs['verbose_name'] + del kwargs['max_length'] + del kwargs['unique'] + del kwargs['validators'] + del kwargs['error_messages'] + return name, path, args, kwargs + + def field_to_activity(self, value): + return value.split('@')[0] + + +PrivacyLevels = models.TextChoices('Privacy', [ + 'public', + 'unlisted', + 'followers', + 'direct' +]) + +class PrivacyField(ActivitypubFieldMixin, models.CharField): + ''' this maps to two differente activitypub fields ''' + public = 'https://www.w3.org/ns/activitystreams#Public' + def __init__(self, *args, **kwargs): + super().__init__( + *args, max_length=255, + choices=PrivacyLevels.choices, default='public') + + def set_field_from_activity(self, instance, data): + to = data.to + cc = data.cc + if to == [self.public]: + setattr(instance, self.name, 'public') + elif cc == []: + setattr(instance, self.name, 'direct') + elif self.public in cc: + setattr(instance, self.name, 'unlisted') + else: + setattr(instance, self.name, 'followers') + + def set_activity_from_field(self, activity, instance): + mentions = [u.remote_id for u in instance.mention_users.all()] + # this is a link to the followers list + followers = instance.user.__class__._meta.get_field('followers')\ + .field_to_activity(instance.user.followers) + if instance.privacy == 'public': + activity['to'] = [self.public] + activity['cc'] = [followers] + mentions + elif instance.privacy == 'unlisted': + activity['to'] = [followers] + activity['cc'] = [self.public] + mentions + elif instance.privacy == 'followers': + activity['to'] = [followers] + activity['cc'] = mentions + if instance.privacy == 'direct': + activity['to'] = mentions + activity['cc'] = [] + + +class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey): + ''' activitypub-aware foreign key field ''' + def field_to_activity(self, value): + if not value: + return None + return value.remote_id + + +class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField): + ''' activitypub-aware foreign key field ''' + def field_to_activity(self, value): + if not value: + return None + return value.to_activity() + + +class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): + ''' activitypub-aware many to many field ''' + def __init__(self, *args, link_only=False, **kwargs): + self.link_only = link_only + super().__init__(*args, **kwargs) + + def set_field_from_activity(self, instance, data): + ''' helper function for assinging a value to the field ''' + value = getattr(data, self.get_activitypub_field()) + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + getattr(instance, self.name).set(formatted) + + def field_to_activity(self, value): + if self.link_only: + return '%s/%s' % (value.instance.remote_id, self.name) + return [i.remote_id for i in value.all()] + + def field_from_activity(self, value): + items = [] + for remote_id in value: + try: + validate_remote_id(remote_id) + except ValidationError: + continue + items.append( + activitypub.resolve_remote_id(self.related_model, remote_id) + ) + return items + + +class TagField(ManyToManyField): + ''' special case of many to many that uses Tags ''' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.activitypub_field = 'tag' + + def field_to_activity(self, value): + tags = [] + for item in value.all(): + activity_type = item.__class__.__name__ + if activity_type == 'User': + activity_type = 'Mention' + tags.append(activitypub.Link( + href=item.remote_id, + name=getattr(item, item.name_field), + type=activity_type + )) + return tags + + def field_from_activity(self, value): + if not isinstance(value, list): + return None + items = [] + for link_json in value: + link = activitypub.Link(**link_json) + tag_type = link.type if link.type != 'Mention' else 'Person' + if tag_type != self.related_model.activity_serializer.type: + # tags can contain multiple types + continue + items.append( + activitypub.resolve_remote_id(self.related_model, link.href) + ) + return items + + +def image_serializer(value): + ''' helper for serializing images ''' + if value and hasattr(value, 'url'): + url = value.url + else: + return None + url = 'https://%s%s' % (DOMAIN, url) + return activitypub.Image(url=url) + + +class ImageField(ActivitypubFieldMixin, models.ImageField): + ''' activitypub-aware image field ''' + # pylint: disable=arguments-differ + def set_field_from_activity(self, instance, data, save=True): + ''' helper function for assinging a value to the field ''' + value = getattr(data, self.get_activitypub_field()) + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + getattr(instance, self.name).save(*formatted, save=save) + + + def field_to_activity(self, value): + return image_serializer(value) + + + def field_from_activity(self, value): + image_slug = value + # when it's an inline image (User avatar/icon, Book cover), it's a json + # blob, but when it's an attached image, it's just a url + if isinstance(image_slug, dict): + url = image_slug.get('url') + elif isinstance(image_slug, str): + url = image_slug + else: + return None + + try: + validate_remote_id(url) + except ValidationError: + return None + + response = get_image(url) + if not response: + return None + + image_name = str(uuid4()) + '.' + url.split('.')[-1] + image_content = ContentFile(response.content) + return [image_name, image_content] + + +class DateTimeField(ActivitypubFieldMixin, models.DateTimeField): + ''' activitypub-aware datetime field ''' + def field_to_activity(self, value): + if not value: + return None + return value.isoformat() + + def field_from_activity(self, value): + try: + date_value = dateutil.parser.parse(value) + try: + return timezone.make_aware(date_value) + except ValueError: + return date_value + except (ParserError, TypeError): + return None + +class ArrayField(ActivitypubFieldMixin, DjangoArrayField): + ''' activitypub-aware array field ''' + def field_to_activity(self, value): + return [str(i) for i in value] + +class CharField(ActivitypubFieldMixin, models.CharField): + ''' activitypub-aware char field ''' + +class TextField(ActivitypubFieldMixin, models.TextField): + ''' activitypub-aware text field ''' + +class BooleanField(ActivitypubFieldMixin, models.BooleanField): + ''' activitypub-aware boolean field ''' + +class IntegerField(ActivitypubFieldMixin, models.IntegerField): + ''' activitypub-aware boolean field ''' diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index fe39325f..835094cd 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -2,14 +2,13 @@ import re import dateutil.parser +from django.contrib.postgres.fields import JSONField from django.db import models from django.utils import timezone from bookwyrm import books_manager -from bookwyrm.connectors import ConnectorException from bookwyrm.models import ReadThrough, User, Book -from bookwyrm.utils.fields import JSONField -from .base_model import PrivacyLevels +from .fields import PrivacyLevels # Mapping goodreads -> bookwyrm shelf titles. diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index dbf99778..debe2ace 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -2,20 +2,23 @@ from django.db import models from bookwyrm import activitypub -from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel +from .base_model import ActivitypubMixin, BookWyrmModel +from . import fields class UserRelationship(ActivitypubMixin, BookWyrmModel): ''' many-to-many through table for followers ''' - user_subject = models.ForeignKey( + user_subject = fields.ForeignKey( 'User', on_delete=models.PROTECT, - related_name='%(class)s_user_subject' + related_name='%(class)s_user_subject', + activitypub_field='actor', ) - user_object = models.ForeignKey( + user_object = fields.ForeignKey( 'User', on_delete=models.PROTECT, - related_name='%(class)s_user_object' + related_name='%(class)s_user_object', + activitypub_field='object', ) class Meta: @@ -32,14 +35,9 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel): ) ] - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user_subject'), - ActivityMapping('object', 'user_object'), - ] activity_serializer = activitypub.Follow - def get_remote_id(self, status=None): + def get_remote_id(self, status=None):# pylint: disable=arguments-differ ''' use shelf identifier in remote_id ''' status = status or 'follows' base_path = self.user_subject.remote_id diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index e85294ba..68f3614f 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -3,19 +3,22 @@ import re from django.db import models from bookwyrm import activitypub -from .base_model import BookWyrmModel, OrderedCollectionMixin, PrivacyLevels +from .base_model import BookWyrmModel +from .base_model import OrderedCollectionMixin +from . import fields class Shelf(OrderedCollectionMixin, BookWyrmModel): ''' a list of books owned by a user ''' - name = models.CharField(max_length=100) + name = fields.CharField(max_length=100) identifier = models.CharField(max_length=100) - user = models.ForeignKey('User', on_delete=models.PROTECT) + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='owner') editable = models.BooleanField(default=True) - privacy = models.CharField( + privacy = fields.CharField( max_length=255, default='public', - choices=PrivacyLevels.choices + choices=fields.PrivacyLevels.choices ) books = models.ManyToManyField( 'Edition', @@ -50,15 +53,20 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): class ShelfBook(BookWyrmModel): ''' many to many join table for books and shelves ''' - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT) - added_by = models.ForeignKey( + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='object') + shelf = fields.ForeignKey( + 'Shelf', on_delete=models.PROTECT, activitypub_field='target') + added_by = fields.ForeignKey( 'User', blank=True, null=True, - on_delete=models.PROTECT + on_delete=models.PROTECT, + activitypub_field='actor' ) + activity_serializer = activitypub.AddBook + def to_add_activity(self, user): ''' AP for shelving a book''' return activitypub.Add( diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 07e25119..cbc89a06 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -6,26 +6,23 @@ from model_utils.managers import InheritanceManager from bookwyrm import activitypub from .base_model import ActivitypubMixin, OrderedCollectionPageMixin -from .base_model import ActivityMapping, BookWyrmModel, PrivacyLevels -from .base_model import tag_formatter, image_attachments_formatter - +from .base_model import BookWyrmModel +from . import fields +from .fields import image_serializer class Status(OrderedCollectionPageMixin, BookWyrmModel): ''' any post, like a reply to a review, etc ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - content = models.TextField(blank=True, null=True) - mention_users = models.ManyToManyField('User', related_name='mention_user') - mention_books = models.ManyToManyField( - 'Edition', related_name='mention_book') + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='attributedTo') + content = fields.TextField(blank=True, null=True) + mention_users = fields.TagField('User', related_name='mention_user') + mention_books = fields.TagField('Edition', related_name='mention_book') local = models.BooleanField(default=True) - privacy = models.CharField( - max_length=255, - default='public', - choices=PrivacyLevels.choices - ) - sensitive = models.BooleanField(default=False) - # the created date can't be this, because of receiving federated posts - published_date = models.DateTimeField(default=timezone.now) + privacy = fields.PrivacyField(max_length=255) + sensitive = fields.BooleanField(default=False) + # created date is different than publish date because of federated posts + published_date = fields.DateTimeField( + default=timezone.now, activitypub_field='published') deleted = models.BooleanField(default=False) deleted_date = models.DateTimeField(blank=True, null=True) favorites = models.ManyToManyField( @@ -35,88 +32,25 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): through_fields=('status', 'user'), related_name='user_favorites' ) - reply_parent = models.ForeignKey( + reply_parent = fields.ForeignKey( 'self', null=True, - on_delete=models.PROTECT + on_delete=models.PROTECT, + activitypub_field='inReplyTo', ) objects = InheritanceManager() - # ---- activitypub serialization settings for this model ----- # - @property - def ap_to(self): - ''' should be related to post privacy I think ''' - return ['https://www.w3.org/ns/activitystreams#Public'] - - @property - def ap_cc(self): - ''' should be related to post privacy I think ''' - return [self.user.ap_followers] - - @property - def ap_replies(self): - ''' structured replies block ''' - return self.to_replies() - - @property - def ap_status_image(self): - ''' attach a book cover, if relevent ''' - if hasattr(self, 'book'): - return self.book.ap_cover - if self.mention_books.first(): - return self.mention_books.first().ap_cover - return None - - - shared_mappings = [ - ActivityMapping('url', 'remote_id', lambda x: None), - ActivityMapping('id', 'remote_id'), - ActivityMapping('inReplyTo', 'reply_parent'), - ActivityMapping('published', 'published_date'), - ActivityMapping('attributedTo', 'user'), - ActivityMapping('to', 'ap_to'), - ActivityMapping('cc', 'ap_cc'), - ActivityMapping('replies', 'ap_replies'), - ActivityMapping( - 'tag', 'mention_books', - lambda x: tag_formatter(x, 'title', 'Book'), - lambda x: activitypub.tag_formatter(x, 'Book') - ), - ActivityMapping( - 'tag', 'mention_users', - lambda x: tag_formatter(x, 'username', 'Mention'), - lambda x: activitypub.tag_formatter(x, 'Mention') - ), - ActivityMapping( - 'attachment', 'attachments', - lambda x: image_attachments_formatter(x.all()), - ) - ] - - # serializing to bookwyrm expanded activitypub - activity_mappings = shared_mappings + [ - ActivityMapping('name', 'name'), - ActivityMapping('inReplyToBook', 'book'), - ActivityMapping('rating', 'rating'), - ActivityMapping('quote', 'quote'), - ActivityMapping('content', 'content'), - ] - - # for serializing to standard activitypub without extended types - pure_activity_mappings = shared_mappings + [ - ActivityMapping('name', 'ap_pure_name'), - ActivityMapping('content', 'ap_pure_content'), - ActivityMapping('attachment', 'ap_status_image'), - ] - activity_serializer = activitypub.Note + serialize_reverse_fields = [('attachments', 'attachment')] + deserialize_reverse_fields = [('attachments', 'attachment')] - #----- replies collection activitypub ----# @classmethod def replies(cls, status): ''' load all replies to a status. idk if there's a better way to write this so it's just a property ''' - return cls.objects.filter(reply_parent=status).select_subclasses() + return cls.objects.filter( + reply_parent=status + ).select_subclasses().order_by('published_date') @property def status_type(self): @@ -131,7 +65,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): **kwargs ) - def to_activity(self, pure=False): + def to_activity(self, pure=False):# pylint: disable=arguments-differ ''' return tombstone if the status is deleted ''' if self.deleted: return activitypub.Tombstone( @@ -140,7 +74,24 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): deleted=self.deleted_date.isoformat(), published=self.deleted_date.isoformat() ).serialize() - return ActivitypubMixin.to_activity(self, pure=pure) + activity = ActivitypubMixin.to_activity(self) + activity['replies'] = self.to_replies() + + # "pure" serialization for non-bookwyrm instances + if pure: + activity['content'] = self.pure_content + if 'name' in activity: + activity['name'] = self.pure_name + activity['type'] = self.pure_type + activity['attachment'] = [ + image_serializer(b.cover) for b in self.mention_books.all() \ + if b.cover] + if hasattr(self, 'book'): + activity['attachment'].append( + image_serializer(self.book.cover) + ) + return activity + def save(self, *args, **kwargs): ''' update user active time ''' @@ -153,40 +104,42 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): class GeneratedNote(Status): ''' these are app-generated messages about user activity ''' @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' message = self.content books = ', '.join( - '"%s"' % (self.book.remote_id, self.book.title) \ + '"%s"' % (book.remote_id, book.title) \ for book in self.mention_books.all() ) - return '%s %s' % (message, books) + return '%s %s %s' % (self.user.display_name, message, books) activity_serializer = activitypub.GeneratedNote - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Comment(Status): ''' like a review but without a rating and transient ''' - book = models.ForeignKey('Edition', on_delete=models.PROTECT) + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' return self.content + '

(comment on "%s")' % \ (self.book.remote_id, self.book.title) activity_serializer = activitypub.Comment - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Quotation(Status): ''' like a review but without a rating and transient ''' - quote = models.TextField() - book = models.ForeignKey('Edition', on_delete=models.PROTECT) + quote = fields.TextField() + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' return '"%s"
-- "%s"

%s' % ( self.quote, @@ -196,14 +149,15 @@ class Quotation(Status): ) activity_serializer = activitypub.Quotation - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Review(Status): ''' a book review ''' - name = models.CharField(max_length=255, null=True) - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - rating = models.IntegerField( + name = fields.CharField(max_length=255, null=True) + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') + rating = fields.IntegerField( default=None, null=True, blank=True, @@ -211,9 +165,10 @@ class Review(Status): ) @property - def ap_pure_name(self): + def pure_name(self): ''' clarify review names for mastodon serialization ''' if self.rating: + #pylint: disable=bad-string-format-type return 'Review of "%s" (%d stars): %s' % ( self.book.title, self.rating, @@ -225,26 +180,21 @@ class Review(Status): ) @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' return self.content + '

("%s")' % \ (self.book.remote_id, self.book.title) activity_serializer = activitypub.Review - pure_activity_serializer = activitypub.Article + pure_type = 'Article' class Favorite(ActivitypubMixin, BookWyrmModel): ''' fav'ing a post ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - status = models.ForeignKey('Status', on_delete=models.PROTECT) - - # ---- activitypub serialization settings for this model ----- # - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user'), - ActivityMapping('object', 'status'), - ] + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='actor') + status = fields.ForeignKey( + 'Status', on_delete=models.PROTECT, activitypub_field='object') activity_serializer = activitypub.Like @@ -254,7 +204,6 @@ class Favorite(ActivitypubMixin, BookWyrmModel): self.user.save() super().save(*args, **kwargs) - class Meta: ''' can't fav things twice ''' unique_together = ('user', 'status') @@ -262,16 +211,12 @@ class Favorite(ActivitypubMixin, BookWyrmModel): class Boost(Status): ''' boost'ing a post ''' - boosted_status = models.ForeignKey( + boosted_status = fields.ForeignKey( 'Status', on_delete=models.PROTECT, - related_name="boosters") - - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user'), - ActivityMapping('object', 'boosted_status'), - ] + related_name='boosters', + activitypub_field='object', + ) activity_serializer = activitypub.Boost diff --git a/bookwyrm/models/tag.py b/bookwyrm/models/tag.py index cd98e2b1..940b4192 100644 --- a/bookwyrm/models/tag.py +++ b/bookwyrm/models/tag.py @@ -6,13 +6,12 @@ from django.db import models from bookwyrm import activitypub from bookwyrm.settings import DOMAIN from .base_model import OrderedCollectionMixin, BookWyrmModel +from . import fields class Tag(OrderedCollectionMixin, BookWyrmModel): ''' freeform tags for books ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - name = models.CharField(max_length=100) + name = fields.CharField(max_length=100, unique=True) identifier = models.CharField(max_length=100) @classmethod @@ -30,6 +29,26 @@ class Tag(OrderedCollectionMixin, BookWyrmModel): base_path = 'https://%s' % DOMAIN return '%s/tag/%s' % (base_path, self.identifier) + + def save(self, *args, **kwargs): + ''' create a url-safe lookup key for the tag ''' + if not self.id: + # add identifiers to new tags + self.identifier = urllib.parse.quote_plus(self.name) + super().save(*args, **kwargs) + + +class UserTag(BookWyrmModel): + ''' an instance of a tag on a book by a user ''' + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='actor') + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='object') + tag = fields.ForeignKey( + 'Tag', on_delete=models.PROTECT, activitypub_field='target') + + activity_serializer = activitypub.AddBook + def to_add_activity(self, user): ''' AP for shelving a book''' return activitypub.Add( @@ -48,13 +67,7 @@ class Tag(OrderedCollectionMixin, BookWyrmModel): target=self.to_activity(), ).serialize() - def save(self, *args, **kwargs): - ''' create a url-safe lookup key for the tag ''' - if not self.id: - # add identifiers to new tags - self.identifier = urllib.parse.quote_plus(self.name) - super().save(*args, **kwargs) class Meta: ''' unqiueness constraint ''' - unique_together = ('user', 'book', 'name') + unique_together = ('user', 'book', 'tag') diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 4d511d56..63549d36 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -6,44 +6,61 @@ from django.db import models from django.dispatch import receiver from bookwyrm import activitypub +from bookwyrm.connectors import get_data from bookwyrm.models.shelf import Shelf -from bookwyrm.models.status import Status +from bookwyrm.models.status import Status, Review from bookwyrm.settings import DOMAIN from bookwyrm.signatures import create_key_pair -from .base_model import ActivityMapping, OrderedCollectionPageMixin -from .base_model import image_formatter +from bookwyrm.tasks import app +from .base_model import OrderedCollectionPageMixin +from .base_model import ActivitypubMixin, BookWyrmModel +from .federated_server import FederatedServer +from . import fields class User(OrderedCollectionPageMixin, AbstractUser): ''' a user who wants to read books ''' - private_key = models.TextField(blank=True, null=True) - public_key = models.TextField(blank=True, null=True) - inbox = models.CharField(max_length=255, unique=True) - shared_inbox = models.CharField(max_length=255, blank=True, null=True) + username = fields.UsernameField() + + key_pair = fields.OneToOneField( + 'KeyPair', + on_delete=models.CASCADE, + blank=True, null=True, + activitypub_field='publicKey', + related_name='owner' + ) + inbox = fields.RemoteIdField(unique=True) + shared_inbox = fields.RemoteIdField( + activitypub_field='sharedInbox', + activitypub_wrapper='endpoints', + deduplication_field=False, + null=True) federated_server = models.ForeignKey( 'FederatedServer', on_delete=models.PROTECT, null=True, blank=True, ) - outbox = models.CharField(max_length=255, unique=True) - summary = models.TextField(blank=True, null=True) - local = models.BooleanField(default=True) - bookwyrm_user = models.BooleanField(default=True) + outbox = fields.RemoteIdField(unique=True) + summary = fields.TextField(default='') + local = models.BooleanField(default=False) + bookwyrm_user = fields.BooleanField(default=True) localname = models.CharField( max_length=255, null=True, unique=True ) # name is your display name, which you can change at will - name = models.CharField(max_length=100, blank=True, null=True) - avatar = models.ImageField(upload_to='avatars/', blank=True, null=True) - following = models.ManyToManyField( + name = fields.CharField(max_length=100, default='') + avatar = fields.ImageField( + upload_to='avatars/', blank=True, null=True, activitypub_field='icon') + followers = fields.ManyToManyField( 'self', + link_only=True, symmetrical=False, through='UserFollows', - through_fields=('user_subject', 'user_object'), - related_name='followers' + through_fields=('user_object', 'user_subject'), + related_name='following' ) follow_requests = models.ManyToManyField( 'self', @@ -66,60 +83,20 @@ class User(OrderedCollectionPageMixin, AbstractUser): through_fields=('user', 'status'), related_name='favorite_statuses' ) - remote_id = models.CharField(max_length=255, null=True, unique=True) + remote_id = fields.RemoteIdField( + null=True, unique=True, activitypub_field='id') created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) last_active_date = models.DateTimeField(auto_now=True) - manually_approves_followers = models.BooleanField(default=False) - - # ---- activitypub serialization settings for this model ----- # - @property - def ap_followers(self): - ''' generates url for activitypub followers page ''' - return '%s/followers' % self.remote_id + manually_approves_followers = fields.BooleanField(default=False) @property - def ap_public_key(self): - ''' format the public key block for activitypub ''' - return activitypub.PublicKey(**{ - 'id': '%s/#main-key' % self.remote_id, - 'owner': self.remote_id, - 'publicKeyPem': self.public_key, - }) + def display_name(self): + ''' show the cleanest version of the user's name possible ''' + if self.name != '': + return self.name + return self.localname or self.username - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping( - 'preferredUsername', - 'username', - activity_formatter=lambda x: x.split('@')[0] - ), - ActivityMapping('name', 'name'), - ActivityMapping('bookwyrmUser', 'bookwyrm_user'), - ActivityMapping('inbox', 'inbox'), - ActivityMapping('outbox', 'outbox'), - ActivityMapping('followers', 'ap_followers'), - ActivityMapping('summary', 'summary'), - ActivityMapping( - 'publicKey', - 'public_key', - model_formatter=lambda x: x.get('publicKeyPem') - ), - ActivityMapping('publicKey', 'ap_public_key'), - ActivityMapping( - 'endpoints', - 'shared_inbox', - activity_formatter=lambda x: {'sharedInbox': x}, - model_formatter=lambda x: x.get('sharedInbox') - ), - ActivityMapping('icon', 'avatar'), - ActivityMapping( - 'manuallyApprovesFollowers', - 'manually_approves_followers' - ), - # this field isn't in the activity but should always be false - ActivityMapping(None, 'local', model_formatter=lambda x: False), - ] activity_serializer = activitypub.Person def to_outbox(self, **kwargs): @@ -127,23 +104,23 @@ class User(OrderedCollectionPageMixin, AbstractUser): queryset = Status.objects.filter( user=self, deleted=False, - ).select_subclasses() + ).select_subclasses().order_by('-published_date') return self.to_ordered_collection(queryset, \ remote_id=self.outbox, **kwargs) def to_following_activity(self, **kwargs): ''' activitypub following list ''' remote_id = '%s/following' % self.remote_id - return self.to_ordered_collection(self.following, \ + return self.to_ordered_collection(self.following.all(), \ remote_id=remote_id, id_only=True, **kwargs) def to_followers_activity(self, **kwargs): ''' activitypub followers list ''' remote_id = '%s/followers' % self.remote_id - return self.to_ordered_collection(self.followers, \ + return self.to_ordered_collection(self.followers.all(), \ remote_id=remote_id, id_only=True, **kwargs) - def to_activity(self, pure=False): + def to_activity(self): ''' override default AP serializer to add context object idk if this is the best way to go about this ''' activity_object = super().to_activity() @@ -180,18 +157,53 @@ class User(OrderedCollectionPageMixin, AbstractUser): self.inbox = '%s/inbox' % self.remote_id self.shared_inbox = 'https://%s/inbox' % DOMAIN self.outbox = '%s/outbox' % self.remote_id - if not self.private_key: - self.private_key, self.public_key = create_key_pair() return super().save(*args, **kwargs) +class KeyPair(ActivitypubMixin, BookWyrmModel): + ''' public and private keys for a user ''' + private_key = models.TextField(blank=True, null=True) + public_key = fields.TextField( + blank=True, null=True, activitypub_field='publicKeyPem') + + activity_serializer = activitypub.PublicKey + serialize_reverse_fields = [('owner', 'owner')] + + def get_remote_id(self): + # self.owner is set by the OneToOneField on User + return '%s/#main-key' % self.owner.remote_id + + def save(self, *args, **kwargs): + ''' create a key pair ''' + if not self.public_key: + self.private_key, self.public_key = create_key_pair() + return super().save(*args, **kwargs) + + def to_activity(self): + ''' override default AP serializer to add context object + idk if this is the best way to go about this ''' + activity_object = super().to_activity() + del activity_object['@context'] + del activity_object['type'] + return activity_object + + @receiver(models.signals.post_save, sender=User) +#pylint: disable=unused-argument def execute_after_save(sender, instance, created, *args, **kwargs): ''' create shelves for new users ''' - if not instance.local or not created: + if not created: return + if not instance.local: + set_remote_server.delay(instance.id) + return + + instance.key_pair = KeyPair.objects.create( + remote_id='%s/#main-key' % instance.remote_id) + instance.save() + shelves = [{ 'name': 'To Read', 'identifier': 'to-read', @@ -210,3 +222,54 @@ def execute_after_save(sender, instance, created, *args, **kwargs): user=instance, editable=False ).save() + + +@app.task +def set_remote_server(user_id): + ''' figure out the user's remote server in the background ''' + user = User.objects.get(id=user_id) + actor_parts = urlparse(user.remote_id) + user.federated_server = \ + get_or_create_remote_server(actor_parts.netloc) + user.save() + if user.bookwyrm_user: + get_remote_reviews.delay(user.outbox) + + +def get_or_create_remote_server(domain): + ''' get info on a remote server ''' + try: + return FederatedServer.objects.get( + server_name=domain + ) + except FederatedServer.DoesNotExist: + pass + + data = get_data('https://%s/.well-known/nodeinfo' % domain) + + try: + nodeinfo_url = data.get('links')[0].get('href') + except (TypeError, KeyError): + return None + + data = get_data(nodeinfo_url) + + server = FederatedServer.objects.create( + server_name=domain, + application_type=data['software']['name'], + application_version=data['software']['version'], + ) + return server + + +@app.task +def get_remote_reviews(outbox): + ''' ingest reviews by a new remote bookwyrm user ''' + outbox_page = outbox + '?page=true' + data = get_data(outbox_page) + + # TODO: pagination? + for activity in data['orderedItems']: + if not activity['type'] == 'Review': + continue + activitypub.Review(**activity).to_model(Review) diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py index a196fcec..38b48282 100644 --- a/bookwyrm/outgoing.py +++ b/bookwyrm/outgoing.py @@ -4,15 +4,15 @@ import re from django.db import IntegrityError, transaction from django.http import HttpResponseNotFound, JsonResponse from django.views.decorators.csrf import csrf_exempt -import requests +from requests import HTTPError from bookwyrm import activitypub from bookwyrm import models +from bookwyrm.connectors import get_data, ConnectorException from bookwyrm.broadcast import broadcast from bookwyrm.status import create_notification from bookwyrm.status import create_generated_note from bookwyrm.status import delete_status -from bookwyrm.remote_user import get_or_create_remote_user from bookwyrm.settings import DOMAIN from bookwyrm.utils import regex @@ -54,16 +54,16 @@ def handle_remote_webfinger(query): url = 'https://%s/.well-known/webfinger?resource=acct:%s' % \ (domain, query) try: - response = requests.get(url) - except requests.exceptions.ConnectionError: + data = get_data(url) + except (ConnectorException, HTTPError): return None - if not response.ok: - return None - data = response.json() - for link in data['links']: - if link['rel'] == 'self': + + for link in data.get('links'): + if link.get('rel') == 'self': try: - user = get_or_create_remote_user(link['href']) + user = activitypub.resolve_remote_id( + models.User, link['href'] + ) except KeyError: return None return user diff --git a/bookwyrm/remote_user.py b/bookwyrm/remote_user.py deleted file mode 100644 index 23a805b3..00000000 --- a/bookwyrm/remote_user.py +++ /dev/null @@ -1,111 +0,0 @@ -''' manage remote users ''' -from urllib.parse import urlparse -import requests - -from django.db import transaction - -from bookwyrm import activitypub, models -from bookwyrm import status as status_builder -from bookwyrm.tasks import app - - -def get_or_create_remote_user(actor): - ''' look up a remote user or add them ''' - try: - return models.User.objects.get(remote_id=actor) - except models.User.DoesNotExist: - pass - - data = fetch_user_data(actor) - - actor_parts = urlparse(actor) - with transaction.atomic(): - user = activitypub.Person(**data).to_model(models.User) - user.federated_server = get_or_create_remote_server(actor_parts.netloc) - user.save() - if user.bookwyrm_user: - get_remote_reviews.delay(user.id) - return user - - -def fetch_user_data(actor): - ''' load the user's info from the actor url ''' - try: - response = requests.get( - actor, - headers={'Accept': 'application/activity+json'} - ) - except ConnectionError: - return None - - if not response.ok: - response.raise_for_status() - data = response.json() - - # make sure our actor is who they say they are - if actor != data['id']: - raise ValueError("Remote actor id must match url.") - return data - - -def refresh_remote_user(user): - ''' get updated user data from its home instance ''' - data = fetch_user_data(user.remote_id) - - activity = activitypub.Person(**data) - activity.to_model(models.User, instance=user) - - -@app.task -def get_remote_reviews(user_id): - ''' ingest reviews by a new remote bookwyrm user ''' - try: - user = models.User.objects.get(id=user_id) - except models.User.DoesNotExist: - return - outbox_page = user.outbox + '?page=true' - response = requests.get( - outbox_page, - headers={'Accept': 'application/activity+json'} - ) - data = response.json() - # TODO: pagination? - for activity in data['orderedItems']: - status_builder.create_status(activity) - - -def get_or_create_remote_server(domain): - ''' get info on a remote server ''' - try: - return models.FederatedServer.objects.get( - server_name=domain - ) - except models.FederatedServer.DoesNotExist: - pass - - response = requests.get( - 'https://%s/.well-known/nodeinfo' % domain, - headers={'Accept': 'application/activity+json'} - ) - - if response.status_code != 200: - return None - - data = response.json() - try: - nodeinfo_url = data.get('links')[0].get('href') - except (TypeError, KeyError): - return None - - response = requests.get( - nodeinfo_url, - headers={'Accept': 'application/activity+json'} - ) - data = response.json() - - server = models.FederatedServer.objects.create( - server_name=domain, - application_type=data['software']['name'], - application_version=data['software']['version'], - ) - return server diff --git a/bookwyrm/routine_book_tasks.py b/bookwyrm/routine_book_tasks.py deleted file mode 100644 index eaa28d90..00000000 --- a/bookwyrm/routine_book_tasks.py +++ /dev/null @@ -1,16 +0,0 @@ -''' Routine tasks for keeping your library tidy ''' -from datetime import timedelta -from django.utils import timezone -from bookwyrm import books_manager -from bookwyrm import models - -def sync_book_data(): - ''' update books with any changes to their canonical source ''' - expiry = timezone.now() - timedelta(days=1) - books = models.Edition.objects.filter( - sync=True, - last_sync_date__lte=expiry - ).all() - for book in books: - # TODO: create background tasks - books_manager.update_book(book) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 3784158c..c42215b4 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -99,10 +99,6 @@ BOOKWYRM_DBS = { 'HOST': env('POSTGRES_HOST', ''), 'PORT': 5432 }, - 'sqlite': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'fedireads.db') - } } DATABASES = { diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py index 57c181df..ff281664 100644 --- a/bookwyrm/signatures.py +++ b/bookwyrm/signatures.py @@ -31,7 +31,7 @@ def make_signature(sender, destination, date, digest): 'digest: %s' % digest, ] message_to_sign = '\n'.join(signature_headers) - signer = pkcs1_15.new(RSA.import_key(sender.private_key)) + signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key)) signed_message = signer.sign(SHA256.new(message_to_sign.encode('utf8'))) signature = { 'keyId': '%s#main-key' % sender.remote_id, diff --git a/bookwyrm/status.py b/bookwyrm/status.py index 6a86209f..648f2e7d 100644 --- a/bookwyrm/status.py +++ b/bookwyrm/status.py @@ -1,7 +1,7 @@ ''' Handle user activity ''' from django.utils import timezone -from bookwyrm import activitypub, books_manager, models +from bookwyrm import models from bookwyrm.sanitize_html import InputHtmlParser @@ -12,37 +12,6 @@ def delete_status(status): status.save() -def create_status(activity): - ''' unfortunately, it's not QUITE as simple as deserializing it ''' - # render the json into an activity object - serializer = activitypub.activity_objects[activity['type']] - activity = serializer(**activity) - try: - model = models.activity_models[activity.type] - except KeyError: - # not a type of status we are prepared to deserialize - return None - - # ignore notes that aren't replies to known statuses - if activity.type == 'Note': - reply = models.Status.objects.filter( - remote_id=activity.inReplyTo - ).first() - if not reply: - return None - - # look up books - book_urls = [] - if hasattr(activity, 'inReplyToBook'): - book_urls.append(activity.inReplyToBook) - if hasattr(activity, 'tag'): - book_urls += [t['href'] for t in activity.tag if t['type'] == 'Book'] - for remote_id in book_urls: - books_manager.get_or_create_book(remote_id) - - return activity.to_model(model) - - def create_generated_note(user, content, mention_books=None, privacy='public'): ''' a note created by the app about user activity ''' # sanitize input html diff --git a/bookwyrm/templates/author.html b/bookwyrm/templates/author.html index fb4970e4..9a7a20ab 100644 --- a/bookwyrm/templates/author.html +++ b/bookwyrm/templates/author.html @@ -1,8 +1,8 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
-

{{ author.display_name }}

+

{{ author.name }}

{% if author.bio %}

@@ -12,7 +12,7 @@

-

Books by {{ author.display_name }}

+

Books by {{ author.name }}

{% include 'snippets/book_tiles.html' with books=books %}
{% endblock %} diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html index c7112016..3263c2b1 100644 --- a/bookwyrm/templates/book.html +++ b/bookwyrm/templates/book.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% load humanize %} {% block content %} @@ -86,8 +86,8 @@ {% endif %} - {% if book.parent_work.edition_set.count > 1 %} -

{{ book.parent_work.edition_set.count }} editions

+ {% if book.parent_work.editions.count > 1 %} +

{{ book.parent_work.editions.count }} editions

{% endif %} diff --git a/bookwyrm/templates/direct_messages.html b/bookwyrm/templates/direct_messages.html new file mode 100644 index 00000000..6a20b111 --- /dev/null +++ b/bookwyrm/templates/direct_messages.html @@ -0,0 +1,37 @@ +{% extends 'layout.html' %} +{% block content %} + +
+

Direct Messages

+ + {% if not activities %} +

You have no messages right now.

+ {% endif %} + {% for activity in activities %} +
+ {% include 'snippets/status.html' with status=activity %} +
+ {% endfor %} + + +
+ +{% endblock %} diff --git a/bookwyrm/templates/editions.html b/bookwyrm/templates/editions.html index 273b2cd6..619ceafb 100644 --- a/bookwyrm/templates/editions.html +++ b/bookwyrm/templates/editions.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

Editions of "{{ work.title }}"

diff --git a/bookwyrm/templates/feed.html b/bookwyrm/templates/feed.html index 6e49943a..07ad8d0f 100644 --- a/bookwyrm/templates/feed.html +++ b/bookwyrm/templates/feed.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
diff --git a/bookwyrm/templates/followers.html b/bookwyrm/templates/followers.html index 645e46a1..00cb13ca 100644 --- a/bookwyrm/templates/followers.html +++ b/bookwyrm/templates/followers.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

diff --git a/bookwyrm/templates/following.html b/bookwyrm/templates/following.html index 2cca9127..478ca813 100644 --- a/bookwyrm/templates/following.html +++ b/bookwyrm/templates/following.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index f91e2cce..6bb903b0 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% load humanize %} {% block content %}
diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index b37c9cda..ab113ad0 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %} @@ -68,6 +68,9 @@ {% include 'snippets/username.html' with user=request.user %}

+ + Direct messages + Profile diff --git a/bookwyrm/templates/notifications.html b/bookwyrm/templates/notifications.html index f9494c31..f31df76d 100644 --- a/bookwyrm/templates/notifications.html +++ b/bookwyrm/templates/notifications.html @@ -1,6 +1,6 @@ {% extends 'layout.html' %} {% load humanize %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

Notifications

diff --git a/bookwyrm/templates/shelf.html b/bookwyrm/templates/shelf.html index d6842d13..390b9fc6 100644 --- a/bookwyrm/templates/shelf.html +++ b/bookwyrm/templates/shelf.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
diff --git a/bookwyrm/templates/snippets/authors.html b/bookwyrm/templates/snippets/authors.html index 165c4cda..e8106f5d 100644 --- a/bookwyrm/templates/snippets/authors.html +++ b/bookwyrm/templates/snippets/authors.html @@ -1 +1 @@ -{{ book.authors.first.display_name }} +{{ book.authors.first.name }} diff --git a/bookwyrm/templates/snippets/avatar.html b/bookwyrm/templates/snippets/avatar.html index cb0a12ea..da4b5fd2 100644 --- a/bookwyrm/templates/snippets/avatar.html +++ b/bookwyrm/templates/snippets/avatar.html @@ -1,3 +1,3 @@ -{% load fr_display %} +{% load bookwyrm_tags %} avatar for {{ user|username }} diff --git a/bookwyrm/templates/snippets/book_cover.html b/bookwyrm/templates/snippets/book_cover.html index 3fd83616..ceeef426 100644 --- a/bookwyrm/templates/snippets/book_cover.html +++ b/bookwyrm/templates/snippets/book_cover.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %}
{% if book.cover %} {% include 'snippets/cover_alt.html' with book=book %} diff --git a/bookwyrm/templates/snippets/book_preview.html b/bookwyrm/templates/snippets/book_preview.html index c675c45f..e7eca455 100644 --- a/bookwyrm/templates/snippets/book_preview.html +++ b/bookwyrm/templates/snippets/book_preview.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %}
diff --git a/bookwyrm/templates/snippets/boost_button.html b/bookwyrm/templates/snippets/boost_button.html index 57765bed..bf06cef7 100644 --- a/bookwyrm/templates/snippets/boost_button.html +++ b/bookwyrm/templates/snippets/boost_button.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %} {% with status.id|uuid as uuid %}
{% csrf_token %} diff --git a/bookwyrm/templates/snippets/cover_alt.html b/bookwyrm/templates/snippets/cover_alt.html index 52bd52f2..0cccc2e1 100644 --- a/bookwyrm/templates/snippets/cover_alt.html +++ b/bookwyrm/templates/snippets/cover_alt.html @@ -1,2 +1,2 @@ -{% load fr_display %} +{% load bookwyrm_tags %} '{{ book.title }}' Cover ({{ book|edition_info }}) diff --git a/bookwyrm/templates/snippets/create_status.html b/bookwyrm/templates/snippets/create_status.html index e36b1b19..ac8c0b75 100644 --- a/bookwyrm/templates/snippets/create_status.html +++ b/bookwyrm/templates/snippets/create_status.html @@ -1,5 +1,5 @@ {% load humanize %} -{% load fr_display %} +{% load bookwyrm_tags %}
    diff --git a/bookwyrm/templates/snippets/fav_button.html b/bookwyrm/templates/snippets/fav_button.html index 58ece1e4..11e03cdb 100644 --- a/bookwyrm/templates/snippets/fav_button.html +++ b/bookwyrm/templates/snippets/fav_button.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %} {% with status.id|uuid as uuid %} {% csrf_token %} diff --git a/bookwyrm/templates/snippets/finish_reading_modal.html b/bookwyrm/templates/snippets/finish_reading_modal.html index 8f8aeeff..d04d508d 100644 --- a/bookwyrm/templates/snippets/finish_reading_modal.html +++ b/bookwyrm/templates/snippets/finish_reading_modal.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %}