diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index cfbe0524..5662d1d5 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,7 +1,7 @@ # These are supported funding model platforms github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -patreon: bookwrym +patreon: bookwyrm open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel diff --git a/README.md b/README.md index 51e39541..2dfedebb 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,6 @@ cp .env.example .env For most testing, you'll want to use ngrok. Remember to set the DOMAIN in `.env` to your ngrok domain. - -#### With Docker You'll have to install the Docker and docker-compose. When you're ready, run: ```bash @@ -70,33 +68,7 @@ docker-compose run --rm web python manage.py migrate docker-compose run --rm web python manage.py initdb ``` -### Without Docker -You will need postgres installed and running on your computer. - -``` bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -createdb bookwyrm -``` - -Create the psql user in `psql bookwyrm`: -``` psql -CREATE ROLE bookwyrm WITH LOGIN PASSWORD 'bookwyrm'; -GRANT ALL PRIVILEGES ON DATABASE bookwyrm TO bookwyrm; -``` - -Initialize the database (or, more specifically, delete the existing database, run migrations, and start fresh): -``` bash -./rebuilddb.sh -``` -This creates two users, `mouse` with password `password123` and `rat` with password `ratword`. - -The application uses Celery and Redis for task management, which must also be installed and configured. - -And go to the app at `localhost:8000` - - +Once the build is complete, you can access the instance at `localhost:1333` ## Installing in Production diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py index b5b124ec..a4fef41e 100644 --- a/bookwyrm/activitypub/__init__.py +++ b/bookwyrm/activitypub/__init__.py @@ -11,6 +11,7 @@ from .note import Tombstone from .interaction import Boost, Like from .ordered_collection import OrderedCollection, OrderedCollectionPage from .person import Person, PublicKey +from .response import ActivitypubResponse from .book import Edition, Work, Author from .verbs import Create, Delete, Undo, Update from .verbs import Follow, Accept, Reject diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index ed19af99..7ef0920f 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -3,9 +3,7 @@ from dataclasses import dataclass, fields, MISSING from json import JSONEncoder from django.apps import apps -from django.db import transaction -from django.db.models.fields.files import ImageFileDescriptor -from django.db.models.fields.related_descriptors import ManyToManyDescriptor +from django.db import IntegrityError, transaction from bookwyrm.connectors import ConnectorException, get_data from bookwyrm.tasks import app @@ -65,7 +63,6 @@ class ActivityObject: setattr(self, field.name, value) - @transaction.atomic def to_model(self, model, instance=None, save=True): ''' convert from an activity to a model instance ''' if not isinstance(self, model.activity_serializer): @@ -76,74 +73,54 @@ class ActivityObject: model.activity_serializer) ) + if hasattr(model, 'ignore_activity') and model.ignore_activity(self): + return instance + # check for an existing instance, if we're not updating a known obj - if not instance: - instance = model.find_existing(self.serialize()) or model() + instance = instance or model.find_existing(self.serialize()) or model() - many_to_many_fields = {} - image_fields = {} - for field in model._meta.get_fields(): - # check if it's an activitypub field - if not hasattr(field, 'field_to_activity'): - continue - # call the formatter associated with the model field class - value = field.field_from_activity( - getattr(self, field.get_activitypub_field()) - ) - if value is None or value is MISSING: - continue + for field in instance.simple_fields: + field.set_field_from_activity(instance, self) - model_field = getattr(model, field.name) - - if isinstance(model_field, ManyToManyDescriptor): - # status mentions book/users for example, stash this for later - many_to_many_fields[field.name] = value - elif isinstance(model_field, ImageFileDescriptor): - # image fields need custom handling - image_fields[field.name] = value - else: - # just a good old fashioned model.field = value - setattr(instance, field.name, value) - - # if this isn't here, it messes up saving users. who even knows. - for (model_key, value) in image_fields.items(): - getattr(instance, model_key).save(*value, save=save) + # image fields have to be set after other fields because they can save + # too early and jank up users + for field in instance.image_fields: + field.set_field_from_activity(instance, self, save=save) if not save: + return instance + + with transaction.atomic(): # we can't set many to many and reverse fields on an unsaved object - return instance + try: + instance.save() + except IntegrityError as e: + raise ActivitySerializerError(e) - instance.save() - - # add many to many fields, which have to be set post-save - for (model_key, values) in many_to_many_fields.items(): - # mention books/users, for example - getattr(instance, model_key).set(values) - - if not save or not hasattr(model, 'deserialize_reverse_fields'): - return instance + # add many to many fields, which have to be set post-save + for field in instance.many_to_many_fields: + # mention books/users, for example + field.set_field_from_activity(instance, self) # reversed relationships in the models for (model_field_name, activity_field_name) in \ - model.deserialize_reverse_fields: + instance.deserialize_reverse_fields: # attachments on Status, for example values = getattr(self, activity_field_name) if values is None or values is MISSING: continue - try: - # this is for one to many - related_model = getattr(model, model_field_name).field.model - except AttributeError: - # it's a one to one or foreign key - related_model = getattr(model, model_field_name)\ - .related.related_model - values = [values] + + model_field = getattr(model, model_field_name) + # creating a Work, model_field is 'editions' + # creating a User, model field is 'key_pair' + related_model = model_field.field.model + related_field_name = model_field.field.name for item in values: set_related_field.delay( related_model.__name__, instance.__class__.__name__, - instance.__class__.__name__.lower(), + related_field_name, instance.remote_id, item ) @@ -160,8 +137,8 @@ class ActivityObject: @app.task @transaction.atomic def set_related_field( - model_name, origin_model_name, - related_field_name, related_remote_id, data): + model_name, origin_model_name, related_field_name, + related_remote_id, data): ''' load reverse related fields (editions, attachments) without blocking ''' model = apps.get_model('bookwyrm.%s' % model_name, require_ready=True) origin_model = apps.get_model( @@ -169,23 +146,38 @@ def set_related_field( require_ready=True ) - if isinstance(data, str): - item = resolve_remote_id(model, data, save=False) - else: - # look for a match based on all the available data - item = model.find_existing(data) - if not item: - # create a new model instance - item = model.activity_serializer(**data) - item = item.to_model(model, save=False) - # this must exist because it's the object that triggered this function - instance = origin_model.find_existing_by_remote_id(related_remote_id) - if not instance: - raise ValueError('Invalid related remote id: %s' % related_remote_id) + with transaction.atomic(): + if isinstance(data, str): + existing = model.find_existing_by_remote_id(data) + if existing: + data = existing.to_activity() + else: + data = get_data(data) + activity = model.activity_serializer(**data) - # edition.parent_work = instance, for example - setattr(item, related_field_name, instance) - item.save() + # this must exist because it's the object that triggered this function + instance = origin_model.find_existing_by_remote_id(related_remote_id) + if not instance: + raise ValueError( + 'Invalid related remote id: %s' % related_remote_id) + + # set the origin's remote id on the activity so it will be there when + # the model instance is created + # edition.parentWork = instance, for example + model_field = getattr(model, related_field_name) + if hasattr(model_field, 'activitypub_field'): + setattr( + activity, + getattr(model_field, 'activitypub_field'), + instance.remote_id + ) + item = activity.to_model(model) + + # if the related field isn't serialized (attachments on Status), then + # we have to set it post-creation + if not hasattr(model_field, 'activitypub_field'): + setattr(item, related_field_name, instance) + item.save() def resolve_remote_id(model, remote_id, refresh=False, save=True): diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index ae9c334d..6fa80b32 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -38,7 +38,7 @@ class Edition(Book): isbn13: str = '' oclcNumber: str = '' asin: str = '' - pages: str = '' + pages: int = None physicalFormat: str = '' publishers: List[str] = field(default_factory=lambda: []) @@ -50,7 +50,7 @@ class Work(Book): ''' work instance of a book object ''' lccn: str = '' defaultEdition: str = '' - editions: List[str] + editions: List[str] = field(default_factory=lambda: []) type: str = 'Work' @@ -58,10 +58,12 @@ class Work(Book): class Author(ActivityObject): ''' author of a book ''' name: str - born: str = '' - died: str = '' - aliases: str = '' + born: str = None + died: str = None + aliases: List[str] = field(default_factory=lambda: []) bio: str = '' openlibraryKey: str = '' + librarythingKey: str = '' + goodreadsKey: str = '' wikipediaLink: str = '' type: str = 'Person' diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py index df28bf8d..72fbe5fc 100644 --- a/bookwyrm/activitypub/note.py +++ b/bookwyrm/activitypub/note.py @@ -23,6 +23,7 @@ class Note(ActivityObject): cc: List[str] = field(default_factory=lambda: []) replies: Dict = field(default_factory=lambda: {}) inReplyTo: str = '' + summary: str = '' tag: List[Link] = field(default_factory=lambda: []) attachment: List[Image] = field(default_factory=lambda: []) sensitive: bool = False @@ -52,8 +53,8 @@ class Comment(Note): @dataclass(init=False) class Review(Comment): ''' a full book review ''' - name: str - rating: int + name: str = None + rating: int = None type: str = 'Review' diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py index 88349c02..7e7d027e 100644 --- a/bookwyrm/activitypub/person.py +++ b/bookwyrm/activitypub/person.py @@ -18,13 +18,13 @@ class PublicKey(ActivityObject): class Person(ActivityObject): ''' actor activitypub json ''' preferredUsername: str - name: str inbox: str outbox: str followers: str - summary: str publicKey: PublicKey endpoints: Dict + name: str = None + summary: str = None icon: Image = field(default_factory=lambda: {}) bookwyrmUser: bool = False manuallyApprovesFollowers: str = False diff --git a/bookwyrm/activitypub/response.py b/bookwyrm/activitypub/response.py new file mode 100644 index 00000000..bbc44c4d --- /dev/null +++ b/bookwyrm/activitypub/response.py @@ -0,0 +1,18 @@ +from django.http import JsonResponse + +from .base_activity import ActivityEncoder + +class ActivitypubResponse(JsonResponse): + """ + A class to be used in any place that's serializing responses for + Activitypub enabled clients. Uses JsonResponse under the hood, but already + configures some stuff beforehand. Made to be a drop-in replacement of + JsonResponse. + """ + def __init__(self, data, encoder=ActivityEncoder, safe=True, + json_dumps_params=None, **kwargs): + + if 'content_type' not in kwargs: + kwargs['content_type'] = 'application/activity+json' + + super().__init__(data, encoder, safe, json_dumps_params, **kwargs) diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py index e890d81f..7c627927 100644 --- a/bookwyrm/activitypub/verbs.py +++ b/bookwyrm/activitypub/verbs.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import List from .base_activity import ActivityObject, Signature -from .book import Book +from .book import Edition @dataclass(init=False) class Verb(ActivityObject): @@ -73,7 +73,7 @@ class Add(Verb): @dataclass(init=False) class AddBook(Verb): '''Add activity that's aware of the book obj ''' - target: Book + target: Edition type: str = 'Add' diff --git a/bookwyrm/broadcast.py b/bookwyrm/broadcast.py index a98b6774..f4186c4d 100644 --- a/bookwyrm/broadcast.py +++ b/bookwyrm/broadcast.py @@ -3,7 +3,7 @@ import json from django.utils.http import http_date import requests -from bookwyrm import models +from bookwyrm import models, settings from bookwyrm.activitypub import ActivityEncoder from bookwyrm.tasks import app from bookwyrm.signatures import make_signature, make_digest @@ -79,6 +79,7 @@ def sign_and_send(sender, data, destination): 'Digest': digest, 'Signature': make_signature(sender, destination, now, digest), 'Content-Type': 'application/activity+json; charset=utf-8', + 'User-Agent': settings.USER_AGENT, }, ) if not response.ok: diff --git a/bookwyrm/connectors/__init__.py b/bookwyrm/connectors/__init__.py index 4eb91de4..cfafd286 100644 --- a/bookwyrm/connectors/__init__.py +++ b/bookwyrm/connectors/__init__.py @@ -2,3 +2,5 @@ from .settings import CONNECTORS from .abstract_connector import ConnectorException from .abstract_connector import get_data, get_image + +from .connector_manager import search, local_search, first_search_result diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index c9f1ad2e..d63bd135 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -1,22 +1,18 @@ ''' functionality outline for a book data connector ''' from abc import ABC, abstractmethod -from dataclasses import dataclass -import pytz +from dataclasses import asdict, dataclass +import logging from urllib3.exceptions import RequestError from django.db import transaction -from dateutil import parser import requests -from requests import HTTPError from requests.exceptions import SSLError -from bookwyrm import models - - -class ConnectorException(HTTPError): - ''' when the connector can't do what was asked ''' +from bookwyrm import activitypub, models, settings +from .connector_manager import load_more_data, ConnectorException +logger = logging.getLogger(__name__) class AbstractMinimalConnector(ABC): ''' just the bare bones, for other bookwyrm instances ''' def __init__(self, identifier): @@ -38,17 +34,22 @@ class AbstractMinimalConnector(ABC): for field in self_fields: setattr(self, field, getattr(info, field)) - def search(self, query, min_confidence=None): + def search(self, query, min_confidence=None):# pylint: disable=unused-argument ''' free text search ''' resp = requests.get( '%s%s' % (self.search_url, query), headers={ 'Accept': 'application/json; charset=utf-8', + 'User-Agent': settings.USER_AGENT, }, ) if not resp.ok: resp.raise_for_status() - data = resp.json() + try: + data = resp.json() + except ValueError as e: + logger.exception(e) + raise ConnectorException('Unable to parse json response', e) results = [] for doc in self.parse_search_data(data)[:10]: @@ -72,9 +73,6 @@ class AbstractConnector(AbstractMinimalConnector): ''' generic book data connector ''' def __init__(self, identifier): super().__init__(identifier) - - self.key_mappings = [] - # fields we want to look for in book data to copy over # title we handle separately. self.book_mappings = [] @@ -89,216 +87,112 @@ class AbstractConnector(AbstractMinimalConnector): def get_or_create_book(self, remote_id): - # try to load the book - book = models.Book.objects.select_subclasses().filter( - origin_id=remote_id - ).first() - if book: - if isinstance(book, models.Work): - return book.default_edition - return book + ''' translate arbitrary json into an Activitypub dataclass ''' + # first, check if we have the origin_id saved + existing = models.Edition.find_existing_by_remote_id(remote_id) or \ + models.Work.find_existing_by_remote_id(remote_id) + if existing: + if hasattr(existing, 'get_default_editon'): + return existing.get_default_editon() + return existing - # no book was found, so we start creating a new one + # load the json data = get_data(remote_id) - - work = None - edition = None + mapped_data = dict_from_mappings(data, self.book_mappings) if self.is_work_data(data): - work_data = data - # if we requested a work and there's already an edition, we're set - work = self.match_from_mappings(work_data, models.Work) - if work and work.default_edition: - return work.default_edition - - # no such luck, we need more information. try: - edition_data = self.get_edition_from_work_data(work_data) + edition_data = self.get_edition_from_work_data(data) except KeyError: # hack: re-use the work data as the edition data # this is why remote ids aren't necessarily unique edition_data = data + work_data = mapped_data else: - edition_data = data - edition = self.match_from_mappings(edition_data, models.Edition) - # no need to figure out about the work if we already know about it - if edition and edition.parent_work: - return edition - - # no such luck, we need more information. try: - work_data = self.get_work_from_edition_date(edition_data) + work_data = self.get_work_from_edition_data(data) + work_data = dict_from_mappings(work_data, self.book_mappings) except KeyError: - # remember this hack: re-use the work data as the edition data - work_data = data + work_data = mapped_data + edition_data = data if not work_data or not edition_data: raise ConnectorException('Unable to load book data: %s' % remote_id) - # at this point, we need to figure out the work, edition, or both - # atomic so that we don't save a work with no edition for vice versa with transaction.atomic(): - if not work: - work_key = self.get_remote_id_from_data(work_data) - work = self.create_book(work_key, work_data, models.Work) + # create activitypub object + work_activity = activitypub.Work(**work_data) + # this will dedupe automatically + work = work_activity.to_model(models.Work) + for author in self.get_authors_from_data(data): + work.authors.add(author) - if not edition: - ed_key = self.get_remote_id_from_data(edition_data) - edition = self.create_book(ed_key, edition_data, models.Edition) - edition.parent_work = work - edition.save() - work.default_edition = edition - work.save() + edition = self.create_edition_from_data(work, edition_data) + load_more_data.delay(self.connector.id, work.id) + return edition - # now's our change to fill in author gaps + + def create_edition_from_data(self, work, edition_data): + ''' if we already have the work, we're ready ''' + mapped_data = dict_from_mappings(edition_data, self.book_mappings) + mapped_data['work'] = work.remote_id + edition_activity = activitypub.Edition(**mapped_data) + edition = edition_activity.to_model(models.Edition) + edition.connector = self.connector + edition.save() + + work.default_edition = edition + work.save() + + for author in self.get_authors_from_data(edition_data): + edition.authors.add(author) if not edition.authors.exists() and work.authors.exists(): edition.authors.set(work.authors.all()) - edition.author_text = work.author_text - edition.save() - - if not edition: - raise ConnectorException('Unable to create book: %s' % remote_id) return edition - def create_book(self, remote_id, data, model): - ''' create a work or edition from data ''' - book = model.objects.create( - origin_id=remote_id, - title=data['title'], - connector=self.connector, - ) - return self.update_book_from_data(book, data) + def get_or_create_author(self, remote_id): + ''' load that author ''' + existing = models.Author.find_existing_by_remote_id(remote_id) + if existing: + return existing + data = get_data(remote_id) - def update_book_from_data(self, book, data, update_cover=True): - ''' for creating a new book or syncing with data ''' - book = update_from_mappings(book, data, self.book_mappings) - - author_text = [] - for author in self.get_authors_from_data(data): - book.authors.add(author) - author_text.append(author.name) - book.author_text = ', '.join(author_text) - book.save() - - if not update_cover: - return book - - cover = self.get_cover_from_data(data) - if cover: - book.cover.save(*cover, save=True) - return book - - - def update_book(self, book, data=None): - ''' load new data ''' - if not book.sync and not book.sync_cover: - return book - - if not data: - key = getattr(book, self.key_name) - data = self.load_book_data(key) - - if book.sync: - book = self.update_book_from_data( - book, data, update_cover=book.sync_cover) - else: - cover = self.get_cover_from_data(data) - if cover: - book.cover.save(*cover, save=True) - - return book - - - def match_from_mappings(self, data, model): - ''' try to find existing copies of this book using various keys ''' - relevent_mappings = [m for m in self.key_mappings if \ - not m.model or model == m.model] - for mapping in relevent_mappings: - # check if this field is present in the data - value = data.get(mapping.remote_field) - if not value: - continue - - # extract the value in the right format - value = mapping.formatter(value) - - # search our database for a matching book - kwargs = {mapping.local_field: value} - match = model.objects.filter(**kwargs).first() - if match: - return match - return None - - - @abstractmethod - def get_remote_id_from_data(self, data): - ''' otherwise we won't properly set the remote_id in the db ''' + mapped_data = dict_from_mappings(data, self.author_mappings) + activity = activitypub.Author(**mapped_data) + # this will dedupe + return activity.to_model(models.Author) @abstractmethod def is_work_data(self, data): ''' differentiate works and editions ''' - @abstractmethod def get_edition_from_work_data(self, data): ''' every work needs at least one edition ''' - @abstractmethod - def get_work_from_edition_date(self, data): + def get_work_from_edition_data(self, data): ''' every edition needs a work ''' - @abstractmethod def get_authors_from_data(self, data): ''' load author data ''' - - @abstractmethod - def get_cover_from_data(self, data): - ''' load cover ''' - @abstractmethod def expand_book_data(self, book): ''' get more info on a book ''' -def update_from_mappings(obj, data, mappings): - ''' assign data to model with mappings ''' +def dict_from_mappings(data, mappings): + ''' create a dict in Activitypub format, using mappings supplies by + the subclass ''' + result = {} for mapping in mappings: - # check if this field is present in the data - value = data.get(mapping.remote_field) - if not value: - continue - - # extract the value in the right format - try: - value = mapping.formatter(value) - except: - continue - - # assign the formatted value to the model - obj.__setattr__(mapping.local_field, value) - return obj - - -def get_date(date_string): - ''' helper function to try to interpret dates ''' - if not date_string: - return None - - try: - return pytz.utc.localize(parser.parse(date_string)) - except ValueError: - pass - - try: - return parser.parse(date_string) - except ValueError: - return None + result[mapping.local_field] = mapping.get_value(data) + return result def get_data(url): @@ -308,9 +202,10 @@ def get_data(url): url, headers={ 'Accept': 'application/json; charset=utf-8', + 'User-Agent': settings.USER_AGENT, }, ) - except RequestError: + except (RequestError, SSLError): raise ConnectorException() if not resp.ok: resp.raise_for_status() @@ -325,7 +220,12 @@ def get_data(url): def get_image(url): ''' wrapper for requesting an image ''' try: - resp = requests.get(url) + resp = requests.get( + url, + headers={ + 'User-Agent': settings.USER_AGENT, + }, + ) except (RequestError, SSLError): return None if not resp.ok: @@ -340,20 +240,35 @@ class SearchResult: key: str author: str year: str + connector: object confidence: int = 1 def __repr__(self): return "".format( self.key, self.title, self.author) + def json(self): + ''' serialize a connector for json response ''' + serialized = asdict(self) + del serialized['connector'] + return serialized + class Mapping: ''' associate a local database field with a field in an external dataset ''' - def __init__( - self, local_field, remote_field=None, formatter=None, model=None): + def __init__(self, local_field, remote_field=None, formatter=None): noop = lambda x: x self.local_field = local_field self.remote_field = remote_field or local_field self.formatter = formatter or noop - self.model = model + + def get_value(self, data): + ''' pull a field from incoming json and return the formatted version ''' + value = data.get(self.remote_field) + if not value: + return None + try: + return self.formatter(value) + except:# pylint: disable=bare-except + return None diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index e4d32fd3..3c6f4614 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -13,4 +13,5 @@ class Connector(AbstractMinimalConnector): return data def format_search_result(self, search_result): + search_result['connector'] = self return SearchResult(**search_result) diff --git a/bookwyrm/books_manager.py b/bookwyrm/connectors/connector_manager.py similarity index 87% rename from bookwyrm/books_manager.py rename to bookwyrm/connectors/connector_manager.py index 3b865768..d3b01f7a 100644 --- a/bookwyrm/books_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -1,4 +1,4 @@ -''' select and call a connector for whatever book task needs doing ''' +''' interface with whatever connectors the app has ''' import importlib from urllib.parse import urlparse @@ -8,43 +8,8 @@ from bookwyrm import models from bookwyrm.tasks import app -def get_edition(book_id): - ''' look up a book in the db and return an edition ''' - book = models.Book.objects.select_subclasses().get(id=book_id) - if isinstance(book, models.Work): - book = book.default_edition - return book - - -def get_or_create_connector(remote_id): - ''' get the connector related to the author's server ''' - url = urlparse(remote_id) - identifier = url.netloc - if not identifier: - raise ValueError('Invalid remote id') - - try: - connector_info = models.Connector.objects.get(identifier=identifier) - except models.Connector.DoesNotExist: - connector_info = models.Connector.objects.create( - identifier=identifier, - connector_file='bookwyrm_connector', - base_url='https://%s' % identifier, - books_url='https://%s/book' % identifier, - covers_url='https://%s/images/covers' % identifier, - search_url='https://%s/search?q=' % identifier, - priority=2 - ) - - return load_connector(connector_info) - - -@app.task -def load_more_data(book_id): - ''' background the work of getting all 10,000 editions of LoTR ''' - book = models.Book.objects.select_subclasses().get(id=book_id) - connector = load_connector(book.connector) - connector.expand_book_data(book) +class ConnectorException(HTTPError): + ''' when the connector can't do what was asked ''' def search(query, min_confidence=0.1): @@ -55,7 +20,7 @@ def search(query, min_confidence=0.1): for connector in get_connectors(): try: result_set = connector.search(query, min_confidence=min_confidence) - except HTTPError: + except (HTTPError, ConnectorException): continue result_set = [r for r in result_set \ @@ -91,6 +56,38 @@ def get_connectors(): yield load_connector(info) +def get_or_create_connector(remote_id): + ''' get the connector related to the author's server ''' + url = urlparse(remote_id) + identifier = url.netloc + if not identifier: + raise ValueError('Invalid remote id') + + try: + connector_info = models.Connector.objects.get(identifier=identifier) + except models.Connector.DoesNotExist: + connector_info = models.Connector.objects.create( + identifier=identifier, + connector_file='bookwyrm_connector', + base_url='https://%s' % identifier, + books_url='https://%s/book' % identifier, + covers_url='https://%s/images/covers' % identifier, + search_url='https://%s/search?q=' % identifier, + priority=2 + ) + + return load_connector(connector_info) + + +@app.task +def load_more_data(connector_id, book_id): + ''' background the work of getting all 10,000 editions of LoTR ''' + connector_info = models.Connector.objects.get(id=connector_id) + connector = load_connector(connector_info) + book = models.Book.objects.select_subclasses().get(id=book_id) + connector.expand_book_data(book) + + def load_connector(connector_info): ''' instantiate the connector class ''' connector = importlib.import_module( diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 28eb1ea0..55355131 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -1,13 +1,10 @@ ''' openlibrary data connector ''' import re -import requests - -from django.core.files.base import ContentFile from bookwyrm import models from .abstract_connector import AbstractConnector, SearchResult, Mapping -from .abstract_connector import ConnectorException -from .abstract_connector import get_date, get_data, update_from_mappings +from .abstract_connector import get_data +from .connector_manager import ConnectorException from .openlibrary_languages import languages @@ -17,67 +14,62 @@ class Connector(AbstractConnector): super().__init__(identifier) get_first = lambda a: a[0] - self.key_mappings = [ - Mapping('isbn_13', model=models.Edition, formatter=get_first), - Mapping('isbn_10', model=models.Edition, formatter=get_first), - Mapping('lccn', model=models.Work, formatter=get_first), + get_remote_id = lambda a: self.base_url + a + self.book_mappings = [ + Mapping('title'), + Mapping('id', remote_field='key', formatter=get_remote_id), Mapping( - 'oclc_number', - remote_field='oclc_numbers', - model=models.Edition, - formatter=get_first - ), - Mapping( - 'openlibrary_key', - remote_field='key', - formatter=get_openlibrary_key - ), - Mapping('goodreads_key'), - Mapping('asin'), - ] - - self.book_mappings = self.key_mappings + [ - Mapping('sort_title'), + 'cover', remote_field='covers', formatter=self.get_cover_url), + Mapping('sortTitle', remote_field='sort_title'), Mapping('subtitle'), Mapping('description', formatter=get_description), Mapping('languages', formatter=get_languages), Mapping('series', formatter=get_first), - Mapping('series_number'), + Mapping('seriesNumber', remote_field='series_number'), Mapping('subjects'), - Mapping('subject_places'), + Mapping('subjectPlaces'), + Mapping('isbn13', formatter=get_first), + Mapping('isbn10', formatter=get_first), + Mapping('lccn', formatter=get_first), Mapping( - 'first_published_date', - remote_field='first_publish_date', - formatter=get_date + 'oclcNumber', remote_field='oclc_numbers', + formatter=get_first ), Mapping( - 'published_date', - remote_field='publish_date', - formatter=get_date + 'openlibraryKey', remote_field='key', + formatter=get_openlibrary_key ), + Mapping('goodreadsKey', remote_field='goodreads_key'), + Mapping('asin'), Mapping( - 'pages', - model=models.Edition, - remote_field='number_of_pages' + 'firstPublishedDate', remote_field='first_publish_date', ), - Mapping('physical_format', model=models.Edition), + Mapping('publishedDate', remote_field='publish_date'), + Mapping('pages', remote_field='number_of_pages'), + Mapping('physicalFormat', remote_field='physical_format'), Mapping('publishers'), ] self.author_mappings = [ + Mapping('id', remote_field='key', formatter=get_remote_id), Mapping('name'), - Mapping('born', remote_field='birth_date', formatter=get_date), - Mapping('died', remote_field='death_date', formatter=get_date), + Mapping( + 'openlibraryKey', remote_field='key', + formatter=get_openlibrary_key + ), + Mapping('born', remote_field='birth_date'), + Mapping('died', remote_field='death_date'), Mapping('bio', formatter=get_description), ] def get_remote_id_from_data(self, data): + ''' format a url from an openlibrary id field ''' try: key = data['key'] except KeyError: raise ConnectorException('Invalid book data') - return '%s/%s' % (self.books_url, key) + return '%s%s' % (self.books_url, key) def is_work_data(self, data): @@ -89,17 +81,17 @@ class Connector(AbstractConnector): key = data['key'] except KeyError: raise ConnectorException('Invalid book data') - url = '%s/%s/editions' % (self.books_url, key) + url = '%s%s/editions' % (self.books_url, key) data = get_data(url) return pick_default_edition(data['entries']) - def get_work_from_edition_date(self, data): + def get_work_from_edition_data(self, data): try: key = data['works'][0]['key'] except (IndexError, KeyError): raise ConnectorException('No work found for edition') - url = '%s/%s' % (self.books_url, key) + url = '%s%s' % (self.books_url, key) return get_data(url) @@ -107,24 +99,17 @@ class Connector(AbstractConnector): ''' parse author json and load or create authors ''' for author_blob in data.get('authors', []): author_blob = author_blob.get('author', author_blob) - # this id is "/authors/OL1234567A" and we want just "OL1234567A" - author_id = author_blob['key'].split('/')[-1] - yield self.get_or_create_author(author_id) + # this id is "/authors/OL1234567A" + author_id = author_blob['key'] + url = '%s%s' % (self.base_url, author_id) + yield self.get_or_create_author(url) - def get_cover_from_data(self, data): + def get_cover_url(self, cover_blob): ''' ask openlibrary for the cover ''' - if not data.get('covers'): - return None - - cover_id = data.get('covers')[0] - image_name = '%s-M.jpg' % cover_id - url = '%s/b/id/%s' % (self.covers_url, image_name) - response = requests.get(url) - if not response.ok: - response.raise_for_status() - image_content = ContentFile(response.content) - return [image_name, image_content] + cover_id = cover_blob[0] + image_name = '%s-L.jpg' % cover_id + return '%s/b/id/%s' % (self.covers_url, image_name) def parse_search_data(self, data): @@ -139,13 +124,14 @@ class Connector(AbstractConnector): title=search_result.get('title'), key=key, author=', '.join(author), + connector=self, year=search_result.get('first_publish_year'), ) def load_edition_data(self, olkey): ''' query openlibrary for editions of a work ''' - url = '%s/works/%s/editions.json' % (self.books_url, olkey) + url = '%s/works/%s/editions' % (self.books_url, olkey) return get_data(url) @@ -158,44 +144,14 @@ class Connector(AbstractConnector): # we can mass download edition data from OL to avoid repeatedly querying edition_options = self.load_edition_data(work.openlibrary_key) for edition_data in edition_options.get('entries'): - olkey = edition_data.get('key').split('/')[-1] - # make sure the edition isn't already in the database - if models.Edition.objects.filter(openlibrary_key=olkey).count(): - continue - - # creates and populates the book from the data - edition = self.create_book(olkey, edition_data, models.Edition) - # ensures that the edition is associated with the work - edition.parent_work = work - edition.save() - # get author data from the work if it's missing from the edition - if not edition.authors and work.authors: - edition.authors.set(work.authors.all()) - - - def get_or_create_author(self, olkey): - ''' load that author ''' - if not re.match(r'^OL\d+A$', olkey): - raise ValueError('Invalid OpenLibrary author ID') - author = models.Author.objects.filter(openlibrary_key=olkey).first() - if author: - return author - - url = '%s/authors/%s.json' % (self.base_url, olkey) - data = get_data(url) - - author = models.Author(openlibrary_key=olkey) - author = update_from_mappings(author, data, self.author_mappings) - author.save() - - return author + self.create_edition_from_data(work, edition_data) def get_description(description_blob): ''' descriptions can be a string or a dict ''' if isinstance(description_blob, dict): return description_blob.get('value') - return description_blob + return description_blob def get_openlibrary_key(key): @@ -220,7 +176,7 @@ def pick_default_edition(options): if len(options) == 1: return options[0] - options = [e for e in options if e.get('cover')] or options + options = [e for e in options if e.get('covers')] or options options = [e for e in options if \ '/languages/eng' in str(e.get('languages'))] or options formats = ['paperback', 'hardcover', 'mass market paperback'] diff --git a/bookwyrm/connectors/self_connector.py b/bookwyrm/connectors/self_connector.py index 80d3a67d..0c21e7bc 100644 --- a/bookwyrm/connectors/self_connector.py +++ b/bookwyrm/connectors/self_connector.py @@ -1,6 +1,9 @@ ''' using a bookwyrm instance as a source of book data ''' +from functools import reduce +import operator + from django.contrib.postgres.search import SearchRank, SearchVector -from django.db.models import F +from django.db.models import Count, F, Q from bookwyrm import models from .abstract_connector import AbstractConnector, SearchResult @@ -9,38 +12,18 @@ from .abstract_connector import AbstractConnector, SearchResult class Connector(AbstractConnector): ''' instantiate a connector ''' def search(self, query, min_confidence=0.1): - ''' right now you can't search bookwyrm sorry, but when - that gets implemented it will totally rule ''' - vector = SearchVector('title', weight='A') +\ - SearchVector('subtitle', weight='B') +\ - SearchVector('author_text', weight='C') +\ - SearchVector('isbn_13', weight='A') +\ - SearchVector('isbn_10', weight='A') +\ - SearchVector('openlibrary_key', weight='C') +\ - SearchVector('goodreads_key', weight='C') +\ - SearchVector('asin', weight='C') +\ - SearchVector('oclc_number', weight='C') +\ - SearchVector('remote_id', weight='C') +\ - SearchVector('description', weight='D') +\ - SearchVector('series', weight='D') - - results = models.Edition.objects.annotate( - search=vector - ).annotate( - rank=SearchRank(vector, query) - ).filter( - rank__gt=min_confidence - ).order_by('-rank') - - # remove non-default editions, if possible - results = results.filter(parent_work__default_edition__id=F('id')) \ - or results - + ''' search your local database ''' + # first, try searching unqiue identifiers + results = search_identifiers(query) + if not results: + # then try searching title/author + results = search_title_author(query, min_confidence) search_results = [] - for book in results[:10]: - search_results.append( - self.format_search_result(book) - ) + for result in results: + search_results.append(self.format_search_result(result)) + if len(search_results) >= 10: + break + search_results.sort(key=lambda r: r.confidence, reverse=True) return search_results @@ -51,31 +34,74 @@ class Connector(AbstractConnector): author=search_result.author_text, year=search_result.published_date.year if \ search_result.published_date else None, - confidence=search_result.rank, + connector=self, + confidence=search_result.rank if \ + hasattr(search_result, 'rank') else 1, ) - def get_remote_id_from_data(self, data): - pass - def is_work_data(self, data): pass def get_edition_from_work_data(self, data): pass - def get_work_from_edition_date(self, data): + def get_work_from_edition_data(self, data): pass def get_authors_from_data(self, data): return None - def get_cover_from_data(self, data): - return None - def parse_search_data(self, data): ''' it's already in the right format, don't even worry about it ''' return data def expand_book_data(self, book): pass + + +def search_identifiers(query): + ''' tries remote_id, isbn; defined as dedupe fields on the model ''' + filters = [{f.name: query} for f in models.Edition._meta.get_fields() \ + if hasattr(f, 'deduplication_field') and f.deduplication_field] + results = models.Edition.objects.filter( + reduce(operator.or_, (Q(**f) for f in filters)) + ).distinct() + + # when there are multiple editions of the same work, pick the default. + # it would be odd for this to happen. + return results.filter(parent_work__default_edition__id=F('id')) \ + or results + + +def search_title_author(query, min_confidence): + ''' searches for title and author ''' + vector = SearchVector('title', weight='A') +\ + SearchVector('subtitle', weight='B') +\ + SearchVector('authors__name', weight='C') +\ + SearchVector('series', weight='D') + + results = models.Edition.objects.annotate( + search=vector + ).annotate( + rank=SearchRank(vector, query) + ).filter( + rank__gt=min_confidence + ).order_by('-rank') + + # when there are multiple editions of the same work, pick the closest + editions_of_work = results.values( + 'parent_work' + ).annotate( + Count('parent_work') + ).values_list('parent_work') + + for work_id in set(editions_of_work): + editions = results.filter(parent_work=work_id) + default = editions.filter(parent_work__default_edition=F('id')) + default_rank = default.first().rank if default.exists() else 0 + # if mutliple books have the top rank, pick the default edition + if default_rank == editions.first().rank: + yield default.first() + else: + yield editions.first() diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py index 72839dce..a1471ac4 100644 --- a/bookwyrm/context_processors.py +++ b/bookwyrm/context_processors.py @@ -1,7 +1,7 @@ ''' customize the info available in context for rendering templates ''' from bookwyrm import models -def site_settings(request): +def site_settings(request):# pylint: disable=unused-argument ''' include the custom info about the site ''' return { 'site': models.SiteSettings.objects.get() diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py index 784f1038..152c2d76 100644 --- a/bookwyrm/forms.py +++ b/bookwyrm/forms.py @@ -31,10 +31,11 @@ class CustomForm(ModelForm): visible.field.widget.attrs['class'] = css_classes[input_type] +# pylint: disable=missing-class-docstring class LoginForm(CustomForm): class Meta: model = models.User - fields = ['username', 'password'] + fields = ['localname', 'password'] help_texts = {f: None for f in fields} widgets = { 'password': PasswordInput(), @@ -44,7 +45,7 @@ class LoginForm(CustomForm): class RegisterForm(CustomForm): class Meta: model = models.User - fields = ['username', 'email', 'password'] + fields = ['localname', 'email', 'password'] help_texts = {f: None for f in fields} widgets = { 'password': PasswordInput() @@ -60,25 +61,36 @@ class RatingForm(CustomForm): class ReviewForm(CustomForm): class Meta: model = models.Review - fields = ['user', 'book', 'name', 'content', 'rating', 'privacy'] + fields = [ + 'user', 'book', + 'name', 'content', 'rating', + 'content_warning', 'sensitive', + 'privacy'] class CommentForm(CustomForm): class Meta: model = models.Comment - fields = ['user', 'book', 'content', 'privacy'] + fields = [ + 'user', 'book', 'content', + 'content_warning', 'sensitive', + 'privacy'] class QuotationForm(CustomForm): class Meta: model = models.Quotation - fields = ['user', 'book', 'quote', 'content', 'privacy'] + fields = [ + 'user', 'book', 'quote', 'content', + 'content_warning', 'sensitive', 'privacy'] class ReplyForm(CustomForm): class Meta: model = models.Status - fields = ['user', 'content', 'reply_parent', 'privacy'] + fields = [ + 'user', 'content', 'content_warning', 'sensitive', + 'reply_parent', 'privacy'] class EditUserForm(CustomForm): @@ -110,14 +122,13 @@ class EditionForm(CustomForm): model = models.Edition exclude = [ 'remote_id', + 'origin_id', 'created_date', 'updated_date', - 'last_sync_date', 'authors',# TODO 'parent_work', 'shelves', - 'misc_identifiers', 'subjects',# TODO 'subject_places',# TODO @@ -125,12 +136,23 @@ class EditionForm(CustomForm): 'connector', ] +class AuthorForm(CustomForm): + class Meta: + model = models.Author + exclude = [ + 'remote_id', + 'origin_id', + 'created_date', + 'updated_date', + ] + class ImportForm(forms.Form): csv_file = forms.FileField() class ExpiryWidget(widgets.Select): def value_from_datadict(self, data, files, name): + ''' human-readable exiration time buckets ''' selected_string = super().value_from_datadict(data, files, name) if selected_string == 'day': diff --git a/bookwyrm/goodreads_import.py b/bookwyrm/goodreads_import.py index 3fd330ab..9b8a4f01 100644 --- a/bookwyrm/goodreads_import.py +++ b/bookwyrm/goodreads_import.py @@ -8,8 +8,6 @@ from bookwyrm.models import ImportJob, ImportItem from bookwyrm.status import create_notification logger = logging.getLogger(__name__) -# TODO: remove or increase once we're confident it's not causing problems. -MAX_ENTRIES = 500 def create_job(user, csv_file, include_reviews, privacy): @@ -19,12 +17,13 @@ def create_job(user, csv_file, include_reviews, privacy): include_reviews=include_reviews, privacy=privacy ) - for index, entry in enumerate(list(csv.DictReader(csv_file))[:MAX_ENTRIES]): + for index, entry in enumerate(list(csv.DictReader(csv_file))): if not all(x in entry for x in ('ISBN13', 'Title', 'Author')): raise ValueError('Author, title, and isbn must be in data.') ImportItem(job=job, index=index, data=entry).save() return job + def create_retry_job(user, original_job, items): ''' retry items that didn't import ''' job = ImportJob.objects.create( @@ -37,6 +36,7 @@ def create_retry_job(user, original_job, items): ImportItem(job=job, index=item.index, data=item.data).save() return job + def start_import(job): ''' initalizes a csv import job ''' result = import_data.delay(job.id) @@ -49,11 +49,10 @@ def import_data(job_id): ''' does the actual lookup work in a celery task ''' job = ImportJob.objects.get(id=job_id) try: - results = [] for item in job.items.all(): try: item.resolve() - except Exception as e: + except Exception as e:# pylint: disable=broad-except logger.exception(e) item.fail_reason = 'Error loading book' item.save() @@ -61,7 +60,6 @@ def import_data(job_id): if item.book: item.save() - results.append(item) # shelves book and handles reviews outgoing.handle_imported_book( diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index bbbebf0f..5e42fe45 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -6,6 +6,7 @@ import django.db.utils from django.http import HttpResponse from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST import requests from bookwyrm import activitypub, models, outgoing @@ -15,11 +16,9 @@ from bookwyrm.signatures import Signature @csrf_exempt +@require_POST def inbox(request, username): ''' incoming activitypub events ''' - # TODO: should do some kind of checking if the user accepts - # this action from the sender probably? idk - # but this will just throw a 404 if the user doesn't exist try: models.User.objects.get(localname=username) except models.User.DoesNotExist: @@ -29,11 +28,9 @@ def inbox(request, username): @csrf_exempt +@require_POST def shared_inbox(request): ''' incoming activitypub events ''' - if request.method == 'GET': - return HttpResponseNotFound() - try: resp = request.body activity = json.loads(resp) @@ -60,7 +57,6 @@ def shared_inbox(request): 'Announce': handle_boost, 'Add': { 'Edition': handle_add, - 'Work': handle_add, }, 'Undo': { 'Follow': handle_unfollow, @@ -69,8 +65,8 @@ def shared_inbox(request): }, 'Update': { 'Person': handle_update_user, - 'Edition': handle_update_book, - 'Work': handle_update_book, + 'Edition': handle_update_edition, + 'Work': handle_update_work, }, } activity_type = activity['type'] @@ -144,7 +140,7 @@ def handle_follow(activity): def handle_unfollow(activity): ''' unfollow a local user ''' obj = activity['object'] - requester = activitypub.resolve_remote_id(models.user, obj['actor']) + requester = activitypub.resolve_remote_id(models.User, obj['actor']) to_unfollow = models.User.objects.get(remote_id=obj['object']) # raises models.User.DoesNotExist @@ -188,34 +184,48 @@ def handle_follow_reject(activity): def handle_create(activity): ''' someone did something, good on them ''' # deduplicate incoming activities - status_id = activity['object']['id'] + activity = activity['object'] + status_id = activity.get('id') if models.Status.objects.filter(remote_id=status_id).count(): return - serializer = activitypub.activity_objects[activity['type']] - status = serializer(**activity) + try: + serializer = activitypub.activity_objects[activity['type']] + except KeyError: + return + + activity = serializer(**activity) try: model = models.activity_models[activity.type] except KeyError: # not a type of status we are prepared to deserialize return - if activity.type == 'Note': - reply = models.Status.objects.filter( - remote_id=activity.inReplyTo - ).first() - if not reply: - return + status = activity.to_model(model) + if not status: + # it was discarded because it's not a bookwyrm type + return - activity.to_model(model) # create a notification if this is a reply + notified = [] if status.reply_parent and status.reply_parent.user.local: + notified.append(status.reply_parent.user) status_builder.create_notification( status.reply_parent.user, 'REPLY', related_user=status.user, related_status=status, ) + if status.mention_users.exists(): + for mentioned_user in status.mention_users.all(): + if not mentioned_user.local or mentioned_user in notified: + continue + status_builder.create_notification( + mentioned_user, + 'MENTION', + related_user=status.user, + related_status=status, + ) @app.task @@ -228,11 +238,12 @@ def handle_delete_status(activity): # is trying to delete a user. return try: - status = models.Status.objects.select_subclasses().get( + status = models.Status.objects.get( remote_id=status_id ) except models.Status.DoesNotExist: return + models.Notification.objects.filter(related_status=status).all().delete() status_builder.delete_status(status) @@ -317,6 +328,12 @@ def handle_update_user(activity): @app.task -def handle_update_book(activity): +def handle_update_edition(activity): ''' a remote instance changed a book (Document) ''' activitypub.Edition(**activity['object']).to_model(models.Edition) + + +@app.task +def handle_update_work(activity): + ''' a remote instance changed a book (Document) ''' + activitypub.Work(**activity['object']).to_model(models.Work) diff --git a/bookwyrm/management/commands/deduplicate_book_data.py b/bookwyrm/management/commands/deduplicate_book_data.py new file mode 100644 index 00000000..044b2a98 --- /dev/null +++ b/bookwyrm/management/commands/deduplicate_book_data.py @@ -0,0 +1,83 @@ +''' PROCEED WITH CAUTION: uses deduplication fields to permanently +merge book data objects ''' +from django.core.management.base import BaseCommand +from django.db.models import Count +from bookwyrm import models + + +def update_related(canonical, obj): + ''' update all the models with fk to the object being removed ''' + # move related models to canonical + related_models = [ + (r.remote_field.name, r.related_model) for r in \ + canonical._meta.related_objects] + for (related_field, related_model) in related_models: + related_objs = related_model.objects.filter( + **{related_field: obj}) + for related_obj in related_objs: + print( + 'replacing in', + related_model.__name__, + related_field, + related_obj.id + ) + try: + setattr(related_obj, related_field, canonical) + related_obj.save() + except TypeError: + getattr(related_obj, related_field).add(canonical) + getattr(related_obj, related_field).remove(obj) + + +def copy_data(canonical, obj): + ''' try to get the most data possible ''' + for data_field in obj._meta.get_fields(): + if not hasattr(data_field, 'activitypub_field'): + continue + data_value = getattr(obj, data_field.name) + if not data_value: + continue + if not getattr(canonical, data_field.name): + print('setting data field', data_field.name, data_value) + setattr(canonical, data_field.name, data_value) + canonical.save() + + +def dedupe_model(model): + ''' combine duplicate editions and update related models ''' + fields = model._meta.get_fields() + dedupe_fields = [f for f in fields if \ + hasattr(f, 'deduplication_field') and f.deduplication_field] + for field in dedupe_fields: + dupes = model.objects.values(field.name).annotate( + Count(field.name) + ).filter(**{'%s__count__gt' % field.name: 1}) + + for dupe in dupes: + value = dupe[field.name] + if not value or value == '': + continue + print('----------') + print(dupe) + objs = model.objects.filter( + **{field.name: value} + ).order_by('id') + canonical = objs.first() + print('keeping', canonical.remote_id) + for obj in objs[1:]: + print(obj.remote_id) + copy_data(canonical, obj) + update_related(canonical, obj) + # remove the outdated entry + obj.delete() + + +class Command(BaseCommand): + ''' dedplucate allllll the book data models ''' + help = 'merges duplicate book data' + # pylint: disable=no-self-use,unused-argument + def handle(self, *args, **options): + ''' run deudplications ''' + dedupe_model(models.Edition) + dedupe_model(models.Work) + dedupe_model(models.Author) diff --git a/bookwyrm/migrations/0001_initial.py b/bookwyrm/migrations/0001_initial.py index b1aba7df..347057e1 100644 --- a/bookwyrm/migrations/0001_initial.py +++ b/bookwyrm/migrations/0001_initial.py @@ -7,7 +7,7 @@ import django.core.validators from django.db import migrations, models import django.db.models.deletion import django.utils.timezone -import bookwyrm.utils.fields +from django.contrib.postgres.fields import JSONField class Migration(migrations.Migration): @@ -62,7 +62,7 @@ class Migration(migrations.Migration): ('content', models.TextField(blank=True, null=True)), ('created_date', models.DateTimeField(auto_now_add=True)), ('openlibrary_key', models.CharField(max_length=255)), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ], options={ 'abstract': False, @@ -75,7 +75,7 @@ class Migration(migrations.Migration): ('content', models.TextField(blank=True, null=True)), ('created_date', models.DateTimeField(auto_now_add=True)), ('openlibrary_key', models.CharField(max_length=255, unique=True)), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ('cover', models.ImageField(blank=True, null=True, upload_to='covers/')), ('added_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), ('authors', models.ManyToManyField(to='bookwyrm.Author')), diff --git a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py index 13cb1406..6a149ab5 100644 --- a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py +++ b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py @@ -2,7 +2,6 @@ import bookwyrm.models.connector import bookwyrm.models.site -import bookwyrm.utils.fields from django.conf import settings import django.contrib.postgres.operations import django.core.validators @@ -10,6 +9,7 @@ from django.db import migrations, models import django.db.models.deletion import django.db.models.expressions import django.utils.timezone +from django.contrib.postgres.fields import JSONField, ArrayField import uuid @@ -148,7 +148,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='misc_identifiers', - field=bookwyrm.utils.fields.JSONField(null=True), + field=JSONField(null=True), ), migrations.AddField( model_name='book', @@ -226,7 +226,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='author', name='aliases', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='user', @@ -394,17 +394,17 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='subject_places', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='book', name='subjects', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='edition', name='publishers', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AlterField( model_name='connector', @@ -578,7 +578,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='languages', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='edition', @@ -676,7 +676,7 @@ class Migration(migrations.Migration): name='ImportItem', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ], ), migrations.CreateModel( diff --git a/bookwyrm/migrations/0016_auto_20201129_0304.py b/bookwyrm/migrations/0016_auto_20201129_0304.py index 2bf820e1..1e715969 100644 --- a/bookwyrm/migrations/0016_auto_20201129_0304.py +++ b/bookwyrm/migrations/0016_auto_20201129_0304.py @@ -1,10 +1,9 @@ # Generated by Django 3.0.7 on 2020-11-29 03:04 -import bookwyrm.utils.fields from django.conf import settings from django.db import migrations, models import django.db.models.deletion - +from django.contrib.postgres.fields import ArrayField class Migration(migrations.Migration): @@ -16,12 +15,12 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='book', name='subject_places', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), ), migrations.AlterField( model_name='book', name='subjects', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), ), migrations.AlterField( model_name='edition', diff --git a/bookwyrm/migrations/0017_auto_20201212_0059.py b/bookwyrm/migrations/0017_auto_20201212_0059.py new file mode 100644 index 00000000..c9e3fcf4 --- /dev/null +++ b/bookwyrm/migrations/0017_auto_20201212_0059.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-12-12 00:59 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0016_auto_20201211_2026'), + ] + + operations = [ + migrations.AlterField( + model_name='readthrough', + name='book', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + ] diff --git a/bookwyrm/migrations/0023_auto_20201214_0511.py b/bookwyrm/migrations/0023_auto_20201214_0511.py new file mode 100644 index 00000000..e811bded --- /dev/null +++ b/bookwyrm/migrations/0023_auto_20201214_0511.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-12-14 05:11 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0022_auto_20201212_1744'), + ] + + operations = [ + migrations.AlterField( + model_name='status', + name='privacy', + field=bookwyrm.models.fields.PrivacyField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255), + ), + ] diff --git a/bookwyrm/migrations/0023_merge_20201216_0112.py b/bookwyrm/migrations/0023_merge_20201216_0112.py new file mode 100644 index 00000000..e3af4849 --- /dev/null +++ b/bookwyrm/migrations/0023_merge_20201216_0112.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-12-16 01:12 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0017_auto_20201212_0059'), + ('bookwyrm', '0022_auto_20201212_1744'), + ] + + operations = [ + ] diff --git a/bookwyrm/migrations/0024_merge_20201216_1721.py b/bookwyrm/migrations/0024_merge_20201216_1721.py new file mode 100644 index 00000000..41f81335 --- /dev/null +++ b/bookwyrm/migrations/0024_merge_20201216_1721.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-12-16 17:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0023_auto_20201214_0511'), + ('bookwyrm', '0023_merge_20201216_0112'), + ] + + operations = [ + ] diff --git a/bookwyrm/migrations/0025_auto_20201217_0046.py b/bookwyrm/migrations/0025_auto_20201217_0046.py new file mode 100644 index 00000000..a3ffe8c1 --- /dev/null +++ b/bookwyrm/migrations/0025_auto_20201217_0046.py @@ -0,0 +1,39 @@ +# Generated by Django 3.0.7 on 2020-12-17 00:46 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0024_merge_20201216_1721'), + ] + + operations = [ + migrations.AlterField( + model_name='author', + name='bio', + field=bookwyrm.models.fields.HtmlField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='description', + field=bookwyrm.models.fields.HtmlField(blank=True, null=True), + ), + migrations.AlterField( + model_name='quotation', + name='quote', + field=bookwyrm.models.fields.HtmlField(), + ), + migrations.AlterField( + model_name='status', + name='content', + field=bookwyrm.models.fields.HtmlField(blank=True, null=True), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.HtmlField(default=''), + ), + ] diff --git a/bookwyrm/migrations/0026_status_content_warning.py b/bookwyrm/migrations/0026_status_content_warning.py new file mode 100644 index 00000000..f4e494db --- /dev/null +++ b/bookwyrm/migrations/0026_status_content_warning.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-12-17 03:17 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0025_auto_20201217_0046'), + ] + + operations = [ + migrations.AddField( + model_name='status', + name='content_warning', + field=bookwyrm.models.fields.CharField(blank=True, max_length=500, null=True), + ), + ] diff --git a/bookwyrm/migrations/0027_auto_20201220_2007.py b/bookwyrm/migrations/0027_auto_20201220_2007.py new file mode 100644 index 00000000..a3ad4dda --- /dev/null +++ b/bookwyrm/migrations/0027_auto_20201220_2007.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.7 on 2020-12-20 20:07 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0026_status_content_warning'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='name', + field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.HtmlField(blank=True, null=True), + ), + ] diff --git a/bookwyrm/migrations/0028_remove_book_author_text.py b/bookwyrm/migrations/0028_remove_book_author_text.py new file mode 100644 index 00000000..8743c910 --- /dev/null +++ b/bookwyrm/migrations/0028_remove_book_author_text.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.7 on 2020-12-21 19:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0027_auto_20201220_2007'), + ] + + operations = [ + migrations.RemoveField( + model_name='book', + name='author_text', + ), + ] diff --git a/bookwyrm/migrations/0029_auto_20201221_2014.py b/bookwyrm/migrations/0029_auto_20201221_2014.py new file mode 100644 index 00000000..ebf27a74 --- /dev/null +++ b/bookwyrm/migrations/0029_auto_20201221_2014.py @@ -0,0 +1,61 @@ +# Generated by Django 3.0.7 on 2020-12-21 20:14 + +import bookwyrm.models.fields +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0028_remove_book_author_text'), + ] + + operations = [ + migrations.RemoveField( + model_name='author', + name='last_sync_date', + ), + migrations.RemoveField( + model_name='author', + name='sync', + ), + migrations.RemoveField( + model_name='book', + name='last_sync_date', + ), + migrations.RemoveField( + model_name='book', + name='sync', + ), + migrations.RemoveField( + model_name='book', + name='sync_cover', + ), + migrations.AddField( + model_name='author', + name='goodreads_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='author', + name='last_edited_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='author', + name='librarything_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='last_edited_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='author', + name='origin_id', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/migrations/0030_auto_20201224_1939.py b/bookwyrm/migrations/0030_auto_20201224_1939.py new file mode 100644 index 00000000..6de5d37f --- /dev/null +++ b/bookwyrm/migrations/0030_auto_20201224_1939.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-12-24 19:39 + +import bookwyrm.models.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0029_auto_20201221_2014'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='localname', + field=models.CharField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_localname]), + ), + ] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index b9a2814e..48852cfe 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -2,15 +2,18 @@ import inspect import sys -from .book import Book, Work, Edition +from .book import Book, Work, Edition, BookDataModel from .author import Author from .connector import Connector from .shelf import Shelf, ShelfBook from .status import Status, GeneratedNote, Review, Comment, Quotation -from .status import Favorite, Boost, Notification, ReadThrough +from .status import Boost from .attachment import Image +from .favorite import Favorite +from .notification import Notification +from .readthrough import ReadThrough from .tag import Tag, UserTag @@ -26,7 +29,5 @@ cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass) activity_models = {c[1].activity_serializer.__name__: c[1] \ for c in cls_members if hasattr(c[1], 'activity_serializer')} -def to_activity(activity_json): - ''' link up models and activities ''' - activity_type = activity_json.get('type') - return activity_models[activity_type].to_activity(activity_json) +status_models = [ + c.__name__ for (_, c) in activity_models.items() if issubclass(c, Status)] diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 79973a37..d0cb8d19 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -1,40 +1,25 @@ ''' database schema for info about authors ''' from django.db import models -from django.utils import timezone from bookwyrm import activitypub from bookwyrm.settings import DOMAIN -from .base_model import ActivitypubMixin, BookWyrmModel +from .book import BookDataModel from . import fields -class Author(ActivitypubMixin, BookWyrmModel): +class Author(BookDataModel): ''' basic biographic info ''' - origin_id = models.CharField(max_length=255, null=True) - openlibrary_key = fields.CharField( + wikipedia_link = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True) - sync = models.BooleanField(default=True) - last_sync_date = models.DateTimeField(default=timezone.now) - wikipedia_link = fields.CharField(max_length=255, blank=True, null=True, deduplication_field=True) # idk probably other keys would be useful here? born = fields.DateTimeField(blank=True, null=True) died = fields.DateTimeField(blank=True, null=True) - name = fields.CharField(max_length=255) + name = fields.CharField(max_length=255, deduplication_field=True) aliases = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) - bio = fields.TextField(null=True, blank=True) - - def save(self, *args, **kwargs): - ''' can't be abstract for query reasons, but you shouldn't USE it ''' - if self.id and not self.remote_id: - self.remote_id = self.get_remote_id() - - if not self.id: - self.origin_id = self.remote_id - self.remote_id = None - return super().save(*args, **kwargs) + bio = fields.HtmlField(null=True, blank=True) def get_remote_id(self): ''' editions and works both use "book" instead of model_name ''' diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index f44797ab..b212d693 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -14,16 +14,9 @@ from django.dispatch import receiver from bookwyrm import activitypub from bookwyrm.settings import DOMAIN, PAGE_LENGTH -from .fields import RemoteIdField +from .fields import ImageField, ManyToManyField, RemoteIdField -PrivacyLevels = models.TextChoices('Privacy', [ - 'public', - 'unlisted', - 'followers', - 'direct' -]) - class BookWyrmModel(models.Model): ''' shared fields ''' created_date = models.DateTimeField(auto_now_add=True) @@ -42,8 +35,14 @@ class BookWyrmModel(models.Model): ''' this is just here to provide default fields for other models ''' abstract = True + @property + def local_path(self): + ''' how to link to this object in the local app ''' + return self.get_remote_id().replace('https://%s' % DOMAIN, '') + @receiver(models.signals.post_save) +#pylint: disable=unused-argument def execute_after_save(sender, instance, created, *args, **kwargs): ''' set the remote_id after save (when the id is available) ''' if not created or not hasattr(instance, 'get_remote_id'): @@ -67,6 +66,33 @@ class ActivitypubMixin: activity_serializer = lambda: {} reverse_unfurl = False + def __init__(self, *args, **kwargs): + ''' collect some info on model fields ''' + self.image_fields = [] + self.many_to_many_fields = [] + self.simple_fields = [] # "simple" + for field in self._meta.get_fields(): + if not hasattr(field, 'field_to_activity'): + continue + + if isinstance(field, ImageField): + self.image_fields.append(field) + elif isinstance(field, ManyToManyField): + self.many_to_many_fields.append(field) + else: + self.simple_fields.append(field) + + self.activity_fields = self.image_fields + \ + self.many_to_many_fields + self.simple_fields + + self.deserialize_reverse_fields = self.deserialize_reverse_fields \ + if hasattr(self, 'deserialize_reverse_fields') else [] + self.serialize_reverse_fields = self.serialize_reverse_fields \ + if hasattr(self, 'serialize_reverse_fields') else [] + + super().__init__(*args, **kwargs) + + @classmethod def find_existing_by_remote_id(cls, remote_id): ''' look up a remote id in the db ''' @@ -83,7 +109,7 @@ class ActivitypubMixin: not field.deduplication_field: continue - value = data.get(field.activitypub_field) + value = data.get(field.get_activitypub_field()) if not value: continue filters.append({field.name: value}) @@ -114,19 +140,8 @@ class ActivitypubMixin: def to_activity(self): ''' convert from a model to an activity ''' activity = {} - for field in self._meta.get_fields(): - if not hasattr(field, 'field_to_activity'): - continue - value = field.field_to_activity(getattr(self, field.name)) - if value is None: - continue - - key = field.get_activitypub_field() - if key in activity and isinstance(activity[key], list): - # handles tags on status, which accumulate across fields - activity[key] += value - else: - activity[key] = value + for field in self.activity_fields: + field.set_activity_from_field(activity, self) if hasattr(self, 'serialize_reverse_fields'): # for example, editions of a work @@ -141,9 +156,9 @@ class ActivitypubMixin: return self.activity_serializer(**activity).serialize() - def to_create_activity(self, user): + def to_create_activity(self, user, **kwargs): ''' returns the object wrapped in a Create activity ''' - activity_object = self.to_activity() + activity_object = self.to_activity(**kwargs) signer = pkcs1_15.new(RSA.import_key(user.key_pair.private_key)) content = activity_object['content'] @@ -227,7 +242,9 @@ class OrderedCollectionPageMixin(ActivitypubMixin): ).serialize() -def to_ordered_collection_page(queryset, remote_id, id_only=False, page=1): +# pylint: disable=unused-argument +def to_ordered_collection_page( + queryset, remote_id, id_only=False, page=1, **kwargs): ''' serialize and pagiante a queryset ''' paginated = Paginator(queryset, PAGE_LENGTH) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index bcd4bc04..08189510 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -2,7 +2,6 @@ import re from django.db import models -from django.utils import timezone from model_utils.managers import InheritanceManager from bookwyrm import activitypub @@ -12,10 +11,9 @@ from .base_model import BookWyrmModel from .base_model import ActivitypubMixin, OrderedCollectionPageMixin from . import fields -class Book(ActivitypubMixin, BookWyrmModel): - ''' a generic book, which can mean either an edition or a work ''' +class BookDataModel(ActivitypubMixin, BookWyrmModel): + ''' fields shared between editable book data (books, works, authors) ''' origin_id = models.CharField(max_length=255, null=True, blank=True) - # these identifiers apply to both works and editions openlibrary_key = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True) librarything_key = fields.CharField( @@ -23,20 +21,33 @@ class Book(ActivitypubMixin, BookWyrmModel): goodreads_key = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True) - # info about where the data comes from and where/if to sync - sync = models.BooleanField(default=True) - sync_cover = models.BooleanField(default=True) - last_sync_date = models.DateTimeField(default=timezone.now) + last_edited_by = models.ForeignKey( + 'User', on_delete=models.PROTECT, null=True) + + class Meta: + ''' can't initialize this model, that wouldn't make sense ''' + abstract = True + + def save(self, *args, **kwargs): + ''' ensure that the remote_id is within this instance ''' + if self.id: + self.remote_id = self.get_remote_id() + else: + self.origin_id = self.remote_id + self.remote_id = None + return super().save(*args, **kwargs) + + +class Book(BookDataModel): + ''' a generic book, which can mean either an edition or a work ''' connector = models.ForeignKey( 'Connector', on_delete=models.PROTECT, null=True) - # TODO: edit history - # book/work metadata title = fields.CharField(max_length=255) sort_title = fields.CharField(max_length=255, blank=True, null=True) subtitle = fields.CharField(max_length=255, blank=True, null=True) - description = fields.TextField(blank=True, null=True) + description = fields.HtmlField(blank=True, null=True) languages = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) @@ -48,27 +59,42 @@ class Book(ActivitypubMixin, BookWyrmModel): subject_places = fields.ArrayField( models.CharField(max_length=255), blank=True, null=True, default=list ) - # TODO: include an annotation about the type of authorship (ie, translator) authors = fields.ManyToManyField('Author') - # preformatted authorship string for search and easier display - author_text = models.CharField(max_length=255, blank=True, null=True) - cover = fields.ImageField(upload_to='covers/', blank=True, null=True) + cover = fields.ImageField( + upload_to='covers/', blank=True, null=True, alt_field='alt_text') first_published_date = fields.DateTimeField(blank=True, null=True) published_date = fields.DateTimeField(blank=True, null=True) objects = InheritanceManager() + @property + def author_text(self): + ''' format a list of authors ''' + return ', '.join(a.name for a in self.authors.all()) + + @property + def edition_info(self): + ''' properties of this edition, as a string ''' + items = [ + self.physical_format if hasattr(self, 'physical_format') else None, + self.languages[0] + ' language' if self.languages and \ + self.languages[0] != 'English' else None, + str(self.published_date.year) if self.published_date else None, + ] + return ', '.join(i for i in items if i) + + @property + def alt_text(self): + ''' image alt test ''' + text = '%s cover' % self.title + if self.edition_info: + text += ' (%s)' % self.edition_info + return text + def save(self, *args, **kwargs): ''' can't be abstract for query reasons, but you shouldn't USE it ''' if not isinstance(self, Edition) and not isinstance(self, Work): raise ValueError('Books should be added as Editions or Works') - - if self.id and not self.remote_id: - self.remote_id = self.get_remote_id() - - if not self.id: - self.origin_id = self.remote_id - self.remote_id = None return super().save(*args, **kwargs) def get_remote_id(self): @@ -92,13 +118,22 @@ class Work(OrderedCollectionPageMixin, Book): default_edition = fields.ForeignKey( 'Edition', on_delete=models.PROTECT, - null=True + null=True, + load_remote=False ) def get_default_edition(self): ''' in case the default edition is not set ''' return self.default_edition or self.editions.first() + def to_edition_list(self, **kwargs): + ''' an ordered collection of editions ''' + return self.to_ordered_collection( + self.editions.order_by('-updated_date').all(), + remote_id='%s/editions' % self.remote_id, + **kwargs + ) + activity_serializer = activitypub.Work serialize_reverse_fields = [('editions', 'editions')] deserialize_reverse_fields = [('editions', 'editions')] diff --git a/bookwyrm/models/favorite.py b/bookwyrm/models/favorite.py new file mode 100644 index 00000000..8373b016 --- /dev/null +++ b/bookwyrm/models/favorite.py @@ -0,0 +1,26 @@ +''' like/fav/star a status ''' +from django.db import models +from django.utils import timezone + +from bookwyrm import activitypub +from .base_model import ActivitypubMixin, BookWyrmModel +from . import fields + +class Favorite(ActivitypubMixin, BookWyrmModel): + ''' fav'ing a post ''' + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='actor') + status = fields.ForeignKey( + 'Status', on_delete=models.PROTECT, activitypub_field='object') + + activity_serializer = activitypub.Like + + def save(self, *args, **kwargs): + ''' update user active time ''' + self.user.last_active_date = timezone.now() + self.user.save() + super().save(*args, **kwargs) + + class Meta: + ''' can't fav things twice ''' + unique_together = ('user', 'status') diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index e6878fb9..c6571ff4 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -1,10 +1,10 @@ ''' activitypub-aware django model fields ''' +from dataclasses import MISSING import re from uuid import uuid4 import dateutil.parser from dateutil.parser import ParserError -from django.contrib.auth.models import AbstractUser from django.contrib.postgres.fields import ArrayField as DjangoArrayField from django.core.exceptions import ValidationError from django.core.files.base import ContentFile @@ -12,8 +12,9 @@ from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ from bookwyrm import activitypub -from bookwyrm.settings import DOMAIN from bookwyrm.connectors import get_image +from bookwyrm.sanitize_html import InputHtmlParser +from bookwyrm.settings import DOMAIN def validate_remote_id(value): @@ -25,6 +26,24 @@ def validate_remote_id(value): ) +def validate_localname(value): + ''' make sure localnames look okay ''' + if not re.match(r'^[A-Za-z\-_\.0-9]+$', value): + raise ValidationError( + _('%(value)s is not a valid username'), + params={'value': value}, + ) + + +def validate_username(value): + ''' make sure usernames look okay ''' + if not re.match(r'^[A-Za-z\-_\.0-9]+@[A-Za-z\-_\.0-9]+\.[a-z]{2,}$', value): + raise ValidationError( + _('%(value)s is not a valid username'), + params={'value': value}, + ) + + class ActivitypubFieldMixin: ''' make a database field serializable ''' def __init__(self, *args, \ @@ -38,6 +57,39 @@ class ActivitypubFieldMixin: self.activitypub_field = activitypub_field super().__init__(*args, **kwargs) + + def set_field_from_activity(self, instance, data): + ''' helper function for assinging a value to the field ''' + try: + value = getattr(data, self.get_activitypub_field()) + except AttributeError: + # masssively hack-y workaround for boosts + if self.get_activitypub_field() != 'attributedTo': + raise + value = getattr(data, 'actor') + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + setattr(instance, self.name, formatted) + + + def set_activity_from_field(self, activity, instance): + ''' update the json object ''' + value = getattr(instance, self.name) + formatted = self.field_to_activity(value) + if formatted is None: + return + + key = self.get_activitypub_field() + # TODO: surely there's a better way + if instance.__class__.__name__ == 'Boost' and key == 'attributedTo': + key = 'actor' + if isinstance(activity.get(key), list): + activity[key] += formatted + else: + activity[key] = formatted + + def field_to_activity(self, value): ''' formatter to convert a model value into activitypub ''' if hasattr(self, 'activitypub_wrapper'): @@ -61,12 +113,19 @@ class ActivitypubFieldMixin: class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin): ''' default (de)serialization for foreign key and one to one ''' + def __init__(self, *args, load_remote=True, **kwargs): + self.load_remote = load_remote + super().__init__(*args, **kwargs) + def field_from_activity(self, value): if not value: return None related_model = self.related_model if isinstance(value, dict) and value.get('id'): + if not self.load_remote: + # only look in the local database + return related_model.find_existing(value) # this is an activitypub object, which we can deserialize activity_serializer = related_model.activity_serializer return activity_serializer(**value).to_model(related_model) @@ -77,6 +136,9 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin): # we don't know what this is, ignore it return None # gets or creates the model field from the remote id + if not self.load_remote: + # only look in the local database + return related_model.find_existing_by_remote_id(value) return activitypub.resolve_remote_id(related_model, value) @@ -94,7 +156,7 @@ class RemoteIdField(ActivitypubFieldMixin, models.CharField): class UsernameField(ActivitypubFieldMixin, models.CharField): ''' activitypub-aware username field ''' - def __init__(self, activitypub_field='preferredUsername'): + def __init__(self, activitypub_field='preferredUsername', **kwargs): self.activitypub_field = activitypub_field # I don't totally know why pylint is mad at this, but it makes it work super( #pylint: disable=bad-super-call @@ -103,7 +165,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): _('username'), max_length=150, unique=True, - validators=[AbstractUser.username_validator], + validators=[validate_username], error_messages={ 'unique': _('A user with that username already exists.'), }, @@ -123,6 +185,52 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): return value.split('@')[0] +PrivacyLevels = models.TextChoices('Privacy', [ + 'public', + 'unlisted', + 'followers', + 'direct' +]) + +class PrivacyField(ActivitypubFieldMixin, models.CharField): + ''' this maps to two differente activitypub fields ''' + public = 'https://www.w3.org/ns/activitystreams#Public' + def __init__(self, *args, **kwargs): + super().__init__( + *args, max_length=255, + choices=PrivacyLevels.choices, default='public') + + def set_field_from_activity(self, instance, data): + to = data.to + cc = data.cc + if to == [self.public]: + setattr(instance, self.name, 'public') + elif cc == []: + setattr(instance, self.name, 'direct') + elif self.public in cc: + setattr(instance, self.name, 'unlisted') + else: + setattr(instance, self.name, 'followers') + + def set_activity_from_field(self, activity, instance): + mentions = [u.remote_id for u in instance.mention_users.all()] + # this is a link to the followers list + followers = instance.user.__class__._meta.get_field('followers')\ + .field_to_activity(instance.user.followers) + if instance.privacy == 'public': + activity['to'] = [self.public] + activity['cc'] = [followers] + mentions + elif instance.privacy == 'unlisted': + activity['to'] = [followers] + activity['cc'] = [self.public] + mentions + elif instance.privacy == 'followers': + activity['to'] = [followers] + activity['cc'] = mentions + if instance.privacy == 'direct': + activity['to'] = mentions + activity['cc'] = [] + + class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey): ''' activitypub-aware foreign key field ''' def field_to_activity(self, value): @@ -145,6 +253,14 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): self.link_only = link_only super().__init__(*args, **kwargs) + def set_field_from_activity(self, instance, data): + ''' helper function for assinging a value to the field ''' + value = getattr(data, self.get_activitypub_field()) + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + getattr(instance, self.name).set(formatted) + def field_to_activity(self, value): if self.link_only: return '%s/%s' % (value.instance.remote_id, self.name) @@ -152,6 +268,8 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): def field_from_activity(self, value): items = [] + if value is None or value is MISSING: + return [] for remote_id in value: try: validate_remote_id(remote_id) @@ -189,6 +307,8 @@ class TagField(ManyToManyField): for link_json in value: link = activitypub.Link(**link_json) tag_type = link.type if link.type != 'Mention' else 'Person' + if tag_type == 'Book': + tag_type = 'Edition' if tag_type != self.related_model.activity_serializer.type: # tags can contain multiple types continue @@ -198,20 +318,45 @@ class TagField(ManyToManyField): return items -def image_serializer(value): +def image_serializer(value, alt): ''' helper for serializing images ''' if value and hasattr(value, 'url'): url = value.url else: return None url = 'https://%s%s' % (DOMAIN, url) - return activitypub.Image(url=url) + return activitypub.Image(url=url, name=alt) class ImageField(ActivitypubFieldMixin, models.ImageField): ''' activitypub-aware image field ''' - def field_to_activity(self, value): - return image_serializer(value) + def __init__(self, *args, alt_field=None, **kwargs): + self.alt_field = alt_field + super().__init__(*args, **kwargs) + + # pylint: disable=arguments-differ + def set_field_from_activity(self, instance, data, save=True): + ''' helper function for assinging a value to the field ''' + value = getattr(data, self.get_activitypub_field()) + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + getattr(instance, self.name).save(*formatted, save=save) + + def set_activity_from_field(self, activity, instance): + value = getattr(instance, self.name) + if value is None: + return + alt_text = getattr(instance, self.alt_field) + formatted = self.field_to_activity(value, alt_text) + + key = self.get_activitypub_field() + activity[key] = formatted + + + def field_to_activity(self, value, alt=None): + return image_serializer(value, alt) + def field_from_activity(self, value): image_slug = value @@ -255,6 +400,15 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField): except (ParserError, TypeError): return None +class HtmlField(ActivitypubFieldMixin, models.TextField): + ''' a text field for storing html ''' + def field_from_activity(self, value): + if not value or value == MISSING: + return None + sanitizer = InputHtmlParser() + sanitizer.feed(value) + return sanitizer.get_output() + class ArrayField(ActivitypubFieldMixin, DjangoArrayField): ''' activitypub-aware array field ''' def field_to_activity(self, value): diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index fe39325f..1ebe9b31 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -2,14 +2,13 @@ import re import dateutil.parser +from django.contrib.postgres.fields import JSONField from django.db import models from django.utils import timezone -from bookwyrm import books_manager -from bookwyrm.connectors import ConnectorException +from bookwyrm.connectors import connector_manager from bookwyrm.models import ReadThrough, User, Book -from bookwyrm.utils.fields import JSONField -from .base_model import PrivacyLevels +from .fields import PrivacyLevels # Mapping goodreads -> bookwyrm shelf titles. @@ -72,12 +71,12 @@ class ImportItem(models.Model): def get_book_from_isbn(self): ''' search by isbn ''' - search_result = books_manager.first_search_result( + search_result = connector_manager.first_search_result( self.isbn, min_confidence=0.999 ) if search_result: # raises ConnectorException - return books_manager.get_or_create_book(search_result.key) + return search_result.connector.get_or_create_book(search_result.key) return None @@ -87,12 +86,12 @@ class ImportItem(models.Model): self.data['Title'], self.data['Author'] ) - search_result = books_manager.first_search_result( + search_result = connector_manager.first_search_result( search_term, min_confidence=0.999 ) if search_result: # raises ConnectorException - return books_manager.get_or_create_book(search_result.key) + return search_result.connector.get_or_create_book(search_result.key) return None diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py new file mode 100644 index 00000000..4ce5dcea --- /dev/null +++ b/bookwyrm/models/notification.py @@ -0,0 +1,33 @@ +''' alert a user to activity ''' +from django.db import models +from .base_model import BookWyrmModel + + +NotificationType = models.TextChoices( + 'NotificationType', + 'FAVORITE REPLY MENTION TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT') + +class Notification(BookWyrmModel): + ''' you've been tagged, liked, followed, etc ''' + user = models.ForeignKey('User', on_delete=models.PROTECT) + related_book = models.ForeignKey( + 'Edition', on_delete=models.PROTECT, null=True) + related_user = models.ForeignKey( + 'User', + on_delete=models.PROTECT, null=True, related_name='related_user') + related_status = models.ForeignKey( + 'Status', on_delete=models.PROTECT, null=True) + related_import = models.ForeignKey( + 'ImportJob', on_delete=models.PROTECT, null=True) + read = models.BooleanField(default=False) + notification_type = models.CharField( + max_length=255, choices=NotificationType.choices) + + class Meta: + ''' checks if notifcation is in enum list for valid types ''' + constraints = [ + models.CheckConstraint( + check=models.Q(notification_type__in=NotificationType.values), + name="notification_type_valid", + ) + ] diff --git a/bookwyrm/models/readthrough.py b/bookwyrm/models/readthrough.py new file mode 100644 index 00000000..61cac7e6 --- /dev/null +++ b/bookwyrm/models/readthrough.py @@ -0,0 +1,26 @@ +''' progress in a book ''' +from django.db import models +from django.utils import timezone + +from .base_model import BookWyrmModel + + +class ReadThrough(BookWyrmModel): + ''' Store progress through a book in the database. ''' + user = models.ForeignKey('User', on_delete=models.PROTECT) + book = models.ForeignKey('Edition', on_delete=models.PROTECT) + pages_read = models.IntegerField( + null=True, + blank=True) + start_date = models.DateTimeField( + blank=True, + null=True) + finish_date = models.DateTimeField( + blank=True, + null=True) + + def save(self, *args, **kwargs): + ''' update user active time ''' + self.user.last_active_date = timezone.now() + self.user.save() + super().save(*args, **kwargs) diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index 8913b9ab..0f3c1dab 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -37,7 +37,7 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel): activity_serializer = activitypub.Follow - def get_remote_id(self, status=None): + def get_remote_id(self, status=None):# pylint: disable=arguments-differ ''' use shelf identifier in remote_id ''' status = status or 'follows' base_path = self.user_subject.remote_id @@ -54,7 +54,7 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel): def to_reject_activity(self): - ''' generate an Accept for this follow request ''' + ''' generate a Reject for this follow request ''' return activitypub.Reject( id=self.get_remote_id(status='rejects'), actor=self.user_object.remote_id, diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index fc63d198..69df43b4 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -3,8 +3,8 @@ import re from django.db import models from bookwyrm import activitypub -from .base_model import BookWyrmModel -from .base_model import OrderedCollectionMixin, PrivacyLevels +from .base_model import ActivitypubMixin, BookWyrmModel +from .base_model import OrderedCollectionMixin from . import fields @@ -18,7 +18,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): privacy = fields.CharField( max_length=255, default='public', - choices=PrivacyLevels.choices + choices=fields.PrivacyLevels.choices ) books = models.ManyToManyField( 'Edition', @@ -51,7 +51,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): unique_together = ('user', 'identifier') -class ShelfBook(BookWyrmModel): +class ShelfBook(ActivitypubMixin, BookWyrmModel): ''' many to many join table for books and shelves ''' book = fields.ForeignKey( 'Edition', on_delete=models.PROTECT, activitypub_field='object') diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 55036f2c..2494c458 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -1,12 +1,16 @@ ''' models for storing different kinds of Activities ''' -from django.utils import timezone +from dataclasses import MISSING +import re + +from django.apps import apps from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from django.utils import timezone from model_utils.managers import InheritanceManager from bookwyrm import activitypub from .base_model import ActivitypubMixin, OrderedCollectionPageMixin -from .base_model import BookWyrmModel, PrivacyLevels +from .base_model import BookWyrmModel from . import fields from .fields import image_serializer @@ -14,17 +18,15 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ''' any post, like a reply to a review, etc ''' user = fields.ForeignKey( 'User', on_delete=models.PROTECT, activitypub_field='attributedTo') - content = fields.TextField(blank=True, null=True) + content = fields.HtmlField(blank=True, null=True) mention_users = fields.TagField('User', related_name='mention_user') mention_books = fields.TagField('Edition', related_name='mention_book') local = models.BooleanField(default=True) - privacy = models.CharField( - max_length=255, - default='public', - choices=PrivacyLevels.choices - ) + content_warning = fields.CharField( + max_length=500, blank=True, null=True, activitypub_field='summary') + privacy = fields.PrivacyField(max_length=255) sensitive = fields.BooleanField(default=False) - # the created date can't be this, because of receiving federated posts + # created date is different than publish date because of federated posts published_date = fields.DateTimeField( default=timezone.now, activitypub_field='published') deleted = models.BooleanField(default=False) @@ -48,18 +50,45 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): serialize_reverse_fields = [('attachments', 'attachment')] deserialize_reverse_fields = [('attachments', 'attachment')] - #----- replies collection activitypub ----# + @classmethod + def ignore_activity(cls, activity): + ''' keep notes if they are replies to existing statuses ''' + if activity.type != 'Note': + return False + if cls.objects.filter( + remote_id=activity.inReplyTo).exists(): + return False + + # keep notes if they mention local users + if activity.tag == MISSING or activity.tag is None: + return True + tags = [l['href'] for l in activity.tag if l['type'] == 'Mention'] + for tag in tags: + user_model = apps.get_model('bookwyrm.User', require_ready=True) + if user_model.objects.filter( + remote_id=tag, local=True).exists(): + # we found a mention of a known use boost + return False + return True + @classmethod def replies(cls, status): ''' load all replies to a status. idk if there's a better way to write this so it's just a property ''' - return cls.objects.filter(reply_parent=status).select_subclasses() + return cls.objects.filter( + reply_parent=status + ).select_subclasses().order_by('published_date') @property def status_type(self): ''' expose the type of status for the ui using activity type ''' return self.activity_serializer.__name__ + @property + def boostable(self): + ''' you can't boost dms ''' + return self.privacy in ['unlisted', 'public'] + def to_replies(self, **kwargs): ''' helper function for loading AP serialized replies to a status ''' return self.to_ordered_collection( @@ -68,7 +97,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): **kwargs ) - def to_activity(self, pure=False): + def to_activity(self, pure=False):# pylint: disable=arguments-differ ''' return tombstone if the status is deleted ''' if self.deleted: return activitypub.Tombstone( @@ -80,37 +109,18 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): activity = ActivitypubMixin.to_activity(self) activity['replies'] = self.to_replies() - # privacy controls - public = 'https://www.w3.org/ns/activitystreams#Public' - mentions = [u.remote_id for u in self.mention_users.all()] - # this is a link to the followers list: - followers = self.user.__class__._meta.get_field('followers')\ - .field_to_activity(self.user.followers) - if self.privacy == 'public': - activity['to'] = [public] - activity['cc'] = [followers] + mentions - elif self.privacy == 'unlisted': - activity['to'] = [followers] - activity['cc'] = [public] + mentions - elif self.privacy == 'followers': - activity['to'] = [followers] - activity['cc'] = mentions - if self.privacy == 'direct': - activity['to'] = mentions - activity['cc'] = [] - # "pure" serialization for non-bookwyrm instances - if pure: + if pure and hasattr(self, 'pure_content'): activity['content'] = self.pure_content if 'name' in activity: activity['name'] = self.pure_name activity['type'] = self.pure_type activity['attachment'] = [ - image_serializer(b.cover) for b in self.mention_books.all() \ - if b.cover] - if hasattr(self, 'book'): + image_serializer(b.cover, b.alt_text) \ + for b in self.mention_books.all()[:4] if b.cover] + if hasattr(self, 'book') and self.book.cover: activity['attachment'].append( - image_serializer(self.book.cover) + image_serializer(self.book.cover, self.book.alt_text) ) return activity @@ -147,8 +157,8 @@ class Comment(Status): @property def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' - return self.content + '

(comment on "%s")' % \ - (self.book.remote_id, self.book.title) + return '%s

(comment on "%s")

' % \ + (self.content, self.book.remote_id, self.book.title) activity_serializer = activitypub.Comment pure_type = 'Note' @@ -156,15 +166,17 @@ class Comment(Status): class Quotation(Status): ''' like a review but without a rating and transient ''' - quote = fields.TextField() + quote = fields.HtmlField() book = fields.ForeignKey( 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') @property def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' - return '"%s"
-- "%s"

%s' % ( - self.quote, + quote = re.sub(r'^

', '

"', self.quote) + quote = re.sub(r'

$', '"

', quote) + return '%s

-- "%s"

%s' % ( + quote, self.book.remote_id, self.book.title, self.content, @@ -190,6 +202,7 @@ class Review(Status): def pure_name(self): ''' clarify review names for mastodon serialization ''' if self.rating: + #pylint: disable=bad-string-format-type return 'Review of "%s" (%d stars): %s' % ( self.book.title, self.rating, @@ -203,33 +216,12 @@ class Review(Status): @property def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' - return self.content + '

("%s")' % \ - (self.book.remote_id, self.book.title) + return self.content activity_serializer = activitypub.Review pure_type = 'Article' -class Favorite(ActivitypubMixin, BookWyrmModel): - ''' fav'ing a post ''' - user = fields.ForeignKey( - 'User', on_delete=models.PROTECT, activitypub_field='actor') - status = fields.ForeignKey( - 'Status', on_delete=models.PROTECT, activitypub_field='object') - - activity_serializer = activitypub.Like - - def save(self, *args, **kwargs): - ''' update user active time ''' - self.user.last_active_date = timezone.now() - self.user.save() - super().save(*args, **kwargs) - - class Meta: - ''' can't fav things twice ''' - unique_together = ('user', 'status') - - class Boost(Status): ''' boost'ing a post ''' boosted_status = fields.ForeignKey( @@ -239,59 +231,20 @@ class Boost(Status): activitypub_field='object', ) + def __init__(self, *args, **kwargs): + ''' the user field is "actor" here instead of "attributedTo" ''' + super().__init__(*args, **kwargs) + + reserve_fields = ['user', 'boosted_status'] + self.simple_fields = [f for f in self.simple_fields if \ + f.name in reserve_fields] + self.activity_fields = self.simple_fields + self.many_to_many_fields = [] + self.image_fields = [] + self.deserialize_reverse_fields = [] + activity_serializer = activitypub.Boost # This constraint can't work as it would cross tables. # class Meta: # unique_together = ('user', 'boosted_status') - - -class ReadThrough(BookWyrmModel): - ''' Store progress through a book in the database. ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - book = models.ForeignKey('Book', on_delete=models.PROTECT) - pages_read = models.IntegerField( - null=True, - blank=True) - start_date = models.DateTimeField( - blank=True, - null=True) - finish_date = models.DateTimeField( - blank=True, - null=True) - - def save(self, *args, **kwargs): - ''' update user active time ''' - self.user.last_active_date = timezone.now() - self.user.save() - super().save(*args, **kwargs) - - -NotificationType = models.TextChoices( - 'NotificationType', - 'FAVORITE REPLY MENTION TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT') - -class Notification(BookWyrmModel): - ''' you've been tagged, liked, followed, etc ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - related_book = models.ForeignKey( - 'Edition', on_delete=models.PROTECT, null=True) - related_user = models.ForeignKey( - 'User', - on_delete=models.PROTECT, null=True, related_name='related_user') - related_status = models.ForeignKey( - 'Status', on_delete=models.PROTECT, null=True) - related_import = models.ForeignKey( - 'ImportJob', on_delete=models.PROTECT, null=True) - read = models.BooleanField(default=False) - notification_type = models.CharField( - max_length=255, choices=NotificationType.choices) - - class Meta: - ''' checks if notifcation is in enum list for valid types ''' - constraints = [ - models.CheckConstraint( - check=models.Q(notification_type__in=NotificationType.values), - name="notification_type_valid", - ) - ] diff --git a/bookwyrm/models/tag.py b/bookwyrm/models/tag.py index 940b4192..6e0ba8ab 100644 --- a/bookwyrm/models/tag.py +++ b/bookwyrm/models/tag.py @@ -17,7 +17,9 @@ class Tag(OrderedCollectionMixin, BookWyrmModel): @classmethod def book_queryset(cls, identifier): ''' county of books associated with this tag ''' - return cls.objects.filter(identifier=identifier) + return cls.objects.filter( + identifier=identifier + ).order_by('-updated_date') @property def collection_queryset(self): @@ -64,7 +66,7 @@ class UserTag(BookWyrmModel): id='%s#remove' % self.remote_id, actor=user.remote_id, object=self.book.to_activity(), - target=self.to_activity(), + target=self.remote_id, ).serialize() diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 63549d36..4cbe387f 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -1,6 +1,8 @@ ''' database schema for user data ''' +import re from urllib.parse import urlparse +from django.apps import apps from django.contrib.auth.models import AbstractUser from django.db import models from django.dispatch import receiver @@ -12,6 +14,7 @@ from bookwyrm.models.status import Status, Review from bookwyrm.settings import DOMAIN from bookwyrm.signatures import create_key_pair from bookwyrm.tasks import app +from bookwyrm.utils import regex from .base_model import OrderedCollectionPageMixin from .base_model import ActivitypubMixin, BookWyrmModel from .federated_server import FederatedServer @@ -42,18 +45,20 @@ class User(OrderedCollectionPageMixin, AbstractUser): blank=True, ) outbox = fields.RemoteIdField(unique=True) - summary = fields.TextField(default='') + summary = fields.HtmlField(null=True, blank=True) local = models.BooleanField(default=False) bookwyrm_user = fields.BooleanField(default=True) localname = models.CharField( max_length=255, null=True, - unique=True + unique=True, + validators=[fields.validate_localname], ) # name is your display name, which you can change at will - name = fields.CharField(max_length=100, default='') + name = fields.CharField(max_length=100, null=True, blank=True) avatar = fields.ImageField( - upload_to='avatars/', blank=True, null=True, activitypub_field='icon') + upload_to='avatars/', blank=True, null=True, + activitypub_field='icon', alt_field='alt_text') followers = fields.ManyToManyField( 'self', link_only=True, @@ -90,20 +95,37 @@ class User(OrderedCollectionPageMixin, AbstractUser): last_active_date = models.DateTimeField(auto_now=True) manually_approves_followers = fields.BooleanField(default=False) + name_field = 'username' + @property + def alt_text(self): + ''' alt text with username ''' + return 'avatar for %s' % (self.localname or self.username) + @property def display_name(self): ''' show the cleanest version of the user's name possible ''' - if self.name != '': + if self.name and self.name != '': return self.name return self.localname or self.username activity_serializer = activitypub.Person - def to_outbox(self, **kwargs): + def to_outbox(self, filter_type=None, **kwargs): ''' an ordered collection of statuses ''' - queryset = Status.objects.filter( + if filter_type: + filter_class = apps.get_model( + 'bookwyrm.%s' % filter_type, require_ready=True) + if not issubclass(filter_class, Status): + raise TypeError( + 'filter_status_class must be a subclass of models.Status') + queryset = filter_class.objects + else: + queryset = Status.objects + + queryset = queryset.filter( user=self, deleted=False, + privacy__in=['public', 'unlisted'], ).select_subclasses().order_by('-published_date') return self.to_ordered_collection(queryset, \ remote_id=self.outbox, **kwargs) @@ -111,14 +133,22 @@ class User(OrderedCollectionPageMixin, AbstractUser): def to_following_activity(self, **kwargs): ''' activitypub following list ''' remote_id = '%s/following' % self.remote_id - return self.to_ordered_collection(self.following.all(), \ - remote_id=remote_id, id_only=True, **kwargs) + return self.to_ordered_collection( + self.following.order_by('-updated_date').all(), + remote_id=remote_id, + id_only=True, + **kwargs + ) def to_followers_activity(self, **kwargs): ''' activitypub followers list ''' remote_id = '%s/followers' % self.remote_id - return self.to_ordered_collection(self.followers.all(), \ - remote_id=remote_id, id_only=True, **kwargs) + return self.to_ordered_collection( + self.followers.order_by('-updated_date').all(), + remote_id=remote_id, + id_only=True, + **kwargs + ) def to_activity(self): ''' override default AP serializer to add context object @@ -140,26 +170,28 @@ class User(OrderedCollectionPageMixin, AbstractUser): def save(self, *args, **kwargs): ''' populate fields for new local users ''' # this user already exists, no need to populate fields - if self.id: - return super().save(*args, **kwargs) - - if not self.local: + if not self.local and not re.match(regex.full_username, self.username): # generate a username that uses the domain (webfinger format) actor_parts = urlparse(self.remote_id) self.username = '%s@%s' % (self.username, actor_parts.netloc) return super().save(*args, **kwargs) + if self.id or not self.local: + return super().save(*args, **kwargs) + # populate fields for local users - self.remote_id = 'https://%s/user/%s' % (DOMAIN, self.username) - self.localname = self.username - self.username = '%s@%s' % (self.username, DOMAIN) - self.actor = self.remote_id + self.remote_id = 'https://%s/user/%s' % (DOMAIN, self.localname) self.inbox = '%s/inbox' % self.remote_id self.shared_inbox = 'https://%s/inbox' % DOMAIN self.outbox = '%s/outbox' % self.remote_id return super().save(*args, **kwargs) + @property + def local_path(self): + ''' this model doesn't inherit bookwyrm model, so here we are ''' + return '/user/%s' % (self.localname or self.username) + class KeyPair(ActivitypubMixin, BookWyrmModel): ''' public and private keys for a user ''' @@ -265,7 +297,7 @@ def get_or_create_remote_server(domain): @app.task def get_remote_reviews(outbox): ''' ingest reviews by a new remote bookwyrm user ''' - outbox_page = outbox + '?page=true' + outbox_page = outbox + '?page=true&type=Review' data = get_data(outbox_page) # TODO: pagination? diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py index 38b48282..88377d33 100644 --- a/bookwyrm/outgoing.py +++ b/bookwyrm/outgoing.py @@ -2,14 +2,18 @@ import re from django.db import IntegrityError, transaction -from django.http import HttpResponseNotFound, JsonResponse +from django.http import JsonResponse +from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_GET +from markdown import markdown from requests import HTTPError from bookwyrm import activitypub from bookwyrm import models from bookwyrm.connectors import get_data, ConnectorException from bookwyrm.broadcast import broadcast +from bookwyrm.sanitize_html import InputHtmlParser from bookwyrm.status import create_notification from bookwyrm.status import create_generated_note from bookwyrm.status import delete_status @@ -18,19 +22,16 @@ from bookwyrm.utils import regex @csrf_exempt +@require_GET def outbox(request, username): ''' outbox for the requested user ''' - if request.method != 'GET': - return HttpResponseNotFound() + user = get_object_or_404(models.User, localname=username) + filter_type = request.GET.get('type') + if filter_type not in models.status_models: + filter_type = None - try: - user = models.User.objects.get(localname=username) - except models.User.DoesNotExist: - return HttpResponseNotFound() - - # collection overview return JsonResponse( - user.to_outbox(**request.GET), + user.to_outbox(**request.GET, filter_type=filter_type), encoder=activitypub.ActivityEncoder ) @@ -40,6 +41,9 @@ def handle_remote_webfinger(query): user = None # usernames could be @user@domain or user@domain + if not query: + return None + if query[0] == '@': query = query[1:] @@ -162,22 +166,23 @@ def handle_imported_book(user, item, include_reviews, privacy): if not item.book: return - if item.shelf: + existing_shelf = models.ShelfBook.objects.filter( + book=item.book, added_by=user).exists() + + # shelve the book if it hasn't been shelved already + if item.shelf and not existing_shelf: desired_shelf = models.Shelf.objects.get( identifier=item.shelf, user=user ) - # shelve the book if it hasn't been shelved already - shelf_book, created = models.ShelfBook.objects.get_or_create( + shelf_book = models.ShelfBook.objects.create( book=item.book, shelf=desired_shelf, added_by=user) - if created: - broadcast(user, shelf_book.to_add_activity(user), privacy=privacy) + broadcast(user, shelf_book.to_add_activity(user), privacy=privacy) - # only add new read-throughs if the item isn't already shelved - for read in item.reads: - read.book = item.book - read.user = user - read.save() + for read in item.reads: + read.book = item.book + read.user = user + read.save() if include_reviews and (item.rating or item.review): review_title = 'Review of {!r} on Goodreads'.format( @@ -209,15 +214,72 @@ def handle_delete_status(user, status): def handle_status(user, form): ''' generic handler for statuses ''' - status = form.save() + status = form.save(commit=False) + if not status.sensitive and status.content_warning: + # the cw text field remains populated when you click "remove" + status.content_warning = None + status.save() # inspect the text for user tags - text = status.content - matches = re.finditer( - regex.username, - text - ) - for match in matches: + content = status.content + for (mention_text, mention_user) in find_mentions(content): + # add them to status mentions fk + status.mention_users.add(mention_user) + + # turn the mention into a link + content = re.sub( + r'%s([^@]|$)' % mention_text, + r'%s\g<1>' % \ + (mention_user.remote_id, mention_text), + content) + + # add reply parent to mentions and notify + if status.reply_parent: + status.mention_users.add(status.reply_parent.user) + for mention_user in status.reply_parent.mention_users.all(): + status.mention_users.add(mention_user) + + if status.reply_parent.user.local: + create_notification( + status.reply_parent.user, + 'REPLY', + related_user=user, + related_status=status + ) + + # deduplicate mentions + status.mention_users.set(set(status.mention_users.all())) + # create mention notifications + for mention_user in status.mention_users.all(): + if status.reply_parent and mention_user == status.reply_parent.user: + continue + if mention_user.local: + create_notification( + mention_user, + 'MENTION', + related_user=user, + related_status=status + ) + + # don't apply formatting to generated notes + if not isinstance(status, models.GeneratedNote): + status.content = to_markdown(content) + # do apply formatting to quotes + if hasattr(status, 'quote'): + status.quote = to_markdown(status.quote) + + status.save() + + broadcast(user, status.to_create_activity(user), software='bookwyrm') + + # re-format the activity for non-bookwyrm servers + remote_activity = status.to_create_activity(user, pure=True) + broadcast(user, remote_activity, software='other') + + +def find_mentions(content): + ''' detect @mentions in raw status content ''' + for match in re.finditer(regex.strict_username, content): username = match.group().strip().split('@')[1:] if len(username) == 1: # this looks like a local user (@user), fill in the domain @@ -228,48 +290,21 @@ def handle_status(user, form): if not mention_user: # we can ignore users we don't know about continue - # add them to status mentions fk - status.mention_users.add(mention_user) - # create notification if the mentioned user is local - if mention_user.local: - create_notification( - mention_user, - 'MENTION', - related_user=user, - related_status=status - ) - status.save() - - # notify reply parent or tagged users - if status.reply_parent and status.reply_parent.user.local: - create_notification( - status.reply_parent.user, - 'REPLY', - related_user=user, - related_status=status - ) - - broadcast(user, status.to_create_activity(user), software='bookwyrm') - - # re-format the activity for non-bookwyrm servers - if hasattr(status, 'pure_activity_serializer'): - remote_activity = status.to_create_activity(user, pure=True) - broadcast(user, remote_activity, software='other') + yield (match.group(), mention_user) -def handle_tag(user, tag): - ''' tag a book ''' - broadcast(user, tag.to_add_activity(user)) - - -def handle_untag(user, book, name): - ''' tag a book ''' - book = models.Book.objects.get(id=book) - tag = models.Tag.objects.get(name=name, book=book, user=user) - tag_activity = tag.to_remove_activity(user) - tag.delete() - - broadcast(user, tag_activity) +def to_markdown(content): + ''' catch links and convert to markdown ''' + content = re.sub( + r'([^(href=")])(https?:\/\/([A-Za-z\.\-_\/]+' \ + r'\.[A-Za-z]{2,}[A-Za-z\.\-_\/]+))', + r'\g<1>\g<3>', + content) + content = markdown(content) + # sanitize resulting html + sanitizer = InputHtmlParser() + sanitizer.feed(content) + return sanitizer.get_output() def handle_favorite(user, status): @@ -312,15 +347,19 @@ def handle_unfavorite(user, status): def handle_boost(user, status): ''' a user wishes to boost a status ''' + # is it boostable? + if not status.boostable: + return + if models.Boost.objects.filter( boosted_status=status, user=user).exists(): # you already boosted that. return boost = models.Boost.objects.create( boosted_status=status, + privacy=status.privacy, user=user, ) - boost.save() boost_activity = boost.to_activity() broadcast(user, boost_activity) @@ -344,9 +383,9 @@ def handle_unboost(user, status): broadcast(user, activity) -def handle_update_book(user, book): +def handle_update_book_data(user, item): ''' broadcast the news about our book ''' - broadcast(user, book.to_update_activity(user)) + broadcast(user, item.to_update_activity(user)) def handle_update_user(user): diff --git a/bookwyrm/sanitize_html.py b/bookwyrm/sanitize_html.py index 9c5ca73a..de13ede8 100644 --- a/bookwyrm/sanitize_html.py +++ b/bookwyrm/sanitize_html.py @@ -1,12 +1,16 @@ ''' html parser to clean up incoming text from unknown sources ''' from html.parser import HTMLParser -class InputHtmlParser(HTMLParser): +class InputHtmlParser(HTMLParser):#pylint: disable=abstract-method ''' Removes any html that isn't allowed_tagsed from a block ''' def __init__(self): HTMLParser.__init__(self) - self.allowed_tags = ['p', 'b', 'i', 'pre', 'a', 'span'] + self.allowed_tags = [ + 'p', 'br', + 'b', 'i', 'strong', 'em', 'pre', + 'a', 'span', 'ul', 'ol', 'li' + ] self.tag_stack = [] self.output = [] # if the html appears invalid, we just won't allow any at all diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 3784158c..46c38b5a 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -3,8 +3,11 @@ import os from environs import Env +import requests + env = Env() DOMAIN = env('DOMAIN') +VERSION = '0.0.1' PAGE_LENGTH = env('PAGE_LENGTH', 15) @@ -99,10 +102,6 @@ BOOKWYRM_DBS = { 'HOST': env('POSTGRES_HOST', ''), 'PORT': 5432 }, - 'sqlite': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'fedireads.db') - } } DATABASES = { @@ -154,3 +153,6 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, env('STATIC_ROOT', 'static')) MEDIA_URL = '/images/' MEDIA_ROOT = os.path.join(BASE_DIR, env('MEDIA_ROOT', 'images')) + +USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % ( + requests.utils.default_user_agent(), VERSION, DOMAIN) diff --git a/bookwyrm/static/css/format.css b/bookwyrm/static/css/format.css index 9e8a24ba..7dab69b0 100644 --- a/bookwyrm/static/css/format.css +++ b/bookwyrm/static/css/format.css @@ -65,6 +65,14 @@ input.toggle-control:checked ~ .modal.toggle-content { .cover-container { height: 250px; width: max-content; + max-width: 250px; +} +.cover-container.is-large { + height: max-content; + max-width: 500px; +} +.cover-container.is-large img { + max-height: 500px; } .cover-container.is-medium { height: 150px; @@ -136,8 +144,3 @@ input.toggle-control:checked ~ .modal.toggle-content { content: "\e904"; right: 0; } - -/* --- BLOCKQUOTE --- */ -blockquote { - white-space: pre-line; -} diff --git a/bookwyrm/status.py b/bookwyrm/status.py index 83a106e5..648f2e7d 100644 --- a/bookwyrm/status.py +++ b/bookwyrm/status.py @@ -1,7 +1,7 @@ ''' Handle user activity ''' from django.utils import timezone -from bookwyrm import activitypub, books_manager, models +from bookwyrm import models from bookwyrm.sanitize_html import InputHtmlParser diff --git a/bookwyrm/templates/author.html b/bookwyrm/templates/author.html index 3e3e0018..4235b266 100644 --- a/bookwyrm/templates/author.html +++ b/bookwyrm/templates/author.html @@ -1,14 +1,32 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
-

{{ author.name }}

+
+
+

{{ author.name }}

+
+ {% if request.user.is_authenticated and perms.bookwyrm.edit_book %} +
+ + + Edit Author + + +
+ {% endif %} +
+
+
{% if author.bio %}

- {{ author.bio }} + {{ author.bio | to_markdown | safe }}

{% endif %} + {% if author.wikipedia_link %} +

Wikipedia

+ {% endif %}
diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html index 8b21b88c..4bbc8d10 100644 --- a/bookwyrm/templates/book.html +++ b/bookwyrm/templates/book.html @@ -1,16 +1,24 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% load humanize %} {% block content %}
-
-

- {% include 'snippets/book_titleby.html' with book=book %} -

+
+
+

+ {{ book.title }}{% if book.subtitle %}: + {{ book.subtitle }}{% endif %} +

+ {% if book.authors %} +

+ by {% include 'snippets/authors.html' with book=book %} +

+ {% endif %} +
{% if request.user.is_authenticated and perms.bookwyrm.edit_book %} -
+
Edit Book @@ -91,87 +99,26 @@ {% endif %}
- {% for readthrough in readthroughs %} -
- - -
- -
- - -
- + {# user's relationship to the book #}
- - + {% for shelf in user_shelves %} +

+ This edition is on your {{ shelf.shelf.name }} shelf. + {% include 'snippets/shelf_selector.html' with current=shelf.shelf %} +

+ {% endfor %} + + {% for shelf in other_edition_shelves %} +

+ A different edition of this book is on your {{ shelf.shelf.name }} shelf. + {% include 'snippets/switch_edition_button.html' with edition=book %} +

+ {% endfor %} + + {% for readthrough in readthroughs %} + {% include 'snippets/readthrough.html' with readthrough=readthrough %} + {% endfor %}
- {% endfor %} {% if request.user.is_authenticated %}
@@ -219,10 +166,10 @@ {% for rating in ratings %}
-
{% include 'snippets/avatar.html' %}
+
{% include 'snippets/avatar.html' with user=rating.user %}
- {% include 'snippets/username.html' %} + {% include 'snippets/username.html' with user=rating.user %}
rated it
diff --git a/bookwyrm/templates/direct_messages.html b/bookwyrm/templates/direct_messages.html new file mode 100644 index 00000000..6a20b111 --- /dev/null +++ b/bookwyrm/templates/direct_messages.html @@ -0,0 +1,37 @@ +{% extends 'layout.html' %} +{% block content %} + +
+

Direct Messages

+ + {% if not activities %} +

You have no messages right now.

+ {% endif %} + {% for activity in activities %} +
+ {% include 'snippets/status.html' with status=activity %} +
+ {% endfor %} + + +
+ +{% endblock %} diff --git a/bookwyrm/templates/discover.html b/bookwyrm/templates/discover.html new file mode 100644 index 00000000..a28a67bd --- /dev/null +++ b/bookwyrm/templates/discover.html @@ -0,0 +1,80 @@ +{% extends 'layout.html' %} +{% block content %} + +{% if not request.user.is_authenticated %} +
+

{{ site.name }}: Social Reading and Reviewing

+
+ +
+
+
+ {% include 'snippets/about.html' %} +
+
+
+
+ {% if site.allow_registration %} +

Join {{ site.name }}

+
+ {% include 'snippets/register_form.html' %} +
+ {% else %} +

This instance is closed

+

Contact an administrator to get an invite

+ {% endif %} +
+
+
+{% else %} +
+

Discover

+
+{% endif %} + +
+

Recent Books

+
+ +
+
+
+
+ {% include 'snippets/discover/large-book.html' with book=books.0 %} +
+
+
+
+
+ {% include 'snippets/discover/small-book.html' with book=books.1 %} +
+
+
+
+ {% include 'snippets/discover/small-book.html' with book=books.2 %} +
+
+
+
+
+
+
+
+ {% include 'snippets/discover/small-book.html' with book=books.3 %} +
+
+
+
+ {% include 'snippets/discover/small-book.html' with book=books.4 %} +
+
+
+
+
+ {% include 'snippets/discover/large-book.html' with book=books.5 %} +
+
+
+
+ +{% endblock %} diff --git a/bookwyrm/templates/edit_author.html b/bookwyrm/templates/edit_author.html new file mode 100644 index 00000000..b08aa983 --- /dev/null +++ b/bookwyrm/templates/edit_author.html @@ -0,0 +1,89 @@ +{% extends 'layout.html' %} +{% load humanize %} +{% block content %} +
+
+

+ Edit "{{ author.name }}" +

+
+ + + Close + + +
+
+
+

Added: {{ author.created_date | naturaltime }}

+

Updated: {{ author.updated_date | naturaltime }}

+

Last edited by: {{ author.last_edited_by.display_name }}

+
+
+ +{% if form.non_field_errors %} +
+

{{ form.non_field_errors }}

+
+{% endif %} + +
+ {% csrf_token %} + + +
+
+

Metadata

+

{{ form.name }}

+ {% for error in form.name.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.bio }}

+ {% for error in form.bio.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.wikipedia_link }}

+ {% for error in form.wikipedia_link.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.born }}

+ {% for error in form.born.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.died }}

+ {% for error in form.died.errors %} +

{{ error | escape }}

+ {% endfor %} +
+
+

Author Identifiers

+

{{ form.openlibrary_key }}

+ {% for error in form.openlibrary_key.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.librarything_key }}

+ {% for error in form.librarything_key.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.goodreads_key }}

+ {% for error in form.goodreads_key.errors %} +

{{ error | escape }}

+ {% endfor %} + +
+
+ +
+ + Cancel +
+
+ +{% endblock %} + diff --git a/bookwyrm/templates/edit_book.html b/bookwyrm/templates/edit_book.html index 54cefb0a..b730f3e4 100644 --- a/bookwyrm/templates/edit_book.html +++ b/bookwyrm/templates/edit_book.html @@ -17,63 +17,51 @@

Added: {{ book.created_date | naturaltime }}

Updated: {{ book.updated_date | naturaltime }}

+

Last edited by: {{ book.last_edited_by.display_name }}

-{% if login_form.non_field_errors %} +{% if form.non_field_errors %}
-

{{ login_form.non_field_errors }}

+

{{ form.non_field_errors }}

{% endif %}
{% csrf_token %} -
-

Data sync -

If sync is enabled, any changes will be over-written

-

-
-
- -
-
- -
-
-
- +

Metadata

-

{{ form.title }}

+

{{ form.title }}

{% for error in form.title.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.sort_title }}

+

{{ form.sort_title }}

{% for error in form.sort_title.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.subtitle }}

+

{{ form.subtitle }}

{% for error in form.subtitle.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.description }}

+

{{ form.description }}

{% for error in form.description.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.series }}

+

{{ form.series }}

{% for error in form.series.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.series_number }}

+

{{ form.series_number }}

{% for error in form.series_number.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.first_published_date }}

+

{{ form.first_published_date }}

{% for error in form.first_published_date.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.published_date }}

+

{{ form.published_date }}

{% for error in form.published_date.errors %}

{{ error | escape }}

{% endfor %} @@ -97,7 +85,7 @@

Physical Properties

-

{{ form.physical_format }}

+

{{ form.physical_format }}

{% for error in form.physical_format.errors %}

{{ error | escape }}

{% endfor %} @@ -105,7 +93,7 @@

{{ error | escape }}

{% endfor %} -

{{ form.pages }}

+

{{ form.pages }}

{% for error in form.pages.errors %}

{{ error | escape }}

{% endfor %} @@ -113,23 +101,23 @@

Book Identifiers

-

{{ form.isbn_13 }}

+

{{ form.isbn_13 }}

{% for error in form.isbn_13.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.isbn_10 }}

+

{{ form.isbn_10 }}

{% for error in form.isbn_10.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.openlibrary_key }}

+

{{ form.openlibrary_key }}

{% for error in form.openlibrary_key.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.librarything_key }}

+

{{ form.librarything_key }}

{% for error in form.librarything_key.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.goodreads_key }}

+

{{ form.goodreads_key }}

{% for error in form.goodreads_key.errors %}

{{ error | escape }}

{% endfor %} diff --git a/bookwyrm/templates/editions.html b/bookwyrm/templates/editions.html index 273b2cd6..619ceafb 100644 --- a/bookwyrm/templates/editions.html +++ b/bookwyrm/templates/editions.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

Editions of "{{ work.title }}"

diff --git a/bookwyrm/templates/feed.html b/bookwyrm/templates/feed.html index 6e49943a..f02219bc 100644 --- a/bookwyrm/templates/feed.html +++ b/bookwyrm/templates/feed.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
@@ -44,16 +44,28 @@
{% endfor %} {% endwith %} {% endfor %} +
+ +
{% endif %}
diff --git a/bookwyrm/templates/followers.html b/bookwyrm/templates/followers.html index 645e46a1..00cb13ca 100644 --- a/bookwyrm/templates/followers.html +++ b/bookwyrm/templates/followers.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

diff --git a/bookwyrm/templates/following.html b/bookwyrm/templates/following.html index 2cca9127..bdf02c74 100644 --- a/bookwyrm/templates/following.html +++ b/bookwyrm/templates/following.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

@@ -31,7 +31,7 @@

{% endfor %} {% if not following.count %} -
No one is following {{ user|username }}
+
{{ user|username }} isn't following any users
{% endif %}

diff --git a/bookwyrm/templates/import.html b/bookwyrm/templates/import.html index 8e3f5eb4..bfa8d3ec 100644 --- a/bookwyrm/templates/import.html +++ b/bookwyrm/templates/import.html @@ -21,8 +21,6 @@
-

- Imports are limited in size, and only the first {{ limit }} items will be imported.

diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index f91e2cce..6bb903b0 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% load humanize %} {% block content %}
diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index b37c9cda..a2565840 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -1,4 +1,4 @@ -{% load fr_display %} +{% load bookwyrm_tags %} @@ -18,7 +18,7 @@ -