Use remote_id resolver to load books, user

This commit is contained in:
Mouse Reeve 2020-11-28 10:18:24 -08:00
parent 81bdd2b3f1
commit a93b5cf5bc
15 changed files with 115 additions and 93 deletions

View file

@ -4,7 +4,7 @@ import sys
from .base_activity import ActivityEncoder, PublicKey, Signature
from .base_activity import Link, Mention
from .base_activity import ActivitySerializerError
from .base_activity import ActivitySerializerError, resolve_remote_id
from .image import Image
from .note import Note, GeneratedNote, Article, Comment, Review, Quotation
from .note import Tombstone
@ -14,7 +14,7 @@ from .person import Person
from .book import Edition, Work, Author
from .verbs import Create, Delete, Undo, Update
from .verbs import Follow, Accept, Reject
from .verbs import Add, Remove
from .verbs import Add, AddBook, Remove
# this creates a list of all the Activity types that we can serialize,
# so when an Activity comes in from outside, we can check if it's known

View file

@ -3,15 +3,20 @@ from dataclasses import dataclass, fields, MISSING
from json import JSONEncoder
from uuid import uuid4
import dateutil.parser
from dateutil.parser import ParserError
from django.core.files.base import ContentFile
from django.db import transaction
from django.db.models.fields.related_descriptors \
import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
ReverseManyToOneDescriptor
from django.db.models.fields import DateTimeField
from django.db.models.fields.files import ImageFileDescriptor
from django.db.models.query_utils import DeferredAttribute
from django.utils import timezone
import requests
from bookwyrm import books_manager, models
from bookwyrm import models
class ActivitySerializerError(ValueError):
@ -106,11 +111,27 @@ class ActivityObject:
model_field = getattr(model, mapping.model_key)
formatted_value = mapping.model_formatter(value)
if isinstance(model_field, ForwardManyToOneDescriptor) and \
if isinstance(model_field, DeferredAttribute) and \
isinstance(model_field.field, DateTimeField):
print("DATE")
try:
formatted_value = timezone.make_aware(
dateutil.parser.parse(formatted_value)
)
except ParserError:
formatted_value = None
elif isinstance(model_field, ForwardManyToOneDescriptor) and \
formatted_value:
# foreign key remote id reolver (work on Edition, for example)
fk_model = model_field.field.related_model
reference = resolve_foreign_key(fk_model, formatted_value)
if isinstance(formatted_value, dict) and \
formatted_value.get('id'):
# if the AP field is a serialized object (as in Add)
remote_id = formatted_value['id']
else:
# if the AP field is just a remote_id (as in every other case)
remote_id = formatted_value
reference = resolve_remote_id(fk_model, remote_id)
mapped_fields[mapping.model_key] = reference
elif isinstance(model_field, ManyToManyDescriptor):
# status mentions book/users
@ -122,6 +143,8 @@ class ActivityObject:
# image fields need custom handling
image_fields[mapping.model_key] = formatted_value
else:
if formatted_value == MISSING:
formatted_value = None
mapped_fields[mapping.model_key] = formatted_value
with transaction.atomic():
@ -153,12 +176,15 @@ class ActivityObject:
model = model_field.model
items = []
for link in values:
# check that the Type matches the model (because Status
# tags contain both user mentions and book tags)
if not model.activity_serializer.type == link.get('type'):
continue
items.append(
resolve_foreign_key(model, link.get('href'))
resolve_remote_id(model, link.get('href'))
)
getattr(instance, model_key).set(items)
# add one to many fields
for (model_key, values) in one_to_many_fields.items():
if values == MISSING:
@ -183,11 +209,8 @@ class ActivityObject:
return data
def resolve_foreign_key(model, remote_id):
''' look up the remote_id on an activity json field '''
if model in [models.Edition, models.Work, models.Book]:
return books_manager.get_or_create_book(remote_id)
def resolve_remote_id(model, remote_id, refresh=False):
''' look up the remote_id in the database or load it remotely '''
result = model.objects
if hasattr(model.objects, 'select_subclasses'):
result = result.select_subclasses()
@ -196,10 +219,10 @@ def resolve_foreign_key(model, remote_id):
result = result.filter(
remote_id=remote_id
).first()
if result:
if result and not refresh:
return result
# failing that, load the data and create the object
# load the data and create the object
try:
response = requests.get(
remote_id,
@ -215,7 +238,8 @@ def resolve_foreign_key(model, remote_id):
(model.__name__, remote_id))
item = model.activity_serializer(**response.json())
return item.to_model(model)
# if we're refreshing, "result" will be set and we'll update it
return item.to_model(model, instance=result)
def image_formatter(image_slug):

View file

@ -12,13 +12,13 @@ class Book(ActivityObject):
sortTitle: str = ''
subtitle: str = ''
description: str = ''
languages: List[str]
languages: List[str] = field(default_factory=lambda: [])
series: str = ''
seriesNumber: str = ''
subjects: List[str]
subjectPlaces: List[str]
subjects: List[str] = field(default_factory=lambda: [])
subjectPlaces: List[str] = field(default_factory=lambda: [])
authors: List[str]
authors: List[str] = field(default_factory=lambda: [])
firstPublishedDate: str = ''
publishedDate: str = ''
@ -33,22 +33,22 @@ class Book(ActivityObject):
@dataclass(init=False)
class Edition(Book):
''' Edition instance of a book object '''
isbn10: str
isbn13: str
oclcNumber: str
asin: str
pages: str
physicalFormat: str
publishers: List[str]
work: str
isbn10: str = ''
isbn13: str = ''
oclcNumber: str = ''
asin: str = ''
pages: str = ''
physicalFormat: str = ''
publishers: List[str] = field(default_factory=lambda: [])
type: str = 'Edition'
@dataclass(init=False)
class Work(Book):
''' work instance of a book object '''
lccn: str
lccn: str = ''
editions: List[str]
type: str = 'Work'

View file

@ -12,6 +12,7 @@ class OrderedCollection(ActivityObject):
first: str
last: str = ''
name: str = ''
owner: str = ''
type: str = 'OrderedCollection'

View file

@ -3,6 +3,7 @@ from dataclasses import dataclass
from typing import List
from .base_activity import ActivityObject, Signature
from .book import Book
@dataclass(init=False)
class Verb(ActivityObject):
@ -69,6 +70,13 @@ class Add(Verb):
type: str = 'Add'
@dataclass(init=False)
class AddBook(Verb):
'''Add activity that's aware of the book obj '''
target: Book
type: str = 'Add'
@dataclass(init=False)
class Remove(Verb):
'''Remove activity '''

View file

@ -16,23 +16,6 @@ def get_edition(book_id):
return book
def get_or_create_book(remote_id):
''' pull up a book record by whatever means possible '''
book = models.Book.objects.select_subclasses().filter(
remote_id=remote_id
).first()
if book:
return book
connector = get_or_create_connector(remote_id)
# raises ConnectorException
book = connector.get_or_create_book(remote_id)
if book:
load_more_data.delay(book.id)
return book
def get_or_create_connector(remote_id):
''' get the connector related to the author's server '''
url = urlparse(remote_id)

View file

@ -317,7 +317,7 @@ def handle_tag(activity):
user = get_or_create_remote_user(activity['actor'])
if not user.local:
# ordered collection weirndess so we can't just to_model
book = books_manager.get_or_create_book(activity['object']['id'])
book = (activity['object']['id'])
name = activity['object']['target'].split('/')[-1]
name = unquote_plus(name)
models.Tag.objects.get_or_create(
@ -330,17 +330,7 @@ def handle_tag(activity):
@app.task
def handle_shelve(activity):
''' putting a book on a shelf '''
user = get_or_create_remote_user(activity['actor'])
book = books_manager.get_or_create_book(activity['object'])
try:
shelf = models.Shelf.objects.get(remote_id=activity['target'])
except models.Shelf.DoesNotExist:
return
if shelf.user != user:
# this doesn't add up.
return
shelf.books.add(book)
shelf.save()
activitypub.AddBook(**activity).to_model(models.ShelfBook)
@app.task

View file

@ -0,0 +1,24 @@
# Generated by Django 3.0.7 on 2020-11-28 18:04
import bookwyrm.utils.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('bookwyrm', '0015_auto_20201128_0349'),
]
operations = [
migrations.AlterField(
model_name='book',
name='subject_places',
field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
),
migrations.AlterField(
model_name='book',
name='subjects',
field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None),
),
]

View file

@ -227,12 +227,16 @@ class OrderedCollectionPageMixin(ActivitypubMixin):
name = ''
if hasattr(self, 'name'):
name = self.name
owner = ''
if hasattr(self, 'user'):
owner = self.user.remote_id
size = queryset.count()
return activitypub.OrderedCollection(
id=remote_id,
totalItems=size,
name=name,
owner=owner,
first='%s%s' % (remote_id, self.page()),
last='%s%s' % (remote_id, self.page(min_id=0))
).serialize()

View file

@ -41,10 +41,10 @@ class Book(ActivitypubMixin, BookWyrmModel):
series = models.CharField(max_length=255, blank=True, null=True)
series_number = models.CharField(max_length=255, blank=True, null=True)
subjects = ArrayField(
models.CharField(max_length=255), blank=True, default=list
models.CharField(max_length=255), blank=True, null=True, default=list
)
subject_places = ArrayField(
models.CharField(max_length=255), blank=True, default=list
models.CharField(max_length=255), blank=True, null=True, default=list
)
# TODO: include an annotation about the type of authorship (ie, translator)
authors = models.ManyToManyField('Author')
@ -132,7 +132,8 @@ class Work(OrderedCollectionPageMixin, Book):
''' it'd be nice to serialize the edition instead but, recursion '''
default = self.default_edition
ed_list = [
e.remote_id for e in self.edition_set.filter(~Q(id=default.id)).all()
e.remote_id for e in \
self.edition_set.filter(~Q(id=default.id)).all()
]
return [default.remote_id] + ed_list

View file

@ -3,7 +3,8 @@ import re
from django.db import models
from bookwyrm import activitypub
from .base_model import BookWyrmModel, OrderedCollectionMixin, PrivacyLevels
from .base_model import ActivityMapping, BookWyrmModel
from .base_model import OrderedCollectionMixin, PrivacyLevels
class Shelf(OrderedCollectionMixin, BookWyrmModel):
@ -47,6 +48,12 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
''' user/shelf unqiueness '''
unique_together = ('user', 'identifier')
activity_mappings = [
ActivityMapping('id', 'remote_id'),
ActivityMapping('owner', 'user'),
ActivityMapping('name', 'name'),
]
class ShelfBook(BookWyrmModel):
''' many to many join table for books and shelves '''
@ -59,6 +66,15 @@ class ShelfBook(BookWyrmModel):
on_delete=models.PROTECT
)
activity_mappings = [
ActivityMapping('id', 'remote_id'),
ActivityMapping('actor', 'added_by'),
ActivityMapping('object', 'book'),
ActivityMapping('target', 'shelf')
]
activity_serializer = activitypub.AddBook
def to_add_activity(self, user):
''' AP for shelving a book'''
return activitypub.Add(

View file

@ -80,12 +80,10 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
ActivityMapping(
'tag', 'mention_books',
lambda x: tag_formatter(x, 'title', 'Book'),
lambda x: [i for i in x if x.get('type') == 'Book']
),
ActivityMapping(
'tag', 'mention_users',
lambda x: tag_formatter(x, 'username', 'Mention'),
lambda x: [i for i in x if x.get('type') == 'Mention']
),
ActivityMapping(
'attachment', 'attachments',

View file

@ -11,7 +11,6 @@ from bookwyrm.models.status import Status
from bookwyrm.settings import DOMAIN
from bookwyrm.signatures import create_key_pair
from .base_model import ActivityMapping, OrderedCollectionPageMixin
from .base_model import image_formatter
class User(OrderedCollectionPageMixin, AbstractUser):

View file

@ -16,11 +16,9 @@ def get_or_create_remote_user(actor):
except models.User.DoesNotExist:
pass
data = fetch_user_data(actor)
actor_parts = urlparse(actor)
with transaction.atomic():
user = activitypub.Person(**data).to_model(models.User)
user = activitypub.resolve_remote_id(models.User, actor)
user.federated_server = get_or_create_remote_server(actor_parts.netloc)
user.save()
if user.bookwyrm_user:
@ -28,32 +26,9 @@ def get_or_create_remote_user(actor):
return user
def fetch_user_data(actor):
''' load the user's info from the actor url '''
try:
response = requests.get(
actor,
headers={'Accept': 'application/activity+json'}
)
except ConnectionError:
return None
if not response.ok:
response.raise_for_status()
data = response.json()
# make sure our actor is who they say they are
if actor != data['id']:
raise ValueError("Remote actor id must match url.")
return data
def refresh_remote_user(user):
''' get updated user data from its home instance '''
data = fetch_user_data(user.remote_id)
activity = activitypub.Person(**data)
activity.to_model(models.User, instance=user)
activitypub.resolve_remote_id(user.remote_id, refresh=True)
@app.task

View file

@ -5,8 +5,7 @@ from django.contrib.auth.decorators import login_required, permission_required
from django.contrib.postgres.search import TrigramSimilarity
from django.core.paginator import Paginator
from django.db.models import Avg, Q
from django.http import HttpResponseBadRequest, HttpResponseNotFound,\
JsonResponse
from django.http import HttpResponseNotFound, JsonResponse
from django.core.exceptions import PermissionDenied
from django.shortcuts import get_object_or_404, redirect
from django.template.response import TemplateResponse