From f28ee934fc9b784f36af721c84303a601262e103 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 16 Feb 2020 18:22:01 -0800 Subject: [PATCH] Stop storing activitypub json in db entirely --- fedireads/activity.py | 113 ++++++++++++++++++++++++- fedireads/incoming.py | 72 +++------------- fedireads/models/__init__.py | 3 +- fedireads/models/activity.py | 63 ++------------ fedireads/outgoing.py | 154 ++++------------------------------ fedireads/templates/feed.html | 31 ++----- fedireads/views.py | 2 +- 7 files changed, 150 insertions(+), 288 deletions(-) diff --git a/fedireads/activity.py b/fedireads/activity.py index a50b585d5..43611592d 100644 --- a/fedireads/activity.py +++ b/fedireads/activity.py @@ -1,7 +1,14 @@ ''' Handle user activity ''' +from base64 import b64encode +from Crypto.PublicKey import RSA +from Crypto.Signature import pkcs1_15 +from Crypto.Hash import SHA256 +from uuid import uuid4 + from fedireads import models from fedireads.openlibrary import get_or_create_book from fedireads.sanitize_html import InputHtmlParser +from fedireads.settings import DOMAIN def create_review(user, possible_book, name, content, rating): @@ -17,7 +24,7 @@ def create_review(user, possible_book, name, content, rating): # no ratings outside of 0-5 rating = rating if 0 <= rating <= 5 else 0 - review = models.Review.objects.create( + return models.Review.objects.create( user=user, book=book, name=name, @@ -25,6 +32,108 @@ def create_review(user, possible_book, name, content, rating): content=content, ) - return review + +def create_status(user, content, reply_parent=None): + ''' a status update ''' + # TODO: handle @'ing users + + # sanitize input html + parser = InputHtmlParser() + parser.feed(content) + content = parser.get_output() + + return models.Status.objects.create( + user=user, + content=content, + reply_parent=reply_parent, + ) + + +def get_status_json(status): + ''' create activitypub json for a status ''' + user = status.user + uri = 'https://%s/user/%s/status/%d' % (DOMAIN, user.localname, status.id) + reply_parent_id = status.reply_parent.id if status.reply_parent else None + status_json = { + 'id': uri, + 'url': uri, + 'inReplyTo': reply_parent_id, + 'published': status.created_date.isoformat(), + 'attributedTo': user.actor, + # TODO: assuming all posts are public -- should check privacy db field + 'to': ['https://www.w3.org/ns/activitystreams#Public'], + 'cc': ['https://%s/user/%s/followers' % (DOMAIN, user.localname)], + 'sensitive': status.sensitive, + 'content': status.content, + 'type': status.activity_type, + 'fedireadsType': status.status_type, + 'attachment': [], # TODO: the book cover + 'replies': { + 'id': '%s/replies' % uri, + 'type': 'Collection', + 'first': { + 'type': 'CollectionPage', + 'next': '%s/replies?only_other_accounts=true&page=true' % uri, + 'partOf': '%s/replies' % uri, + 'items': [], # TODO: populate with replies + } + } + } + + if status.status_type == 'Review': + status_json['name'] = status.name, + status_json['rating'] = status.rating + status_json['fedireadsType'] = status.status_type + + return status_json + + +def get_create_json(user, status_json): + ''' create activitypub json for a Create activity ''' + signer = pkcs1_15.new(RSA.import_key(user.private_key)) + content = status_json['content'] + signed_message = signer.sign(SHA256.new(content.encode('utf8'))) + return { + '@context': 'https://www.w3.org/ns/activitystreams', + + 'id': '%s/activity' % status_json['id'], + 'type': 'Create', + 'actor': user.actor, + 'published': status_json['published'], + + 'to': ['%s/followers' % user.actor], + 'cc': ['https://www.w3.org/ns/activitystreams#Public'], + + 'object': status_json, + 'signature': { + 'type': 'RsaSignature2017', + 'creator': 'https://%s/user/%s#main-key' % (DOMAIN, user.localname), + 'created': status_json['published'], + 'signatureValue': b64encode(signed_message).decode('utf8'), + } + } + + + +def get_add_remove_json(user, book, shelf, action='Add'): + ''' format an Add or Remove json blob ''' + uuid = uuid4() + return { + '@context': 'https://www.w3.org/ns/activitystreams', + 'id': str(uuid), + 'type': action, + 'actor': user.actor, + 'object': { + 'type': 'Document', + 'name': book.data['title'], + 'url': book.openlibrary_key + }, + 'target': { + 'type': 'Collection', + 'name': shelf.name, + 'id': 'https://%s/user/%s/shelf/%s' % \ + (DOMAIN, user.localname, shelf.identifier) + } + } diff --git a/fedireads/incoming.py b/fedireads/incoming.py index 786163f21..380efed45 100644 --- a/fedireads/incoming.py +++ b/fedireads/incoming.py @@ -11,7 +11,7 @@ import requests from fedireads import models from fedireads import outgoing -from fedireads.activity import create_review +from fedireads.activity import create_review, create_status from fedireads.remote_user import get_or_create_remote_user @@ -33,10 +33,7 @@ def shared_inbox(request): return HttpResponse(status=401) response = HttpResponseNotFound() - if activity['type'] == 'Add': - response = handle_incoming_shelve(activity) - - elif activity['type'] == 'Follow': + if activity['type'] == 'Follow': response = handle_incoming_follow(activity) elif activity['type'] == 'Create': @@ -45,7 +42,7 @@ def shared_inbox(request): elif activity['type'] == 'Accept': response = handle_incoming_follow_accept(activity) - # TODO: Undo, Remove, etc + # TODO: Add, Undo, Remove, etc return response @@ -217,43 +214,12 @@ def get_follow_page(user_list, id_slug, page): return data -def handle_incoming_shelve(activity): - ''' receiving an Add activity (to shelve a book) ''' - # TODO what happens here? If it's a remote over, then I think - # I should save both the activity and the ShelfBook entry. But - # I'll do that later. - uuid = activity['id'] - models.ShelveActivity.objects.get(uuid=uuid) - ''' - book_id = activity['object']['url'] - book = openlibrary.get_or_create_book(book_id) - user_ap_id = activity['actor'].replace('https//:', '') - user = models.User.objects.get(actor=user_ap_id) - if not user or not user.local: - return HttpResponseBadRequest() - - shelf = models.Shelf.objects.get(activitypub_id=activity['target']['id']) - models.ShelfBook( - shelf=shelf, - book=book, - added_by=user, - ).save() - ''' - return HttpResponse() - - def handle_incoming_follow(activity): ''' someone wants to follow a local user ''' # figure out who they want to follow to_follow = models.User.objects.get(actor=activity['object']) # figure out who they are user = get_or_create_remote_user(activity['actor']) - models.FollowActivity( - uuid=activity['id'], - user=user, - followed=to_follow, - content=activity, - ) # TODO: allow users to manually approve requests outgoing.handle_outgoing_accept(user, to_follow, activity) return HttpResponse() @@ -278,36 +244,27 @@ def handle_incoming_create(activity): return HttpResponseBadRequest() response = HttpResponse() + content = activity['object'].get('content') if activity['object'].get('fedireadsType') == 'Review' and \ 'inReplyTo' in activity['object']: book = activity['object']['inReplyTo'] book = book.split('/')[-1] name = activity['object'].get('name') - content = activity['object'].get('content') rating = activity['object'].get('rating') if user.local: review_id = activity['object']['id'].split('/')[-1] - review = models.Review.objects.get(id=review_id) + models.Review.objects.get(id=review_id) else: try: - review = create_review(user, book, name, content, rating) + create_review(user, book, name, content, rating) except ValueError: return HttpResponseBadRequest() - models.ReviewActivity.objects.create( - uuid=activity['id'], - user=user, - content=activity['object'], - activity_type=activity['object']['type'], - book=review.book, - ) - else: - models.Activity.objects.create( - uuid=activity['id'], - user=user, - content=activity, - activity_type=activity['object']['type'] - ) + try: + create_status(user, content) + except ValueError: + return HttpResponseBadRequest() + return response @@ -321,12 +278,5 @@ def handle_incoming_accept(activity): # save this relationship in the db followed.followers.add(user) - # save the activity record - models.FollowActivity( - uuid=activity['id'], - user=user, - followed=followed, - content=activity, - ).save() return HttpResponse() diff --git a/fedireads/models/__init__.py b/fedireads/models/__init__.py index a3e286754..b54cee606 100644 --- a/fedireads/models/__init__.py +++ b/fedireads/models/__init__.py @@ -1,6 +1,5 @@ ''' bring all the models into the app namespace ''' from .book import Shelf, ShelfBook, Book, Author from .user import User, FederatedServer -from .activity import Activity, ShelveActivity, FollowActivity, \ - ReviewActivity, Status, Review +from .activity import Status, Review diff --git a/fedireads/models/activity.py b/fedireads/models/activity.py index 67d35cf01..e91734f82 100644 --- a/fedireads/models/activity.py +++ b/fedireads/models/activity.py @@ -3,66 +3,15 @@ from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from model_utils.managers import InheritanceManager -from fedireads.utils.fields import JSONField - -# TODO: I don't know that these Activity models should exist, at least in this way -# but I'm not sure what the right approach is for now. - -class Activity(models.Model): - ''' basic fields for storing activities ''' - uuid = models.CharField(max_length=255, unique=True) - user = models.ForeignKey('User', on_delete=models.PROTECT) - content = JSONField(max_length=5000) - # the activitypub activity type (Create, Add, Follow, ...) - activity_type = models.CharField(max_length=255) - # custom types internal to fedireads (Review, Shelve, ...) - fedireads_type = models.CharField(max_length=255, blank=True, null=True) - local = models.BooleanField(default=True) - created_date = models.DateTimeField(auto_now_add=True) - updated_date = models.DateTimeField(auto_now=True) - objects = InheritanceManager() - - -class ShelveActivity(Activity): - ''' someone put a book on a shelf ''' - book = models.ForeignKey('Book', on_delete=models.PROTECT) - shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT) - - def save(self, *args, **kwargs): - if not self.activity_type: - self.activity_type = 'Add' - self.fedireads_type = 'Shelve' - super().save(*args, **kwargs) - - -class FollowActivity(Activity): - ''' record follow requests sent out ''' - followed = models.ForeignKey( - 'User', - related_name='followed', - on_delete=models.PROTECT - ) - - def save(self, *args, **kwargs): - self.activity_type = 'Follow' - super().save(*args, **kwargs) - - -class ReviewActivity(Activity): - book = models.ForeignKey('Book', on_delete=models.PROTECT) - - def save(self, *args, **kwargs): - self.activity_type = 'Note' - self.fedireads_type = 'Review' - super().save(*args, **kwargs) - class Status(models.Model): - ''' reply to a review, etc ''' + ''' any post, like a reply to a review, etc ''' user = models.ForeignKey('User', on_delete=models.PROTECT) status_type = models.CharField(max_length=255, default='Note') - activity = JSONField(max_length=5000, null=True) + activity_type = models.CharField(max_length=255, default='Note') local = models.BooleanField(default=True) + privacy = models.CharField(max_length=255, default='public') + sensitive = models.BooleanField(default=False) reply_parent = models.ForeignKey( 'self', null=True, @@ -70,14 +19,13 @@ class Status(models.Model): ) content = models.TextField(blank=True, null=True) created_date = models.DateTimeField(auto_now_add=True) - updated_date = models.DateTimeField(auto_now=True) objects = InheritanceManager() class Review(Status): ''' a book review ''' - book = models.ForeignKey('Book', on_delete=models.PROTECT) name = models.CharField(max_length=255) + book = models.ForeignKey('Book', on_delete=models.PROTECT) rating = models.IntegerField( default=0, validators=[MinValueValidator(0), MaxValueValidator(5)] @@ -87,4 +35,3 @@ class Review(Status): self.status_type = 'Review' super().save(*args, **kwargs) - diff --git a/fedireads/outgoing.py b/fedireads/outgoing.py index df76eb9a2..d134d47de 100644 --- a/fedireads/outgoing.py +++ b/fedireads/outgoing.py @@ -1,9 +1,4 @@ ''' handles all the activity coming out of the server ''' -from base64 import b64encode -from Crypto.PublicKey import RSA -from Crypto.Signature import pkcs1_15 -from Crypto.Hash import SHA256 -from datetime import datetime from django.http import HttpResponseNotFound, JsonResponse from django.views.decorators.csrf import csrf_exempt import requests @@ -11,7 +6,8 @@ from urllib.parse import urlencode from uuid import uuid4 from fedireads import models -from fedireads.activity import create_review +from fedireads.activity import create_review, get_status_json, get_create_json +from fedireads.activity import get_add_remove_json from fedireads.remote_user import get_or_create_remote_user from fedireads.broadcast import get_recipients, broadcast from fedireads.settings import DOMAIN @@ -43,19 +39,17 @@ def outbox(request, username): filters['id__lte'] = max_id collection_id = query_path + urlencode(params) - messages = models.Activity.objects.filter( - user=user, - activity_type__in=['Article', 'Note'], - **filters - ).all()[:limit] - outbox_page = { '@context': 'https://www.w3.org/ns/activitystreams', 'id': collection_id, 'type': 'OrderedCollectionPage', 'partOf': user.outbox, - 'orderedItems': [m.content for m in messages], + 'orderedItems': [], } + statuses = models.Status.objects.filter(user=user, **filters).all() + for status in statuses[:limit]: + outbox_page['orderedItems'].append(get_status_json(status)) + if max_id: outbox_page['next'] = query_path + \ urlencode({'min_id': max_id, 'page': 'true'}) @@ -65,7 +59,7 @@ def outbox(request, username): return JsonResponse(outbox_page) # collection overview - size = models.Review.objects.filter(user=user).count() + size = models.Status.objects.filter(user=user).count() return JsonResponse({ '@context': 'https://www.w3.org/ns/activitystreams', 'id': '%s/outbox' % user.actor, @@ -134,90 +128,25 @@ def handle_outgoing_accept(user, to_follow, activity): def handle_shelve(user, book, shelf): ''' a local user is getting a book put on their shelf ''' # update the database + # TODO: this should probably happen in incoming instead models.ShelfBook(book=book, shelf=shelf, added_by=user).save() - # send out the activitypub action - summary = '%s marked %s as %s' % ( - user.username, - book.data['title'], - shelf.name - ) - - uuid = uuid4() - activity = { - '@context': 'https://www.w3.org/ns/activitystreams', - 'id': str(uuid), - 'summary': summary, - 'type': 'Add', - 'actor': user.actor, - 'object': { - 'type': 'Document', - 'name': book.data['title'], - 'url': book.openlibrary_key - }, - 'target': { - 'type': 'Collection', - 'name': shelf.name, - 'id': 'https://%s/user/%s/shelf/%s' % \ - (DOMAIN, user.localname, shelf.identifier) - } - } + activity = get_add_remove_json(user, book, shelf, 'Add') recipients = get_recipients(user, 'public') - models.ShelveActivity( - uuid=uuid, - user=user, - content=activity, - shelf=shelf, - book=book, - ).save() - broadcast(user, activity, recipients) def handle_unshelve(user, book, shelf): ''' a local user is getting a book put on their shelf ''' # update the database + # TODO: this should probably happen in incoming instead row = models.ShelfBook.objects.get(book=book, shelf=shelf) row.delete() - # send out the activitypub action - summary = '%s removed %s from %s' % ( - user.username, - book.data['title'], - shelf.name - ) - - uuid = uuid4() - activity = { - '@context': 'https://www.w3.org/ns/activitystreams', - 'id': str(uuid), - 'summary': summary, - 'type': 'Remove', - 'actor': user.actor, - 'object': { - 'type': 'Document', - 'name': book.data['title'], - 'url': book.openlibrary_key - }, - 'target': { - 'type': 'Collection', - 'name': shelf.name, - 'id': 'https://%s/user/%s/shelf/%s' % \ - (DOMAIN, user.localname, shelf.identifier) - } - } + activity = get_add_remove_json(user, book, shelf, 'Remove') recipients = get_recipients(user, 'public') - models.ShelveActivity( - uuid=uuid, - user=user, - content=activity, - shelf=shelf, - book=book, - activity_type='Remove', - ).save() - broadcast(user, activity, recipients) @@ -226,63 +155,10 @@ def handle_review(user, book, name, content, rating): # validated and saves the review in the database so it has an id review = create_review(user, book, name, content, rating) - review_path = 'https://%s/user/%s/status/%d' % \ - (DOMAIN, user.localname, review.id) - book_path = 'https://%s/book/%s' % (DOMAIN, review.book.openlibrary_key) + #book_path = 'https://%s/book/%s' % (DOMAIN, review.book.openlibrary_key) - now = datetime.utcnow().isoformat() #TODO: should this be http_date? - review_activity = { - 'id': review_path, - 'url': review_path, - 'inReplyTo': book_path, - 'published': now, - 'attributedTo': user.actor, - # TODO: again, assuming all posts are public - 'to': ['https://www.w3.org/ns/activitystreams#Public'], - 'cc': ['https://%s/user/%s/followers' % (DOMAIN, user.localname)], - 'sensitive': False, # TODO: allow content warning/sensitivity - 'content': content, - 'type': 'Note', - 'fedireadsType': 'Review', - 'name': name, - 'rating': rating, # fedireads-only custom field - 'attachment': [], # TODO: the book cover - 'replies': { - 'id': '%s/replies' % review_path, - 'type': 'Collection', - 'first': { - 'type': 'CollectionPage', - 'next': '%s/replies?only_other_accounts=true&page=true' % \ - review_path, - 'partOf': '%s/replies' % review_path, - 'items': [], # TODO: populate with replies - } - } - } - review.activity = review_activity - review.save() - - signer = pkcs1_15.new(RSA.import_key(user.private_key)) - signed_message = signer.sign(SHA256.new(content.encode('utf8'))) - create_activity = { - '@context': 'https://www.w3.org/ns/activitystreams', - - 'id': '%s/activity' % review_path, - 'type': 'Create', - 'actor': user.actor, - 'published': now, - - 'to': ['%s/followers' % user.actor], - 'cc': ['https://www.w3.org/ns/activitystreams#Public'], - - 'object': review_activity, - 'signature': { - 'type': 'RsaSignature2017', - 'creator': 'https://%s/user/%s#main-key' % (DOMAIN, user.localname), - 'created': now, - 'signatureValue': b64encode(signed_message).decode('utf8'), - } - } + review_activity = get_status_json(review) + create_activity = get_create_json(user, review_activity) recipients = get_recipients(user, 'public') broadcast(user, create_activity, recipients) diff --git a/fedireads/templates/feed.html b/fedireads/templates/feed.html index 6ecb65368..33f649ee1 100644 --- a/fedireads/templates/feed.html +++ b/fedireads/templates/feed.html @@ -55,40 +55,21 @@

{% include 'snippets/avatar.html' with user=activity.user %} {% include 'snippets/username.html' with user=activity.user %} - {% if activity.fedireads_type == 'Shelve' %} - {# display a reading/shelving activity #} - {% if activity.shelf.identifier == 'to-read' %} - wants to read - {% elif activity.shelf.identifier == 'read' %} - finished reading - {% elif activity.shelf.identifier == 'reading' %} - started reading - {% else %} - shelved in "{{ activity.shelf.name }}" - {% endif %} -

-
- {% include 'snippets/book.html' with book=activity.book size=large description=True %} -
-
- {% elif activity.fedireads_type == 'Review' %} + {% if activity.status_type == 'Review' %} {# display a review #} reviewed {{ activity.book.data.title }}
{% include 'snippets/book.html' with book=activity.book size=large %} -

{{ activity.content.name }}

-

{{ activity.content.rating | stars }}

-

{{ activity.content.content | safe }}

+

{{ activity.name }}

+

{{ activity.rating | stars }}

+

{{ activity.content | safe }}

- {% elif activity.activity_type == 'Follow' %} - started following someone - - {% elif activity.activity_type == 'Note' %} + {% elif activity.status_type == 'Note' %} posted - {{ activity.content.object.content | safe }} + {{ activity.content | safe }} {% else %} {# generic handling for a misc activity, which perhaps should not be displayed at all #} did {{ activity.activity_type }} diff --git a/fedireads/views.py b/fedireads/views.py index 8292f6eca..1ad12aa9e 100644 --- a/fedireads/views.py +++ b/fedireads/views.py @@ -39,7 +39,7 @@ def home(request): # TODO: this is fundamentally not how the feed should work I think? it # should do something smart with inboxes. (in this implementation it would # show DMs meant for other local users) - activities = models.Activity.objects.filter( + activities = models.Status.objects.filter( user__in=following, ).select_subclasses().order_by( '-created_date'