Updates to_model to use fields

bookwyrm/activitypub/base_activity.py

@@ -1,20 +1,13 @@
 ''' basics for an activitypub serializer '''
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
-from uuid import uuid4
 
-import dateutil.parser
-from dateutil.parser import ParserError
-from django.core.files.base import ContentFile
 from django.db.models.fields.related_descriptors \
     import ForwardManyToOneDescriptor, ManyToManyDescriptor, \
         ReverseManyToOneDescriptor
-from django.db.models.fields import DateTimeField
 from django.db.models.fields.files import ImageFileDescriptor
-from django.db.models.query_utils import DeferredAttribute
-from django.utils import timezone
 
-from bookwyrm.connectors import ConnectorException, get_data, get_image
+from bookwyrm.connectors import ConnectorException, get_data
 
 class ActivitySerializerError(ValueError):
     ''' routine problems serializing activitypub json '''
@@ -84,35 +77,28 @@ class ActivityObject:
         # check for an existing instance, if we're not updating a known obj
         if not instance:
             instance = find_existing_by_remote_id(model, self.id)
+	# TODO: deduplicate books by identifiers
 
-        model_fields = [m.name for m in model._meta.get_fields()]
         mapped_fields = {}
         many_to_many_fields = {}
         one_to_many_fields = {}
         image_fields = {}
 
-        for mapping in model.activity_mappings:
+        for field in model._meta.get_fields():
-            if mapping.model_key not in model_fields:
+            if not hasattr(field, 'field_to_activity'):
                 continue
+            activitypub_field = field.get_activitypub_field()
+            value = field.field_from_activity(getattr(self, activitypub_field))
+            if value is None:
+                continue
+
             # value is None if there's a default that isn't supplied
             # in the activity but is supplied in the formatter
-            value = None
+            value = getattr(self, activitypub_field)
-            if mapping.activity_key:
+            model_field = getattr(model, field.name)
-                value = getattr(self, mapping.activity_key)
-            model_field = getattr(model, mapping.model_key)
 
-            formatted_value = mapping.model_formatter(value)
+            formatted_value = field.field_from_activity(value)
-            if isinstance(model_field, DeferredAttribute) and \
+            if isinstance(model_field, ForwardManyToOneDescriptor):
-                    isinstance(model_field.field, DateTimeField):
-                try:
-                    date_value = dateutil.parser.parse(formatted_value)
-                    try:
-                        formatted_value = timezone.make_aware(date_value)
-                    except ValueError:
-                        formatted_value = date_value
-                except (ParserError, TypeError):
-                    formatted_value = None
-            elif isinstance(model_field, ForwardManyToOneDescriptor):
                 if not formatted_value:
                     continue
                 # foreign key remote id reolver (work on Edition, for example)
@@ -120,25 +106,30 @@ class ActivityObject:
                 if isinstance(formatted_value, dict) and \
                         formatted_value.get('id'):
                     # if the AP field is a serialized object (as in Add)
-                    remote_id = formatted_value['id']
+                    # or PublicKey
+                    related_model = field.related_model
+                    related_activity = related_model.activity_serializer
+                    mapped_fields[field.name] = related_activity(
+                        **formatted_value
+                    ).to_model(related_model)
                 else:
                     # if the field is just a remote_id (as in every other case)
                     remote_id = formatted_value
-                reference = resolve_remote_id(fk_model, remote_id)
+                    reference = resolve_remote_id(fk_model, remote_id)
-                mapped_fields[mapping.model_key] = reference
+                    mapped_fields[field.name] = reference
             elif isinstance(model_field, ManyToManyDescriptor):
                 # status mentions book/users
-                many_to_many_fields[mapping.model_key] = formatted_value
+                many_to_many_fields[field.name] = formatted_value
             elif isinstance(model_field, ReverseManyToOneDescriptor):
                 # attachments on Status, for example
-                one_to_many_fields[mapping.model_key] = formatted_value
+                one_to_many_fields[field.name] = formatted_value
             elif isinstance(model_field, ImageFileDescriptor):
                 # image fields need custom handling
-                image_fields[mapping.model_key] = formatted_value
+                image_fields[field.name] = formatted_value
             else:
                 if formatted_value == MISSING:
                     formatted_value = None
-                mapped_fields[mapping.model_key] = formatted_value
+                mapped_fields[field.name] = formatted_value
 
         if instance:
             # updating an existing model instance
@@ -154,7 +145,6 @@ class ActivityObject:
 
         # add images
         for (model_key, value) in image_fields.items():
-            formatted_value = image_formatter(value)
             if not formatted_value:
                 continue
             getattr(instance, model_key).save(*formatted_value, save=True)
@@ -243,25 +233,3 @@ def resolve_remote_id(model, remote_id, refresh=False):
     item = model.activity_serializer(**data)
     # if we're refreshing, "result" will be set and we'll update it
     return item.to_model(model, instance=result)
-
-
-def image_formatter(image_slug):
-    ''' helper function to load images and format them for a model '''
-    # when it's an inline image (User avatar/icon, Book cover), it's a json
-    # blob, but when it's an attached image, it's just a url
-    if isinstance(image_slug, dict):
-        url = image_slug.get('url')
-    elif isinstance(image_slug, str):
-        url = image_slug
-    else:
-        return None
-    if not url:
-        return None
-
-    response = get_image(url)
-    if not response:
-        return None
-
-    image_name = str(uuid4()) + '.' + url.split('.')[-1]
-    image_content = ContentFile(response.content)
-    return [image_name, image_content]

bookwyrm/connectors/abstract_connector.py

@@ -8,6 +8,7 @@ from django.db import transaction
 from dateutil import parser
 import requests
 from requests import HTTPError
+from requests.exceptions import SSLError
 
 from bookwyrm import models
 
@@ -322,7 +323,7 @@ def get_image(url):
     ''' wrapper for requesting an image '''
     try:
         resp = requests.get(url)
-    except RequestError:
+    except (RequestError, SSLError):
         return None
     if not resp.ok:
         return None

bookwyrm/migrations/0019_auto_20201130_1939.py

@@ -3,6 +3,17 @@
 import bookwyrm.models.fields
 from django.db import migrations
 
+def update_notnull(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    users = app_registry.get_model('bookwyrm', 'User')
+    for user in users.objects.using(db_alias):
+        if user.name and user.summary:
+            continue
+        if not user.summary:
+            user.summary = ''
+        if not user.name:
+            user.name = ''
+        user.save()
 
 class Migration(migrations.Migration):
 
@@ -11,6 +22,7 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.RunPython(update_notnull),
         migrations.AlterField(
             model_name='user',
             name='name',

bookwyrm/models/__init__.py

@@ -25,3 +25,8 @@ from .site import SiteSettings, SiteInvite, PasswordReset
 cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
 activity_models = {c[1].activity_serializer.__name__: c[1] \
     for c in cls_members if hasattr(c[1], 'activity_serializer')}
+
+def to_activity(activity_json):
+    ''' link up models and activities '''
+    activity_type = activity_json.get('type')
+    return activity_models[activity_type].to_activity(activity_json)

bookwyrm/models/base_model.py

@@ -80,7 +80,9 @@ class ActivitypubMixin:
                 activity[key] += value
             else:
                 activity[key] = value
+
         if hasattr(self, 'serialize_reverse_fields'):
+            # for example, editions of a work
             for field_name in self.serialize_reverse_fields:
                 related_field = getattr(self, field_name)
                 activity[field_name] = unfurl_related_field(related_field)

bookwyrm/models/fields.py

@@ -2,11 +2,14 @@
 import re
 from uuid import uuid4
 
+import dateutil.parser
+from dateutil.parser import ParserError
 from django.contrib.auth.models import AbstractUser
 from django.contrib.postgres.fields import ArrayField as DjangoArrayField
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.db import models
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
@@ -39,10 +42,9 @@ class ActivitypubFieldMixin:
             return {self.activitypub_wrapper: value}
         return value
 
-    def from_activity(self, activity_data):
+    def field_from_activity(self, value):
         ''' formatter to convert activitypub into a model value '''
-        value = activity_data.get(self.activitypub_field)
+        if hasattr(self, 'activitypub_wrapper'):
-        if self.activitypub_wrapper:
             value = value.get(self.activitypub_wrapper)
         return value
 
@@ -100,6 +102,16 @@ class ForeignKey(ActivitypubFieldMixin, models.ForeignKey):
             return None
         return value.remote_id
 
+    def field_from_activity(self, value):
+        if isinstance(value, dict) and value.get('id'):
+            # if the AP field is a serialized object (as in Add)
+            remote_id = value['id']
+        else:
+            # if the field is just a remote_id (as in every other case)
+            remote_id = value
+
+        return resolve_remote_id(remote_id)
+
 
 class OneToOneField(ActivitypubFieldMixin, models.OneToOneField):
     ''' activitypub-aware foreign key field '''
@@ -120,10 +132,10 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
             return '%s/followers' % value.instance.remote_id
         return [i.remote_id for i in value.all()]
 
-    def from_activity(self, activity_data):
+    def field_from_activity(self, valueactivity_data):
         if self.link_only:
             return None
-        values = super().from_activity(activity_data)
+        values = super().field_from_activity(values)
         return values# TODO
 
 
@@ -162,8 +174,8 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
     def field_to_activity(self, value):
         return image_serializer(value)
 
-    def from_activity(self, activity_data):
+    def field_from_activity(self, value):
-        image_slug = super().from_activity(activity_data)
+        image_slug = value
         # when it's an inline image (User avatar/icon, Book cover), it's a json
         # blob, but when it's an attached image, it's just a url
         if isinstance(image_slug, dict):
@@ -191,6 +203,16 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
             return None
         return value.isoformat()
 
+    def field_from_activity(self, value):
+        try:
+            date_value = dateutil.parser.parse(value)
+            try:
+                return timezone.make_aware(date_value)
+            except ValueError:
+                return date_value
+        except (ParserError, TypeError):
+            return None
+
 class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
     ''' activitypub-aware array field '''
     def field_to_activity(self, value):

bookwyrm/models/user.py

@@ -174,7 +174,8 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
 
     def save(self, *args, **kwargs):
         ''' create a key pair '''
-        self.private_key, self.public_key = create_key_pair()
+        if not self.public_key:
+            self.private_key, self.public_key = create_key_pair()
         return super().save(*args, **kwargs)
 
     def to_activity(self):
@@ -194,6 +195,7 @@ def execute_after_save(sender, instance, created, *args, **kwargs):
 
     if not instance.local:
         set_remote_server.delay(instance.id)
+        return
 
     instance.key_pair = KeyPair.objects.create(
         remote_id='%s/#main-key' % instance.remote_id)