Store book data

fedireads/activitypub/collection.py

@@ -120,7 +120,7 @@ def get_add_remove(user, book, shelf, action='Add'):
         'object': {
             # TODO: document??
             'type': 'Document',
-            'name': book.data['title'],
+            'name': book.title,
             'url': book.openlibrary_key
         },
         'target': {

fedireads/activitypub/status.py

@@ -16,7 +16,7 @@ def get_review_article(review):
     ''' a book review formatted for a non-fedireads isntance (mastodon) '''
     status = get_status(review)
     name = 'Review of "%s" (%d stars): %s' % (
-        review.book.data['title'],
+        review.book.title,
         review.rating,
         review.name
     )

fedireads/migrations/0010_auto_20200307_0655.py

@@ -0,0 +1,190 @@
+# Generated by Django 3.0.3 on 2020-03-07 06:55
+
+import datetime
+from django.db import migrations, models
+import django.db.models.deletion
+import fedireads.utils.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fedireads', '0009_status_published_date'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Edition',
+            fields=[
+                ('book_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='fedireads.Book')),
+                ('isbn', models.CharField(max_length=255, null=True, unique=True)),
+                ('oclc_number', models.CharField(max_length=255, null=True, unique=True)),
+                ('pages', models.IntegerField(null=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=('fedireads.book',),
+        ),
+        migrations.CreateModel(
+            name='Work',
+            fields=[
+                ('book_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='fedireads.Book')),
+                ('lccn', models.CharField(max_length=255, null=True, unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=('fedireads.book',),
+        ),
+        migrations.RemoveField(
+            model_name='author',
+            name='data',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='added_by',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='data',
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='aliases',
+            field=fedireads.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='bio',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='born',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='died',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='first_name',
+            field=models.CharField(max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='last_name',
+            field=models.CharField(max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='name',
+            field=models.CharField(default='Unknown', max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='wikipedia_link',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='description',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='first_published_date',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='language',
+            field=models.CharField(max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='last_sync_date',
+            field=models.DateTimeField(default=datetime.datetime.now),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='librarything_key',
+            field=models.CharField(max_length=255, null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='local_edits',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='local_key',
+            field=models.CharField(default=uuid.uuid4, max_length=255, unique=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='misc_identifiers',
+            field=fedireads.utils.fields.JSONField(null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='origin',
+            field=models.CharField(max_length=255, null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='published_date',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='series',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='series_number',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='sort_title',
+            field=models.CharField(max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='subtitle',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='sync',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='title',
+            field=models.CharField(default='Unknown', max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='author',
+            name='openlibrary_key',
+            field=models.CharField(max_length=255, null=True, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='book',
+            name='openlibrary_key',
+            field=models.CharField(max_length=255, null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='parent_work',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='fedireads.Work'),
+        ),
+    ]

fedireads/models/__init__.py

@@ -1,5 +1,5 @@
 ''' bring all the models into the app namespace '''
-from .book import Shelf, ShelfBook, Book, Author
+from .book import Book, Work, Edition, Author
-from .user import User, UserRelationship, FederatedServer
+from .shelf import Shelf, ShelfBook
 from .status import Status, Review, Favorite, Tag
-
+from .user import User, UserRelationship, FederatedServer

fedireads/models/book.py

@@ -1,68 +1,65 @@
 ''' database schema for books and shelves '''
+from datetime import datetime
 from django.db import models
+from uuid import uuid4
 
 from fedireads.settings import DOMAIN
-from fedireads.utils.fields import JSONField
+from fedireads.utils.fields import JSONField, ArrayField
 from fedireads.utils.models import FedireadsModel
 
 
-class Shelf(FedireadsModel):
-    name = models.CharField(max_length=100)
-    identifier = models.CharField(max_length=100)
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    editable = models.BooleanField(default=True)
-    books = models.ManyToManyField(
-        'Book',
-        symmetrical=False,
-        through='ShelfBook',
-        through_fields=('shelf', 'book')
-    )
-
-    @property
-    def absolute_id(self):
-        ''' use shelf identifier as absolute id '''
-        base_path = self.user.absolute_id
-        model_name = type(self).__name__.lower()
-        return '%s/%s/%s' % (base_path, model_name, self.identifier)
-
-    class Meta:
-        unique_together = ('user', 'identifier')
-
-
-class ShelfBook(FedireadsModel):
-    # many to many join table for books and shelves
-    book = models.ForeignKey('Book', on_delete=models.PROTECT)
-    shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT)
-    added_by = models.ForeignKey(
-        'User',
-        blank=True,
-        null=True,
-        on_delete=models.PROTECT
-    )
-
-    class Meta:
-        unique_together = ('book', 'shelf')
-
-
 class Book(FedireadsModel):
-    ''' a non-canonical copy of a work (not book) from open library '''
+    ''' a generic book, which can mean either an edition or a work '''
-    openlibrary_key = models.CharField(max_length=255, unique=True)
+    # these identifiers apply to both works and editions
-    data = JSONField()
+    openlibrary_key = models.CharField(max_length=255, unique=True, null=True)
+    librarything_key = models.CharField(max_length=255, unique=True, null=True)
+    local_key = models.CharField(max_length=255, unique=True, default=uuid4)
+    misc_identifiers = JSONField(null=True)
+
+    # info about where the data comes from and where/if to sync
+    origin = models.CharField(max_length=255, unique=True, null=True)
+    local_edits = models.BooleanField(default=False)
+    sync = models.BooleanField(default=True)
+    last_sync_date = models.DateTimeField(default=datetime.now)
+
+    # TODO: edit history
+
+    # book/work metadata
+    title = models.CharField(max_length=255)
+    sort_title = models.CharField(max_length=255, null=True)
+    subtitle = models.TextField(blank=True, null=True)
+    description = models.TextField(blank=True, null=True)
+    language = models.CharField(max_length=255, null=True)
+    series = models.CharField(max_length=255, blank=True, null=True)
+    series_number = models.CharField(max_length=255, blank=True, null=True)
+    # TODO: include an annotation about the type of authorship (ie, translator)
     authors = models.ManyToManyField('Author')
     # TODO: also store cover thumbnail
     cover = models.ImageField(upload_to='covers/', blank=True, null=True)
+    first_published_date = models.DateTimeField(null=True)
+    published_date = models.DateTimeField(null=True)
     shelves = models.ManyToManyField(
         'Shelf',
         symmetrical=False,
         through='ShelfBook',
         through_fields=('book', 'shelf')
     )
-    added_by = models.ForeignKey(
+    # TODO: why can't I just call this work????
-        'User',
+    parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True)
-        blank=True,
+
-        null=True,
+
-        on_delete=models.PROTECT
+class Work(Book):
-    )
+    ''' a work (an abstract concept of a book that manifests in an edition) '''
+    # library of congress catalog control number
+    lccn = models.CharField(max_length=255, unique=True, null=True)
+
+
+class Edition(Book):
+    ''' an edition of a book '''
+    # these identifiers only apply to work
+    isbn = models.CharField(max_length=255, unique=True, null=True)
+    oclc_number = models.CharField(max_length=255, unique=True, null=True)
+    pages = models.IntegerField(null=True)
 
     @property
     def absolute_id(self):
@@ -74,6 +71,14 @@ class Book(FedireadsModel):
 
 class Author(FedireadsModel):
     ''' copy of an author from OL '''
-    openlibrary_key = models.CharField(max_length=255)
+    openlibrary_key = models.CharField(max_length=255, null=True, unique=True)
-    data = JSONField()
+    wikipedia_link = models.CharField(max_length=255, blank=True, null=True)
+    # idk probably other keys would be useful here?
+    born = models.DateTimeField(null=True)
+    died = models.DateTimeField(null=True)
+    name = models.CharField(max_length=255)
+    last_name = models.CharField(max_length=255, null=True)
+    first_name = models.CharField(max_length=255, null=True)
+    aliases = ArrayField(models.CharField(max_length=255), blank=True)
+    bio = models.TextField(null=True, blank=True)
 

fedireads/models/shelf.py

@@ -0,0 +1,42 @@
+''' puttin' books on shelves '''
+from django.db import models
+
+from fedireads.utils.models import FedireadsModel
+
+
+class Shelf(FedireadsModel):
+    name = models.CharField(max_length=100)
+    identifier = models.CharField(max_length=100)
+    user = models.ForeignKey('User', on_delete=models.PROTECT)
+    editable = models.BooleanField(default=True)
+    books = models.ManyToManyField(
+        'Book',
+        symmetrical=False,
+        through='ShelfBook',
+        through_fields=('shelf', 'book')
+    )
+
+    @property
+    def absolute_id(self):
+        ''' use shelf identifier as absolute id '''
+        base_path = self.user.absolute_id
+        model_name = type(self).__name__.lower()
+        return '%s/%s/%s' % (base_path, model_name, self.identifier)
+
+    class Meta:
+        unique_together = ('user', 'identifier')
+
+
+class ShelfBook(FedireadsModel):
+    # many to many join table for books and shelves
+    book = models.ForeignKey('Book', on_delete=models.PROTECT)
+    shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT)
+    added_by = models.ForeignKey(
+        'User',
+        blank=True,
+        null=True,
+        on_delete=models.PROTECT
+    )
+
+    class Meta:
+        unique_together = ('book', 'shelf')

fedireads/openlibrary.py

@@ -4,7 +4,7 @@ from django.core.files.base import ContentFile
 import re
 import requests
 
-from fedireads.models import Author, Book
+from fedireads import models
 from fedireads.settings import OL_URL
 
 
@@ -29,24 +29,25 @@ def book_search(query):
     return results
 
 
-def get_or_create_book(olkey, user=None, update=False):
+def get_or_create_book(olkey, update=False):
-    ''' add a book by looking up its open library "work" key. I'm conflating
+    ''' create a book or work '''
-    "book" and "work" here a bit; the table is called "book" in fedireads, but
-    in open library parlance, it's a "work," which is the canonical umbrella
-    item that contains all the editions ("book"s) '''
     # check if this is in the format of an OL book identifier
-    if not re.match(r'^OL\d+W$', olkey):
+    if re.match(r'^OL\d+W$', olkey):
-        raise ValueError('Invalid OpenLibrary work ID')
+        model = models.Work
+    elif re.match(r'^OL\d+M$', olkey):
+        model = models.Edition
+    else:
+        raise ValueError('Invalid OpenLibrary ID')
 
     # get the existing entry from our db, if it exists
     try:
-        book = Book.objects.get(openlibrary_key=olkey)
+        book = model.objects.get(openlibrary_key=olkey)
         if not update:
             return book
         # we have the book, but still want to update it from OL
     except ObjectDoesNotExist:
         # no book was found, so we start creating a new one
-        book = Book(openlibrary_key=olkey)
+        book = model(openlibrary_key=olkey)
 
     # load the book json from openlibrary.org
     response = requests.get('%s/works/%s.json' % (OL_URL, olkey))
@@ -54,18 +55,30 @@ def get_or_create_book(olkey, user=None, update=False):
         response.raise_for_status()
 
     data = response.json()
-    book.data = data
-
-    if user and user.is_authenticated:
-        book.added_by = user
 
     # great, we can update our book.
+    book.title = data['title']
+    description = data.get('description')
+    if description:
+        if isinstance(description, dict):
+            description = description.get('value')
+        book.description = description
+    book.pages = data.get('pages')
+    #book.published_date = data.get('publish_date')
+
+    # this book sure as heck better be an edition
+    if data.get('works'):
+        key = data.get('works')[0]['key']
+        key = key.split('/')[-1]
+        work = get_or_create_book(key)
+        book.parent_work = work
     book.save()
 
     # we also need to know the author get the cover
-    for author_blob in data['authors']:
+    for author_blob in data.get('authors'):
         # this id starts as "/authors/OL1234567A" and we want just "OL1234567A"
-        author_id = author_blob['author']['key']
+        author_blob = author_blob.get('author', author_blob)
+        author_id = author_blob['key']
         author_id = author_id.split('/')[-1]
         book.authors.add(get_or_create_author(author_id))
 
@@ -92,7 +105,7 @@ def get_or_create_author(olkey, update=False):
     if not re.match(r'^OL\d+A$', olkey):
         raise ValueError('Invalid OpenLibrary author ID')
     try:
-        author = Author.objects.get(openlibrary_key=olkey)
+        author = models.Author.objects.get(openlibrary_key=olkey)
         if not update:
             return author
     except ObjectDoesNotExist:
@@ -103,7 +116,20 @@ def get_or_create_author(olkey, update=False):
         response.raise_for_status()
 
     data = response.json()
-    author = Author(openlibrary_key=olkey, data=data)
+    author = models.Author(openlibrary_key=olkey)
+    bio = data.get('bio')
+    if bio:
+        if isinstance(bio, dict):
+            bio = bio.get('value')
+        author.bio = bio
+    name = data['name']
+    author.name = name
+    # TODO this is making some BOLD assumption
+    author.last_name = name.split(' ')[-1]
+    author.first_name = ' '.join(name.split(' ')[:-1])
+    #author.born = data.get('birth_date')
+    #author.died = data.get('death_date')
     author.save()
+
     return author
 

fedireads/outgoing.py

@@ -126,7 +126,7 @@ def handle_shelve(user, book, shelf):
         'read': 'finished reading'
     }[shelf.identifier]
     name = user.name if user.name else user.localname
-    message = '%s %s %s' % (name, verb, book.data['title'])
+    message = '%s %s %s' % (name, verb, book.title)
     status = create_status(user, message, mention_books=[book])
 
     activity = activitypub.get_status(status)
@@ -150,7 +150,7 @@ def handle_unshelve(user, book, shelf):
 def handle_review(user, book, name, content, rating):
     ''' post a review '''
     # validated and saves the review in the database so it has an id
-    review = create_review(user, book, name, content, rating)
+    review = create_review(user, book, name, content, rating, None)
 
     review_activity = activitypub.get_review(review)
     review_create_activity = activitypub.get_create(user, review_activity)

fedireads/status.py

@@ -15,14 +15,17 @@ def create_review(user, possible_book, name, content, rating, published):
     # no ratings outside of 0-5
     rating = rating if 0 <= rating <= 5 else 0
 
-    return models.Review.objects.create(
+    review = models.Review(
         user=user,
         book=book,
         name=name,
         rating=rating,
         content=content,
-        published_date=published,
     )
+    if published:
+        review.published_date = published
+    review.save()
+    return review
 
 
 def create_status(user, content, reply_parent=None, mention_books=None):

fedireads/templates/author.html

@@ -3,9 +3,9 @@
 {% block content %}
 <div id="content">
     <div>
-        <h2>{{ author.data.name }}</h2>
+        <h2>{{ author.name }}</h2>
-        {% if author.data.bio %}
+        {% if author.bio %}
-        <blockquote>{{ author.data.bio | author_bio }}
+        <blockquote>{{ author.bio | author_bio }}
         </blockquote>
         {% endif %}
         {% for book in books %}

fedireads/templates/book.html

@@ -3,7 +3,7 @@
 {% block content %}
 <div id="sidebar">
     <div>
-        <h2><q>{{ book.data.title }}</q> and You</h2>
+        <h2><q>{{ book.title }}</q> and You</h2>
         <p>{% if shelf %}On shelf <q>{{ shelf.name }}</q>{% endif %}</p>
         {% include 'snippets/shelve-button.html' with book=book pulldown=True %}
 
@@ -27,13 +27,15 @@
 
 <div id="content">
     <div>
-        <h2><q>{{ book.data.title }}</q> by
+        <h2><q>{{ book.title }}</q> by
             {% include 'snippets/authors.html' with book=book %}</h2>
+        {% if book.parent_work %}<p>Edition of <a href="/book/{{ book.parent_work.openlibrary_key }}">{{ book.parent_work.title }}</a></p>{% endif %}
         <div class="book-preview">
+
             {% include 'snippets/book_cover.html' with book=book size=large %}
             <p>{{ active_tab }} rating: {{ rating | stars }}</p>
             {% if description %}
-            <blockquote>{{ book.data.description | description }}</blockquote>
+            <blockquote>{{ book.description | description }}</blockquote>
             {% endif %}
             <div>
                 <div id="tag-cloud">

fedireads/templates/snippets/authors.html

@@ -1 +1 @@
-by <a href="/author/{{ book.authors.first.openlibrary_key }}" class="author">{{ book.authors.first.data.name }}</a>
+<a href="/author/{{ book.authors.first.openlibrary_key }}" class="author">{{ book.authors.first.name }}</a>

fedireads/templates/snippets/book.html

@@ -1,10 +1,10 @@
 {% load fr_display %}
 {% include 'snippets/book_cover.html' with book=book %}
 <p class="title">
-    <a href="/book/{{ book.openlibrary_key }}">{{ book.data.title }}</a>
+    <a href="/book/{{ book.openlibrary_key }}">{{ book.title }}</a>
 </p>
 <p>
-    {% include 'snippets/authors.html' with book=book %}
+    by {% include 'snippets/authors.html' with book=book %}
 </p>
 
 {% if rating %}
@@ -12,7 +12,7 @@
 {% endif %}
 
 {% if description %}
-<blockquote>{{ book.data.description | description }}</blockquote>
+<blockquote>{{ book.description | description }}</blockquote>
 {% endif %}
 
 {% include 'snippets/shelve-button.html' with book=book pulldown=shelf_pulldown %}

fedireads/templates/snippets/shelf.html

@@ -33,19 +33,19 @@
         {% include 'snippets/book_cover.html' with book=book %}
     </td>
     <td>
-        <a href="/book/{{ book.openlibrary_key }}">{{ book.data.title }}</a>
+        <a href="/book/{{ book.openlibrary_key }}">{{ book.title }}</a>
     </td>
     <td>
         {{ book.authors.first.data.name }}
     </td>
     <td>
-        {{ book.data.first_publish_date }}
+        {{ book.first_publish_date }}
     </td>
     <td>
         {{ book.added_date | naturalday }}
     </td>
     <td>
-        <a href="https://openlibrary.org{{ book.data.key }}" target="_blank">OpenLibrary</a>
+        <a href="https://openlibrary.org{{ book.key }}" target="_blank">OpenLibrary</a>
     </td>
     {% if ratings %}
     <td>

fedireads/templates/snippets/status.html

@@ -1,7 +1,7 @@
 {% load fr_display %}
 <div class="update">
     {% if activity.status_type == 'Review' %}
-        {% include 'snippets/status_banner.html' with content="reviewed <i>"|add:activity.book.data.title|add:"</i>"  %}
+        {% include 'snippets/status_banner.html' with content="reviewed <i>"|add:activity.book.title|add:"</i>"  %}
         <div class="book-preview review">
             {% include 'snippets/book.html' with book=activity.book size=large %}
 

fedireads/templatetags/fr_display.py

@@ -11,6 +11,7 @@ def dict_key(d, k):
     '''Returns the given key from a dictionary.'''
     return d.get(k) or 0
 
+
 @register.filter(name='stars')
 def stars(number):
     ''' turn integers into stars '''
@@ -20,16 +21,19 @@ def stars(number):
         number = 0
     return ('★' * number) + '☆' * (5 - number)
 
+
 @register.filter(name='description')
 def description_format(description):
     ''' handle the various OL description formats '''
-    if isinstance(description, dict) and 'value' in description:
+    if not description:
-        description = description['value']
+        return ''
+
     if '----------' in description:
         description = description.split('----------')[0]
 
     return description.strip()
 
+
 @register.filter(name='author_bio')
 def bio_format(bio):
     ''' clean up OL author bios '''

fedireads/utils/models.py

@@ -1,3 +1,4 @@
+''' base model with default fields '''
 from django.db import models
 
 from fedireads.settings import DOMAIN

fedireads/views.py

@@ -190,26 +190,33 @@ def book_page(request, book_identifier, tab='friends'):
     ''' info about a book '''
     book = openlibrary.get_or_create_book(book_identifier)
 
-    user_reviews = models.Review.objects.filter(user=request.user, book=book).all()
+    if isinstance(book, models.Work):
+        book_reviews = models.Review.objects.filter(
+            Q(book=book) | Q(book__parent_work=book),
+        )
+    else:
+        book_reviews = models.Review.objects.filter(book=book)
+
+    user_reviews = book_reviews.filter(
+        user=request.user,
+    ).all()
 
     if tab == 'friends':
-        reviews = models.Review.objects.filter(
+        reviews = book_reviews.filter(
             Q(user__followers=request.user, privacy='public') | \
+                Q(user=request.user) | \
                 Q(mention_users=request.user),
-            book=book,
         )
     elif tab == 'local':
-        reviews = models.Review.objects.filter(
+        reviews = book_reviews.filter(
             Q(privacy='public') | \
                 Q(mention_users=request.user),
             user__local=True,
-            book=book,
         )
     else:
-        reviews = models.Review.objects.filter(
+        reviews = book_reviews.filter(
             Q(privacy='public') | \
                 Q(mention_users=request.user),
-            book=book,
         )
 
     try: