Merge pull request #162 from mouse-reeve/fedireads_connector

Fedireads connector

fedireads/activitypub/__init__.py

@@ -1,6 +1,6 @@
 ''' bring activitypub functions into the namespace '''
 from .actor import get_actor
-from .book import get_book
+from .book import get_book, get_author
 from .create import get_create, get_update
 from .follow import get_following, get_followers
 from .follow import get_follow_request, get_unfollow, get_accept, get_reject

fedireads/activitypub/book.py

@@ -1,25 +1,23 @@
 ''' federate book data '''
 from fedireads.settings import DOMAIN
 
-def get_book(book):
+def get_book(book, recursive=True):
     ''' activitypub serialize a book '''
 
     fields = [
+        'title',
         'sort_title',
         'subtitle',
         'isbn_13',
         'oclc_number',
         'openlibrary_key',
         'librarything_key',
-        'fedireads_key',
         'lccn',
         'oclc_number',
         'pages',
         'physical_format',
         'misc_identifiers',
 
-        'source_url',
-
         'description',
         'languages',
         'series',
@@ -30,21 +28,28 @@ def get_book(book):
         'physical_format',
     ]
 
+    book_type = type(book).__name__
     activity = {
         '@context': 'https://www.w3.org/ns/activitystreams',
         'type': 'Document',
-        'book_type': type(book).__name__,
+        'book_type': book_type,
         'name': book.title,
         'url': book.absolute_id,
 
-        'authors': [get_author(a) for a in book.authors.all()],
+        'authors': [a.absolute_id for a in book.authors.all()],
         'first_published_date': book.first_published_date.isoformat() if \
                 book.first_published_date else None,
         'published_date': book.published_date.isoformat() if \
                 book.published_date else None,
-        'parent_work': book.parent_work.absolute_id if \
-                hasattr(book, 'parent_work') else None,
     }
+    if recursive:
+        if book_type == 'Edition':
+            activity['work'] = get_book(book.parent_work, recursive=False)
+        else:
+            editions = book.edition_set.order_by('default')
+            activity['editions'] = [
+                get_book(b, recursive=False) for b in editions]
+
     for field in fields:
         if hasattr(book, field):
             activity[field] = book.__getattribute__(field)
@@ -63,7 +68,21 @@ def get_book(book):
 
 def get_author(author):
     ''' serialize an author '''
-    return {
+    fields = [
-        'name': author.name,
+        'name',
+        'born',
+        'died',
+        'aliases',
+        'bio'
+        'openlibrary_key',
+        'wikipedia_link',
+    ]
+    activity = {
+        '@context': 'https://www.w3.org/ns/activitystreams',
         'url': author.absolute_id,
+        'type': 'Person',
     }
+    for field in fields:
+        if hasattr(author, field):
+            activity[field] = author.__getattribute__(field)
+    return activity

fedireads/books_manager.py

@@ -1,31 +1,100 @@
 ''' select and call a connector for whatever book task needs doing '''
-import importlib
+from requests import HTTPError
 
-from fedireads import models
+import importlib
+from urllib.parse import urlparse
+
+from fedireads import models, settings
 from fedireads.tasks import app
 
 
-def get_or_create_book(key):
+def get_or_create_book(value, key='id', connector_id=None):
     ''' pull up a book record by whatever means possible '''
-    try:
+    book = models.Book.objects.select_subclasses().filter(
-        book = models.Book.objects.select_subclasses().get(
+        **{key: value}
-            fedireads_key=key
+    ).first()
-        )
+    if book:
+        if not isinstance(book, models.Edition):
+            return book.default_edition
         return book
-    except models.Book.DoesNotExist:
-        pass
 
-    connector = get_connector()
+    if key == 'remote_id':
-    book = connector.get_or_create_book(key)
+        book = get_by_absolute_id(value, models.Book)
+        if book:
+            return book
+
+    if connector_id:
+        connector_info = models.Connector.objects.get(id=connector_id)
+        connector = load_connector(connector_info)
+    else:
+        connector = get_or_create_connector(value)
+
+    book = connector.get_or_create_book(value)
     load_more_data.delay(book.id)
     return book
 
 
+def get_or_create_connector(remote_id):
+    ''' get the connector related to the author's server '''
+    url = urlparse(remote_id)
+    identifier = url.netloc
+    if not identifier:
+        raise ValueError('Invalid remote id')
+
+    try:
+        connector_info = models.Connector.objects.get(identifier=identifier)
+    except models.Connector.DoesNotExist:
+        connector_info = models.Connector.objects.create(
+            identifier=identifier,
+            connector_file='fedireads_connector',
+            base_url='https://%s' % identifier,
+            books_url='https://%s/book' % identifier,
+            covers_url='https://%s/images/covers' % identifier,
+            search_url='https://%s/search?q=' % identifier,
+            key_name='remote_id',
+            priority=3
+        )
+
+    return load_connector(connector_info)
+
+
+def get_by_absolute_id(absolute_id, model):
+    ''' generalized function to get from a model with a remote_id field '''
+    if not absolute_id:
+        return None
+
+    # check if it's a remote status
+    try:
+        return model.objects.get(remote_id=absolute_id)
+    except model.DoesNotExist:
+        pass
+
+    url = urlparse(absolute_id)
+    if url.netloc != settings.DOMAIN:
+        return None
+
+    # try finding a local status with that id
+    local_id = absolute_id.split('/')[-1]
+    try:
+        if hasattr(model.objects, 'select_subclasses'):
+            possible_match = model.objects.select_subclasses().get(id=local_id)
+        else:
+            possible_match = model.objects.get(id=local_id)
+    except model.DoesNotExist:
+        return None
+
+    # make sure it's not actually a remote status with an id that
+    # clashes with a local id
+    if possible_match.absolute_id == absolute_id:
+        return possible_match
+    return None
+
+
 @app.task
 def load_more_data(book_id):
     ''' background the work of getting all 10,000 editions of LoTR '''
     book = models.Book.objects.select_subclasses().get(id=book_id)
-    connector = get_connector(book)
+    connector = load_connector(book.connector)
     connector.expand_book_data(book)
 
 
@@ -35,7 +104,10 @@ def search(query):
     dedup_slug = lambda r: '%s/%s/%s' % (r.title, r.author, r.year)
     result_index = set()
     for connector in get_connectors():
-        result_set = connector.search(query)
+        try:
+            result_set = connector.search(query)
+        except HTTPError:
+            continue
 
         result_set = [r for r in result_set \
                 if dedup_slug(r) not in result_index]
@@ -49,6 +121,13 @@ def search(query):
     return results
 
 
+def local_search(query):
+    ''' only look at local search results '''
+    connector = load_connector(models.Connector.objects.get(local=True))
+    return connector.search(query)
+
+
+
 def first_search_result(query):
     ''' search until you find a result that fits '''
     for connector in get_connectors():
@@ -58,10 +137,10 @@ def first_search_result(query):
     return None
 
 
-def update_book(book):
+def update_book(book, data=None):
     ''' re-sync with the original data source '''
-    connector = get_connector(book)
+    connector = load_connector(book.connector)
-    connector.update_book(book)
+    connector.update_book(book, data=data)
 
 
 def get_connectors():
@@ -70,18 +149,6 @@ def get_connectors():
     return [load_connector(c) for c in connectors_info]
 
 
-def get_connector(book=None):
-    ''' pick a book data connector '''
-    if book and book.connector:
-        connector_info = book.connector
-    else:
-        # only select from external connectors
-        connector_info = models.Connector.objects.filter(
-            local=False
-        ).order_by('priority').first()
-    return load_connector(connector_info)
-
-
 def load_connector(connector_info):
     ''' instantiate the connector class '''
     connector = importlib.import_module(

fedireads/connectors/abstract_connector.py

@@ -2,6 +2,9 @@
 from abc import ABC, abstractmethod
 from dateutil import parser
 import pytz
+import requests
+
+from django.db import transaction
 
 from fedireads import models
 
@@ -14,34 +17,233 @@ class AbstractConnector(ABC):
         info = models.Connector.objects.get(identifier=identifier)
         self.connector = info
 
-        self.base_url = info.base_url
+        self.book_mappings = {}
-        self.books_url = info.books_url
+        self.key_mappings = {
-        self.covers_url = info.covers_url
+            'isbn_13': ('isbn_13', None),
-        self.search_url = info.search_url
+            'isbn_10': ('isbn_10', None),
-        self.key_name = info.key_name
+            'oclc_numbers': ('oclc_number', None),
-        self.max_query_count = info.max_query_count
+            'lccn': ('lccn', None),
-        self.name = info.name
+        }
-        self.local = info.local
 
+        fields = [
+            'base_url',
+            'books_url',
+            'covers_url',
+            'search_url',
+            'key_name',
+            'max_query_count',
+            'name',
+            'identifier',
+            'local'
+        ]
+        for field in fields:
+            setattr(self, field, getattr(info, field))
 
     def is_available(self):
         ''' check if you're allowed to use this connector '''
-        if self.connector.max_query_count is not None:
+        if self.max_query_count is not None:
-            if self.connector.query_count >= self.connector.max_query_count:
+            if self.connector.query_count >= self.max_query_count:
                 return False
         return True
 
 
-    @abstractmethod
     def search(self, query):
         ''' free text search '''
-        # return list of search result objs
+        resp = requests.get(
+            '%s%s' % (self.search_url, query),
+            headers={
+                'Accept': 'application/json; charset=utf-8',
+            },
+        )
+        if not resp.ok:
+            resp.raise_for_status()
+        data = resp.json()
+        results = []
+
+        for doc in self.parse_search_data(data)[:10]:
+            results.append(self.format_search_result(doc))
+        return results
+
+
+    def get_or_create_book(self, remote_id):
+        ''' pull up a book record by whatever means possible '''
+        # try to load the book
+        book = models.Book.objects.select_subclasses().filter(
+            remote_id=remote_id
+        ).first()
+        if book:
+            if isinstance(book, models.Work):
+                return book.default_edition
+            return book
+
+        # no book was found, so we start creating a new one
+        data = get_data(remote_id)
+
+        work = None
+        edition = None
+        if self.is_work_data(data):
+            work_data = data
+            # if we requested a work and there's already an edition, we're set
+            work = self.match_from_mappings(work_data)
+            if work and work.default_edition:
+                return work.default_edition
+
+            # no such luck, we need more information.
+            try:
+                edition_data = self.get_edition_from_work_data(work_data)
+            except KeyError:
+                # hack: re-use the work data as the edition data
+                # this is why remote ids aren't necessarily unique
+                edition_data = data
+        else:
+            edition_data = data
+            edition = self.match_from_mappings(edition_data)
+            # no need to figure out about the work if we already know about it
+            if edition and edition.parent_work:
+                return edition
+
+            # no such luck, we need more information.
+            try:
+                work_data = self.get_work_from_edition_date(edition_data)
+            except KeyError:
+                # remember this hack: re-use the work data as the edition data
+                work_data = data
+
+        # at this point, we need to figure out the work, edition, or both
+        # atomic so that we don't save a work with no edition for vice versa
+        with transaction.atomic():
+            if not work:
+                work_key = work_data.get('url')
+                work = self.create_book(work_key, work_data, models.Work)
+
+            if not edition:
+                ed_key = edition_data.get('url')
+                edition = self.create_book(ed_key, edition_data, models.Edition)
+                edition.default = True
+                edition.parent_work = work
+                edition.save()
+
+        # now's our change to fill in author gaps
+        if not edition.authors and work.authors:
+            edition.authors.set(work.authors.all())
+            edition.author_text = work.author_text
+            edition.save()
+
+        return edition
+
+
+    def create_book(self, remote_id, data, model):
+        ''' create a work or edition from data '''
+        book = model.objects.create(
+            remote_id=remote_id,
+            title=data['title'],
+            connector=self.connector,
+        )
+        return self.update_book_from_data(book, data)
+
+
+    def update_book_from_data(self, book, data, update_cover=True):
+        ''' for creating a new book or syncing with data '''
+        book = update_from_mappings(book, data, self.book_mappings)
+
+        for author in self.get_authors_from_data(data):
+            book.authors.add(author)
+        book.author_text = ', '.join(a.name for a in book.authors.all())
+        book.save()
+
+        if not update_cover:
+            return book
+
+        cover = self.get_cover_from_data(data)
+        if cover:
+            book.cover.save(*cover, save=True)
+        return book
+
+
+    def update_book(self, book, data=None):
+        ''' load new data '''
+        if not book.sync and not book.sync_cover:
+            return
+
+        if not data:
+            key = getattr(book, self.key_name)
+            data = self.load_book_data(key)
+
+        if book.sync:
+            book = self.update_book_from_data(
+                book, data, update_cover=book.sync_cover)
+        else:
+            cover = self.get_cover_from_data(data)
+            if cover:
+                book.cover.save(*cover, save=True)
+
+        return book
+
+
+    def match_from_mappings(self, data):
+        ''' try to find existing copies of this book using various keys '''
+        keys = [
+            ('openlibrary_key', models.Book),
+            ('librarything_key', models.Book),
+            ('goodreads_key', models.Book),
+            ('lccn', models.Work),
+            ('isbn_10', models.Edition),
+            ('isbn_13', models.Edition),
+            ('oclc_number', models.Edition),
+            ('asin', models.Edition),
+        ]
+        noop = lambda x: x
+        for key, model in keys:
+            formatter = None
+            if key in self.key_mappings:
+                key, formatter = self.key_mappings[key]
+            if not formatter:
+                formatter = noop
+
+            value = data.get(key)
+            if not value:
+                continue
+            value = formatter(value)
+
+            match = model.objects.select_subclasses().filter(
+                **{key: value}).first()
+            if match:
+                return match
 
 
     @abstractmethod
-    def get_or_create_book(self, book_id):
+    def is_work_data(self, data):
-        ''' request and format a book given an identifier '''
+        ''' differentiate works and editions '''
-        # return book model obj
+
+
+    @abstractmethod
+    def get_edition_from_work_data(self, data):
+        ''' every work needs at least one edition '''
+
+
+    @abstractmethod
+    def get_work_from_edition_date(self, data):
+        ''' every edition needs a work '''
+
+
+    @abstractmethod
+    def get_authors_from_data(self, data):
+        ''' load author data '''
+
+
+    @abstractmethod
+    def get_cover_from_data(self, data):
+        ''' load cover '''
+
+
+    @abstractmethod
+    def parse_search_data(self, data):
+        ''' turn the result json from a search into a list '''
+
+
+    @abstractmethod
+    def format_search_result(self, search_result):
+        ''' create a SearchResult obj from json '''
 
 
     @abstractmethod
@@ -49,18 +251,6 @@ class AbstractConnector(ABC):
         ''' get more info on a book '''
 
 
-    @abstractmethod
-    def get_or_create_author(self, book_id):
-        ''' request and format a book given an identifier '''
-        # return book model obj
-
-
-    @abstractmethod
-    def update_book(self, book_obj):
-        ''' sync a book with the canonical remote copy '''
-        # return book model obj
-
-
 def update_from_mappings(obj, data, mappings):
     ''' assign data to model with mappings '''
     noop = lambda x: x
@@ -91,20 +281,35 @@ def has_attr(obj, key):
 
 def get_date(date_string):
     ''' helper function to try to interpret dates '''
+    if not date_string:
+        return None
     try:
         return pytz.utc.localize(parser.parse(date_string))
     except ValueError:
         return None
 
 
-class SearchResult:
+def get_data(url):
+    ''' wrapper for request.get '''
+    resp = requests.get(
+        url,
+        headers={
+            'Accept': 'application/json; charset=utf-8',
+        },
+    )
+    if not resp.ok:
+        resp.raise_for_status()
+    data = resp.json()
+    return data
+
+
+class SearchResult(object):
     ''' standardized search result object '''
-    def __init__(self, title, key, author, year, raw_data):
+    def __init__(self, title, key, author, year):
         self.title = title
         self.key = key
         self.author = author
         self.year = year
-        self.raw_data = raw_data
 
     def __repr__(self):
         return "<SearchResult key={!r} title={!r} author={!r}>".format(

fedireads/connectors/fedireads_connector.py

@@ -1,101 +1,68 @@
 ''' using another fedireads instance as a source of book data '''
+from uuid import uuid4
+
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files.base import ContentFile
 import requests
 
 from fedireads import models
-from .abstract_connector import AbstractConnector
+from .abstract_connector import AbstractConnector, SearchResult
-from .abstract_connector import update_from_mappings, get_date
+from .abstract_connector import update_from_mappings, get_date, get_data
 
 
 class Connector(AbstractConnector):
-    def search(self, query):
+    ''' interact with other instances '''
-        ''' right now you can't search fedireads, but... '''
+    def __init__(self, identifier):
-        resp = requests.get(
+        super().__init__(identifier)
-            '%s%s' % (self.search_url, query),
+        self.book_mappings = self.key_mappings.copy()
-            headers={
+        self.book_mappings.update({
-                'Accept': 'application/activity+json; charset=utf-8',
+            'published_date': ('published_date', get_date),
-            },
+            'first_published_date': ('first_published_date', get_date),
-        )
+        })
-        if not resp.ok:
-            resp.raise_for_status()
-
-        return resp.json()
 
 
-    def get_or_create_book(self, fedireads_key):
+    def is_work_data(self, data):
-        ''' pull up a book record by whatever means possible '''
+        return data['book_type'] == 'Work'
-        try:
-            book = models.Book.objects.select_subclasses().get(
-                fedireads_key=fedireads_key
-            )
-            return book
-        except ObjectDoesNotExist:
-            if self.model.is_self:
-                # we can't load a book from a remote server, this is it
-                return None
-            # no book was found, so we start creating a new one
-            book = models.Book(fedireads_key=fedireads_key)
 
 
-    def update_book(self, book):
+    def get_edition_from_work_data(self, data):
-        ''' add remote data to a local book '''
+        return data['editions'][0]
-        fedireads_key = book.fedireads_key
+
-        response = requests.get(
+
-            '%s/%s' % (self.base_url, fedireads_key),
+    def get_work_from_edition_date(self, data):
-            headers={
+        return data['work']
-                'Accept': 'application/activity+json; charset=utf-8',
+
-            },
+
-        )
+    def get_authors_from_data(self, data):
+        for author_url in data.get('authors', []):
+            yield self.get_or_create_author(author_url)
+
+
+    def get_cover_from_data(self, data):
+        cover_data = data.get('attachment')
+        if not cover_data:
+            return None
+        cover_url = cover_data[0].get('url')
+        response = requests.get(cover_url)
         if not response.ok:
             response.raise_for_status()
 
-        data = response.json()
+        image_name = str(uuid4()) + cover_url.split('.')[-1]
-
+        image_content = ContentFile(response.content)
-        # great, we can update our book.
+        return [image_name, image_content]
-        mappings = {
-            'published_date': ('published_date', get_date),
-            'first_published_date': ('first_published_date', get_date),
-        }
-        book = update_from_mappings(book, data, mappings)
-
-        if not book.source_url:
-            book.source_url = response.url
-        if not book.connector:
-            book.connector = self.connector
-        book.save()
-
-        if data.get('parent_work'):
-            work = self.get_or_create_book(data.get('parent_work'))
-            book.parent_work = work
-
-        for author_blob in data.get('authors', []):
-            author_blob = author_blob.get('author', author_blob)
-            author_id = author_blob['key']
-            author_id = author_id.split('/')[-1]
-            book.authors.add(self.get_or_create_author(author_id))
-
-        if book.sync_cover and data.get('covers') and data['covers']:
-            book.cover.save(*get_cover(data['covers'][0]), save=True)
-
-        return book
 
 
-    def get_or_create_author(self, fedireads_key):
+    def get_or_create_author(self, remote_id):
         ''' load that author '''
         try:
-            return models.Author.objects.get(fedireads_key=fedireads_key)
+            return models.Author.objects.get(remote_id=remote_id)
         except ObjectDoesNotExist:
             pass
 
-        resp = requests.get('%s/authors/%s.json' % (self.url, fedireads_key))
+        data = get_data(remote_id)
-        if not resp.ok:
-            resp.raise_for_status()
-
-        data = resp.json()
 
         # ingest a new author
-        author = models.Author(fedireads_key=fedireads_key)
+        author = models.Author(remote_id=remote_id)
         mappings = {
             'born': ('born', get_date),
             'died': ('died', get_date),
@@ -106,11 +73,14 @@ class Connector(AbstractConnector):
         return author
 
 
-def get_cover(cover_url):
+    def parse_search_data(self, data):
-    ''' ask openlibrary for the cover '''
+        return data
-    image_name = cover_url.split('/')[-1]
+
-    response = requests.get(cover_url)
+
-    if not response.ok:
+    def format_search_result(self, search_result):
-        response.raise_for_status()
+        return SearchResult(**search_result)
-    image_content = ContentFile(response.content)
+
-    return [image_name, image_content]
+
+    def expand_book_data(self, book):
+        # TODO
+        pass

fedireads/connectors/openlibrary.py

@@ -3,184 +3,105 @@ import re
 import requests
 
 from django.core.files.base import ContentFile
-from django.db import transaction
 
 from fedireads import models
 from .abstract_connector import AbstractConnector, SearchResult
-from .abstract_connector import update_from_mappings, get_date
+from .abstract_connector import update_from_mappings
+from .abstract_connector import get_date, get_data
 from .openlibrary_languages import languages
 
 
 class Connector(AbstractConnector):
     ''' instantiate a connector for OL '''
     def __init__(self, identifier):
+        super().__init__(identifier)
         get_first = lambda a: a[0]
-        self.book_mappings = {
+        self.key_mappings = {
+            'isbn_13': ('isbn_13', get_first),
+            'isbn_10': ('isbn_10', get_first),
+            'oclc_numbers': ('oclc_number', get_first),
+            'lccn': ('lccn', get_first),
+        }
+
+        self.book_mappings = self.key_mappings.copy()
+        self.book_mappings.update({
             'publish_date': ('published_date', get_date),
             'first_publish_date': ('first_published_date', get_date),
             'description': ('description', get_description),
-            'isbn_13': ('isbn_13', get_first),
-            'oclc_numbers': ('oclc_number', get_first),
-            'lccn': ('lccn', get_first),
             'languages': ('languages', get_languages),
             'number_of_pages': ('pages', None),
             'series': ('series', get_first),
-        }
+        })
-        super().__init__(identifier)
 
 
-    def search(self, query):
+    def is_work_data(self, data):
-        ''' query openlibrary search '''
+        return not re.match(r'^OL\d+M$', data['key'])
-        resp = requests.get(
-            '%s%s' % (self.search_url, query),
-            headers={
-                'Accept': 'application/json; charset=utf-8',
-            },
-        )
-        if not resp.ok:
-            resp.raise_for_status()
-        data = resp.json()
-        results = []
-
-        for doc in data['docs'][:5]:
-            key = doc['key']
-            key = key.split('/')[-1]
-            author = doc.get('author_name') or ['Unknown']
-            results.append(SearchResult(
-                doc.get('title'),
-                key,
-                author[0],
-                doc.get('first_publish_year'),
-                doc
-            ))
-        return results
 
 
-    def get_or_create_book(self, olkey):
+    def get_edition_from_work_data(self, data):
-        ''' pull up a book record by whatever means possible.
+        try:
-        if you give a work key, it should give you the default edition,
+            key = data['key']
-        annotated with work data. '''
+        except KeyError:
-
+            return False
-        book = models.Book.objects.select_subclasses().filter(
+        url = '%s/%s/editions' % (self.books_url, key)
-            openlibrary_key=olkey
+        data = get_data(url)
-        ).first()
+        return pick_default_edition(data['entries'])
-        if book:
-            if isinstance(book, models.Work):
-                return book.default_edition
-            return book
-
-        # no book was found, so we start creating a new one
-        if re.match(r'^OL\d+W$', olkey):
-            with transaction.atomic():
-                # create both work and a default edition
-                work_data = self.load_book_data(olkey)
-                work = self.create_book(olkey, work_data, models.Work)
-
-                edition_options = self.load_edition_data(olkey).get('entries')
-                edition_data = pick_default_edition(edition_options)
-                if not edition_data:
-                    # hack: re-use the work data as the edition data
-                    edition_data = work_data
-                key = edition_data.get('key').split('/')[-1]
-                edition = self.create_book(key, edition_data, models.Edition)
-                edition.default = True
-                edition.parent_work = work
-                edition.save()
-        else:
-            with transaction.atomic():
-                edition_data = self.load_book_data(olkey)
-                edition = self.create_book(olkey, edition_data, models.Edition)
-
-                work_data = edition_data.get('works')
-                if not work_data:
-                    # hack: we're re-using the edition data as the work data
-                    work_key = olkey
-                else:
-                    work_key = work_data[0]['key'].split('/')[-1]
-
-                work = models.Work.objects.filter(
-                    openlibrary_key=work_key
-                ).first()
-                if not work:
-                    work_data = self.load_book_data(work_key)
-                    work = self.create_book(work_key, work_data, models.Work)
-                edition.parent_work = work
-                edition.save()
-        if not edition.authors and work.authors:
-            edition.authors.set(work.authors.all())
-            edition.author_text = ', '.join(a.name for a in edition.authors)
-
-        return edition
 
 
-    def create_book(self, key, data, model):
+    def get_work_from_edition_date(self, data):
-        ''' create a work or edition from data '''
+        try:
-        book = model.objects.create(
+            key = data['works'][0]['key']
-            openlibrary_key=key,
+        except (IndexError, KeyError):
-            title=data['title'],
+            return False
-            connector=self.connector,
+        url = '%s/%s' % (self.books_url, key)
-        )
+        return get_data(url)
-        return self.update_book_from_data(book, data)
-
-
-    def update_book_from_data(self, book, data):
-        ''' updaet a book model instance from ol data '''
-        # populate the simple data fields
-        update_from_mappings(book, data, self.book_mappings)
-        book.save()
-
-        authors = self.get_authors_from_data(data)
-        for author in authors:
-            book.authors.add(author)
-        if authors:
-            book.author_text = ', '.join(a.name for a in authors)
-
-        if data.get('covers'):
-            book.cover.save(*self.get_cover(data['covers'][0]), save=True)
-        return book
-
-
-    def update_book(self, book):
-        ''' load new data '''
-        if not book.sync and not book.sync_cover:
-            return
-
-        data = self.load_book_data(book.openlibrary_key)
-        if book.sync_cover and data.get('covers'):
-            book.cover.save(*self.get_cover(data['covers'][0]), save=True)
-        if book.sync:
-            book = self.update_book_from_data(book, data)
-        return book
 
 
     def get_authors_from_data(self, data):
         ''' parse author json and load or create authors '''
-        authors = []
         for author_blob in data.get('authors', []):
-            # this id is "/authors/OL1234567A" and we want just "OL1234567A"
             author_blob = author_blob.get('author', author_blob)
+            # this id is "/authors/OL1234567A" and we want just "OL1234567A"
             author_id = author_blob['key'].split('/')[-1]
-            authors.append(self.get_or_create_author(author_id))
+            yield self.get_or_create_author(author_id)
-        return authors
 
 
-    def load_book_data(self, olkey):
+    def get_cover_from_data(self, data):
-        ''' query openlibrary for data on a book '''
+        ''' ask openlibrary for the cover '''
-        response = requests.get('%s/works/%s.json' % (self.books_url, olkey))
+        if not data.get('covers'):
+            return None
+
+        cover_id = data.get('covers')[0]
+        image_name = '%s-M.jpg' % cover_id
+        url = '%s/b/id/%s' % (self.covers_url, image_name)
+        response = requests.get(url)
         if not response.ok:
             response.raise_for_status()
-        data = response.json()
+        image_content = ContentFile(response.content)
-        return data
+        return [image_name, image_content]
+
+
+    def parse_search_data(self, data):
+        return data.get('docs')
+
+
+    def format_search_result(self, doc):
+        key = doc['key']
+        # build the absolute id from the openlibrary key
+        key = self.books_url + key
+        author = doc.get('author_name') or ['Unknown']
+        return SearchResult(
+            doc.get('title'),
+            key,
+            author[0],
+            doc.get('first_publish_year'),
+        )
 
 
     def load_edition_data(self, olkey):
         ''' query openlibrary for editions of a work '''
-        response = requests.get(
+        url = '%s/works/%s/editions.json' % (self.books_url, olkey)
-            '%s/works/%s/editions.json' % (self.books_url, olkey))
+        return get_data(url)
-        if not response.ok:
-            response.raise_for_status()
-        data = response.json()
-        return data
 
 
     def expand_book_data(self, book):
@@ -209,11 +130,9 @@ class Connector(AbstractConnector):
         except models.Author.DoesNotExist:
             pass
 
-        response = requests.get('%s/authors/%s.json' % (self.base_url, olkey))
+        url = '%s/authors/%s.json' % (self.base_url, olkey)
-        if not response.ok:
+        data = get_data(url)
-            response.raise_for_status()
 
-        data = response.json()
         author = models.Author(openlibrary_key=olkey)
         mappings = {
             'birth_date': ('born', get_date),
@@ -221,8 +140,8 @@ class Connector(AbstractConnector):
             'bio': ('bio', get_description),
         }
         author = update_from_mappings(author, data, mappings)
-        # TODO this is making some BOLD assumption
         name = data.get('name')
+        # TODO this is making some BOLD assumption
         if name:
             author.last_name = name.split(' ')[-1]
             author.first_name = ' '.join(name.split(' ')[:-1])
@@ -231,18 +150,6 @@ class Connector(AbstractConnector):
         return author
 
 
-    def get_cover(self, cover_id):
-        ''' ask openlibrary for the cover '''
-        # TODO: get medium and small versions
-        image_name = '%s-M.jpg' % cover_id
-        url = '%s/b/id/%s' % (self.covers_url, image_name)
-        response = requests.get(url)
-        if not response.ok:
-            response.raise_for_status()
-        image_content = ContentFile(response.content)
-        return [image_name, image_content]
-
-
 def get_description(description_blob):
     ''' descriptions can be a string or a dict '''
     if isinstance(description_blob, dict):

fedireads/connectors/self_connector.py

@@ -23,9 +23,9 @@ class Connector(AbstractConnector):
                    SearchVector('isbn_10', weight='A') +\
                    SearchVector('openlibrary_key', weight='B') +\
                    SearchVector('goodreads_key', weight='B') +\
-                   SearchVector('source_url', weight='B') +\
                    SearchVector('asin', weight='B') +\
                    SearchVector('oclc_number', weight='B') +\
+                   SearchVector('remote_id', weight='B') +\
                    SearchVector('description', weight='C') +\
                    SearchVector('series', weight='C')
         ).filter(search=query)
@@ -34,39 +34,49 @@ class Connector(AbstractConnector):
         search_results = []
         for book in results[:10]:
             search_results.append(
-                SearchResult(
+                self.format_search_result(book)
-                    book.title,
-                    book.fedireads_key,
-                    book.author_text,
-                    book.published_date.year if book.published_date else None,
-                    None
-                )
             )
         return search_results
 
 
-    def get_or_create_book(self, fedireads_key):
+    def format_search_result(self, book):
+        return SearchResult(
+            book.title,
+            book.absolute_id,
+            book.author_text,
+            book.published_date.year if book.published_date else None,
+        )
+
+
+    def get_or_create_book(self, book_id):
         ''' since this is querying its own data source, it can only
         get a book, not load one from an external source '''
         try:
             return models.Book.objects.select_subclasses().get(
-                fedireads_key=fedireads_key
+                id=book_id
             )
         except ObjectDoesNotExist:
             return None
 
 
-    def get_or_create_author(self, fedireads_key):
+    def is_work_data(self, data):
-        ''' load that author '''
-        try:
-            return models.Author.objects.get(fedreads_key=fedireads_key)
-        except ObjectDoesNotExist:
-            pass
-
-
-    def update_book(self, book_obj):
         pass
 
+    def get_edition_from_work_data(self, data):
+        pass
+
+    def get_work_from_edition_date(self, data):
+        pass
+
+    def get_authors_from_data(self, data):
+        return None
+
+    def get_cover_from_data(self, data):
+        return None
+
+    def parse_search_data(self, data):
+        ''' it's already in the right format, don't even worry about it '''
+        return data
 
     def expand_book_data(self, book):
         pass

fedireads/forms.py

@@ -109,7 +109,6 @@ class EditionForm(ModelForm):
             'subjects',# TODO
             'subject_places',# TODO
 
-            'source_url',
             'connector',
         ]
 

fedireads/incoming.py

@@ -10,7 +10,7 @@ from django.http import HttpResponseBadRequest, HttpResponseNotFound
 from django.views.decorators.csrf import csrf_exempt
 import requests
 
-from fedireads import models, outgoing
+from fedireads import books_manager, models, outgoing
 from fedireads import status as status_builder
 from fedireads.remote_user import get_or_create_remote_user
 from fedireads.tasks import app
@@ -63,7 +63,7 @@ def shared_inbox(request):
         },
         'Update': {
             'Person': None,# TODO: handle_update_user
-            'Document': None# TODO: handle_update_book
+            'Document': handle_update_book,
         },
     }
     activity_type = activity['type']
@@ -320,3 +320,18 @@ def handle_tag(activity):
     if not user.local:
         book = activity['target']['id'].split('/')[-1]
         status_builder.create_tag(user, book, activity['object']['name'])
+
+
+@app.task
+def handle_update_book(activity):
+    ''' a remote instance changed a book (Document) '''
+    document = activity['object']
+    # check if we have their copy and care about their updates
+    book = models.Book.objects.select_subclasses().filter(
+        remote_id=document['url'],
+        sync=True,
+    ).first()
+    if not book:
+        return
+
+    books_manager.update_book(book, data=document)

fedireads/migrations/0037_auto_20200504_0154.py

@@ -0,0 +1,41 @@
+# Generated by Django 3.0.3 on 2020-05-04 01:54
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fedireads', '0036_auto_20200503_2007'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='author',
+            name='fedireads_key',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='fedireads_key',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='source_url',
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='last_sync_date',
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AddField(
+            model_name='author',
+            name='sync',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='remote_id',
+            field=models.CharField(max_length=255, null=True),
+        ),
+    ]

fedireads/migrations/0038_author_remote_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.3 on 2020-05-09 19:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fedireads', '0037_auto_20200504_0154'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='author',
+            name='remote_id',
+            field=models.CharField(max_length=255, null=True),
+        ),
+    ]

fedireads/models/book.py

@@ -1,6 +1,4 @@
 ''' database schema for books and shelves '''
-from uuid import uuid4
-
 from django.utils import timezone
 from django.db import models
 from model_utils.managers import InheritanceManager
@@ -50,15 +48,14 @@ class Connector(FedireadsModel):
 
 class Book(FedireadsModel):
     ''' a generic book, which can mean either an edition or a work '''
+    remote_id = models.CharField(max_length=255, null=True)
     # these identifiers apply to both works and editions
-    fedireads_key = models.CharField(max_length=255, unique=True, default=uuid4)
     openlibrary_key = models.CharField(max_length=255, blank=True, null=True)
     librarything_key = models.CharField(max_length=255, blank=True, null=True)
     goodreads_key = models.CharField(max_length=255, blank=True, null=True)
     misc_identifiers = JSONField(null=True)
 
     # info about where the data comes from and where/if to sync
-    source_url = models.CharField(max_length=255, unique=True, null=True)
     sync = models.BooleanField(default=True)
     sync_cover = models.BooleanField(default=True)
     last_sync_date = models.DateTimeField(default=timezone.now)
@@ -95,9 +92,10 @@ class Book(FedireadsModel):
     @property
     def absolute_id(self):
         ''' constructs the absolute reference to any db object '''
+        if self.sync and self.remote_id:
+            return self.remote_id
         base_path = 'https://%s' % DOMAIN
-        model_name = type(self).__name__.lower()
+        return '%s/book/%d' % (base_path, self.id)
-        return '%s/book/%s' % (base_path, self.openlibrary_key)
 
     def save(self, *args, **kwargs):
         ''' can't be abstract for query reasons, but you shouldn't USE it '''
@@ -130,6 +128,7 @@ class Edition(Book):
     ''' an edition of a book '''
     # default -> this is what gets displayed for a work
     default = models.BooleanField(default=False)
+
     # these identifiers only apply to editions, not works
     isbn_10 = models.CharField(max_length=255, blank=True, null=True)
     isbn_13 = models.CharField(max_length=255, blank=True, null=True)
@@ -151,8 +150,10 @@ class Edition(Book):
 
 class Author(FedireadsModel):
     ''' copy of an author from OL '''
-    fedireads_key = models.CharField(max_length=255, unique=True, default=uuid4)
+    remote_id = models.CharField(max_length=255, null=True)
     openlibrary_key = models.CharField(max_length=255, blank=True, null=True)
+    sync = models.BooleanField(default=True)
+    last_sync_date = models.DateTimeField(default=timezone.now)
     wikipedia_link = models.CharField(max_length=255, blank=True, null=True)
     # idk probably other keys would be useful here?
     born = models.DateTimeField(blank=True, null=True)

fedireads/outgoing.py

@@ -7,7 +7,7 @@ from django.http import HttpResponseNotFound, JsonResponse
 from django.views.decorators.csrf import csrf_exempt
 import requests
 
-from fedireads import activitypub
+from fedireads import activitypub, books_manager
 from fedireads import models
 from fedireads.broadcast import broadcast
 from fedireads.status import create_review, create_status
@@ -260,9 +260,10 @@ def handle_comment(user, book, content):
         user, book, builder, fr_serializer, ap_serializer, content)
 
 
-def handle_status(user, book, \
+def handle_status(user, book_id, \
         builder, fr_serializer, ap_serializer, *args):
     ''' generic handler for statuses '''
+    book = books_manager.get_or_create_book(book_id)
     status = builder(user, book, *args)
 
     activity = fr_serializer(status)
@@ -286,7 +287,7 @@ def handle_tag(user, book, name):
 
 def handle_untag(user, book, name):
     ''' tag a book '''
-    book = models.Book.objects.get(fedireads_key=book)
+    book = models.Book.objects.get(id=book)
     tag = models.Tag.objects.get(name=name, book=book, user=user)
     tag_activity = activitypub.get_remove_tag(tag)
     tag.delete()

fedireads/status.py

@@ -2,22 +2,20 @@
 from django.db import IntegrityError
 
 from fedireads import models
-from fedireads.books_manager import get_or_create_book
+from fedireads.books_manager import get_or_create_book, get_by_absolute_id
 from fedireads.sanitize_html import InputHtmlParser
 
 
 def create_review_from_activity(author, activity):
     ''' parse an activity json blob into a status '''
     book_id = activity['inReplyToBook']
-    book_id = book_id.split('/')[-1]
+    book = get_or_create_book(book_id, key='remote_id')
     name = activity.get('name')
     rating = activity.get('rating')
     content = activity.get('content')
     published = activity.get('published')
     remote_id = activity['id']
 
-    book = get_or_create_book(book_id)
-
     review = create_review(author, book, name, content, rating)
     review.published_date = published
     review.remote_id = remote_id
@@ -58,8 +56,8 @@ def create_review(user, book, name, content, rating):
 
 def create_quotation_from_activity(author, activity):
     ''' parse an activity json blob into a status '''
-    book = activity['inReplyToBook']
+    book_id = activity['inReplyToBook']
-    book = book.split('/')[-1]
+    book = get_or_create_book(book_id, key='remote_id')
     quote = activity.get('quote')
     content = activity.get('content')
     published = activity.get('published')
@@ -72,10 +70,9 @@ def create_quotation_from_activity(author, activity):
     return quotation
 
 
-def create_quotation(user, possible_book, content, quote):
+def create_quotation(user, book, content, quote):
     ''' a quotation has been added '''
     # throws a value error if the book is not found
-    book = get_or_create_book(possible_book)
     content = sanitize(content)
     quote = sanitize(quote)
 
@@ -87,11 +84,10 @@ def create_quotation(user, possible_book, content, quote):
     )
 
 
-
 def create_comment_from_activity(author, activity):
     ''' parse an activity json blob into a status '''
-    book = activity['inReplyToBook']
+    book_id = activity['inReplyToBook']
-    book = book.split('/')[-1]
+    book = get_or_create_book(book_id, key='remote_id')
     content = activity.get('content')
     published = activity.get('published')
     remote_id = activity['id']
@@ -103,10 +99,9 @@ def create_comment_from_activity(author, activity):
     return comment
 
 
-def create_comment(user, possible_book, content):
+def create_comment(user, book, content):
     ''' a book comment has been added '''
     # throws a value error if the book is not found
-    book = get_or_create_book(possible_book)
     content = sanitize(content)
 
     return models.Comment.objects.create(
@@ -170,34 +165,6 @@ def get_favorite(absolute_id):
     return get_by_absolute_id(absolute_id, models.Favorite)
 
 
-def get_by_absolute_id(absolute_id, model):
-    ''' generalized function to get from a model with a remote_id field '''
-    if not absolute_id:
-        return None
-
-    # check if it's a remote status
-    try:
-        return model.objects.get(remote_id=absolute_id)
-    except model.DoesNotExist:
-        pass
-
-    # try finding a local status with that id
-    local_id = absolute_id.split('/')[-1]
-    try:
-        if hasattr(model.objects, 'select_subclasses'):
-            possible_match = model.objects.select_subclasses().get(id=local_id)
-        else:
-            possible_match = model.objects.get(id=local_id)
-    except model.DoesNotExist:
-        return None
-
-    # make sure it's not actually a remote status with an id that
-    # clashes with a local id
-    if possible_match.absolute_id == absolute_id:
-        return possible_match
-    return None
-
-
 def create_status(user, content, reply_parent=None, mention_books=None,
                   remote_id=None):
     ''' a status update '''
@@ -224,7 +191,7 @@ def create_status(user, content, reply_parent=None, mention_books=None,
 
 def create_tag(user, possible_book, name):
     ''' add a tag to a book '''
-    book = get_or_create_book(possible_book)
+    book = get_or_create_book(possible_book, key='remote_id')
 
     try:
         tag = models.Tag.objects.create(name=name, book=book, user=user)

fedireads/templates/author.html

@@ -16,7 +16,7 @@
     <div class="book-grid row shrink wrap">
         {% for book in books %}
         <div class="book-preview">
-            <a href="{{ book.absolute_id }}">
+            <a href="/book/{{ book.id }}">
             {% include 'snippets/book_cover.html' with book=book %}
             </a>
             {% include 'snippets/shelve_button.html' with book=book %}

fedireads/templates/book.html

@@ -6,7 +6,7 @@
         {% include 'snippets/book_titleby.html' with book=book %}
 
         {% if request.user.is_authenticated %}
-        <a href="{{ book.fedireads_key }}/edit" class="edit-link">edit
+        <a href="{{ book.id }}/edit" class="edit-link">edit
             <span class="icon icon-pencil">
                 <span class="hidden-text">Edit Book</span>
             </span>
@@ -22,7 +22,7 @@
             {% include 'snippets/shelve_button.html' %}
 
             {% if request.user.is_authenticated and not book.cover %}
-            <form name="add-cover" method="POST" action="/upload_cover/{{book.id}}" enctype="multipart/form-data">
+            <form name="add-cover" method="POST" action="/upload_cover/{{ book.id }}" enctype="multipart/form-data">
                 {% csrf_token %}
                 {{ cover_form.as_p }}
                 <button type="submit">Add cover</button>
@@ -57,7 +57,7 @@
                 <h3>Tags</h3>
                 <form name="tag" action="/tag/" method="post">
                     {% csrf_token %}
-                    <input type="hidden" name="book" value="{{ book.fedireads_key }}">
+                    <input type="hidden" name="book" value="{{ book.id }}">
                     <input type="text" name="name">
                     <button type="submit">Add tag</button>
                 </form>

fedireads/templates/book_results.html

@@ -3,19 +3,25 @@
 <div class="content-container">
     <h2>Search results</h2>
     {% for result_set in results %}
+    {% if result_set.results %}
     <section>
         {% if not result_set.connector.local %}
         <h3>
-            Results from <a href="{{ result_set.connector.base_url }}" target="_blank">{{ result_set.connector.name }}</a>
+            Results from <a href="{{ result_set.connector.base_url }}" target="_blank">{% if result_set.connector.name %}{{ result_set.connector.name }}{% else %}{{ result_set.connector.identifier }}{% endif %}</a>
         </h3>
         {% endif %}
 
         {% for result in result_set.results %}
         <div>
-            <a href="/book/{{ result.key }}">{{ result.title }}</a> by {{ result.author }} ({{ result.year }})
+            <form action="/resolve_book" method="POST">
+                {% csrf_token %}
+                <input type="hidden" name="remote_id" value="{{ result.key }}">
+                <button type="submit">{{ result.title }} by {{ result.author }} ({{ result.year }})</button>
+            </form>
         </div>
         {% endfor %}
     </section>
+    {% endif %}
     {% endfor %}
 </div>
 {% endblock %}

fedireads/templates/books.html

@@ -6,7 +6,7 @@
     <div class="book-grid row wrap shrink">
         {% for book in books %}
         <div class="cover-container">
-            <a href="{{ book.absolute_id }}">
+            <a href="/book/{{ book.id }}">
             {% include 'snippets/book_cover.html' with book=book %}
             </a>
             {% include 'snippets/shelve_button.html' with book=book %}

fedireads/templates/edit_book.html

@@ -4,7 +4,7 @@
 <div class="content-container">
     <h2>
         Edit "{{ book.title }}"
-        <a href="/book/{{ book.fedireads_key }}">
+        <a href="/book/{{ book.id }}">
             <span class="edit-link icon icon-close">
                 <span class="hidden-text">Close</span>
             </span>
@@ -40,8 +40,8 @@
 
     <h3>Book Identifiers</h2>
     <div>
-        <p><label for="id_isbn">ISBN 13:</label> {{ form.isbn_13 }} </p>
+        <p><label for="id_isbn_13">ISBN 13:</label> {{ form.isbn_13 }} </p>
-        <p><label for="id_fedireads_key">Fedireads key:</label> {{ form.fedireads_key }} </p>
+        <p><label for="id_isbn_10">ISBN 10:</label> {{ form.isbn_10 }} </p>
         <p><label for="id_openlibrary_key">Openlibrary key:</label> {{ form.openlibrary_key }} </p>
         <p><label for="id_librarything_key">Librarything key:</label> {{ form.librarything_key }} </p>
         <p><label for="id_goodreads_key">Goodreads key:</label> {{ form.goodreads_key }} </p>

fedireads/templates/editions.html

@@ -2,11 +2,11 @@
 {% load fr_display %}
 {% block content %}
 <div class="content-container">
-    <h2>Editions of <a href="/book/{{ work.fedireads_key }}">"{{ work.title }}"</a></h2>
+    <h2>Editions of <a href="/book/{{ work.id }}">"{{ work.title }}"</a></h2>
     <ol class="book-grid row wrap">
         {% for book in editions %}
         <li class="book-preview">
-            <a href="{{ book.absolute_id }}">
+            <a href="/book/{{ book.id }}">
                 {% include 'snippets/book_cover.html' with book=book %}
             </a>
             {% include 'snippets/shelve_button.html' with book=book %}

fedireads/templates/import_status.html

@@ -44,7 +44,7 @@
             <tr>
                 <td>
                     {% if item.book %}
-                    <a href="{{ item.book.absolute_id }}">
+                    <a href="/book/{{ item.book.id }}">
                     {% include 'snippets/book_cover.html' with book=item.book size='small' %}
                     </a>
                     {% endif %}

fedireads/templates/snippets/authors.html

@@ -1 +1 @@
-<a href="/author/{{ book.authors.first.fedireads_key }}" class="author">{{ book.authors.first.name }}</a>
+<a href="/author/{{ book.authors.first.id }}" class="author">{{ book.authors.first.name }}</a>

fedireads/templates/snippets/book_titleby.html

@@ -1,5 +1,5 @@
 <span class="title">
-    <a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
+    <a href="/book/{{ book.id }}">{{ book.title }}</a>
 </span>
 {% if book.authors %}
 <span class="author">

fedireads/templates/snippets/covers_shelf.html

@@ -37,7 +37,7 @@
             <h2>
                 {% include 'snippets/avatar.html' with user=user %}
                 Your thoughts on
-            a   <a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
+            a   <a href="/book/{{ book.id }}">{{ book.title }}</a>
                 by {% include 'snippets/authors.html' with book=book %}
             </h2>
 

fedireads/templates/snippets/create_status.html

@@ -3,13 +3,13 @@
 
 <div class="tabs secondary">
     <div class="tab active" data-id="tab-review-{{ book.id }}" data-category="tab-option-{{ book.id }}">
-        <a href="{{ book.absolute_id }}/review" onclick="tabChange(event)">Review</a>
+        <a href="/book/{{ book.id }}/review" onclick="tabChange(event)">Review</a>
     </div>
     <div class="tab" data-id="tab-comment-{{ book.id }}" data-category="tab-option-{{ book.id }}">
-        <a href="{{ book.absolute_id }}/comment" onclick="tabChange(event)">Comment</a>
+        <a href="/book/{{ book.id }}/comment" onclick="tabChange(event)">Comment</a>
     </div>
     <div class="tab" data-id="tab-quotation-{{ book.id }}" data-category="tab-option-{{ book.id }}">
-        <a href="{{ book.absolute_id }}/quotation" onclick="tabChange(event)">Quote</a>
+        <a href="/book/{{ book.id }}/quotation" onclick="tabChange(event)">Quote</a>
     </div>
 </div>
 
@@ -21,7 +21,7 @@
     {% endif %}
     <form class="tab-option-{{ book.id }} review-form" name="review" action="/review/" method="post" id="tab-review-{{ book.id }}">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.fedireads_key }}">
+        <input type="hidden" name="book" value="{{ book.id }}">
         {% include 'snippets/rate_form.html' with book=book %}
         {{ review_form.as_p }}
         <button type="submit">post review</button>
@@ -29,14 +29,14 @@
 
     <form class="hidden tab-option-{{ book.id }} review-form" name="comment" action="/comment/" method="post" id="tab-comment-{{ book.id }}">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.fedireads_key }}">
+        <input type="hidden" name="book" value="{{ book.id }}">
         {{ comment_form.as_p }}
         <button type="submit">post comment</button>
     </form>
 
     <form class="hidden tab-option-{{ book.id }} review-form quote-form" name="quotation" action="/quotate/" method="post" id="tab-quotation-{{ book.id }}">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.fedireads_key }}">
+        <input type="hidden" name="book" value="{{ book.id }}">
         {{ quotation_form.as_p }}
         <button typr="submit">post quote</button>
     </form>

fedireads/templates/snippets/rate_action.html

@@ -4,7 +4,7 @@
     {% for i in '12345'|make_list %}
     <form name="rate" action="/rate/" method="POST" onsubmit="return rate_stars(event)">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.fedireads_key }}">
+        <input type="hidden" name="book" value="{{ book.id }}">
         <input type="hidden" name="rating" value="{{ forloop.counter }}">
         <button type="submit" class="icon icon-star-{% if book|rating:user < forloop.counter %}empty{% else %}full{% endif %}">
             <span class="hidden-text">{{ forloop.counter }} star{{ forloop.counter | pluralize }}</span>

fedireads/templates/snippets/shelf.html

@@ -39,7 +39,7 @@
         {% include 'snippets/book_cover.html' with book=book size="small" %}
     </td>
     <td>
-        <a href="/book/{{ book.fedireads_key }}">{{ book.title }}</a>
+        <a href="/book/{{ book.id }}">{{ book.title }}</a>
     </td>
     <td>
         {{ book.authors.first.name }}

fedireads/templates/snippets/status_header.html

@@ -6,13 +6,13 @@
 {% if status.status_type == 'Update' %}
     {{ status.content | safe }}
 {% elif status.status_type == 'Review' and not status.name and not status.content%}
-    rated <a href="{{ status.book.absolute_id }}">{{ status.book.title }}</a>
+    rated <a href="/book/{{ status.book.id }}">{{ status.book.title }}</a>
 {% elif status.status_type == 'Review' %}
-    reviewed <a href="{{ status.book.absolute_id }}">{{ status.book.title }}</a>
+    reviewed <a href="/book/{{ status.book.id }}">{{ status.book.title }}</a>
 {% elif status.status_type == 'Comment' %}
-    commented on <a href="{{ status.book.absolute_id }}">{{ status.book.title }}</a>
+    commented on <a href="/book/{{ status.book.id }}">{{ status.book.title }}</a>
 {% elif status.status_type == 'Quotation' %}
-    quoted <a href="{{ status.book.absolute_id }}">{{ status.book.title }}</a>
+    quoted <a href="/book/{{ status.book.id }}">{{ status.book.title }}</a>
 {% elif status.status_type == 'Boost' %}
     boosted
 {% elif status.reply_parent %}

fedireads/templates/snippets/tag.html

@@ -3,14 +3,14 @@
     {% if tag.identifier in user_tags %}
     <form class="tag-form" name="tag" action="/untag/" method="post">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.fedireads_key }}">
+        <input type="hidden" name="book" value="{{ book.id }}">
         <input type="hidden" name="name" value="{{ tag.name }}">
         <button type="submit">x</button>
     </form>
     {% else %}
     <form class="tag-form" name="tag" action="/tag/" method="post">
         {% csrf_token %}
-        <input type="hidden" name="book" value="{{ book.fedireads_key }}">
+        <input type="hidden" name="book" value="{{ book.id }}">
         <input type="hidden" name="name" value="{{ tag.name }}">
         <button type="submit">+</button>
     </form>

fedireads/templates/tag.html

@@ -6,7 +6,7 @@
     <div class="book-grid row wrap shrink">
         {% for book in books.all %}
         <div class="cover-container">
-            <a href="{{ book.absolute_id }}">
+            <a href="/book/{{ book.id }}">
             {% include 'snippets/book_cover.html' with book=book %}
             </a>
             {% include 'snippets/rate_action.html' with user=request.user book=book %}

fedireads/urls.py

@@ -11,7 +11,7 @@ localname_regex = r'(?P<username>[\w\-_]+)'
 user_path = r'^user/%s' % username_regex
 local_user_path = r'^user/%s' % localname_regex
 status_path = r'%s/(status|review|comment)/(?P<status_id>\d+)' % local_user_path
-book_path = r'^book/(?P<book_identifier>[\w\-]+)'
+book_path = r'^book/(?P<book_id>\d+)'
 
 handler404 = 'fedireads.views.not_found_page'
 handler500 = 'fedireads.views.server_error_page'
@@ -63,19 +63,21 @@ urlpatterns = [
     re_path(r'%s/edit/?$' % book_path, views.edit_book_page),
     re_path(r'^editions/(?P<work_id>\d+)/?$', views.editions_page),
 
-    re_path(r'^author/(?P<author_identifier>[\w\-]+)/?$', views.author_page),
+    re_path(r'^author/(?P<author_id>[\w\-]+)(.json)?/?$', views.author_page),
     re_path(r'^tag/(?P<tag_id>.+)/?$', views.tag_page),
     re_path(r'^shelf/%s/(?P<shelf_identifier>[\w-]+)(.json)?/?$' % username_regex, views.shelf_page),
     re_path(r'^shelf/%s/(?P<shelf_identifier>[\w-]+)(.json)?/?$' % localname_regex, views.shelf_page),
 
+    re_path(r'^search/?$', views.search),
+
     # internal action endpoints
     re_path(r'^logout/?$', actions.user_logout),
     re_path(r'^user-login/?$', actions.user_login),
     re_path(r'^register/?$', actions.register),
     re_path(r'^edit_profile/?$', actions.edit_profile),
 
-    re_path(r'^search/?$', actions.search),
     re_path(r'^import_data/?', actions.import_data),
+    re_path(r'^resolve_book/?', actions.resolve_book),
     re_path(r'^edit_book/(?P<book_id>\d+)/?', actions.edit_book),
     re_path(r'^upload_cover/(?P<book_id>\d+)/?', actions.upload_cover),
 

fedireads/view_actions.py

@@ -1,6 +1,5 @@
 ''' views for actions you can take in the application '''
 from io import BytesIO, TextIOWrapper
-import re
 from PIL import Image
 
 from django.contrib.auth import authenticate, login, logout
@@ -10,7 +9,7 @@ from django.http import HttpResponseBadRequest, HttpResponseNotFound
 from django.shortcuts import redirect
 from django.template.response import TemplateResponse
 
-from fedireads import forms, models, books_manager, outgoing
+from fedireads import forms, models, outgoing
 from fedireads import goodreads_import
 from fedireads.settings import DOMAIN
 from fedireads.views import get_user_from_username
@@ -116,6 +115,13 @@ def edit_profile(request):
     return redirect('/user/%s' % request.user.localname)
 
 
+def resolve_book(request):
+    ''' figure out the local path to a book from a remote_id '''
+    remote_id = request.POST.get('remote_id')
+    book = get_or_create_book(remote_id, key='remote_id')
+    return redirect('/book/%d' % book.id)
+
+
 @login_required
 def edit_book(request, book_id):
     ''' edit a book cool '''
@@ -133,7 +139,7 @@ def edit_book(request, book_id):
     form.save()
 
     outgoing.handle_update_book(request.user, book)
-    return redirect('/book/%s' % book.fedireads_key)
+    return redirect('/book/%s' % book.id)
 
 
 @login_required
@@ -157,7 +163,7 @@ def upload_cover(request, book_id):
     book.save()
 
     outgoing.handle_update_book(request.user, book)
-    return redirect('/book/%s' % book.fedireads_key)
+    return redirect('/book/%s' % book.id)
 
 
 @login_required
@@ -190,27 +196,25 @@ def shelve(request):
 def rate(request):
     ''' just a star rating for a book '''
     form = forms.RatingForm(request.POST)
-    book_identifier = request.POST.get('book')
+    book_id = request.POST.get('book')
     # TODO: better failure behavior
     if not form.is_valid():
-        return redirect('/book/%s' % book_identifier)
+        return redirect('/book/%s' % book_id)
 
     rating = form.cleaned_data.get('rating')
     # throws a value error if the book is not found
-    book = get_or_create_book(book_identifier)
 
-    outgoing.handle_rate(request.user, book, rating)
+    outgoing.handle_rate(request.user, book_id, rating)
-    return redirect('/book/%s' % book_identifier)
+    return redirect('/book/%s' % book_id)
 
 
 @login_required
 def review(request):
     ''' create a book review '''
     form = forms.ReviewForm(request.POST)
-    book_identifier = request.POST.get('book')
+    book_id = request.POST.get('book')
-    # TODO: better failure behavior
     if not form.is_valid():
-        return redirect('/book/%s' % book_identifier)
+        return redirect('/book/%s' % book_id)
 
     # TODO: validation, htmlification
     name = form.cleaned_data.get('name')
@@ -221,42 +225,39 @@ def review(request):
     except ValueError:
         rating = None
 
-    # throws a value error if the book is not found
+    outgoing.handle_review(request.user, book_id, name, content, rating)
-    book = get_or_create_book(book_identifier)
+    return redirect('/book/%s' % book_id)
-
-    outgoing.handle_review(request.user, book, name, content, rating)
-    return redirect('/book/%s' % book_identifier)
 
 
 @login_required
 def quotate(request):
     ''' create a book quotation '''
     form = forms.QuotationForm(request.POST)
-    book_identifier = request.POST.get('book')
+    book_id = request.POST.get('book')
     if not form.is_valid():
-        return redirect('/book/%s' % book_identifier)
+        return redirect('/book/%s' % book_id)
 
     quote = form.cleaned_data.get('quote')
     content = form.cleaned_data.get('content')
 
-    outgoing.handle_quotation(request.user, book_identifier, content, quote)
+    outgoing.handle_quotation(request.user, book_id, content, quote)
-    return redirect('/book/%s' % book_identifier)
+    return redirect('/book/%s' % book_id)
 
 
 @login_required
 def comment(request):
     ''' create a book comment '''
     form = forms.CommentForm(request.POST)
-    book_identifier = request.POST.get('book')
+    book_id = request.POST.get('book')
     # TODO: better failure behavior
     if not form.is_valid():
-        return redirect('/book/%s' % book_identifier)
+        return redirect('/book/%s' % book_id)
 
     # TODO: validation, htmlification
     content = form.data.get('content')
 
-    outgoing.handle_comment(request.user, book_identifier, content)
+    outgoing.handle_comment(request.user, book_id, content)
-    return redirect('/book/%s' % book_identifier)
+    return redirect('/book/%s' % book_id)
 
 
 @login_required
@@ -265,20 +266,20 @@ def tag(request):
     # I'm not using a form here because sometimes "name" is sent as a hidden
     # field which doesn't validate
     name = request.POST.get('name')
-    book_identifier = request.POST.get('book')
+    book_id = request.POST.get('book')
 
-    outgoing.handle_tag(request.user, book_identifier, name)
+    outgoing.handle_tag(request.user, book_id, name)
-    return redirect('/book/%s' % book_identifier)
+    return redirect('/book/%s' % book_id)
 
 
 @login_required
 def untag(request):
     ''' untag a book '''
     name = request.POST.get('name')
-    book_identifier = request.POST.get('book')
+    book_id = request.POST.get('book')
 
-    outgoing.handle_untag(request.user, book_identifier, name)
+    outgoing.handle_untag(request.user, book_id, name)
-    return redirect('/book/%s' % book_identifier)
+    return redirect('/book/%s' % book_id)
 
 
 @login_required
@@ -346,22 +347,6 @@ def unfollow(request):
     return redirect('/user/%s' % user_slug)
 
 
-@login_required
-def search(request):
-    ''' that search bar up top '''
-    query = request.GET.get('q')
-    if re.match(r'\w+@\w+.\w+', query):
-        # if something looks like a username, search with webfinger
-        results = [outgoing.handle_account_search(query)]
-        template = 'user_results.html'
-    else:
-        # just send the question over to book search
-        results = books_manager.search(query)
-        template = 'book_results.html'
-
-    return TemplateResponse(request, template, {'results': results})
-
-
 @login_required
 def clear_notifications(request):
     ''' permanently delete notification for user '''

fedireads/views.py

@@ -1,15 +1,19 @@
 ''' views for pages you can go to in the application '''
+import re
+
 from django.contrib.auth.decorators import login_required
 from django.db.models import Avg, Q
 from django.http import HttpResponseBadRequest, HttpResponseNotFound,\
         JsonResponse
 from django.core.exceptions import PermissionDenied
+from django.shortcuts import redirect
 from django.template.response import TemplateResponse
 from django.views.decorators.csrf import csrf_exempt
 
-from fedireads import activitypub
+from fedireads import activitypub, outgoing
 from fedireads import forms, models, books_manager
 from fedireads import goodreads_import
+from fedireads.books_manager import get_or_create_book
 from fedireads.tasks import app
 
 
@@ -139,6 +143,27 @@ def get_activity_feed(user, filter_level, model=models.Status):
     return activities
 
 
+def search(request):
+    ''' that search bar up top '''
+    query = request.GET.get('q')
+    if re.match(r'\w+@\w+.\w+', query):
+        # if something looks like a username, search with webfinger
+        results = [outgoing.handle_account_search(query)]
+        return TemplateResponse(
+            request, 'user_results.html', {'results': results}
+        )
+
+    # or just send the question over to book search
+
+    if is_api_request(request):
+        # only return local results via json so we don't cause a cascade
+        results = books_manager.local_search(query)
+        return JsonResponse([r.__dict__ for r in results], safe=False)
+
+    results = books_manager.search(query)
+    return TemplateResponse(request, 'book_results.html', {'results': results})
+
+
 def books_page(request):
     ''' discover books '''
     recent_books = models.Work.objects
@@ -363,10 +388,9 @@ def edit_profile_page(request):
     return TemplateResponse(request, 'edit_user.html', data)
 
 
-def book_page(request, book_identifier, tab='friends'):
+def book_page(request, book_id, tab='friends'):
     ''' info about a book '''
-    book = books_manager.get_or_create_book(book_identifier)
+    book = get_or_create_book(book_id)
-
     if is_api_request(request):
         return JsonResponse(activitypub.get_book(book))
 
@@ -430,7 +454,7 @@ def book_page(request, book_identifier, tab='friends'):
             {'id': 'federated', 'display': 'Federated'}
         ],
         'active_tab': tab,
-        'path': '/book/%s' % book_identifier,
+        'path': '/book/%s' % book_id,
         'cover_form': forms.CoverForm(instance=book),
         'info_fields': [
             {'name': 'ISBN', 'value': book.isbn_13},
@@ -445,9 +469,9 @@ def book_page(request, book_identifier, tab='friends'):
 
 
 @login_required
-def edit_book_page(request, book_identifier):
+def edit_book_page(request, book_id):
     ''' info about a book '''
-    book = books_manager.get_or_create_book(book_identifier)
+    book = get_or_create_book(book_id)
     if not book.description:
         book.description = book.parent_work.description
     data = {
@@ -468,13 +492,16 @@ def editions_page(request, work_id):
     return TemplateResponse(request, 'editions.html', data)
 
 
-def author_page(request, author_identifier):
+def author_page(request, author_id):
     ''' landing page for an author '''
     try:
-        author = models.Author.objects.get(fedireads_key=author_identifier)
+        author = models.Author.objects.get(id=author_id)
     except ValueError:
         return HttpResponseNotFound()
 
+    if is_api_request(request):
+        return JsonResponse(activitypub.get_author(author))
+
     books = models.Work.objects.filter(authors=author)
     data = {
         'author': author,

init_db.py

@@ -31,10 +31,10 @@ Connector.objects.create(
     books_url='https://%s/book' % DOMAIN,
     covers_url='https://%s/images/covers' % DOMAIN,
     search_url='https://%s/search?q=' % DOMAIN,
-    key_name='openlibrary_key',
+    key_name='id',
     priority=1,
 )
 
 
-get_or_create_book('OL1715344W')
+get_or_create_book('OL1715344W', key='openlibrary_key', connector_id=1)
-get_or_create_book('OL102749W')
+get_or_create_book('OL102749W', key='openlibrary_key', connector_id=1)