Merge branch 'main' into review-rate

bookwyrm/connectors/__init__.py

@@ -2,3 +2,5 @@
 from .settings import CONNECTORS
 from .abstract_connector import ConnectorException
 from .abstract_connector import get_data, get_image
+
+from .connector_manager import search, local_search, first_search_result

bookwyrm/connectors/abstract_connector.py

@@ -6,17 +6,13 @@ from urllib3.exceptions import RequestError
 
 from django.db import transaction
 import requests
-from requests import HTTPError
 from requests.exceptions import SSLError
 
 from bookwyrm import activitypub, models, settings
+from .connector_manager import load_more_data, ConnectorException
 
 
 logger = logging.getLogger(__name__)
-class ConnectorException(HTTPError):
-    ''' when the connector can't do what was asked '''
-
-
 class AbstractMinimalConnector(ABC):
     ''' just the bare bones, for other bookwyrm instances '''
     def __init__(self, identifier):
@@ -90,7 +86,6 @@ class AbstractConnector(AbstractMinimalConnector):
         return True
 
 
-    @transaction.atomic
     def get_or_create_book(self, remote_id):
         ''' translate arbitrary json into an Activitypub dataclass '''
         # first, check if we have the origin_id saved
@@ -123,13 +118,17 @@ class AbstractConnector(AbstractMinimalConnector):
         if not work_data or not edition_data:
             raise ConnectorException('Unable to load book data: %s' % remote_id)
 
-        # create activitypub object
-        work_activity = activitypub.Work(**work_data)
-        # this will dedupe automatically
-        work = work_activity.to_model(models.Work)
-        for author in self.get_authors_from_data(data):
-            work.authors.add(author)
-        return self.create_edition_from_data(work, edition_data)
+        with transaction.atomic():
+            # create activitypub object
+            work_activity = activitypub.Work(**work_data)
+            # this will dedupe automatically
+            work = work_activity.to_model(models.Work)
+            for author in self.get_authors_from_data(data):
+                work.authors.add(author)
+
+            edition = self.create_edition_from_data(work, edition_data)
+        load_more_data.delay(self.connector.id, work.id)
+        return edition
 
 
     def create_edition_from_data(self, work, edition_data):
@@ -206,7 +205,7 @@ def get_data(url):
                 'User-Agent': settings.USER_AGENT,
             },
         )
-    except RequestError:
+    except (RequestError, SSLError):
         raise ConnectorException()
     if not resp.ok:
         resp.raise_for_status()

bookwyrm/connectors/connector_manager.py

@@ -1,51 +1,15 @@
-''' select and call a connector for whatever book task needs doing '''
+''' interface with whatever connectors the app has '''
 import importlib
 from urllib.parse import urlparse
 
 from requests import HTTPError
 
 from bookwyrm import models
-from bookwyrm.connectors import ConnectorException
 from bookwyrm.tasks import app
 
 
-def get_edition(book_id):
-    ''' look up a book in the db and return an edition '''
-    book = models.Book.objects.select_subclasses().get(id=book_id)
-    if isinstance(book, models.Work):
-        book = book.default_edition
-    return book
-
-
-def get_or_create_connector(remote_id):
-    ''' get the connector related to the author's server '''
-    url = urlparse(remote_id)
-    identifier = url.netloc
-    if not identifier:
-        raise ValueError('Invalid remote id')
-
-    try:
-        connector_info = models.Connector.objects.get(identifier=identifier)
-    except models.Connector.DoesNotExist:
-        connector_info = models.Connector.objects.create(
-            identifier=identifier,
-            connector_file='bookwyrm_connector',
-            base_url='https://%s' % identifier,
-            books_url='https://%s/book' % identifier,
-            covers_url='https://%s/images/covers' % identifier,
-            search_url='https://%s/search?q=' % identifier,
-            priority=2
-        )
-
-    return load_connector(connector_info)
-
-
-@app.task
-def load_more_data(book_id):
-    ''' background the work of getting all 10,000 editions of LoTR '''
-    book = models.Book.objects.select_subclasses().get(id=book_id)
-    connector = load_connector(book.connector)
-    connector.expand_book_data(book)
+class ConnectorException(HTTPError):
+    ''' when the connector can't do what was asked '''
 
 
 def search(query, min_confidence=0.1):
@@ -92,6 +56,38 @@ def get_connectors():
         yield load_connector(info)
 
 
+def get_or_create_connector(remote_id):
+    ''' get the connector related to the author's server '''
+    url = urlparse(remote_id)
+    identifier = url.netloc
+    if not identifier:
+        raise ValueError('Invalid remote id')
+
+    try:
+        connector_info = models.Connector.objects.get(identifier=identifier)
+    except models.Connector.DoesNotExist:
+        connector_info = models.Connector.objects.create(
+            identifier=identifier,
+            connector_file='bookwyrm_connector',
+            base_url='https://%s' % identifier,
+            books_url='https://%s/book' % identifier,
+            covers_url='https://%s/images/covers' % identifier,
+            search_url='https://%s/search?q=' % identifier,
+            priority=2
+        )
+
+    return load_connector(connector_info)
+
+
+@app.task
+def load_more_data(connector_id, book_id):
+    ''' background the work of getting all 10,000 editions of LoTR '''
+    connector_info = models.Connector.objects.get(id=connector_id)
+    connector = load_connector(connector_info)
+    book = models.Book.objects.select_subclasses().get(id=book_id)
+    connector.expand_book_data(book)
+
+
 def load_connector(connector_info):
     ''' instantiate the connector class '''
     connector = importlib.import_module(

bookwyrm/connectors/openlibrary.py

@@ -3,7 +3,8 @@ import re
 
 from bookwyrm import models
 from .abstract_connector import AbstractConnector, SearchResult, Mapping
-from .abstract_connector import ConnectorException, get_data
+from .abstract_connector import get_data
+from .connector_manager import ConnectorException
 from .openlibrary_languages import languages
 
 
@@ -107,7 +108,7 @@ class Connector(AbstractConnector):
     def get_cover_url(self, cover_blob):
         ''' ask openlibrary for the cover '''
         cover_id = cover_blob[0]
-        image_name = '%s-M.jpg' % cover_id
+        image_name = '%s-L.jpg' % cover_id
         return '%s/b/id/%s' % (self.covers_url, image_name)
 
 

bookwyrm/connectors/self_connector.py

@@ -1,6 +1,9 @@
 ''' using a bookwyrm instance as a source of book data '''
+from functools import reduce
+import operator
+
 from django.contrib.postgres.search import SearchRank, SearchVector
-from django.db.models import F
+from django.db.models import Count, F, Q
 
 from bookwyrm import models
 from .abstract_connector import AbstractConnector, SearchResult
@@ -9,38 +12,18 @@ from .abstract_connector import AbstractConnector, SearchResult
 class Connector(AbstractConnector):
     ''' instantiate a connector  '''
     def search(self, query, min_confidence=0.1):
-        ''' right now you can't search bookwyrm sorry, but when
-        that gets implemented it will totally rule '''
-        vector = SearchVector('title', weight='A') +\
-            SearchVector('subtitle', weight='B') +\
-            SearchVector('authors__name', weight='C') +\
-            SearchVector('isbn_13', weight='A') +\
-            SearchVector('isbn_10', weight='A') +\
-            SearchVector('openlibrary_key', weight='C') +\
-            SearchVector('goodreads_key', weight='C') +\
-            SearchVector('asin', weight='C') +\
-            SearchVector('oclc_number', weight='C') +\
-            SearchVector('remote_id', weight='C') +\
-            SearchVector('description', weight='D') +\
-            SearchVector('series', weight='D')
-
-        results = models.Edition.objects.annotate(
-            search=vector
-        ).annotate(
-            rank=SearchRank(vector, query)
-        ).filter(
-            rank__gt=min_confidence
-        ).order_by('-rank')
-
-        # remove non-default editions, if possible
-        results = results.filter(parent_work__default_edition__id=F('id')) \
-                    or results
-
+        ''' search your local database '''
+        # first, try searching unqiue identifiers
+        results = search_identifiers(query)
+        if not results:
+            # then try searching title/author
+            results = search_title_author(query, min_confidence)
         search_results = []
-        for book in results[:10]:
-            search_results.append(
-                self.format_search_result(book)
-            )
+        for result in results:
+            search_results.append(self.format_search_result(result))
+            if len(search_results) >= 10:
+                break
+        search_results.sort(key=lambda r: r.confidence, reverse=True)
         return search_results
 
 
@@ -52,7 +35,8 @@ class Connector(AbstractConnector):
             year=search_result.published_date.year if \
                     search_result.published_date else None,
             connector=self,
-            confidence=search_result.rank,
+            confidence=search_result.rank if \
+                    hasattr(search_result, 'rank') else 1,
         )
 
 
@@ -74,3 +58,50 @@ class Connector(AbstractConnector):
 
     def expand_book_data(self, book):
         pass
+
+
+def search_identifiers(query):
+    ''' tries remote_id, isbn; defined as dedupe fields on the model '''
+    filters = [{f.name: query} for f in models.Edition._meta.get_fields() \
+        if hasattr(f, 'deduplication_field') and f.deduplication_field]
+    results = models.Edition.objects.filter(
+        reduce(operator.or_, (Q(**f) for f in filters))
+    ).distinct()
+
+    # when there are multiple editions of the same work, pick the default.
+    # it would be odd for this to happen.
+    return results.filter(parent_work__default_edition__id=F('id')) \
+            or results
+
+
+def search_title_author(query, min_confidence):
+    ''' searches for title and author '''
+    vector = SearchVector('title', weight='A') +\
+        SearchVector('subtitle', weight='B') +\
+        SearchVector('authors__name', weight='C') +\
+        SearchVector('series', weight='D')
+
+    results = models.Edition.objects.annotate(
+        search=vector
+    ).annotate(
+        rank=SearchRank(vector, query)
+    ).filter(
+        rank__gt=min_confidence
+    ).order_by('-rank')
+
+    # when there are multiple editions of the same work, pick the closest
+    editions_of_work = results.values(
+        'parent_work'
+    ).annotate(
+        Count('parent_work')
+    ).values_list('parent_work')
+
+    for work_id in set(editions_of_work):
+        editions = results.filter(parent_work=work_id)
+        default = editions.filter(parent_work__default_edition=F('id'))
+        default_rank = default.first().rank if default.exists() else 0
+        # if mutliple books have the top rank, pick the default edition
+        if default_rank == editions.first().rank:
+            yield default.first()
+        else:
+            yield editions.first()

bookwyrm/forms.py

@@ -35,7 +35,7 @@ class CustomForm(ModelForm):
 class LoginForm(CustomForm):
     class Meta:
         model = models.User
-        fields = ['username', 'password']
+        fields = ['localname', 'password']
         help_texts = {f: None for f in fields}
         widgets = {
             'password': PasswordInput(),
@@ -45,7 +45,7 @@ class LoginForm(CustomForm):
 class RegisterForm(CustomForm):
     class Meta:
         model = models.User
-        fields = ['username', 'email', 'password']
+        fields = ['localname', 'email', 'password']
         help_texts = {f: None for f in fields}
         widgets = {
             'password': PasswordInput()

bookwyrm/goodreads_import.py

@@ -8,8 +8,6 @@ from bookwyrm.models import ImportJob, ImportItem
 from bookwyrm.status import create_notification
 
 logger = logging.getLogger(__name__)
-# TODO: remove or increase once we're confident it's not causing problems.
-MAX_ENTRIES = 500
 
 
 def create_job(user, csv_file, include_reviews, privacy):
@@ -19,12 +17,13 @@ def create_job(user, csv_file, include_reviews, privacy):
         include_reviews=include_reviews,
         privacy=privacy
     )
-    for index, entry in enumerate(list(csv.DictReader(csv_file))[:MAX_ENTRIES]):
+    for index, entry in enumerate(list(csv.DictReader(csv_file))):
         if not all(x in entry for x in ('ISBN13', 'Title', 'Author')):
             raise ValueError('Author, title, and isbn must be in data.')
         ImportItem(job=job, index=index, data=entry).save()
     return job
 
+
 def create_retry_job(user, original_job, items):
     ''' retry items that didn't import '''
     job = ImportJob.objects.create(
@@ -37,6 +36,7 @@ def create_retry_job(user, original_job, items):
         ImportItem(job=job, index=item.index, data=item.data).save()
     return job
 
+
 def start_import(job):
     ''' initalizes a csv import job '''
     result = import_data.delay(job.id)
@@ -49,7 +49,6 @@ def import_data(job_id):
     ''' does the actual lookup work in a celery task '''
     job = ImportJob.objects.get(id=job_id)
     try:
-        results = []
         for item in job.items.all():
             try:
                 item.resolve()
@@ -61,7 +60,6 @@ def import_data(job_id):
 
             if item.book:
                 item.save()
-                results.append(item)
 
                 # shelves book and handles reviews
                 outgoing.handle_imported_book(

bookwyrm/migrations/0030_auto_20201224_1939.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.7 on 2020-12-24 19:39
+
+import bookwyrm.models.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0029_auto_20201221_2014'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='localname',
+            field=models.CharField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_localname]),
+        ),
+    ]

bookwyrm/migrations/0031_auto_20210104_2040.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.0.7 on 2021-01-04 20:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0030_auto_20201224_1939'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sitesettings',
+            name='favicon',
+            field=models.ImageField(blank=True, null=True, upload_to='logos/'),
+        ),
+        migrations.AddField(
+            model_name='sitesettings',
+            name='logo',
+            field=models.ImageField(blank=True, null=True, upload_to='logos/'),
+        ),
+        migrations.AddField(
+            model_name='sitesettings',
+            name='logo_small',
+            field=models.ImageField(blank=True, null=True, upload_to='logos/'),
+        ),
+    ]

bookwyrm/migrations/0032_auto_20210104_2055.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.7 on 2021-01-04 20:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookwyrm', '0031_auto_20210104_2040'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sitesettings',
+            name='instance_tagline',
+            field=models.CharField(default='Social Reading and Reviewing', max_length=150),
+        ),
+        migrations.AddField(
+            model_name='sitesettings',
+            name='registration_closed_text',
+            field=models.TextField(default='Contact an administrator to get an invite'),
+        ),
+    ]

bookwyrm/models/fields.py

@@ -26,11 +26,20 @@ def validate_remote_id(value):
         )
 
 
+def validate_localname(value):
+    ''' make sure localnames look okay '''
+    if not re.match(r'^[A-Za-z\-_\.0-9]+$', value):
+        raise ValidationError(
+            _('%(value)s is not a valid username'),
+            params={'value': value},
+        )
+
+
 def validate_username(value):
     ''' make sure usernames look okay '''
-    if not re.match(r'^[A-Za-z\-_\.]+$', value):
+    if not re.match(r'^[A-Za-z\-_\.0-9]+@[A-Za-z\-_\.0-9]+\.[a-z]{2,}$', value):
         raise ValidationError(
-            _('%(value)s is not a valid remote_id'),
+            _('%(value)s is not a valid username'),
             params={'value': value},
         )
 
@@ -147,7 +156,7 @@ class RemoteIdField(ActivitypubFieldMixin, models.CharField):
 
 class UsernameField(ActivitypubFieldMixin, models.CharField):
     ''' activitypub-aware username field '''
-    def __init__(self, activitypub_field='preferredUsername'):
+    def __init__(self, activitypub_field='preferredUsername', **kwargs):
         self.activitypub_field = activitypub_field
         # I don't totally know why pylint is mad at this, but it makes it work
         super( #pylint: disable=bad-super-call

bookwyrm/models/import_job.py

@@ -6,7 +6,7 @@ from django.contrib.postgres.fields import JSONField
 from django.db import models
 from django.utils import timezone
 
-from bookwyrm import books_manager
+from bookwyrm.connectors import connector_manager
 from bookwyrm.models import ReadThrough, User, Book
 from .fields import PrivacyLevels
 
@@ -71,7 +71,7 @@ class ImportItem(models.Model):
 
     def get_book_from_isbn(self):
         ''' search by isbn '''
-        search_result = books_manager.first_search_result(
+        search_result = connector_manager.first_search_result(
             self.isbn, min_confidence=0.999
         )
         if search_result:
@@ -86,7 +86,7 @@ class ImportItem(models.Model):
             self.data['Title'],
             self.data['Author']
         )
-        search_result = books_manager.first_search_result(
+        search_result = connector_manager.first_search_result(
             search_term, min_confidence=0.999
         )
         if search_result:

bookwyrm/models/site.py

@@ -12,11 +12,24 @@ from .user import User
 class SiteSettings(models.Model):
     ''' customized settings for this instance '''
     name = models.CharField(default='BookWyrm', max_length=100)
+    instance_tagline = models.CharField(
+        max_length=150, default='Social Reading and Reviewing')
     instance_description = models.TextField(
-        default="This instance has no description.")
+        default='This instance has no description.')
+    registration_closed_text = models.TextField(
+        default='Contact an administrator to get an invite')
     code_of_conduct = models.TextField(
-        default="Add a code of conduct here.")
+        default='Add a code of conduct here.')
     allow_registration = models.BooleanField(default=True)
+    logo = models.ImageField(
+        upload_to='logos/', null=True, blank=True
+    )
+    logo_small = models.ImageField(
+        upload_to='logos/', null=True, blank=True
+    )
+    favicon = models.ImageField(
+        upload_to='logos/', null=True, blank=True
+    )
     support_link = models.CharField(max_length=255, null=True, blank=True)
     support_title = models.CharField(max_length=100, null=True, blank=True)
     admin_email = models.EmailField(max_length=255, null=True, blank=True)
@@ -52,7 +65,7 @@ class SiteInvite(models.Model):
     @property
     def link(self):
         ''' formats the invite link '''
-        return "https://{}/invite/{}".format(DOMAIN, self.code)
+        return 'https://{}/invite/{}'.format(DOMAIN, self.code)
 
 
 def get_passowrd_reset_expiry():
@@ -74,4 +87,4 @@ class PasswordReset(models.Model):
     @property
     def link(self):
         ''' formats the invite link '''
-        return "https://{}/password-reset/{}".format(DOMAIN, self.code)
+        return 'https://{}/password-reset/{}'.format(DOMAIN, self.code)

bookwyrm/models/status.py

@@ -118,7 +118,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
             activity['attachment'] = [
                 image_serializer(b.cover, b.alt_text) \
                     for b in self.mention_books.all()[:4] if b.cover]
-            if hasattr(self, 'book'):
+            if hasattr(self, 'book') and self.book.cover:
                 activity['attachment'].append(
                     image_serializer(self.book.cover, self.book.alt_text)
                 )

bookwyrm/models/user.py

@@ -1,4 +1,5 @@
 ''' database schema for user data '''
+import re
 from urllib.parse import urlparse
 
 from django.apps import apps
@@ -13,6 +14,7 @@ from bookwyrm.models.status import Status, Review
 from bookwyrm.settings import DOMAIN
 from bookwyrm.signatures import create_key_pair
 from bookwyrm.tasks import app
+from bookwyrm.utils import regex
 from .base_model import OrderedCollectionPageMixin
 from .base_model import ActivitypubMixin, BookWyrmModel
 from .federated_server import FederatedServer
@@ -49,7 +51,8 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     localname = models.CharField(
         max_length=255,
         null=True,
-        unique=True
+        unique=True,
+        validators=[fields.validate_localname],
     )
     # name is your display name, which you can change at will
     name = fields.CharField(max_length=100, null=True, blank=True)
@@ -167,20 +170,17 @@ class User(OrderedCollectionPageMixin, AbstractUser):
     def save(self, *args, **kwargs):
         ''' populate fields for new local users '''
         # this user already exists, no need to populate fields
-        if self.id:
-            return super().save(*args, **kwargs)
-
-        if not self.local:
+        if not self.local and not re.match(regex.full_username, self.username):
             # generate a username that uses the domain (webfinger format)
             actor_parts = urlparse(self.remote_id)
             self.username = '%s@%s' % (self.username, actor_parts.netloc)
             return super().save(*args, **kwargs)
 
+        if self.id or not self.local:
+            return super().save(*args, **kwargs)
+
         # populate fields for local users
-        self.remote_id = 'https://%s/user/%s' % (DOMAIN, self.username)
-        self.localname = self.username
-        self.username = '%s@%s' % (self.username, DOMAIN)
-        self.actor = self.remote_id
+        self.remote_id = 'https://%s/user/%s' % (DOMAIN, self.localname)
         self.inbox = '%s/inbox' % self.remote_id
         self.shared_inbox = 'https://%s/inbox' % DOMAIN
         self.outbox = '%s/outbox' % self.remote_id

bookwyrm/outgoing.py

@@ -166,22 +166,23 @@ def handle_imported_book(user, item, include_reviews, privacy):
     if not item.book:
         return
 
-    if item.shelf:
+    existing_shelf = models.ShelfBook.objects.filter(
+        book=item.book, added_by=user).exists()
+
+    # shelve the book if it hasn't been shelved already
+    if item.shelf and not existing_shelf:
         desired_shelf = models.Shelf.objects.get(
             identifier=item.shelf,
             user=user
         )
-        # shelve the book if it hasn't been shelved already
-        shelf_book, created = models.ShelfBook.objects.get_or_create(
+        shelf_book = models.ShelfBook.objects.create(
             book=item.book, shelf=desired_shelf, added_by=user)
-        if created:
-            broadcast(user, shelf_book.to_add_activity(user), privacy=privacy)
+        broadcast(user, shelf_book.to_add_activity(user), privacy=privacy)
 
-            # only add new read-throughs if the item isn't already shelved
-            for read in item.reads:
-                read.book = item.book
-                read.user = user
-                read.save()
+    for read in item.reads:
+        read.book = item.book
+        read.user = user
+        read.save()
 
     if include_reviews and (item.rating or item.review):
         review_title = 'Review of {!r} on Goodreads'.format(
@@ -220,8 +221,65 @@ def handle_status(user, form):
     status.save()
 
     # inspect the text for user tags
-    matches = []
-    for match in re.finditer(regex.strict_username, status.content):
+    content = status.content
+    for (mention_text, mention_user) in find_mentions(content):
+        # add them to status mentions fk
+        status.mention_users.add(mention_user)
+
+        # turn the mention into a link
+        content = re.sub(
+            r'%s([^@]|$)' % mention_text,
+            r'<a href="%s">%s</a>\g<1>' % \
+                (mention_user.remote_id, mention_text),
+            content)
+
+    # add reply parent to mentions and notify
+    if status.reply_parent:
+        status.mention_users.add(status.reply_parent.user)
+        for mention_user in status.reply_parent.mention_users.all():
+            status.mention_users.add(mention_user)
+
+        if status.reply_parent.user.local:
+            create_notification(
+                status.reply_parent.user,
+                'REPLY',
+                related_user=user,
+                related_status=status
+            )
+
+    # deduplicate mentions
+    status.mention_users.set(set(status.mention_users.all()))
+    # create mention notifications
+    for mention_user in status.mention_users.all():
+        if status.reply_parent and mention_user == status.reply_parent.user:
+            continue
+        if mention_user.local:
+            create_notification(
+                mention_user,
+                'MENTION',
+                related_user=user,
+                related_status=status
+            )
+
+    # don't apply formatting to generated notes
+    if not isinstance(status, models.GeneratedNote):
+        status.content = to_markdown(content)
+    # do apply formatting to quotes
+    if hasattr(status, 'quote'):
+        status.quote = to_markdown(status.quote)
+
+    status.save()
+
+    broadcast(user, status.to_create_activity(user), software='bookwyrm')
+
+    # re-format the activity for non-bookwyrm servers
+    remote_activity = status.to_create_activity(user, pure=True)
+    broadcast(user, remote_activity, software='other')
+
+
+def find_mentions(content):
+    ''' detect @mentions in raw status content '''
+    for match in re.finditer(regex.strict_username, content):
         username = match.group().strip().split('@')[1:]
         if len(username) == 1:
             # this looks like a local user (@user), fill in the domain
@@ -232,44 +290,7 @@ def handle_status(user, form):
         if not mention_user:
             # we can ignore users we don't know about
             continue
-        matches.append((match.group(), mention_user.remote_id))
-        # add them to status mentions fk
-        status.mention_users.add(mention_user)
-        # create notification if the mentioned user is local
-        if mention_user.local:
-            create_notification(
-                mention_user,
-                'MENTION',
-                related_user=user,
-                related_status=status
-            )
-    # add mentions
-    content = status.content
-    for (username, url) in matches:
-        content = re.sub(
-            r'%s([^@])' % username,
-            r'<a href="%s">%s</a>\g<1>' % (url, username),
-            content)
-    if not isinstance(status, models.GeneratedNote):
-        status.content = to_markdown(content)
-    if hasattr(status, 'quote'):
-        status.quote = to_markdown(status.quote)
-    status.save()
-
-    # notify reply parent or tagged users
-    if status.reply_parent and status.reply_parent.user.local:
-        create_notification(
-            status.reply_parent.user,
-            'REPLY',
-            related_user=user,
-            related_status=status
-        )
-
-    broadcast(user, status.to_create_activity(user), software='bookwyrm')
-
-    # re-format the activity for non-bookwyrm servers
-    remote_activity = status.to_create_activity(user, pure=True)
-    broadcast(user, remote_activity, software='other')
+        yield (match.group(), mention_user)
 
 
 def to_markdown(content):

bookwyrm/static/css/format.css

@@ -67,6 +67,13 @@ input.toggle-control:checked ~ .modal.toggle-content {
     width: max-content;
     max-width: 250px;
 }
+.cover-container.is-large {
+    height: max-content;
+    max-width: 500px;
+}
+.cover-container.is-large img {
+    max-height: 500px;
+}
 .cover-container.is-medium {
     height: 150px;
 }

bookwyrm/templates/discover.html

@@ -0,0 +1,80 @@
+{% extends 'layout.html' %}
+{% block content %}
+
+{% if not request.user.is_authenticated %}
+<div class="block">
+    <h1 class="title has-text-centered">{{ site.name }}: {{ site.instance_tagline }}</h1>
+</div>
+
+<section class="tile is-ancestor">
+    <div class="tile is-7 is-parent">
+        <div class="tile is-child box">
+            {% include 'snippets/about.html' %}
+        </div>
+    </div>
+    <div class="tile is-5 is-parent">
+        <div class="tile is-child box has-background-primary-light">
+            {% if site.allow_registration %}
+            <h2 class="title">Join {{ site.name }}</h2>
+            <form name="register" method="post" action="/user-register">
+                {% include 'snippets/register_form.html' %}
+            </form>
+            {% else %}
+            <h2 class="title">This instance is closed</h2>
+            <p>{{ site.registration_closed_text }}</p>
+            {% endif %}
+        </div>
+    </div>
+</section>
+{% else %}
+<div class="block">
+    <h1 class="title has-text-centered">Discover</h1>
+</div>
+{% endif %}
+
+<div class="block is-hidden-tablet">
+    <h2 class="title has-text-centered">Recent Books</h2>
+</div>
+
+<section class="tile is-ancestor">
+    <div class="tile is-vertical">
+        <div class="tile is-parent">
+            <div class="tile is-child box has-background-white-ter">
+                {% include 'snippets/discover/large-book.html' with book=books.0 %}
+            </div>
+        </div>
+        <div class="tile">
+            <div class="tile is-parent is-6">
+                <div class="tile is-child box has-background-white-ter">
+                    {% include 'snippets/discover/small-book.html' with book=books.1 %}
+                </div>
+            </div>
+            <div class="tile is-parent is-6">
+                <div class="tile is-child box has-background-white-ter">
+                    {% include 'snippets/discover/small-book.html' with book=books.2 %}
+                </div>
+            </div>
+        </div>
+    </div>
+    <div class="tile is-vertical">
+        <div class="tile">
+            <div class="tile is-parent is-6">
+                <div class="tile is-child box has-background-white-ter">
+                    {% include 'snippets/discover/small-book.html' with book=books.3 %}
+                </div>
+            </div>
+            <div class="tile is-parent is-6">
+                <div class="tile is-child box has-background-white-ter">
+                    {% include 'snippets/discover/small-book.html' with book=books.4 %}
+                </div>
+            </div>
+        </div>
+        <div class="tile is-parent">
+            <div class="tile is-child box has-background-white-ter">
+                {% include 'snippets/discover/large-book.html' with book=books.5 %}
+            </div>
+        </div>
+    </div>
+</section>
+
+{% endblock %}

bookwyrm/templates/import.html

@@ -21,8 +21,6 @@
         </div>
         <button class="button is-primary" type="submit">Import</button>
     </form>
-    <p>
-    Imports are limited in size, and only the first {{ limit }} items will be imported.
 </div>
 
 <div class="content block">

bookwyrm/templates/layout.html

@@ -8,20 +8,23 @@
     <link type="text/css" rel="stylesheet" href="/static/css/format.css">
     <link type="text/css" rel="stylesheet" href="/static/css/icons.css">
 
-    <link rel="shortcut icon" type="image/x-icon" href="/static/images/favicon.ico">
+    <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}/images/{{ site.favicon }}{% else %}/static/images/favicon.ico{% endif %}">
 
     <meta name="twitter:card" content="summary">
-    <meta name="twitter:title" content="{{ site.name }}">
-    <meta name="og:title" content="{{ site.name }}">
-    <meta name="twitter:description" content="Federated Social Reading">
-    <meta name="og:description" content="Federated Social Reading">
+    <meta name="twitter:title" content="{% if title %}{{ title }} | {% endif %}{{ site.name }}">
+    <meta name="og:title" content="{% if title %}{{ title }} | {% endif %}{{ site.name }}">
+    <meta name="twitter:description" content="{{ site.instance_tagline }}">
+    <meta name="og:description" content="{{ site.instance_tagline }}">
+
+    <meta name="twitter:image" content="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}">
+    <meta name="og:image" content="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}">
+    <meta name="twitter:image:alt" content="BookWyrm Logo">
 </head>
 <body>
-
-<nav class="navbar" role="navigation" aria-label="main navigation">
+<nav class="navbar container" role="navigation" aria-label="main navigation">
     <div class="navbar-brand">
         <a class="navbar-item" href="/">
-            <img class="image logo" src="/static/images/logo-small.png" alt="Home page">
+            <img class="image logo" src="{% if site.logo_small %}/images/{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" alt="Home page">
         </a>
         <form class="navbar-item column" action="/search/">
             <div class="field has-addons">
@@ -63,33 +66,45 @@
         <div class="navbar-end">
 	    {% if request.user.is_authenticated %}
             <div class="navbar-item has-dropdown is-hoverable">
-                <div class="navbar-link"><p>
+                <div class="navbar-link" role="button" aria-expanded=false" onclick="toggleMenu(this)" tabindex="0" aria-haspopup="true" aria-controls="navbar-dropdown"><p>
                     {% include 'snippets/avatar.html' with user=request.user %}
                     {% include 'snippets/username.html' with user=request.user %}
                 </p></div>
-                <div class="navbar-dropdown">
-                    <a href="/direct-messages" class="navbar-item">
-                        Direct messages
-                    </a>
-                    <a href="/user/{{request.user.localname}}" class="navbar-item">
-                        Profile
-                    </a>
-                    <a href="/user-edit" class="navbar-item">
-                        Settings
-                    </a>
-                    <a href="/import" class="navbar-item">
-                        Import books
-                    </a>
+                <ul class="navbar-dropdown" id="navbar-dropdown">
+                    <li>
+                        <a href="/direct-messages" class="navbar-item">
+                            Direct messages
+                        </a>
+                    </li>
+                    <li>
+                        <a href="/user/{{request.user.localname}}" class="navbar-item">
+                            Profile
+                        </a>
+                    </li>
+                    <li>
+                        <a href="/user-edit" class="navbar-item">
+                            Settings
+                        </a>
+                    </li>
+                    <li>
+                        <a href="/import" class="navbar-item">
+                            Import books
+                        </a>
+                    </li>
                     {% if perms.bookwyrm.create_invites %}
-                    <a href="/invite" class="navbar-item">
-                        Invites
-                    </a>
+                    <li>
+                        <a href="/invite" class="navbar-item">
+                            Invites
+                        </a>
+                    </li>
                     {% endif %}
                     <hr class="navbar-divider">
-                    <a href="/logout" class="navbar-item">
-                        Log out
-                    </a>
-                </div>
+                    <li>
+                        <a href="/logout" class="navbar-item">
+                            Log out
+                        </a>
+                    </li>
+                </ul>
             </div>
             <div class="navbar-item">
                 <a href="/notifications">
@@ -105,48 +120,73 @@
                     </div>
                 </a>
             </div>
-	    {% else %}
+            {% else %}
             <div class="navbar-item">
-                <div class="buttons">
-                    <a href="/login" class="button is-primary">
-                        Join
-                    </a>
+                {% if request.path != '/login' and request.path != '/login/' and request.path != '/user-login' %}
+                <div class="columns">
+                    <div class="column">
+                        <form name="login" method="post" action="/user-login">
+                            {% csrf_token %}
+                            <div class="field is-grouped">
+                                <div class="control">
+                                    <label class="is-sr-only" for="id_localname">Username:</label>
+                                    <input type="text" name="localname" maxlength="150" class="input" required="" id="id_localname" placeholder="username">
+                                </div>
+                                <div class="control">
+                                    <label class="is-sr-only" for="id_password">Username:</label>
+                                    <input type="password" name="password" maxlength="128" class="input" required="" id="id_password" placeholder="password">
+                                    <p class="help"><a href="/password-reset">Forgot your password?</a></p>
+                                </div>
+                                <button class="button is-primary" type="submit">Log in</button>
+                            </div>
+                        </form>
+                    </div>
+                    {% if site.allow_registration and request.path != '' and request.path != '/' %}
+                    <div class="column is-narrow">
+                        <a href="/" class="button is-link">
+                            Join
+                        </a>
+                    </div>
+                    {% endif %}
                 </div>
+                {% endif %}
             </div>
-	    {% endif %}
+            {% endif %}
         </div>
     </div>
 </nav>
 
 
-<div class="section">
+<div class="section container">
     {% block content %}
     {% endblock %}
 </div>
 
-<div class="footer">
-    <div class="columns">
-        <div class="column">
-            <p>
-                <a href="/about">About this server</a>
-            </p>
-            {% if site.admin_email %}
-            <p>
-                <a href="mailto:{{ site.admin_email }}">Contact site admin</a>
-            </p>
+<footer class="footer">
+    <div class="container">
+        <div class="columns">
+            <div class="column">
+                <p>
+                    <a href="/about">About this server</a>
+                </p>
+                {% if site.admin_email %}
+                <p>
+                    <a href="mailto:{{ site.admin_email }}">Contact site admin</a>
+                </p>
+                {% endif %}
+            </div>
+            {% if site.support_link %}
+            <div class="column">
+                <span class="icon icon-heart"></span>
+                Support {{ site.name }} on <a href="{{ site.support_link }}" target="_blank">{{ site.support_title }}</a>
+            </div>
             {% endif %}
-        </div>
-        {% if site.support_link %}
-        <div class="column">
-            <span class="icon icon-heart"></span>
-            Support {{ site.name }} on <a href="{{ site.support_link }}" target="_blank">{{ site.support_title }}</a>
-        </div>
-        {% endif %}
-        <div class="column">
-            BookWyrm is open source software. You can contribute or report issues on <a href="https://github.com/mouse-reeve/bookwyrm">GitHub</a>.
+            <div class="column">
+                BookWyrm is open source software. You can contribute or report issues on <a href="https://github.com/mouse-reeve/bookwyrm">GitHub</a>.
+            </div>
         </div>
     </div>
-</div>
+</footer>
 
 <script>
     var csrf_token = '{{ csrf_token }}';
@@ -154,4 +194,3 @@
 <script src="/static/js/shared.js"></script>
 </body>
 </html>
-

bookwyrm/templates/login.html

@@ -11,9 +11,9 @@
             <form name="login" method="post" action="/user-login">
                 {% csrf_token %}
                 <div class="field">
-                    <label class="label" for="id_username">Username:</label>
+                    <label class="label" for="id_localname">Username:</label>
                     <div class="control">
-                        {{ login_form.username }}
+                        {{ login_form.localname }}
                     </div>
                 </div>
                 <div class="field">

bookwyrm/templates/snippets/about.html

@@ -1,7 +1,12 @@
-<h1 class="title">About {{ site.name }}</h1>
-<div class="block">
-    <img src="/static/images/logo.png" alt="BookWyrm">
+<div class="columns">
+    <div class="column is-narrow is-hidden-mobile">
+        <figure class="block">
+            <img src="{% if site.logo_small %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}" alt="BookWyrm logo">
+        </figure>
+    </div>
+    <div class="content">
+        <p class="block">
+            {{ site.instance_description | safe }}
+        </p>
+    </div>
 </div>
-<p class="block">
-    {{ site.instance_description }}
-</p>

bookwyrm/templates/snippets/discover/large-book.html

@@ -0,0 +1,18 @@
+{% load bookwyrm_tags %}
+{% if book %}
+<div class="columns">
+    <div class="column is-narrow">
+        {% include 'snippets/book_cover.html' with book=book size="large" %}
+        {% include 'snippets/stars.html' with rating=ratings|dict_key:book.id %}
+    </div>
+    <div class="column">
+        <h3 class="title is-5"><a href="/book/{{ book.id }}">{{ book.title }}</a></h3>
+        {% if book.authors %}
+        <p class="subtitle is-5">by {% include 'snippets/authors.html' with book=book %}</p>
+        {% endif %}
+        {% if book|book_description %}
+        <blockquote class="content">{{ book|book_description|to_markdown|safe|truncatewords_html:50 }}</blockquote>
+        {% endif %}
+    </div>
+</div>
+{% endif %}

bookwyrm/templates/snippets/discover/small-book.html

@@ -0,0 +1,11 @@
+{% load bookwyrm_tags %}
+{% if book %}
+{% include 'snippets/book_cover.html' with book=book %}
+{% include 'snippets/stars.html' with rating=ratings|dict_key:book.id %}
+
+<h3 class="title is-6"><a href="/book/{{ book.id }}">{{ book.title }}</a></h3>
+{% if book.authors %}
+<p class="subtitle is-6">by {% include 'snippets/authors.html' with book=book %}</p>
+{% endif %}
+
+{% endif %}

bookwyrm/templates/snippets/register_form.html

@@ -1,10 +1,10 @@
 {% csrf_token %}
 <div class="field">
-    <label class="label" for="id_username_register">Username:</label>
+    <label class="label" for="id_localname_register">Username:</label>
     <div class="control">
-        <input type="text" name="username" maxlength="150" class="input" required="" id="id_username_register" value="{% if register_form.username.value %}{{ register_form.username.value }} {% endif %}">
+        <input type="text" name="localname" maxlength="150" class="input" required="" id="id_localname_register" value="{% if register_form.localname.value %}{{ register_form.localname.value }}{% endif %}">
     </div>
-    {% for error in register_form.username.errors %}
+    {% for error in register_form.localname.errors %}
     <p class="help is-danger">{{ error | escape }}</p>
     {% endfor %}
 </div>

bookwyrm/templates/snippets/reply_form.html

@@ -10,7 +10,7 @@
             {% include 'snippets/content_warning_field.html' with parent_status=status %}
             <label for="id_content_{{ status.id }}-{{ uuid }}" class="is-sr-only">Reply</label>
             <div class="field">
-                <textarea class="textarea" name="content" placeholder="Leave a comment..." id="id_content_{{ status.id }}-{{ uuid }}" required="true">{{ status|mentions:request.user }}</textarea>
+                <textarea class="textarea" name="content" placeholder="Leave a comment..." id="id_content_{{ status.id }}-{{ uuid }}" required="true"></textarea>
             </div>
         </div>
         <div class="column is-narrow">

bookwyrm/tests/activitypub/test_base_activity.py

@@ -20,7 +20,8 @@ class BaseActivity(TestCase):
     def setUp(self):
         ''' we're probably going to re-use this so why copy/paste '''
         self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         self.user.remote_id = 'http://example.com/a/b'
         self.user.save()
 

bookwyrm/tests/connectors/test_abstract_connector.py

@@ -1,4 +1,5 @@
 ''' testing book data connectors '''
+from unittest.mock import patch
 from django.test import TestCase
 import responses
 
@@ -104,8 +105,10 @@ class AbstractConnector(TestCase):
             'https://example.com/book/abcd',
             json=self.edition_data
         )
-        result = self.connector.get_or_create_book(
-            'https://example.com/book/abcd')
+        with patch(
+                'bookwyrm.connectors.abstract_connector.load_more_data.delay'):
+            result = self.connector.get_or_create_book(
+                'https://example.com/book/abcd')
         self.assertEqual(result, self.book)
         self.assertEqual(models.Edition.objects.count(), 1)
         self.assertEqual(models.Edition.objects.count(), 1)

bookwyrm/tests/connectors/test_connector_manager.py

@@ -1,12 +1,18 @@
+''' interface between the app and various connectors '''
 from django.test import TestCase
 
-from bookwyrm import books_manager, models
-from bookwyrm.connectors.bookwyrm_connector import Connector as BookWyrmConnector
-from bookwyrm.connectors.self_connector import Connector as SelfConnector
+from bookwyrm import models
+from bookwyrm.connectors import connector_manager
+from bookwyrm.connectors.bookwyrm_connector \
+    import Connector as BookWyrmConnector
+from bookwyrm.connectors.self_connector \
+    import Connector as SelfConnector
 
 
-class Book(TestCase):
+class ConnectorManager(TestCase):
+    ''' interface between the app and various connectors '''
     def setUp(self):
+        ''' we'll need some books and a connector info entry '''
         self.work = models.Work.objects.create(
             title='Example Work'
         )
@@ -28,53 +34,50 @@ class Book(TestCase):
             covers_url='http://test.com/',
         )
 
-    def test_get_edition(self):
-        edition = books_manager.get_edition(self.edition.id)
-        self.assertEqual(edition, self.edition)
-
-
-    def test_get_edition_work(self):
-        edition = books_manager.get_edition(self.work.id)
-        self.assertEqual(edition, self.edition)
-
 
     def test_get_or_create_connector(self):
+        ''' loads a connector if the data source is known or creates one '''
         remote_id = 'https://example.com/object/1'
-        connector = books_manager.get_or_create_connector(remote_id)
+        connector = connector_manager.get_or_create_connector(remote_id)
         self.assertIsInstance(connector, BookWyrmConnector)
         self.assertEqual(connector.identifier, 'example.com')
         self.assertEqual(connector.base_url, 'https://example.com')
 
-        same_connector = books_manager.get_or_create_connector(remote_id)
+        same_connector = connector_manager.get_or_create_connector(remote_id)
         self.assertEqual(connector.identifier, same_connector.identifier)
 
     def test_get_connectors(self):
+        ''' load all connectors '''
         remote_id = 'https://example.com/object/1'
-        books_manager.get_or_create_connector(remote_id)
-        connectors = list(books_manager.get_connectors())
+        connector_manager.get_or_create_connector(remote_id)
+        connectors = list(connector_manager.get_connectors())
         self.assertEqual(len(connectors), 2)
         self.assertIsInstance(connectors[0], SelfConnector)
         self.assertIsInstance(connectors[1], BookWyrmConnector)
 
     def test_search(self):
-        results = books_manager.search('Example')
+        ''' search all connectors '''
+        results = connector_manager.search('Example')
         self.assertEqual(len(results), 1)
         self.assertIsInstance(results[0]['connector'], SelfConnector)
         self.assertEqual(len(results[0]['results']), 1)
         self.assertEqual(results[0]['results'][0].title, 'Example Edition')
 
     def test_local_search(self):
-        results = books_manager.local_search('Example')
+        ''' search only the local database '''
+        results = connector_manager.local_search('Example')
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0].title, 'Example Edition')
 
     def test_first_search_result(self):
-        result = books_manager.first_search_result('Example')
+        ''' only get one search result '''
+        result = connector_manager.first_search_result('Example')
         self.assertEqual(result.title, 'Example Edition')
-        no_result = books_manager.first_search_result('dkjfhg')
+        no_result = connector_manager.first_search_result('dkjfhg')
         self.assertIsNone(no_result)
 
     def test_load_connector(self):
-        connector = books_manager.load_connector(self.connector)
+        ''' load a connector object from the database entry '''
+        connector = connector_manager.load_connector(self.connector)
         self.assertIsInstance(connector, SelfConnector)
         self.assertEqual(connector.identifier, 'test_connector')

bookwyrm/tests/connectors/test_openlibrary_connector.py

@@ -12,7 +12,7 @@ from bookwyrm.connectors.openlibrary import get_languages, get_description
 from bookwyrm.connectors.openlibrary import pick_default_edition, \
         get_openlibrary_key
 from bookwyrm.connectors.abstract_connector import SearchResult
-from bookwyrm.connectors.abstract_connector import ConnectorException
+from bookwyrm.connectors.connector_manager import ConnectorException
 
 
 class Openlibrary(TestCase):
@@ -104,7 +104,7 @@ class Openlibrary(TestCase):
         blob = ['image']
         result = self.connector.get_cover_url(blob)
         self.assertEqual(
-            result, 'https://covers.openlibrary.org/b/id/image-M.jpg')
+            result, 'https://covers.openlibrary.org/b/id/image-L.jpg')
 
     def test_parse_search_result(self):
         ''' extract the results from the search json response '''

bookwyrm/tests/connectors/test_self_connector.py

@@ -9,7 +9,9 @@ from bookwyrm.settings import DOMAIN
 
 
 class SelfConnector(TestCase):
+    ''' just uses local data '''
     def setUp(self):
+        ''' creating the connector '''
         models.Connector.objects.create(
             identifier=DOMAIN,
             name='Local',
@@ -22,58 +24,85 @@ class SelfConnector(TestCase):
             priority=1,
         )
         self.connector = Connector(DOMAIN)
-        self.work = models.Work.objects.create(
-            title='Example Work',
-        )
-        author = models.Author.objects.create(name='Anonymous')
-        self.edition = models.Edition.objects.create(
-            title='Edition of Example Work',
-            published_date=datetime.datetime(1980, 5, 10, tzinfo=timezone.utc),
-            parent_work=self.work,
-        )
-        self.edition.authors.add(author)
-        models.Edition.objects.create(
-            title='Another Edition',
-            parent_work=self.work,
-            series='Anonymous'
-        )
-        models.Edition.objects.create(
-            title='More Editions',
-            subtitle='The Anonymous Edition',
-            parent_work=self.work,
-        )
-
-        edition = models.Edition.objects.create(
-            title='An Edition',
-            parent_work=self.work
-        )
-        edition.authors.add(models.Author.objects.create(name='Fish'))
 
 
     def test_format_search_result(self):
+        ''' create a SearchResult '''
+        author = models.Author.objects.create(name='Anonymous')
+        edition = models.Edition.objects.create(
+            title='Edition of Example Work',
+            published_date=datetime.datetime(1980, 5, 10, tzinfo=timezone.utc),
+        )
+        edition.authors.add(author)
         result = self.connector.search('Edition of Example')[0]
         self.assertEqual(result.title, 'Edition of Example Work')
-        self.assertEqual(result.key, self.edition.remote_id)
+        self.assertEqual(result.key, edition.remote_id)
         self.assertEqual(result.author, 'Anonymous')
         self.assertEqual(result.year, 1980)
+        self.assertEqual(result.connector, self.connector)
 
 
     def test_search_rank(self):
+        ''' prioritize certain results '''
+        author = models.Author.objects.create(name='Anonymous')
+        edition = models.Edition.objects.create(
+            title='Edition of Example Work',
+            published_date=datetime.datetime(1980, 5, 10, tzinfo=timezone.utc),
+            parent_work=models.Work.objects.create(title='')
+        )
+        # author text is rank C
+        edition.authors.add(author)
+
+        # series is rank D
+        models.Edition.objects.create(
+            title='Another Edition',
+            series='Anonymous',
+            parent_work=models.Work.objects.create(title='')
+        )
+        # subtitle is rank B
+        models.Edition.objects.create(
+            title='More Editions',
+            subtitle='The Anonymous Edition',
+            parent_work=models.Work.objects.create(title='')
+        )
+        # title is rank A
+        models.Edition.objects.create(title='Anonymous')
+        # doesn't rank in this search
+        edition = models.Edition.objects.create(
+            title='An Edition',
+            parent_work=models.Work.objects.create(title='')
+        )
+
         results = self.connector.search('Anonymous')
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results[0].title, 'More Editions')
-        self.assertEqual(results[1].title, 'Edition of Example Work')
+        self.assertEqual(len(results), 3)
+        self.assertEqual(results[0].title, 'Anonymous')
+        self.assertEqual(results[1].title, 'More Editions')
+        self.assertEqual(results[2].title, 'Edition of Example Work')
 
 
-    def test_search_default_filter(self):
+    def test_search_multiple_editions(self):
         ''' it should get rid of duplicate editions for the same work '''
-        self.work.default_edition = self.edition
-        self.work.save()
+        work = models.Work.objects.create(title='Work Title')
+        edition_1 = models.Edition.objects.create(
+            title='Edition 1 Title', parent_work=work)
+        edition_2 = models.Edition.objects.create(
+            title='Edition 2 Title', parent_work=work)
+        edition_3 = models.Edition.objects.create(
+            title='Fish', parent_work=work)
+        work.default_edition = edition_2
+        work.save()
 
-        results = self.connector.search('Anonymous')
+        # pick the best edition
+        results = self.connector.search('Edition 1 Title')
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0].title, 'Edition of Example Work')
+        self.assertEqual(results[0].key, edition_1.remote_id)
 
+        # pick the default edition when no match is best
+        results = self.connector.search('Edition Title')
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].key, edition_2.remote_id)
+
+        # only matches one edition, so no deduplication takes place
         results = self.connector.search('Fish')
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0].title, 'An Edition')
+        self.assertEqual(results[0].key, edition_3.remote_id)

bookwyrm/tests/data/goodreads.csv

@@ -0,0 +1,4 @@
+Book Id,Title,Author,Author l-f,Additional Authors,ISBN,ISBN13,My Rating,Average Rating,Publisher,Binding,Number of Pages,Year Published,Original Publication Year,Date Read,Date Added,Bookshelves,Bookshelves with positions,Exclusive Shelf,My Review,Spoiler,Private Notes,Read Count,Recommended For,Recommended By,Owned Copies,Original Purchase Date,Original Purchase Location,Condition,Condition Description,BCID
+42036538,Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,"Muir, Tamsyn",,"=""1250313198""","=""9781250313195""",0,4.20,Tor,Hardcover,448,2019,2019,2020/10/25,2020/10/21,,,read,,,,1,,,0,,,,,
+52691223,Subcutanean,Aaron A. Reed,"Reed, Aaron A.",,"=""""","=""""",0,4.45,,Paperback,232,2020,,2020/03/06,2020/03/05,,,read,,,,1,,,0,,,,,
+28694510,Patisserie at Home,Mélanie Dupuis,"Dupuis, Mélanie",Anne Cazor,"=""0062445316""","=""9780062445315""",2,4.60,Harper Design,Hardcover,288,2016,,,2019/07/08,,,read,"mixed feelings",,,2,,,0,,,,,

bookwyrm/tests/models/test_base_model.py

@@ -22,7 +22,8 @@ class BaseModel(TestCase):
     def test_remote_id_with_user(self):
         ''' format of remote id when there's a user object '''
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse')
         instance = base_model.BookWyrmModel()
         instance.user = user
         instance.id = 1
@@ -51,7 +52,8 @@ class BaseModel(TestCase):
     def test_to_create_activity(self):
         ''' wrapper for ActivityPub "create" action '''
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse')
 
         object_activity = {
             'to': 'to field', 'cc': 'cc field',
@@ -81,7 +83,8 @@ class BaseModel(TestCase):
     def test_to_delete_activity(self):
         ''' wrapper for Delete activity '''
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse')
 
         MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
         mock_self = MockSelf(
@@ -105,7 +108,8 @@ class BaseModel(TestCase):
     def test_to_update_activity(self):
         ''' ditto above but for Update '''
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse')
 
         MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
         mock_self = MockSelf(
@@ -129,7 +133,8 @@ class BaseModel(TestCase):
     def test_to_undo_activity(self):
         ''' and again, for Undo '''
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse')
 
         MockSelf = namedtuple('Self', ('remote_id', 'to_activity'))
         mock_self = MockSelf(
@@ -173,7 +178,8 @@ class BaseModel(TestCase):
         book = models.Edition.objects.create(
             title='Test Edition', remote_id='http://book.com/book')
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         user.remote_id = 'http://example.com/a/b'
         user.save()
 

bookwyrm/tests/models/test_fields.py

@@ -111,10 +111,17 @@ class ActivitypubFields(TestCase):
         self.assertEqual(instance.max_length, 150)
         self.assertEqual(instance.unique, True)
         with self.assertRaises(ValidationError):
-            instance.run_validators('one two')
-            instance.run_validators('a*&')
-            instance.run_validators('trailingwhite ')
-        self.assertIsNone(instance.run_validators('aksdhf'))
+            instance.run_validators('mouse')
+            instance.run_validators('mouseexample.com')
+            instance.run_validators('mouse@example.c')
+            instance.run_validators('@example.com')
+            instance.run_validators('mouse@examplecom')
+            instance.run_validators('one two@fish.aaaa')
+            instance.run_validators('a*&@exampke.com')
+            instance.run_validators('trailingwhite@example.com ')
+        self.assertIsNone(instance.run_validators('mouse@example.com'))
+        self.assertIsNone(instance.run_validators('mo-2use@ex3ample.com'))
+        self.assertIsNone(instance.run_validators('aksdhf@sdkjf-df.cm'))
 
         self.assertEqual(instance.field_to_activity('test@example.com'), 'test')
 
@@ -173,7 +180,8 @@ class ActivitypubFields(TestCase):
     def test_privacy_field_set_activity_from_field(self):
         ''' translate between to/cc fields and privacy '''
         user = User.objects.create_user(
-            'rat', 'rat@rat.rat', 'ratword', local=True)
+            'rat', 'rat@rat.rat', 'ratword',
+            local=True, localname='rat')
         public = 'https://www.w3.org/ns/activitystreams#Public'
         followers = '%s/followers' % user.remote_id
 
@@ -230,7 +238,8 @@ class ActivitypubFields(TestCase):
 
         # it shouldn't match with this unrelated user:
         unrelated_user = User.objects.create_user(
-            'rat', 'rat@rat.rat', 'ratword', local=True)
+            'rat', 'rat@rat.rat', 'ratword',
+            local=True, localname='rat')
 
         # test receiving an unknown remote id and loading data
         responses.add(
@@ -258,7 +267,8 @@ class ActivitypubFields(TestCase):
 
         # it shouldn't match with this unrelated user:
         unrelated_user = User.objects.create_user(
-            'rat', 'rat@rat.rat', 'ratword', local=True)
+            'rat', 'rat@rat.rat', 'ratword',
+            local=True, localname='rat')
         with patch('bookwyrm.models.user.set_remote_server.delay'):
             value = instance.field_from_activity(userdata)
         self.assertIsInstance(value, User)
@@ -276,11 +286,13 @@ class ActivitypubFields(TestCase):
         )
         userdata = json.loads(datafile.read_bytes())
         user = User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         user.remote_id = 'https://example.com/user/mouse'
         user.save()
         User.objects.create_user(
-            'rat', 'rat@rat.rat', 'ratword', local=True)
+            'rat', 'rat@rat.rat', 'ratword',
+            local=True, localname='rat')
 
         value = instance.field_from_activity(userdata)
         self.assertEqual(value, user)
@@ -290,9 +302,11 @@ class ActivitypubFields(TestCase):
         ''' test receiving a remote id of an existing object in the db '''
         instance = fields.ForeignKey(User, on_delete=models.CASCADE)
         user = User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         User.objects.create_user(
-            'rat', 'rat@rat.rat', 'ratword', local=True)
+            'rat', 'rat@rat.rat', 'ratword',
+            local=True, localname='rat')
 
         value = instance.field_from_activity(user.remote_id)
         self.assertEqual(value, user)
@@ -382,7 +396,8 @@ class ActivitypubFields(TestCase):
     def test_image_field(self):
         ''' storing images '''
         user = User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         image_file = pathlib.Path(__file__).parent.joinpath(
             '../../static/images/default_avi.jpg')
         image = Image.open(image_file)

bookwyrm/tests/models/test_import_model.py

@@ -8,7 +8,8 @@ from django.utils import timezone
 from django.test import TestCase
 import responses
 
-from bookwyrm import books_manager, models
+from bookwyrm import models
+from bookwyrm.connectors import connector_manager
 from bookwyrm.connectors.abstract_connector import SearchResult
 
 
@@ -58,7 +59,8 @@ class ImportJob(TestCase):
         unknown_read_data['Date Read'] = ''
 
         user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         job = models.ImportJob.objects.create(user=user)
         self.item_1 = models.ImportItem.objects.create(
             job=job, index=1, data=currently_reading_data)
@@ -134,7 +136,7 @@ class ImportJob(TestCase):
             search_url='https://openlibrary.org/search?q=',
             priority=3,
         )
-        connector = books_manager.load_connector(connector_info)
+        connector = connector_manager.load_connector(connector_info)
         result = SearchResult(
             title='Test Result',
             key='https://openlibrary.org/works/OL1234W',
@@ -163,8 +165,12 @@ class ImportJob(TestCase):
             json={'name': 'test author'},
             status=200)
 
-        with patch('bookwyrm.books_manager.first_search_result') as search:
-            search.return_value = result
-            book = self.item_1.get_book_from_isbn()
+        with patch(
+                'bookwyrm.connectors.abstract_connector.load_more_data.delay'):
+            with patch(
+                    'bookwyrm.connectors.connector_manager.first_search_result'
+                ) as search:
+                search.return_value = result
+                book = self.item_1.get_book_from_isbn()
 
         self.assertEqual(book.title, 'Sabriel')

bookwyrm/tests/models/test_relationship_models.py

@@ -16,7 +16,8 @@ class Relationship(TestCase):
                 outbox='https://example.com/users/rat/outbox',
             )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse')
         self.local_user.remote_id = 'http://local.com/user/mouse'
         self.local_user.save()
 

bookwyrm/tests/models/test_shelf_model.py

@@ -9,7 +9,8 @@ class Shelf(TestCase):
     def setUp(self):
         ''' look, a shelf '''
         self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         self.shelf = models.Shelf.objects.create(
             name='Test Shelf', identifier='test-shelf', user=self.user)
 

bookwyrm/tests/models/test_status_model.py

@@ -16,7 +16,8 @@ class Status(TestCase):
     def setUp(self):
         ''' useful things for creating a status '''
         self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         self.book = models.Edition.objects.create(title='Test Edition')
 
         image_file = pathlib.Path(__file__).parent.joinpath(

bookwyrm/tests/models/test_user_model.py

@@ -9,7 +9,8 @@ from bookwyrm.settings import DOMAIN
 class User(TestCase):
     def setUp(self):
         self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse@%s' % DOMAIN, 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
 
     def test_computed_fields(self):
         ''' username instead of id here '''

bookwyrm/tests/test_broadcast.py

@@ -7,10 +7,12 @@ from bookwyrm import models, broadcast
 class Book(TestCase):
     def setUp(self):
         self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
 
         local_follower = models.User.objects.create_user(
-            'joe', 'joe@mouse.mouse', 'jeoword', local=True)
+            'joe', 'joe@mouse.mouse', 'jeoword',
+            local=True, localname='joe')
         self.user.followers.add(local_follower)
 
         with patch('bookwyrm.models.user.set_remote_server.delay'):

bookwyrm/tests/test_goodreads_import.py

@@ -0,0 +1,104 @@
+''' testing import '''
+from collections import namedtuple
+import pathlib
+from unittest.mock import patch
+
+from django.test import TestCase
+import responses
+
+from bookwyrm import goodreads_import, models
+from bookwyrm.settings import DOMAIN
+
+class GoodreadsImport(TestCase):
+    ''' importing from goodreads csv '''
+    def setUp(self):
+        ''' use a test csv '''
+        datafile = pathlib.Path(__file__).parent.joinpath(
+            'data/goodreads.csv')
+        self.csv = open(datafile, 'r')
+        self.user = models.User.objects.create_user(
+            'mouse', 'mouse@mouse.mouse', 'password', local=True)
+
+        models.Connector.objects.create(
+            identifier=DOMAIN,
+            name='Local',
+            local=True,
+            connector_file='self_connector',
+            base_url='https://%s' % DOMAIN,
+            books_url='https://%s/book' % DOMAIN,
+            covers_url='https://%s/images/covers' % DOMAIN,
+            search_url='https://%s/search?q=' % DOMAIN,
+            priority=1,
+        )
+
+
+    def test_create_job(self):
+        ''' creates the import job entry and checks csv '''
+        import_job = goodreads_import.create_job(
+            self.user, self.csv, False, 'public')
+        self.assertEqual(import_job.user, self.user)
+        self.assertEqual(import_job.include_reviews, False)
+        self.assertEqual(import_job.privacy, 'public')
+
+        import_items = models.ImportItem.objects.filter(job=import_job).all()
+        self.assertEqual(len(import_items), 3)
+        self.assertEqual(import_items[0].index, 0)
+        self.assertEqual(import_items[0].data['Book Id'], '42036538')
+        self.assertEqual(import_items[1].index, 1)
+        self.assertEqual(import_items[1].data['Book Id'], '52691223')
+        self.assertEqual(import_items[2].index, 2)
+        self.assertEqual(import_items[2].data['Book Id'], '28694510')
+
+
+    def test_create_retry_job(self):
+        ''' trying again with items that didn't import '''
+        import_job = goodreads_import.create_job(
+            self.user, self.csv, False, 'unlisted')
+        import_items = models.ImportItem.objects.filter(
+            job=import_job
+            ).all()[:2]
+
+        retry = goodreads_import.create_retry_job(
+            self.user, import_job, import_items)
+        self.assertNotEqual(import_job, retry)
+        self.assertEqual(retry.user, self.user)
+        self.assertEqual(retry.include_reviews, False)
+        self.assertEqual(retry.privacy, 'unlisted')
+
+        retry_items = models.ImportItem.objects.filter(job=retry).all()
+        self.assertEqual(len(retry_items), 2)
+        self.assertEqual(retry_items[0].index, 0)
+        self.assertEqual(retry_items[0].data['Book Id'], '42036538')
+        self.assertEqual(retry_items[1].index, 1)
+        self.assertEqual(retry_items[1].data['Book Id'], '52691223')
+
+
+    def test_start_import(self):
+        ''' begin loading books '''
+        import_job = goodreads_import.create_job(
+            self.user, self.csv, False, 'unlisted')
+        MockTask = namedtuple('Task', ('id'))
+        mock_task = MockTask(7)
+        with patch('bookwyrm.goodreads_import.import_data.delay') as start:
+            start.return_value = mock_task
+            goodreads_import.start_import(import_job)
+        import_job.refresh_from_db()
+        self.assertEqual(import_job.task_id, '7')
+
+
+    @responses.activate
+    def test_import_data(self):
+        ''' resolve entry '''
+        import_job = goodreads_import.create_job(
+            self.user, self.csv, False, 'unlisted')
+        book = models.Edition.objects.create(title='Test Book')
+
+        with patch(
+                'bookwyrm.models.import_job.ImportItem.get_book_from_isbn'
+                ) as resolve:
+            resolve.return_value = book
+            with patch('bookwyrm.outgoing.handle_imported_book'):
+                goodreads_import.import_data(import_job.id)
+
+        import_item = models.ImportItem.objects.get(job=import_job, index=0)
+        self.assertEqual(import_item.book.id, book.id)

bookwyrm/tests/test_incoming.py

@@ -19,7 +19,8 @@ class Incoming(TestCase):
     def setUp(self):
         ''' we need basic things, like users '''
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+            'mouse@example.com', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse')
         self.local_user.remote_id = 'https://example.com/user/mouse'
         self.local_user.save()
         with patch('bookwyrm.models.user.set_remote_server.delay'):
@@ -486,6 +487,10 @@ class Incoming(TestCase):
 
     def test_handle_update_user(self):
         ''' update an existing user '''
+        # we only do this with remote users
+        self.local_user.local = False
+        self.local_user.save()
+
         datafile = pathlib.Path(__file__).parent.joinpath(
             'data/ap_user.json')
         userdata = json.loads(datafile.read_bytes())
@@ -494,6 +499,8 @@ class Incoming(TestCase):
         incoming.handle_update_user({'object': userdata})
         user = models.User.objects.get(id=self.local_user.id)
         self.assertEqual(user.name, 'MOUSE?? MOUSE!!')
+        self.assertEqual(user.username, 'mouse@example.com')
+        self.assertEqual(user.localname, 'mouse')
 
 
     def test_handle_update_edition(self):

bookwyrm/tests/test_outgoing.py

@@ -1,4 +1,5 @@
 ''' sending out activities '''
+import csv
 import json
 import pathlib
 from unittest.mock import patch
@@ -8,10 +9,11 @@ from django.test import TestCase
 from django.test.client import RequestFactory
 import responses
 
-from bookwyrm import models, outgoing
+from bookwyrm import forms, models, outgoing
 from bookwyrm.settings import DOMAIN
 
 
+# pylint: disable=too-many-public-methods
 class Outgoing(TestCase):
     ''' sends out activities '''
     def setUp(self):
@@ -19,15 +21,16 @@ class Outgoing(TestCase):
         self.factory = RequestFactory()
         with patch('bookwyrm.models.user.set_remote_server'):
             self.remote_user = models.User.objects.create_user(
-                'rat', 'rat@rat.com', 'ratword',
+                'rat', 'rat@email.com', 'ratword',
                 local=False,
                 remote_id='https://example.com/users/rat',
                 inbox='https://example.com/users/rat/inbox',
                 outbox='https://example.com/users/rat/outbox',
             )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True,
-            localname='mouse', remote_id='https://example.com/users/mouse',
+            'mouse@local.com', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse',
+            remote_id='https://example.com/users/mouse',
         )
 
         datafile = pathlib.Path(__file__).parent.joinpath(
@@ -173,10 +176,10 @@ class Outgoing(TestCase):
 
     def test_existing_user(self):
         ''' simple database lookup by username '''
-        result = outgoing.handle_remote_webfinger('@mouse@%s' % DOMAIN)
+        result = outgoing.handle_remote_webfinger('@mouse@local.com')
         self.assertEqual(result, self.local_user)
 
-        result = outgoing.handle_remote_webfinger('mouse@%s' % DOMAIN)
+        result = outgoing.handle_remote_webfinger('mouse@local.com')
         self.assertEqual(result, self.local_user)
 
 
@@ -255,3 +258,192 @@ class Outgoing(TestCase):
         with patch('bookwyrm.broadcast.broadcast_task.delay'):
             outgoing.handle_unshelve(self.local_user, self.book, self.shelf)
         self.assertEqual(self.shelf.books.count(), 0)
+
+
+    def test_handle_imported_book(self):
+        ''' goodreads import added a book, this adds related connections '''
+        shelf = self.local_user.shelf_set.filter(identifier='read').first()
+        self.assertIsNone(shelf.books.first())
+
+        import_job = models.ImportJob.objects.create(user=self.local_user)
+        datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
+        csv_file = open(datafile, 'r')
+        for index, entry in enumerate(list(csv.DictReader(csv_file))):
+            import_item = models.ImportItem.objects.create(
+                job_id=import_job.id, index=index, data=entry, book=self.book)
+            break
+
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_imported_book(
+                self.local_user, import_item, False, 'public')
+
+        shelf.refresh_from_db()
+        self.assertEqual(shelf.books.first(), self.book)
+
+        readthrough = models.ReadThrough.objects.get(user=self.local_user)
+        self.assertEqual(readthrough.book, self.book)
+        # I can't remember how to create dates and I don't want to look it up.
+        self.assertEqual(readthrough.start_date.year, 2020)
+        self.assertEqual(readthrough.start_date.month, 10)
+        self.assertEqual(readthrough.start_date.day, 21)
+        self.assertEqual(readthrough.finish_date.year, 2020)
+        self.assertEqual(readthrough.finish_date.month, 10)
+        self.assertEqual(readthrough.finish_date.day, 25)
+
+
+    def test_handle_imported_book_already_shelved(self):
+        ''' goodreads import added a book, this adds related connections '''
+        shelf = self.local_user.shelf_set.filter(identifier='to-read').first()
+        models.ShelfBook.objects.create(
+            shelf=shelf, added_by=self.local_user, book=self.book)
+
+        import_job = models.ImportJob.objects.create(user=self.local_user)
+        datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
+        csv_file = open(datafile, 'r')
+        for index, entry in enumerate(list(csv.DictReader(csv_file))):
+            import_item = models.ImportItem.objects.create(
+                job_id=import_job.id, index=index, data=entry, book=self.book)
+            break
+
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_imported_book(
+                self.local_user, import_item, False, 'public')
+
+        shelf.refresh_from_db()
+        self.assertEqual(shelf.books.first(), self.book)
+        self.assertIsNone(
+            self.local_user.shelf_set.get(identifier='read').books.first())
+        readthrough = models.ReadThrough.objects.get(user=self.local_user)
+        self.assertEqual(readthrough.book, self.book)
+        self.assertEqual(readthrough.start_date.year, 2020)
+        self.assertEqual(readthrough.start_date.month, 10)
+        self.assertEqual(readthrough.start_date.day, 21)
+        self.assertEqual(readthrough.finish_date.year, 2020)
+        self.assertEqual(readthrough.finish_date.month, 10)
+        self.assertEqual(readthrough.finish_date.day, 25)
+
+
+    def test_handle_imported_book_review(self):
+        ''' goodreads review import '''
+        import_job = models.ImportJob.objects.create(user=self.local_user)
+        datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
+        csv_file = open(datafile, 'r')
+        entry = list(csv.DictReader(csv_file))[2]
+        import_item = models.ImportItem.objects.create(
+            job_id=import_job.id, index=0, data=entry, book=self.book)
+
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_imported_book(
+                self.local_user, import_item, True, 'unlisted')
+        review = models.Review.objects.get(book=self.book, user=self.local_user)
+        self.assertEqual(review.content, 'mixed feelings')
+        self.assertEqual(review.rating, 2)
+        self.assertEqual(review.published_date.year, 2019)
+        self.assertEqual(review.published_date.month, 7)
+        self.assertEqual(review.published_date.day, 8)
+        self.assertEqual(review.privacy, 'unlisted')
+
+
+    def test_handle_imported_book_reviews_disabled(self):
+        ''' goodreads review import '''
+        import_job = models.ImportJob.objects.create(user=self.local_user)
+        datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
+        csv_file = open(datafile, 'r')
+        entry = list(csv.DictReader(csv_file))[2]
+        import_item = models.ImportItem.objects.create(
+            job_id=import_job.id, index=0, data=entry, book=self.book)
+
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_imported_book(
+                self.local_user, import_item, False, 'unlisted')
+        self.assertFalse(models.Review.objects.filter(
+            book=self.book, user=self.local_user
+        ).exists())
+
+
+    def test_handle_status(self):
+        ''' create a status '''
+        form = forms.CommentForm({
+            'content': 'hi',
+            'user': self.local_user.id,
+            'book': self.book.id,
+            'privacy': 'public',
+        })
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_status(self.local_user, form)
+        status = models.Comment.objects.get()
+        self.assertEqual(status.content, '<p>hi</p>')
+        self.assertEqual(status.user, self.local_user)
+        self.assertEqual(status.book, self.book)
+
+    def test_handle_status_reply(self):
+        ''' create a status in reply to an existing status '''
+        user = models.User.objects.create_user(
+            'rat', 'rat@rat.com', 'password', local=True)
+        parent = models.Status.objects.create(
+            content='parent status', user=self.local_user)
+        form = forms.ReplyForm({
+            'content': 'hi',
+            'user': user.id,
+            'reply_parent': parent.id,
+            'privacy': 'public',
+        })
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_status(user, form)
+        status = models.Status.objects.get(user=user)
+        self.assertEqual(status.content, '<p>hi</p>')
+        self.assertEqual(status.user, user)
+        self.assertEqual(
+            models.Notification.objects.get().user, self.local_user)
+
+    def test_handle_status_mentions(self):
+        ''' @mention a user in a post '''
+        user = models.User.objects.create_user(
+            'rat@%s' % DOMAIN, 'rat@rat.com', 'password',
+            local=True, localname='rat')
+        form = forms.CommentForm({
+            'content': 'hi @rat',
+            'user': self.local_user.id,
+            'book': self.book.id,
+            'privacy': 'public',
+        })
+
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_status(self.local_user, form)
+        status = models.Status.objects.get()
+        self.assertEqual(list(status.mention_users.all()), [user])
+        self.assertEqual(models.Notification.objects.get().user, user)
+        self.assertEqual(
+            status.content,
+            '<p>hi <a href="%s">@rat</a></p>' % user.remote_id)
+
+    def test_handle_status_reply_with_mentions(self):
+        ''' reply to a post with an @mention'ed user '''
+        user = models.User.objects.create_user(
+            'rat', 'rat@rat.com', 'password',
+            local=True, localname='rat')
+        form = forms.CommentForm({
+            'content': 'hi @rat@example.com',
+            'user': self.local_user.id,
+            'book': self.book.id,
+            'privacy': 'public',
+        })
+
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_status(self.local_user, form)
+        status = models.Status.objects.get()
+
+        form = forms.ReplyForm({
+            'content': 'right',
+            'user': user,
+            'privacy': 'public',
+            'reply_parent': status.id
+        })
+        with patch('bookwyrm.broadcast.broadcast_task.delay'):
+            outgoing.handle_status(user, form)
+
+        reply = models.Status.replies(status).first()
+        self.assertEqual(reply.content, '<p>right</p>')
+        self.assertEqual(reply.user, user)
+        self.assertTrue(self.remote_user in reply.mention_users.all())
+        self.assertTrue(self.local_user in reply.mention_users.all())

bookwyrm/tests/test_signing.py

@@ -31,11 +31,14 @@ Sender = namedtuple('Sender', ('remote_id', 'key_pair'))
 class Signature(TestCase):
     def setUp(self):
         self.mouse = User.objects.create_user(
-            'mouse', 'mouse@example.com', '', local=True)
+            'mouse@%s' % DOMAIN, 'mouse@example.com', '',
+            local=True, localname='mouse')
         self.rat = User.objects.create_user(
-            'rat', 'rat@example.com', '', local=True)
+            'rat@%s' % DOMAIN, 'rat@example.com', '',
+            local=True, localname='rat')
         self.cat = User.objects.create_user(
-            'cat', 'cat@example.com', '', local=True)
+            'cat@%s' % DOMAIN, 'cat@example.com', '',
+            local=True, localname='cat')
 
         private_key, public_key = create_key_pair()
 

bookwyrm/tests/test_templatetags.py

@@ -16,7 +16,8 @@ class TemplateTags(TestCase):
     def setUp(self):
         ''' create some filler objects '''
         self.user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'mouseword', local=True)
+            'mouse@example.com', 'mouse@mouse.mouse', 'mouseword',
+            local=True, localname='mouse')
         with patch('bookwyrm.models.user.set_remote_server.delay'):
             self.remote_user = models.User.objects.create_user(
                 'rat', 'rat@rat.rat', 'ratword',

bookwyrm/tests/test_view_actions.py

@@ -18,7 +18,8 @@ class ViewActions(TestCase):
     def setUp(self):
         ''' we need basic things, like users '''
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.com', 'mouseword', local=True)
+            'mouse', 'mouse@mouse.com', 'mouseword',
+            local=True, localname='mouse')
         self.local_user.remote_id = 'https://example.com/user/mouse'
         self.local_user.save()
         self.group = Group.objects.create(name='editor')
@@ -54,7 +55,7 @@ class ViewActions(TestCase):
         request = self.factory.post(
             'register/',
             {
-                'username': 'nutria-user.user_nutria',
+                'localname': 'nutria-user.user_nutria',
                 'password': 'mouseword',
                 'email': 'aa@bb.cccc'
             })
@@ -72,7 +73,7 @@ class ViewActions(TestCase):
         request = self.factory.post(
             'register/',
             {
-                'username': 'nutria ',
+                'localname': 'nutria ',
                 'password': 'mouseword',
                 'email': 'aa@bb.ccc'
             })
@@ -91,7 +92,7 @@ class ViewActions(TestCase):
         request = self.factory.post(
             'register/',
             {
-                'username': 'nutria',
+                'localname': 'nutria',
                 'password': 'mouseword',
                 'email': 'aa'
             })
@@ -105,7 +106,7 @@ class ViewActions(TestCase):
         request = self.factory.post(
             'register/',
             {
-                'username': 'nut@ria',
+                'localname': 'nut@ria',
                 'password': 'mouseword',
                 'email': 'aa@bb.ccc'
             })
@@ -116,7 +117,7 @@ class ViewActions(TestCase):
         request = self.factory.post(
             'register/',
             {
-                'username': 'nutr ia',
+                'localname': 'nutr ia',
                 'password': 'mouseword',
                 'email': 'aa@bb.ccc'
             })
@@ -127,7 +128,7 @@ class ViewActions(TestCase):
         request = self.factory.post(
             'register/',
             {
-                'username': 'nut@ria',
+                'localname': 'nut@ria',
                 'password': 'mouseword',
                 'email': 'aa@bb.ccc'
             })
@@ -143,7 +144,7 @@ class ViewActions(TestCase):
         request = self.factory.post(
             'register/',
             {
-                'username': 'nutria ',
+                'localname': 'nutria ',
                 'password': 'mouseword',
                 'email': 'aa@bb.ccc'
             })
@@ -161,7 +162,7 @@ class ViewActions(TestCase):
         request = self.factory.post(
             'register/',
             {
-                'username': 'nutria',
+                'localname': 'nutria',
                 'password': 'mouseword',
                 'email': 'aa@bb.ccc',
                 'invite_code': 'testcode'
@@ -172,23 +173,24 @@ class ViewActions(TestCase):
         self.assertEqual(response.status_code, 302)
         self.assertEqual(models.SiteInvite.objects.get().times_used, 1)
 
-        # invalid invite
+        # invite already used to max capacity
         request = self.factory.post(
             'register/',
             {
-                'username': 'nutria2',
+                'localname': 'nutria2',
                 'password': 'mouseword',
                 'email': 'aa@bb.ccc',
                 'invite_code': 'testcode'
             })
-        response = actions.register(request)
+        with self.assertRaises(PermissionDenied):
+            response = actions.register(request)
         self.assertEqual(models.User.objects.count(), 3)
 
         # bad invite code
         request = self.factory.post(
             'register/',
             {
-                'username': 'nutria3',
+                'localname': 'nutria3',
                 'password': 'mouseword',
                 'email': 'aa@bb.ccc',
                 'invite_code': 'dkfkdjgdfkjgkdfj'
@@ -379,7 +381,7 @@ class ViewActions(TestCase):
     def test_untag(self):
         ''' remove a tag from a book '''
         tag = models.Tag.objects.create(name='A Tag!?')
-        user_tag = models.UserTag.objects.create(
+        models.UserTag.objects.create(
             user=self.local_user, book=self.book, tag=tag)
         request = self.factory.post(
             '', {

bookwyrm/tests/test_views.py

@@ -9,8 +9,9 @@ from django.test import TestCase
 from django.test.client import RequestFactory
 
 from bookwyrm import models, views
+from bookwyrm.activitypub import ActivitypubResponse
 from bookwyrm.connectors import abstract_connector
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import DOMAIN, USER_AGENT
 
 
 # pylint: disable=too-many-public-methods
@@ -28,7 +29,8 @@ class Views(TestCase):
             local=True
         )
         self.local_user = models.User.objects.create_user(
-            'mouse', 'mouse@mouse.mouse', 'password', local=True)
+            'mouse@local.com', 'mouse@mouse.mouse', 'password',
+            local=True, localname='mouse')
         with patch('bookwyrm.models.user.set_remote_server.delay'):
             self.remote_user = models.User.objects.create_user(
                 'rat', 'rat@rat.com', 'ratword',
@@ -39,12 +41,20 @@ class Views(TestCase):
             )
 
 
+    def test_get_edition(self):
+        ''' given an edition or a work, returns an edition '''
+        self.assertEqual(
+            views.get_edition(self.book.id), self.book)
+        self.assertEqual(
+            views.get_edition(self.work.id), self.book)
+
+
     def test_get_user_from_username(self):
         ''' works for either localname or username '''
         self.assertEqual(
             views.get_user_from_username('mouse'), self.local_user)
         self.assertEqual(
-            views.get_user_from_username('mouse@%s' % DOMAIN), self.local_user)
+            views.get_user_from_username('mouse@local.com'), self.local_user)
         with self.assertRaises(models.User.DoesNotExist):
             views.get_user_from_username('mojfse@example.com')
 
@@ -193,7 +203,8 @@ class Views(TestCase):
         request = self.factory.get('', {'q': 'Test Book'})
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = False
-            with patch('bookwyrm.books_manager.search') as manager:
+            with patch(
+                    'bookwyrm.connectors.connector_manager.search') as manager:
                 manager.return_value = [search_result]
                 response = views.search(request)
         self.assertIsInstance(response, TemplateResponse)
@@ -202,6 +213,19 @@ class Views(TestCase):
             response.context_data['book_results'][0].title, 'Gideon the Ninth')
 
 
+    def test_search_html_response_users(self):
+        ''' searches remote connectors '''
+        request = self.factory.get('', {'q': 'mouse'})
+        with patch('bookwyrm.views.is_api_request') as is_api:
+            is_api.return_value = False
+            with patch('bookwyrm.connectors.connector_manager.search'):
+                response = views.search(request)
+        self.assertIsInstance(response, TemplateResponse)
+        self.assertEqual(response.template_name, 'search_results.html')
+        self.assertEqual(
+            response.context_data['user_results'][0], self.local_user)
+
+
     def test_import_page(self):
         ''' there are so many views, this just makes sure it LOADS '''
         request = self.factory.get('')
@@ -321,7 +345,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.user_page(request, 'mouse')
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -339,7 +363,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.followers_page(request, 'mouse')
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -357,7 +381,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.following_page(request, 'mouse')
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -377,7 +401,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.status_page(request, 'mouse', status.id)
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -397,7 +421,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.replies_page(request, 'mouse', status.id)
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -426,7 +450,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.book_page(request, self.book.id)
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -467,7 +491,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.editions_page(request, self.work.id)
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -486,7 +510,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.author_page(request, author.id)
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -507,7 +531,7 @@ class Views(TestCase):
         with patch('bookwyrm.views.is_api_request') as is_api:
             is_api.return_value = True
             result = views.tag_page(request, tag.identifier)
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
 
 
@@ -528,5 +552,22 @@ class Views(TestCase):
             is_api.return_value = True
             result = views.shelf_page(
                 request, self.local_user.username, shelf.identifier)
-        self.assertIsInstance(result, JsonResponse)
+        self.assertIsInstance(result, ActivitypubResponse)
         self.assertEqual(result.status_code, 200)
+
+
+    def test_is_bookwyrm_request(self):
+        ''' checks if a request came from a bookwyrm instance '''
+        request = self.factory.get('', {'q': 'Test Book'})
+        self.assertFalse(views.is_bookworm_request(request))
+
+        request = self.factory.get(
+            '', {'q': 'Test Book'},
+            HTTP_USER_AGENT=\
+                "http.rb/4.4.1 (Mastodon/3.3.0; +https://mastodon.social/)"
+        )
+        self.assertFalse(views.is_bookworm_request(request))
+
+        request = self.factory.get(
+            '', {'q': 'Test Book'}, HTTP_USER_AGENT=USER_AGENT)
+        self.assertTrue(views.is_bookworm_request(request))

bookwyrm/urls.py

@@ -52,6 +52,7 @@ urlpatterns = [
 
     path('', views.home),
     re_path(r'^(?P<tab>home|local|federated)/?$', views.home_tab),
+    re_path(r'^discover/?$', views.discover_page),
     re_path(r'^notifications/?$', views.notifications_page),
     re_path(r'^direct-messages/?$', views.direct_messages_page),
     re_path(r'^import/?$', views.import_page),

bookwyrm/utils/regex.py

@@ -6,3 +6,5 @@ strict_localname = r'@[a-zA-Z_\-\.0-9]+'
 username = r'%s(@%s)?' % (localname, domain)
 strict_username = r'%s(@%s)?' % (strict_localname, domain)
 full_username = r'%s@%s' % (localname, domain)
+# should match (BookWyrm/1.0.0; or (BookWyrm/99.1.2;
+bookwyrm_user_agent = r'\(BookWyrm/[0-9]+\.[0-9]+\.[0-9]+;'

bookwyrm/view_actions.py

@@ -17,11 +17,12 @@ from django.template.response import TemplateResponse
 from django.utils import timezone
 from django.views.decorators.http import require_GET, require_POST
 
-from bookwyrm import books_manager, forms, models, outgoing, goodreads_import
+from bookwyrm import forms, models, outgoing, goodreads_import
+from bookwyrm.connectors import connector_manager
 from bookwyrm.broadcast import broadcast
 from bookwyrm.emailing import password_reset_email
 from bookwyrm.settings import DOMAIN
-from bookwyrm.views import get_user_from_username
+from bookwyrm.views import get_user_from_username, get_edition
 
 
 @require_POST
@@ -29,8 +30,8 @@ def user_login(request):
     ''' authenticate user login '''
     login_form = forms.LoginForm(request.POST)
 
-    username = login_form.data['username']
-    username = '%s@%s' % (username, DOMAIN)
+    localname = login_form.data['localname']
+    username = '%s@%s' % (localname, DOMAIN)
     password = login_form.data['password']
     user = authenticate(request, username=username, password=password)
     if user is not None:
@@ -58,6 +59,8 @@ def register(request):
             raise PermissionDenied
 
         invite = get_object_or_404(models.SiteInvite, code=invite_code)
+        if not invite.valid():
+            raise PermissionDenied
     else:
         invite = None
 
@@ -66,13 +69,13 @@ def register(request):
     if not form.is_valid():
         errors = True
 
-    username = form.data['username'].strip()
+    localname = form.data['localname'].strip()
     email = form.data['email']
     password = form.data['password']
 
-    # check username and email uniqueness
-    if models.User.objects.filter(localname=username).first():
-        form.add_error('username', 'User with this username already exists')
+    # check localname and email uniqueness
+    if models.User.objects.filter(localname=localname).first():
+        form.errors['localname'] = ['User with this username already exists']
         errors = True
 
     if errors:
@@ -82,8 +85,9 @@ def register(request):
         }
         return TemplateResponse(request, 'login.html', data)
 
+    username = '%s@%s' % (localname, DOMAIN)
     user = models.User.objects.create_user(
-        username, email, password, local=True)
+        username, email, password, localname=localname, local=True)
     if invite:
         invite.times_used += 1
         invite.save()
@@ -210,10 +214,8 @@ def edit_profile(request):
 def resolve_book(request):
     ''' figure out the local path to a book from a remote_id '''
     remote_id = request.POST.get('remote_id')
-    connector = books_manager.get_or_create_connector(remote_id)
+    connector = connector_manager.get_or_create_connector(remote_id)
     book = connector.get_or_create_book(remote_id)
-    if book.connector:
-        books_manager.load_more_data.delay(book.id)
 
     return redirect('/book/%d' % book.id)
 
@@ -371,7 +373,7 @@ def delete_shelf(request, shelf_id):
 @require_POST
 def shelve(request):
     ''' put a  on a user's shelf '''
-    book = books_manager.get_edition(request.POST['book'])
+    book = get_edition(request.POST['book'])
 
     desired_shelf = models.Shelf.objects.filter(
         identifier=request.POST['shelf'],
@@ -417,7 +419,7 @@ def unshelve(request):
 @require_POST
 def start_reading(request, book_id):
     ''' begin reading a book '''
-    book = books_manager.get_edition(book_id)
+    book = get_edition(book_id)
     shelf = models.Shelf.objects.filter(
         identifier='reading',
         user=request.user
@@ -453,7 +455,7 @@ def start_reading(request, book_id):
 @require_POST
 def finish_reading(request, book_id):
     ''' a user completed a book, yay '''
-    book = books_manager.get_edition(book_id)
+    book = get_edition(book_id)
     shelf = models.Shelf.objects.filter(
         identifier='read',
         user=request.user

bookwyrm/views.py

@@ -4,7 +4,8 @@ import re
 from django.contrib.auth.decorators import login_required, permission_required
 from django.contrib.postgres.search import TrigramSimilarity
 from django.core.paginator import Paginator
-from django.db.models import Avg, Q
+from django.db.models import Avg, Q, Max
+from django.db.models.functions import Greatest
 from django.http import HttpResponseNotFound, JsonResponse
 from django.core.exceptions import PermissionDenied
 from django.shortcuts import get_object_or_404, redirect
@@ -13,14 +14,21 @@ from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_GET
 
 from bookwyrm import outgoing
+from bookwyrm import forms, models
 from bookwyrm.activitypub import ActivitypubResponse
-from bookwyrm import forms, models, books_manager
-from bookwyrm import goodreads_import
+from bookwyrm.connectors import connector_manager
 from bookwyrm.settings import PAGE_LENGTH
 from bookwyrm.tasks import app
 from bookwyrm.utils import regex
 
 
+def get_edition(book_id):
+    ''' look up a book in the db and return an edition '''
+    book = models.Book.objects.select_subclasses().get(id=book_id)
+    if isinstance(book, models.Work):
+        book = book.get_default_edition()
+    return book
+
 def get_user_from_username(username):
     ''' helper function to resolve a localname or a username to a user '''
     # raises DoesNotExist if user is now found
@@ -35,6 +43,14 @@ def is_api_request(request):
     return 'json' in request.headers.get('Accept') or \
             request.path[-5:] == '.json'
 
+def is_bookworm_request(request):
+    ''' check if the request is coming from another bookworm instance '''
+    user_agent = request.headers.get('User-Agent')
+    if user_agent is None or \
+            re.search(regex.bookwyrm_user_agent, user_agent) is None:
+        return False
+
+    return True
 
 def server_error_page(request):
     ''' 500 errors '''
@@ -48,11 +64,12 @@ def not_found_page(request, _):
         request, 'notfound.html', {'title': 'Not found'}, status=404)
 
 
-@login_required
 @require_GET
 def home(request):
     ''' this is the same as the feed on the home tab '''
-    return home_tab(request, 'home')
+    if request.user.is_authenticated:
+        return home_tab(request, 'home')
+    return discover_page(request)
 
 
 @login_required
@@ -115,6 +132,36 @@ def get_suggested_books(user, max_books=5):
     return suggested_books
 
 
+@require_GET
+def discover_page(request):
+    ''' tiled book activity page '''
+    books = models.Edition.objects.filter(
+        review__published_date__isnull=False,
+        review__user__local=True,
+        review__privacy__in=['public', 'unlisted'],
+    ).exclude(
+        cover__exact=''
+    ).annotate(
+        Max('review__published_date')
+    ).order_by('-review__published_date__max')[:6]
+
+    ratings = {}
+    for book in books:
+        reviews = models.Review.objects.filter(
+            book__in=book.parent_work.editions.all()
+        )
+        reviews = get_activity_feed(
+            request.user, 'federated', model=reviews)
+        ratings[book.id] = reviews.aggregate(Avg('rating'))['rating__avg']
+    data = {
+        'title': 'Discover',
+        'register_form': forms.RegisterForm(),
+        'books': list(set(books)),
+        'ratings': ratings
+    }
+    return TemplateResponse(request, 'discover.html', data)
+
+
 @login_required
 @require_GET
 def direct_messages_page(request, page=1):
@@ -166,7 +213,7 @@ def get_activity_feed(user, filter_level, model=models.Status):
         return activities.filter(
             Q(user=user) | Q(mention_users=user),
             privacy='direct'
-        )
+        ).distinct()
 
     # never show DMs in the regular feed
     activities = activities.filter(~Q(privacy='direct'))
@@ -181,7 +228,7 @@ def get_activity_feed(user, filter_level, model=models.Status):
             Q(user__in=following, privacy__in=[
                 'public', 'unlisted', 'followers'
             ]) | Q(mention_users=user) | Q(user=user)
-        )
+        ).distinct()
     elif filter_level == 'self':
         activities = activities.filter(user=user, privacy='public')
     elif filter_level == 'local':
@@ -211,7 +258,7 @@ def search(request):
 
     if is_api_request(request):
         # only return local book results via json so we don't cause a cascade
-        book_results = books_manager.local_search(query)
+        book_results = connector_manager.local_search(query)
         return JsonResponse([r.json() for r in book_results], safe=False)
 
     # use webfinger for mastodon style account@domain.com username
@@ -220,12 +267,15 @@ def search(request):
 
     # do a local user search
     user_results = models.User.objects.annotate(
-        similarity=TrigramSimilarity('username', query),
+        similarity=Greatest(
+            TrigramSimilarity('username', query),
+            TrigramSimilarity('localname', query),
+        )
     ).filter(
         similarity__gt=0.5,
     ).order_by('-similarity')[:10]
 
-    book_results = books_manager.search(query)
+    book_results = connector_manager.search(query)
     data = {
         'title': 'Search Results',
         'book_results': book_results,
@@ -244,7 +294,6 @@ def import_page(request):
         'import_form': forms.ImportForm(),
         'jobs': models.ImportJob.
                 objects.filter(user=request.user).order_by('-created_date'),
-        'limit': goodreads_import.MAX_ENTRIES,
     })
 
 
@@ -499,7 +548,8 @@ def status_page(request, username, status_id):
         return HttpResponseNotFound()
 
     if is_api_request(request):
-        return ActivitypubResponse(status.to_activity())
+        return ActivitypubResponse(
+            status.to_activity(pure=not is_bookworm_request(request)))
 
     data = {
         'title': 'Status by %s' % user.username,
@@ -645,7 +695,7 @@ def book_page(request, book_id):
 @require_GET
 def edit_book_page(request, book_id):
     ''' info about a book '''
-    book = books_manager.get_edition(book_id)
+    book = get_edition(book_id)
     if not book.description:
         book.description = book.parent_work.description
     data = {

bw-dev

@@ -83,7 +83,7 @@ case "$1" in
         ;;
     pytest)
         shift 1
-        execweb pytest "$@"
+        execweb pytest --no-cov-on-fail "$@"
         ;;
     test_report)
         execweb coverage report

celerywyrm/celery.py

@@ -20,8 +20,9 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks()
 app.autodiscover_tasks(['bookwyrm'], related_name='activitypub.base_activity')
-app.autodiscover_tasks(['bookwyrm'], related_name='books_manager')
 app.autodiscover_tasks(['bookwyrm'], related_name='broadcast')
+app.autodiscover_tasks(
+    ['bookwyrm'], related_name='connectors.abstract_connector')
 app.autodiscover_tasks(['bookwyrm'], related_name='emailing')
 app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import')
 app.autodiscover_tasks(['bookwyrm'], related_name='incoming')