First draft async imports.

This commit is contained in:
Adam Kelly 2020-04-20 17:10:19 +01:00
parent bba63e3515
commit 881cc4d64b
8 changed files with 57 additions and 56 deletions

View file

@ -1,11 +1,13 @@
''' handle reading a csv from goodreads ''' ''' handle reading a csv from goodreads '''
import re import re
import csv import csv
import itertools
import dateutil.parser import dateutil.parser
from requests import HTTPError
from fedireads import books_manager from fedireads import books_manager
from fedireads.models import Edition, ReadThrough from fedireads import outgoing
from fedireads.models import Edition, ReadThrough, User
from fedireads.tasks import app
# Mapping goodreads -> fedireads shelf titles. # Mapping goodreads -> fedireads shelf titles.
@ -36,14 +38,42 @@ def construct_search_term(title, author):
return ' '.join([title, author]) return ' '.join([title, author])
class GoodreadsCsv: def async_import(user, csv_file):
''' define a goodreads csv ''' entries = list(csv.DictReader(csv_file))[:MAX_ENTRIES]
def __init__(self, csv_file): return import_data.delay(user.id, entries)
self.reader = csv.DictReader(csv_file)
@app.task
def import_data(user_id, entries):
user = User.objects.get(pk=user_id)
results = []
reviews = []
failures = []
for item in entries:
item = GoodreadsItem(item)
try:
item.resolve()
except HTTPError:
pass
if item.book:
results.append(item)
if item.rating or item.review:
reviews.append(item)
else:
failures.append(item)
outgoing.handle_import_books(user, results)
for item in reviews:
review_title = "Review of {!r} on Goodreads".format(
item.book.title,
) if item.review else ""
outgoing.handle_review(
user,
item.book,
review_title,
item.review,
item.rating,
)
def __iter__(self):
for line in itertools.islice(self.reader, MAX_ENTRIES):
yield GoodreadsItem(line)
class GoodreadsItem: class GoodreadsItem:
''' a processed line in a goodreads csv ''' ''' a processed line in a goodreads csv '''

View file

@ -14,6 +14,7 @@ from fedireads import models, outgoing
from fedireads import status as status_builder from fedireads import status as status_builder
from fedireads.remote_user import get_or_create_remote_user from fedireads.remote_user import get_or_create_remote_user
from fedireads.tasks import app from fedireads.tasks import app
from fedireads.status import create_notification
@csrf_exempt @csrf_exempt

View file

@ -150,7 +150,8 @@ class ReadThrough(FedireadsModel):
NotificationType = models.TextChoices( NotificationType = models.TextChoices(
'NotificationType', 'FAVORITE REPLY TAG FOLLOW FOLLOW_REQUEST BOOST') 'NotificationType',
'FAVORITE REPLY TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT_RESULT')
class Notification(FedireadsModel): class Notification(FedireadsModel):
''' you've been tagged, liked, followed, etc ''' ''' you've been tagged, liked, followed, etc '''

View file

@ -203,6 +203,7 @@ def handle_import_books(user, items):
status.status_type = 'Update' status.status_type = 'Update'
status.save() status.save()
create_notification(user, 'IMPORT_RESULT', related_status=status)
create_activity = activitypub.get_create( create_activity = activitypub.get_create(
user, activitypub.get_status(status)) user, activitypub.get_status(status))
broadcast(user, create_activity) broadcast(user, create_activity)
@ -356,4 +357,3 @@ def handle_update_user(user):
actor = activitypub.get_actor(user) actor = activitypub.get_actor(user)
update_activity = activitypub.get_update(user, actor) update_activity = activitypub.get_update(user, actor)
broadcast(user, update_activity) broadcast(user, update_activity)

View file

@ -2,17 +2,9 @@
{% block content %} {% block content %}
<div id="content"> <div id="content">
<div> <div>
<h1>The following books could not be imported: </h1> <h1>Import</h1>
<ul> Import uploaded successfully. The import is being processed.
{% for item in failures %}
<li>
{{ item }}
</li>
{% endfor %}
</ul>
<p>{{ success_count }} books imported successfully</p>
</div> </div>
</div> </div>
{% endblock %} {% endblock %}

View file

@ -14,6 +14,7 @@
{% for notification in notifications %} {% for notification in notifications %}
<div class="notification{% if notification.id in unread %} unread{% endif %}"> <div class="notification{% if notification.id in unread %} unread{% endif %}">
<small class="time-ago">{{ notification.created_date | naturaltime }}</small> <small class="time-ago">{{ notification.created_date | naturaltime }}</small>
{% if notification.related_user %}
{% include 'snippets/username.html' with user=notification.related_user %} {% include 'snippets/username.html' with user=notification.related_user %}
{% if notification.notification_type == 'FAVORITE' %} {% if notification.notification_type == 'FAVORITE' %}
favorited your favorited your
@ -36,6 +37,10 @@
{% elif notification.notification_type == 'BOOST' %} {% elif notification.notification_type == 'BOOST' %}
boosted your <a href="{{ notification.related_status.absolute_id}}">status</a> boosted your <a href="{{ notification.related_status.absolute_id}}">status</a>
{% endif %} {% endif %}
{% else %}
your <a href="{{ notification.related_status.absolute_id }}">import</a> succeeded.
{% endif %}
</div> </div>
{% endfor %} {% endfor %}
{% if not notifications %} {% if not notifications %}

View file

@ -2,7 +2,6 @@
from io import BytesIO, TextIOWrapper from io import BytesIO, TextIOWrapper
import re import re
from PIL import Image from PIL import Image
from requests import HTTPError
from django.contrib.auth import authenticate, login, logout from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
@ -12,7 +11,7 @@ from django.shortcuts import redirect
from django.template.response import TemplateResponse from django.template.response import TemplateResponse
from fedireads import forms, models, books_manager, outgoing from fedireads import forms, models, books_manager, outgoing
from fedireads.goodreads_import import GoodreadsCsv from fedireads import goodreads_import
from fedireads.settings import DOMAIN from fedireads.settings import DOMAIN
from fedireads.views import get_user_from_username from fedireads.views import get_user_from_username
from fedireads.books_manager import get_or_create_book from fedireads.books_manager import get_or_create_book
@ -419,38 +418,10 @@ def import_data(request):
''' ingest a goodreads csv ''' ''' ingest a goodreads csv '''
form = forms.ImportForm(request.POST, request.FILES) form = forms.ImportForm(request.POST, request.FILES)
if form.is_valid(): if form.is_valid():
results = [] goodreads_import.async_import(
reviews = [] request.user,
failures = [] TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding)
for item in GoodreadsCsv(TextIOWrapper( )
request.FILES['csv_file'], return TemplateResponse(request, 'import_results.html', {})
encoding=request.encoding)):
try:
item.resolve()
except HTTPError:
pass
if item.book:
results.append(item)
if item.rating or item.review:
reviews.append(item)
else:
failures.append(item)
outgoing.handle_import_books(request.user, results)
for item in reviews:
review_title = "Review of {!r} on Goodreads".format(
item.book.title,
) if item.review else ""
outgoing.handle_review(
request.user,
item.book,
review_title,
item.review,
item.rating,
)
return TemplateResponse(request, 'import_results.html', {
'success_count': len(results),
'failures': failures,
})
return HttpResponseBadRequest() return HttpResponseBadRequest()

View file

@ -21,4 +21,5 @@ app.autodiscover_tasks()
app.autodiscover_tasks(['fedireads'], related_name='incoming') app.autodiscover_tasks(['fedireads'], related_name='incoming')
app.autodiscover_tasks(['fedireads'], related_name='broadcast') app.autodiscover_tasks(['fedireads'], related_name='broadcast')
app.autodiscover_tasks(['fedireads'], related_name='books_manager') app.autodiscover_tasks(['fedireads'], related_name='books_manager')
app.autodiscover_tasks(['fedireads'], related_name='goodreads_import')