mirror of
https://github.com/bookwyrm-social/bookwyrm.git
synced 2024-11-15 13:24:04 +00:00
First draft async imports.
This commit is contained in:
parent
bba63e3515
commit
881cc4d64b
8 changed files with 57 additions and 56 deletions
|
@ -1,11 +1,13 @@
|
||||||
''' handle reading a csv from goodreads '''
|
''' handle reading a csv from goodreads '''
|
||||||
import re
|
import re
|
||||||
import csv
|
import csv
|
||||||
import itertools
|
|
||||||
import dateutil.parser
|
import dateutil.parser
|
||||||
|
from requests import HTTPError
|
||||||
|
|
||||||
from fedireads import books_manager
|
from fedireads import books_manager
|
||||||
from fedireads.models import Edition, ReadThrough
|
from fedireads import outgoing
|
||||||
|
from fedireads.models import Edition, ReadThrough, User
|
||||||
|
from fedireads.tasks import app
|
||||||
|
|
||||||
|
|
||||||
# Mapping goodreads -> fedireads shelf titles.
|
# Mapping goodreads -> fedireads shelf titles.
|
||||||
|
@ -36,14 +38,42 @@ def construct_search_term(title, author):
|
||||||
return ' '.join([title, author])
|
return ' '.join([title, author])
|
||||||
|
|
||||||
|
|
||||||
class GoodreadsCsv:
|
def async_import(user, csv_file):
|
||||||
''' define a goodreads csv '''
|
entries = list(csv.DictReader(csv_file))[:MAX_ENTRIES]
|
||||||
def __init__(self, csv_file):
|
return import_data.delay(user.id, entries)
|
||||||
self.reader = csv.DictReader(csv_file)
|
|
||||||
|
@app.task
|
||||||
|
def import_data(user_id, entries):
|
||||||
|
user = User.objects.get(pk=user_id)
|
||||||
|
results = []
|
||||||
|
reviews = []
|
||||||
|
failures = []
|
||||||
|
for item in entries:
|
||||||
|
item = GoodreadsItem(item)
|
||||||
|
try:
|
||||||
|
item.resolve()
|
||||||
|
except HTTPError:
|
||||||
|
pass
|
||||||
|
if item.book:
|
||||||
|
results.append(item)
|
||||||
|
if item.rating or item.review:
|
||||||
|
reviews.append(item)
|
||||||
|
else:
|
||||||
|
failures.append(item)
|
||||||
|
|
||||||
|
outgoing.handle_import_books(user, results)
|
||||||
|
for item in reviews:
|
||||||
|
review_title = "Review of {!r} on Goodreads".format(
|
||||||
|
item.book.title,
|
||||||
|
) if item.review else ""
|
||||||
|
outgoing.handle_review(
|
||||||
|
user,
|
||||||
|
item.book,
|
||||||
|
review_title,
|
||||||
|
item.review,
|
||||||
|
item.rating,
|
||||||
|
)
|
||||||
|
|
||||||
def __iter__(self):
|
|
||||||
for line in itertools.islice(self.reader, MAX_ENTRIES):
|
|
||||||
yield GoodreadsItem(line)
|
|
||||||
|
|
||||||
class GoodreadsItem:
|
class GoodreadsItem:
|
||||||
''' a processed line in a goodreads csv '''
|
''' a processed line in a goodreads csv '''
|
||||||
|
|
|
@ -14,6 +14,7 @@ from fedireads import models, outgoing
|
||||||
from fedireads import status as status_builder
|
from fedireads import status as status_builder
|
||||||
from fedireads.remote_user import get_or_create_remote_user
|
from fedireads.remote_user import get_or_create_remote_user
|
||||||
from fedireads.tasks import app
|
from fedireads.tasks import app
|
||||||
|
from fedireads.status import create_notification
|
||||||
|
|
||||||
|
|
||||||
@csrf_exempt
|
@csrf_exempt
|
||||||
|
|
|
@ -150,7 +150,8 @@ class ReadThrough(FedireadsModel):
|
||||||
|
|
||||||
|
|
||||||
NotificationType = models.TextChoices(
|
NotificationType = models.TextChoices(
|
||||||
'NotificationType', 'FAVORITE REPLY TAG FOLLOW FOLLOW_REQUEST BOOST')
|
'NotificationType',
|
||||||
|
'FAVORITE REPLY TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT_RESULT')
|
||||||
|
|
||||||
class Notification(FedireadsModel):
|
class Notification(FedireadsModel):
|
||||||
''' you've been tagged, liked, followed, etc '''
|
''' you've been tagged, liked, followed, etc '''
|
||||||
|
|
|
@ -203,6 +203,7 @@ def handle_import_books(user, items):
|
||||||
status.status_type = 'Update'
|
status.status_type = 'Update'
|
||||||
status.save()
|
status.save()
|
||||||
|
|
||||||
|
create_notification(user, 'IMPORT_RESULT', related_status=status)
|
||||||
create_activity = activitypub.get_create(
|
create_activity = activitypub.get_create(
|
||||||
user, activitypub.get_status(status))
|
user, activitypub.get_status(status))
|
||||||
broadcast(user, create_activity)
|
broadcast(user, create_activity)
|
||||||
|
@ -356,4 +357,3 @@ def handle_update_user(user):
|
||||||
actor = activitypub.get_actor(user)
|
actor = activitypub.get_actor(user)
|
||||||
update_activity = activitypub.get_update(user, actor)
|
update_activity = activitypub.get_update(user, actor)
|
||||||
broadcast(user, update_activity)
|
broadcast(user, update_activity)
|
||||||
|
|
||||||
|
|
|
@ -2,17 +2,9 @@
|
||||||
{% block content %}
|
{% block content %}
|
||||||
<div id="content">
|
<div id="content">
|
||||||
<div>
|
<div>
|
||||||
<h1>The following books could not be imported: </h1>
|
<h1>Import</h1>
|
||||||
|
|
||||||
<ul>
|
Import uploaded successfully. The import is being processed.
|
||||||
{% for item in failures %}
|
|
||||||
<li>
|
|
||||||
{{ item }}
|
|
||||||
</li>
|
|
||||||
{% endfor %}
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
<p>{{ success_count }} books imported successfully</p>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
|
@ -14,6 +14,7 @@
|
||||||
{% for notification in notifications %}
|
{% for notification in notifications %}
|
||||||
<div class="notification{% if notification.id in unread %} unread{% endif %}">
|
<div class="notification{% if notification.id in unread %} unread{% endif %}">
|
||||||
<small class="time-ago">{{ notification.created_date | naturaltime }}</small>
|
<small class="time-ago">{{ notification.created_date | naturaltime }}</small>
|
||||||
|
{% if notification.related_user %}
|
||||||
{% include 'snippets/username.html' with user=notification.related_user %}
|
{% include 'snippets/username.html' with user=notification.related_user %}
|
||||||
{% if notification.notification_type == 'FAVORITE' %}
|
{% if notification.notification_type == 'FAVORITE' %}
|
||||||
favorited your
|
favorited your
|
||||||
|
@ -36,6 +37,10 @@
|
||||||
{% elif notification.notification_type == 'BOOST' %}
|
{% elif notification.notification_type == 'BOOST' %}
|
||||||
boosted your <a href="{{ notification.related_status.absolute_id}}">status</a>
|
boosted your <a href="{{ notification.related_status.absolute_id}}">status</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
{% else %}
|
||||||
|
your <a href="{{ notification.related_status.absolute_id }}">import</a> succeeded.
|
||||||
|
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
{% if not notifications %}
|
{% if not notifications %}
|
||||||
|
|
|
@ -2,7 +2,6 @@
|
||||||
from io import BytesIO, TextIOWrapper
|
from io import BytesIO, TextIOWrapper
|
||||||
import re
|
import re
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from requests import HTTPError
|
|
||||||
|
|
||||||
from django.contrib.auth import authenticate, login, logout
|
from django.contrib.auth import authenticate, login, logout
|
||||||
from django.contrib.auth.decorators import login_required
|
from django.contrib.auth.decorators import login_required
|
||||||
|
@ -12,7 +11,7 @@ from django.shortcuts import redirect
|
||||||
from django.template.response import TemplateResponse
|
from django.template.response import TemplateResponse
|
||||||
|
|
||||||
from fedireads import forms, models, books_manager, outgoing
|
from fedireads import forms, models, books_manager, outgoing
|
||||||
from fedireads.goodreads_import import GoodreadsCsv
|
from fedireads import goodreads_import
|
||||||
from fedireads.settings import DOMAIN
|
from fedireads.settings import DOMAIN
|
||||||
from fedireads.views import get_user_from_username
|
from fedireads.views import get_user_from_username
|
||||||
from fedireads.books_manager import get_or_create_book
|
from fedireads.books_manager import get_or_create_book
|
||||||
|
@ -419,38 +418,10 @@ def import_data(request):
|
||||||
''' ingest a goodreads csv '''
|
''' ingest a goodreads csv '''
|
||||||
form = forms.ImportForm(request.POST, request.FILES)
|
form = forms.ImportForm(request.POST, request.FILES)
|
||||||
if form.is_valid():
|
if form.is_valid():
|
||||||
results = []
|
goodreads_import.async_import(
|
||||||
reviews = []
|
request.user,
|
||||||
failures = []
|
TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding)
|
||||||
for item in GoodreadsCsv(TextIOWrapper(
|
)
|
||||||
request.FILES['csv_file'],
|
return TemplateResponse(request, 'import_results.html', {})
|
||||||
encoding=request.encoding)):
|
|
||||||
try:
|
|
||||||
item.resolve()
|
|
||||||
except HTTPError:
|
|
||||||
pass
|
|
||||||
if item.book:
|
|
||||||
results.append(item)
|
|
||||||
if item.rating or item.review:
|
|
||||||
reviews.append(item)
|
|
||||||
else:
|
|
||||||
failures.append(item)
|
|
||||||
|
|
||||||
outgoing.handle_import_books(request.user, results)
|
|
||||||
for item in reviews:
|
|
||||||
review_title = "Review of {!r} on Goodreads".format(
|
|
||||||
item.book.title,
|
|
||||||
) if item.review else ""
|
|
||||||
outgoing.handle_review(
|
|
||||||
request.user,
|
|
||||||
item.book,
|
|
||||||
review_title,
|
|
||||||
item.review,
|
|
||||||
item.rating,
|
|
||||||
)
|
|
||||||
return TemplateResponse(request, 'import_results.html', {
|
|
||||||
'success_count': len(results),
|
|
||||||
'failures': failures,
|
|
||||||
})
|
|
||||||
return HttpResponseBadRequest()
|
return HttpResponseBadRequest()
|
||||||
|
|
||||||
|
|
|
@ -21,4 +21,5 @@ app.autodiscover_tasks()
|
||||||
app.autodiscover_tasks(['fedireads'], related_name='incoming')
|
app.autodiscover_tasks(['fedireads'], related_name='incoming')
|
||||||
app.autodiscover_tasks(['fedireads'], related_name='broadcast')
|
app.autodiscover_tasks(['fedireads'], related_name='broadcast')
|
||||||
app.autodiscover_tasks(['fedireads'], related_name='books_manager')
|
app.autodiscover_tasks(['fedireads'], related_name='books_manager')
|
||||||
|
app.autodiscover_tasks(['fedireads'], related_name='goodreads_import')
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue