Merge branch 'main' into production

This commit is contained in:
Mouse Reeve 2022-11-05 13:48:27 -07:00
commit 2cbb80305d
4 changed files with 119 additions and 29 deletions

View file

@ -1,4 +1,5 @@
""" track progress of goodreads imports """ """ track progress of goodreads imports """
import math
import re import re
import dateutil.parser import dateutil.parser
@ -53,6 +54,14 @@ class ImportJob(models.Model):
"""How many books do you want to import???""" """How many books do you want to import???"""
return self.items.count() return self.items.count()
@property
def percent_complete(self):
"""How far along?"""
item_count = self.item_count
if not item_count:
return 0
return math.floor((item_count - self.pending_item_count) / item_count * 100)
@property @property
def pending_item_count(self): def pending_item_count(self):
"""And how many pending items??""" """And how many pending items??"""

View file

@ -7,12 +7,28 @@
{% block content %} {% block content %}
<div class="block"> <div class="block">
<h1 class="title">{% trans "Import Books" %}</h1> <h1 class="title">{% trans "Import Books" %}</h1>
{% if recent_avg_hours or recent_avg_minutes %}
<div class="notification">
<p>
{% if recent_avg_hours %}
{% blocktrans trimmed with hours=recent_avg_hours|floatformat:0|intcomma %}
On average, recent imports have taken {{ hours }} hours.
{% endblocktrans %}
{% else %}
{% blocktrans trimmed with minutes=recent_avg_minutes|floatformat:0|intcomma %}
On average, recent imports have taken {{ minutes }} minutes.
{% endblocktrans %}
{% endif %}
</p>
</div>
{% endif %}
<form class="box" name="import" action="/import" method="post" enctype="multipart/form-data"> <form class="box" name="import" action="/import" method="post" enctype="multipart/form-data">
{% csrf_token %} {% csrf_token %}
<div class="columns"> <div class="columns">
<div class="column is-half"> <div class="column is-half">
<div class="field"> <div class="field">
<label class="label" for="source"> <label class="label" for="source">
{% trans "Data source:" %} {% trans "Data source:" %}
@ -21,19 +37,19 @@
<div class="select"> <div class="select">
<select name="source" id="source" aria-describedby="desc_source"> <select name="source" id="source" aria-describedby="desc_source">
<option value="Goodreads" {% if current == 'Goodreads' %}selected{% endif %}> <option value="Goodreads" {% if current == 'Goodreads' %}selected{% endif %}>
Goodreads (CSV) {% trans "Goodreads (CSV)" %}
</option> </option>
<option value="Storygraph" {% if current == 'Storygraph' %}selected{% endif %}> <option value="Storygraph" {% if current == 'Storygraph' %}selected{% endif %}>
Storygraph (CSV) {% trans "Storygraph (CSV)" %}
</option> </option>
<option value="LibraryThing" {% if current == 'LibraryThing' %}selected{% endif %}> <option value="LibraryThing" {% if current == 'LibraryThing' %}selected{% endif %}>
LibraryThing (TSV) {% trans "LibraryThing (TSV)" %}
</option> </option>
<option value="OpenLibrary" {% if current == 'OpenLibrary' %}selected{% endif %}> <option value="OpenLibrary" {% if current == 'OpenLibrary' %}selected{% endif %}>
OpenLibrary (CSV) {% trans "OpenLibrary (CSV)" %}
</option> </option>
<option value="Calibre" {% if current == 'Calibre' %}selected{% endif %}> <option value="Calibre" {% if current == 'Calibre' %}selected{% endif %}>
Calibre (CSV) {% trans "Calibre (CSV)" %}
</option> </option>
</select> </select>
</div> </div>
@ -73,13 +89,54 @@
<div class="content block"> <div class="content block">
<h2 class="title">{% trans "Recent Imports" %}</h2> <h2 class="title">{% trans "Recent Imports" %}</h2>
{% if not jobs %} <div class="table-container">
<p><em>{% trans "No recent imports" %}</em></p> <table class="table is-striped is-fullwidth">
{% endif %} <tr>
<ul> <th>
{% for job in jobs %} {% trans "Date Created" %}
<li><a href="{% url 'import-status' job.id %}">{{ job.created_date | naturaltime }}</a></li> </th>
{% endfor %} <th>
</ul> {% trans "Last Updated" %}
</th>
<th>
{% trans "Items" %}
</th>
<th>
{% trans "Status" %}
</th>
</tr>
{% if not jobs %}
<tr>
<td colspan="4">
<em>{% trans "No recent imports" %}</em>
</td>
</tr>
{% endif %}
{% for job in jobs %}
<tr>
<td>
<a href="{% url 'import-status' job.id %}">{{ job.created_date }}</a>
</td>
<td>{{ job.updated_date }}</td>
<td>{{ job.item_count|intcomma }}</td>
<td>
{% if job.complete %}
<span class="tag is-success">
{% trans "Completed" %}
</span>
{% else %}
<span class="tag is-warning">
{% blocktrans trimmed with percent=job.percent_complete %}
Active, {{ percent }}% complete
{% endblocktrans %}
</span>
{% endif %}
</td>
</tr>
{% endfor %}
</table>
</div>
{% include 'snippets/pagination.html' with page=jobs path=request.path %}
</div> </div>
{% endblock %} {% endblock %}

View file

@ -1,10 +1,14 @@
""" import books from another app """ """ import books from another app """
from io import TextIOWrapper from io import TextIOWrapper
import datetime
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
from django.db.models import Avg, ExpressionWrapper, F, fields
from django.core.paginator import Paginator
from django.http import HttpResponseBadRequest from django.http import HttpResponseBadRequest
from django.shortcuts import redirect from django.shortcuts import redirect
from django.template.response import TemplateResponse from django.template.response import TemplateResponse
from django.utils import timezone
from django.utils.decorators import method_decorator from django.utils.decorators import method_decorator
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from django.views import View from django.views import View
@ -17,6 +21,7 @@ from bookwyrm.importers import (
StorygraphImporter, StorygraphImporter,
OpenLibraryImporter, OpenLibraryImporter,
) )
from bookwyrm.settings import PAGE_LENGTH
# pylint: disable= no-self-use # pylint: disable= no-self-use
@method_decorator(login_required, name="dispatch") @method_decorator(login_required, name="dispatch")
@ -25,16 +30,39 @@ class Import(View):
def get(self, request): def get(self, request):
"""load import page""" """load import page"""
return TemplateResponse( jobs = models.ImportJob.objects.filter(user=request.user).order_by(
request, "-created_date"
"import/import.html",
{
"import_form": forms.ImportForm(),
"jobs": models.ImportJob.objects.filter(user=request.user).order_by(
"-created_date"
),
},
) )
paginated = Paginator(jobs, PAGE_LENGTH)
page = paginated.get_page(request.GET.get("page"))
data = {
"import_form": forms.ImportForm(),
"jobs": page,
"page_range": paginated.get_elided_page_range(
page.number, on_each_side=2, on_ends=1
),
}
last_week = timezone.now() - datetime.timedelta(days=7)
recent_avg = (
models.ImportJob.objects.filter(created_date__gte=last_week, complete=True)
.annotate(
runtime=ExpressionWrapper(
F("updated_date") - F("created_date"),
output_field=fields.DurationField(),
)
)
.aggregate(Avg("runtime"))
.get("runtime__avg")
)
if recent_avg:
seconds = recent_avg.total_seconds()
if seconds > 60**2:
data["recent_avg_hours"] = recent_avg.seconds / (60**2)
else:
data["recent_avg_minutes"] = recent_avg.seconds / 60
return TemplateResponse(request, "import/import.html", data)
def post(self, request): def post(self, request):
"""ingest a goodreads csv""" """ingest a goodreads csv"""

View file

@ -1,6 +1,4 @@
""" import books from another app """ """ import books from another app """
import math
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied from django.core.exceptions import PermissionDenied
from django.core.paginator import Paginator from django.core.paginator import Paginator
@ -38,7 +36,7 @@ class ImportStatus(View):
fail_count = items.filter( fail_count = items.filter(
fail_reason__isnull=False, book_guess__isnull=True fail_reason__isnull=False, book_guess__isnull=True
).count() ).count()
pending_item_count = job.pending_items.count() pending_item_count = job.pending_item_count
data = { data = {
"job": job, "job": job,
"items": page, "items": page,
@ -50,9 +48,7 @@ class ImportStatus(View):
"show_progress": True, "show_progress": True,
"item_count": item_count, "item_count": item_count,
"complete_count": item_count - pending_item_count, "complete_count": item_count - pending_item_count,
"percent": math.floor( # pylint: disable=c-extension-no-member "percent": job.percent_complete,
(item_count - pending_item_count) / item_count * 100
),
# hours since last import item update # hours since last import item update
"inactive_time": (job.updated_date - timezone.now()).seconds / 60 / 60, "inactive_time": (job.updated_date - timezone.now()).seconds / 60 / 60,
"legacy": not job.mappings, "legacy": not job.mappings,