admin view for user imports

- makes user_import_time_limit a site setting rather than a value in settings.py (note this applies to exports as well as imports)
- admins can change user_import_time_limit from UI
- admins can cancel stuck user imports
- disabling new imports also disables user imports
This commit is contained in:
Hugh Rundle 2023-10-22 15:07:49 +11:00
parent 836127f369
commit a27c652501
No known key found for this signature in database
GPG key ID: A7E35779918253F9
11 changed files with 374 additions and 160 deletions

View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.20 on 2023-10-22 02:27
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('bookwyrm', '0183_auto_20231021_2050'),
]
operations = [
migrations.AddField(
model_name='sitesettings',
name='user_import_time_limit',
field=models.IntegerField(default=48),
),
]

View file

@ -96,6 +96,7 @@ class SiteSettings(SiteModel):
imports_enabled = models.BooleanField(default=True)
import_size_limit = models.IntegerField(default=0)
import_limit_reset = models.IntegerField(default=0)
user_import_time_limit = models.IntegerField(default=48)
field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])

View file

@ -422,7 +422,4 @@ if HTTP_X_FORWARDED_PROTO:
# Mastodon servers.
# Do not change this setting unless you already have an existing
# user with the same username - in which case you should change it!
INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"
# exports
USER_EXPORT_COOLDOWN_HOURS = 48
INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"

View file

@ -10,81 +10,89 @@
{% if invalid %}
<div class="notification is-danger">
{% trans "Not a valid JSON file" %}
{% trans "Not a valid import file" %}
</div>
{% endif %}
{% if not site.imports_enabled %}
<div class="box notification has-text-centered is-warning m-6 content">
<p class="mt-5">
<span class="icon icon-warning is-size-2" aria-hidden="true"></span>
</p>
<p class="mb-5">
{% trans "Imports are temporarily disabled; thank you for your patience." %}
</p>
</div>
{% elif next_available %}
<div class="notification is-warning">
<p>{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}</p>
<p>{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}</p>
</div>
{% else %}
<form class="box" name="import-user" action="/user-import" method="post" enctype="multipart/form-data">
{% csrf_token %}
{% if next_available %}
<div class="notification is-warning">
<p>{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}</p>
<p>{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}</p>
<div class="columns">
<div class="column is-half">
<div class="field">
<label class="label" for="id_archive_file">{% trans "Data file:" %}</label>
{{ import_form.archive_file }}
</div>
<div>
<p class="block"> {% trans "Importing this file will overwrite any data you currently have saved." %}</p>
<p class="block">{% trans "Deselect any data you do not wish to include in your import. Books will always be imported" %}</p>
</div>
</div>
<div class="column is-half">
<div class="field">
<label class="label">
<input type="checkbox" name="include_user_profile" checked> {% trans "Include user profile" %}
</label>
<label class="label">
<input type="checkbox" name="include_user_settings" checked> {% trans "Include user settings" %}
</label>
<label class="label">
<input type="checkbox" name="include_goals" checked> {% trans "Include reading goals" %}
</label>
<label class="label">
<input type="checkbox" name="include_shelves" checked> {% trans "Include shelves" %}
</label>
<label class="label">
<input type="checkbox" name="include_readthroughs" checked> {% trans "Include 'readthroughs'" %}
</label>
<label class="label">
<input type="checkbox" name="include_reviews" checked> {% trans "Include book reviews" %}
</label>
<label class="label">
<input type="checkbox" name="include_quotes" checked> {% trans "Include quotations" %}
</label>
<label class="label">
<input type="checkbox" name="include_comments" checked> {% trans "Include comments about books" %}
</label>
<label class="label">
<input type="checkbox" name="include_lists" checked> {% trans "Include book lists" %}
</label>
<label class="label">
<input type="checkbox" name="include_saved_lists" checked> {% trans "Include saved lists" %}
</label>
<label class="label">
<input type="checkbox" name="include_follows" checked> {% trans "Include follows" %}
</label>
<label class="label">
<input type="checkbox" name="include_blocks" checked> {% trans "Include user blocks" %}
</label>
</div>
</div>
</div>
{% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %}
<button class="button is-primary" type="submit">{% trans "Import" %}</button>
{% else %}
<form class="box" name="import-user" action="/user-import" method="post" enctype="multipart/form-data">
{% csrf_token %}
<div class="columns">
<div class="column is-half">
<div class="field">
<label class="label" for="id_archive_file">{% trans "Data file:" %}</label>
{{ import_form.archive_file }}
</div>
<div>
<p class="block"> {% trans "Importing this file will overwrite any data you currently have saved." %}</p>
<p class="block">{% trans "Deselect any data you do not wish to include in your import. Books will always be imported" %}</p>
</div>
</div>
<div class="column is-half">
<div class="field">
<label class="label">
<input type="checkbox" name="include_user_profile" checked> {% trans "Include user profile" %}
</label>
<label class="label">
<input type="checkbox" name="include_user_settings" checked> {% trans "Include user settings" %}
</label>
<label class="label">
<input type="checkbox" name="include_goals" checked> {% trans "Include reading goals" %}
</label>
<label class="label">
<input type="checkbox" name="include_shelves" checked> {% trans "Include shelves" %}
</label>
<label class="label">
<input type="checkbox" name="include_readthroughs" checked> {% trans "Include 'readthroughs'" %}
</label>
<label class="label">
<input type="checkbox" name="include_reviews" checked> {% trans "Include book reviews" %}
</label>
<label class="label">
<input type="checkbox" name="include_quotes" checked> {% trans "Include quotations" %}
</label>
<label class="label">
<input type="checkbox" name="include_comments" checked> {% trans "Include comments about books" %}
</label>
<label class="label">
<input type="checkbox" name="include_lists" checked> {% trans "Include book lists" %}
</label>
<label class="label">
<input type="checkbox" name="include_saved_lists" checked> {% trans "Include saved lists" %}
</label>
<label class="label">
<input type="checkbox" name="include_follows" checked> {% trans "Include follows" %}
</label>
<label class="label">
<input type="checkbox" name="include_blocks" checked> {% trans "Include user blocks" %}
</label>
</div>
</div>
</div>
{% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %}
<button class="button is-primary" type="submit">{% trans "Import" %}</button>
{% else %}
<button class="button is-primary is-disabled" type="submit">{% trans "Import" %}</button>
<p>{% trans "You've reached the import limit." %}</p>
{% endif%}
</form>
{% endif %}
<button class="button is-primary is-disabled" type="submit">{% trans "Import" %}</button>
<p>{% trans "You've reached the import limit." %}</p>
{% endif%}
</form>
{% endif %}
</div>

View file

@ -0,0 +1,23 @@
{% extends 'components/modal.html' %}
{% load i18n %}
{% block modal-title %}{% trans "Stop import?" %}{% endblock %}
{% block modal-body %}
{% trans "This action will stop the user import before it is complete and cannot be un-done" %}
{% endblock %}
{% block modal-footer %}
<form name="complete-import-{{ import.id }}" action="{% url 'settings-user-import-complete' import.id %}" method="POST" class="is-flex-grow-1">
{% csrf_token %}
<input type="hidden" name="id" value="{{ import.id }}">
<div class="buttons is-right is-flex-grow-1">
<button type="button" class="button" data-modal-close>
{% trans "Cancel" %}
</button>
<button class="button is-danger" type="submit">
{% trans "Confirm" %}
</button>
</div>
</form>
{% endblock %}

View file

@ -29,6 +29,7 @@
<div class="notification">
{% trans "This is only intended to be used when things have gone very wrong with imports and you need to pause the feature while addressing issues." %}
{% trans "While imports are disabled, users will not be allowed to start new imports, but existing imports will not be affected." %}
{% trans "This setting prevents both book imports and user imports." %}
</div>
{% csrf_token %}
<div class="control">
@ -89,91 +90,214 @@
</div>
</form>
</details>
<details class="details-panel box">
<summary>
<span role="heading" aria-level="2" class="title is-6">
{% trans "Limit how often users can import and export" %}
</span>
<span class="details-close icon icon-x" aria-hidden="true"></span>
</summary>
<form
name="user-imports-set-limit"
id="user-imports-set-limit"
method="POST"
action="{% url 'settings-user-imports-set-limit' %}"
>
<div class="notification">
{% trans "Some users might try to run user imports or exports very frequently, which you want to limit." %}
{% trans "Set the value to 0 to not enforce any limit." %}
</div>
<div class="align.to-t">
<label for="limit">{% trans "Restrict user imports and exports to once every " %}</label>
<input name="limit" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ user_import_time_limit }}">
<label>{% trans "hours" %}</label>
{% csrf_token %}
<div class="control">
<button type="submit" class="button is-warning">
{% trans "Change limit" %}
</button>
</div>
</div>
</form>
</details>
</div>
<div class="block">
<div class="tabs">
<ul>
{% url 'settings-imports' as url %}
<li {% if request.path in url %}class="is-active" aria-current="page"{% endif %}>
<a href="{{ url }}">{% trans "Active" %}</a>
</li>
{% url 'settings-imports' status="complete" as url %}
<li {% if url in request.path %}class="is-active" aria-current="page"{% endif %}>
<a href="{{ url }}">{% trans "Completed" %}</a>
</li>
</ul>
<h4 class="title is-4">{% trans "Book Imports" %}</h4>
<div class="block">
<div class="tabs">
<ul>
{% url 'settings-imports' as url %}
<li {% if request.path in url %}class="is-active" aria-current="page"{% endif %}>
<a href="{{ url }}">{% trans "Active" %}</a>
</li>
{% url 'settings-imports' status="complete" as url %}
<li {% if url in request.path %}class="is-active" aria-current="page"{% endif %}>
<a href="{{ url }}">{% trans "Completed" %}</a>
</li>
</ul>
</div>
</div>
<div class="table-container block content">
<table class="table is-striped is-fullwidth">
<tr>
{% url 'settings-imports' status as url %}
<th>
{% trans "ID" %}
</th>
<th>
{% trans "User" as text %}
{% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %}
</th>
<th>
{% trans "Date Created" as text %}
{% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %}
</th>
{% if status != "active" %}
<th>
{% trans "Date Updated" %}
</th>
{% endif %}
<th>
{% trans "Items" %}
</th>
<th>
{% trans "Pending items" %}
</th>
<th>
{% trans "Successful items" %}
</th>
<th>
{% trans "Failed items" %}
</th>
{% if status == "active" %}
<th>{% trans "Actions" %}</th>
{% endif %}
</tr>
{% for import in imports %}
<tr>
<td>{{ import.id }}</td>
<td class="overflow-wrap-anywhere">
<a href="{% url 'settings-user' import.user.id %}">{{ import.user|username }}</a>
</td>
<td>{{ import.created_date }}</td>
{% if status != "active" %}
<td>{{ import.updated_date }}</td>
{% endif %}
<td>{{ import.item_count|intcomma }}</td>
<td>{{ import.pending_item_count|intcomma }}</td>
<td>{{ import.successful_item_count|intcomma }}</td>
<td>{{ import.failed_item_count|intcomma }}</td>
{% if status == "active" %}
<td>
{% join "complete" import.id as modal_id %}
<button type="button" data-modal-open="{{ modal_id }}" class="button is-danger">{% trans "Stop import" %}</button>
{% include "settings/imports/complete_import_modal.html" with id=modal_id %}
</td>
{% endif %}
</tr>
{% endfor %}
{% if not imports %}
<tr>
<td colspan="6">
<em>{% trans "No matching imports found." %} </em>
</td>
</tr>
{% endif %}
</table>
</div>
{% include 'snippets/pagination.html' with page=imports path=request.path %}
</div>
<div class="table-container block content">
<table class="table is-striped is-fullwidth">
<tr>
{% url 'settings-imports' status as url %}
<th>
{% trans "ID" %}
</th>
<th>
{% trans "User" as text %}
{% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %}
</th>
<th>
{% trans "Date Created" as text %}
{% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %}
</th>
{% if status != "active" %}
<th>
{% trans "Date Updated" %}
</th>
<div class="block">
<h4 class="title is-4">{% trans "User Imports" %}</h4>
<div class="block">
<div class="tabs">
<ul>
{% url 'settings-imports' as url %}
<li {% if request.path in url %}class="is-active" aria-current="page"{% endif %}>
<a href="{{ url }}">{% trans "Active" %}</a>
</li>
{% url 'settings-imports' status="complete" as url %}
<li {% if url in request.path %}class="is-active" aria-current="page"{% endif %}>
<a href="{{ url }}">{% trans "Completed" %}</a>
</li>
</ul>
</div>
</div>
<div class="table-container block content">
<table class="table is-striped is-fullwidth">
<tr>
{% url 'settings-imports' status as url %}
<th>
{% trans "ID" %}
</th>
<th>
{% trans "User" as text %}
{% include 'snippets/table-sort-header.html' with field="user" sort=sort text=text %}
</th>
<th>
{% trans "Date Created" as text %}
{% include 'snippets/table-sort-header.html' with field="created_date" sort=sort text=text %}
</th>
{% if status != "active" %}
<th>
{% trans "Date Updated" %}
</th>
{% endif %}
{% if status == "active" %}
<th>{% trans "Actions" %}</th>
{% else %}
<th>{% trans "Status" %}</th>
{% endif %}
</tr>
{% for import in user_imports %}
<tr>
<td>{{ import.id }}</td>
<td class="overflow-wrap-anywhere">
<a href="{% url 'settings-user' import.user.id %}">{{ import.user|username }}</a>
</td>
<td>{{ import.created_date }}</td>
{% if status != "active" %}
<td>{{ import.updated_date }}</td>
{% endif %}
{% if status == "active" %}
<td>
{% join "complete" import.id as modal_id %}
<button type="button" data-modal-open="{{ modal_id }}" class="button is-danger">{% trans "Stop import" %}</button>
{% include "settings/imports/complete_user_import_modal.html" with id=modal_id %}
</td>
{% else %}
<td>
<span
{% if import.status == "stopped" or import.status == "failed" %}
class="tag is-danger"
{% elif import.status == "pending" %}
class="tag is-warning"
{% elif import.complete %}
class="tag"
{% else %}
class="tag is-success"
{% endif %}
>{{ import.status }}</span></td>
</td>
{% endif %}
</tr>
{% endfor %}
{% if not user_imports %}
<tr>
<td colspan="6">
<em>{% trans "No matching imports found." %} </em>
</td>
</tr>
{% endif %}
<th>
{% trans "Items" %}
</th>
<th>
{% trans "Pending items" %}
</th>
<th>
{% trans "Successful items" %}
</th>
<th>
{% trans "Failed items" %}
</th>
{% if status == "active" %}
<th>{% trans "Actions" %}</th>
{% endif %}
</tr>
{% for import in imports %}
<tr>
<td>{{ import.id }}</td>
<td class="overflow-wrap-anywhere">
<a href="{% url 'settings-user' import.user.id %}">{{ import.user|username }}</a>
</td>
<td>{{ import.created_date }}</td>
{% if status != "active" %}
<td>{{ import.updated_date }}</td>
{% endif %}
<td>{{ import.item_count|intcomma }}</td>
<td>{{ import.pending_item_count|intcomma }}</td>
<td>{{ import.successful_item_count|intcomma }}</td>
<td>{{ import.failed_item_count|intcomma }}</td>
{% if status == "active" %}
<td>
{% join "complete" import.id as modal_id %}
<button type="button" data-modal-open="{{ modal_id }}" class="button is-danger">{% trans "Stop import" %}</button>
{% include "settings/imports/complete_import_modal.html" with id=modal_id %}
</td>
{% endif %}
</tr>
{% endfor %}
{% if not imports %}
<tr>
<td colspan="6">
<em>{% trans "No matching imports found." %} </em>
</td>
</tr>
{% endif %}
</table>
</table>
</div>
{% include 'snippets/pagination.html' with page=user_imports path=request.path %}
{% endblock %}
</div>
{% include 'snippets/pagination.html' with page=imports path=request.path %}
{% endblock %}

View file

@ -316,6 +316,11 @@ urlpatterns = [
views.ImportList.as_view(),
name="settings-imports-complete",
),
re_path(
r"^settings/user-imports/(?P<import_id>\d+)/complete/?$",
views.set_user_import_completed,
name="settings-user-import-complete",
),
re_path(
r"^settings/imports/disable/?$",
views.disable_imports,
@ -331,6 +336,11 @@ urlpatterns = [
views.set_import_size_limit,
name="settings-imports-set-limit",
),
re_path(
r"^settings/user-imports/set-limit/?$",
views.set_user_import_limit,
name="settings-user-imports-set-limit",
),
re_path(
r"^settings/celery/?$", views.CeleryStatus.as_view(), name="settings-celery"
),

View file

@ -16,6 +16,8 @@ from .admin.imports import (
disable_imports,
enable_imports,
set_import_size_limit,
set_user_import_completed,
set_user_import_limit
)
from .admin.ip_blocklist import IPBlocklist
from .admin.invite import ManageInvites, Invite, InviteRequest

View file

@ -40,9 +40,17 @@ class ImportList(View):
paginated = Paginator(imports, PAGE_LENGTH)
page = paginated.get_page(request.GET.get("page"))
user_imports = models.BookwyrmImportJob.objects.filter(complete=complete).order_by(
"created_date"
)
user_paginated = Paginator(user_imports, PAGE_LENGTH)
user_page = user_paginated.get_page(request.GET.get("page"))
site_settings = models.SiteSettings.objects.get()
data = {
"imports": page,
"user_imports": user_page,
"page_range": paginated.get_elided_page_range(
page.number, on_each_side=2, on_ends=1
),
@ -50,6 +58,7 @@ class ImportList(View):
"sort": sort,
"import_size_limit": site_settings.import_size_limit,
"import_limit_reset": site_settings.import_limit_reset,
"user_import_time_limit": site_settings.user_import_time_limit,
}
return TemplateResponse(request, "settings/imports/imports.html", data)
@ -95,3 +104,24 @@ def set_import_size_limit(request):
site.import_limit_reset = import_limit_reset
site.save(update_fields=["import_size_limit", "import_limit_reset"])
return redirect("settings-imports")
@require_POST
@login_required
@permission_required("bookwyrm.moderate_user", raise_exception=True)
# pylint: disable=unused-argument
def set_user_import_completed(request, import_id):
"""Mark a user import as complete"""
import_job = get_object_or_404(models.BookwyrmImportJob, id=import_id)
import_job.stop_job()
return redirect("settings-imports")
@require_POST
@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
# pylint: disable=unused-argument
def set_user_import_limit(request):
"""Limit how ofter users can import and export their account"""
site = models.SiteSettings.objects.get()
site.user_import_time_limit = int(request.POST.get("limit"))
site.save(update_fields=["user_import_time_limit"])
return redirect("settings-imports")

View file

@ -23,7 +23,7 @@ from bookwyrm.importers import (
OpenLibraryImporter,
)
from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
from bookwyrm.settings import PAGE_LENGTH, USER_EXPORT_COOLDOWN_HOURS
from bookwyrm.settings import PAGE_LENGTH
from bookwyrm.utils.cache import get_or_set
# pylint: disable= no-self-use
@ -142,8 +142,9 @@ class UserImport(View):
jobs = BookwyrmImportJob.objects.filter(user=request.user).order_by(
"-created_date"
)
hours = USER_EXPORT_COOLDOWN_HOURS
allowed = jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours)
site = models.SiteSettings.objects.get()
hours = site.user_import_time_limit
allowed = jobs.first().created_date < timezone.now() - datetime.timedelta(hours=hours) if jobs.first() else True
next_available = jobs.first().created_date + datetime.timedelta(hours=hours) if not allowed else False
paginated = Paginator(jobs, PAGE_LENGTH)
page = paginated.get_page(request.GET.get("page"))

View file

@ -103,10 +103,10 @@ class ExportUser(View):
jobs = BookwyrmExportJob.objects.filter(user=request.user).order_by(
"-created_date"
)
hours = settings.USER_EXPORT_COOLDOWN_HOURS
allowed = jobs.first().created_date < timezone.now() - timedelta(hours=hours)
site = models.SiteSettings.objects.get()
hours = site.user_import_time_limit
allowed = jobs.first().created_date < timezone.now() - timedelta(hours=hours) if jobs.first() else True
next_available = jobs.first().created_date + timedelta(hours=hours) if not allowed else False
paginated = Paginator(jobs, PAGE_LENGTH)
page = paginated.get_page(request.GET.get("page"))
data = {