Merge pull request #2511 from Giebisch/import-limit

Added Import Limit

bookwyrm/importers/importer.py

@@ -1,7 +1,8 @@
 """ handle reading a csv from an external service, defaults are from Goodreads """
 import csv
+from datetime import timedelta
 from django.utils import timezone
-from bookwyrm.models import ImportJob, ImportItem
+from bookwyrm.models import ImportJob, ImportItem, SiteSettings
 
 
 class Importer:
@@ -33,6 +34,7 @@ class Importer:
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
+    # pylint: disable=too-many-locals
     def create_job(self, user, csv_file, include_reviews, privacy):
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
@@ -49,7 +51,13 @@ class Importer:
             source=self.service,
         )
 
+        enforce_limit, allowed_imports = self.get_import_limit(user)
+        if enforce_limit and allowed_imports <= 0:
+            job.complete_job()
+            return job
         for index, entry in rows:
+            if enforce_limit and index >= allowed_imports:
+                break
             self.create_item(job, index, entry)
         return job
 
@@ -99,6 +107,24 @@ class Importer:
         """use the dataclass to create the formatted row of data"""
         return {k: entry.get(v) for k, v in mappings.items()}
 
+    def get_import_limit(self, user):  # pylint: disable=no-self-use
+        """check if import limit is set and return how many imports are left"""
+        site_settings = SiteSettings.objects.get()
+        import_size_limit = site_settings.import_size_limit
+        import_limit_reset = site_settings.import_limit_reset
+        enforce_limit = import_size_limit and import_limit_reset
+        allowed_imports = 0
+
+        if enforce_limit:
+            time_range = timezone.now() - timedelta(days=import_limit_reset)
+            import_jobs = ImportJob.objects.filter(
+                user=user, created_date__gte=time_range
+            )
+            # pylint: disable=consider-using-generator
+            imported_books = sum([job.successful_item_count for job in import_jobs])
+            allowed_imports = import_size_limit - imported_books
+        return enforce_limit, allowed_imports
+
     def create_retry_job(self, user, original_job, items):
         """retry items that didn't import"""
         job = ImportJob.objects.create(
@@ -110,7 +136,13 @@ class Importer:
             mappings=original_job.mappings,
             retry=True,
         )
-        for item in items:
+        enforce_limit, allowed_imports = self.get_import_limit(user)
+        if enforce_limit and allowed_imports <= 0:
+            job.complete_job()
+            return job
+        for index, item in enumerate(items):
+            if enforce_limit and index >= allowed_imports:
+                break
             # this will re-normalize the raw data
             self.create_item(job, item.index, item.data)
         return job

bookwyrm/migrations/0167_sitesettings_import_size_limit.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-12-05 13:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0166_sitesettings_imports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="import_size_limit",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="import_limit_reset",
+            field=models.IntegerField(default=0),
+        ),
+    ]

bookwyrm/migrations/0171_merge_20221219_2020.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2022-12-19 20:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0167_sitesettings_import_size_limit"),
+        ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+    ]
+
+    operations = []

bookwyrm/migrations/0173_merge_20230102_1444.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2023-01-02 14:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0171_merge_20221219_2020"),
+        ("bookwyrm", "0172_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0174_merge_20230111_1523.py

@@ -0,0 +1,12 @@
+# Generated by Django 3.2.16 on 2023-01-11 15:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0173_merge_20230102_1444"),
+    ]
+
+    operations = []

bookwyrm/models/site.py

@@ -90,6 +90,8 @@ class SiteSettings(SiteModel):
 
     # controls
     imports_enabled = models.BooleanField(default=True)
+    import_size_limit = models.IntegerField(default=0)
+    import_limit_reset = models.IntegerField(default=0)
 
     field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])
 

bookwyrm/static/css/bookwyrm/utilities/_size.scss

@@ -40,6 +40,10 @@
     width: 500px !important;
 }
 
+.is-h-em {
+    height: 1em !important;
+}
+
 .is-h-xs {
     height: 80px !important;
 }

bookwyrm/templates/import/import.html

@@ -15,6 +15,12 @@
     {% endif %}
 
     {% if site.imports_enabled %}
+        {% if import_size_limit and import_limit_reset %}
+            <div class="notification">
+                <p>{% blocktrans %}Currently you are allowed to import {{ import_size_limit }} books every {{ import_limit_reset }} days.{% endblocktrans %}</p>
+                <p>{% blocktrans %}You have {{ allowed_imports }} left.{% endblocktrans %}</p>
+            </div>
+        {% endif %}
         {% if recent_avg_hours or recent_avg_minutes %}
         <div class="notification">
             <p>
@@ -90,7 +96,12 @@
                 </div>
             </div>
             </div>
-            <button class="button is-primary" type="submit">{% trans "Import" %}</button>
+            {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %}
+                <button class="button is-primary" type="submit">{% trans "Import" %}</button>
+            {% else %}
+                <button class="button is-primary is-disabled" type="submit">{% trans "Import" %}</button>
+                <p>{% trans "You've reached the import limit." %}</p>
+            {% endif%}
         </form>
     {% else %}
     <div class="box notification has-text-centered is-warning m-6 content">

bookwyrm/templates/settings/imports/imports.html

@@ -57,8 +57,39 @@
         </div>
     </form>
     {% endif %}
+    <details class="details-panel box">
+        <summary>
+            <span role="heading" aria-level="2" class="title is-6">
+                {% trans "Limit the amount of imports" %}
+            </span>
+            <span class="details-close icon icon-x" aria-hidden="true"></span>
+        </summary>
+        <form
+            name="imports-set-limit"
+            id="imports-set-limit"
+            method="POST"
+            action="{% url 'settings-imports-set-limit' %}"
+        >
+            <div class="notification">
+                {% trans "Some users might try to import a large number of books, which you want to limit." %}
+                {% trans "Set the value to 0 to not enforce any limit." %}
+            </div>
+            <div class="align.to-t">
+                <label for="limit">{% trans "Set import limit to" %}</label>
+                <input name="limit" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ import_size_limit }}">
+                <label for="reset">{% trans "books every" %}</label>
+                <input name="reset" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ import_limit_reset }}">
+                <label>{% trans "days." %}</label>
+                {% csrf_token %}
+                <div class="control">
+                    <button type="submit" class="button is-warning">
+                        {% trans "Set limit" %}
+                    </button>
+                </div>
+            </div>
+        </form>
+    </details>
 </div>
-
 <div class="block">
     <div class="tabs">
         <ul>

bookwyrm/tests/importers/test_calibre_import.py

@@ -28,7 +28,7 @@ class CalibreImport(TestCase):
             self.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
-
+        models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
         self.book = models.Edition.objects.create(
             title="Example Edition",

bookwyrm/tests/importers/test_goodreads_import.py

@@ -35,7 +35,7 @@ class GoodreadsImport(TestCase):
             self.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
-
+        models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
         self.book = models.Edition.objects.create(
             title="Example Edition",

bookwyrm/tests/importers/test_importer.py

@@ -39,7 +39,7 @@ class GenericImporter(TestCase):
             self.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
-
+        models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
         self.book = models.Edition.objects.create(
             title="Example Edition",
@@ -360,3 +360,16 @@ class GenericImporter(TestCase):
         self.assertFalse(
             models.Review.objects.filter(book=self.book, user=self.local_user).exists()
         )
+
+    def test_import_limit(self, *_):
+        """checks if import limit works"""
+        site_settings = models.SiteSettings.objects.get()
+        site_settings.import_size_limit = 2
+        site_settings.import_limit_reset = 2
+        site_settings.save()
+
+        import_job = self.importer.create_job(
+            self.local_user, self.csv, False, "public"
+        )
+        import_items = models.ImportItem.objects.filter(job=import_job).all()
+        self.assertEqual(len(import_items), 2)

bookwyrm/tests/importers/test_librarything_import.py

@@ -37,6 +37,7 @@ class LibrarythingImport(TestCase):
             self.local_user = models.User.objects.create_user(
                 "mmai", "mmai@mmai.mmai", "password", local=True
             )
+        models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
         self.book = models.Edition.objects.create(
             title="Example Edition",

bookwyrm/tests/importers/test_openlibrary_import.py

@@ -35,7 +35,7 @@ class OpenLibraryImport(TestCase):
             self.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
-
+        models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
         self.book = models.Edition.objects.create(
             title="Example Edition",

bookwyrm/tests/importers/test_storygraph_import.py

@@ -35,7 +35,7 @@ class StorygraphImport(TestCase):
             self.local_user = models.User.objects.create_user(
                 "mouse", "mouse@mouse.mouse", "password", local=True
             )
-
+        models.SiteSettings.objects.create()
         work = models.Work.objects.create(title="Test Work")
         self.book = models.Edition.objects.create(
             title="Example Edition",

bookwyrm/urls.py

@@ -321,6 +321,11 @@ urlpatterns = [
         views.enable_imports,
         name="settings-imports-enable",
     ),
+    re_path(
+        r"^settings/imports/set-limit/?$",
+        views.set_import_size_limit,
+        name="settings-imports-set-limit",
+    ),
     re_path(
         r"^settings/celery/?$", views.CeleryStatus.as_view(), name="settings-celery"
     ),

bookwyrm/views/__init__.py

@@ -11,7 +11,12 @@ from .admin.federation import AddFederatedServer, ImportServerBlocklist
 from .admin.federation import block_server, unblock_server, refresh_server
 from .admin.email_blocklist import EmailBlocklist
 from .admin.email_config import EmailConfig
-from .admin.imports import ImportList, disable_imports, enable_imports
+from .admin.imports import (
+    ImportList,
+    disable_imports,
+    enable_imports,
+    set_import_size_limit,
+)
 from .admin.ip_blocklist import IPBlocklist
 from .admin.invite import ManageInvites, Invite, InviteRequest
 from .admin.invite import ManageInviteRequests, ignore_invite_request

bookwyrm/views/admin/imports.py

@@ -38,6 +38,8 @@ class ImportList(View):
 
         paginated = Paginator(imports, PAGE_LENGTH)
         page = paginated.get_page(request.GET.get("page"))
+
+        site_settings = models.SiteSettings.objects.get()
         data = {
             "imports": page,
             "page_range": paginated.get_elided_page_range(
@@ -45,6 +47,8 @@ class ImportList(View):
             ),
             "status": status,
             "sort": sort,
+            "import_size_limit": site_settings.import_size_limit,
+            "import_limit_reset": site_settings.import_limit_reset,
         }
         return TemplateResponse(request, "settings/imports/imports.html", data)
 
@@ -76,3 +80,17 @@ def enable_imports(request):
     site.imports_enabled = True
     site.save(update_fields=["imports_enabled"])
     return redirect("settings-imports")
+
+
+@require_POST
+@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
+# pylint: disable=unused-argument
+def set_import_size_limit(request):
+    """Limit the amount of books users can import at once"""
+    site = models.SiteSettings.objects.get()
+    import_size_limit = int(request.POST.get("limit"))
+    import_limit_reset = int(request.POST.get("reset"))
+    site.import_size_limit = import_size_limit
+    site.import_limit_reset = import_limit_reset
+    site.save(update_fields=["import_size_limit", "import_limit_reset"])
+    return redirect("settings-imports")

bookwyrm/views/imports/import_data.py

@@ -51,6 +51,19 @@ class Import(View):
         elif seconds:
             data["recent_avg_minutes"] = seconds / 60
 
+        site_settings = models.SiteSettings.objects.get()
+        time_range = timezone.now() - datetime.timedelta(
+            days=site_settings.import_limit_reset
+        )
+        import_jobs = models.ImportJob.objects.filter(
+            user=request.user, created_date__gte=time_range
+        )
+        # pylint: disable=consider-using-generator
+        imported_books = sum([job.successful_item_count for job in import_jobs])
+        data["import_size_limit"] = site_settings.import_size_limit
+        data["import_limit_reset"] = site_settings.import_limit_reset
+        data["allowed_imports"] = site_settings.import_size_limit - imported_books
+
         return TemplateResponse(request, "import/import.html", data)
 
     def post(self, request):