Merge branch 'main' into issue-3187

This commit is contained in:
Rohan 2024-01-18 21:19:20 +05:30 committed by GitHub
commit d73141792d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 178 additions and 21 deletions

View file

@ -137,3 +137,6 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60
# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default. # and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
# Value should be a comma-separated list of host names. # Value should be a comma-separated list of host names.
CSP_ADDITIONAL_HOSTS= CSP_ADDITIONAL_HOSTS=
# The last number here means "megabytes"
# Increase if users are having trouble uploading BookWyrm export files.
DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 100)

View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.23 on 2024-01-16 10:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0191_merge_20240102_0326"),
]
operations = [
migrations.AddField(
model_name="sitesettings",
name="user_exports_enabled",
field=models.BooleanField(default=False),
),
]

View file

@ -96,6 +96,7 @@ class SiteSettings(SiteModel):
imports_enabled = models.BooleanField(default=True) imports_enabled = models.BooleanField(default=True)
import_size_limit = models.IntegerField(default=0) import_size_limit = models.IntegerField(default=0)
import_limit_reset = models.IntegerField(default=0) import_limit_reset = models.IntegerField(default=0)
user_exports_enabled = models.BooleanField(default=False)
user_import_time_limit = models.IntegerField(default=48) user_import_time_limit = models.IntegerField(default=48)
field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])

View file

@ -442,3 +442,5 @@ if HTTP_X_FORWARDED_PROTO:
# Do not change this setting unless you already have an existing # Do not change this setting unless you already have an existing
# user with the same username - in which case you should change it! # user with the same username - in which case you should change it!
INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"
DATA_UPLOAD_MAX_MEMORY_SIZE = env.int("DATA_UPLOAD_MAX_MEMORY_SIZE", (1024**2 * 100))

View file

@ -31,10 +31,10 @@
</p> </p>
</div> </div>
<div class="columns"> <div class="columns is-multiline">
{% if superlatives.top_rated %} {% if superlatives.top_rated %}
{% with book=superlatives.top_rated.default_edition rating=superlatives.top_rated.rating %} {% with book=superlatives.top_rated.default_edition rating=superlatives.top_rated.rating %}
<div class="column is-one-third is-flex"> <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
<div class="media notification is-clipped"> <div class="media notification is-clipped">
<div class="media-left"> <div class="media-left">
<a href="{{ book.local_path }}"> <a href="{{ book.local_path }}">
@ -53,7 +53,7 @@
{% if superlatives.wanted %} {% if superlatives.wanted %}
{% with book=superlatives.wanted.default_edition %} {% with book=superlatives.wanted.default_edition %}
<div class="column is-one-third is-flex"> <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
<div class="media notification is-clipped"> <div class="media notification is-clipped">
<div class="media-left"> <div class="media-left">
<a href="{{ book.local_path }}"> <a href="{{ book.local_path }}">
@ -72,7 +72,7 @@
{% if superlatives.controversial %} {% if superlatives.controversial %}
{% with book=superlatives.controversial.default_edition %} {% with book=superlatives.controversial.default_edition %}
<div class="column is-one-third is-flex"> <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
<div class="media notification is-clipped"> <div class="media notification is-clipped">
<div class="media-left"> <div class="media-left">
<a href="{{ book.local_path }}"> <a href="{{ book.local_path }}">

View file

@ -23,7 +23,7 @@
<div class="notification is-warning"> <div class="notification is-warning">
<p> <p>
{% id_to_username request.user.moved_to as username %} {% id_to_username request.user.moved_to as username %}
{% blocktrans trimmed with moved_to=user.moved_to %} {% blocktrans trimmed with moved_to=user.moved_to %}
<strong>You have moved your account</strong> to <a href="{{ moved_to }}">{{ username }}</a> <strong>You have moved your account</strong> to <a href="{{ moved_to }}">{{ username }}</a>
{% endblocktrans %} {% endblocktrans %}

View file

@ -14,7 +14,7 @@
{% block description %} {% block description %}
{% if related_user_moved_to %} {% if related_user_moved_to %}
{% id_to_username request.user.moved_to as username %} {% id_to_username related_user_moved_to as username %}
{% blocktrans trimmed %} {% blocktrans trimmed %}
{{ related_user }} has moved to <a href="{{ related_user_moved_to }}">{{ username }}</a> {{ related_user }} has moved to <a href="{{ related_user_moved_to }}">{{ username }}</a>
{% endblocktrans %} {% endblocktrans %}

View file

@ -46,7 +46,11 @@
{% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an <strong>alias</strong> of this one, or <strong>move</strong> this account to the new account, before you import your user data." %} {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an <strong>alias</strong> of this one, or <strong>move</strong> this account to the new account, before you import your user data." %}
{% endspaceless %} {% endspaceless %}
</p> </p>
{% if next_available %} {% if not site.user_exports_enabled %}
<p class="notification is-danger">
{% trans "New user exports are currently disabled." %}
</p>
{% elif next_available %}
<p class="notification is-warning"> <p class="notification is-warning">
{% blocktrans trimmed %} {% blocktrans trimmed %}
You will be able to create a new export file at {{ next_available }} You will be able to create a new export file at {{ next_available }}

View file

@ -90,6 +90,33 @@
</div> </div>
</form> </form>
</details> </details>
{% if site.user_exports_enabled %}
<details class="details-panel box">
<summary>
<span role="heading" aria-level="2" class="title is-6">
{% trans "Disable starting new user exports" %}
</span>
<span class="details-close icon icon-x" aria-hidden="true"></span>
</summary>
<form
name="disable-user-exports"
id="disable-user-exports"
method="POST"
action="{% url 'settings-user-exports-disable' %}"
>
<div class="notification">
{% trans "This is only intended to be used when things have gone very wrong with exports and you need to pause the feature while addressing issues." %}
{% trans "While exports are disabled, users will not be allowed to start new user exports, but existing exports will not be affected." %}
</div>
{% csrf_token %}
<div class="control">
<button type="submit" class="button is-danger">
{% trans "Disable user exports" %}
</button>
</div>
</form>
</details>
<details class="details-panel box"> <details class="details-panel box">
<summary> <summary>
<span role="heading" aria-level="2" class="title is-6"> <span role="heading" aria-level="2" class="title is-6">
@ -108,7 +135,7 @@
{% trans "Set the value to 0 to not enforce any limit." %} {% trans "Set the value to 0 to not enforce any limit." %}
</div> </div>
<div class="align.to-t"> <div class="align.to-t">
<label for="limit">{% trans "Restrict user imports and exports to once every " %}</label> <label for="limit">{% trans "Limit how often users can import and export user data" %}</label>
<input name="limit" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ user_import_time_limit }}"> <input name="limit" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ user_import_time_limit }}">
<label>{% trans "hours" %}</label> <label>{% trans "hours" %}</label>
{% csrf_token %} {% csrf_token %}
@ -120,6 +147,28 @@
</div> </div>
</form> </form>
</details> </details>
{% else %}
<form
name="enable-user-imports"
id="enable-user-imports"
method="POST"
action="{% url 'settings-user-exports-enable' %}"
class="box"
>
<div class="notification is-danger is-light">
<p class="my-2">{% trans "Users are currently unable to start new user exports. This is the default setting." %}</p>
{% if use_s3 %}
<p>{% trans "It is not currently possible to provide user exports when using s3 storage. The BookWyrm development team are working on a fix for this." %}</p>
{% endif %}
</div>
{% csrf_token %}
<div class="control">
<button type="submit" class="button is-success" {% if use_s3 %}disabled{% endif %}>
{% trans "Enable user exports" %}
</button>
</div>
</form>
{% endif %}
</div> </div>
<div class="block"> <div class="block">
<h4 class="title is-4">{% trans "Book Imports" %}</h4> <h4 class="title is-4">{% trans "Book Imports" %}</h4>

View file

@ -125,7 +125,8 @@ def id_to_username(user_id):
name = parts[-1] name = parts[-1]
value = f"{name}@{domain}" value = f"{name}@{domain}"
return value return value
return "a new user account"
@register.filter(name="get_file_size") @register.filter(name="get_file_size")

View file

@ -18,7 +18,9 @@ class ExportViews(TestCase):
"""viewing and creating statuses""" """viewing and creating statuses"""
@classmethod @classmethod
def setUpTestData(self): # pylint: disable=bad-classmethod-argument def setUpTestData(
self,
): # pylint: disable=bad-classmethod-argument, disable=invalid-name
"""we need basic test data and mocks""" """we need basic test data and mocks"""
with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
"bookwyrm.activitystreams.populate_stream_task.delay" "bookwyrm.activitystreams.populate_stream_task.delay"
@ -40,6 +42,7 @@ class ExportViews(TestCase):
bnf_id="beep", bnf_id="beep",
) )
# pylint: disable=invalid-name
def setUp(self): def setUp(self):
"""individual test setup""" """individual test setup"""
self.factory = RequestFactory() self.factory = RequestFactory()
@ -53,11 +56,12 @@ class ExportViews(TestCase):
def test_export_file(self, *_): def test_export_file(self, *_):
"""simple export""" """simple export"""
models.ShelfBook.objects.create( shelfbook = models.ShelfBook.objects.create(
shelf=self.local_user.shelf_set.first(), shelf=self.local_user.shelf_set.first(),
user=self.local_user, user=self.local_user,
book=self.book, book=self.book,
) )
book_date = str.encode(f"{shelfbook.shelved_date.date()}")
request = self.factory.post("") request = self.factory.post("")
request.user = self.local_user request.user = self.local_user
export = views.Export.as_view()(request) export = views.Export.as_view()(request)
@ -66,7 +70,7 @@ class ExportViews(TestCase):
# pylint: disable=line-too-long # pylint: disable=line-too-long
self.assertEqual( self.assertEqual(
export.content, export.content,
b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content\r\nTest Book,," b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date\r\n"
+ self.book.remote_id.encode("utf-8") + b"Test Book,,%b,,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,,,to-read,To Read,%b\r\n"
+ b",,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,\r\n", % (self.book.remote_id.encode("utf-8"), book_date),
) )

View file

@ -338,6 +338,16 @@ urlpatterns = [
views.disable_imports, views.disable_imports,
name="settings-imports-disable", name="settings-imports-disable",
), ),
re_path(
r"^settings/user-exports/enable/?$",
views.enable_user_exports,
name="settings-user-exports-enable",
),
re_path(
r"^settings/user-exports/disable/?$",
views.disable_user_exports,
name="settings-user-exports-disable",
),
re_path( re_path(
r"^settings/imports/enable/?$", r"^settings/imports/enable/?$",
views.enable_imports, views.enable_imports,

View file

@ -18,6 +18,8 @@ from .admin.imports import (
set_import_size_limit, set_import_size_limit,
set_user_import_completed, set_user_import_completed,
set_user_import_limit, set_user_import_limit,
enable_user_exports,
disable_user_exports,
) )
from .admin.ip_blocklist import IPBlocklist from .admin.ip_blocklist import IPBlocklist
from .admin.invite import ManageInvites, Invite, InviteRequest from .admin.invite import ManageInvites, Invite, InviteRequest

View file

@ -9,7 +9,7 @@ from django.views.decorators.http import require_POST
from bookwyrm import models from bookwyrm import models
from bookwyrm.views.helpers import redirect_to_referer from bookwyrm.views.helpers import redirect_to_referer
from bookwyrm.settings import PAGE_LENGTH from bookwyrm.settings import PAGE_LENGTH, USE_S3
# pylint: disable=no-self-use # pylint: disable=no-self-use
@ -59,6 +59,7 @@ class ImportList(View):
"import_size_limit": site_settings.import_size_limit, "import_size_limit": site_settings.import_size_limit,
"import_limit_reset": site_settings.import_limit_reset, "import_limit_reset": site_settings.import_limit_reset,
"user_import_time_limit": site_settings.user_import_time_limit, "user_import_time_limit": site_settings.user_import_time_limit,
"use_s3": USE_S3,
} }
return TemplateResponse(request, "settings/imports/imports.html", data) return TemplateResponse(request, "settings/imports/imports.html", data)
@ -126,3 +127,25 @@ def set_user_import_limit(request):
site.user_import_time_limit = int(request.POST.get("limit")) site.user_import_time_limit = int(request.POST.get("limit"))
site.save(update_fields=["user_import_time_limit"]) site.save(update_fields=["user_import_time_limit"])
return redirect("settings-imports") return redirect("settings-imports")
@require_POST
@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
# pylint: disable=unused-argument
def enable_user_exports(request):
"""Allow users to export account data"""
site = models.SiteSettings.objects.get()
site.user_exports_enabled = True
site.save(update_fields=["user_exports_enabled"])
return redirect("settings-imports")
@require_POST
@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
# pylint: disable=unused-argument
def disable_user_exports(request):
"""Don't allow users to export account data"""
site = models.SiteSettings.objects.get()
site.user_exports_enabled = False
site.save(update_fields=["user_exports_enabled"])
return redirect("settings-imports")

View file

@ -17,6 +17,7 @@ from bookwyrm import models
from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob
from bookwyrm.settings import PAGE_LENGTH from bookwyrm.settings import PAGE_LENGTH
# pylint: disable=no-self-use,too-many-locals # pylint: disable=no-self-use,too-many-locals
@method_decorator(login_required, name="dispatch") @method_decorator(login_required, name="dispatch")
class Export(View): class Export(View):
@ -54,8 +55,19 @@ class Export(View):
fields = ( fields = (
["title", "author_text"] ["title", "author_text"]
+ deduplication_fields + deduplication_fields
+ ["start_date", "finish_date", "stopped_date"] + [
+ ["rating", "review_name", "review_cw", "review_content"] "start_date",
"finish_date",
"stopped_date",
"rating",
"review_name",
"review_cw",
"review_content",
"review_published",
"shelf",
"shelf_name",
"shelf_date",
]
) )
writer.writerow(fields) writer.writerow(fields)
@ -97,9 +109,27 @@ class Export(View):
.first() .first()
) )
if review: if review:
book.review_published = (
review.published_date.date() if review.published_date else None
)
book.review_name = review.name book.review_name = review.name
book.review_cw = review.content_warning book.review_cw = review.content_warning
book.review_content = review.raw_content book.review_content = (
review.raw_content if review.raw_content else review.content
) # GoodReads imported reviews do not have raw_content, but content.
shelfbook = (
models.ShelfBook.objects.filter(user=request.user, book=book)
.order_by("-shelved_date", "-created_date", "-updated_date")
.last()
)
if shelfbook:
book.shelf = shelfbook.shelf.identifier
book.shelf_name = shelfbook.shelf.name
book.shelf_date = (
shelfbook.shelved_date.date() if shelfbook.shelved_date else None
)
writer.writerow([getattr(book, field, "") or "" for field in fields]) writer.writerow([getattr(book, field, "") or "" for field in fields])
return HttpResponse( return HttpResponse(

View file

@ -1,4 +1,4 @@
FROM python:3.9 FROM python:3.9-bookworm
WORKDIR /app/dev-tools WORKDIR /app/dev-tools
ENV PATH="/app/dev-tools/node_modules/.bin:$PATH" ENV PATH="/app/dev-tools/node_modules/.bin:$PATH"

View file

@ -64,13 +64,18 @@ server {
# directly serve images and static files from the # directly serve images and static files from the
# bookwyrm filesystem using sendfile. # bookwyrm filesystem using sendfile.
# make the logs quieter by not reporting these requests # make the logs quieter by not reporting these requests
location ~ ^/(images|static)/ { location ~ \.(bmp|ico|jpg|jpeg|png|tif|tiff|webp|css|js)$ {
root /app; root /app;
try_files $uri =404; try_files $uri =404;
add_header X-Cache-Status STATIC; add_header X-Cache-Status STATIC;
access_log off; access_log off;
} }
# block access to any non-image files from images or static
location ~ ^/images/ {
return 403;
}
# monitor the celery queues with flower, no caching enabled # monitor the celery queues with flower, no caching enabled
location /flower/ { location /flower/ {
proxy_pass http://flower:8888; proxy_pass http://flower:8888;

View file

@ -96,12 +96,17 @@ server {
# # directly serve images and static files from the # # directly serve images and static files from the
# # bookwyrm filesystem using sendfile. # # bookwyrm filesystem using sendfile.
# # make the logs quieter by not reporting these requests # # make the logs quieter by not reporting these requests
# location ~ ^/(images|static)/ { # location ~ \.(bmp|ico|jpg|jpeg|png|tif|tiff|webp|css|js)$ {
# root /app; # root /app;
# try_files $uri =404; # try_files $uri =404;
# add_header X-Cache-Status STATIC; # add_header X-Cache-Status STATIC;
# access_log off; # access_log off;
# } # }
# # block access to any non-image files from images or static
# location ~ ^/images/ {
# return 403;
# }
# #
# # monitor the celery queues with flower, no caching enabled # # monitor the celery queues with flower, no caching enabled
# location /flower/ { # location /flower/ {