Merge branch 'main' into production

This commit is contained in:
Mouse Reeve 2023-10-02 10:23:13 -07:00
commit 1093e95de7
88 changed files with 8249 additions and 5385 deletions

View file

@ -4,7 +4,11 @@ import sys
from .base_activity import ActivityEncoder, Signature, naive_parse
from .base_activity import Link, Mention, Hashtag
from .base_activity import ActivitySerializerError, resolve_remote_id
from .base_activity import (
ActivitySerializerError,
resolve_remote_id,
get_representative,
)
from .image import Document, Image
from .note import Note, GeneratedNote, Article, Comment, Quotation
from .note import Review, Rating

View file

@ -1,4 +1,5 @@
""" basics for an activitypub serializer """
from __future__ import annotations
from dataclasses import dataclass, fields, MISSING
from json import JSONEncoder
import logging
@ -72,8 +73,10 @@ class ActivityObject:
def __init__(
self,
activity_objects: Optional[list[str, base_model.BookWyrmModel]] = None,
**kwargs: dict[str, Any],
activity_objects: Optional[
dict[str, Union[str, list[str], ActivityObject, base_model.BookWyrmModel]]
] = None,
**kwargs: Any,
):
"""this lets you pass in an object with fields that aren't in the
dataclass, which it ignores. Any field in the dataclass is required or

View file

@ -112,7 +112,7 @@ class ActivityStream(RedisStore):
trace.get_current_span().set_attribute("status_privacy", status.privacy)
trace.get_current_span().set_attribute(
"status_reply_parent_privacy",
status.reply_parent.privacy if status.reply_parent else None,
status.reply_parent.privacy if status.reply_parent else status.privacy,
)
# direct messages don't appear in feeds, direct comments/reviews/etc do
if status.privacy == "direct" and status.status_type == "Note":

View file

@ -1,4 +1,6 @@
""" handle reading a csv from calibre """
from typing import Any, Optional
from bookwyrm.models import Shelf
from . import Importer
@ -9,7 +11,7 @@ class CalibreImporter(Importer):
service = "Calibre"
def __init__(self, *args, **kwargs):
def __init__(self, *args: Any, **kwargs: Any):
# Add timestamp to row_mappings_guesses for date_added to avoid
# integrity error
row_mappings_guesses = []
@ -23,6 +25,6 @@ class CalibreImporter(Importer):
self.row_mappings_guesses = row_mappings_guesses
super().__init__(*args, **kwargs)
def get_shelf(self, normalized_row):
def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
# Calibre export does not indicate which shelf to use. Use a default one for now
return Shelf.TO_READ

View file

@ -1,8 +1,10 @@
""" handle reading a csv from an external service, defaults are from Goodreads """
import csv
from datetime import timedelta
from typing import Iterable, Optional
from django.utils import timezone
from bookwyrm.models import ImportJob, ImportItem, SiteSettings
from bookwyrm.models import ImportJob, ImportItem, SiteSettings, User
class Importer:
@ -35,19 +37,26 @@ class Importer:
}
# pylint: disable=too-many-locals
def create_job(self, user, csv_file, include_reviews, privacy):
def create_job(
self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str
) -> ImportJob:
"""check over a csv and creates a database entry for the job"""
csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
rows = list(csv_reader)
if len(rows) < 1:
raise ValueError("CSV file is empty")
rows = enumerate(rows)
mappings = (
self.create_row_mappings(list(fieldnames))
if (fieldnames := csv_reader.fieldnames)
else {}
)
job = ImportJob.objects.create(
user=user,
include_reviews=include_reviews,
privacy=privacy,
mappings=self.create_row_mappings(csv_reader.fieldnames),
mappings=mappings,
source=self.service,
)
@ -55,16 +64,20 @@ class Importer:
if enforce_limit and allowed_imports <= 0:
job.complete_job()
return job
for index, entry in rows:
for index, entry in enumerate(rows):
if enforce_limit and index >= allowed_imports:
break
self.create_item(job, index, entry)
return job
def update_legacy_job(self, job):
def update_legacy_job(self, job: ImportJob) -> None:
"""patch up a job that was in the old format"""
items = job.items
headers = list(items.first().data.keys())
first_item = items.first()
if first_item is None:
return
headers = list(first_item.data.keys())
job.mappings = self.create_row_mappings(headers)
job.updated_date = timezone.now()
job.save()
@ -75,24 +88,24 @@ class Importer:
item.normalized_data = normalized
item.save()
def create_row_mappings(self, headers):
def create_row_mappings(self, headers: list[str]) -> dict[str, Optional[str]]:
"""guess what the headers mean"""
mappings = {}
for (key, guesses) in self.row_mappings_guesses:
value = [h for h in headers if h.lower() in guesses]
value = value[0] if len(value) else None
values = [h for h in headers if h.lower() in guesses]
value = values[0] if len(values) else None
if value:
headers.remove(value)
mappings[key] = value
return mappings
def create_item(self, job, index, data):
def create_item(self, job: ImportJob, index: int, data: dict[str, str]) -> None:
"""creates and saves an import item"""
normalized = self.normalize_row(data, job.mappings)
normalized["shelf"] = self.get_shelf(normalized)
ImportItem(job=job, index=index, data=data, normalized_data=normalized).save()
def get_shelf(self, normalized_row):
def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
"""determine which shelf to use"""
shelf_name = normalized_row.get("shelf")
if not shelf_name:
@ -103,11 +116,15 @@ class Importer:
]
return shelf[0] if shelf else None
def normalize_row(self, entry, mappings): # pylint: disable=no-self-use
# pylint: disable=no-self-use
def normalize_row(
self, entry: dict[str, str], mappings: dict[str, Optional[str]]
) -> dict[str, Optional[str]]:
"""use the dataclass to create the formatted row of data"""
return {k: entry.get(v) for k, v in mappings.items()}
return {k: entry.get(v) if v else None for k, v in mappings.items()}
def get_import_limit(self, user): # pylint: disable=no-self-use
# pylint: disable=no-self-use
def get_import_limit(self, user: User) -> tuple[int, int]:
"""check if import limit is set and return how many imports are left"""
site_settings = SiteSettings.objects.get()
import_size_limit = site_settings.import_size_limit
@ -125,7 +142,9 @@ class Importer:
allowed_imports = import_size_limit - imported_books
return enforce_limit, allowed_imports
def create_retry_job(self, user, original_job, items):
def create_retry_job(
self, user: User, original_job: ImportJob, items: list[ImportItem]
) -> ImportJob:
"""retry items that didn't import"""
job = ImportJob.objects.create(
user=user,

View file

@ -1,11 +1,16 @@
""" handle reading a tsv from librarything """
import re
from typing import Optional
from bookwyrm.models import Shelf
from . import Importer
def _remove_brackets(value: Optional[str]) -> Optional[str]:
return re.sub(r"\[|\]", "", value) if value else None
class LibrarythingImporter(Importer):
"""csv downloads from librarything"""
@ -13,16 +18,19 @@ class LibrarythingImporter(Importer):
delimiter = "\t"
encoding = "ISO-8859-1"
def normalize_row(self, entry, mappings): # pylint: disable=no-self-use
def normalize_row(
self, entry: dict[str, str], mappings: dict[str, Optional[str]]
) -> dict[str, Optional[str]]: # pylint: disable=no-self-use
"""use the dataclass to create the formatted row of data"""
remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None
normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()}
isbn_13 = normalized.get("isbn_13")
isbn_13 = isbn_13.split(", ") if isbn_13 else []
normalized = {
k: _remove_brackets(entry.get(v) if v else None)
for k, v in mappings.items()
}
isbn_13 = value.split(", ") if (value := normalized.get("isbn_13")) else []
normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 1 else None
return normalized
def get_shelf(self, normalized_row):
def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
if normalized_row["date_finished"]:
return Shelf.READ_FINISHED
if normalized_row["date_started"]:

View file

@ -1,4 +1,6 @@
""" handle reading a csv from openlibrary"""
from typing import Any
from . import Importer
@ -7,7 +9,7 @@ class OpenLibraryImporter(Importer):
service = "OpenLibrary"
def __init__(self, *args, **kwargs):
def __init__(self, *args: Any, **kwargs: Any):
self.row_mappings_guesses.append(("openlibrary_key", ["edition id"]))
self.row_mappings_guesses.append(("openlibrary_work_key", ["work id"]))
super().__init__(*args, **kwargs)

View file

@ -1,11 +1,20 @@
""" Use the range message from isbn-international to hyphenate ISBNs """
import os
from typing import Optional
from xml.etree import ElementTree
from xml.etree.ElementTree import Element
import requests
from bookwyrm import settings
def _get_rules(element: Element) -> list[Element]:
if (rules_el := element.find("Rules")) is not None:
return rules_el.findall("Rule")
return []
class IsbnHyphenator:
"""Class to manage the range message xml file and use it to hyphenate ISBNs"""
@ -15,58 +24,94 @@ class IsbnHyphenator:
)
__element_tree = None
def update_range_message(self):
def update_range_message(self) -> None:
"""Download the range message xml file and save it locally"""
response = requests.get(self.__range_message_url)
with open(self.__range_file_path, "w", encoding="utf-8") as file:
file.write(response.text)
self.__element_tree = None
def hyphenate(self, isbn_13):
def hyphenate(self, isbn_13: Optional[str]) -> Optional[str]:
"""hyphenate the given ISBN-13 number using the range message"""
if isbn_13 is None:
return None
if self.__element_tree is None:
self.__element_tree = ElementTree.parse(self.__range_file_path)
gs1_prefix = isbn_13[:3]
reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
if reg_group is None:
return isbn_13 # failed to hyphenate
registrant = self.__find_registrant(isbn_13, gs1_prefix, reg_group)
if registrant is None:
return isbn_13 # failed to hyphenate
publication = isbn_13[len(gs1_prefix) + len(reg_group) + len(registrant) : -1]
check_digit = isbn_13[-1:]
return "-".join((gs1_prefix, reg_group, registrant, publication, check_digit))
def __find_reg_group(self, isbn_13, gs1_prefix):
for ean_ucc_el in self.__element_tree.find("EAN.UCCPrefixes").findall(
"EAN.UCC"
):
if ean_ucc_el.find("Prefix").text == gs1_prefix:
for rule_el in ean_ucc_el.find("Rules").findall("Rule"):
length = int(rule_el.find("Length").text)
def __find_reg_group(self, isbn_13: str, gs1_prefix: str) -> Optional[str]:
if self.__element_tree is None:
self.__element_tree = ElementTree.parse(self.__range_file_path)
ucc_prefixes_el = self.__element_tree.find("EAN.UCCPrefixes")
if ucc_prefixes_el is None:
return None
for ean_ucc_el in ucc_prefixes_el.findall("EAN.UCC"):
if (
prefix_el := ean_ucc_el.find("Prefix")
) is not None and prefix_el.text == gs1_prefix:
for rule_el in _get_rules(ean_ucc_el):
length_el = rule_el.find("Length")
if length_el is None:
continue
length = int(text) if (text := length_el.text) else 0
if length == 0:
continue
reg_grp_range = [
int(x[:length]) for x in rule_el.find("Range").text.split("-")
]
range_el = rule_el.find("Range")
if range_el is None or range_el.text is None:
continue
reg_grp_range = [int(x[:length]) for x in range_el.text.split("-")]
reg_group = isbn_13[len(gs1_prefix) : len(gs1_prefix) + length]
if reg_grp_range[0] <= int(reg_group) <= reg_grp_range[1]:
return reg_group
return None
return None
def __find_registrant(self, isbn_13, gs1_prefix, reg_group):
def __find_registrant(
self, isbn_13: str, gs1_prefix: str, reg_group: str
) -> Optional[str]:
from_ind = len(gs1_prefix) + len(reg_group)
for group_el in self.__element_tree.find("RegistrationGroups").findall("Group"):
if group_el.find("Prefix").text == "-".join((gs1_prefix, reg_group)):
for rule_el in group_el.find("Rules").findall("Rule"):
length = int(rule_el.find("Length").text)
if self.__element_tree is None:
self.__element_tree = ElementTree.parse(self.__range_file_path)
reg_groups_el = self.__element_tree.find("RegistrationGroups")
if reg_groups_el is None:
return None
for group_el in reg_groups_el.findall("Group"):
if (
prefix_el := group_el.find("Prefix")
) is not None and prefix_el.text == "-".join((gs1_prefix, reg_group)):
for rule_el in _get_rules(group_el):
length_el = rule_el.find("Length")
if length_el is None:
continue
length = int(text) if (text := length_el.text) else 0
if length == 0:
continue
range_el = rule_el.find("Range")
if range_el is None or range_el.text is None:
continue
registrant_range = [
int(x[:length]) for x in rule_el.find("Range").text.split("-")
int(x[:length]) for x in range_el.text.split("-")
]
registrant = isbn_13[from_ind : from_ind + length]
if registrant_range[0] <= int(registrant) <= registrant_range[1]:

View file

@ -1,5 +1,7 @@
""" database schema for info about authors """
import re
from typing import Tuple, Any
from django.contrib.postgres.indexes import GinIndex
from django.db import models
@ -38,7 +40,7 @@ class Author(BookDataModel):
)
bio = fields.HtmlField(null=True, blank=True)
def save(self, *args, **kwargs):
def save(self, *args: Tuple[Any, ...], **kwargs: dict[str, Any]) -> None:
"""normalize isni format"""
if self.isni:
self.isni = re.sub(r"\s", "", self.isni)

View file

@ -217,6 +217,13 @@ class Book(BookDataModel):
"""editions and works both use "book" instead of model_name"""
return f"https://{DOMAIN}/book/{self.id}"
def guess_sort_title(self):
"""Get a best-guess sort title for the current book"""
articles = chain(
*(LANGUAGE_ARTICLES.get(language, ()) for language in tuple(self.languages))
)
return re.sub(f'^{" |^".join(articles)} ', "", str(self.title).lower())
def __repr__(self):
# pylint: disable=consider-using-f-string
return "<{} key={!r} title={!r}>".format(
@ -374,16 +381,7 @@ class Edition(Book):
# Create sort title by removing articles from title
if self.sort_title in [None, ""]:
if self.sort_title in [None, ""]:
articles = chain(
*(
LANGUAGE_ARTICLES.get(language, ())
for language in tuple(self.languages)
)
)
self.sort_title = re.sub(
f'^{" |^".join(articles)} ', "", str(self.title).lower()
)
self.sort_title = self.guess_sort_title()
return super().save(*args, **kwargs)

View file

@ -54,10 +54,10 @@ ImportStatuses = [
class ImportJob(models.Model):
"""entry for a specific request for book data import"""
user = models.ForeignKey(User, on_delete=models.CASCADE)
user: User = models.ForeignKey(User, on_delete=models.CASCADE)
created_date = models.DateTimeField(default=timezone.now)
updated_date = models.DateTimeField(default=timezone.now)
include_reviews = models.BooleanField(default=True)
include_reviews: bool = models.BooleanField(default=True)
mappings = models.JSONField()
source = models.CharField(max_length=100)
privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels)
@ -76,7 +76,7 @@ class ImportJob(models.Model):
self.save(update_fields=["task_id"])
def complete_job(self):
def complete_job(self) -> None:
"""Report that the job has completed"""
self.status = "complete"
self.complete = True

View file

@ -1,5 +1,6 @@
""" models for storing different kinds of Activities """
from dataclasses import MISSING
from typing import Optional
import re
from django.apps import apps
@ -269,7 +270,7 @@ class GeneratedNote(Status):
"""indicate the book in question for mastodon (or w/e) users"""
message = self.content
books = ", ".join(
f'<a href="{book.remote_id}">"{book.title}"</a>'
f'<a href="{book.remote_id}"><i>{book.title}</i></a>'
for book in self.mention_books.all()
)
return f"{self.user.display_name} {message} {books}"
@ -320,17 +321,14 @@ class Comment(BookStatus):
@property
def pure_content(self):
"""indicate the book in question for mastodon (or w/e) users"""
if self.progress_mode == "PG" and self.progress and (self.progress > 0):
return_value = (
f'{self.content}<p>(comment on <a href="{self.book.remote_id}">'
f'"{self.book.title}"</a>, page {self.progress})</p>'
)
else:
return_value = (
f'{self.content}<p>(comment on <a href="{self.book.remote_id}">'
f'"{self.book.title}"</a>)</p>'
)
return return_value
progress = self.progress or 0
citation = (
f'comment on <a href="{self.book.remote_id}">'
f"<i>{self.book.title}</i></a>"
)
if self.progress_mode == "PG" and progress > 0:
citation += f", p. {progress}"
return f"{self.content}<p>({citation})</p>"
activity_serializer = activitypub.Comment
@ -354,22 +352,24 @@ class Quotation(BookStatus):
blank=True,
)
def _format_position(self) -> Optional[str]:
"""serialize page position"""
beg = self.position
end = self.endposition or 0
if self.position_mode != "PG" or not beg:
return None
return f"pp. {beg}-{end}" if end > beg else f"p. {beg}"
@property
def pure_content(self):
"""indicate the book in question for mastodon (or w/e) users"""
quote = re.sub(r"^<p>", '<p>"', self.quote)
quote = re.sub(r"</p>$", '"</p>', quote)
if self.position_mode == "PG" and self.position and (self.position > 0):
return_value = (
f'{quote} <p>-- <a href="{self.book.remote_id}">'
f'"{self.book.title}"</a>, page {self.position}</p>{self.content}'
)
else:
return_value = (
f'{quote} <p>-- <a href="{self.book.remote_id}">'
f'"{self.book.title}"</a></p>{self.content}'
)
return return_value
title, href = self.book.title, self.book.remote_id
citation = f'— <a href="{href}"><i>{title}</i></a>'
if position := self._format_position():
citation += f", {position}"
return f"{quote} <p>{citation}</p>{self.content}"
activity_serializer = activitypub.Quotation

View file

@ -1,5 +1,7 @@
""" bookwyrm settings and configuration """
import os
from typing import AnyStr
from environs import Env
import requests
@ -12,7 +14,7 @@ from django.core.exceptions import ImproperlyConfigured
env = Env()
env.read_env()
DOMAIN = env("DOMAIN")
VERSION = "0.6.4"
VERSION = "0.6.6"
RELEASE_API = env(
"RELEASE_API",
@ -22,7 +24,7 @@ RELEASE_API = env(
PAGE_LENGTH = env.int("PAGE_LENGTH", 15)
DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English")
JS_CACHE = "b972a43c"
JS_CACHE = "ac315a3b"
# email
EMAIL_BACKEND = env("EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend")
@ -37,7 +39,7 @@ EMAIL_SENDER_DOMAIN = env("EMAIL_SENDER_DOMAIN", DOMAIN)
EMAIL_SENDER = f"{EMAIL_SENDER_NAME}@{EMAIL_SENDER_DOMAIN}"
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BASE_DIR: AnyStr = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LOCALE_PATHS = [
os.path.join(BASE_DIR, "locale"),
]
@ -315,6 +317,7 @@ LANGUAGES = [
LANGUAGE_ARTICLES = {
"English": {"the", "a", "an"},
"Español (Spanish)": {"un", "una", "unos", "unas", "el", "la", "los", "las"},
}
TIME_ZONE = "UTC"

View file

@ -106,7 +106,7 @@ const tries = {
e: {
p: {
u: {
b: "ePub",
b: "EPUB",
},
},
},

View file

@ -254,7 +254,8 @@ def rerank_suggestions_task(user_id):
def rerank_user_task(user_id, update_only=False):
"""do the hard work in celery"""
user = models.User.objects.get(id=user_id)
suggested_users.rerank_obj(user, update_only=update_only)
if user:
suggested_users.rerank_obj(user, update_only=update_only)
@app.task(queue=SUGGESTED_USERS)

View file

@ -10,7 +10,7 @@
{% csrf_token %}
<input type="hidden" name="last_edited_by" value="{{ request.user.id }}">
{% if form.parent_work %}
{% if book.parent_work.id or form.parent_work %}
<input type="hidden" name="parent_work" value="{% firstof book.parent_work.id form.parent_work %}">
{% endif %}

View file

@ -5,7 +5,7 @@
{% block content %}
<div class="block">
<h1 class="title">{% blocktrans with work_path=work.local_path work_title=work|book_title %}Editions of <a href="{{ work_path }}">"{{ work_title }}"</a>{% endblocktrans %}</h1>
<h1 class="title">{% blocktrans with work_path=work.local_path work_title=work|book_title %}Editions of <a href="{{ work_path }}"><i>{{ work_title }}</i></a>{% endblocktrans %}</h1>
</div>
{% include 'book/editions/edition_filters.html' %}

View file

@ -35,7 +35,7 @@
required=""
id="id_filetype"
value="{% firstof file_link_form.filetype.value '' %}"
placeholder="ePub"
placeholder="EPUB"
list="mimetypes-list"
data-autocomplete="mimetype"
>

View file

@ -4,11 +4,11 @@
{% block filter %}
<legend class="label">{% trans "Community" %}</legend>
<label class="is-block">
<input type="radio" class="radio" name="scope" value="local" {% if request.GET.scope == "local" %}checked{% endif %}>
<input type="radio" class="radio" name="scope" value="local" {% if scope == "local" %}checked{% endif %}>
{% trans "Local users" %}
</label>
<label class="is-block">
<input type="radio" class="radio" name="scope" value="federated" {% if request.GET.scope == "federated" %}checked{% endif %}>
<input type="radio" class="radio" name="scope" value="federated" {% if scope == "federated" %}checked{% endif %}>
{% trans "Federated community" %}
</label>
{% endblock %}

View file

@ -6,8 +6,8 @@
<div class="control">
<div class="select">
<select name="sort" id="id_sort">
<option value="recent" {% if request.GET.sort == "recent" %}selected{% endif %}>{% trans "Recently active" %}</option>
<option value="suggested" {% if request.GET.sort == "suggested" %}selected{% endif %}>{% trans "Suggested" %}</option>
<option value="recent" {% if sort == "recent" %}selected{% endif %}>{% trans "Recently active" %}</option>
<option value="suggested" {% if sort == "suggested" %}selected{% endif %}>{% trans "Suggested" %}</option>
</select>
</div>
</div>

View file

@ -31,5 +31,7 @@
</div>
</div>
</div>
{% empty %}
<p class="column"><em>{% trans "No groups found." %}</em></p>
{% endfor %}
</div>

View file

@ -18,7 +18,7 @@
{% if import_size_limit and import_limit_reset %}
<div class="notification">
<p>
{% blocktrans count days=import_limit_reset with display_size=import_size_limit|intcomma %}
{% blocktrans trimmed count days=import_limit_reset with display_size=import_size_limit|intcomma %}
Currently, you are allowed to import {{ display_size }} books every {{ import_limit_reset }} day.
{% plural %}
Currently, you are allowed to import {{ import_size_limit }} books every {{ import_limit_reset }} days.

View file

@ -46,5 +46,7 @@
</div>
</div>
</div>
{% empty %}
<p class="column"><em>{% trans "No lists found." %}</em></p>
{% endfor %}
</div>

View file

@ -43,7 +43,6 @@
</nav>
{% endif %}
{% if lists %}
<section class="block">
{% include 'lists/list_items.html' with lists=lists %}
</section>
@ -51,7 +50,6 @@
<div>
{% include 'snippets/pagination.html' with page=lists path=path %}
</div>
{% endif %}
{% endblock %}

View file

@ -3,14 +3,13 @@
xmlns="http://a9.com/-/spec/opensearch/1.1/"
xmlns:moz="http://www.mozilla.org/2006/browser/search/"
>
<ShortName>{{ site_name }}</ShortName>
<ShortName>{{ site.name }}</ShortName>
<Description>{% blocktrans trimmed with site_name=site.name %}
{{ site_name }} search
{% endblocktrans %}</Description>
<Image width="16" height="16" type="image/x-icon">{{ image }}</Image>
<Url
type="text/html"
method="get"
template="https://{{ DOMAIN }}{% url 'search' %}?q={searchTerms}"
/>
</OpenSearchDescription>

View file

@ -75,13 +75,13 @@
{% include 'snippets/form_errors.html' with errors_list=form.invite_request_text.errors id="desc_invite_request_text" %}
</div>
<div class="field">
<label class="label" for="id_invite_requests_question">
<label class="label">
{{ form.invite_request_question }}
{% trans "Set a question for invite requests" %}
</label>
</div>
<div class="field">
<label class="label" for="id_invite_question_text">
<label class="label">
{% trans "Question:" %}
{{ form.invite_question_text }}
</label>

View file

@ -45,7 +45,7 @@
{% include 'snippets/form_errors.html' with errors_list=form.invite_request_text.errors id="desc_invite_request_text" %}
</div>
<div class="field">
<label class="label" for="id_invite_requests_question">
<label class="label">
{{ form.invite_request_question }}
{% trans "Set a question for invite requests" %}
</label>

View file

@ -1,6 +1,10 @@
{% extends 'user/layout.html' %}
{% load i18n %}
{% load utilities %}
{% load i18n %}
{% block title %}
{% trans "Reading Goal" %} - {{ user|username }}
{% endblock %}
{% block header %}
<div class="columns is-mobile">

View file

@ -1,6 +1,11 @@
{% extends 'user/layout.html' %}
{% load utilities %}
{% load i18n %}
{% block title %}
{% trans "Groups" %} - {{ user|username }}
{% endblock %}
{% block header %}
<div class="columns is-mobile">
<div class="column">

View file

@ -1,6 +1,11 @@
{% extends 'user/layout.html' %}
{% load utilities %}
{% load i18n %}
{% block title %}
{% trans "Lists" %} - {{ user|username }}
{% endblock %}
{% block header %}
<div class="columns is-mobile">
<div class="column">

View file

@ -2,7 +2,9 @@
{% load i18n %}
{% load utilities %}
{% block title %}{{ user.display_name }}{% endblock %}
{% block title %}
{% trans "Reviews and Comments" %} - {{ user|username }}
{% endblock %}
{% block header %}
<div class="columns is-mobile">
@ -21,7 +23,7 @@
{% endfor %}
{% if not activities %}
<div class="block">
<p>{% trans "No reviews or comments yet!" %}</p>
<p><em>{% trans "No reviews or comments yet!" %}</em></p>
</div>
{% endif %}

View file

@ -51,9 +51,15 @@
{% endfor %}
</div>
</div>
{% empty %}
<p class="column">
<em>No books found.</em>
</p>
{% endfor %}
</div>
{% if shelves.exists %}
<small><a href="{% url 'user-shelves' user|username %}">{% trans "View all books" %}</a></small>
{% endif %}
</div>
{% endif %}
@ -119,16 +125,16 @@
</div>
{% endif %}
</div>
{% for activity in activities %}
<div class="block" id="feed_{{ activity.id }}">
{% include 'snippets/status/status.html' with status=activity %}
</div>
{% endfor %}
{% if not activities %}
{% empty %}
<div class="block">
<p>{% trans "No activities yet!" %}</p>
<p><em>{% trans "No activities yet!" %}</em></p>
</div>
{% endif %}
{% endfor %}
{% include 'snippets/pagination.html' with page=activities path=user.local_path anchor="#feed" mode="chronological" %}
</div>

View file

@ -23,12 +23,12 @@
<p>
{% if request.user.id == user.id or admin_mode %}
<a href="{% url 'user-relationships' user|username 'followers' %}">{% blocktrans trimmed count counter=user.followers.count %}
{{ counter }} follower
<a href="{% url 'user-relationships' user|username 'followers' %}">{% blocktrans trimmed count counter=user.followers.count with display_count=user.followers.count|intcomma %}
{{ display_count }} follower
{% plural %}
{{ counter }} followers
{{ display_count }} followers
{% endblocktrans %}</a>,
<a href="{% url 'user-relationships' user|username 'following' %}">{% blocktrans trimmed with counter=user.following.count %}
<a href="{% url 'user-relationships' user|username 'following' %}">{% blocktrans trimmed with counter=user.following.count|intcomma %}
{{ counter }} following
{% endblocktrans %}</a>

View file

@ -212,7 +212,7 @@ class Status(TestCase):
def test_generated_note_to_pure_activity(self, *_):
"""subclass of the base model version with a "pure" serializer"""
status = models.GeneratedNote.objects.create(
content="test content", user=self.local_user
content="reads", user=self.local_user
)
status.mention_books.set([self.book])
status.mention_users.set([self.local_user])
@ -220,7 +220,7 @@ class Status(TestCase):
self.assertEqual(activity["id"], status.remote_id)
self.assertEqual(
activity["content"],
f'mouse test content <a href="{self.book.remote_id}">"Test Edition"</a>',
f'mouse reads <a href="{self.book.remote_id}"><i>Test Edition</i></a>',
)
self.assertEqual(len(activity["tag"]), 2)
self.assertEqual(activity["type"], "Note")
@ -249,14 +249,18 @@ class Status(TestCase):
def test_comment_to_pure_activity(self, *_):
"""subclass of the base model version with a "pure" serializer"""
status = models.Comment.objects.create(
content="test content", user=self.local_user, book=self.book
content="test content", user=self.local_user, book=self.book, progress=27
)
activity = status.to_activity(pure=True)
self.assertEqual(activity["id"], status.remote_id)
self.assertEqual(activity["type"], "Note")
self.assertEqual(
activity["content"],
f'test content<p>(comment on <a href="{self.book.remote_id}">"Test Edition"</a>)</p>',
(
"test content"
f'<p>(comment on <a href="{self.book.remote_id}">'
"<i>Test Edition</i></a>, p. 27)</p>"
),
)
self.assertEqual(activity["attachment"][0]["type"], "Document")
# self.assertTrue(
@ -295,7 +299,11 @@ class Status(TestCase):
self.assertEqual(activity["type"], "Note")
self.assertEqual(
activity["content"],
f'a sickening sense <p>-- <a href="{self.book.remote_id}">"Test Edition"</a></p>test content',
(
"a sickening sense "
f'<p>— <a href="{self.book.remote_id}">'
"<i>Test Edition</i></a></p>test content"
),
)
self.assertEqual(activity["attachment"][0]["type"], "Document")
self.assertTrue(
@ -306,6 +314,29 @@ class Status(TestCase):
)
self.assertEqual(activity["attachment"][0]["name"], "Test Edition")
def test_quotation_page_serialization(self, *_):
"""serialization of quotation page position"""
tests = [
("single pos", 7, None, "p. 7"),
("page range", 7, 10, "pp. 7-10"),
]
for desc, beg, end, pages in tests:
with self.subTest(desc):
status = models.Quotation.objects.create(
quote="<p>my quote</p>",
content="",
user=self.local_user,
book=self.book,
position=beg,
endposition=end,
position_mode="PG",
)
activity = status.to_activity(pure=True)
self.assertRegex(
activity["content"],
f'^<p>"my quote"</p> <p>— <a .+</a>, {pages}</p>$',
)
def test_review_to_activity(self, *_):
"""subclass of the base model version with a "pure" serializer"""
status = models.Review.objects.create(

View file

@ -0,0 +1,31 @@
""" test ISBN hyphenator for books """
from django.test import TestCase
from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator
class TestISBN(TestCase):
"""isbn hyphenator"""
def test_isbn_hyphenation(self):
"""different isbn hyphenations"""
# nothing
self.assertEqual(hyphenator.hyphenate(None), None)
# 978-0 (English language) 3700000-6389999
self.assertEqual(hyphenator.hyphenate("9780439554930"), "978-0-439-55493-0")
# 978-2 (French language) 0000000-1999999
self.assertEqual(hyphenator.hyphenate("9782070100927"), "978-2-07-010092-7")
# 978-3 (German language) 2000000-6999999
self.assertEqual(hyphenator.hyphenate("9783518188125"), "978-3-518-18812-5")
# 978-4 (Japan) 0000000-1999999
self.assertEqual(hyphenator.hyphenate("9784101050454"), "978-4-10-105045-4")
# 978-626 (Taiwan) 9500000-9999999
self.assertEqual(hyphenator.hyphenate("9786269533251"), "978-626-95332-5-1")
# 979-8 (United States) 4000000-8499999
self.assertEqual(hyphenator.hyphenate("9798627974040"), "979-8-6279-7404-0")
# 978-626 (Taiwan) 8000000-9499999 (unassigned)
self.assertEqual(hyphenator.hyphenate("9786268533251"), "9786268533251")
# 978 range 6600000-6999999 (unassigned)
self.assertEqual(hyphenator.hyphenate("9786769533251"), "9786769533251")
# 979-8 (United States) 2300000-3499999 (unassigned)
self.assertEqual(hyphenator.hyphenate("9798311111111"), "9798311111111")

View file

@ -156,7 +156,7 @@ class Views(TestCase):
response = view(request)
validate_html(response.render())
self.assertFalse("results" in response.context_data)
self.assertTrue("results" in response.context_data)
def test_search_lists(self):
"""searches remote connectors"""

View file

@ -72,7 +72,7 @@ class SetupViews(TestCase):
self.site.refresh_from_db()
self.assertFalse(self.site.install_mode)
user = models.User.objects.get()
user = models.User.objects.first()
self.assertTrue(user.is_active)
self.assertTrue(user.is_superuser)
self.assertTrue(user.is_staff)

View file

@ -1,6 +1,7 @@
""" url routing for the app and api """
from django.conf.urls.static import static
from django.contrib import admin
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.urls import path, re_path
from django.views.generic.base import TemplateView
@ -774,5 +775,8 @@ urlpatterns = [
path("guided-tour/<tour>", views.toggle_guided_tour),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
# Serves /static when DEBUG is true.
urlpatterns.extend(staticfiles_urlpatterns())
# pylint: disable=invalid-name
handler500 = "bookwyrm.views.server_error"

View file

@ -1,8 +1,15 @@
""" Custom handler for caching """
from typing import Any, Callable, Tuple, Union
from django.core.cache import cache
def get_or_set(cache_key, function, *args, timeout=None):
def get_or_set(
cache_key: str,
function: Callable[..., Any],
*args: Tuple[Any, ...],
timeout: Union[float, None] = None
) -> Any:
"""Django's built-in get_or_set isn't cutting it"""
value = cache.get(cache_key)
if value is None:

View file

@ -1,15 +1,24 @@
"""ISNI author checking utilities"""
import xml.etree.ElementTree as ET
from typing import Union, Optional
import requests
from bookwyrm import activitypub, models
def request_isni_data(search_index, search_term, max_records=5):
def get_element_text(element: Optional[ET.Element]) -> str:
"""If the element is not None and there is a text attribute return this"""
if element is not None and element.text is not None:
return element.text
return ""
def request_isni_data(search_index: str, search_term: str, max_records: int = 5) -> str:
"""Request data from the ISNI API"""
search_string = f'{search_index}="{search_term}"'
query_params = {
query_params: dict[str, Union[str, int]] = {
"query": search_string,
"version": "1.1",
"operation": "searchRetrieve",
@ -26,41 +35,52 @@ def request_isni_data(search_index, search_term, max_records=5):
return result.text
def make_name_string(element):
def make_name_string(element: ET.Element) -> str:
"""create a string of form 'personal_name surname'"""
# NOTE: this will often be incorrect, many naming systems
# list "surname" before personal name
forename = element.find(".//forename")
surname = element.find(".//surname")
if forename is not None:
return "".join([forename.text, " ", surname.text])
return surname.text
forename_text = get_element_text(forename)
surname_text = get_element_text(surname)
return "".join(
[forename_text, " " if forename_text and surname_text else "", surname_text]
)
def get_other_identifier(element, code):
def get_other_identifier(element: ET.Element, code: str) -> str:
"""Get other identifiers associated with an author from their ISNI record"""
identifiers = element.findall(".//otherIdentifierOfIdentity")
for section_head in identifiers:
if (
section_head.find(".//type") is not None
and section_head.find(".//type").text == code
and section_head.find(".//identifier") is not None
(section_type := section_head.find(".//type")) is not None
and section_type.text is not None
and section_type.text == code
and (identifier := section_head.find(".//identifier")) is not None
and identifier.text is not None
):
return section_head.find(".//identifier").text
return identifier.text
# if we can't find it in otherIdentifierOfIdentity,
# try sources
for source in element.findall(".//sources"):
code_of_source = source.find(".//codeOfSource")
if code_of_source is not None and code_of_source.text.lower() == code.lower():
return source.find(".//sourceIdentifier").text
if (
(code_of_source := source.find(".//codeOfSource")) is not None
and code_of_source.text is not None
and code_of_source.text.lower() == code.lower()
and (source_identifier := source.find(".//sourceIdentifier")) is not None
and source_identifier.text is not None
):
return source_identifier.text
return ""
def get_external_information_uri(element, match_string):
def get_external_information_uri(element: ET.Element, match_string: str) -> str:
"""Get URLs associated with an author from their ISNI record"""
sources = element.findall(".//externalInformation")
@ -69,14 +89,18 @@ def get_external_information_uri(element, match_string):
uri = source.find(".//URI")
if (
uri is not None
and uri.text is not None
and information is not None
and information.text is not None
and information.text.lower() == match_string.lower()
):
return uri.text
return ""
def find_authors_by_name(name_string, description=False):
def find_authors_by_name(
name_string: str, description: bool = False
) -> list[activitypub.Author]:
"""Query the ISNI database for possible author matches by name"""
payload = request_isni_data("pica.na", name_string)
@ -92,7 +116,11 @@ def find_authors_by_name(name_string, description=False):
if not personal_name:
continue
author = get_author_from_isni(element.find(".//isniUnformatted").text)
author = get_author_from_isni(
get_element_text(element.find(".//isniUnformatted"))
)
if author is None:
continue
if bool(description):
@ -111,22 +139,23 @@ def find_authors_by_name(name_string, description=False):
# some of the "titles" in ISNI are a little ...iffy
# @ is used by ISNI/OCLC to index the starting point ignoring stop words
# (e.g. "The @Government of no one")
title_elements = [
e
for e in titles
if hasattr(e, "text") and not e.text.replace("@", "").isnumeric()
]
if len(title_elements):
author.bio = title_elements[0].text.replace("@", "")
else:
author.bio = None
author.bio = ""
for title in titles:
if (
title is not None
and hasattr(title, "text")
and title.text is not None
and not title.text.replace("@", "").isnumeric()
):
author.bio = title.text.replace("@", "")
break
possible_authors.append(author)
return possible_authors
def get_author_from_isni(isni):
def get_author_from_isni(isni: str) -> Optional[activitypub.Author]:
"""Find data to populate a new author record from their ISNI"""
payload = request_isni_data("pica.isn", isni)
@ -135,25 +164,30 @@ def get_author_from_isni(isni):
# there should only be a single responseRecord
# but let's use the first one just in case
element = root.find(".//responseRecord")
name = make_name_string(element.find(".//forename/.."))
if element is None:
return None
name = (
make_name_string(forename)
if (forename := element.find(".//forename/..")) is not None
else ""
)
viaf = get_other_identifier(element, "viaf")
# use a set to dedupe aliases in ISNI
aliases = set()
aliases_element = element.findall(".//personalNameVariant")
for entry in aliases_element:
aliases.add(make_name_string(entry))
# aliases needs to be list not set
aliases = list(aliases)
bio = element.find(".//nameTitle")
bio = bio.text if bio is not None else ""
bio = get_element_text(element.find(".//nameTitle"))
wikipedia = get_external_information_uri(element, "Wikipedia")
author = activitypub.Author(
id=element.find(".//isniURI").text,
id=get_element_text(element.find(".//isniURI")),
name=name,
isni=isni,
viafId=viaf,
aliases=aliases,
# aliases needs to be list not set
aliases=list(aliases),
bio=bio,
wikipediaLink=wikipedia,
)
@ -161,21 +195,26 @@ def get_author_from_isni(isni):
return author
def build_author_from_isni(match_value):
def build_author_from_isni(match_value: str) -> dict[str, activitypub.Author]:
"""Build basic author class object from ISNI URL"""
# if it is an isni value get the data
if match_value.startswith("https://isni.org/isni/"):
isni = match_value.replace("https://isni.org/isni/", "")
return {"author": get_author_from_isni(isni)}
author = get_author_from_isni(isni)
if author is not None:
return {"author": author}
# otherwise it's a name string
return {}
def augment_author_metadata(author, isni):
def augment_author_metadata(author: models.Author, isni: str) -> None:
"""Update any missing author fields from ISNI data"""
isni_author = get_author_from_isni(isni)
if isni_author is None:
return
isni_author.to_model(model=models.Author, instance=author, overwrite=False)
# we DO want to overwrite aliases because we're adding them to the

View file

@ -10,7 +10,7 @@ class IgnoreVariableDoesNotExist(logging.Filter):
these errors are not useful to us.
"""
def filter(self, record):
def filter(self, record: logging.LogRecord) -> bool:
if record.exc_info:
(_, err_value, _) = record.exc_info
while err_value:

View file

@ -1,8 +1,10 @@
"""Validations"""
from typing import Optional
from bookwyrm.settings import DOMAIN, USE_HTTPS
def validate_url_domain(url):
def validate_url_domain(url: str) -> Optional[str]:
"""Basic check that the URL starts with the instance domain name"""
if not url:
return None

View file

@ -5,6 +5,7 @@ from django.shortcuts import get_object_or_404, redirect
from django.template.response import TemplateResponse
from django.utils.decorators import method_decorator
from django.views import View
from django.views.decorators.http import require_POST
from bookwyrm import forms, models
from bookwyrm.settings import PAGE_LENGTH
@ -108,6 +109,7 @@ class EditAnnouncement(View):
@login_required
@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
@require_POST
def delete_announcement(_, announcement_id):
"""delete announcement"""
announcement = get_object_or_404(models.Announcement, id=announcement_id)

View file

@ -32,6 +32,9 @@ class EditBook(View):
def get(self, request, book_id):
"""info about a book"""
book = get_edition(book_id)
# This doesn't update the sort title, just pre-populates it in the form
if book.sort_title in ["", None]:
book.sort_title = book.guess_sort_title()
if not book.description:
book.description = book.parent_work.description
data = {"book": book, "form": forms.EditionForm(instance=book)}
@ -40,6 +43,7 @@ class EditBook(View):
def post(self, request, book_id):
"""edit a book cool"""
book = get_object_or_404(models.Edition, id=book_id)
form = forms.EditionForm(request.POST, request.FILES, instance=book)
data = {"book": book, "form": form}

View file

@ -19,7 +19,7 @@ class Directory(View):
software = request.GET.get("software")
if not software or software == "bookwyrm":
filters["bookwyrm_user"] = True
scope = request.GET.get("scope")
scope = request.GET.get("scope", "federated")
if scope == "local":
filters["local"] = True
@ -38,6 +38,8 @@ class Directory(View):
page.number, on_each_side=2, on_ends=1
),
"users": page,
"sort": sort,
"scope": scope,
}
return TemplateResponse(request, "directory/directory.html", data)

View file

@ -91,18 +91,15 @@ def book_search(request):
def user_search(request):
"""cool kids members only user search"""
"""user search: search for a user"""
viewer = request.user
query = request.GET.get("q")
query = query.strip()
data = {"type": "user", "query": query}
# logged out viewers can't search users
if not viewer.is_authenticated:
return TemplateResponse(request, "search/user.html", data)
# use webfinger for mastodon style account@domain.com username to load the user if
# they don't exist locally (handle_remote_webfinger will check the db)
if re.match(regex.FULL_USERNAME, query):
if re.match(regex.FULL_USERNAME, query) and viewer.is_authenticated:
handle_remote_webfinger(query)
results = (
@ -118,6 +115,11 @@ def user_search(request):
)
.order_by("-similarity")
)
# don't expose remote users
if not viewer.is_authenticated:
results = results.filter(local=True)
paginated = Paginator(results, PAGE_LENGTH)
page = paginated.get_page(request.GET.get("page"))
data["results"] = page

View file

@ -9,6 +9,7 @@ from django.shortcuts import redirect
from django.template.response import TemplateResponse
from django.views import View
from bookwyrm.activitypub import get_representative
from bookwyrm import forms, models
from bookwyrm import settings
from bookwyrm.utils import regex
@ -96,4 +97,5 @@ class CreateAdmin(View):
login(request, user)
site.install_mode = False
site.save()
get_representative() # create the instance user
return redirect("settings-site")

View file

@ -9,7 +9,7 @@ x-logging:
services:
nginx:
image: nginx:latest
image: nginx:1.25.2
logging: *default-logging
restart: unless-stopped
ports:
@ -62,7 +62,7 @@ services:
ports:
- "8000"
redis_activity:
image: redis
image: redis:7.2.1
command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT}
logging: *default-logging
volumes:
@ -73,7 +73,7 @@ services:
- main
restart: on-failure
redis_broker:
image: redis
image: redis:7.2.1
command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT}
logging: *default-logging
volumes:

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

Binary file not shown.

File diff suppressed because it is too large Load diff

View file

@ -13,6 +13,15 @@ implicit_reexport = True
[mypy-bookwyrm.connectors.*]
ignore_errors = False
[mypy-bookwyrm.utils.*]
ignore_errors = False
[mypy-bookwyrm.importers.*]
ignore_errors = False
[mypy-bookwyrm.isbn.*]
ignore_errors = False
[mypy-celerywyrm.*]
ignore_errors = False