mirrors/bookwyrm
1
0
Fork 1
mirror of https://github.com/bookwyrm-social/bookwyrm.git synced 2025-02-17 19:45:17 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'main' into flowerproxy

Browse source
This commit is contained in:
Mouse Reeve 2024-04-24 15:34:46 -07:00 committed by GitHub
commit 3f8cf2e134
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
466 changed files with 47762 additions and 12807 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
.env.example
View file

@ -16,6 +16,11 @@ DEFAULT_LANGUAGE="English"
## Leave unset to allow all hosts
# ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
# Specify when the site is served from a port that is not the default
# for the protocol (80 for HTTP or 443 for HTTPS).
# Probably only necessary in development.
# PORT=1333
MEDIA_ROOT=images/
# Database configuration
@ -71,14 +76,20 @@ ENABLE_THUMBNAIL_GENERATION=true
USE_S3=false
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
# seconds for signed S3 urls to expire
# this is currently only used for user export files
S3_SIGNED_URL_EXPIRY=900
# Commented are example values if you use a non-AWS, S3-compatible service
# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL.
# AWS_S3_URL_PROTOCOL must end in ":" and defaults to the same protocol as
# the BookWyrm instance ("http:" or "https:", based on USE_SSL).
# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name"
# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud"
# AWS_S3_URL_PROTOCOL=None # "http:"
# AWS_S3_REGION_NAME=None # "fr-par"
# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud"
@ -133,7 +144,14 @@ HTTP_X_FORWARDED_PROTO=false
TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
TWO_FACTOR_LOGIN_MAX_SECONDS=60
# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
# Value should be a comma-separated list of host names.
# Additional hosts to allow in the Content-Security-Policy, "self" (should be
# DOMAIN with optionally ":" + PORT) and AWS_S3_CUSTOM_DOMAIN (if used) are
# added by default. Value should be a comma-separated list of host names.
CSP_ADDITIONAL_HOSTS=
# Time before being logged out (in seconds)
# SESSION_COOKIE_AGE=2592000 # current default: 30 days
# Maximum allowed memory for file uploads (increase if users are having trouble
# uploading BookWyrm export files).
# DATA_UPLOAD_MAX_MEMORY_MiB=100

17
.github/workflows/black.yml vendored
View file

@ -1,17 +0,0 @@
name: Python Formatting (run ./bw-dev black to fix)
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
- uses: psf/black@22.12.0
with:
version: 22.12.0

8
.github/workflows/codeql-analysis.yml vendored
View file

@ -36,11 +36,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@ -51,7 +51,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
uses: github/codeql-action/autobuild@v3
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@ -65,4 +65,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
uses: github/codeql-action/analyze@v3

2
.github/workflows/curlylint.yaml vendored
View file

@ -10,7 +10,7 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install curlylint
run: pip install curlylint

61
.github/workflows/django-tests.yml vendored
View file

@ -1,61 +0,0 @@
name: Run Python Tests
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-20.04
services:
postgres:
image: postgres:13
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: hunter2
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Run Tests
env:
SECRET_KEY: beepbeep
DEBUG: false
USE_HTTPS: true
DOMAIN: your.domain.here
BOOKWYRM_DATABASE_BACKEND: postgres
MEDIA_ROOT: images/
POSTGRES_PASSWORD: hunter2
POSTGRES_USER: postgres
POSTGRES_DB: github_actions
POSTGRES_HOST: 127.0.0.1
CELERY_BROKER: ""
REDIS_BROKER_PORT: 6379
REDIS_BROKER_PASSWORD: beep
USE_DUMMY_CACHE: true
FLOWER_PORT: 8888
EMAIL_HOST: "smtp.mailgun.org"
EMAIL_PORT: 587
EMAIL_HOST_USER: ""
EMAIL_HOST_PASSWORD: ""
EMAIL_USE_TLS: true
ENABLE_PREVIEW_IMAGES: false
ENABLE_THUMBNAIL_GENERATION: true
HTTP_X_FORWARDED_PROTO: false
run: |
pytest -n 3

5
.github/workflows/lint-frontend.yaml vendored
View file

@ -19,10 +19,11 @@ jobs:
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install modules
run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
# run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
run: npm install eslint@^8.9.0
# See .stylelintignore for files that are not linted.
# - name: Run stylelint

50
.github/workflows/mypy.yml vendored
View file

@ -1,50 +0,0 @@
name: Mypy
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Analysing the code with mypy
env:
SECRET_KEY: beepbeep
DEBUG: false
USE_HTTPS: true
DOMAIN: your.domain.here
BOOKWYRM_DATABASE_BACKEND: postgres
MEDIA_ROOT: images/
POSTGRES_PASSWORD: hunter2
POSTGRES_USER: postgres
POSTGRES_DB: github_actions
POSTGRES_HOST: 127.0.0.1
CELERY_BROKER: ""
REDIS_BROKER_PORT: 6379
REDIS_BROKER_PASSWORD: beep
USE_DUMMY_CACHE: true
FLOWER_PORT: 8888
EMAIL_HOST: "smtp.mailgun.org"
EMAIL_PORT: 587
EMAIL_HOST_USER: ""
EMAIL_HOST_PASSWORD: ""
EMAIL_USE_TLS: true
ENABLE_PREVIEW_IMAGES: false
ENABLE_THUMBNAIL_GENERATION: true
HTTP_X_FORWARDED_PROTO: false
run: |
mypy bookwyrm celerywyrm

2
.github/workflows/prettier.yaml vendored
View file

@ -14,7 +14,7 @@ jobs:
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install modules
run: npm install prettier@2.5.1

27
.github/workflows/pylint.yml vendored
View file

@ -1,27 +0,0 @@
name: Pylint
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Analysing the code with pylint
run: |
pylint bookwyrm/

99
.github/workflows/python.yml vendored Normal file
View file

@ -0,0 +1,99 @@
name: Python
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
# overrides for .env.example
env:
POSTGRES_HOST: 127.0.0.1
PGPORT: 5432
POSTGRES_USER: postgres
POSTGRES_PASSWORD: hunter2
POSTGRES_DB: github_actions
SECRET_KEY: beepbeep
EMAIL_HOST_USER: ""
EMAIL_HOST_PASSWORD: ""
jobs:
pytest:
name: Tests (pytest)
runs-on: ubuntu-latest
services:
postgres:
image: postgres:13
env: # does not inherit from jobs.build.env
POSTGRES_USER: postgres
POSTGRES_PASSWORD: hunter2
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: 3.11
cache: pip
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install pytest-github-actions-annotate-failures
- name: Set up .env
run: cp .env.example .env
- name: Check migrations up-to-date
run: python ./manage.py makemigrations --check
- name: Run Tests
run: pytest -n 3
pylint:
name: Linting (pylint)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: 3.11
cache: pip
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Analyse code with pylint
run: pylint bookwyrm/
mypy:
name: Typing (mypy)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: 3.11
cache: pip
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Set up .env
run: cp .env.example .env
- name: Analyse code with mypy
run: mypy bookwyrm celerywyrm
black:
name: Formatting (black; run ./bw-dev black to fix)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
- uses: psf/black@stable
with:
version: "22.*"

5
.gitignore vendored
View file

@ -16,6 +16,8 @@
# BookWyrm
.env
/images/
/exports/
/static/
bookwyrm/static/css/bookwyrm.css
bookwyrm/static/css/themes/
!bookwyrm/static/css/themes/bookwyrm-*.scss
@ -36,3 +38,6 @@ nginx/default.conf
#macOS
**/.DS_Store
# Docker
docker-compose.override.yml

1
.prettierrc Normal file
View file

@ -0,0 +1 @@
'trailingComma': 'es5'

2
Dockerfile
View file

@ -1,4 +1,4 @@
FROM python:3.9
FROM python:3.11
ENV PYTHONUNBUFFERED 1

5
FEDERATION.md
View file

@ -13,14 +13,15 @@ User relationship interactions follow the standard ActivityPub spec.
- `Block`: prevent users from seeing one another's statuses, and prevents the blocked user from viewing the actor's profile
- `Update`: updates a user's profile and settings
- `Delete`: deactivates a user
- `Undo`: reverses a `Follow` or `Block`
- `Undo`: reverses a `Block` or `Follow`
### Activities
- `Create/Status`: saves a new status in the database.
- `Delete/Status`: Removes a status
- `Like/Status`: Creates a favorite on the status
- `Announce/Status`: Boosts the status into the actor's timeline
- `Undo/*`,: Reverses a `Like` or `Announce`
- `Undo/*`,: Reverses an `Announce`, `Like`, or `Move`
- `Move/User`: Moves a user from one ActivityPub id to another.
### Collections
User's books and lists are represented by [`OrderedCollection`](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection)

1
README.md
View file

@ -10,7 +10,6 @@ BookWyrm is a social network for tracking your reading, talking about books, wri
## Links
[![Mastodon Follow](https://img.shields.io/mastodon/follow/000146121?domain=https%3A%2F%2Ftech.lgbt&style=social)](https://tech.lgbt/@bookwyrm)
[![Twitter Follow](https://img.shields.io/twitter/follow/BookWyrmSocial?style=social)](https://twitter.com/BookWyrmSocial)
- [Project homepage](https://joinbookwyrm.com/)
- [Support](https://patreon.com/bookwyrm)

1
VERSION Normal file
View file

@ -0,0 +1 @@
0.7.3

7
bookwyrm/activitypub/__init__.py
View file

@ -4,7 +4,11 @@ import sys
from .base_activity import ActivityEncoder, Signature, naive_parse
from .base_activity import Link, Mention, Hashtag
from .base_activity import ActivitySerializerError, resolve_remote_id
from .base_activity import (
ActivitySerializerError,
resolve_remote_id,
get_representative,
)
from .image import Document, Image
from .note import Note, GeneratedNote, Article, Comment, Quotation
from .note import Review, Rating
@ -19,6 +23,7 @@ from .verbs import Create, Delete, Undo, Update
from .verbs import Follow, Accept, Reject, Block
from .verbs import Add, Remove
from .verbs import Announce, Like
from .verbs import Move
# this creates a list of all the Activity types that we can serialize,
# so when an Activity comes in from outside, we can check if it's known

29
bookwyrm/activitypub/base_activity.py
View file

@ -1,4 +1,5 @@
""" basics for an activitypub serializer """
from __future__ import annotations
from dataclasses import dataclass, fields, MISSING
from json import JSONEncoder
import logging
@ -19,6 +20,7 @@ from bookwyrm.tasks import app, MISC
logger = logging.getLogger(__name__)
# pylint: disable=invalid-name
TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel)
@ -72,8 +74,10 @@ class ActivityObject:
def __init__(
self,
activity_objects: Optional[list[str, base_model.BookWyrmModel]] = None,
**kwargs: dict[str, Any],
activity_objects: Optional[
dict[str, Union[str, list[str], ActivityObject, base_model.BookWyrmModel]]
] = None,
**kwargs: Any,
):
"""this lets you pass in an object with fields that aren't in the
dataclass, which it ignores. Any field in the dataclass is required or
@ -233,7 +237,7 @@ class ActivityObject:
omit = kwargs.get("omit", ())
data = self.__dict__.copy()
# recursively serialize
for (k, v) in data.items():
for k, v in data.items():
try:
if issubclass(type(v), ActivityObject):
data[k] = v.serialize()
@ -393,19 +397,15 @@ def resolve_remote_id(
def get_representative():
"""Get or create an actor representing the instance
to sign requests to 'secure mastodon' servers"""
username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
email = "bookwyrm@localhost"
try:
user = models.User.objects.get(username=username)
except models.User.DoesNotExist:
user = models.User.objects.create_user(
username=username,
email=email,
to sign outgoing HTTP GET requests"""
return models.User.objects.get_or_create(
username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}",
defaults=dict(
email="bookwyrm@localhost",
local=True,
localname=INSTANCE_ACTOR_USERNAME,
)
return user
),
)[0]
def get_activitypub_data(url):
@ -424,6 +424,7 @@ def get_activitypub_data(url):
"Date": now,
"Signature": make_signature("get", sender, url, now),
},
timeout=15,
)
except requests.RequestException:
raise ConnectorException()

4
bookwyrm/activitypub/book.py
View file

@ -22,8 +22,6 @@ class BookData(ActivityObject):
aasin: Optional[str] = None
isfdb: Optional[str] = None
lastEditedBy: Optional[str] = None
links: list[str] = field(default_factory=list)
fileLinks: list[str] = field(default_factory=list)
# pylint: disable=invalid-name
@ -45,6 +43,8 @@ class Book(BookData):
firstPublishedDate: str = ""
publishedDate: str = ""
fileLinks: list[str] = field(default_factory=list)
cover: Optional[Document] = None
type: str = "Book"

6
bookwyrm/activitypub/person.py
View file

@ -1,5 +1,5 @@
""" actor serializer """
from dataclasses import dataclass, field
from dataclasses import dataclass
from typing import Dict
from .base_activity import ActivityObject
@ -35,9 +35,11 @@ class Person(ActivityObject):
endpoints: Dict = None
name: str = None
summary: str = None
icon: Image = field(default_factory=lambda: {})
icon: Image = None
bookwyrmUser: bool = False
manuallyApprovesFollowers: str = False
discoverable: str = False
hideFollows: str = False
movedTo: str = None
alsoKnownAs: dict[str] = None
type: str = "Person"

43
bookwyrm/activitypub/verbs.py
View file

@ -171,9 +171,19 @@ class Reject(Verb):
type: str = "Reject"
def action(self, allow_external_connections=True):
"""reject a follow request"""
obj = self.object.to_model(save=False, allow_create=False)
obj.reject()
"""reject a follow or follow request"""
for model_name in ["UserFollowRequest", "UserFollows", None]:
model = apps.get_model(f"bookwyrm.{model_name}") if model_name else None
if obj := self.object.to_model(
model=model,
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
):
# Reject the first model that can be built.
obj.reject()
break
@dataclass(init=False)
@ -231,3 +241,30 @@ class Announce(Verb):
def action(self, allow_external_connections=True):
"""boost"""
self.to_model(allow_external_connections=allow_external_connections)
@dataclass(init=False)
class Move(Verb):
"""a user moving an object"""
object: str
type: str = "Move"
origin: str = None
target: str = None
def action(self, allow_external_connections=True):
"""move"""
object_is_user = resolve_remote_id(remote_id=self.object, model="User")
if object_is_user:
model = apps.get_model("bookwyrm.MoveUser")
self.to_model(
model=model,
save=True,
allow_external_connections=allow_external_connections,
)
else:
# we might do something with this to move other objects at some point
pass

14
bookwyrm/activitystreams.py
View file

@ -139,14 +139,14 @@ class ActivityStream(RedisStore):
| (
Q(following=status.user) & Q(following=status.reply_parent.user)
) # if the user is following both authors
).distinct()
)
# only visible to the poster's followers and tagged users
elif status.privacy == "followers":
audience = audience.filter(
Q(following=status.user) # if the user is following the author
)
return audience.distinct()
return audience.distinct("id")
@tracer.start_as_current_span("ActivityStream.get_audience")
def get_audience(self, status):
@ -156,7 +156,7 @@ class ActivityStream(RedisStore):
status_author = models.User.objects.filter(
is_active=True, local=True, id=status.user.id
).values_list("id", flat=True)
return list(set(list(audience) + list(status_author)))
return list(set(audience) | set(status_author))
def get_stores_for_users(self, user_ids):
"""convert a list of user ids into redis store ids"""
@ -183,15 +183,13 @@ class HomeStream(ActivityStream):
def get_audience(self, status):
trace.get_current_span().set_attribute("stream_id", self.key)
audience = super()._get_audience(status)
if not audience:
return []
# if the user is following the author
audience = audience.filter(following=status.user).values_list("id", flat=True)
# if the user is the post's author
status_author = models.User.objects.filter(
is_active=True, local=True, id=status.user.id
).values_list("id", flat=True)
return list(set(list(audience) + list(status_author)))
return list(set(audience) | set(status_author))
def get_statuses_for_user(self, user):
return models.Status.privacy_filter(
@ -239,9 +237,7 @@ class BooksStream(ActivityStream):
)
audience = super()._get_audience(status)
if not audience:
return models.User.objects.none()
return audience.filter(shelfbook__book__parent_work=work).distinct()
return audience.filter(shelfbook__book__parent_work=work)
def get_audience(self, status):
# only show public statuses on the books feed,

15
bookwyrm/apps.py
View file

@ -1,4 +1,5 @@
"""Do further startup configuration and initialization"""
import os
import urllib
import logging
@ -14,16 +15,16 @@ def download_file(url, destination):
"""Downloads a file to the given path"""
try:
# Ensure our destination directory exists
os.makedirs(os.path.dirname(destination))
os.makedirs(os.path.dirname(destination), exist_ok=True)
with urllib.request.urlopen(url) as stream:
with open(destination, "b+w") as outfile:
outfile.write(stream.read())
except (urllib.error.HTTPError, urllib.error.URLError):
logger.info("Failed to download file %s", url)
except OSError:
logger.info("Couldn't open font file %s for writing", destination)
except: # pylint: disable=bare-except
logger.info("Unknown error in file download")
except (urllib.error.HTTPError, urllib.error.URLError) as err:
logger.error("Failed to download file %s: %s", url, err)
except OSError as err:
logger.error("Couldn't open font file %s for writing: %s", destination, err)
except Exception as err: # pylint:disable=broad-except
logger.error("Unknown error in file download: %s", err)
class BookwyrmConfig(AppConfig):

32
bookwyrm/book_search.py
View file

@ -43,6 +43,7 @@ def search(
min_confidence: float = 0,
filters: Optional[list[Any]] = None,
return_first: bool = False,
books: Optional[QuerySet[models.Edition]] = None,
) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
"""search your local database"""
filters = filters or []
@ -54,13 +55,15 @@ def search(
# first, try searching unique identifiers
# unique identifiers never have spaces, title/author usually do
if not " " in query:
results = search_identifiers(query, *filters, return_first=return_first)
results = search_identifiers(
query, *filters, return_first=return_first, books=books
)
# if there were no identifier results...
if not results:
# then try searching title/author
results = search_title_author(
query, min_confidence, *filters, return_first=return_first
query, min_confidence, *filters, return_first=return_first, books=books
)
return results
@ -98,9 +101,17 @@ def format_search_result(search_result):
def search_identifiers(
query, *filters, return_first=False
query,
*filters,
return_first=False,
books=None,
) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
"""tries remote_id, isbn; defined as dedupe fields on the model"""
"""search Editions by deduplication fields
Best for cases when we can assume someone is searching for an exact match on
commonly unique data identifiers like isbn or specific library ids.
"""
books = books or models.Edition.objects
if connectors.maybe_isbn(query):
# Oh did you think the 'S' in ISBN stood for 'standard'?
normalized_isbn = query.strip().upper().rjust(10, "0")
@ -111,7 +122,7 @@ def search_identifiers(
for f in models.Edition._meta.get_fields()
if hasattr(f, "deduplication_field") and f.deduplication_field
]
results = models.Edition.objects.filter(
results = books.filter(
*filters, reduce(operator.or_, (Q(**f) for f in or_filters))
).distinct()
@ -121,12 +132,17 @@ def search_identifiers(
def search_title_author(
query, min_confidence, *filters, return_first=False
query,
min_confidence,
*filters,
return_first=False,
books=None,
) -> QuerySet[models.Edition]:
"""searches for title and author"""
books = books or models.Edition.objects
query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
results = (
models.Edition.objects.filter(*filters, search_vector=query)
books.filter(*filters, search_vector=query)
.annotate(rank=SearchRank(F("search_vector"), query))
.filter(rank__gt=min_confidence)
.order_by("-rank")
@ -137,7 +153,7 @@ def search_title_author(
# filter out multiple editions of the same work
list_results = []
for work_id in set(editions_of_work[:30]):
for work_id in editions_of_work[:30]:
result = (
results.filter(parent_work=work_id)
.order_by("-rank", "-edition_rank")

4
bookwyrm/connectors/abstract_connector.py
View file

@ -3,7 +3,9 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Optional, TypedDict, Any, Callable, Union, Iterator
from urllib.parse import quote_plus
import imghdr
# pylint: disable-next=deprecated-module
import imghdr # Deprecated in 3.11 for removal in 3.13; no good alternative yet
import logging
import re
import asyncio

19
bookwyrm/connectors/connector_manager.py
View file

@ -118,9 +118,11 @@ def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
"""get the connector related to the object's server"""
url = urlparse(remote_id)
identifier = url.netloc
identifier = url.hostname
if not identifier:
raise ValueError("Invalid remote id")
raise ValueError(f"Invalid remote id: {remote_id}")
base_url = f"{url.scheme}://{url.netloc}"
try:
connector_info = models.Connector.objects.get(identifier=identifier)
@ -128,10 +130,10 @@ def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnec
connector_info = models.Connector.objects.create(
identifier=identifier,
connector_file="bookwyrm_connector",
base_url=f"https://{identifier}",
books_url=f"https://{identifier}/book",
covers_url=f"https://{identifier}/images/covers",
search_url=f"https://{identifier}/search?q=",
base_url=base_url,
books_url=f"{base_url}/book",
covers_url=f"{base_url}/images/covers",
search_url=f"{base_url}/search?q=",
priority=2,
)
@ -188,8 +190,11 @@ def raise_not_valid_url(url: str) -> None:
if not parsed.scheme in ["http", "https"]:
raise ConnectorException("Invalid scheme: ", url)
if not parsed.hostname:
raise ConnectorException("Hostname missing: ", url)
try:
ipaddress.ip_address(parsed.netloc)
ipaddress.ip_address(parsed.hostname)
raise ConnectorException("Provided url is an IP address: ", url)
except ValueError:
# it's not an IP address, which is good

3
bookwyrm/emailing.py
View file

@ -4,7 +4,7 @@ from django.template.loader import get_template
from bookwyrm import models, settings
from bookwyrm.tasks import app, EMAIL
from bookwyrm.settings import DOMAIN
from bookwyrm.settings import DOMAIN, BASE_URL
def email_data():
@ -14,6 +14,7 @@ def email_data():
"site_name": site.name,
"logo": site.logo_small_url,
"domain": DOMAIN,
"base_url": BASE_URL,
"user": None,
}

2
bookwyrm/forms/author.py
View file

@ -15,6 +15,7 @@ class AuthorForm(CustomForm):
"aliases",
"bio",
"wikipedia_link",
"wikidata",
"website",
"born",
"died",
@ -32,6 +33,7 @@ class AuthorForm(CustomForm):
"wikipedia_link": forms.TextInput(
attrs={"aria-describedby": "desc_wikipedia_link"}
),
"wikidata": forms.TextInput(attrs={"aria-describedby": "desc_wikidata"}),
"website": forms.TextInput(attrs={"aria-describedby": "desc_website"}),
"born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
"died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),

7
bookwyrm/forms/books.py
View file

@ -1,8 +1,9 @@
""" using django model forms """
from django import forms
from file_resubmit.widgets import ResubmitImageWidget
from bookwyrm import models
from bookwyrm.models.fields import ClearableFileInputWithWarning
from .custom_form import CustomForm
from .widgets import ArrayWidget, SelectDateWidget, Select
@ -70,9 +71,7 @@ class EditionForm(CustomForm):
"published_date": SelectDateWidget(
attrs={"aria-describedby": "desc_published_date"}
),
"cover": ClearableFileInputWithWarning(
attrs={"aria-describedby": "desc_cover"}
),
"cover": ResubmitImageWidget(attrs={"aria-describedby": "desc_cover"}),
"physical_format": Select(
attrs={"aria-describedby": "desc_physical_format"}
),

16
bookwyrm/forms/edit_user.py
View file

@ -70,6 +70,22 @@ class DeleteUserForm(CustomForm):
fields = ["password"]
class MoveUserForm(CustomForm):
target = forms.CharField(widget=forms.TextInput)
class Meta:
model = models.User
fields = ["password"]
class AliasUserForm(CustomForm):
username = forms.CharField(widget=forms.TextInput)
class Meta:
model = models.User
fields = ["password"]
class ChangePasswordForm(CustomForm):
current_password = forms.CharField(widget=forms.PasswordInput)
confirm_password = forms.CharField(widget=forms.PasswordInput)

4
bookwyrm/forms/forms.py
View file

@ -25,6 +25,10 @@ class ImportForm(forms.Form):
csv_file = forms.FileField()
class ImportUserForm(forms.Form):
archive_file = forms.FileField()
class ShelfForm(CustomForm):
class Meta:
model = models.Shelf

10
bookwyrm/forms/links.py
View file

@ -1,4 +1,5 @@
""" using django model forms """
from urllib.parse import urlparse
from django.utils.translation import gettext_lazy as _
@ -25,7 +26,7 @@ class FileLinkForm(CustomForm):
url = cleaned_data.get("url")
filetype = cleaned_data.get("filetype")
book = cleaned_data.get("book")
domain = urlparse(url).netloc
domain = urlparse(url).hostname
if models.LinkDomain.objects.filter(domain=domain).exists():
status = models.LinkDomain.objects.get(domain=domain).status
if status == "blocked":
@ -37,10 +38,9 @@ class FileLinkForm(CustomForm):
),
)
if (
not self.instance
and models.FileLink.objects.filter(
url=url, book=book, filetype=filetype
).exists()
models.FileLink.objects.filter(url=url, book=book, filetype=filetype)
.exclude(pk=self.instance)
.exists()
):
# pylint: disable=line-too-long
self.add_error(

1
bookwyrm/importers/__init__.py
View file

@ -1,6 +1,7 @@
""" import classes """
from .importer import Importer
from .bookwyrm_import import BookwyrmImporter
from .calibre_import import CalibreImporter
from .goodreads_import import GoodreadsImporter
from .librarything_import import LibrarythingImporter

24
bookwyrm/importers/bookwyrm_import.py Normal file
View file

@ -0,0 +1,24 @@
"""Import data from Bookwyrm export files"""
from django.http import QueryDict
from bookwyrm.models import User
from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
class BookwyrmImporter:
"""Import a Bookwyrm User export file.
This is kind of a combination of an importer and a connector.
"""
# pylint: disable=no-self-use
def process_import(
self, user: User, archive_file: bytes, settings: QueryDict
) -> BookwyrmImportJob:
"""import user data from a Bookwyrm export file"""
required = [k for k in settings if settings.get(k) == "on"]
job = BookwyrmImportJob.objects.create(
user=user, archive_file=archive_file, required=required
)
return job

9
bookwyrm/isbn/isbn.py
View file

@ -26,7 +26,7 @@ class IsbnHyphenator:
def update_range_message(self) -> None:
"""Download the range message xml file and save it locally"""
response = requests.get(self.__range_message_url)
response = requests.get(self.__range_message_url, timeout=15)
with open(self.__range_file_path, "w", encoding="utf-8") as file:
file.write(response.text)
self.__element_tree = None
@ -40,7 +40,12 @@ class IsbnHyphenator:
self.__element_tree = ElementTree.parse(self.__range_file_path)
gs1_prefix = isbn_13[:3]
reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
try:
reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
except ValueError:
# if the reg groups are invalid, just return the original isbn
return isbn_13
if reg_group is None:
return isbn_13 # failed to hyphenate

37
bookwyrm/management/commands/deduplicate_book_data.py
View file

@ -1,13 +1,14 @@
""" PROCEED WITH CAUTION: uses deduplication fields to permanently
merge book data objects """
from django.core.management.base import BaseCommand
from django.db.models import Count
from bookwyrm import models
from bookwyrm.management.merge import merge_objects
def dedupe_model(model):
def dedupe_model(model, dry_run=False):
"""combine duplicate editions and update related models"""
print(f"deduplicating {model.__name__}:")
fields = model._meta.get_fields()
dedupe_fields = [
f for f in fields if hasattr(f, "deduplication_field") and f.deduplication_field
@ -16,30 +17,42 @@ def dedupe_model(model):
dupes = (
model.objects.values(field.name)
.annotate(Count(field.name))
.filter(**{"%s__count__gt" % field.name: 1})
.filter(**{f"{field.name}__count__gt": 1})
.exclude(**{field.name: ""})
.exclude(**{f"{field.name}__isnull": True})
)
for dupe in dupes:
value = dupe[field.name]
if not value or value == "":
continue
print("----------")
print(dupe)
objs = model.objects.filter(**{field.name: value}).order_by("id")
canonical = objs.first()
print("keeping", canonical.remote_id)
action = "would merge" if dry_run else "merging"
print(
f"{action} into {model.__name__} {canonical.remote_id} based on {field.name} {value}:"
)
for obj in objs[1:]:
print(obj.remote_id)
merge_objects(canonical, obj)
print(f"- {obj.remote_id}")
absorbed_fields = obj.merge_into(canonical, dry_run=dry_run)
print(f" absorbed fields: {absorbed_fields}")
class Command(BaseCommand):
"""deduplicate allllll the book data models"""
help = "merges duplicate book data"
def add_arguments(self, parser):
"""add the arguments for this command"""
parser.add_argument(
"--dry_run",
action="store_true",
help="don't actually merge, only print what would happen",
)
# pylint: disable=no-self-use,unused-argument
def handle(self, *args, **options):
"""run deduplications"""
dedupe_model(models.Edition)
dedupe_model(models.Work)
dedupe_model(models.Author)
dedupe_model(models.Edition, dry_run=options["dry_run"])
dedupe_model(models.Work, dry_run=options["dry_run"])
dedupe_model(models.Author, dry_run=options["dry_run"])

43
bookwyrm/management/commands/erase_deleted_user_data.py Normal file
View file

@ -0,0 +1,43 @@
""" Erase any data stored about deleted users """
import sys
from django.core.management.base import BaseCommand, CommandError
from bookwyrm import models
from bookwyrm.models.user import erase_user_data
# pylint: disable=missing-function-docstring
class Command(BaseCommand):
"""command-line options"""
help = "Remove Two Factor Authorisation from user"
def add_arguments(self, parser): # pylint: disable=no-self-use
parser.add_argument(
"--dryrun",
action="store_true",
help="Preview users to be cleared without altering the database",
)
def handle(self, *args, **options): # pylint: disable=unused-argument
# Check for anything fishy
bad_state = models.User.objects.filter(is_deleted=True, is_active=True)
if bad_state.exists():
raise CommandError(
f"{bad_state.count()} user(s) marked as both active and deleted"
)
deleted_users = models.User.objects.filter(is_deleted=True)
self.stdout.write(f"Found {deleted_users.count()} deleted users")
if options["dryrun"]:
self.stdout.write("\n".join(u.username for u in deleted_users[:5]))
if deleted_users.count() > 5:
self.stdout.write("... and more")
sys.exit()
self.stdout.write("Erasing user data:")
for user_id in deleted_users.values_list("id", flat=True):
erase_user_data.delay(user_id)
self.stdout.write(".", ending="")
self.stdout.write("")
self.stdout.write("Tasks created successfully")

54
bookwyrm/management/commands/instance_version.py
View file

@ -1,54 +0,0 @@
""" Get your admin code to allow install """
from django.core.management.base import BaseCommand
from bookwyrm import models
from bookwyrm.settings import VERSION
# pylint: disable=no-self-use
class Command(BaseCommand):
"""command-line options"""
help = "What version is this?"
def add_arguments(self, parser):
"""specify which function to run"""
parser.add_argument(
"--current",
action="store_true",
help="Version stored in database",
)
parser.add_argument(
"--target",
action="store_true",
help="Version stored in settings",
)
parser.add_argument(
"--update",
action="store_true",
help="Update database version",
)
# pylint: disable=unused-argument
def handle(self, *args, **options):
"""execute init"""
site = models.SiteSettings.objects.get()
current = site.version or "0.0.1"
target = VERSION
if options.get("current"):
print(current)
return
if options.get("target"):
print(target)
return
if options.get("update"):
site.version = target
site.save()
return
if current != target:
print(f"{current}/{target}")
else:
print(current)

50
bookwyrm/management/merge.py
View file

@ -1,50 +0,0 @@
from django.db.models import ManyToManyField
def update_related(canonical, obj):
"""update all the models with fk to the object being removed"""
# move related models to canonical
related_models = [
(r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
]
for (related_field, related_model) in related_models:
# Skip the ManyToMany fields that aren’t auto-created. These
# should have a corresponding OneToMany field in the model for
# the linking table anyway. If we update it through that model
# instead then we won’t lose the extra fields in the linking
# table.
related_field_obj = related_model._meta.get_field(related_field)
if isinstance(related_field_obj, ManyToManyField):
through = related_field_obj.remote_field.through
if not through._meta.auto_created:
continue
related_objs = related_model.objects.filter(**{related_field: obj})
for related_obj in related_objs:
print("replacing in", related_model.__name__, related_field, related_obj.id)
try:
setattr(related_obj, related_field, canonical)
related_obj.save()
except TypeError:
getattr(related_obj, related_field).add(canonical)
getattr(related_obj, related_field).remove(obj)
def copy_data(canonical, obj):
"""try to get the most data possible"""
for data_field in obj._meta.get_fields():
if not hasattr(data_field, "activitypub_field"):
continue
data_value = getattr(obj, data_field.name)
if not data_value:
continue
if not getattr(canonical, data_field.name):
print("setting data field", data_field.name, data_value)
setattr(canonical, data_field.name, data_value)
canonical.save()
def merge_objects(canonical, obj):
copy_data(canonical, obj)
update_related(canonical, obj)
# remove the outdated entry
obj.delete()

12
bookwyrm/management/merge_command.py
View file

@ -1,4 +1,3 @@
from bookwyrm.management.merge import merge_objects
from django.core.management.base import BaseCommand
@ -9,6 +8,11 @@ class MergeCommand(BaseCommand):
"""add the arguments for this command"""
parser.add_argument("--canonical", type=int, required=True)
parser.add_argument("--other", type=int, required=True)
parser.add_argument(
"--dry_run",
action="store_true",
help="don't actually merge, only print what would happen",
)
# pylint: disable=no-self-use,unused-argument
def handle(self, *args, **options):
@ -26,4 +30,8 @@ class MergeCommand(BaseCommand):
print("other book doesn’t exist!")
return
merge_objects(canonical, other)
absorbed_fields = other.merge_into(canonical, dry_run=options["dry_run"])
action = "would be" if options["dry_run"] else "has been"
print(f"{other.remote_id} {action} merged into {canonical.remote_id}")
print(f"absorbed fields: {absorbed_fields}")

1
bookwyrm/middleware/__init__.py
View file

@ -1,3 +1,4 @@
""" look at all this nice middleware! """
from .timezone_middleware import TimezoneMiddleware
from .ip_middleware import IPBlocklistMiddleware
from .file_too_big import FileTooBig

30
bookwyrm/middleware/file_too_big.py Normal file
View file

@ -0,0 +1,30 @@
"""Middleware to display a custom 413 error page"""
from django.http import HttpResponse
from django.shortcuts import render
from django.core.exceptions import RequestDataTooBig
class FileTooBig:
"""Middleware to display a custom page when a
RequestDataTooBig exception is thrown"""
def __init__(self, get_response):
"""boilerplate __init__ from Django docs"""
self.get_response = get_response
def __call__(self, request):
"""If RequestDataTooBig is thrown, render the 413 error page"""
try:
body = request.body # pylint: disable=unused-variable
except RequestDataTooBig:
rendered = render(request, "413.html")
response = HttpResponse(rendered)
return response
response = self.get_response(request)
return response

4
bookwyrm/migrations/0179_populate_sort_title.py
View file

@ -45,5 +45,7 @@ class Migration(migrations.Migration):
]
operations = [
migrations.RunPython(populate_sort_title),
migrations.RunPython(
populate_sort_title, reverse_code=migrations.RunPython.noop
),
]

130
bookwyrm/migrations/0182_auto_20231027_1122.py Normal file
View file

@ -0,0 +1,130 @@
# Generated by Django 3.2.20 on 2023-10-27 11:22
import bookwyrm.models.activitypub_mixin
import bookwyrm.models.fields
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0181_merge_20230806_2302"),
]
operations = [
migrations.AddField(
model_name="user",
name="also_known_as",
field=bookwyrm.models.fields.ManyToManyField(to=settings.AUTH_USER_MODEL),
),
migrations.AddField(
model_name="user",
name="moved_to",
field=bookwyrm.models.fields.RemoteIdField(
max_length=255,
null=True,
validators=[bookwyrm.models.fields.validate_remote_id],
),
),
migrations.AlterField(
model_name="notification",
name="notification_type",
field=models.CharField(
choices=[
("FAVORITE", "Favorite"),
("REPLY", "Reply"),
("MENTION", "Mention"),
("TAG", "Tag"),
("FOLLOW", "Follow"),
("FOLLOW_REQUEST", "Follow Request"),
("BOOST", "Boost"),
("IMPORT", "Import"),
("ADD", "Add"),
("REPORT", "Report"),
("LINK_DOMAIN", "Link Domain"),
("INVITE", "Invite"),
("ACCEPT", "Accept"),
("JOIN", "Join"),
("LEAVE", "Leave"),
("REMOVE", "Remove"),
("GROUP_PRIVACY", "Group Privacy"),
("GROUP_NAME", "Group Name"),
("GROUP_DESCRIPTION", "Group Description"),
("MOVE", "Move"),
],
max_length=255,
),
),
migrations.CreateModel(
name="Move",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created_date", models.DateTimeField(auto_now_add=True)),
("updated_date", models.DateTimeField(auto_now=True)),
(
"remote_id",
bookwyrm.models.fields.RemoteIdField(
max_length=255,
null=True,
validators=[bookwyrm.models.fields.validate_remote_id],
),
),
("object", bookwyrm.models.fields.CharField(max_length=255)),
(
"origin",
bookwyrm.models.fields.CharField(
blank=True, default="", max_length=255, null=True
),
),
(
"user",
bookwyrm.models.fields.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"abstract": False,
},
bases=(bookwyrm.models.activitypub_mixin.ActivityMixin, models.Model),
),
migrations.CreateModel(
name="MoveUser",
fields=[
(
"move_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.move",
),
),
(
"target",
bookwyrm.models.fields.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
related_name="move_target",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"abstract": False,
},
bases=("bookwyrm.move",),
),
]

18
bookwyrm/migrations/0183_auto_20231105_1607.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.20 on 2023-11-05 16:07
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0182_auto_20231027_1122"),
]
operations = [
migrations.AddField(
model_name="user",
name="is_deleted",
field=models.BooleanField(default=False),
),
]

35
bookwyrm/migrations/0184_auto_20231106_0421.py Normal file
View file

@ -0,0 +1,35 @@
# Generated by Django 3.2.20 on 2023-11-06 04:21
from django.db import migrations
from bookwyrm.models import User
def update_deleted_users(apps, schema_editor):
"""Find all the users who are deleted, not just inactive, and set deleted"""
users = apps.get_model("bookwyrm", "User")
db_alias = schema_editor.connection.alias
users.objects.using(db_alias).filter(
is_active=False,
deactivation_reason__in=[
"self_deletion",
"moderator_deletion",
],
).update(is_deleted=True)
# differente rules for remote users
users.objects.using(db_alias).filter(is_active=False, local=False,).exclude(
deactivation_reason="moderator_deactivation",
).update(is_deleted=True)
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0183_auto_20231105_1607"),
]
operations = [
migrations.RunPython(
update_deleted_users, reverse_code=migrations.RunPython.noop
),
]

42
bookwyrm/migrations/0185_alter_notification_notification_type.py Normal file
View file

@ -0,0 +1,42 @@
# Generated by Django 3.2.20 on 2023-11-13 22:39
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0184_auto_20231106_0421"),
]
operations = [
migrations.AlterField(
model_name="notification",
name="notification_type",
field=models.CharField(
choices=[
("FAVORITE", "Favorite"),
("BOOST", "Boost"),
("REPLY", "Reply"),
("MENTION", "Mention"),
("TAG", "Tag"),
("FOLLOW", "Follow"),
("FOLLOW_REQUEST", "Follow Request"),
("IMPORT", "Import"),
("ADD", "Add"),
("REPORT", "Report"),
("LINK_DOMAIN", "Link Domain"),
("INVITE", "Invite"),
("ACCEPT", "Accept"),
("JOIN", "Join"),
("LEAVE", "Leave"),
("REMOVE", "Remove"),
("GROUP_PRIVACY", "Group Privacy"),
("GROUP_NAME", "Group Name"),
("GROUP_DESCRIPTION", "Group Description"),
("MOVE", "Move"),
],
max_length=255,
),
),
]

212
bookwyrm/migrations/0186_auto_20231116_0048.py Normal file
View file

@ -0,0 +1,212 @@
# Generated by Django 3.2.20 on 2023-11-16 00:48
from django.conf import settings
import django.contrib.postgres.fields
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0185_alter_notification_notification_type"),
]
operations = [
migrations.CreateModel(
name="ParentJob",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("task_id", models.UUIDField(blank=True, null=True, unique=True)),
(
"created_date",
models.DateTimeField(default=django.utils.timezone.now),
),
(
"updated_date",
models.DateTimeField(default=django.utils.timezone.now),
),
("complete", models.BooleanField(default=False)),
(
"status",
models.CharField(
choices=[
("pending", "Pending"),
("active", "Active"),
("complete", "Complete"),
("stopped", "Stopped"),
("failed", "Failed"),
],
default="pending",
max_length=50,
null=True,
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"abstract": False,
},
),
migrations.AddField(
model_name="sitesettings",
name="user_import_time_limit",
field=models.IntegerField(default=48),
),
migrations.AlterField(
model_name="notification",
name="notification_type",
field=models.CharField(
choices=[
("FAVORITE", "Favorite"),
("BOOST", "Boost"),
("REPLY", "Reply"),
("MENTION", "Mention"),
("TAG", "Tag"),
("FOLLOW", "Follow"),
("FOLLOW_REQUEST", "Follow Request"),
("IMPORT", "Import"),
("USER_IMPORT", "User Import"),
("USER_EXPORT", "User Export"),
("ADD", "Add"),
("REPORT", "Report"),
("LINK_DOMAIN", "Link Domain"),
("INVITE", "Invite"),
("ACCEPT", "Accept"),
("JOIN", "Join"),
("LEAVE", "Leave"),
("REMOVE", "Remove"),
("GROUP_PRIVACY", "Group Privacy"),
("GROUP_NAME", "Group Name"),
("GROUP_DESCRIPTION", "Group Description"),
("MOVE", "Move"),
],
max_length=255,
),
),
migrations.CreateModel(
name="BookwyrmExportJob",
fields=[
(
"parentjob_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.parentjob",
),
),
("export_data", models.FileField(null=True, upload_to="")),
],
options={
"abstract": False,
},
bases=("bookwyrm.parentjob",),
),
migrations.CreateModel(
name="BookwyrmImportJob",
fields=[
(
"parentjob_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.parentjob",
),
),
("archive_file", models.FileField(blank=True, null=True, upload_to="")),
("import_data", models.JSONField(null=True)),
(
"required",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(blank=True, max_length=50),
blank=True,
size=None,
),
),
],
options={
"abstract": False,
},
bases=("bookwyrm.parentjob",),
),
migrations.CreateModel(
name="ChildJob",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("task_id", models.UUIDField(blank=True, null=True, unique=True)),
(
"created_date",
models.DateTimeField(default=django.utils.timezone.now),
),
(
"updated_date",
models.DateTimeField(default=django.utils.timezone.now),
),
("complete", models.BooleanField(default=False)),
(
"status",
models.CharField(
choices=[
("pending", "Pending"),
("active", "Active"),
("complete", "Complete"),
("stopped", "Stopped"),
("failed", "Failed"),
],
default="pending",
max_length=50,
null=True,
),
),
(
"parent_job",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="child_jobs",
to="bookwyrm.parentjob",
),
),
],
options={
"abstract": False,
},
),
migrations.AddField(
model_name="notification",
name="related_user_export",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
to="bookwyrm.bookwyrmexportjob",
),
),
]

48
bookwyrm/migrations/0186_invite_request_notification.py Normal file
View file

@ -0,0 +1,48 @@
# Generated by Django 3.2.20 on 2023-11-14 10:02
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0185_alter_notification_notification_type"),
]
operations = [
migrations.AddField(
model_name="notification",
name="related_invite_requests",
field=models.ManyToManyField(to="bookwyrm.InviteRequest"),
),
migrations.AlterField(
model_name="notification",
name="notification_type",
field=models.CharField(
choices=[
("FAVORITE", "Favorite"),
("BOOST", "Boost"),
("REPLY", "Reply"),
("MENTION", "Mention"),
("TAG", "Tag"),
("FOLLOW", "Follow"),
("FOLLOW_REQUEST", "Follow Request"),
("IMPORT", "Import"),
("ADD", "Add"),
("REPORT", "Report"),
("LINK_DOMAIN", "Link Domain"),
("INVITE_REQUEST", "Invite Request"),
("INVITE", "Invite"),
("ACCEPT", "Accept"),
("JOIN", "Join"),
("LEAVE", "Leave"),
("REMOVE", "Remove"),
("GROUP_PRIVACY", "Group Privacy"),
("GROUP_NAME", "Group Name"),
("GROUP_DESCRIPTION", "Group Description"),
("MOVE", "Move"),
],
max_length=255,
),
),
]

54
bookwyrm/migrations/0187_partial_publication_dates.py Normal file
View file

@ -0,0 +1,54 @@
# Generated by Django 3.2.20 on 2023-11-09 16:57
import bookwyrm.models.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0186_invite_request_notification"),
]
operations = [
migrations.AddField(
model_name="book",
name="first_published_date_precision",
field=models.CharField(
blank=True,
choices=[
("DAY", "Day prec."),
("MONTH", "Month prec."),
("YEAR", "Year prec."),
],
editable=False,
max_length=10,
null=True,
),
),
migrations.AddField(
model_name="book",
name="published_date_precision",
field=models.CharField(
blank=True,
choices=[
("DAY", "Day prec."),
("MONTH", "Month prec."),
("YEAR", "Year prec."),
],
editable=False,
max_length=10,
null=True,
),
),
migrations.AlterField(
model_name="book",
name="first_published_date",
field=bookwyrm.models.fields.PartialDateField(blank=True, null=True),
),
migrations.AlterField(
model_name="book",
name="published_date",
field=bookwyrm.models.fields.PartialDateField(blank=True, null=True),
),
]

18
bookwyrm/migrations/0188_theme_loads.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.23 on 2023-11-20 18:02
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0187_partial_publication_dates"),
]
operations = [
migrations.AddField(
model_name="theme",
name="loads",
field=models.BooleanField(blank=True, null=True),
),
]

45
bookwyrm/migrations/0189_alter_user_preferred_language.py Normal file
View file

@ -0,0 +1,45 @@
# Generated by Django 3.2.23 on 2023-12-12 23:42
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0188_theme_loads"),
]
operations = [
migrations.AlterField(
model_name="user",
name="preferred_language",
field=models.CharField(
blank=True,
choices=[
("en-us", "English"),
("ca-es", "Català (Catalan)"),
("de-de", "Deutsch (German)"),
("eo-uy", "Esperanto (Esperanto)"),
("es-es", "Español (Spanish)"),
("eu-es", "Euskara (Basque)"),
("gl-es", "Galego (Galician)"),
("it-it", "Italiano (Italian)"),
("fi-fi", "Suomi (Finnish)"),
("fr-fr", "Français (French)"),
("lt-lt", "Lietuvių (Lithuanian)"),
("nl-nl", "Nederlands (Dutch)"),
("no-no", "Norsk (Norwegian)"),
("pl-pl", "Polski (Polish)"),
("pt-br", "Português do Brasil (Brazilian Portuguese)"),
("pt-pt", "Português Europeu (European Portuguese)"),
("ro-ro", "Română (Romanian)"),
("sv-se", "Svenska (Swedish)"),
("uk-ua", "Українська (Ukrainian)"),
("zh-hans", "简体中文 (Simplified Chinese)"),
("zh-hant", "繁體中文 (Traditional Chinese)"),
],
max_length=255,
null=True,
),
),
]

13
bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.23 on 2023-11-22 10:16
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0186_auto_20231116_0048"),
("bookwyrm", "0188_theme_loads"),
]
operations = []

45
bookwyrm/migrations/0190_alter_notification_notification_type.py Normal file
View file

@ -0,0 +1,45 @@
# Generated by Django 3.2.23 on 2023-11-23 19:49
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0189_merge_0186_auto_20231116_0048_0188_theme_loads"),
]
operations = [
migrations.AlterField(
model_name="notification",
name="notification_type",
field=models.CharField(
choices=[
("FAVORITE", "Favorite"),
("BOOST", "Boost"),
("REPLY", "Reply"),
("MENTION", "Mention"),
("TAG", "Tag"),
("FOLLOW", "Follow"),
("FOLLOW_REQUEST", "Follow Request"),
("IMPORT", "Import"),
("USER_IMPORT", "User Import"),
("USER_EXPORT", "User Export"),
("ADD", "Add"),
("REPORT", "Report"),
("LINK_DOMAIN", "Link Domain"),
("INVITE_REQUEST", "Invite Request"),
("INVITE", "Invite"),
("ACCEPT", "Accept"),
("JOIN", "Join"),
("LEAVE", "Leave"),
("REMOVE", "Remove"),
("GROUP_PRIVACY", "Group Privacy"),
("GROUP_NAME", "Group Name"),
("GROUP_DESCRIPTION", "Group Description"),
("MOVE", "Move"),
],
max_length=255,
),
),
]

16
bookwyrm/migrations/0190_book_search_updates.py Normal file
View file

@ -0,0 +1,16 @@
# Generated by Django 3.2.20 on 2023-11-24 17:11
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0188_theme_loads"),
]
operations = [
migrations.RemoveIndex(
model_name="author",
name="bookwyrm_au_search__b050a8_gin",
),
]

13
bookwyrm/migrations/0191_merge_20240102_0326.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.23 on 2024-01-02 03:26
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0189_alter_user_preferred_language"),
("bookwyrm", "0190_alter_notification_notification_type"),
]
operations = []

76
bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py Normal file
View file

@ -0,0 +1,76 @@
# Generated by Django 3.2.20 on 2023-11-25 00:47
from importlib import import_module
import re
from django.db import migrations
import pgtrigger.compiler
import pgtrigger.migrations
trigger_migration = import_module("bookwyrm.migrations.0077_auto_20210623_2155")
# it's _very_ convenient for development that this migration be reversible
search_vector_trigger = trigger_migration.Migration.operations[4]
author_search_vector_trigger = trigger_migration.Migration.operations[5]
assert re.search(r"\bCREATE TRIGGER search_vector_trigger\b", search_vector_trigger.sql)
assert re.search(
r"\bCREATE TRIGGER author_search_vector_trigger\b",
author_search_vector_trigger.sql,
)
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0190_book_search_updates"),
]
operations = [
pgtrigger.migrations.AddTrigger(
model_name="book",
trigger=pgtrigger.compiler.Trigger(
name="update_search_vector_on_book_edit",
sql=pgtrigger.compiler.UpsertTriggerSql(
func="new.search_vector := setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(bookwyrm_author.name), ' '), '')), 'C') FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id ) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D');RETURN NEW;",
hash="77d6399497c0a89b0bf09d296e33c396da63705c",
operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"',
pgid="pgtrigger_update_search_vector_on_book_edit_bec58",
table="bookwyrm_book",
when="BEFORE",
),
),
),
pgtrigger.migrations.AddTrigger(
model_name="author",
trigger=pgtrigger.compiler.Trigger(
name="reset_search_vector_on_author_edit",
sql=pgtrigger.compiler.UpsertTriggerSql(
func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;",
hash="e7bbf08711ff3724c58f4d92fb7a082ffb3d7826",
operation='UPDATE OF "name"',
pgid="pgtrigger_reset_search_vector_on_author_edit_a447c",
table="bookwyrm_author",
when="AFTER",
),
),
),
migrations.RunSQL(
sql="""DROP TRIGGER IF EXISTS search_vector_trigger ON bookwyrm_book;
DROP FUNCTION IF EXISTS book_trigger;
""",
reverse_sql=search_vector_trigger.sql,
),
migrations.RunSQL(
sql="""DROP TRIGGER IF EXISTS author_search_vector_trigger ON bookwyrm_author;
DROP FUNCTION IF EXISTS author_trigger;
""",
reverse_sql=author_search_vector_trigger.sql,
),
migrations.RunSQL(
# Recalculate book search vector for any missed author name changes
# due to bug in JOIN in the old trigger.
sql="UPDATE bookwyrm_book SET search_vector = NULL;",
reverse_sql=migrations.RunSQL.noop,
),
]

23
bookwyrm/migrations/0192_make_page_positions_text.py Normal file
View file

@ -0,0 +1,23 @@
# Generated by Django 3.2.23 on 2024-01-04 23:56
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0191_merge_20240102_0326"),
]
operations = [
migrations.AlterField(
model_name="quotation",
name="endposition",
field=models.TextField(blank=True, null=True),
),
migrations.AlterField(
model_name="quotation",
name="position",
field=models.TextField(blank=True, null=True),
),
]

18
bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.23 on 2024-01-02 19:36
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0191_merge_20240102_0326"),
]
operations = [
migrations.RenameField(
model_name="sitesettings",
old_name="version",
new_name="available_version",
),
]

18
bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.23 on 2024-01-16 10:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0191_merge_20240102_0326"),
]
operations = [
migrations.AddField(
model_name="sitesettings",
name="user_exports_enabled",
field=models.BooleanField(default=False),
),
]

92
bookwyrm/migrations/0193_auto_20240128_0249.py Normal file
View file

@ -0,0 +1,92 @@
# Generated by Django 3.2.23 on 2024-01-28 02:49
import bookwyrm.storage_backends
import django.core.serializers.json
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0192_sitesettings_user_exports_enabled"),
]
operations = [
migrations.AddField(
model_name="bookwyrmexportjob",
name="export_json",
field=models.JSONField(
encoder=django.core.serializers.json.DjangoJSONEncoder, null=True
),
),
migrations.AddField(
model_name="bookwyrmexportjob",
name="json_completed",
field=models.BooleanField(default=False),
),
migrations.AlterField(
model_name="bookwyrmexportjob",
name="export_data",
field=models.FileField(
null=True,
storage=bookwyrm.storage_backends.ExportsFileStorage,
upload_to="",
),
),
migrations.CreateModel(
name="AddFileToTar",
fields=[
(
"childjob_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.childjob",
),
),
(
"parent_export_job",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="child_edition_export_jobs",
to="bookwyrm.bookwyrmexportjob",
),
),
],
options={
"abstract": False,
},
bases=("bookwyrm.childjob",),
),
migrations.CreateModel(
name="AddBookToUserExportJob",
fields=[
(
"childjob_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="bookwyrm.childjob",
),
),
(
"edition",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="bookwyrm.edition",
),
),
],
options={
"abstract": False,
},
bases=("bookwyrm.childjob",),
),
]

13
bookwyrm/migrations/0193_merge_20240203_1539.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.23 on 2024-02-03 15:39
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0192_make_page_positions_text"),
("bookwyrm", "0192_sitesettings_user_exports_enabled"),
]
operations = []

13
bookwyrm/migrations/0194_merge_20240203_1619.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.23 on 2024-02-03 16:19
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0192_rename_version_sitesettings_available_version"),
("bookwyrm", "0193_merge_20240203_1539"),
]
operations = []

46
bookwyrm/migrations/0195_alter_user_preferred_language.py Normal file
View file

@ -0,0 +1,46 @@
# Generated by Django 3.2.23 on 2024-02-21 00:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0194_merge_20240203_1619"),
]
operations = [
migrations.AlterField(
model_name="user",
name="preferred_language",
field=models.CharField(
blank=True,
choices=[
("en-us", "English"),
("ca-es", "Català (Catalan)"),
("de-de", "Deutsch (German)"),
("eo-uy", "Esperanto (Esperanto)"),
("es-es", "Español (Spanish)"),
("eu-es", "Euskara (Basque)"),
("gl-es", "Galego (Galician)"),
("it-it", "Italiano (Italian)"),
("ko-kr", "한국어 (Korean)"),
("fi-fi", "Suomi (Finnish)"),
("fr-fr", "Français (French)"),
("lt-lt", "Lietuvių (Lithuanian)"),
("nl-nl", "Nederlands (Dutch)"),
("no-no", "Norsk (Norwegian)"),
("pl-pl", "Polski (Polish)"),
("pt-br", "Português do Brasil (Brazilian Portuguese)"),
("pt-pt", "Português Europeu (European Portuguese)"),
("ro-ro", "Română (Romanian)"),
("sv-se", "Svenska (Swedish)"),
("uk-ua", "Українська (Ukrainian)"),
("zh-hans", "简体中文 (Simplified Chinese)"),
("zh-hant", "繁體中文 (Traditional Chinese)"),
],
max_length=255,
null=True,
),
),
]

13
bookwyrm/migrations/0196_merge_20240318_1737.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.23 on 2024-03-18 17:37
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0193_auto_20240128_0249"),
("bookwyrm", "0195_alter_user_preferred_language"),
]
operations = []

13
bookwyrm/migrations/0196_merge_pr3134_into_main.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.23 on 2024-03-18 00:48
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0191_migrate_search_vec_triggers_to_pgtriggers"),
("bookwyrm", "0195_alter_user_preferred_language"),
]
operations = []

41
bookwyrm/migrations/0197_author_search_vector.py Normal file
View file

@ -0,0 +1,41 @@
# Generated by Django 3.2.25 on 2024-03-20 15:15
import django.contrib.postgres.indexes
from django.db import migrations
import pgtrigger.compiler
import pgtrigger.migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0196_merge_pr3134_into_main"),
]
operations = [
migrations.AddIndex(
model_name="author",
index=django.contrib.postgres.indexes.GinIndex(
fields=["search_vector"], name="bookwyrm_au_search__b050a8_gin"
),
),
pgtrigger.migrations.AddTrigger(
model_name="author",
trigger=pgtrigger.compiler.Trigger(
name="update_search_vector_on_author_edit",
sql=pgtrigger.compiler.UpsertTriggerSql(
func="new.search_vector := setweight(to_tsvector('simple', new.name), 'A') || setweight(to_tsvector('simple', coalesce(array_to_string(new.aliases, ' '), '')), 'B');RETURN NEW;",
hash="b97919016236d74d0ade51a0769a173ea269da64",
operation='INSERT OR UPDATE OF "name", "aliases", "search_vector"',
pgid="pgtrigger_update_search_vector_on_author_edit_c61cb",
table="bookwyrm_author",
when="BEFORE",
),
),
),
migrations.RunSQL(
# Calculate search vector for all Authors.
sql="UPDATE bookwyrm_author SET search_vector = NULL;",
reverse_sql="UPDATE bookwyrm_author SET search_vector = NULL;",
),
]

13
bookwyrm/migrations/0197_merge_20240324_0235.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.25 on 2024-03-24 02:35
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0196_merge_20240318_1737"),
("bookwyrm", "0196_merge_pr3134_into_main"),
]
operations = []

48
bookwyrm/migrations/0197_mergedauthor_mergedbook.py Normal file
View file

@ -0,0 +1,48 @@
# Generated by Django 3.2.24 on 2024-02-28 21:30
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0196_merge_pr3134_into_main"),
]
operations = [
migrations.CreateModel(
name="MergedBook",
fields=[
("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
(
"merged_into",
models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
related_name="absorbed",
to="bookwyrm.book",
),
),
],
options={
"abstract": False,
},
),
migrations.CreateModel(
name="MergedAuthor",
fields=[
("deleted_id", models.IntegerField(primary_key=True, serialize=False)),
(
"merged_into",
models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
related_name="absorbed",
to="bookwyrm.author",
),
),
],
options={
"abstract": False,
},
),
]

23
bookwyrm/migrations/0198_alter_bookwyrmexportjob_export_data.py Normal file
View file

@ -0,0 +1,23 @@
# Generated by Django 3.2.25 on 2024-03-26 11:37
import bookwyrm.models.bookwyrm_export_job
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0197_merge_20240324_0235"),
]
operations = [
migrations.AlterField(
model_name="bookwyrmexportjob",
name="export_data",
field=models.FileField(
null=True,
storage=bookwyrm.models.bookwyrm_export_job.select_exports_storage,
upload_to="",
),
),
]

57
bookwyrm/migrations/0198_book_search_vector_author_aliases.py Normal file
View file

@ -0,0 +1,57 @@
# Generated by Django 3.2.25 on 2024-03-20 15:52
from django.db import migrations
import pgtrigger.compiler
import pgtrigger.migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0197_author_search_vector"),
]
operations = [
pgtrigger.migrations.RemoveTrigger(
model_name="author",
name="reset_search_vector_on_author_edit",
),
pgtrigger.migrations.RemoveTrigger(
model_name="book",
name="update_search_vector_on_book_edit",
),
pgtrigger.migrations.AddTrigger(
model_name="author",
trigger=pgtrigger.compiler.Trigger(
name="reset_book_search_vector_on_author_edit",
sql=pgtrigger.compiler.UpsertTriggerSql(
func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;",
hash="68422c0f29879c5802b82159dde45297eff53e73",
operation='UPDATE OF "name", "aliases"',
pgid="pgtrigger_reset_book_search_vector_on_author_edit_a50c7",
table="bookwyrm_author",
when="AFTER",
),
),
),
pgtrigger.migrations.AddTrigger(
model_name="book",
trigger=pgtrigger.compiler.Trigger(
name="update_search_vector_on_book_edit",
sql=pgtrigger.compiler.UpsertTriggerSql(
func="WITH author_names AS (SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id ) SELECT setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(name_and_aliases), ' '), '')), 'C') FROM author_names) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D') INTO new.search_vector;RETURN NEW;",
hash="9324f5ca76a6f5e63931881d62d11da11f595b2c",
operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"',
pgid="pgtrigger_update_search_vector_on_book_edit_bec58",
table="bookwyrm_book",
when="BEFORE",
),
),
),
migrations.RunSQL(
# Recalculate search vector for all Books because it now includes
# Author aliases.
sql="UPDATE bookwyrm_book SET search_vector = NULL;",
reverse_sql="UPDATE bookwyrm_book SET search_vector = NULL;",
),
]

13
bookwyrm/migrations/0199_merge_20240326_1217.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.25 on 2024-03-26 12:17
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0198_alter_bookwyrmexportjob_export_data"),
("bookwyrm", "0198_book_search_vector_author_aliases"),
]
operations = []

19
bookwyrm/migrations/0199_status_bookwyrm_st_remote__06aeba_idx.py Normal file
View file

@ -0,0 +1,19 @@
# Generated by Django 3.2.25 on 2024-04-02 19:53
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0198_book_search_vector_author_aliases"),
]
operations = [
migrations.AddIndex(
model_name="status",
index=models.Index(
fields=["remote_id"], name="bookwyrm_st_remote__06aeba_idx"
),
),
]

27
bookwyrm/migrations/0200_auto_20240327_1914.py Normal file
View file

@ -0,0 +1,27 @@
# Generated by Django 3.2.25 on 2024-03-27 19:14
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0199_merge_20240326_1217"),
]
operations = [
migrations.RemoveField(
model_name="addfiletotar",
name="childjob_ptr",
),
migrations.RemoveField(
model_name="addfiletotar",
name="parent_export_job",
),
migrations.DeleteModel(
name="AddBookToUserExportJob",
),
migrations.DeleteModel(
name="AddFileToTar",
),
]

19
bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py Normal file
View file

@ -0,0 +1,19 @@
# Generated by Django 3.2.25 on 2024-04-03 19:05
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0199_status_bookwyrm_st_remote__06aeba_idx"),
]
operations = [
migrations.AddIndex(
model_name="status",
index=models.Index(
fields=["thread_id"], name="bookwyrm_st_thread__cf064f_idx"
),
),
]

19
bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py Normal file
View file

@ -0,0 +1,19 @@
# Generated by Django 3.2.25 on 2024-04-03 19:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0200_status_bookwyrm_st_thread__cf064f_idx"),
]
operations = [
migrations.AddIndex(
model_name="keypair",
index=models.Index(
fields=["remote_id"], name="bookwyrm_ke_remote__472927_idx"
),
),
]

19
bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py Normal file
View file

@ -0,0 +1,19 @@
# Generated by Django 3.2.25 on 2024-04-03 19:14
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0201_keypair_bookwyrm_ke_remote__472927_idx"),
]
operations = [
migrations.AddIndex(
model_name="user",
index=models.Index(
fields=["username"], name="bookwyrm_us_usernam_b2546d_idx"
),
),
]

19
bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py Normal file
View file

@ -0,0 +1,19 @@
# Generated by Django 3.2.25 on 2024-04-03 19:22
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0202_user_bookwyrm_us_usernam_b2546d_idx"),
]
operations = [
migrations.AddIndex(
model_name="user",
index=models.Index(
fields=["is_active", "local"], name="bookwyrm_us_is_acti_972dc4_idx"
),
),
]

13
bookwyrm/migrations/0204_merge_20240409_1042.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.25 on 2024-04-09 10:42
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0197_mergedauthor_mergedbook"),
("bookwyrm", "0203_user_bookwyrm_us_is_acti_972dc4_idx"),
]
operations = []

13
bookwyrm/migrations/0205_merge_20240413_0232.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.25 on 2024-04-13 02:32
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0200_auto_20240327_1914"),
("bookwyrm", "0204_merge_20240409_1042"),
]
operations = []

6
bookwyrm/models/__init__.py
View file

@ -26,13 +26,17 @@ from .federated_server import FederatedServer
from .group import Group, GroupMember, GroupMemberInvitation
from .import_job import ImportJob, ImportItem
from .bookwyrm_import_job import BookwyrmImportJob
from .bookwyrm_export_job import BookwyrmExportJob
from .move import MoveUser
from .site import SiteSettings, Theme, SiteInvite
from .site import PasswordReset, InviteRequest
from .announcement import Announcement
from .antispam import EmailBlocklist, IPBlocklist, AutoMod, automod_task
from .notification import Notification
from .notification import Notification, NotificationType
from .hashtag import Hashtag

27
bookwyrm/models/activitypub_mixin.py
View file

@ -152,8 +152,9 @@ class ActivitypubMixin:
# find anyone who's tagged in a status, for example
mentions = self.recipients if hasattr(self, "recipients") else []
# we always send activities to explicitly mentioned users' inboxes
recipients = [u.inbox for u in mentions or [] if not u.local]
# we always send activities to explicitly mentioned users (using shared inboxes
# where available to avoid duplicate submissions to a given instance)
recipients = {u.shared_inbox or u.inbox for u in mentions if not u.local}
# unless it's a dm, all the followers should receive the activity
if privacy != "direct":
@ -173,18 +174,18 @@ class ActivitypubMixin:
if user:
queryset = queryset.filter(following=user)
# ideally, we will send to shared inboxes for efficiency
shared_inboxes = (
queryset.filter(shared_inbox__isnull=False)
.values_list("shared_inbox", flat=True)
.distinct()
# as above, we prefer shared inboxes if available
recipients.update(
queryset.filter(shared_inbox__isnull=False).values_list(
"shared_inbox", flat=True
)
)
# but not everyone has a shared inbox
inboxes = queryset.filter(shared_inbox__isnull=True).values_list(
"inbox", flat=True
recipients.update(
queryset.filter(shared_inbox__isnull=True).values_list(
"inbox", flat=True
)
)
recipients += list(shared_inboxes) + list(inboxes)
return list(set(recipients))
return list(recipients)
def to_activity_dataclass(self):
"""convert from a model to an activity"""
@ -602,7 +603,7 @@ def to_ordered_collection_page(
if activity_page.has_next():
next_page = f"{remote_id}?page={activity_page.next_page_number()}"
if activity_page.has_previous():
prev_page = f"{remote_id}?page=%d{activity_page.previous_page_number()}"
prev_page = f"{remote_id}?page={activity_page.previous_page_number()}"
return activitypub.OrderedCollectionPage(
id=f"{remote_id}?page={page}",
partOf=remote_id,

3
bookwyrm/models/antispam.py
View file

@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _
from bookwyrm.tasks import app, MISC
from .base_model import BookWyrmModel
from .notification import NotificationType
from .user import User
@ -80,7 +81,7 @@ def automod_task():
with transaction.atomic():
for admin in admins:
notification, _ = notification_model.objects.get_or_create(
user=admin, notification_type=notification_model.REPORT, read=False
user=admin, notification_type=NotificationType.REPORT, read=False
)
notification.related_reports.set(reports)

60
bookwyrm/models/author.py
View file

@ -1,18 +1,25 @@
""" database schema for info about authors """
import re
from django.contrib.postgres.indexes import GinIndex
from typing import Tuple, Any
from django.db import models
from django.contrib.postgres.indexes import GinIndex
import pgtrigger
from bookwyrm import activitypub
from bookwyrm.settings import DOMAIN
from bookwyrm.settings import BASE_URL
from bookwyrm.utils.db import format_trigger
from .book import BookDataModel
from .book import BookDataModel, MergedAuthor
from . import fields
class Author(BookDataModel):
"""basic biographic info"""
merged_model = MergedAuthor
wikipedia_link = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
@ -38,7 +45,7 @@ class Author(BookDataModel):
)
bio = fields.HtmlField(null=True, blank=True)
def save(self, *args, **kwargs):
def save(self, *args: Tuple[Any, ...], **kwargs: dict[str, Any]) -> None:
"""normalize isni format"""
if self.isni:
self.isni = re.sub(r"\s", "", self.isni)
@ -63,11 +70,48 @@ class Author(BookDataModel):
def get_remote_id(self):
"""editions and works both use "book" instead of model_name"""
return f"https://{DOMAIN}/author/{self.id}"
activity_serializer = activitypub.Author
return f"{BASE_URL}/author/{self.id}"
class Meta:
"""sets up postgres GIN index field"""
"""sets up indexes and triggers"""
# pylint: disable=line-too-long
indexes = (GinIndex(fields=["search_vector"]),)
triggers = [
pgtrigger.Trigger(
name="update_search_vector_on_author_edit",
when=pgtrigger.Before,
operation=pgtrigger.Insert
| pgtrigger.UpdateOf("name", "aliases", "search_vector"),
func=format_trigger(
"""new.search_vector :=
-- author name, with priority A
setweight(to_tsvector('simple', new.name), 'A') ||
-- author aliases, with priority B
setweight(to_tsvector('simple', coalesce(array_to_string(new.aliases, ' '), '')), 'B');
RETURN new;
"""
),
),
pgtrigger.Trigger(
name="reset_book_search_vector_on_author_edit",
when=pgtrigger.After,
operation=pgtrigger.UpdateOf("name", "aliases"),
func=format_trigger(
"""WITH updated_books AS (
SELECT book_id
FROM bookwyrm_book_authors
WHERE author_id = new.id
)
UPDATE bookwyrm_book
SET search_vector = ''
FROM updated_books
WHERE id = updated_books.book_id;
RETURN new;
"""
),
),
]
activity_serializer = activitypub.Author

6
bookwyrm/models/base_model.py
View file

@ -10,7 +10,7 @@ from django.http import Http404
from django.utils.translation import gettext_lazy as _
from django.utils.text import slugify
from bookwyrm.settings import DOMAIN
from bookwyrm.settings import BASE_URL
from .fields import RemoteIdField
@ -38,7 +38,7 @@ class BookWyrmModel(models.Model):
def get_remote_id(self):
"""generate the url that resolves to the local object, without a slug"""
base_path = f"https://{DOMAIN}"
base_path = BASE_URL
if hasattr(self, "user"):
base_path = f"{base_path}{self.user.local_path}"
@ -53,7 +53,7 @@ class BookWyrmModel(models.Model):
@property
def local_path(self):
"""how to link to this object in the local app, with a slug"""
local = self.get_remote_id().replace(f"https://{DOMAIN}", "")
local = self.get_remote_id().replace(BASE_URL, "")
name = None
if hasattr(self, "name_field"):

191
bookwyrm/models/book.py
View file

@ -1,29 +1,33 @@
""" database schema for books and shelves """
from itertools import chain
import re
from typing import Any
from typing import Any, Dict
from typing_extensions import Self
from django.contrib.postgres.search import SearchVectorField
from django.contrib.postgres.indexes import GinIndex
from django.core.cache import cache
from django.db import models, transaction
from django.db.models import Prefetch
from django.db.models import Prefetch, ManyToManyField
from django.dispatch import receiver
from django.utils.translation import gettext_lazy as _
from model_utils import FieldTracker
from model_utils.managers import InheritanceManager
from imagekit.models import ImageSpecField
import pgtrigger
from bookwyrm import activitypub
from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator
from bookwyrm.preview_images import generate_edition_preview_image_task
from bookwyrm.settings import (
DOMAIN,
BASE_URL,
DEFAULT_LANGUAGE,
LANGUAGE_ARTICLES,
ENABLE_PREVIEW_IMAGES,
ENABLE_THUMBNAIL_GENERATION,
)
from bookwyrm.utils.db import format_trigger
from .activitypub_mixin import OrderedCollectionPageMixin, ObjectMixin
from .base_model import BookWyrmModel
@ -106,10 +110,115 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
"""only send book data updates to other bookwyrm instances"""
super().broadcast(activity, sender, software=software, **kwargs)
def merge_into(self, canonical: Self, dry_run=False) -> Dict[str, Any]:
"""merge this entity into another entity"""
if canonical.id == self.id:
raise ValueError(f"Cannot merge {self} into itself")
absorbed_fields = canonical.absorb_data_from(self, dry_run=dry_run)
if dry_run:
return absorbed_fields
canonical.save()
self.merged_model.objects.create(deleted_id=self.id, merged_into=canonical)
# move related models to canonical
related_models = [
(r.remote_field.name, r.related_model) for r in self._meta.related_objects
]
# pylint: disable=protected-access
for related_field, related_model in related_models:
# Skip the ManyToMany fields that aren’t auto-created. These
# should have a corresponding OneToMany field in the model for
# the linking table anyway. If we update it through that model
# instead then we won’t lose the extra fields in the linking
# table.
# pylint: disable=protected-access
related_field_obj = related_model._meta.get_field(related_field)
if isinstance(related_field_obj, ManyToManyField):
through = related_field_obj.remote_field.through
if not through._meta.auto_created:
continue
related_objs = related_model.objects.filter(**{related_field: self})
for related_obj in related_objs:
try:
setattr(related_obj, related_field, canonical)
related_obj.save()
except TypeError:
getattr(related_obj, related_field).add(canonical)
getattr(related_obj, related_field).remove(self)
self.delete()
return absorbed_fields
def absorb_data_from(self, other: Self, dry_run=False) -> Dict[str, Any]:
"""fill empty fields with values from another entity"""
absorbed_fields = {}
for data_field in self._meta.get_fields():
if not hasattr(data_field, "activitypub_field"):
continue
canonical_value = getattr(self, data_field.name)
other_value = getattr(other, data_field.name)
if not other_value:
continue
if isinstance(data_field, fields.ArrayField):
if new_values := list(set(other_value) - set(canonical_value)):
# append at the end (in no particular order)
if not dry_run:
setattr(self, data_field.name, canonical_value + new_values)
absorbed_fields[data_field.name] = new_values
elif isinstance(data_field, fields.PartialDateField):
if (
(not canonical_value)
or (other_value.has_day and not canonical_value.has_day)
or (other_value.has_month and not canonical_value.has_month)
):
if not dry_run:
setattr(self, data_field.name, other_value)
absorbed_fields[data_field.name] = other_value
else:
if not canonical_value:
if not dry_run:
setattr(self, data_field.name, other_value)
absorbed_fields[data_field.name] = other_value
return absorbed_fields
class MergedBookDataModel(models.Model):
"""a BookDataModel instance that has been merged into another instance. kept
to be able to redirect old URLs"""
deleted_id = models.IntegerField(primary_key=True)
class Meta:
"""abstract just like BookDataModel"""
abstract = True
class MergedBook(MergedBookDataModel):
"""an Book that has been merged into another one"""
merged_into = models.ForeignKey(
"Book", on_delete=models.PROTECT, related_name="absorbed"
)
class MergedAuthor(MergedBookDataModel):
"""an Author that has been merged into another one"""
merged_into = models.ForeignKey(
"Author", on_delete=models.PROTECT, related_name="absorbed"
)
class Book(BookDataModel):
"""a generic book, which can mean either an edition or a work"""
merged_model = MergedBook
connector = models.ForeignKey("Connector", on_delete=models.PROTECT, null=True)
# book/work metadata
@ -135,8 +244,8 @@ class Book(BookDataModel):
preview_image = models.ImageField(
upload_to="previews/covers/", blank=True, null=True
)
first_published_date = fields.DateTimeField(blank=True, null=True)
published_date = fields.DateTimeField(blank=True, null=True)
first_published_date = fields.PartialDateField(blank=True, null=True)
published_date = fields.PartialDateField(blank=True, null=True)
objects = InheritanceManager()
field_tracker = FieldTracker(fields=["authors", "title", "subtitle", "cover"])
@ -190,9 +299,13 @@ class Book(BookDataModel):
"""properties of this edition, as a string"""
items = [
self.physical_format if hasattr(self, "physical_format") else None,
f"{self.languages[0]} language"
if self.languages and self.languages[0] and self.languages[0] != "English"
else None,
(
f"{self.languages[0]} language"
if self.languages
and self.languages[0]
and self.languages[0] != "English"
else None
),
str(self.published_date.year) if self.published_date else None,
", ".join(self.publishers) if hasattr(self, "publishers") else None,
]
@ -201,21 +314,20 @@ class Book(BookDataModel):
@property
def alt_text(self):
"""image alt test"""
text = self.title
if self.edition_info:
text += f" ({self.edition_info})"
return text
author = f"{name}: " if (name := self.author_text) else ""
edition = f" ({info})" if (info := self.edition_info) else ""
return f"{author}{self.title}{edition}"
def save(self, *args: Any, **kwargs: Any) -> None:
"""can't be abstract for query reasons, but you shouldn't USE it"""
if not isinstance(self, Edition) and not isinstance(self, Work):
if not isinstance(self, (Edition, Work)):
raise ValueError("Books should be added as Editions or Works")
return super().save(*args, **kwargs)
def get_remote_id(self):
"""editions and works both use "book" instead of model_name"""
return f"https://{DOMAIN}/book/{self.id}"
return f"{BASE_URL}/book/{self.id}"
def guess_sort_title(self):
"""Get a best-guess sort title for the current book"""
@ -233,9 +345,49 @@ class Book(BookDataModel):
)
class Meta:
"""sets up postgres GIN index field"""
"""set up indexes and triggers"""
# pylint: disable=line-too-long
indexes = (GinIndex(fields=["search_vector"]),)
triggers = [
pgtrigger.Trigger(
name="update_search_vector_on_book_edit",
when=pgtrigger.Before,
operation=pgtrigger.Insert
| pgtrigger.UpdateOf("title", "subtitle", "series", "search_vector"),
func=format_trigger(
"""
WITH author_names AS (
SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases
FROM bookwyrm_author
LEFT JOIN bookwyrm_book_authors
ON bookwyrm_author.id = bookwyrm_book_authors.author_id
WHERE bookwyrm_book_authors.book_id = new.id
)
SELECT
-- title, with priority A (parse in English, default to simple if empty)
setweight(COALESCE(nullif(
to_tsvector('english', new.title), ''),
to_tsvector('simple', new.title)), 'A') ||
-- subtitle, with priority B (always in English?)
setweight(to_tsvector('english', COALESCE(new.subtitle, '')), 'B') ||
-- list of authors names and aliases (with priority C)
(SELECT setweight(to_tsvector('simple', COALESCE(array_to_string(ARRAY_AGG(name_and_aliases), ' '), '')), 'C')
FROM author_names
) ||
--- last: series name, with lowest priority
setweight(to_tsvector('english', COALESCE(new.series, '')), 'D')
INTO new.search_vector;
RETURN new;
"""
),
)
]
class Work(OrderedCollectionPageMixin, Book):
@ -367,9 +519,9 @@ class Edition(Book):
# normalize isbn format
if self.isbn_10:
self.isbn_10 = re.sub(r"[^0-9X]", "", self.isbn_10)
self.isbn_10 = normalize_isbn(self.isbn_10)
if self.isbn_13:
self.isbn_13 = re.sub(r"[^0-9X]", "", self.isbn_13)
self.isbn_13 = normalize_isbn(self.isbn_13)
# set rank
self.edition_rank = self.get_rank()
@ -464,6 +616,11 @@ def isbn_13_to_10(isbn_13):
return converted + str(checkdigit)
def normalize_isbn(isbn):
"""Remove unexpected characters from ISBN 10 or 13"""
return re.sub(r"[^0-9X]", "", isbn)
# pylint: disable=unused-argument
@receiver(models.signals.post_save, sender=Edition)
def preview_image(instance, *args, **kwargs):

334
bookwyrm/models/bookwyrm_export_job.py Normal file
View file

@ -0,0 +1,334 @@
"""Export user account to tar.gz file for import into another Bookwyrm instance"""
import logging
import os
from boto3.session import Session as BotoSession
from s3_tar import S3Tar
from django.db.models import BooleanField, FileField, JSONField
from django.db.models import Q
from django.core.serializers.json import DjangoJSONEncoder
from django.core.files.base import ContentFile
from django.utils.module_loading import import_string
from bookwyrm import settings, storage_backends
from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, ListItem
from bookwyrm.models import Review, Comment, Quotation
from bookwyrm.models import Edition
from bookwyrm.models import UserFollows, User, UserBlocks
from bookwyrm.models.job import ParentJob
from bookwyrm.tasks import app, IMPORTS
from bookwyrm.utils.tar import BookwyrmTarFile
logger = logging.getLogger(__name__)
class BookwyrmAwsSession(BotoSession):
"""a boto session that always uses settings.AWS_S3_ENDPOINT_URL"""
def client(self, *args, **kwargs): # pylint: disable=arguments-differ
kwargs["endpoint_url"] = settings.AWS_S3_ENDPOINT_URL
return super().client("s3", *args, **kwargs)
def select_exports_storage():
"""callable to allow for dependency on runtime configuration"""
cls = import_string(settings.EXPORTS_STORAGE)
return cls()
class BookwyrmExportJob(ParentJob):
"""entry for a specific request to export a bookwyrm user"""
export_data = FileField(null=True, storage=select_exports_storage)
export_json = JSONField(null=True, encoder=DjangoJSONEncoder)
json_completed = BooleanField(default=False)
def start_job(self):
"""schedule the first task"""
task = create_export_json_task.delay(job_id=self.id)
self.task_id = task.id
self.save(update_fields=["task_id"])
@app.task(queue=IMPORTS)
def create_export_json_task(job_id):
"""create the JSON data for the export"""
job = BookwyrmExportJob.objects.get(id=job_id)
# don't start the job if it was stopped from the UI
if job.complete:
return
try:
job.set_status("active")
# generate JSON structure
job.export_json = export_json(job.user)
job.save(update_fields=["export_json"])
# create archive in separate task
create_archive_task.delay(job_id=job.id)
except Exception as err: # pylint: disable=broad-except
logger.exception(
"create_export_json_task for %s failed with error: %s", job, err
)
job.set_status("failed")
def archive_file_location(file, directory="") -> str:
"""get the relative location of a file inside the archive"""
return os.path.join(directory, file.name)
def add_file_to_s3_tar(s3_tar: S3Tar, storage, file, directory=""):
"""
add file to S3Tar inside directory, keeping any directories under its
storage location
"""
s3_tar.add_file(
os.path.join(storage.location, file.name),
folder=os.path.dirname(archive_file_location(file, directory=directory)),
)
@app.task(queue=IMPORTS)
def create_archive_task(job_id):
"""create the archive containing the JSON file and additional files"""
job = BookwyrmExportJob.objects.get(id=job_id)
# don't start the job if it was stopped from the UI
if job.complete:
return
try:
export_task_id = str(job.task_id)
archive_filename = f"{export_task_id}.tar.gz"
export_json_bytes = DjangoJSONEncoder().encode(job.export_json).encode("utf-8")
user = job.user
editions = get_books_for_user(user)
if settings.USE_S3:
# Storage for writing temporary files
exports_storage = storage_backends.ExportsS3Storage()
# Handle for creating the final archive
s3_tar = S3Tar(
exports_storage.bucket_name,
os.path.join(exports_storage.location, archive_filename),
session=BookwyrmAwsSession(),
)
# Save JSON file to a temporary location
export_json_tmp_file = os.path.join(export_task_id, "archive.json")
exports_storage.save(
export_json_tmp_file,
ContentFile(export_json_bytes),
)
s3_tar.add_file(
os.path.join(exports_storage.location, export_json_tmp_file)
)
# Add images to TAR
images_storage = storage_backends.ImagesStorage()
if user.avatar:
add_file_to_s3_tar(s3_tar, images_storage, user.avatar)
for edition in editions:
if edition.cover:
add_file_to_s3_tar(
s3_tar, images_storage, edition.cover, directory="images"
)
# Create archive and store file name
s3_tar.tar()
job.export_data = archive_filename
job.save(update_fields=["export_data"])
# Delete temporary files
exports_storage.delete(export_json_tmp_file)
else:
job.export_data = archive_filename
with job.export_data.open("wb") as tar_file:
with BookwyrmTarFile.open(mode="w:gz", fileobj=tar_file) as tar:
# save json file
tar.write_bytes(export_json_bytes)
# Add avatar image if present
if user.avatar:
tar.add_image(user.avatar)
for edition in editions:
if edition.cover:
tar.add_image(edition.cover, directory="images")
job.save(update_fields=["export_data"])
job.set_status("completed")
except Exception as err: # pylint: disable=broad-except
logger.exception("create_archive_task for %s failed with error: %s", job, err)
job.set_status("failed")
def export_json(user: User):
"""create export JSON"""
data = export_user(user) # in the root of the JSON structure
data["settings"] = export_settings(user)
data["goals"] = export_goals(user)
data["books"] = export_books(user)
data["saved_lists"] = export_saved_lists(user)
data["follows"] = export_follows(user)
data["blocks"] = export_blocks(user)
return data
def export_user(user: User):
"""export user data"""
data = user.to_activity()
if user.avatar:
data["icon"]["url"] = archive_file_location(user.avatar)
else:
data["icon"] = {}
return data
def export_settings(user: User):
"""Additional settings - can't be serialized as AP"""
vals = [
"show_goal",
"preferred_timezone",
"default_post_privacy",
"show_suggested_users",
]
return {k: getattr(user, k) for k in vals}
def export_saved_lists(user: User):
"""add user saved lists to export JSON"""
return [l.remote_id for l in user.saved_lists.all()]
def export_follows(user: User):
"""add user follows to export JSON"""
follows = UserFollows.objects.filter(user_subject=user).distinct()
following = User.objects.filter(userfollows_user_object__in=follows).distinct()
return [f.remote_id for f in following]
def export_blocks(user: User):
"""add user blocks to export JSON"""
blocks = UserBlocks.objects.filter(user_subject=user).distinct()
blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct()
return [b.remote_id for b in blocking]
def export_goals(user: User):
"""add user reading goals to export JSON"""
reading_goals = AnnualGoal.objects.filter(user=user).distinct()
return [
{"goal": goal.goal, "year": goal.year, "privacy": goal.privacy}
for goal in reading_goals
]
def export_books(user: User):
"""add books to export JSON"""
editions = get_books_for_user(user)
return [export_book(user, edition) for edition in editions]
def export_book(user: User, edition: Edition):
"""add book to export JSON"""
data = {}
data["work"] = edition.parent_work.to_activity()
data["edition"] = edition.to_activity()
if edition.cover:
data["edition"]["cover"]["url"] = archive_file_location(
edition.cover, directory="images"
)
# authors
data["authors"] = [author.to_activity() for author in edition.authors.all()]
# Shelves this book is on
# Every ShelfItem is this book so we don't other serializing
shelf_books = (
ShelfBook.objects.select_related("shelf")
.filter(user=user, book=edition)
.distinct()
)
data["shelves"] = [shelfbook.shelf.to_activity() for shelfbook in shelf_books]
# Lists and ListItems
# ListItems include "notes" and "approved" so we need them
# even though we know it's this book
list_items = ListItem.objects.filter(book=edition, user=user).distinct()
data["lists"] = []
for item in list_items:
list_info = item.book_list.to_activity()
list_info[
"privacy"
] = item.book_list.privacy # this isn't serialized so we add it
list_info["list_item"] = item.to_activity()
data["lists"].append(list_info)
# Statuses
# Can't use select_subclasses here because
# we need to filter on the "book" value,
# which is not available on an ordinary Status
for status in ["comments", "quotations", "reviews"]:
data[status] = []
comments = Comment.objects.filter(user=user, book=edition).all()
for status in comments:
obj = status.to_activity()
obj["progress"] = status.progress
obj["progress_mode"] = status.progress_mode
data["comments"].append(obj)
quotes = Quotation.objects.filter(user=user, book=edition).all()
for status in quotes:
obj = status.to_activity()
obj["position"] = status.position
obj["endposition"] = status.endposition
obj["position_mode"] = status.position_mode
data["quotations"].append(obj)
reviews = Review.objects.filter(user=user, book=edition).all()
data["reviews"] = [status.to_activity() for status in reviews]
# readthroughs can't be serialized to activity
book_readthroughs = (
ReadThrough.objects.filter(user=user, book=edition).distinct().values()
)
data["readthroughs"] = list(book_readthroughs)
return data
def get_books_for_user(user):
"""Get all the books and editions related to a user"""
editions = (
Edition.objects.select_related("parent_work")
.filter(
Q(shelves__user=user)
| Q(readthrough__user=user)
| Q(review__user=user)
| Q(list__user=user)
| Q(comment__user=user)
| Q(quotation__user=user)
)
.distinct()
)
return editions

462
bookwyrm/models/bookwyrm_import_job.py Normal file
View file

@ -0,0 +1,462 @@
"""Import a user from another Bookwyrm instance"""
import json
import logging
from django.db.models import FileField, JSONField, CharField
from django.utils import timezone
from django.utils.html import strip_tags
from django.contrib.postgres.fields import ArrayField as DjangoArrayField
from bookwyrm import activitypub
from bookwyrm import models
from bookwyrm.tasks import app, IMPORTS
from bookwyrm.models.job import ParentJob, ParentTask, SubTask
from bookwyrm.utils.tar import BookwyrmTarFile
logger = logging.getLogger(__name__)
class BookwyrmImportJob(ParentJob):
"""entry for a specific request for importing a bookwyrm user backup"""
archive_file = FileField(null=True, blank=True)
import_data = JSONField(null=True)
required = DjangoArrayField(CharField(max_length=50, blank=True), blank=True)
def start_job(self):
"""Start the job"""
start_import_task.delay(job_id=self.id, no_children=True)
@app.task(queue=IMPORTS, base=ParentTask)
def start_import_task(**kwargs):
"""trigger the child import tasks for each user data"""
job = BookwyrmImportJob.objects.get(id=kwargs["job_id"])
archive_file = job.archive_file
# don't start the job if it was stopped from the UI
if job.complete:
return
try:
archive_file.open("rb")
with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar:
json_filename = next(
filter(lambda n: n.startswith("archive"), tar.getnames())
)
job.import_data = json.loads(tar.read(json_filename).decode("utf-8"))
if "include_user_profile" in job.required:
update_user_profile(job.user, tar, job.import_data)
if "include_user_settings" in job.required:
update_user_settings(job.user, job.import_data)
if "include_goals" in job.required:
update_goals(job.user, job.import_data.get("goals", []))
if "include_saved_lists" in job.required:
upsert_saved_lists(job.user, job.import_data.get("saved_lists", []))
if "include_follows" in job.required:
upsert_follows(job.user, job.import_data.get("follows", []))
if "include_blocks" in job.required:
upsert_user_blocks(job.user, job.import_data.get("blocks", []))
process_books(job, tar)
job.set_status("complete")
archive_file.close()
except Exception as err: # pylint: disable=broad-except
logger.exception("User Import Job %s Failed with error: %s", job.id, err)
job.set_status("failed")
def process_books(job, tar):
"""
Process user import data related to books
We always import the books even if not assigning
them to shelves, lists etc
"""
books = job.import_data.get("books")
for data in books:
book = get_or_create_edition(data, tar)
if "include_shelves" in job.required:
upsert_shelves(book, job.user, data)
if "include_readthroughs" in job.required:
upsert_readthroughs(data.get("readthroughs"), job.user, book.id)
if "include_comments" in job.required:
upsert_statuses(
job.user, models.Comment, data.get("comments"), book.remote_id
)
if "include_quotations" in job.required:
upsert_statuses(
job.user, models.Quotation, data.get("quotations"), book.remote_id
)
if "include_reviews" in job.required:
upsert_statuses(
job.user, models.Review, data.get("reviews"), book.remote_id
)
if "include_lists" in job.required:
upsert_lists(job.user, data.get("lists"), book.id)
def get_or_create_edition(book_data, tar):
"""Take a JSON string of work and edition data,
find or create the edition and work in the database and
return an edition instance"""
edition = book_data.get("edition")
existing = models.Edition.find_existing(edition)
if existing:
return existing
# make sure we have the authors in the local DB
# replace the old author ids in the edition JSON
edition["authors"] = []
for author in book_data.get("authors"):
parsed_author = activitypub.parse(author)
instance = parsed_author.to_model(
model=models.Author, save=True, overwrite=True
)
edition["authors"].append(instance.remote_id)
# we will add the cover later from the tar
# don't try to load it from the old server
cover = edition.get("cover", {})
cover_path = cover.get("url", None)
edition["cover"] = {}
# first we need the parent work to exist
work = book_data.get("work")
work["editions"] = []
parsed_work = activitypub.parse(work)
work_instance = parsed_work.to_model(model=models.Work, save=True, overwrite=True)
# now we have a work we can add it to the edition
# and create the edition model instance
edition["work"] = work_instance.remote_id
parsed_edition = activitypub.parse(edition)
book = parsed_edition.to_model(model=models.Edition, save=True, overwrite=True)
# set the cover image from the tar
if cover_path:
tar.write_image_to_file(cover_path, book.cover)
return book
def upsert_readthroughs(data, user, book_id):
"""Take a JSON string of readthroughs and
find or create the instances in the database"""
for read_through in data:
obj = {}
keys = [
"progress_mode",
"start_date",
"finish_date",
"stopped_date",
"is_active",
]
for key in keys:
obj[key] = read_through[key]
obj["user_id"] = user.id
obj["book_id"] = book_id
existing = models.ReadThrough.objects.filter(**obj).first()
if not existing:
models.ReadThrough.objects.create(**obj)
def upsert_statuses(user, cls, data, book_remote_id):
"""Take a JSON string of a status and
find or create the instances in the database"""
for status in data:
if is_alias(
user, status["attributedTo"]
): # don't let l33t hax0rs steal other people's posts
# update ids and remove replies
status["attributedTo"] = user.remote_id
status["to"] = update_followers_address(user, status["to"])
status["cc"] = update_followers_address(user, status["cc"])
status[
"replies"
] = (
{}
) # this parses incorrectly but we can't set it without knowing the new id
status["inReplyToBook"] = book_remote_id
parsed = activitypub.parse(status)
if not status_already_exists(
user, parsed
): # don't duplicate posts on multiple import
instance = parsed.to_model(model=cls, save=True, overwrite=True)
for val in [
"progress",
"progress_mode",
"position",
"endposition",
"position_mode",
]:
if status.get(val):
instance.val = status[val]
instance.remote_id = instance.get_remote_id() # update the remote_id
instance.save() # save and broadcast
else:
logger.warning("User does not have permission to import statuses")
def upsert_lists(user, lists, book_id):
"""Take a list of objects each containing
a list and list item as AP objects
Because we are creating new IDs we can't assume the id
will exist or be accurate, so we only use to_model for
adding new items after checking whether they exist .
"""
book = models.Edition.objects.get(id=book_id)
for blist in lists:
booklist = models.List.objects.filter(name=blist["name"], user=user).first()
if not booklist:
blist["owner"] = user.remote_id
parsed = activitypub.parse(blist)
booklist = parsed.to_model(model=models.List, save=True, overwrite=True)
booklist.privacy = blist["privacy"]
booklist.save()
item = models.ListItem.objects.filter(book=book, book_list=booklist).exists()
if not item:
count = booklist.books.count()
models.ListItem.objects.create(
book=book,
book_list=booklist,
user=user,
notes=blist["list_item"]["notes"],
approved=blist["list_item"]["approved"],
order=count + 1,
)
def upsert_shelves(book, user, book_data):
"""Take shelf JSON objects and create
DB entries if they don't already exist"""
shelves = book_data["shelves"]
for shelf in shelves:
book_shelf = models.Shelf.objects.filter(name=shelf["name"], user=user).first()
if not book_shelf:
book_shelf = models.Shelf.objects.create(name=shelf["name"], user=user)
# add the book as a ShelfBook if needed
if not models.ShelfBook.objects.filter(
book=book, shelf=book_shelf, user=user
).exists():
models.ShelfBook.objects.create(
book=book, shelf=book_shelf, user=user, shelved_date=timezone.now()
)
def update_user_profile(user, tar, data):
"""update the user's profile from import data"""
name = data.get("name", None)
username = data.get("preferredUsername")
user.name = name if name else username
user.summary = strip_tags(data.get("summary", None))
user.save(update_fields=["name", "summary"])
if data["icon"].get("url"):
avatar_filename = next(filter(lambda n: n.startswith("avatar"), tar.getnames()))
tar.write_image_to_file(avatar_filename, user.avatar)
def update_user_settings(user, data):
"""update the user's settings from import data"""
update_fields = ["manually_approves_followers", "hide_follows", "discoverable"]
ap_fields = [
("manuallyApprovesFollowers", "manually_approves_followers"),
("hideFollows", "hide_follows"),
("discoverable", "discoverable"),
]
for (ap_field, bw_field) in ap_fields:
setattr(user, bw_field, data[ap_field])
bw_fields = [
"show_goal",
"show_suggested_users",
"default_post_privacy",
"preferred_timezone",
]
for field in bw_fields:
update_fields.append(field)
setattr(user, field, data["settings"][field])
user.save(update_fields=update_fields)
@app.task(queue=IMPORTS, base=SubTask)
def update_user_settings_task(job_id):
"""wrapper task for user's settings import"""
parent_job = BookwyrmImportJob.objects.get(id=job_id)
return update_user_settings(parent_job.user, parent_job.import_data.get("user"))
def update_goals(user, data):
"""update the user's goals from import data"""
for goal in data:
# edit the existing goal if there is one
existing = models.AnnualGoal.objects.filter(
year=goal["year"], user=user
).first()
if existing:
for k in goal.keys():
setattr(existing, k, goal[k])
existing.save()
else:
goal["user"] = user
models.AnnualGoal.objects.create(**goal)
@app.task(queue=IMPORTS, base=SubTask)
def update_goals_task(job_id):
"""wrapper task for user's goals import"""
parent_job = BookwyrmImportJob.objects.get(id=job_id)
return update_goals(parent_job.user, parent_job.import_data.get("goals"))
def upsert_saved_lists(user, values):
"""Take a list of remote ids and add as saved lists"""
for remote_id in values:
book_list = activitypub.resolve_remote_id(remote_id, models.List)
if book_list:
user.saved_lists.add(book_list)
@app.task(queue=IMPORTS, base=SubTask)
def upsert_saved_lists_task(job_id):
"""wrapper task for user's saved lists import"""
parent_job = BookwyrmImportJob.objects.get(id=job_id)
return upsert_saved_lists(
parent_job.user, parent_job.import_data.get("saved_lists")
)
def upsert_follows(user, values):
"""Take a list of remote ids and add as follows"""
for remote_id in values:
followee = activitypub.resolve_remote_id(remote_id, models.User)
if followee:
(follow_request, created,) = models.UserFollowRequest.objects.get_or_create(
user_subject=user,
user_object=followee,
)
if not created:
# this request probably failed to connect with the remote
# and should save to trigger a re-broadcast
follow_request.save()
@app.task(queue=IMPORTS, base=SubTask)
def upsert_follows_task(job_id):
"""wrapper task for user's follows import"""
parent_job = BookwyrmImportJob.objects.get(id=job_id)
return upsert_follows(parent_job.user, parent_job.import_data.get("follows"))
def upsert_user_blocks(user, user_ids):
"""block users"""
for user_id in user_ids:
user_object = activitypub.resolve_remote_id(user_id, models.User)
if user_object:
exists = models.UserBlocks.objects.filter(
user_subject=user, user_object=user_object
).exists()
if not exists:
models.UserBlocks.objects.create(
user_subject=user, user_object=user_object
)
# remove the blocked users's lists from the groups
models.List.remove_from_group(user, user_object)
# remove the blocked user from all blocker's owned groups
models.GroupMember.remove(user, user_object)
@app.task(queue=IMPORTS, base=SubTask)
def upsert_user_blocks_task(job_id):
"""wrapper task for user's blocks import"""
parent_job = BookwyrmImportJob.objects.get(id=job_id)
return upsert_user_blocks(
parent_job.user, parent_job.import_data.get("blocked_users")
)
def update_followers_address(user, field):
"""statuses to or cc followers need to have the followers
address updated to the new local user"""
for i, audience in enumerate(field):
if audience.rsplit("/")[-1] == "followers":
field[i] = user.followers_url
return field
def is_alias(user, remote_id):
"""check that the user is listed as movedTo or also_known_as
in the remote user's profile"""
remote_user = activitypub.resolve_remote_id(
remote_id=remote_id, model=models.User, save=False
)
if remote_user:
if remote_user.moved_to:
return user.remote_id == remote_user.moved_to
if remote_user.also_known_as:
return user in remote_user.also_known_as.all()
return False
def status_already_exists(user, status):
"""check whether this status has already been published
by this user. We can't rely on to_model() because it
only matches on remote_id, which we have to change
*after* saving because it needs the primary key (id)"""
return models.Status.objects.filter(
user=user, content=status.content, published_date=status.published
).exists()

2
bookwyrm/models/connector.py
View file

@ -11,7 +11,7 @@ ConnectorFiles = models.TextChoices("ConnectorFiles", CONNECTORS)
class Connector(BookWyrmModel):
"""book data source connectors"""
identifier = models.CharField(max_length=255, unique=True)
identifier = models.CharField(max_length=255, unique=True) # domain
priority = models.IntegerField(default=2)
name = models.CharField(max_length=255, null=True, blank=True)
connector_file = models.CharField(max_length=255, choices=ConnectorFiles.choices)

5
bookwyrm/models/federated_server.py
View file

@ -16,7 +16,7 @@ FederationStatus = [
class FederatedServer(BookWyrmModel):
"""store which servers we federate with"""
server_name = models.CharField(max_length=255, unique=True)
server_name = models.CharField(max_length=255, unique=True) # domain
status = models.CharField(
max_length=255, default="federated", choices=FederationStatus
)
@ -64,5 +64,4 @@ class FederatedServer(BookWyrmModel):
def is_blocked(cls, url: str) -> bool:
"""look up if a domain is blocked"""
url = urlparse(url)
domain = url.netloc
return cls.objects.filter(server_name=domain, status="blocked").exists()
return cls.objects.filter(server_name=url.hostname, status="blocked").exists()

54
bookwyrm/models/fields.py
View file

@ -1,5 +1,6 @@
""" activitypub-aware django model fields """
from dataclasses import MISSING
from datetime import datetime
import re
from uuid import uuid4
from urllib.parse import urljoin
@ -19,6 +20,11 @@ from markdown import markdown
from bookwyrm import activitypub
from bookwyrm.connectors import get_image
from bookwyrm.utils.sanitizer import clean
from bookwyrm.utils.partial_date import (
PartialDate,
PartialDateModel,
from_partial_isoformat,
)
from bookwyrm.settings import MEDIA_FULL_URL
@ -254,12 +260,12 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
if to == [self.public]:
setattr(instance, self.name, "public")
elif self.public in cc:
setattr(instance, self.name, "unlisted")
elif to == [user.followers_url]:
setattr(instance, self.name, "followers")
elif cc == []:
setattr(instance, self.name, "direct")
elif self.public in cc:
setattr(instance, self.name, "unlisted")
else:
setattr(instance, self.name, "followers")
return original == getattr(instance, self.name)
@ -476,16 +482,18 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
if not url:
return None
return activitypub.Document(url=url, name=alt)
return activitypub.Image(url=url, name=alt)
def field_from_activity(self, value, allow_external_connections=True):
image_slug = value
# when it's an inline image (User avatar/icon, Book cover), it's a json
# blob, but when it's an attached image, it's just a url
if hasattr(image_slug, "url"):
url = image_slug.url
elif isinstance(image_slug, str):
if isinstance(image_slug, str):
url = image_slug
elif isinstance(image_slug, dict):
url = image_slug.get("url")
elif hasattr(image_slug, "url"): # Serialized to Image/Document object?
url = image_slug.url
else:
return None
@ -534,8 +542,9 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
return value.isoformat()
def field_from_activity(self, value, allow_external_connections=True):
missing_fields = datetime(1970, 1, 1) # "2022-10" => "2022-10-01"
try:
date_value = dateutil.parser.parse(value)
date_value = dateutil.parser.parse(value, default=missing_fields)
try:
return timezone.make_aware(date_value)
except ValueError:
@ -544,6 +553,37 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
return None
class PartialDateField(ActivitypubFieldMixin, PartialDateModel):
"""activitypub-aware partial date field"""
def field_to_activity(self, value) -> str:
return value.partial_isoformat() if value else None
def field_from_activity(self, value, allow_external_connections=True):
# pylint: disable=no-else-return
try:
return from_partial_isoformat(value)
except ValueError:
pass
# fallback to full ISO-8601 parsing
try:
parsed = dateutil.parser.isoparse(value)
except (ValueError, ParserError):
return None
if timezone.is_aware(parsed):
return PartialDate.from_datetime(parsed)
else:
# Should not happen on the wire, but truncate down to date parts.
return PartialDate.from_date_parts(parsed.year, parsed.month, parsed.day)
# FIXME: decide whether to fix timestamps like "2023-09-30T21:00:00-03":
# clearly Oct 1st, not Sep 30th (an unwanted side-effect of USE_TZ). It's
# basically the remnants of #3028; there is a data migration pending (see …)
# but over the wire we might get these for an indeterminate amount of time.
class HtmlField(ActivitypubFieldMixin, models.TextField):
"""a text field for storing html"""

17
bookwyrm/models/group.py
View file

@ -1,8 +1,7 @@
""" do book related things with other users """
from django.apps import apps
from django.db import models, IntegrityError, transaction
from django.db.models import Q
from bookwyrm.settings import DOMAIN
from bookwyrm.settings import BASE_URL
from .base_model import BookWyrmModel
from . import fields
from .relationship import UserBlocks
@ -18,7 +17,7 @@ class Group(BookWyrmModel):
def get_remote_id(self):
"""don't want the user to be in there in this case"""
return f"https://{DOMAIN}/group/{self.id}"
return f"{BASE_URL}/group/{self.id}"
@classmethod
def followers_filter(cls, queryset, viewer):
@ -143,26 +142,28 @@ class GroupMemberInvitation(models.Model):
@transaction.atomic
def accept(self):
"""turn this request into the real deal"""
# pylint: disable-next=import-outside-toplevel
from .notification import Notification, NotificationType # circular dependency
GroupMember.from_request(self)
model = apps.get_model("bookwyrm.Notification", require_ready=True)
# tell the group owner
model.notify(
Notification.notify(
self.group.user,
self.user,
related_group=self.group,
notification_type=model.ACCEPT,
notification_type=NotificationType.ACCEPT,
)
# let the other members know about it
for membership in self.group.memberships.all():
member = membership.user
if member not in (self.user, self.group.user):
model.notify(
Notification.notify(
member,
self.user,
related_group=self.group,
notification_type=model.JOIN,
notification_type=NotificationType.JOIN,
)
def reject(self):

35
bookwyrm/models/import_job.py
View file

@ -1,4 +1,5 @@
""" track progress of goodreads imports """
from datetime import datetime
import math
import re
import dateutil.parser
@ -259,38 +260,30 @@ class ImportItem(models.Model):
except ValueError:
return None
def _parse_datefield(self, field, /):
if not (date := self.normalized_data.get(field)):
return None
defaults = datetime(1970, 1, 1) # "2022-10" => "2022-10-01"
parsed = dateutil.parser.parse(date, default=defaults)
# Keep timezone if import already had one, else use default.
return parsed if timezone.is_aware(parsed) else timezone.make_aware(parsed)
@property
def date_added(self):
"""when the book was added to this dataset"""
if self.normalized_data.get("date_added"):
parsed_date_added = dateutil.parser.parse(
self.normalized_data.get("date_added")
)
if timezone.is_aware(parsed_date_added):
# Keep timezone if import already had one
return parsed_date_added
return timezone.make_aware(parsed_date_added)
return None
return self._parse_datefield("date_added")
@property
def date_started(self):
"""when the book was started"""
if self.normalized_data.get("date_started"):
return timezone.make_aware(
dateutil.parser.parse(self.normalized_data.get("date_started"))
)
return None
return self._parse_datefield("date_started")
@property
def date_read(self):
"""the date a book was completed"""
if self.normalized_data.get("date_finished"):
return timezone.make_aware(
dateutil.parser.parse(self.normalized_data.get("date_finished"))
)
return None
return self._parse_datefield("date_finished")
@property
def reads(self):

307
bookwyrm/models/job.py Normal file
View file

@ -0,0 +1,307 @@
"""Everything needed for Celery to multi-thread complex tasks."""
from django.db import models
from django.db import transaction
from django.utils.translation import gettext_lazy as _
from django.utils import timezone
from bookwyrm.models.user import User
from bookwyrm.tasks import app
class Job(models.Model):
"""Abstract model to store the state of a Task."""
class Status(models.TextChoices):
"""Possible job states."""
PENDING = "pending", _("Pending")
ACTIVE = "active", _("Active")
COMPLETE = "complete", _("Complete")
STOPPED = "stopped", _("Stopped")
FAILED = "failed", _("Failed")
task_id = models.UUIDField(unique=True, null=True, blank=True)
created_date = models.DateTimeField(default=timezone.now)
updated_date = models.DateTimeField(default=timezone.now)
complete = models.BooleanField(default=False)
status = models.CharField(
max_length=50, choices=Status.choices, default=Status.PENDING, null=True
)
class Meta:
"""Make it abstract"""
abstract = True
def complete_job(self):
"""Report that the job has completed"""
if self.complete:
return
self.status = self.Status.COMPLETE
self.complete = True
self.updated_date = timezone.now()
self.save(update_fields=["status", "complete", "updated_date"])
def stop_job(self, reason=None):
"""Stop the job"""
if self.complete:
return
self.__terminate_job()
if reason and reason == "failed":
self.status = self.Status.FAILED
else:
self.status = self.Status.STOPPED
self.complete = True
self.updated_date = timezone.now()
self.save(update_fields=["status", "complete", "updated_date"])
def set_status(self, status):
"""Set job status"""
if self.complete:
return
if self.status == status:
return
if status == self.Status.COMPLETE:
self.complete_job()
return
if status == self.Status.STOPPED:
self.stop_job()
return
if status == self.Status.FAILED:
self.stop_job(reason="failed")
return
self.updated_date = timezone.now()
self.status = status
self.save(update_fields=["status", "updated_date"])
def __terminate_job(self):
"""Tell workers to ignore and not execute this task."""
app.control.revoke(self.task_id, terminate=True)
class ParentJob(Job):
"""Store the state of a Task which can spawn many :model:`ChildJob`s to spread
resource load.
Intended to be sub-classed if necessary via proxy or
multi-table inheritance.
Extends :model:`Job`.
"""
user = models.ForeignKey(User, on_delete=models.CASCADE)
def complete_job(self):
"""Report that the job has completed and stop pending
children. Extend.
"""
super().complete_job()
self.__terminate_pending_child_jobs()
def notify_child_job_complete(self):
"""let the job know when the items get work done"""
if self.complete:
return
self.updated_date = timezone.now()
self.save(update_fields=["updated_date"])
if not self.complete and self.has_completed:
self.complete_job()
def __terminate_job(self): # pylint: disable=unused-private-member
"""Tell workers to ignore and not execute this task
& pending child tasks. Extend.
"""
super().__terminate_job()
self.__terminate_pending_child_jobs()
def __terminate_pending_child_jobs(self):
"""Tell workers to ignore and not execute any pending child tasks."""
tasks = self.pending_child_jobs.filter(task_id__isnull=False).values_list(
"task_id", flat=True
)
app.control.revoke(list(tasks))
self.pending_child_jobs.update(status=self.Status.STOPPED)
@property
def has_completed(self):
"""has this job finished"""
return not self.pending_child_jobs.exists()
@property
def pending_child_jobs(self):
"""items that haven't been processed yet"""
return self.child_jobs.filter(complete=False)
class ChildJob(Job):
"""Stores the state of a Task for the related :model:`ParentJob`.
Intended to be sub-classed if necessary via proxy or
multi-table inheritance.
Extends :model:`Job`.
"""
parent_job = models.ForeignKey(
ParentJob, on_delete=models.CASCADE, related_name="child_jobs"
)
def set_status(self, status):
"""Set job and parent_job status. Extend."""
super().set_status(status)
if (
status == self.Status.ACTIVE
and self.parent_job.status == self.Status.PENDING
):
self.parent_job.set_status(self.Status.ACTIVE)
def complete_job(self):
"""Report to parent_job that the job has completed. Extend."""
super().complete_job()
self.parent_job.notify_child_job_complete()
class ParentTask(app.Task):
"""Used with ParentJob, Abstract Tasks execute code at specific points in
a Task's lifecycle, applying to all Tasks with the same 'base'.
All status & ParentJob.task_id assignment is managed here for you.
Usage e.g. @app.task(base=ParentTask)
"""
def before_start(
self, task_id, args, kwargs
): # pylint: disable=no-self-use, unused-argument
"""Handler called before the task starts. Override.
Prepare ParentJob before the task starts.
Arguments:
task_id (str): Unique id of the task to execute.
args (Tuple): Original arguments for the task to execute.
kwargs (Dict): Original keyword arguments for the task to execute.
Keyword Arguments:
job_id (int): Unique 'id' of the ParentJob.
no_children (bool): If 'True' this is the only Task expected to run
for the given ParentJob.
Returns:
None: The return value of this handler is ignored.
"""
job = ParentJob.objects.get(id=kwargs["job_id"])
job.task_id = task_id
job.save(update_fields=["task_id"])
if kwargs["no_children"]:
job.set_status(ChildJob.Status.ACTIVE)
def on_success(
self, retval, task_id, args, kwargs
): # pylint: disable=no-self-use, unused-argument
"""Run by the worker if the task executes successfully. Override.
Update ParentJob on Task complete.
Arguments:
retval (Any): The return value of the task.
task_id (str): Unique id of the executed task.
args (Tuple): Original arguments for the executed task.
kwargs (Dict): Original keyword arguments for the executed task.
Keyword Arguments:
job_id (int): Unique 'id' of the ParentJob.
no_children (bool): If 'True' this is the only Task expected to run
for the given ParentJob.
Returns:
None: The return value of this handler is ignored.
"""
if kwargs["no_children"]:
job = ParentJob.objects.get(id=kwargs["job_id"])
job.complete_job()
class SubTask(app.Task):
"""Used with ChildJob, Abstract Tasks execute code at specific points in
a Task's lifecycle, applying to all Tasks with the same 'base'.
All status & ChildJob.task_id assignment is managed here for you.
Usage e.g. @app.task(base=SubTask)
"""
def before_start(
self, task_id, *args, **kwargs
): # pylint: disable=no-self-use, unused-argument
"""Handler called before the task starts. Override.
Prepare ChildJob before the task starts.
Arguments:
task_id (str): Unique id of the task to execute.
args (Tuple): Original arguments for the task to execute.
kwargs (Dict): Original keyword arguments for the task to execute.
Keyword Arguments:
job_id (int): Unique 'id' of the ParentJob.
child_id (int): Unique 'id' of the ChildJob.
Returns:
None: The return value of this handler is ignored.
"""
child_job = ChildJob.objects.get(id=kwargs["child_id"])
child_job.task_id = task_id
child_job.save(update_fields=["task_id"])
child_job.set_status(ChildJob.Status.ACTIVE)
def on_success(
self, retval, task_id, *args, **kwargs
): # pylint: disable=no-self-use, unused-argument
"""Run by the worker if the task executes successfully. Override.
Notify ChildJob of task completion.
Arguments:
retval (Any): The return value of the task.
task_id (str): Unique id of the executed task.
args (Tuple): Original arguments for the executed task.
kwargs (Dict): Original keyword arguments for the executed task.
Keyword Arguments:
job_id (int): Unique 'id' of the ParentJob.
child_id (int): Unique 'id' of the ChildJob.
Returns:
None: The return value of this handler is ignored.
"""
subtask = ChildJob.objects.get(id=kwargs["child_id"])
subtask.complete_job()
@transaction.atomic
def create_child_job(parent_job, task_callback):
"""Utility method for creating a ChildJob
and running a task to avoid DB race conditions
"""
child_job = ChildJob.objects.create(parent_job=parent_job)
transaction.on_commit(
lambda: task_callback.delay(job_id=parent_job.id, child_id=child_job.id)
)
return child_job

2
bookwyrm/models/link.py
View file

@ -38,7 +38,7 @@ class Link(ActivitypubMixin, BookWyrmModel):
"""create a link"""
# get or create the associated domain
if not self.domain:
domain = urlparse(self.url).netloc
domain = urlparse(self.url).hostname
self.domain, _ = LinkDomain.objects.get_or_create(domain=domain)
# this is never broadcast, the owning model broadcasts an update

4
bookwyrm/models/list.py
View file

@ -7,7 +7,7 @@ from django.db.models import Q
from django.utils import timezone
from bookwyrm import activitypub
from bookwyrm.settings import DOMAIN
from bookwyrm.settings import BASE_URL
from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
from .base_model import BookWyrmModel
@ -50,7 +50,7 @@ class List(OrderedCollectionMixin, BookWyrmModel):
def get_remote_id(self):
"""don't want the user to be in there in this case"""
return f"https://{DOMAIN}/list/{self.id}"
return f"{BASE_URL}/list/{self.id}"
@property
def collection_queryset(self):

71
bookwyrm/models/move.py Normal file
View file

@ -0,0 +1,71 @@
""" move an object including migrating a user account """
from django.core.exceptions import PermissionDenied
from django.db import models
from bookwyrm import activitypub
from .activitypub_mixin import ActivityMixin
from .base_model import BookWyrmModel
from . import fields
from .notification import Notification, NotificationType
class Move(ActivityMixin, BookWyrmModel):
"""migrating an activitypub object"""
user = fields.ForeignKey(
"User", on_delete=models.PROTECT, activitypub_field="actor"
)
object = fields.CharField(
max_length=255,
blank=False,
null=False,
activitypub_field="object",
)
origin = fields.CharField(
max_length=255,
blank=True,
null=True,
default="",
activitypub_field="origin",
)
activity_serializer = activitypub.Move
class MoveUser(Move):
"""migrating an activitypub user account"""
target = fields.ForeignKey(
"User",
on_delete=models.PROTECT,
related_name="move_target",
activitypub_field="target",
)
def save(self, *args, **kwargs):
"""update user info and broadcast it"""
# only allow if the source is listed in the target's alsoKnownAs
if self.user in self.target.also_known_as.all():
self.user.also_known_as.add(self.target.id)
self.user.update_active_date()
self.user.moved_to = self.target.remote_id
self.user.save(update_fields=["moved_to"])
if self.user.local:
kwargs[
"broadcast"
] = True # Only broadcast if we are initiating the Move
super().save(*args, **kwargs)
for follower in self.user.followers.all():
if follower.local:
Notification.notify(
follower, self.user, notification_type=NotificationType.MOVE
)
else:
raise PermissionDenied()

121
bookwyrm/models/notification.py
View file

@ -1,12 +1,21 @@
""" alert a user to activity """
from django.db import models, transaction
from django.dispatch import receiver
from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob
from .base_model import BookWyrmModel
from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain
from . import (
Boost,
Favorite,
GroupMemberInvitation,
ImportJob,
BookwyrmImportJob,
LinkDomain,
)
from . import ListItem, Report, Status, User, UserFollowRequest
from .site import InviteRequest
class Notification(BookWyrmModel):
class NotificationType(models.TextChoices):
"""you've been tagged, liked, followed, etc"""
# Status interactions
@ -22,6 +31,8 @@ class Notification(BookWyrmModel):
# Imports
IMPORT = "IMPORT"
USER_IMPORT = "USER_IMPORT"
USER_EXPORT = "USER_EXPORT"
# List activity
ADD = "ADD"
@ -29,6 +40,7 @@ class Notification(BookWyrmModel):
# Admin
REPORT = "REPORT"
LINK_DOMAIN = "LINK_DOMAIN"
INVITE_REQUEST = "INVITE_REQUEST"
# Groups
INVITE = "INVITE"
@ -40,12 +52,12 @@ class Notification(BookWyrmModel):
GROUP_NAME = "GROUP_NAME"
GROUP_DESCRIPTION = "GROUP_DESCRIPTION"
# pylint: disable=line-too-long
NotificationType = models.TextChoices(
# there has got be a better way to do this
"NotificationType",
f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
)
# Migrations
MOVE = "MOVE"
class Notification(BookWyrmModel):
"""a notification object"""
user = models.ForeignKey("User", on_delete=models.CASCADE)
read = models.BooleanField(default=False)
@ -61,11 +73,15 @@ class Notification(BookWyrmModel):
)
related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True)
related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True)
related_user_export = models.ForeignKey(
"BookwyrmExportJob", on_delete=models.CASCADE, null=True
)
related_list_items = models.ManyToManyField(
"ListItem", symmetrical=False, related_name="notifications"
)
related_reports = models.ManyToManyField("Report", symmetrical=False)
related_link_domains = models.ManyToManyField("LinkDomain", symmetrical=False)
related_reports = models.ManyToManyField("Report")
related_link_domains = models.ManyToManyField("LinkDomain")
related_invite_requests = models.ManyToManyField("InviteRequest")
@classmethod
@transaction.atomic
@ -90,11 +106,11 @@ class Notification(BookWyrmModel):
user=user,
related_users=related_user,
related_list_items__book_list=list_item.book_list,
notification_type=Notification.ADD,
notification_type=NotificationType.ADD,
).first()
if not notification:
notification = cls.objects.create(
user=user, notification_type=Notification.ADD
user=user, notification_type=NotificationType.ADD
)
notification.related_users.add(related_user)
notification.related_list_items.add(list_item)
@ -121,7 +137,7 @@ def notify_on_fav(sender, instance, *args, **kwargs):
instance.status.user,
instance.user,
related_status=instance.status,
notification_type=Notification.FAVORITE,
notification_type=NotificationType.FAVORITE,
)
@ -135,7 +151,7 @@ def notify_on_unfav(sender, instance, *args, **kwargs):
instance.status.user,
instance.user,
related_status=instance.status,
notification_type=Notification.FAVORITE,
notification_type=NotificationType.FAVORITE,
)
@ -160,7 +176,7 @@ def notify_user_on_mention(sender, instance, *args, **kwargs):
instance.reply_parent.user,
instance.user,
related_status=instance,
notification_type=Notification.REPLY,
notification_type=NotificationType.REPLY,
)
for mention_user in instance.mention_users.all():
@ -172,7 +188,7 @@ def notify_user_on_mention(sender, instance, *args, **kwargs):
Notification.notify(
mention_user,
instance.user,
notification_type=Notification.MENTION,
notification_type=NotificationType.MENTION,
related_status=instance,
)
@ -191,7 +207,7 @@ def notify_user_on_boost(sender, instance, *args, **kwargs):
instance.boosted_status.user,
instance.user,
related_status=instance.boosted_status,
notification_type=Notification.BOOST,
notification_type=NotificationType.BOOST,
)
@ -203,7 +219,7 @@ def notify_user_on_unboost(sender, instance, *args, **kwargs):
instance.boosted_status.user,
instance.user,
related_status=instance.boosted_status,
notification_type=Notification.BOOST,
notification_type=NotificationType.BOOST,
)
@ -218,11 +234,41 @@ def notify_user_on_import_complete(
return
Notification.objects.get_or_create(
user=instance.user,
notification_type=Notification.IMPORT,
notification_type=NotificationType.IMPORT,
related_import=instance,
)
@receiver(models.signals.post_save, sender=BookwyrmImportJob)
# pylint: disable=unused-argument
def notify_user_on_user_import_complete(
sender, instance, *args, update_fields=None, **kwargs
):
"""we imported your user details! aren't you proud of us"""
update_fields = update_fields or []
if not instance.complete or "complete" not in update_fields:
return
Notification.objects.create(
user=instance.user, notification_type=NotificationType.USER_IMPORT
)
@receiver(models.signals.post_save, sender=BookwyrmExportJob)
# pylint: disable=unused-argument
def notify_user_on_user_export_complete(
sender, instance, *args, update_fields=None, **kwargs
):
"""we exported your user details! aren't you proud of us"""
update_fields = update_fields or []
if not instance.complete or "complete" not in update_fields:
return
Notification.objects.create(
user=instance.user,
notification_type=NotificationType.USER_EXPORT,
related_user_export=instance,
)
@receiver(models.signals.post_save, sender=Report)
@transaction.atomic
# pylint: disable=unused-argument
@ -233,11 +279,10 @@ def notify_admins_on_report(sender, instance, created, *args, **kwargs):
return
# moderators and superusers should be notified
admins = User.admins()
for admin in admins:
for admin in User.admins():
notification, _ = Notification.objects.get_or_create(
user=admin,
notification_type=Notification.REPORT,
notification_type=NotificationType.REPORT,
read=False,
)
notification.related_reports.add(instance)
@ -253,16 +298,33 @@ def notify_admins_on_link_domain(sender, instance, created, *args, **kwargs):
return
# moderators and superusers should be notified
admins = User.admins()
for admin in admins:
for admin in User.admins():
notification, _ = Notification.objects.get_or_create(
user=admin,
notification_type=Notification.LINK_DOMAIN,
notification_type=NotificationType.LINK_DOMAIN,
read=False,
)
notification.related_link_domains.add(instance)
@receiver(models.signals.post_save, sender=InviteRequest)
@transaction.atomic
# pylint: disable=unused-argument
def notify_admins_on_invite_request(sender, instance, created, *args, **kwargs):
"""need to handle a new invite request"""
if not created:
return
# moderators and superusers should be notified
for admin in User.admins():
notification, _ = Notification.objects.get_or_create(
user=admin,
notification_type=NotificationType.INVITE_REQUEST,
read=False,
)
notification.related_invite_requests.add(instance)
@receiver(models.signals.post_save, sender=GroupMemberInvitation)
# pylint: disable=unused-argument
def notify_user_on_group_invite(sender, instance, *args, **kwargs):
@ -271,7 +333,7 @@ def notify_user_on_group_invite(sender, instance, *args, **kwargs):
instance.user,
instance.group.user,
related_group=instance.group,
notification_type=Notification.INVITE,
notification_type=NotificationType.INVITE,
)
@ -309,11 +371,12 @@ def notify_user_on_follow(sender, instance, created, *args, **kwargs):
notification = Notification.objects.filter(
user=instance.user_object,
related_users=instance.user_subject,
notification_type=Notification.FOLLOW_REQUEST,
notification_type=NotificationType.FOLLOW_REQUEST,
).first()
if not notification:
notification = Notification.objects.create(
user=instance.user_object, notification_type=Notification.FOLLOW_REQUEST
user=instance.user_object,
notification_type=NotificationType.FOLLOW_REQUEST,
)
notification.related_users.set([instance.user_subject])
notification.read = False
@ -323,6 +386,6 @@ def notify_user_on_follow(sender, instance, created, *args, **kwargs):
Notification.notify(
instance.user_object,
instance.user_subject,
notification_type=Notification.FOLLOW,
notification_type=NotificationType.FOLLOW,
read=False,
)

28
bookwyrm/models/relationship.py
View file

@ -65,6 +65,13 @@ class UserRelationship(BookWyrmModel):
base_path = self.user_subject.remote_id
return f"{base_path}#follows/{self.id}"
def get_accept_reject_id(self, status):
"""get id for sending an accept or reject of a local user"""
base_path = self.user_object.remote_id
status_id = self.id or 0
return f"{base_path}#{status}/{status_id}"
class UserFollows(ActivityMixin, UserRelationship):
"""Following a user"""
@ -105,6 +112,20 @@ class UserFollows(ActivityMixin, UserRelationship):
)
return obj
def reject(self):
"""generate a Reject for this follow. This would normally happen
when a user deletes a follow they previously accepted"""
if self.user_object.local:
activity = activitypub.Reject(
id=self.get_accept_reject_id(status="rejects"),
actor=self.user_object.remote_id,
object=self.to_activity(),
).serialize()
self.broadcast(activity, self.user_object)
self.delete()
class UserFollowRequest(ActivitypubMixin, UserRelationship):
"""following a user requires manual or automatic confirmation"""
@ -148,13 +169,6 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
if not manually_approves:
self.accept()
def get_accept_reject_id(self, status):
"""get id for sending an accept or reject of a local user"""
base_path = self.user_object.remote_id
status_id = self.id or 0
return f"{base_path}#{status}/{status_id}"
def accept(self, broadcast_only=False):
"""turn this request into the real deal"""
user = self.user_object

4
bookwyrm/models/report.py
View file

@ -3,7 +3,7 @@ from django.core.exceptions import PermissionDenied
from django.db import models
from django.utils.translation import gettext_lazy as _
from bookwyrm.settings import DOMAIN
from bookwyrm.settings import BASE_URL
from .base_model import BookWyrmModel
@ -46,7 +46,7 @@ class Report(BookWyrmModel):
raise PermissionDenied()
def get_remote_id(self):
return f"https://{DOMAIN}/settings/reports/{self.id}"
return f"{BASE_URL}/settings/reports/{self.id}"
def comment(self, user, note):
"""comment on a report"""

Some files were not shown because too many files have changed in this diff Show more