Merge branch 'main' into production

This commit is contained in:
Mouse Reeve 2022-11-14 12:20:25 -08:00
commit 3578dc839c
10 changed files with 58 additions and 27 deletions

View file

@ -7,7 +7,7 @@ from django.apps import apps
from django.db import IntegrityError, transaction
from bookwyrm.connectors import ConnectorException, get_data
from bookwyrm.tasks import app
from bookwyrm.tasks import app, MEDIUM
logger = logging.getLogger(__name__)
@ -202,7 +202,7 @@ class ActivityObject:
return data
@app.task(queue="medium_priority")
@app.task(queue=MEDIUM)
@transaction.atomic
def set_related_field(
model_name, origin_model_name, related_field_name, related_remote_id, data

View file

@ -13,7 +13,7 @@ from requests import HTTPError
from bookwyrm import book_search, models
from bookwyrm.settings import SEARCH_TIMEOUT, USER_AGENT
from bookwyrm.tasks import app
from bookwyrm.tasks import app, LOW
logger = logging.getLogger(__name__)
@ -143,7 +143,7 @@ def get_or_create_connector(remote_id):
return load_connector(connector_info)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def load_more_data(connector_id, book_id):
"""background the work of getting all 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id)
@ -152,7 +152,7 @@ def load_more_data(connector_id, book_id):
connector.expand_book_data(book)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def create_edition_task(connector_id, work_id, data):
"""separate task for each of the 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id)

View file

@ -3,7 +3,7 @@ from django.core.mail import EmailMultiAlternatives
from django.template.loader import get_template
from bookwyrm import models, settings
from bookwyrm.tasks import app
from bookwyrm.tasks import app, HIGH
from bookwyrm.settings import DOMAIN
@ -68,7 +68,7 @@ def format_email(email_name, data):
return (subject, html_content, text_content)
@app.task(queue="high_priority")
@app.task(queue=HIGH)
def send_email(recipient, subject, html_content, text_content):
"""use a task to send the email"""
email = EmailMultiAlternatives(

View file

@ -0,0 +1,31 @@
""" Actually let's not generate those preview images """
import json
from django.core.management.base import BaseCommand
from bookwyrm.tasks import app
class Command(BaseCommand):
"""Find and revoke image tasks"""
# pylint: disable=unused-argument
def handle(self, *args, **options):
"""reveoke nonessential low priority tasks"""
types = [
"bookwyrm.preview_images.generate_edition_preview_image_task",
"bookwyrm.preview_images.generate_user_preview_image_task",
]
self.stdout.write(" | Finding tasks of types:")
self.stdout.write("\n".join(types))
with app.pool.acquire(block=True) as conn:
tasks = conn.default_channel.client.lrange("low_priority", 0, -1)
self.stdout.write(f" | Found {len(tasks)} task(s) in low priority queue")
revoke_ids = []
for task in tasks:
task_json = json.loads(task)
task_type = task_json.get("headers", {}).get("task")
if task_type in types:
revoke_ids.append(task_json.get("headers", {}).get("id"))
self.stdout.write(".", ending="")
self.stdout.write(f"\n | Revoking {len(revoke_ids)} task(s)")
app.control.revoke(revoke_ids)

View file

@ -8,7 +8,7 @@ from django.db import models, transaction
from django.db.models import Q
from django.utils.translation import gettext_lazy as _
from bookwyrm.tasks import app
from bookwyrm.tasks import app, LOW
from .base_model import BookWyrmModel
from .user import User
@ -65,7 +65,7 @@ class AutoMod(AdminModel):
created_by = models.ForeignKey("User", on_delete=models.PROTECT)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def automod_task():
"""Create reports"""
if not AutoMod.objects.exists():

View file

@ -328,7 +328,7 @@ class ImportItem(models.Model):
)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def start_import_task(job_id):
"""trigger the child tasks for each row"""
job = ImportJob.objects.get(id=job_id)
@ -345,7 +345,7 @@ def start_import_task(job_id):
job.save()
@app.task(queue="low_priority")
@app.task(queue=LOW)
def import_item_task(item_id):
"""resolve a row into a book"""
item = ImportItem.objects.get(id=item_id)

View file

@ -20,7 +20,7 @@ from bookwyrm.models.status import Status
from bookwyrm.preview_images import generate_user_preview_image_task
from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, USE_HTTPS, LANGUAGES
from bookwyrm.signatures import create_key_pair
from bookwyrm.tasks import app
from bookwyrm.tasks import app, LOW
from bookwyrm.utils import regex
from .activitypub_mixin import OrderedCollectionPageMixin, ActivitypubMixin
from .base_model import BookWyrmModel, DeactivationReason, new_access_code
@ -458,7 +458,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
return super().save(*args, **kwargs)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def set_remote_server(user_id):
"""figure out the user's remote server in the background"""
user = User.objects.get(id=user_id)
@ -502,7 +502,7 @@ def get_or_create_remote_server(domain, refresh=False):
return server
@app.task(queue="low_priority")
@app.task(queue=LOW)
def get_remote_reviews(outbox):
"""ingest reviews by a new remote bookwyrm user"""
outbox_page = outbox + "?page=true&type=Review"

View file

@ -16,7 +16,7 @@ from django.core.files.storage import default_storage
from django.db.models import Avg
from bookwyrm import models, settings
from bookwyrm.tasks import app
from bookwyrm.tasks import app, LOW
logger = logging.getLogger(__name__)
@ -401,7 +401,7 @@ def save_and_cleanup(image, instance=None):
# pylint: disable=invalid-name
@app.task(queue="low_priority")
@app.task(queue=LOW)
def generate_site_preview_image_task():
"""generate preview_image for the website"""
if not settings.ENABLE_PREVIEW_IMAGES:
@ -426,7 +426,7 @@ def generate_site_preview_image_task():
# pylint: disable=invalid-name
@app.task(queue="low_priority")
@app.task(queue=LOW)
def generate_edition_preview_image_task(book_id):
"""generate preview_image for a book"""
if not settings.ENABLE_PREVIEW_IMAGES:
@ -451,7 +451,7 @@ def generate_edition_preview_image_task(book_id):
save_and_cleanup(image, instance=book)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def generate_user_preview_image_task(user_id):
"""generate preview_image for a book"""
if not settings.ENABLE_PREVIEW_IMAGES:

View file

@ -7,7 +7,7 @@ from django.db.models import signals, Count, Q, Case, When, IntegerField
from bookwyrm import models
from bookwyrm.redis_store import RedisStore, r
from bookwyrm.tasks import app
from bookwyrm.tasks import app, LOW, MEDIUM
logger = logging.getLogger(__name__)
@ -237,41 +237,41 @@ def domain_level_update(sender, instance, created, update_fields=None, **kwargs)
# ------------------- TASKS
@app.task(queue="low_priority")
@app.task(queue=LOW)
def rerank_suggestions_task(user_id):
"""do the hard work in celery"""
suggested_users.rerank_user_suggestions(user_id)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def rerank_user_task(user_id, update_only=False):
"""do the hard work in celery"""
user = models.User.objects.get(id=user_id)
suggested_users.rerank_obj(user, update_only=update_only)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def remove_user_task(user_id):
"""do the hard work in celery"""
user = models.User.objects.get(id=user_id)
suggested_users.remove_object_from_related_stores(user)
@app.task(queue="medium_priority")
@app.task(queue=MEDIUM)
def remove_suggestion_task(user_id, suggested_user_id):
"""remove a specific user from a specific user's suggestions"""
suggested_user = models.User.objects.get(id=suggested_user_id)
suggested_users.remove_suggestion(user_id, suggested_user)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def bulk_remove_instance_task(instance_id):
"""remove a bunch of users from recs"""
for user in models.User.objects.filter(federated_server__id=instance_id):
suggested_users.remove_object_from_related_stores(user)
@app.task(queue="low_priority")
@app.task(queue=LOW)
def bulk_add_instance_task(instance_id):
"""remove a bunch of users from recs"""
for user in models.User.objects.filter(federated_server__id=instance_id):

View file

@ -14,7 +14,7 @@ from django.views import View
from django.views.decorators.csrf import csrf_exempt
from bookwyrm import activitypub, models
from bookwyrm.tasks import app
from bookwyrm.tasks import app, MEDIUM
from bookwyrm.signatures import Signature
from bookwyrm.utils import regex
@ -98,7 +98,7 @@ def raise_is_blocked_activity(activity_json):
raise PermissionDenied()
@app.task(queue="medium_priority")
@app.task(queue=MEDIUM)
def activity_task(activity_json):
"""do something with this json we think is legit"""
# lets see if the activitypub module can make sense of this json