Merge branch 'main' into import-tombstone

.env.example

@@ -21,6 +21,7 @@ DEFAULT_LANGUAGE="English"
 # Probably only necessary in development.
 # PORT=1333
 
+STATIC_ROOT=static/
 MEDIA_ROOT=images/
 
 # Database configuration

.github/workflows/lint-frontend.yaml

@@ -15,7 +15,7 @@ on:
 jobs:
   lint:
     name: Lint with stylelint and ESLint.
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.

.github/workflows/prettier.yaml

@@ -10,7 +10,7 @@ on:
 jobs:
   lint:
     name: Lint with Prettier
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.

bookwyrm/activitypub/book.py

@@ -13,6 +13,7 @@ class BookData(ActivityObject):
 
     openlibraryKey: Optional[str] = None
     inventaireId: Optional[str] = None
+    finnaKey: Optional[str] = None
     librarythingKey: Optional[str] = None
     goodreadsKey: Optional[str] = None
     bnfId: Optional[str] = None

bookwyrm/connectors/finna.py

@@ -0,0 +1,398 @@
+"""finna data connector"""
+
+import re
+from typing import Iterator
+
+from bookwyrm import models
+from bookwyrm.book_search import SearchResult
+from bookwyrm.models.book import FormatChoices
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
+from .abstract_connector import get_data
+from .connector_manager import ConnectorException, create_edition_task
+from .openlibrary_languages import languages
+
+
+class Connector(AbstractConnector):
+    """instantiate a connector for finna"""
+
+    generated_remote_link_field = "id"
+
+    def __init__(self, identifier: str):
+        super().__init__(identifier)
+
+        get_first = lambda x, *args: x[0] if x else None
+        format_remote_id = lambda x: f"{self.books_url}{x}"
+        format_cover_url = lambda x: f"{self.covers_url}{x[0]}" if x else None
+        self.book_mappings = [
+            Mapping("id", remote_field="id", formatter=format_remote_id),
+            Mapping("finnaKey", remote_field="id"),
+            Mapping("title", remote_field="shortTitle"),
+            Mapping("title", remote_field="title"),
+            Mapping("subtitle", remote_field="subTitle"),
+            Mapping("isbn10", remote_field="cleanIsbn"),
+            Mapping("languages", remote_field="languages", formatter=resolve_languages),
+            Mapping("authors", remote_field="authors", formatter=parse_authors),
+            Mapping("subjects", formatter=join_subject_list),
+            Mapping("publishedDate", remote_field="year"),
+            Mapping("cover", remote_field="images", formatter=format_cover_url),
+            Mapping("description", remote_field="summary", formatter=get_first),
+            Mapping("series", remote_field="series", formatter=parse_series_name),
+            Mapping(
+                "seriesNumber",
+                remote_field="series",
+                formatter=parse_series_number,
+            ),
+            Mapping("publishers", remote_field="publishers"),
+            Mapping(
+                "physicalFormat",
+                remote_field="formats",
+                formatter=describe_physical_format,
+            ),
+            Mapping(
+                "physicalFormatDetail",
+                remote_field="physicalDescriptions",
+                formatter=get_first,
+            ),
+            Mapping(
+                "pages",
+                remote_field="physicalDescriptions",
+                formatter=guess_page_numbers,
+            ),
+        ]
+
+        self.author_mappings = [
+            Mapping("id", remote_field="authors", formatter=self.get_remote_author_id),
+            Mapping("name", remote_field="authors", formatter=get_first_author),
+        ]
+
+    def get_book_data(self, remote_id: str) -> JsonDict:
+        request_parameters = {
+            "field[]": [
+                "authors",
+                "cleanIsbn",
+                "formats",
+                "id",
+                "images",
+                "isbns",
+                "languages",
+                "physicalDescriptions",
+                "publishers",
+                "recordPage",
+                "series",
+                "shortTitle",
+                "subjects",
+                "subTitle",
+                "summary",
+                "title",
+                "year",
+            ]
+        }
+        data = get_data(
+            url=remote_id, params=request_parameters  # type:ignore[arg-type]
+        )
+        extracted = data.get("records", [])
+        try:
+            data = extracted[0]
+        except (KeyError, IndexError):
+            raise ConnectorException("Invalid book data")
+        return data
+
+    def get_remote_author_id(self, data: JsonDict) -> str | None:
+        """return search url for author info, as we don't
+        have way to retrieve author-id with the query"""
+        author = get_first_author(data)
+        if author:
+            return f"{self.search_url}{author}&type=Author"
+        return None
+
+    def get_remote_id(self, data: JsonDict) -> str:
+        """return record-id page as book-id"""
+        return f"{self.books_url}{data.get('id')}"
+
+    def parse_search_data(
+        self, data: JsonDict, min_confidence: float
+    ) -> Iterator[SearchResult]:
+        for idx, search_result in enumerate(data.get("records", [])):
+            authors = search_result.get("authors")
+            author = None
+            if authors:
+                author_list = parse_authors(authors)
+                if author_list:
+                    author = "; ".join(author_list)
+
+            confidence = 1 / (idx + 1)
+            if confidence < min_confidence:
+                break
+
+            # Create some extra info on edition if it is audio-book or e-book
+            edition_info_title = describe_physical_format(search_result.get("formats"))
+            edition_info = ""
+            if edition_info_title and edition_info_title != "Hardcover":
+                for book_format, info_title in FormatChoices:
+                    if book_format == edition_info_title:
+                        edition_info = f" {info_title}"
+                        break
+
+            search_result = SearchResult(
+                title=f"{search_result.get('title')}{edition_info}",
+                key=f"{self.books_url}{search_result.get('id')}",
+                author=author,
+                cover=f"{self.covers_url}{search_result.get('images')[0]}"
+                if search_result.get("images")
+                else None,
+                year=search_result.get("year"),
+                view_link=f"{self.base_url}{search_result.get('recordPage')}",
+                confidence=confidence,
+                connector=self,
+            )
+            yield search_result
+
+    def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
+        """got some data"""
+        for idx, search_result in enumerate(data.get("records", [])):
+            authors = search_result.get("authors")
+            author = None
+            if authors:
+                author_list = parse_authors(authors)
+                if author_list:
+                    author = "; ".join(author_list)
+
+            confidence = 1 / (idx + 1)
+            yield SearchResult(
+                title=search_result.get("title"),
+                key=f"{self.books_url}{search_result.get('id')}",
+                author=author,
+                cover=f"{self.covers_url}{search_result.get('images')[0]}"
+                if search_result.get("images")
+                else None,
+                year=search_result.get("year"),
+                view_link=f"{self.base_url}{search_result.get('recordPage')}",
+                confidence=confidence,
+                connector=self,
+            )
+
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
+        authors = data.get("authors")
+        if authors:
+            for author in parse_authors(authors):
+                model = self.get_or_create_author(
+                    f"{self.search_url}{author}&type=Author"
+                )
+                if model:
+                    yield model
+
+    def expand_book_data(self, book: models.Book) -> None:
+        work = book
+        # go from the edition to the work, if necessary
+        if isinstance(book, models.Edition):
+            work = book.parent_work
+
+        try:
+            edition_options = retrieve_versions(work.finna_key)
+        except ConnectorException:
+            return
+
+        for edition in edition_options:
+            remote_id = self.get_remote_id(edition)
+            if remote_id:
+                create_edition_task.delay(self.connector.id, work.id, edition)
+
+    def get_remote_id_from_model(self, obj: models.BookDataModel) -> str:
+        """use get_remote_id to figure out the link from a model obj"""
+        return f"{self.books_url}{obj.finna_key}"
+
+    def is_work_data(self, data: JsonDict) -> bool:
+        """
+        https://api.finna.fi/v1/search?id=anders.1946700&search=versions&view=&lng=fi&field[]=formats&field[]=series&field[]=title&field[]=authors&field[]=summary&field[]=cleanIsbn&field[]=id
+
+        No real ordering what is work and what is edition, so pick first version as work
+        """
+        edition_list = retrieve_versions(data.get("id"))
+        if edition_list:
+            return data.get("id") == edition_list[0].get("id")
+        return True
+
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
+        """No real distinctions what is work/edition,
+        so check all versions and pick preferred edition"""
+        edition_list = retrieve_versions(data.get("id"))
+        if not edition_list:
+            raise ConnectorException("No editions found for work")
+        edition = pick_preferred_edition(edition_list)
+        if not edition:
+            raise ConnectorException("No editions found for work")
+        return edition
+
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
+        return retrieve_versions(data.get("id"))[0]
+
+
+def guess_page_numbers(data: JsonDict) -> str | None:
+    """Try to retrieve page count of edition"""
+    for row in data:
+        # Try to match page count text in style of '134 pages' or '134 sivua'
+        page_search = re.search(r"(\d+) (sivua|s\.|sidor|pages)", row)
+        page_count = page_search.group(1) if page_search else None
+        if page_count:
+            return page_count
+        # If we didn't match, try starting number
+        page_search = re.search(r"^(\d+)", row)
+        page_count = page_search.group(1) if page_search else None
+        if page_count:
+            return page_count
+    return None
+
+
+def resolve_languages(data: JsonDict) -> list[str]:
+    """Use openlibrary language code list to resolve iso-lang codes"""
+    result_languages = []
+    for language_code in data:
+        result_languages.append(
+            languages.get(f"/languages/{language_code}", language_code)
+        )
+    return result_languages
+
+
+def join_subject_list(data: list[JsonDict]) -> list[str]:
+    """Join list of string list about subject topics as one list"""
+    return [" ".join(info) for info in data]
+
+
+def describe_physical_format(formats: list[JsonDict]) -> str:
+    """Map if book is physical book, eBook or audiobook"""
+    found_format = "Hardcover"
+    # Map finnish finna formats to bookwyrm codes
+    format_mapping = {
+        "1/Book/Book/": "Hardcover",
+        "1/Book/AudioBook/": "AudiobookFormat",
+        "1/Book/eBook/": "EBook",
+    }
+    for format_to_check in formats:
+        format_value = format_to_check.get("value")
+        if not isinstance(format_value, str):
+            continue
+        if (mapping_match := format_mapping.get(format_value, None)) is not None:
+            found_format = mapping_match
+    return found_format
+
+
+def parse_series_name(series: list[JsonDict]) -> str | None:
+    """Parse series name if given"""
+    for info in series:
+        if "name" in info:
+            return info.get("name")
+    return None
+
+
+def parse_series_number(series: list[JsonDict]) -> str | None:
+    """Parse series number from additional info if given"""
+    for info in series:
+        if "additional" in info:
+            return info.get("additional")
+    return None
+
+
+def retrieve_versions(book_id: str | None) -> list[JsonDict]:
+    """
+    https://api.finna.fi/v1/search?id=anders.1946700&search=versions&view=&
+
+    Search all editions/versions of the book that finna is aware of
+    """
+
+    if not book_id:
+        return []
+
+    request_parameters = {
+        "id": book_id,
+        "search": "versions",
+        "view": "",
+        "field[]": [
+            "authors",
+            "cleanIsbn",
+            "edition",
+            "formats",
+            "id",
+            "images",
+            "isbns",
+            "languages",
+            "physicalDescriptions",
+            "publishers",
+            "recordPage",
+            "series",
+            "shortTitle",
+            "subjects",
+            "subTitle",
+            "summary",
+            "title",
+            "year",
+        ],
+    }
+    data = get_data(
+        url="https://api.finna.fi/api/v1/search",
+        params=request_parameters,  # type: ignore[arg-type]
+    )
+    result = data.get("records", [])
+    if isinstance(result, list):
+        return result
+    return []
+
+
+def get_first_author(data: JsonDict) -> str | None:
+    """Parse authors and return first one, usually the main author"""
+    authors = parse_authors(data)
+    if authors:
+        return authors[0]
+    return None
+
+
+def parse_authors(data: JsonDict) -> list[str]:
+    """Search author info, they are given in SurName, FirstName style
+    return them also as FirstName SurName order"""
+    if author_keys := data.get("primary", None):
+        if author_keys:
+            # we search for 'kirjoittaja' role, if any found
+            tulos = list(
+                # Convert from 'Lewis, Michael' to 'Michael Lewis'
+                " ".join(reversed(author_key.split(", ")))
+                for author_key, author_info in author_keys.items()
+                if "kirjoittaja" in author_info.get("role", [])
+            )
+            if tulos:
+                return tulos
+            # if not found, we search any role that is not specificly something
+            tulos = list(
+                " ".join(reversed(author_key.split(", ")))
+                for author_key, author_info in author_keys.items()
+                if "-" in author_info.get("role", [])
+            )
+            return tulos
+    return []
+
+
+def pick_preferred_edition(options: list[JsonDict]) -> JsonDict | None:
+    """favor physical copies with covers in english"""
+    if not options:
+        return None
+    if len(options) == 1:
+        return options[0]
+
+    # pick hardcodver book if present over eBook/audiobook
+    formats = ["1/Book/Book/"]
+    format_selection = []
+    for edition in options:
+        for edition_format in edition.get("formats", []):
+            if edition_format.get("value") in formats:
+                format_selection.append(edition)
+    options = format_selection or options
+
+    # Prefer Finnish/Swedish language editions if any found
+    language_list = ["fin", "swe"]
+    languages_selection = []
+    for edition in options:
+        for edition_language in edition.get("languages", []):
+            if edition_language in language_list:
+                languages_selection.append(edition)
+    options = languages_selection or options
+
+    options = [e for e in options if e.get("cleanIsbn")] or options
+    return options[0]

bookwyrm/connectors/inventaire.py

@@ -222,9 +222,10 @@ class Connector(AbstractConnector):
     def get_description(self, links: JsonDict) -> str:
         """grab an extracted excerpt from wikipedia"""
         link = links.get("enwiki")
-        if not link:
+        if not link or not link.get("title"):
             return ""
-        url = f"{self.base_url}/api/data?action=wp-extract&lang=en&title={link}"
+        title = link.get("title")
+        url = f"{self.base_url}/api/data?action=wp-extract&lang=en&title={title}"
         try:
             data = get_data(url)
         except ConnectorException:

bookwyrm/connectors/settings.py

@@ -1,3 +1,3 @@
 """ settings book data connectors """
 
-CONNECTORS = ["openlibrary", "inventaire", "bookwyrm_connector"]
+CONNECTORS = ["openlibrary", "inventaire", "bookwyrm_connector", "finna"]

bookwyrm/forms/books.py

@@ -40,6 +40,7 @@ class EditionForm(CustomForm):
             "openlibrary_key",
             "inventaire_id",
             "goodreads_key",
+            "finna_key",
             "oclc_number",
             "asin",
             "aasin",
@@ -93,6 +94,7 @@ class EditionForm(CustomForm):
             "oclc_number": forms.TextInput(
                 attrs={"aria-describedby": "desc_oclc_number"}
             ),
+            "finna_key": forms.TextInput(attrs={"aria-describedby": "desc_finna_key"}),
             "ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}),
             "AASIN": forms.TextInput(attrs={"aria-describedby": "desc_AASIN"}),
             "isfdb": forms.TextInput(attrs={"aria-describedby": "desc_isfdb"}),

bookwyrm/forms/landing.py

@@ -64,6 +64,10 @@ class InviteRequestForm(CustomForm):
         if email and models.User.objects.filter(email=email).exists():
             self.add_error("email", _("A user with this email already exists."))
 
+        email_domain = email.split("@")[-1]
+        if email and models.EmailBlocklist.objects.filter(domain=email_domain).exists():
+            self.add_error("email", _("This email address cannot be registered."))
+
     class Meta:
         model = models.InviteRequest
         fields = ["email", "answer"]

bookwyrm/management/commands/add_finna_connector.py

@@ -0,0 +1,58 @@
+""" Add finna connector to connectors """
+from django.core.management.base import BaseCommand
+
+from bookwyrm import models
+
+
+def enable_finna_connector():
+
+    models.Connector.objects.create(
+        identifier="api.finna.fi",
+        name="Finna API",
+        connector_file="finna",
+        base_url="https://www.finna.fi",
+        books_url="https://api.finna.fi/api/v1/record" "?id=",
+        covers_url="https://api.finna.fi",
+        search_url="https://api.finna.fi/api/v1/search?limit=20"
+        "&filter[]=format%3a%220%2fBook%2f%22"
+        "&field[]=title&field[]=recordPage&field[]=authors"
+        "&field[]=year&field[]=id&field[]=formats&field[]=images"
+        "&lookfor=",
+        isbn_search_url="https://api.finna.fi/api/v1/search?limit=1"
+        "&filter[]=format%3a%220%2fBook%2f%22"
+        "&field[]=title&field[]=recordPage&field[]=authors&field[]=year"
+        "&field[]=id&field[]=formats&field[]=images"
+        "&lookfor=isbn:",
+    )
+
+
+def remove_finna_connector():
+    models.Connector.objects.filter(identifier="api.finna.fi").update(
+        active=False, deactivation_reason="Disabled by management command"
+    )
+    print("Finna connector deactivated")
+
+
+# pylint: disable=no-self-use
+# pylint: disable=unused-argument
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Setup Finna API connector"
+
+    def add_arguments(self, parser):
+        """specify argument to remove connector"""
+        parser.add_argument(
+            "--deactivate",
+            action="store_true",
+            help="Deactivate the finna connector from config",
+        )
+
+    def handle(self, *args, **options):
+        """enable or remove connector"""
+        if options.get("deactivate"):
+            print("Deactivate finna connector config if one present")
+            remove_finna_connector()
+        else:
+            print("Adding Finna API connector to configuration")
+            enable_finna_connector()

bookwyrm/migrations/0210_alter_connector_connector_file.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.17 on 2025-02-02 20:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0209_user_show_ratings"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="connector",
+            name="connector_file",
+            field=models.CharField(
+                choices=[
+                    ("openlibrary", "Openlibrary"),
+                    ("inventaire", "Inventaire"),
+                    ("bookwyrm_connector", "Bookwyrm Connector"),
+                    ("finna", "Finna"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0211_author_finna_key_book_finna_key.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2.17 on 2025-02-08 16:14
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0210_alter_connector_connector_file"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="finna_key",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="finna_key",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/models/book.py

@@ -41,6 +41,9 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
     openlibrary_key = fields.CharField(
         max_length=255, blank=True, null=True, deduplication_field=True
     )
+    finna_key = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True
+    )
     inventaire_id = fields.CharField(
         max_length=255, blank=True, null=True, deduplication_field=True
     )

bookwyrm/templates/book/book_identifiers.html

@@ -52,6 +52,13 @@
             <dd>{{ book.goodreads_key }}</dd>
         </div>
     {% endif %}
+
+    {% if book.finna_key %}
+        <div class="is-flex is-flex-wrap-wrap">
+            <dt class="mr-1">{% trans "Finna ID:" %}</dt>
+            <dd>{{ book.finna_key}}</dd>
+        </div>
+    {% endif %}
 </dl>
 {% endif %}
 {% endspaceless %}

bookwyrm/templates/book/edit/edit_book_form.html

@@ -382,6 +382,15 @@
 
                     {% include 'snippets/form_errors.html' with errors_list=form.isfdb.errors id="desc_isfdb" %}
                 </div>
+
+                <div class="field">
+                    <label class="label" for="id_finna_key">
+                        {% trans "Finna ID:" %}
+                    </label>
+                    {{ form.finna_key }}
+
+                    {% include 'snippets/form_errors.html' with errors_list=form.finna_key.errors id="desc_finna_key" %}
+                </div>
             </div>
         </section>
     </div>

bookwyrm/templates/search/author.html

@@ -8,6 +8,9 @@
     <li class="">
         <a href="{{ author.local_path }}" class="author" itemprop="author" itemscope itemtype="https://schema.org/Thing">
             <span itemprop="name">{{ author.name }}</span>
+            {% if author.born or author.died %}
+            <span>({{ author.born|date:"Y" }}-{{ author.died|date:"Y" }})</span>
+            {% endif %}
         </a>
     </li>
 {% endfor %}

bookwyrm/tests/connectors/test_finna_connector.py

@@ -0,0 +1,142 @@
+""" testing book data connectors """
+import json
+import pathlib
+
+from django.test import TestCase
+import responses
+
+from bookwyrm import models
+from bookwyrm.connectors.finna import Connector, guess_page_numbers
+
+
+class Finna(TestCase):
+    """test loading data from finna.fi"""
+
+    @classmethod
+    def setUpTestData(cls):
+        """creates the connector in the database"""
+        models.Connector.objects.create(
+            identifier="api.finna.fi",
+            name="Finna API",
+            connector_file="finna",
+            base_url="https://www.finna.fi",
+            books_url="https://api.finna.fi/api/v1/record" "?id=",
+            covers_url="https://api.finna.fi",
+            search_url="https://api.finna.fi/api/v1/search?limit=20"
+            "&filter[]=format%3a%220%2fBook%2f%22"
+            "&field[]=title&field[]=recordPage&field[]=authors"
+            "&field[]=year&field[]=id&field[]=formats&field[]=images"
+            "&lookfor=",
+            isbn_search_url="https://api.finna.fi/api/v1/search?limit=1"
+            "&filter[]=format%3a%220%2fBook%2f%22"
+            "&field[]=title&field[]=recordPage&field[]=authors&field[]=year"
+            "&field[]=id&field[]=formats&field[]=images"
+            "&lookfor=isbn:",
+        )
+
+    def setUp(self):
+        """connector instance"""
+        self.connector = Connector("api.finna.fi")
+
+    def test_parse_search_data(self):
+        """json to search result objs"""
+        search_file = pathlib.Path(__file__).parent.joinpath(
+            "../data/finna_search.json"
+        )
+        search_results = json.loads(search_file.read_bytes())
+        print(search_results)
+
+        print(self.connector)
+        formatted = list(self.connector.parse_search_data(search_results, 0))
+        print(formatted)
+
+        self.assertEqual(formatted[0].title, "Sarvijumala")
+        self.assertEqual(formatted[0].author, "Magdalena Hai")
+        self.assertEqual(
+            formatted[0].key, "https://api.finna.fi/api/v1/record?id=anders.1920022"
+        )
+        self.assertEqual(
+            formatted[0].cover,
+            None,
+        )
+        # Test that edition info is parsed correctly to title
+        self.assertEqual(formatted[1].title, "Sarvijumala Audiobook")
+        self.assertEqual(formatted[2].title, "Sarvijumala")
+        self.assertEqual(formatted[3].title, "Sarvijumala eBook")
+
+    def test_parse_isbn_search_data(self):
+        """another search type"""
+        search_file = pathlib.Path(__file__).parent.joinpath(
+            "../data/finna_isbn_search.json"
+        )
+        search_results = json.loads(search_file.read_bytes())
+
+        formatted = list(self.connector.parse_isbn_search_data(search_results))[0]
+
+        self.assertEqual(formatted.title, "Ilmakirja : painovoimainen ilmanvaihto")
+        self.assertEqual(
+            formatted.key, "https://api.finna.fi/api/v1/record?id=3amk.308439"
+        )
+
+    def test_parse_isbn_search_data_empty(self):
+        """another search type"""
+        search_results = {"resultCount": 0, "records": []}
+        results = list(self.connector.parse_isbn_search_data(search_results))
+        self.assertEqual(results, [])
+
+    def test_page_count_parsing(self):
+        """Test page count parsing flow"""
+        for data in [
+            "123 sivua",
+            "123, [4] sivua",
+            "sidottu 123 sivua",
+            "123s; [4]; 9cm",
+        ]:
+            page_count = guess_page_numbers([data])
+            self.assertEqual(page_count, "123")
+        for data in [" sivua", "xx, [4] sivua", "sidottu", "[4]; 9cm"]:
+            page_count = guess_page_numbers([data])
+            self.assertEqual(page_count, None)
+
+    @responses.activate
+    def test_get_book_data(self):
+        """Test book data parsing from example json files"""
+        record_file = pathlib.Path(__file__).parent.joinpath(
+            "../data/finna_record.json"
+        )
+        version_file = pathlib.Path(__file__).parent.joinpath(
+            "../data/finna_versions.json"
+        )
+        author_file = pathlib.Path(__file__).parent.joinpath(
+            "../data/finna_author_search.json"
+        )
+        record_result = json.loads(record_file.read_bytes())
+        versions_result = json.loads(version_file.read_bytes())
+        author_search_result = json.loads(author_file.read_bytes())
+        responses.add(
+            responses.GET,
+            "https://api.finna.fi/api/v1/search?id=anders.1819084&search=versions"
+            "&view=&field%5B%5D=authors&field%5B%5D=cleanIsbn&field%5B%5D=edition"
+            "&field%5B%5D=formats&field%5B%5D=id&field%5B%5D=images&field%5B%5D=isbns"
+            "&field%5B%5D=languages&field%5B%5D=physicalDescriptions"
+            "&field%5B%5D=publishers&field%5B%5D=recordPage&field%5B%5D=series"
+            "&field%5B%5D=shortTitle&field%5B%5D=subjects&field%5B%5D=subTitle"
+            "&field%5B%5D=summary&field%5B%5D=title&field%5B%5D=year",
+            json=versions_result,
+        )
+        responses.add(
+            responses.GET,
+            "https://api.finna.fi/api/v1/search?limit=20&filter%5B%5D=format%3A%220%2F"
+            "Book%2F%22&field%5B%5D=title&field%5B%5D=recordPage&field%5B%5D=authors&"
+            "field%5B%5D=year&field%5B%5D=id&field%5B%5D=formats&field%5B%5D=images&"
+            "lookfor=Emmi%20It%C3%A4ranta&type=Author&field%5B%5D=authors&field%5B%5D"
+            "=cleanIsbn&field%5B%5D=formats&field%5B%5D=id&field%5B%5D=images&field"
+            "%5B%5D=isbns&field%5B%5D=languages&field%5B%5D=physicalDescriptions&"
+            "field%5B%5D=publishers&field%5B%5D=recordPage&field%5B%5D=series&field"
+            "%5B%5D=shortTitle&field%5B%5D=subjects&field%5B%5D=subTitle"
+            "&field%5B%5D=summary&field%5B%5D=title&field%5B%5D=year",
+            json=author_search_result,
+        )
+        responses.add(responses.GET, "https://test.url/id", json=record_result)
+        book = self.connector.get_or_create_book("https://test.url/id")
+        self.assertEqual(book.languages[0], "Finnish")

bookwyrm/tests/connectors/test_inventaire_connector.py

@@ -273,7 +273,9 @@ class Inventaire(TestCase):
             json={"extract": "hi hi"},
         )
 
-        extract = self.connector.get_description({"enwiki": "test_path"})
+        extract = self.connector.get_description(
+            {"enwiki": {"title": "test_path", "badges": "hello"}}
+        )
         self.assertEqual(extract, "hi hi")
 
     def test_remote_id_from_model(self):

bookwyrm/tests/data/finna_isbn_search.json

@@ -0,0 +1,54 @@
+{
+  "resultCount": 1,
+  "records": [
+    {
+      "title": "Ilmakirja : painovoimainen ilmanvaihto",
+      "recordPage": "/Record/3amk.308439",
+      "authors": {
+        "primary": {
+          "Mikkola, Juulia": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Kuuluvainen, Leino": {
+            "role": [
+              "-"
+            ]
+          },
+          "Böök, Netta": {
+            "role": [
+              "-"
+            ]
+          },
+          "Moreeni": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": {
+          "Moreeni, kustantaja": {
+            "role": [
+              "kustantaja"
+            ]
+          }
+        }
+      },
+      "year": "2024",
+      "id": "3amk.308439",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    }
+  ],
+  "status": "OK"
+}

bookwyrm/tests/data/finna_record.json

@@ -0,0 +1,86 @@
+{
+  "resultCount": 1,
+  "records": [
+    {
+      "authors": {
+        "primary": {
+          "Itäranta, Emmi": {
+            "role": [
+              "kirjoittaja"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "cleanIsbn": "9523630873",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ],
+      "id": "anders.1819084",
+      "languages": [
+        "fin"
+      ],
+      "physicalDescriptions": [
+        "381 sivua ; 22 cm"
+      ],
+      "publishers": [
+        "Kustannusosakeyhtiö Teos"
+      ],
+      "recordPage": "/Record/anders.1819084",
+      "series": [],
+      "shortTitle": "Kuunpäivän kirjeet",
+      "subjects": [
+        [
+          "Salo, Lumi",
+          "(fiktiivinen hahmo)"
+        ],
+        [
+          "Soli",
+          "(fiktiivinen hahmo)"
+        ],
+        [
+          "katoaminen"
+        ],
+        [
+          "etsintä"
+        ],
+        [
+          "identiteetti"
+        ],
+        [
+          "luokkaerot"
+        ],
+        [
+          "riisto"
+        ],
+        [
+          "puolisot"
+        ],
+        [
+          "sisäkertomukset"
+        ],
+        [
+          "muistikirjat"
+        ],
+        [
+          "siirtokunnat"
+        ],
+        [
+          "vaihtoehtoiset todellisuudet"
+        ]
+      ],
+      "summary": [],
+      "title": "Kuunpäivän kirjeet",
+      "year": "2020"
+    }
+  ],
+  "status": "OK"
+}

bookwyrm/tests/data/finna_search.json

@@ -0,0 +1,677 @@
+{
+  "resultCount": 87,
+  "records": [
+    {
+      "title": "Sarvijumala",
+      "recordPage": "/Record/anders.1920022",
+      "authors": {
+        "primary": {
+          "Hai, Magdalena": {
+            "role": [
+              "kirjoittaja"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2023",
+      "id": "anders.1920022",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Sarvijumala",
+      "recordPage": "/Record/fikka.5591048",
+      "authors": {
+        "primary": {
+          "Hai, Magdalena": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Toiviainen, Miiko": {
+            "role": [
+              "-"
+            ]
+          },
+          "Otava": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Toiviainen, Miiko, lukija": {
+            "role": [
+              "lukija"
+            ]
+          }
+        },
+        "corporate": {
+          "Otava, kustantaja": {
+            "role": [
+              "kustantaja"
+            ]
+          }
+        }
+      },
+      "year": "2023",
+      "id": "fikka.5591048",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/AudioBook/",
+          "translated": "Äänikirja"
+        },
+        {
+          "value": "2/Book/AudioBook/Online/",
+          "translated": "E-äänikirja"
+        }
+      ]
+    },
+    {
+      "title": "Sarvijumala",
+      "recordPage": "/Record/anders.1961374",
+      "authors": {
+        "primary": {
+          "Takala, Tuija": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Otava, kustannusosakeyhtiö": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Hai, Magdalena": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "corporate": {
+          "Otava, kustannusosakeyhtiö, kustantaja": {
+            "role": [
+              "kustantaja"
+            ]
+          }
+        }
+      },
+      "year": "2024",
+      "id": "anders.1961374",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Sarvijumala",
+      "recordPage": "/Record/fikka.5591040",
+      "authors": {
+        "primary": {
+          "Hai, Magdalena": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Otava, kustannusosakeyhtiö": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": {
+          "Otava, kustannusosakeyhtiö, kustantaja": {
+            "role": [
+              "kustantaja"
+            ]
+          }
+        }
+      },
+      "year": "2023",
+      "id": "fikka.5591040",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/eBook/",
+          "translated": "E-kirja"
+        }
+      ]
+    },
+    {
+      "title": "Sarvijumala",
+      "recordPage": "/Record/keski.3324001",
+      "authors": {
+        "primary": {
+          "Hai, Magdalena": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Toiviainen, Miiko": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Toiviainen, Miiko, lukija": {
+            "role": [
+              "lukija"
+            ]
+          }
+        },
+        "corporate": []
+      },
+      "year": "2023",
+      "id": "keski.3324001",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/AudioBook/",
+          "translated": "Äänikirja"
+        },
+        {
+          "value": "2/Book/AudioBook/Daisy/",
+          "translated": "Celia-äänikirja"
+        }
+      ]
+    },
+    {
+      "title": "Kirjapaketti kouluille : Sarvijumala",
+      "recordPage": "/Record/vaski.4353064",
+      "authors": {
+        "primary": {
+          "Hai, Magdalena": {
+            "role": [
+              "kirjoittaja"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2023",
+      "id": "vaski.4353064",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Sarvijumala : Daisy-äänikirja vain lukemisesteisille",
+      "recordPage": "/Record/kyyti.1536498",
+      "authors": {
+        "primary": {
+          "Hai, Magdalena": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Toiviainen, Miiko, lukija": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2023",
+      "id": "kyyti.1536498",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/AudioBook/",
+          "translated": "Äänikirja"
+        },
+        {
+          "value": "2/Book/AudioBook/Daisy/",
+          "translated": "Celia-äänikirja"
+        }
+      ]
+    },
+    {
+      "title": "Tulta, verta, savupatsaita!. Osa 4, New Yorkin tulipätsissä Jumala antoi valokuvata Luciferin sarvet",
+      "recordPage": "/Record/fikka.3958203",
+      "authors": {
+        "primary": {
+          "Meller, Leo": {
+            "role": [
+              "-"
+            ]
+          },
+          "Kuva ja sana (yhtiö)": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": {
+          "Kuva ja sana (yhtiö), kustantaja": {
+            "role": [
+              "kustantaja"
+            ]
+          }
+        }
+      },
+      "year": "2002",
+      "id": "fikka.3958203",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/AudioBook/",
+          "translated": "Äänikirja"
+        },
+        {
+          "value": "2/Book/AudioBook/Cassette/",
+          "translated": "Kasettiäänikirja"
+        }
+      ]
+    },
+    {
+      "title": "Nico Bravo and the trial of Vulcan",
+      "recordPage": "/Record/helmet.2531986",
+      "authors": {
+        "primary": {
+          "Cavallaro, Michael": {
+            "role": [
+              "sarjakuvantekijä"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2022",
+      "id": "helmet.2531986",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Nico Bravo and the cellar dwellers",
+      "recordPage": "/Record/helmet.2536894",
+      "authors": {
+        "primary": {
+          "Cavallaro, Michael": {
+            "role": [
+              "sarjakuvantekijä"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2020",
+      "id": "helmet.2536894",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Nico Bravo and the cellar dwellers",
+      "recordPage": "/Record/helmet.2536962",
+      "authors": {
+        "primary": {
+          "Cavallaro, Michael": {
+            "role": [
+              "sarjakuvantekijä"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2020",
+      "id": "helmet.2536962",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Nico Bravo and the hound of Hades",
+      "recordPage": "/Record/helmet.2536893",
+      "authors": {
+        "primary": {
+          "Cavallaro, Mike": {
+            "role": [
+              "sarjakuvantekijä"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2019",
+      "id": "helmet.2536893",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Monsters : a bestiary of the bizarre",
+      "recordPage": "/Record/karelia.99700956005967",
+      "authors": {
+        "primary": {
+          "Dell, Christopher": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2016",
+      "id": "karelia.99700956005967",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Wendigo ja muita yliluonnollisia kauhukertomuksia",
+      "recordPage": "/Record/anders.1471241",
+      "authors": {
+        "primary": {
+          "Sadelehto, Markku": {
+            "role": [
+              "-"
+            ]
+          },
+          "Rosvall, Matti": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Sadelehto, Markku, toimittaja": {
+            "role": [
+              "toimittaja"
+            ]
+          },
+          "Rosvall, Matti, kääntäjä": {
+            "role": [
+              "kääntäjä"
+            ]
+          }
+        },
+        "corporate": []
+      },
+      "year": "2015",
+      "id": "anders.1471241",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Sairas kertomus",
+      "recordPage": "/Record/jykdok.1506488",
+      "authors": {
+        "primary": {
+          "Sivonen, Hannu": {
+            "role": [
+              "kirjoittaja"
+            ]
+          }
+        },
+        "secondary": {
+          "Sivonen, Hannu, kirjoittaja": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "corporate": []
+      },
+      "year": "2015",
+      "id": "jykdok.1506488",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Kuvitteellisten olentojen kirja",
+      "recordPage": "/Record/anders.891877",
+      "authors": {
+        "primary": {
+          "Borges, Jorge Luis": {
+            "role": [
+              "tekijä"
+            ]
+          },
+          "Guerrero, Margarita": {
+            "role": [
+              "-"
+            ]
+          },
+          "Selander, Sari": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Selander, Sari, kääntäjä": {
+            "role": [
+              "kääntäjä"
+            ]
+          }
+        },
+        "corporate": []
+      },
+      "year": "2009",
+      "id": "anders.891877",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Suutele minulle siivet",
+      "recordPage": "/Record/anders.201550",
+      "authors": {
+        "primary": {
+          "Tabermann, Tommy": {
+            "role": [
+              "kirjoittaja"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "2004",
+      "id": "anders.201550",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Elämänuskon kirja",
+      "recordPage": "/Record/anders.910579",
+      "authors": {
+        "primary": {
+          "Jaatinen, Sanna": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "1996",
+      "id": "anders.910579",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Suomen kansan eläinkirja : kertomus Metsolan ja Ilmolan väestä ja elämästä",
+      "recordPage": "/Record/anders.88891",
+      "authors": {
+        "primary": {
+          "Railo, Eino": {
+            "role": [
+              "kirjoittaja"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "1934",
+      "id": "anders.88891",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    },
+    {
+      "title": "Iloinen laulu Eevasta ja Aatamista : runoja",
+      "recordPage": "/Record/anders.909304",
+      "authors": {
+        "primary": {
+          "Ahti, Risto": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "year": "1995",
+      "id": "anders.909304",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ]
+    }
+  ],
+  "status": "OK"
+}

bookwyrm/tests/data/finna_versions.json

@@ -0,0 +1,507 @@
+{
+  "resultCount": 7,
+  "records": [
+    {
+      "authors": {
+        "primary": {
+          "Itäranta, Emmi": {
+            "role": [
+              "kirjoittaja"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "cleanIsbn": "9523630873",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ],
+      "id": "anders.1819084",
+      "isbns": [
+        "978-952-363-087-1 sidottu"
+      ],
+      "languages": [
+        "fin"
+      ],
+      "physicalDescriptions": [
+        "381 sivua ; 22 cm"
+      ],
+      "publishers": [
+        "Kustannusosakeyhtiö Teos"
+      ],
+      "recordPage": "/Record/anders.1819084",
+      "series": [],
+      "shortTitle": "Kuunpäivän kirjeet",
+      "subjects": [
+        [
+          "Salo, Lumi",
+          "(fiktiivinen hahmo)"
+        ],
+        [
+          "Soli",
+          "(fiktiivinen hahmo)"
+        ],
+        [
+          "katoaminen"
+        ],
+        [
+          "etsintä"
+        ],
+        [
+          "identiteetti"
+        ],
+        [
+          "luokkaerot"
+        ],
+        [
+          "riisto"
+        ],
+        [
+          "puolisot"
+        ],
+        [
+          "sisäkertomukset"
+        ],
+        [
+          "muistikirjat"
+        ],
+        [
+          "siirtokunnat"
+        ],
+        [
+          "vaihtoehtoiset todellisuudet"
+        ]
+      ],
+      "summary": [],
+      "title": "Kuunpäivän kirjeet",
+      "year": "2020"
+    },
+    {
+      "authors": {
+        "primary": {
+          "Itäranta, Emmi": {
+            "role": [
+              "kirjoittaja"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": []
+      },
+      "cleanIsbn": "1803360445",
+      "edition": "First Titan edition",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ],
+      "id": "anders.1906854",
+      "isbns": [
+        "978-1-80336-044-7 pehmeäkantinen"
+      ],
+      "languages": [
+        "eng"
+      ],
+      "physicalDescriptions": [
+        "365 sivua ; 20 cm"
+      ],
+      "publishers": [
+        "Titan Books"
+      ],
+      "recordPage": "/Record/anders.1906854",
+      "series": [],
+      "shortTitle": "The moonday letters",
+      "subjects": [
+        [
+          "Salo, Lumi",
+          "(fiktiivinen hahmo)"
+        ],
+        [
+          "Sol",
+          "(fiktiivinen hahmo)"
+        ],
+        [
+          "katoaminen"
+        ],
+        [
+          "etsintä"
+        ],
+        [
+          "identiteetti"
+        ],
+        [
+          "luokkaerot"
+        ],
+        [
+          "riisto"
+        ],
+        [
+          "puolisot"
+        ],
+        [
+          "sisäkertomukset"
+        ],
+        [
+          "muistikirjat"
+        ],
+        [
+          "siirtokunnat"
+        ],
+        [
+          "vaihtoehtoiset todellisuudet"
+        ]
+      ],
+      "summary": [
+        "Sol has disappeared. Their Earth-born wife Lumi sets out to find them but it is no simple feat: each clue uncovers another enigma. Their disappearance leads back to underground environmental groups and a web of mystery that spans the space between the planets themselves. Told through letters and extracts, the course of Lumi's journey takes her not only from the affluent colonies of Mars to the devastated remnants of Earth, but into the hidden depths of Sol's past and the long-forgotten secrets of her own."
+      ],
+      "title": "The moonday letters",
+      "year": "2022"
+    },
+    {
+      "authors": {
+        "primary": {
+          "Itäranta, Emmi": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Varjomäki, Elina": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Varjomäki, Elina, lukija": {
+            "role": [
+              "lukija"
+            ]
+          }
+        },
+        "corporate": []
+      },
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/AudioBook/",
+          "translated": "Äänikirja"
+        },
+        {
+          "value": "2/Book/AudioBook/Daisy/",
+          "translated": "Celia-äänikirja"
+        }
+      ],
+      "id": "keski.3127858",
+      "isbns": [],
+      "languages": [
+        "fin"
+      ],
+      "physicalDescriptions": [
+        "1 CD-levy (Daisy) (11 h 41 min)"
+      ],
+      "publishers": [
+        "Celia"
+      ],
+      "recordPage": "/Record/keski.3127858",
+      "series": [],
+      "shortTitle": "Kuunpäivän kirjeet",
+      "subjects": [
+        [
+          "Salo, Lumi",
+          "(fiktiivinen hahmo)"
+        ],
+        [
+          "Sol",
+          "(fiktiivinen hahmo)"
+        ],
+        [
+          "katoaminen"
+        ],
+        [
+          "etsintä"
+        ],
+        [
+          "identiteetti"
+        ],
+        [
+          "luokkaerot"
+        ],
+        [
+          "riisto"
+        ],
+        [
+          "puolisot"
+        ],
+        [
+          "sisäkertomukset"
+        ],
+        [
+          "muistikirjat"
+        ],
+        [
+          "siirtokunnat"
+        ],
+        [
+          "vaihtoehtoiset todellisuudet"
+        ]
+      ],
+      "summary": [],
+      "title": "Kuunpäivän kirjeet",
+      "year": "2020"
+    },
+    {
+      "authors": {
+        "primary": {
+          "Itäranta, Emmi": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Varjomäki, Elina": {
+            "role": [
+              "-"
+            ]
+          },
+          "Teos (kustantamo)": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Varjomäki, Elina, lukija": {
+            "role": [
+              "lukija"
+            ]
+          }
+        },
+        "corporate": {
+          "Teos (kustantamo), kustantaja": {
+            "role": [
+              "kustantaja"
+            ]
+          }
+        }
+      },
+      "cleanIsbn": "9523631020",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/AudioBook/",
+          "translated": "Äänikirja"
+        },
+        {
+          "value": "2/Book/AudioBook/Online/",
+          "translated": "E-äänikirja"
+        }
+      ],
+      "id": "fikka.5456913",
+      "isbns": [
+        "978-952-363-102-1 MP3"
+      ],
+      "languages": [
+        "fin"
+      ],
+      "physicalDescriptions": [
+        "1 verkkoaineisto (1 äänitiedosto (11 h 39 min))"
+      ],
+      "publishers": [
+        "Kustannusosakeyhtiö Teos"
+      ],
+      "recordPage": "/Record/fikka.5456913",
+      "series": [],
+      "shortTitle": "Kuunpäivän kirjeet",
+      "subjects": [],
+      "summary": [],
+      "title": "Kuunpäivän kirjeet",
+      "year": "2020"
+    },
+    {
+      "authors": {
+        "primary": {
+          "Itäranta, Emmi": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Kustannusosakeyhtiö Teos": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": [],
+        "corporate": {
+          "Kustannusosakeyhtiö Teos, kustantaja": {
+            "role": [
+              "kustantaja"
+            ]
+          }
+        }
+      },
+      "cleanIsbn": "9523631012",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/eBook/",
+          "translated": "E-kirja"
+        }
+      ],
+      "id": "fikka.5458151",
+      "isbns": [
+        "978-952-363-101-4 EPUB"
+      ],
+      "languages": [
+        "fin"
+      ],
+      "physicalDescriptions": [
+        "1 verkkoaineisto"
+      ],
+      "publishers": [
+        "Teos"
+      ],
+      "recordPage": "/Record/fikka.5458151",
+      "series": [],
+      "shortTitle": "Kuunpäivän kirjeet",
+      "subjects": [],
+      "summary": [],
+      "title": "Kuunpäivän kirjeet",
+      "year": "2020"
+    },
+    {
+      "authors": {
+        "primary": {
+          "Itäranta, Emmi": {
+            "role": [
+              "kirjoittaja"
+            ]
+          },
+          "Švec, Michal": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Švec, Michal, kääntäjä": {
+            "role": [
+              "kääntäjä"
+            ]
+          }
+        },
+        "corporate": []
+      },
+      "cleanIsbn": "807662634X",
+      "edition": "Vydání první",
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/Book/",
+          "translated": "Kirja"
+        }
+      ],
+      "id": "fikka.5773795",
+      "isbns": [
+        "978-80-7662-634-8 kovakantinen"
+      ],
+      "languages": [
+        "ces"
+      ],
+      "physicalDescriptions": [
+        "332 sivua ; 21 cm"
+      ],
+      "publishers": [
+        "Kniha Zlin"
+      ],
+      "recordPage": "/Record/fikka.5773795",
+      "series": [],
+      "shortTitle": "Dopisy měsíčního dne",
+      "subjects": [],
+      "summary": [],
+      "title": "Dopisy měsíčního dne",
+      "year": "2024"
+    },
+    {
+      "authors": {
+        "primary": {
+          "Itäranta, Emmi": {
+            "role": [
+              "-"
+            ]
+          },
+          "Varjomäki, Elina": {
+            "role": [
+              "-"
+            ]
+          }
+        },
+        "secondary": {
+          "Varjomäki, Elina , lukija": {
+            "role": [
+              "lukija"
+            ]
+          }
+        },
+        "corporate": []
+      },
+      "formats": [
+        {
+          "value": "0/Book/",
+          "translated": "Kirja"
+        },
+        {
+          "value": "1/Book/AudioBook/",
+          "translated": "Äänikirja"
+        },
+        {
+          "value": "2/Book/AudioBook/Daisy/",
+          "translated": "Celia-äänikirja"
+        }
+      ],
+      "id": "ratamo.2045073",
+      "isbns": [],
+      "languages": [
+        "fin"
+      ],
+      "physicalDescriptions": [
+        "1 CD-äänilevy (MP3) ( 11 h 41 min)"
+      ],
+      "publishers": [
+        "Celia"
+      ],
+      "recordPage": "/Record/ratamo.2045073",
+      "series": [],
+      "shortTitle": "Kuunpäivän kirjeet",
+      "subjects": [],
+      "summary": [],
+      "title": "Kuunpäivän kirjeet",
+      "year": "2020"
+    }
+  ],
+  "status": "OK"
+}

bookwyrm/tests/management/test_add_finna_connector.py

@@ -0,0 +1,34 @@
+""" test populating user streams """
+from django.test import TestCase
+
+from bookwyrm.models import Connector
+from bookwyrm.management.commands import add_finna_connector
+
+
+class InitDB(TestCase):
+    """Add/remove finna connector"""
+
+    def test_adding_connector(self):
+        """Create groups"""
+        add_finna_connector.enable_finna_connector()
+        self.assertTrue(
+            Connector.objects.filter(identifier="api.finna.fi", active=True).exists()
+        )
+
+    def test_command_no_args(self):
+        """command line calls"""
+        command = add_finna_connector.Command()
+        command.handle()
+        self.assertTrue(
+            Connector.objects.filter(identifier="api.finna.fi", active=True).exists()
+        )
+
+    def test_command_with_args(self):
+        """command line calls"""
+        command = add_finna_connector.Command()
+        command.handle(deactivate=True)
+
+        # everything should have been cleaned
+        self.assertFalse(
+            Connector.objects.filter(identifier="api.finna.fi", active=True).exists()
+        )

bookwyrm/tests/views/lists/test_lists.py

@@ -145,11 +145,21 @@ class ListViews(TestCase):
     def test_user_lists_page_logged_out(self):
         """there are so many views, this just makes sure it LOADS"""
         view = views.UserLists.as_view()
+        with (
+            patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"),
+            patch("bookwyrm.lists_stream.remove_list_task.delay"),
+        ):
+            models.List.objects.create(name="Public list", user=self.local_user)
+            models.List.objects.create(
+                name="Private list", privacy="direct", user=self.local_user
+            )
         request = self.factory.get("")
         request.user = self.anonymous_user
 
         result = view(request, self.local_user.username)
-        self.assertEqual(result.status_code, 302)
+        self.assertIsInstance(result, TemplateResponse)
+        validate_html(result.render())
+        self.assertEqual(result.status_code, 200)
 
     def test_lists_create(self):
         """create list view"""

bookwyrm/tests/views/preferences/test_export.py

@@ -68,7 +68,7 @@ class ExportViews(TestCase):
         # pylint: disable=line-too-long
         self.assertEqual(
             export.content,
-            b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date\r\n"
-            + b"Test Book,,%b,,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,,,to-read,To Read,%b\r\n"
+            b"title,author_text,remote_id,openlibrary_key,finna_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date\r\n"
+            + b"Test Book,,%b,,,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,,,to-read,To Read,%b\r\n"
             % (self.book.remote_id.encode("utf-8"), book_date),
         )

bookwyrm/views/author.py

@@ -7,6 +7,7 @@ from django.template.response import TemplateResponse
 from django.utils.decorators import method_decorator
 from django.views import View
 from django.views.decorators.http import require_POST
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm import forms, models
 from bookwyrm.activitypub import ActivitypubResponse
@@ -24,6 +25,7 @@ class Author(View):
     """this person wrote a book"""
 
     # pylint: disable=unused-argument
+    @vary_on_headers("Accept")
     def get(self, request, author_id, slug=None):
         """landing page for an author"""
         author = get_mergeable_object_or_404(models.Author, id=author_id)

bookwyrm/views/books/books.py

@@ -10,6 +10,7 @@ from django.shortcuts import get_object_or_404, redirect
 from django.template.response import TemplateResponse
 from django.views import View
 from django.views.decorators.http import require_POST
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm import forms, models
 from bookwyrm.activitypub import ActivitypubResponse
@@ -27,6 +28,7 @@ from bookwyrm.views.helpers import (
 class Book(View):
     """a book! this is the stuff"""
 
+    @vary_on_headers("Accept")
     def get(self, request, book_id, **kwargs):
         """info about a book"""
         if is_api_request(request):

bookwyrm/views/books/editions.py

@@ -12,6 +12,7 @@ from django.shortcuts import redirect
 from django.template.response import TemplateResponse
 from django.views import View
 from django.views.decorators.http import require_POST
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm import forms, models
 from bookwyrm.activitypub import ActivitypubResponse
@@ -23,6 +24,7 @@ from bookwyrm.views.helpers import is_api_request, get_mergeable_object_or_404
 class Editions(View):
     """list of editions"""
 
+    @vary_on_headers("Accept")
     def get(self, request, book_id):
         """list of editions of a book"""
         work = get_mergeable_object_or_404(models.Work, id=book_id)

bookwyrm/views/books/series.py

@@ -3,6 +3,7 @@
 from sys import float_info
 from django.views import View
 from django.template.response import TemplateResponse
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm.views.helpers import is_api_request, get_mergeable_object_or_404
 from bookwyrm import models
@@ -20,6 +21,7 @@ def sort_by_series(book):
 class BookSeriesBy(View):
     """book series by author"""
 
+    @vary_on_headers("Accept")
     def get(self, request, author_id):
         """lists all books in a series"""
         series_name = request.GET.get("series_name")

bookwyrm/views/feed.py

@@ -9,6 +9,7 @@ from django.template.response import TemplateResponse
 from django.utils import timezone
 from django.utils.decorators import method_decorator
 from django.views import View
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm import activitystreams, forms, models
 from bookwyrm.models.user import FeedFilterChoices
@@ -130,6 +131,7 @@ class Status(View):
     """get posting"""
 
     # pylint: disable=unused-argument
+    @vary_on_headers("Accept")
     def get(self, request, username, status_id, slug=None):
         """display a particular status (and replies, etc)"""
         user = get_user_from_username(request.user, username)
@@ -217,6 +219,7 @@ class Status(View):
 class Replies(View):
     """replies page (a json view of status)"""
 
+    @vary_on_headers("Accept")
     def get(self, request, username, status_id):
         """ordered collection of replies to a status"""
         # the html view is the same as Status

bookwyrm/views/isbn.py

@@ -3,6 +3,7 @@ from django.core.paginator import Paginator
 from django.http import JsonResponse
 from django.template.response import TemplateResponse
 from django.views import View
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm import book_search
 from bookwyrm.settings import PAGE_LENGTH
@@ -12,6 +13,7 @@ from .helpers import is_api_request
 class Isbn(View):
     """search a book by isbn"""
 
+    @vary_on_headers("Accept")
     def get(self, request, isbn):
         """info about a book"""
         book_results = book_search.isbn_search(isbn)

bookwyrm/views/list/list.py

@@ -14,6 +14,7 @@ from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.views import View
 from django.views.decorators.http import require_POST
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm import book_search, forms, models
 from bookwyrm.activitypub import ActivitypubResponse
@@ -29,6 +30,7 @@ from bookwyrm.views.helpers import (
 class List(View):
     """book list page"""
 
+    @vary_on_headers("Accept")
     def get(self, request, list_id, **kwargs):
         """display a book list"""
         add_failed = kwargs.get("add_failed", False)

bookwyrm/views/list/lists.py

@@ -10,6 +10,9 @@ from bookwyrm import forms, models
 from bookwyrm.lists_stream import ListsStream
 from bookwyrm.views.helpers import get_user_from_username
 
+import logging
+
+logger = logging.getLogger(__name__)
 
 # pylint: disable=no-self-use
 class Lists(View):
@@ -64,12 +67,12 @@ class SavedLists(View):
         return TemplateResponse(request, "lists/lists.html", data)
 
 
-@method_decorator(login_required, name="dispatch")
 class UserLists(View):
     """a user's book list page"""
 
     def get(self, request, username):
         """display a book list"""
+
         user = get_user_from_username(request.user, username)
         lists = models.List.privacy_filter(request.user).filter(user=user)
         paginated = Paginator(lists, 12)

bookwyrm/views/relationships.py

@@ -4,6 +4,7 @@ from django.core.paginator import Paginator
 from django.db.models import Q, Count
 from django.template.response import TemplateResponse
 from django.views import View
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm.activitypub import ActivitypubResponse
 from bookwyrm.settings import PAGE_LENGTH
@@ -14,6 +15,7 @@ from .helpers import get_user_from_username, is_api_request
 class Relationships(View):
     """list of followers/following view"""
 
+    @vary_on_headers("Accept")
     def get(self, request, username, direction):
         """list of followers"""
         user = get_user_from_username(request.user, username)

bookwyrm/views/search.py

@@ -9,6 +9,7 @@ from django.db.models.functions import Greatest
 from django.http import JsonResponse
 from django.template.response import TemplateResponse
 from django.views import View
+from django.views.decorators.vary import vary_on_headers
 
 from csp.decorators import csp_update
 
@@ -26,6 +27,7 @@ class Search(View):
     """search users or books"""
 
     @csp_update(IMG_SRC="*")
+    @vary_on_headers("Accept")
     def get(self, request):
         """that search bar up top"""
         if is_api_request(request):

bookwyrm/views/shelf/shelf.py

@@ -10,6 +10,7 @@ from django.template.response import TemplateResponse
 from django.utils.decorators import method_decorator
 from django.utils.translation import gettext_lazy as _
 from django.views import View
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm import forms, models
 from bookwyrm.activitypub import ActivitypubResponse
@@ -23,6 +24,7 @@ class Shelf(View):
     """shelf page"""
 
     # pylint: disable=R0914
+    @vary_on_headers("Accept")
     def get(self, request, username, shelf_identifier=None):
         """display a shelf"""
         user = get_user_from_username(request.user, username)

bookwyrm/views/user.py

@@ -8,6 +8,7 @@ from django.template.response import TemplateResponse
 from django.utils import timezone
 from django.views import View
 from django.views.decorators.http import require_POST
+from django.views.decorators.vary import vary_on_headers
 
 from bookwyrm import models
 from bookwyrm.activitypub import ActivitypubResponse
@@ -19,6 +20,7 @@ from .helpers import get_user_from_username, is_api_request
 class User(View):
     """user profile page"""
 
+    @vary_on_headers("Accept")
     def get(self, request, username):
         """profile page for a user"""
         user = get_user_from_username(request.user, username)

requirements.txt

@@ -1,10 +1,10 @@
-aiohttp==3.10.2
+aiohttp==3.10.11
 bleach==6.1.0
 boto3==1.34.74
 bw-file-resubmit==0.6.0rc2
 celery==5.3.6
 colorthief==0.2.1
-Django==4.2.16
+Django==4.2.20
 django-celery-beat==2.6.0
 django-compressor==4.4
 django-csp==3.8