From 82c2f2eeb1b0d8a9fa1133c2bd1c626214dfa1d0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 6 Apr 2021 17:46:06 -0700 Subject: [PATCH] Adds more author identifier fields --- bookwyrm/connectors/abstract_connector.py | 8 +-- bookwyrm/connectors/inventaire.py | 47 ++++++++++++------ bookwyrm/connectors/openlibrary.py | 10 ++-- .../migrations/0063_auto_20210406_1738.py | 28 ----------- .../migrations/0063_auto_20210407_0045.py | 49 +++++++++++++++++++ bookwyrm/models/author.py | 9 ++++ bookwyrm/models/book.py | 3 ++ 7 files changed, 103 insertions(+), 51 deletions(-) delete mode 100644 bookwyrm/migrations/0063_auto_20210406_1738.py create mode 100644 bookwyrm/migrations/0063_auto_20210407_0045.py diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 8007bb1a..f66cca7f 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -150,11 +150,11 @@ class AbstractConnector(AbstractMinimalConnector): load_more_data.delay(self.connector.id, work.id) return edition - def get_book_data(self, remote_id): + def get_book_data(self, remote_id): # pylint: disable=no-self-use """ this allows connectors to override the default behavior """ return get_data(remote_id) - def get_search_data(self, remote_id): + def get_search_data(self, remote_id): # pylint: disable=no-self-use """ this allows connectors to override the default behavior """ return get_data(remote_id) @@ -293,7 +293,7 @@ class Mapping: """ associate a local database field with a field in an external dataset """ def __init__(self, local_field, remote_field=None, formatter=None): - noop = lambda x, *_: x + noop = lambda x: x self.local_field = local_field self.remote_field = remote_field or local_field @@ -305,6 +305,6 @@ class Mapping: if not value: return None try: - return self.formatter(value, self.remote_field) + return self.formatter(value) except: # pylint: disable=bare-except return None diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py index 1ff834bf..6eff6508 100644 --- a/bookwyrm/connectors/inventaire.py +++ b/bookwyrm/connectors/inventaire.py @@ -10,32 +10,45 @@ class Connector(AbstractConnector): def __init__(self, identifier): super().__init__(identifier) - get_first = lambda a, *args: a[0] - get_remote_id_list = lambda a, *_: [self.get_remote_id(v) for v in a] + get_first = lambda a: a[0] + shared_mappings = [ + Mapping("id", remote_field="uri", formatter=self.get_remote_id), + Mapping("bnfId", remote_field="wdt:P268", formatter=get_first), + Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_first), + Mapping("openlibraryKey", remote_field="wdt:P648", formatter=get_first), + ] self.book_mappings = [ Mapping("title", remote_field="wdt:P1476", formatter=get_first), Mapping("subtitle", remote_field="wdt:P1680", formatter=get_first), - Mapping("id", remote_field="uri", formatter=self.get_remote_id), - # Mapping("authors", remote_field="wdt:P50", formatter=get_remote_id_list), Mapping("inventaireId", remote_field="uri"), Mapping("cover", remote_field="image", formatter=self.get_cover_url), Mapping("isbn13", remote_field="wdt:P212", formatter=get_first), Mapping("isbn10", remote_field="wdt:P957", formatter=get_first), + Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_first), + Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_first), # Mapping("languages", remote_field="wdt:P407", formatter=get_language), # Mapping("publishers", remote_field="wdt:P123", formatter=resolve_key), Mapping("publishedDate", remote_field="wdt:P577", formatter=get_first), Mapping("pages", remote_field="wdt:P1104", formatter=get_first), - Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_first), - Mapping("openlibraryKey", remote_field="wdt:P648", formatter=get_first), # Mapping("subjectPlaces", remote_field="wdt:P840", formatter=resolve_key), # Mapping("subjects", remote_field="wdt:P921", formatter=resolve_key), - Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_first), - Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_first), Mapping("asin", remote_field="wdt:P5749", formatter=get_first), - ] - # TODO: P136: genre, P268 bnf id, P674 characters, P950 bne + ] + shared_mappings + # TODO: P136: genre, P674 characters, P950 bne - def get_remote_id(self, value, *_): + self.author_mappings = [ + Mapping("id", remote_field="uri", formatter=self.get_remote_id), + Mapping("name", remote_field="label", formatter=get_language_code), + Mapping("goodreadsKey", remote_field="wdt:P2963", formatter=get_first), + Mapping("isni", remote_field="wdt:P213", formatter=get_first), + Mapping("viafId", remote_field="wdt:P214", formatter=get_first), + Mapping("gutenberg_id", remote_field="wdt:P1938", formatter=get_first), + Mapping("born", remote_field="wdt:P569", formatter=get_first), + Mapping("died", remote_field="wdt:P570", formatter=get_first), + ] + shared_mappings + + + def get_remote_id(self, value): """ convert an id/uri into a url """ return "{:s}?action=by-uris&uris={:s}".format(self.books_url, value) @@ -102,7 +115,9 @@ class Connector(AbstractConnector): return self.get_book_data(self.get_remote_id(uri)) def get_authors_from_data(self, data): - return [] + authors = data.get("wdt:P50") + for author in authors: + yield self.get_or_create_author(self.get_remote_id(author)) def expand_book_data(self, book): return @@ -117,11 +132,15 @@ class Connector(AbstractConnector): return "%s%s" % (self.covers_url, cover_id) -def get_language(wikidata_key, *_): +def get_language(wikidata_key): """ who here speaks "wd:Q150" """ return wikidata_key # TODO -def resolve_key(wikidata_key, *_): +def resolve_key(wikidata_key): """ cool, it's "wd:Q3156592" now what the heck does that mean """ return wikidata_key # TODO + +def get_language_code(options, code="en"): + """ when there are a bunch of translation but we need a single field """ + return options.get(code) diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 34357726..466bf1e5 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -95,7 +95,7 @@ class Connector(AbstractConnector): url = "%s%s" % (self.base_url, author_id) yield self.get_or_create_author(url) - def get_cover_url(self, cover_blob, *_, size="L"): + def get_cover_url(self, cover_blob, size="L"): """ ask openlibrary for the cover """ if not cover_blob: return None @@ -181,19 +181,19 @@ def ignore_edition(edition_data): return True -def get_description(description_blob, *_): +def get_description(description_blob): """ descriptions can be a string or a dict """ if isinstance(description_blob, dict): return description_blob.get("value") return description_blob -def get_openlibrary_key(key, *_): +def get_openlibrary_key(key): """ convert /books/OL27320736M into OL27320736M """ return key.split("/")[-1] -def get_languages(language_blob, *_): +def get_languages(language_blob): """ /language/eng -> English """ langs = [] for lang in language_blob: @@ -201,7 +201,7 @@ def get_languages(language_blob, *_): return langs -def pick_default_edition(options, *_): +def pick_default_edition(options): """ favor physical copies with covers in english """ if not options: return None diff --git a/bookwyrm/migrations/0063_auto_20210406_1738.py b/bookwyrm/migrations/0063_auto_20210406_1738.py deleted file mode 100644 index c9f07b3c..00000000 --- a/bookwyrm/migrations/0063_auto_20210406_1738.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 3.1.6 on 2021-04-06 17:38 - -import bookwyrm.models.fields -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("bookwyrm", "0062_auto_20210406_1731"), - ] - - operations = [ - migrations.AddField( - model_name="author", - name="inventaire_id", - field=bookwyrm.models.fields.CharField( - blank=True, max_length=255, null=True - ), - ), - migrations.AddField( - model_name="book", - name="inventaire_id", - field=bookwyrm.models.fields.CharField( - blank=True, max_length=255, null=True - ), - ), - ] diff --git a/bookwyrm/migrations/0063_auto_20210407_0045.py b/bookwyrm/migrations/0063_auto_20210407_0045.py new file mode 100644 index 00000000..2543193b --- /dev/null +++ b/bookwyrm/migrations/0063_auto_20210407_0045.py @@ -0,0 +1,49 @@ +# Generated by Django 3.1.6 on 2021-04-07 00:45 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0062_auto_20210406_1731'), + ] + + operations = [ + migrations.AddField( + model_name='author', + name='bnf_id', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='author', + name='gutenberg_id', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='author', + name='inventaire_id', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='author', + name='isni', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='author', + name='viaf_id', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='bnf_id', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='inventaire_id', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 4c5fe6c8..f7740b1d 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -14,6 +14,15 @@ class Author(BookDataModel): wikipedia_link = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) + isni = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True + ) + viaf_id = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True + ) + gutenberg_id = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True + ) # idk probably other keys would be useful here? born = fields.DateTimeField(blank=True, null=True) died = fields.DateTimeField(blank=True, null=True) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index ccbec9eb..94bbe330 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -28,6 +28,9 @@ class BookDataModel(ObjectMixin, BookWyrmModel): goodreads_key = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) + bnf_id = fields.CharField( # Bibliothèque nationale de France + max_length=255, blank=True, null=True, deduplication_field=True + ) last_edited_by = models.ForeignKey("User", on_delete=models.PROTECT, null=True)