From 8bcd3da25e73b4f874f9c85281f1b168542aa548 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 23 Mar 2020 16:40:09 +0000 Subject: [PATCH 1/8] Add a form to upload goodreads data. --- fedireads/forms.py | 4 ++++ fedireads/templates/import.html | 10 ++++++++++ fedireads/templates/layout.html | 1 + fedireads/urls.py | 2 ++ fedireads/views.py | 8 ++++++++ 5 files changed, 25 insertions(+) create mode 100644 fedireads/templates/import.html diff --git a/fedireads/forms.py b/fedireads/forms.py index e812fc0e6..bddef528f 100644 --- a/fedireads/forms.py +++ b/fedireads/forms.py @@ -1,6 +1,7 @@ ''' usin django model forms ''' from django.core.validators import MaxValueValidator, MinValueValidator from django.forms import ModelForm, PasswordInput, IntegerField +from django import forms from fedireads import models @@ -73,3 +74,6 @@ class TagForm(ModelForm): help_texts = {f: None for f in fields} labels = {'name': 'Add a tag'} + +class ImportForm(forms.Form): + csv_file = forms.FileField() diff --git a/fedireads/templates/import.html b/fedireads/templates/import.html new file mode 100644 index 000000000..694fe9914 --- /dev/null +++ b/fedireads/templates/import.html @@ -0,0 +1,10 @@ +{% extends 'layout.html' %} +{% block content %} +
+
+ {% csrf_token %} + {{ import_form.as_p }} + +
+
+{% endblock %} diff --git a/fedireads/templates/layout.html b/fedireads/templates/layout.html index e76cc2d46..8537a4e7a 100644 --- a/fedireads/templates/layout.html +++ b/fedireads/templates/layout.html @@ -29,6 +29,7 @@
  • Your shelves
  • Updates
  • Discover Books
    • +
  • Import Books
  • diff --git a/fedireads/urls.py b/fedireads/urls.py index eb0aa3d16..677b86823 100644 --- a/fedireads/urls.py +++ b/fedireads/urls.py @@ -34,6 +34,7 @@ urlpatterns = [ re_path(r'^(?Phome|local|federated)/?$', views.home_tab), re_path(r'^notifications/?', views.notifications_page), re_path(r'books/?$', views.books_page), + re_path(r'import/?$', views.import_page), # should return a ui view or activitypub json blob as requested # users @@ -81,5 +82,6 @@ urlpatterns = [ re_path(r'^accept_follow_request/?$', actions.accept_follow_request), re_path(r'^delete_follow_request/?$', actions.delete_follow_request), + re_path(r'import_data', actions.import_data), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/fedireads/views.py b/fedireads/views.py index ec80b726f..83bf3df7f 100644 --- a/fedireads/views.py +++ b/fedireads/views.py @@ -108,6 +108,14 @@ def books_page(request): } return TemplateResponse(request, 'books.html', data) +@login_required +def import_page(request): + ''' import history from goodreads ''' + return TemplateResponse(request, 'import.html', { + 'import_form': forms.ImportForm(), + }) + + def login_page(request): ''' authentication ''' From 03a71b1c37a597133854ae8e5648a1650c39bc35 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 23 Mar 2020 16:43:11 +0000 Subject: [PATCH 2/8] Process upload and create books. --- fedireads/view_actions.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/fedireads/view_actions.py b/fedireads/view_actions.py index dd73ca28c..ab63f9238 100644 --- a/fedireads/view_actions.py +++ b/fedireads/view_actions.py @@ -5,6 +5,7 @@ from django.http import HttpResponseBadRequest from django.shortcuts import redirect from django.template.response import TemplateResponse import re +import csv from fedireads import forms, models, books_manager, outgoing from fedireads.settings import DOMAIN @@ -289,3 +290,27 @@ def delete_follow_request(request): outgoing.handle_outgoing_reject(requester, request.user, follow_request) return redirect('/user/%s' % request.user.localname) +def unquote_string(text): + match = re.match(r'="([^"]*)"', text) + if match: + return match.group(1) + else: + return text + +import itertools +from io import TextIOWrapper + +@login_required +def import_data(request): + form = forms.ImportForm(request.POST, request.FILES) + if form.is_valid(): + reader = csv.DictReader(TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding)) + for line in itertools.islice(reader, 10): + isbn = unquote_string(line['ISBN13']) + print(line['Title'], isbn, line['Exclusive Shelf']) + search_results = books_manager.search(isbn) + if search_results: + book = books_manager.get_or_create_book(search_results[0].key) + return HttpResponse('thanks') + else: + return HttpResponseBadRequest() From 8fb082e2fc49f535c7cb8cc22fefe14b08d4e9a8 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 24 Mar 2020 16:56:53 +0000 Subject: [PATCH 3/8] Add useful reprs to books & search results for debugging. --- fedireads/connectors/abstract_connector.py | 6 +++++- fedireads/models/book.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/fedireads/connectors/abstract_connector.py b/fedireads/connectors/abstract_connector.py index dfb6b99de..045e655ee 100644 --- a/fedireads/connectors/abstract_connector.py +++ b/fedireads/connectors/abstract_connector.py @@ -53,8 +53,12 @@ class AbstractConnector(ABC): class SearchResult(object): ''' standardized search result object ''' - def __init__(self, title, key, author, year): + def __init__(self, title, key, author, year, raw_data): self.title = title self.key = key self.author = author self.year = year + self.raw_data = raw_data + + def __repr__(self): + return "".format(self.key, self.title, self.author) diff --git a/fedireads/models/book.py b/fedireads/models/book.py index 04451d0db..6cdedfe2e 100644 --- a/fedireads/models/book.py +++ b/fedireads/models/book.py @@ -54,6 +54,9 @@ class Book(FedireadsModel): model_name = type(self).__name__.lower() return '%s/%s/%s' % (base_path, model_name, self.openlibrary_key) + def __repr__(self): + return "<{} key={!r} title={!r} author={!r}>".format(self.__class__, self.openlibrary_key, self.title, self.author) + class Work(Book): ''' a work (an abstract concept of a book that manifests in an edition) ''' From 2188371f449be287b13c789659910fb1cc9eb27c Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 24 Mar 2020 16:58:11 +0000 Subject: [PATCH 4/8] Display results of import. --- fedireads/templates/import_results.html | 28 +++++++++++++++++++++ fedireads/view_actions.py | 33 +++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 fedireads/templates/import_results.html diff --git a/fedireads/templates/import_results.html b/fedireads/templates/import_results.html new file mode 100644 index 000000000..73996cfa8 --- /dev/null +++ b/fedireads/templates/import_results.html @@ -0,0 +1,28 @@ +{% extends 'layout.html' %} +{% block content %} +
    +
    +

    Search results

    + {% for book in results %} +
    + + {% include 'snippets/book_cover.html' with book=book %} + + {{ book.title }} + {{ book.author }} + {% include 'snippets/shelve_button.html' with book=book pulldown=True %} +
    + {% endfor %} + +

    Failures

    +
      + {% for book in failures %} +
    • + {{ book.Title }} + {{ book.Author }} +
      • + {% endfor %} +
    +
    +
    +{% endblock %} diff --git a/fedireads/view_actions.py b/fedireads/view_actions.py index ab63f9238..a3e2c2e59 100644 --- a/fedireads/view_actions.py +++ b/fedireads/view_actions.py @@ -297,20 +297,49 @@ def unquote_string(text): else: return text +def construct_search_term(title, author): + # Strip brackets (usually series title from search term) + title = re.sub(r'\s*\([^)]*\)\s*', '', title) + # Open library doesn't like including author initials in search term. + author = re.sub(r'(\w\.)+\s*', '', author) + + return ' '.join([title, author]) + import itertools from io import TextIOWrapper +from requests import HTTPError @login_required def import_data(request): form = forms.ImportForm(request.POST, request.FILES) if form.is_valid(): reader = csv.DictReader(TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding)) - for line in itertools.islice(reader, 10): + results = [] + failures = [] + for line in itertools.islice(reader, 20): isbn = unquote_string(line['ISBN13']) print(line['Title'], isbn, line['Exclusive Shelf']) search_results = books_manager.search(isbn) if search_results: book = books_manager.get_or_create_book(search_results[0].key) - return HttpResponse('thanks') + print(book) + results.append(book) + else: + try: + search_term = construct_search_term(line['Title'], line['Author']) + print("Search term: ", search_term) + search_results = books_manager.search(search_term) + if search_results: + book = books_manager.get_or_create_book(search_results[0].key) + print(book) + results.append(book) + else: + failures.append(line) + except HTTPError: + failures.append(line) # + return TemplateResponse(request, 'import_results.html', { + 'results': results, + 'failures': failures + }) else: return HttpResponseBadRequest() From ce446d57fc7f2d6014b8545cb4c22699d3daafae Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 25 Mar 2020 12:29:21 +0000 Subject: [PATCH 5/8] Handle uploaded CSV and match to openlibrary titles. --- fedireads/connectors/openlibrary.py | 1 + fedireads/goodreads_import.py | 59 +++++++++++++++++++++++++ fedireads/templates/import_results.html | 3 +- fedireads/view_actions.py | 49 +++----------------- 4 files changed, 68 insertions(+), 44 deletions(-) create mode 100644 fedireads/goodreads_import.py diff --git a/fedireads/connectors/openlibrary.py b/fedireads/connectors/openlibrary.py index 2c3b458f7..1beedb08b 100644 --- a/fedireads/connectors/openlibrary.py +++ b/fedireads/connectors/openlibrary.py @@ -31,6 +31,7 @@ class OpenLibraryConnector(AbstractConnector): key, author[0], doc.get('first_publish_year'), + doc )) return results diff --git a/fedireads/goodreads_import.py b/fedireads/goodreads_import.py new file mode 100644 index 000000000..1ba17faf2 --- /dev/null +++ b/fedireads/goodreads_import.py @@ -0,0 +1,59 @@ +import re +import csv +import itertools +from requests import HTTPError + +from fedireads import books_manager + +def unquote_string(text): + match = re.match(r'="([^"]*)"', text) + if match: + return match.group(1) + else: + return text + +def construct_search_term(title, author): + # Strip brackets (usually series title from search term) + title = re.sub(r'\s*\([^)]*\)\s*', '', title) + # Open library doesn't like including author initials in search term. + author = re.sub(r'(\w\.)+\s*', '', author) + + return ' '.join([title, author]) + +class GoodreadsCsv(object): + def __init__(self, csv_file): + self.reader = csv.DictReader(csv_file) + + def __iter__(self): + for line in itertools.islice(self.reader, 20, 30): + entry = GoodreadsItem(line) + try: + entry.resolve() + except HTTPError: + pass + yield entry + +class GoodreadsItem(object): + def __init__(self, line): + self.line = line + self.book = None + + def resolve(self): + self.book = self.get_book_from_isbn() + if not self.book: + self.book = self.get_book_from_title_author() + + def get_book_from_isbn(self): + isbn = unquote_string(self.line['ISBN13']) + search_results = books_manager.search(isbn) + if search_results: + return books_manager.get_or_create_book(search_results[0].key) + + def get_book_from_title_author(self): + search_term = construct_search_term(self.line['Title'], self.line['Author']) + search_results = books_manager.search(search_term) + if search_results: + return books_manager.get_or_create_book(search_results[0].key) + + def __repr__(self): + return "".format(self.line['Title']) diff --git a/fedireads/templates/import_results.html b/fedireads/templates/import_results.html index 73996cfa8..3082878f6 100644 --- a/fedireads/templates/import_results.html +++ b/fedireads/templates/import_results.html @@ -18,8 +18,7 @@
      {% for book in failures %}
    • - {{ book.Title }} - {{ book.Author }} + {{ book }}
      • {% endfor %}
    diff --git a/fedireads/view_actions.py b/fedireads/view_actions.py index a3e2c2e59..dae69559a 100644 --- a/fedireads/view_actions.py +++ b/fedireads/view_actions.py @@ -1,15 +1,16 @@ ''' views for actions you can take in the application ''' +from io import TextIOWrapper + from django.contrib.auth import authenticate, login, logout from django.contrib.auth.decorators import login_required from django.http import HttpResponseBadRequest from django.shortcuts import redirect from django.template.response import TemplateResponse -import re -import csv from fedireads import forms, models, books_manager, outgoing from fedireads.settings import DOMAIN from fedireads.views import get_user_from_username +from fedireads.goodreads_import import GoodreadsCsv def user_login(request): @@ -289,54 +290,18 @@ def delete_follow_request(request): outgoing.handle_outgoing_reject(requester, request.user, follow_request) return redirect('/user/%s' % request.user.localname) - -def unquote_string(text): - match = re.match(r'="([^"]*)"', text) - if match: - return match.group(1) - else: - return text - -def construct_search_term(title, author): - # Strip brackets (usually series title from search term) - title = re.sub(r'\s*\([^)]*\)\s*', '', title) - # Open library doesn't like including author initials in search term. - author = re.sub(r'(\w\.)+\s*', '', author) - - return ' '.join([title, author]) -import itertools -from io import TextIOWrapper -from requests import HTTPError - @login_required def import_data(request): form = forms.ImportForm(request.POST, request.FILES) if form.is_valid(): - reader = csv.DictReader(TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding)) results = [] failures = [] - for line in itertools.islice(reader, 20): - isbn = unquote_string(line['ISBN13']) - print(line['Title'], isbn, line['Exclusive Shelf']) - search_results = books_manager.search(isbn) - if search_results: - book = books_manager.get_or_create_book(search_results[0].key) - print(book) - results.append(book) + for item in GoodreadsCsv(TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding)): + if item.book: + results.append(item.book) else: - try: - search_term = construct_search_term(line['Title'], line['Author']) - print("Search term: ", search_term) - search_results = books_manager.search(search_term) - if search_results: - book = books_manager.get_or_create_book(search_results[0].key) - print(book) - results.append(book) - else: - failures.append(line) - except HTTPError: - failures.append(line) # + failures.append(item) return TemplateResponse(request, 'import_results.html', { 'results': results, 'failures': failures From 323c7f8dbf431e3d05bc0b46f77cad3dbbbb5b5f Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 25 Mar 2020 12:58:27 +0000 Subject: [PATCH 6/8] Shelve the books. --- fedireads/goodreads_import.py | 14 +++++++++++++- fedireads/view_actions.py | 6 ++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/fedireads/goodreads_import.py b/fedireads/goodreads_import.py index 1ba17faf2..3dd29ed12 100644 --- a/fedireads/goodreads_import.py +++ b/fedireads/goodreads_import.py @@ -5,6 +5,13 @@ from requests import HTTPError from fedireads import books_manager +# Mapping goodreads -> fedireads shelf titles. +GOODREADS_SHELVES = { + 'read': 'read', + 'currently-reading': 'reading', + 'to-read': 'to-read', +} + def unquote_string(text): match = re.match(r'="([^"]*)"', text) if match: @@ -25,7 +32,7 @@ class GoodreadsCsv(object): self.reader = csv.DictReader(csv_file) def __iter__(self): - for line in itertools.islice(self.reader, 20, 30): + for line in itertools.islice(self.reader, 30): entry = GoodreadsItem(line) try: entry.resolve() @@ -55,5 +62,10 @@ class GoodreadsItem(object): if search_results: return books_manager.get_or_create_book(search_results[0].key) + @property + def shelf(self): + if self.line['Exclusive Shelf']: + return GOODREADS_SHELVES[self.line['Exclusive Shelf']] + def __repr__(self): return "".format(self.line['Title']) diff --git a/fedireads/view_actions.py b/fedireads/view_actions.py index dae69559a..8eb15ceca 100644 --- a/fedireads/view_actions.py +++ b/fedireads/view_actions.py @@ -300,6 +300,12 @@ def import_data(request): for item in GoodreadsCsv(TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding)): if item.book: results.append(item.book) + if item.shelf: + desired_shelf = models.Shelf.objects.get( + identifier=item.shelf, + user=request.user + ) + outgoing.handle_shelve(request.user, item.book, desired_shelf) else: failures.append(item) return TemplateResponse(request, 'import_results.html', { From 8bf3225fc43d420619155832e37d1433699477df Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 25 Mar 2020 14:14:38 +0000 Subject: [PATCH 7/8] Generated import status rather than individual statuses. --- fedireads/goodreads_import.py | 6 +++++- fedireads/outgoing.py | 25 +++++++++++++++++++++++++ fedireads/templates/import_results.html | 19 +++++-------------- fedireads/view_actions.py | 21 ++++++++++----------- 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/fedireads/goodreads_import.py b/fedireads/goodreads_import.py index 3dd29ed12..5f5f8a7e9 100644 --- a/fedireads/goodreads_import.py +++ b/fedireads/goodreads_import.py @@ -11,6 +11,7 @@ GOODREADS_SHELVES = { 'currently-reading': 'reading', 'to-read': 'to-read', } +MAX_ENTRIES = 20 def unquote_string(text): match = re.match(r'="([^"]*)"', text) @@ -32,7 +33,7 @@ class GoodreadsCsv(object): self.reader = csv.DictReader(csv_file) def __iter__(self): - for line in itertools.islice(self.reader, 30): + for line in itertools.islice(self.reader, MAX_ENTRIES): entry = GoodreadsItem(line) try: entry.resolve() @@ -69,3 +70,6 @@ class GoodreadsItem(object): def __repr__(self): return "".format(self.line['Title']) + + def __str__(self): + return "{} by {}".format(self.line['Title'], self.line['Author']) diff --git a/fedireads/outgoing.py b/fedireads/outgoing.py index b47f0fcde..a58ee5d6b 100644 --- a/fedireads/outgoing.py +++ b/fedireads/outgoing.py @@ -157,6 +157,31 @@ def handle_unshelve(user, book, shelf): broadcast(user, activity, recipients) +def handle_import_books(user, items): + new_books = [] + for item in items: + if item.shelf: + desired_shelf = models.Shelf.objects.get( + identifier=item.shelf, + user=user + ) + shelf, created = models.ShelfBook.objects.get_or_create(book=item.book, shelf=desired_shelf, added_by=user) + if created: + new_books.append(item.book) + activity = activitypub.get_add(user, item.book, desired_shelf) + recipients = get_recipients(user, 'public') + broadcast(user, activity, recipients) + + if new_books: + message = 'imported {} books'.format(len(new_books)) + status = create_status(user, message, mention_books=new_books) + status.status_type = 'Update' + status.save() + + create_activity = activitypub.get_create(user, activitypub.get_status(status)) + broadcast(user, create_activity, get_recipients(user, 'public')) + + def handle_review(user, book, name, content, rating): ''' post a review ''' # validated and saves the review in the database so it has an id diff --git a/fedireads/templates/import_results.html b/fedireads/templates/import_results.html index 3082878f6..2994b6ff2 100644 --- a/fedireads/templates/import_results.html +++ b/fedireads/templates/import_results.html @@ -2,26 +2,17 @@ {% block content %}
    -

    Search results

    - {% for book in results %} -
    - - {% include 'snippets/book_cover.html' with book=book %} - - {{ book.title }} - {{ book.author }} - {% include 'snippets/shelve_button.html' with book=book pulldown=True %} -
    - {% endfor %} +

    The following books could not be imported:

    -

    Failures

      - {% for book in failures %} + {% for item in failures %}
    • - {{ book }} + {{ item }}
      • {% endfor %}
    + +

    {{ success_count }} books imported successfully

    {% endblock %} diff --git a/fedireads/view_actions.py b/fedireads/view_actions.py index 8eb15ceca..c3b272bd1 100644 --- a/fedireads/view_actions.py +++ b/fedireads/view_actions.py @@ -299,18 +299,17 @@ def import_data(request): failures = [] for item in GoodreadsCsv(TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding)): if item.book: - results.append(item.book) - if item.shelf: - desired_shelf = models.Shelf.objects.get( - identifier=item.shelf, - user=request.user - ) - outgoing.handle_shelve(request.user, item.book, desired_shelf) + results.append(item) else: failures.append(item) - return TemplateResponse(request, 'import_results.html', { - 'results': results, - 'failures': failures - }) + + outgoing.handle_import_books(request.user, results) + if failures: + return TemplateResponse(request, 'import_results.html', { + 'success_count': len(results), + 'failures': failures, + }) + else: + return redirect('/') else: return HttpResponseBadRequest() From 7d837ad344ab1ba2b7d0e0b1fd6ab2a4538b9be9 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 25 Mar 2020 14:33:29 +0000 Subject: [PATCH 8/8] Show up to three mentioned books with a status. --- fedireads/templates/snippets/status.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fedireads/templates/snippets/status.html b/fedireads/templates/snippets/status.html index 82284fad1..af4b73fa7 100644 --- a/fedireads/templates/snippets/status.html +++ b/fedireads/templates/snippets/status.html @@ -21,13 +21,15 @@ {% if not hide_book and status.mention_books.count %} + {% for book in status.mention_books.all|slice:"0:3" %}
    {% if status.status_type == 'Review' %} - {% include 'snippets/book.html' with book=status.mention_books.first %} + {% include 'snippets/book.html' with book=book %} {% else %} - {% include 'snippets/book.html' with book=status.mention_books.first description=True %} + {% include 'snippets/book.html' with book=book description=True %} {% endif %}
    + {% endfor %} {% endif %} {% if not hide_book and status.book%}