diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 0e04ffaf2..0dad48e9a 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -4,6 +4,8 @@ from urllib.parse import quote_plus import imghdr import logging import re +import aiohttp +import asyncio from django.core.files.base import ContentFile from django.db import transaction @@ -11,6 +13,7 @@ import requests from requests.exceptions import RequestException from bookwyrm import activitypub, models, settings +from bookwyrm.settings import SEARCH_TIMEOUT, USER_AGENT from .connector_manager import load_more_data, ConnectorException, raise_not_valid_url from .format_mappings import format_mappings @@ -57,6 +60,40 @@ class AbstractMinimalConnector(ABC): return list(self.parse_isbn_search_data(data))[:10] return list(self.parse_search_data(data, min_confidence))[:10] + async def get_results(self, session, url, min_confidence, query): + """try this specific connector""" + # pylint: disable=line-too-long + headers = { + "Accept": ( + 'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8' + ), + "User-Agent": USER_AGENT, + } + params = {"min_confidence": min_confidence} + try: + async with session.get(url, headers=headers, params=params) as response: + if not response.ok: + logger.info("Unable to connect to %s: %s", url, response.reason) + return + + try: + raw_data = await response.json() + except aiohttp.client_exceptions.ContentTypeError as err: + logger.exception(err) + return + + return { + "connector": self, + "results": self.process_search_response( + query, raw_data, min_confidence + ), + } + except asyncio.TimeoutError: + logger.info("Connection timed out for url: %s", url) + except aiohttp.ClientError as err: + logger.info(err) + + @abstractmethod def get_or_create_book(self, remote_id): """pull up a book record by whatever means possible""" diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 4330d4ac2..8e85c37ee 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -21,41 +21,6 @@ logger = logging.getLogger(__name__) class ConnectorException(HTTPError): """when the connector can't do what was asked""" - -async def get_results(session, url, min_confidence, query, connector): - """try this specific connector""" - # pylint: disable=line-too-long - headers = { - "Accept": ( - 'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8' - ), - "User-Agent": USER_AGENT, - } - params = {"min_confidence": min_confidence} - try: - async with session.get(url, headers=headers, params=params) as response: - if not response.ok: - logger.info("Unable to connect to %s: %s", url, response.reason) - return - - try: - raw_data = await response.json() - except aiohttp.client_exceptions.ContentTypeError as err: - logger.exception(err) - return - - return { - "connector": connector, - "results": connector.process_search_response( - query, raw_data, min_confidence - ), - } - except asyncio.TimeoutError: - logger.info("Connection timed out for url: %s", url) - except aiohttp.ClientError as err: - logger.info(err) - - async def async_connector_search(query, items, min_confidence): """Try a number of requests simultaneously""" timeout = aiohttp.ClientTimeout(total=SEARCH_TIMEOUT) @@ -64,7 +29,7 @@ async def async_connector_search(query, items, min_confidence): for url, connector in items: tasks.append( asyncio.ensure_future( - get_results(session, url, min_confidence, query, connector) + connector.get_results(session, url, min_confidence, query) ) )