mirrors/wallabag
1
0
Fork 0
mirror of https://github.com/wallabag/wallabag.git synced 2024-12-22 23:56:29 +00:00
Code Issues Projects Releases Wiki Activity

Make PocketImport use HttpClient

Browse source
This commit is contained in:
Yassine Guedidi 2024-02-08 23:23:11 +01:00
parent ca587fb6b6
commit 62fcfe547a
4 changed files with 110 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
app/config/config.yml
View file

@ -40,6 +40,11 @@ framework:
scoped_clients: scoped_clients:
download_images.client: download_images.client:
scope: '.*' scope: '.*'
pocket.client:
scope: '.*'
headers:
Content-Type: 'application/json'
X-Accept: 'application/json'
# Twig Configuration # Twig Configuration
twig: twig:
@ -426,14 +431,6 @@ httplug:
defaults: defaults:
timeout: 10 timeout: 10
plugins: ['httplug.plugin.logger'] plugins: ['httplug.plugin.logger']
wallabag_core.pocket.client:
factory: 'httplug.factory.auto'
plugins:
- 'httplug.plugin.logger'
- header_defaults:
headers:
'content-type': 'application/json'
'X-Accept': 'application/json'
discovery: discovery:
client: false client: false

5
app/config/services.yml
View file

@ -282,12 +282,9 @@ services:
tags: tags:
- { name: kernel.event_listener, event: security.authentication.failure, method: onAuthenticationFailure } - { name: kernel.event_listener, event: security.authentication.failure, method: onAuthenticationFailure }
wallabag_core.pocket.client:
alias: 'httplug.client.wallabag_core.pocket.client'
Wallabag\CoreBundle\Import\PocketImport: Wallabag\CoreBundle\Import\PocketImport:
calls: calls:
- [ setClient, [ "@wallabag_core.pocket.client" ] ] - [ setClient, [ '@Symfony\Contracts\HttpClient\HttpClientInterface $pocketClient' ] ]
tags: tags:
- { name: wallabag_core.import, alias: pocket } - { name: wallabag_core.import, alias: pocket }

129
src/Wallabag/CoreBundle/Import/PocketImport.php
View file

@ -2,22 +2,16 @@
namespace Wallabag\CoreBundle\Import; namespace Wallabag\CoreBundle\Import;
use Http\Client\Common\HttpMethodsClient; use Symfony\Component\HttpFoundation\Request;
use Http\Client\Common\Plugin\ErrorPlugin; use Symfony\Contracts\HttpClient\Exception\ExceptionInterface;
use Http\Client\Common\PluginClient; use Symfony\Contracts\HttpClient\HttpClientInterface;
use Http\Client\Exception\RequestException;
use Http\Discovery\Psr17FactoryDiscovery;
use Psr\Http\Client\ClientInterface;
use Psr\Http\Message\RequestFactoryInterface;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\StreamFactoryInterface;
use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Entry;
class PocketImport extends AbstractImport class PocketImport extends AbstractImport
{ {
public const NB_ELEMENTS = 5000; public const NB_ELEMENTS = 5000;
/** /**
* @var HttpMethodsClient * @var HttpClientInterface
*/ */
private $client; private $client;
private $accessToken; private $accessToken;
@ -57,17 +51,19 @@ class PocketImport extends AbstractImport
public function getRequestToken($redirectUri) public function getRequestToken($redirectUri)
{ {
try { try {
$response = $this->client->post('https://getpocket.com/v3/oauth/request', [], json_encode([ $response = $this->client->request(Request::METHOD_POST, 'https://getpocket.com/v3/oauth/request', [
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'json' => [
'redirect_uri' => $redirectUri, 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
])); 'redirect_uri' => $redirectUri,
} catch (RequestException $e) { ],
]);
return $response->toArray()['code'];
} catch (ExceptionInterface $e) {
$this->logger->error(sprintf('PocketImport: Failed to request token: %s', $e->getMessage()), ['exception' => $e]); $this->logger->error(sprintf('PocketImport: Failed to request token: %s', $e->getMessage()), ['exception' => $e]);
return false; return false;
} }
return $this->jsonDecode($response)['code'];
} }
/** /**
@ -81,19 +77,21 @@ class PocketImport extends AbstractImport
public function authorize($code) public function authorize($code)
{ {
try { try {
$response = $this->client->post('https://getpocket.com/v3/oauth/authorize', [], json_encode([ $response = $this->client->request(Request::METHOD_POST, 'https://getpocket.com/v3/oauth/authorize', [
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'json' => [
'code' => $code, 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
])); 'code' => $code,
} catch (RequestException $e) { ],
]);
$this->accessToken = $response->toArray()['access_token'];
return true;
} catch (ExceptionInterface $e) {
$this->logger->error(sprintf('PocketImport: Failed to authorize client: %s', $e->getMessage()), ['exception' => $e]); $this->logger->error(sprintf('PocketImport: Failed to authorize client: %s', $e->getMessage()), ['exception' => $e]);
return false; return false;
} }
$this->accessToken = $this->jsonDecode($response)['access_token'];
return true;
} }
public function import($offset = 0) public function import($offset = 0)
@ -101,49 +99,51 @@ class PocketImport extends AbstractImport
static $run = 0; static $run = 0;
try { try {
$response = $this->client->post('https://getpocket.com/v3/get', [], json_encode([ $response = $this->client->request(Request::METHOD_POST, 'https://getpocket.com/v3/get', [
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'json' => [
'access_token' => $this->accessToken, 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'detailType' => 'complete', 'access_token' => $this->accessToken,
'state' => 'all', 'detailType' => 'complete',
'sort' => 'newest', 'state' => 'all',
'count' => self::NB_ELEMENTS, 'sort' => 'newest',
'offset' => $offset, 'count' => self::NB_ELEMENTS,
])); 'offset' => $offset,
} catch (RequestException $e) { ],
]);
$entries = $response->toArray();
if ($this->producer) {
$this->parseEntriesForProducer($entries['list']);
} else {
$this->parseEntries($entries['list']);
}
// if we retrieve exactly the amount of items requested it means we can get more
// re-call import and offset item by the amount previous received:
// - first call get 5k offset 0
// - second call get 5k offset 5k
// - and so on
if (self::NB_ELEMENTS === \count($entries['list'])) {
++$run;
return $this->import(self::NB_ELEMENTS * $run);
}
return true;
} catch (ExceptionInterface $e) {
$this->logger->error(sprintf('PocketImport: Failed to import: %s', $e->getMessage()), ['exception' => $e]); $this->logger->error(sprintf('PocketImport: Failed to import: %s', $e->getMessage()), ['exception' => $e]);
return false; return false;
} }
$entries = $this->jsonDecode($response);
if ($this->producer) {
$this->parseEntriesForProducer($entries['list']);
} else {
$this->parseEntries($entries['list']);
}
// if we retrieve exactly the amount of items requested it means we can get more
// re-call import and offset item by the amount previous received:
// - first call get 5k offset 0
// - second call get 5k offset 5k
// - and so on
if (self::NB_ELEMENTS === \count($entries['list'])) {
++$run;
return $this->import(self::NB_ELEMENTS * $run);
}
return true;
} }
/** /**
* Set the Http client. * Set the Http client.
*/ */
public function setClient(ClientInterface $client, ?RequestFactoryInterface $requestFactory = null, ?StreamFactoryInterface $streamFactory = null) public function setClient(HttpClientInterface $pocketClient)
{ {
$this->client = new HttpMethodsClient(new PluginClient($client, [new ErrorPlugin()]), $requestFactory ?: Psr17FactoryDiscovery::findRequestFactory(), $streamFactory ?: Psr17FactoryDiscovery::findStreamFactory()); $this->client = $pocketClient;
} }
public function validateEntry(array $importedEntry) public function validateEntry(array $importedEntry)
@ -222,15 +222,4 @@ class PocketImport extends AbstractImport
return $importedEntry; return $importedEntry;
} }
protected function jsonDecode(ResponseInterface $response)
{
$data = json_decode((string) $response->getBody(), true);
if (\JSON_ERROR_NONE !== json_last_error()) {
throw new \InvalidArgumentException('Unable to parse JSON data: ' . json_last_error_msg());
}
return $data;
}
} }

92
tests/Wallabag/CoreBundle/Import/PocketImportTest.php
View file

@ -4,8 +4,6 @@ namespace Tests\Wallabag\CoreBundle\Import;
use Doctrine\ORM\EntityManager; use Doctrine\ORM\EntityManager;
use Doctrine\ORM\UnitOfWork; use Doctrine\ORM\UnitOfWork;
use GuzzleHttp\Psr7\Response;
use Http\Mock\Client as HttpMockClient;
use M6Web\Component\RedisMock\RedisMockFactory; use M6Web\Component\RedisMock\RedisMockFactory;
use Monolog\Handler\TestHandler; use Monolog\Handler\TestHandler;
use Monolog\Logger; use Monolog\Logger;
@ -13,6 +11,8 @@ use PHPUnit\Framework\TestCase;
use Predis\Client; use Predis\Client;
use Simpleue\Queue\RedisQueue; use Simpleue\Queue\RedisQueue;
use Symfony\Component\EventDispatcher\EventDispatcher; use Symfony\Component\EventDispatcher\EventDispatcher;
use Symfony\Component\HttpClient\MockHttpClient;
use Symfony\Component\HttpClient\Response\MockResponse;
use Wallabag\CoreBundle\Entity\Config; use Wallabag\CoreBundle\Entity\Config;
use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\User; use Wallabag\CoreBundle\Entity\User;
@ -43,11 +43,10 @@ class PocketImportTest extends TestCase
public function testOAuthRequest() public function testOAuthRequest()
{ {
$httpMockClient = new HttpMockClient(); $mockHttpClient = new MockHttpClient([new MockResponse(json_encode(['code' => 'wunderbar_code']), ['response_headers' => ['Content-Type: application/json']])]);
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['code' => 'wunderbar_code'])));
$pocketImport = $this->getPocketImport(); $pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$code = $pocketImport->getRequestToken('http://0.0.0.0/redirect'); $code = $pocketImport->getRequestToken('http://0.0.0.0/redirect');
@ -56,11 +55,10 @@ class PocketImportTest extends TestCase
public function testOAuthRequestBadResponse() public function testOAuthRequestBadResponse()
{ {
$httpMockClient = new HttpMockClient(); $mockHttpClient = new MockHttpClient([new MockResponse('', ['http_code' => 403])]);
$httpMockClient->addResponse(new Response(403));
$pocketImport = $this->getPocketImport(); $pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$code = $pocketImport->getRequestToken('http://0.0.0.0/redirect'); $code = $pocketImport->getRequestToken('http://0.0.0.0/redirect');
@ -73,11 +71,10 @@ class PocketImportTest extends TestCase
public function testOAuthAuthorize() public function testOAuthAuthorize()
{ {
$httpMockClient = new HttpMockClient(); $mockHttpClient = new MockHttpClient([new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']])]);
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token'])));
$pocketImport = $this->getPocketImport(); $pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$res = $pocketImport->authorize('wunderbar_code'); $res = $pocketImport->authorize('wunderbar_code');
@ -87,11 +84,10 @@ class PocketImportTest extends TestCase
public function testOAuthAuthorizeBadResponse() public function testOAuthAuthorizeBadResponse()
{ {
$httpMockClient = new HttpMockClient(); $mockHttpClient = new MockHttpClient([new MockResponse('', ['http_code' => 403])]);
$httpMockClient->addResponse(new Response(403));
$pocketImport = $this->getPocketImport(); $pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$res = $pocketImport->authorize('wunderbar_code'); $res = $pocketImport->authorize('wunderbar_code');
@ -107,9 +103,9 @@ class PocketImportTest extends TestCase
*/ */
public function testImport() public function testImport()
{ {
$httpMockClient = new HttpMockClient(); $mockHttpClient = new MockHttpClient([
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token']))); new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<'JSON' new MockResponse(<<<'JSON'
{ {
"status": 1, "status": 1,
"list": { "list": {
@ -190,7 +186,8 @@ class PocketImportTest extends TestCase
} }
} }
JSON JSON
)); , ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport('ConsumerKey', 1); $pocketImport = $this->getPocketImport('ConsumerKey', 1);
@ -221,7 +218,7 @@ JSON
->method('updateEntry') ->method('updateEntry')
->willReturn($entry); ->willReturn($entry);
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$pocketImport->authorize('wunderbar_code'); $pocketImport->authorize('wunderbar_code');
$res = $pocketImport->import(); $res = $pocketImport->import();
@ -235,9 +232,9 @@ JSON
*/ */
public function testImportAndMarkAllAsRead() public function testImportAndMarkAllAsRead()
{ {
$httpMockClient = new HttpMockClient(); $mockHttpClient = new MockHttpClient([
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token']))); new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<'JSON' new MockResponse(<<<'JSON'
{ {
"status": 1, "status": 1,
"list": { "list": {
@ -280,7 +277,8 @@ JSON
} }
} }
JSON JSON
)); , ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport('ConsumerKey', 2); $pocketImport = $this->getPocketImport('ConsumerKey', 2);
@ -312,7 +310,7 @@ JSON
->method('updateEntry') ->method('updateEntry')
->willReturn($entry); ->willReturn($entry);
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$pocketImport->authorize('wunderbar_code'); $pocketImport->authorize('wunderbar_code');
$res = $pocketImport->setMarkAsRead(true)->import(); $res = $pocketImport->setMarkAsRead(true)->import();
@ -326,8 +324,6 @@ JSON
*/ */
public function testImportWithRabbit() public function testImportWithRabbit()
{ {
$httpMockClient = new HttpMockClient();
$body = <<<'JSON' $body = <<<'JSON'
{ {
"item_id": "229279689", "item_id": "229279689",
@ -351,8 +347,9 @@ JSON
} }
JSON; JSON;
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token']))); $mockHttpClient = new MockHttpClient([
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<JSON new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
new MockResponse(<<<JSON
{ {
"status": 1, "status": 1,
"list": { "list": {
@ -360,7 +357,8 @@ JSON;
} }
} }
JSON JSON
)); , ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport(); $pocketImport = $this->getPocketImport();
@ -394,7 +392,7 @@ JSON
->method('publish') ->method('publish')
->with(json_encode($bodyAsArray)); ->with(json_encode($bodyAsArray));
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$pocketImport->setProducer($producer); $pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code'); $pocketImport->authorize('wunderbar_code');
@ -409,8 +407,6 @@ JSON
*/ */
public function testImportWithRedis() public function testImportWithRedis()
{ {
$httpMockClient = new HttpMockClient();
$body = <<<'JSON' $body = <<<'JSON'
{ {
"item_id": "229279689", "item_id": "229279689",
@ -434,8 +430,9 @@ JSON
} }
JSON; JSON;
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token']))); $mockHttpClient = new MockHttpClient([
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<JSON new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
new MockResponse(<<<JSON
{ {
"status": 1, "status": 1,
"list": { "list": {
@ -443,7 +440,8 @@ JSON;
} }
} }
JSON JSON
)); , ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport(); $pocketImport = $this->getPocketImport();
@ -470,7 +468,7 @@ JSON
$queue = new RedisQueue($redisMock, 'pocket'); $queue = new RedisQueue($redisMock, 'pocket');
$producer = new Producer($queue); $producer = new Producer($queue);
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$pocketImport->setProducer($producer); $pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code'); $pocketImport->authorize('wunderbar_code');
@ -484,13 +482,13 @@ JSON
public function testImportBadResponse() public function testImportBadResponse()
{ {
$httpMockClient = new HttpMockClient(); $mockHttpClient = new MockHttpClient([
new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token']))); new MockResponse('', ['http_code' => 403]),
$httpMockClient->addResponse(new Response(403)); ]);
$pocketImport = $this->getPocketImport(); $pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$pocketImport->authorize('wunderbar_code'); $pocketImport->authorize('wunderbar_code');
$res = $pocketImport->import(); $res = $pocketImport->import();
@ -504,10 +502,9 @@ JSON
public function testImportWithExceptionFromGraby() public function testImportWithExceptionFromGraby()
{ {
$httpMockClient = new HttpMockClient(); $mockHttpClient = new MockHttpClient([
new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token']))); new MockResponse(<<<'JSON'
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<'JSON'
{ {
"status": 1, "status": 1,
"list": { "list": {
@ -520,7 +517,8 @@ JSON
} }
JSON JSON
)); , ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport('ConsumerKey', 1); $pocketImport = $this->getPocketImport('ConsumerKey', 1);
@ -544,7 +542,7 @@ JSON
->method('updateEntry') ->method('updateEntry')
->will($this->throwException(new \Exception())); ->will($this->throwException(new \Exception()));
$pocketImport->setClient($httpMockClient); $pocketImport->setClient($mockHttpClient);
$pocketImport->authorize('wunderbar_code'); $pocketImport->authorize('wunderbar_code');
$res = $pocketImport->import(); $res = $pocketImport->import();