Enable Redis async import

- using javibravo/simpleue
- internal config value are now `import_with_redis` & `import_with_rabbit` which are more clear
- if both option are enable rabbit will be choosen
- services imports related to async are now splitted into 2 files: `redis.yml` & `rabbit.yml`
-

app/config/config.yml

@@ -254,7 +254,7 @@ old_sound_rabbit_mq:
                 type: topic
             queue_options:
                 name: 'wallabag.import.pocket'
-            callback: wallabag_import.consumer.pocket
+            callback: wallabag_import.consumer.ampq.pocket
         import_readability:
             connection: default
             exchange_options:
@@ -262,7 +262,7 @@ old_sound_rabbit_mq:
                 type: topic
             queue_options:
                 name: 'wallabag.import.readability'
-            callback: wallabag_import.consumer.readability
+            callback: wallabag_import.consumer.ampq.readability
         import_wallabag_v1:
             connection: default
             exchange_options:
@@ -270,7 +270,7 @@ old_sound_rabbit_mq:
                 type: topic
             queue_options:
                 name: 'wallabag.import.wallabag_v1'
-            callback: wallabag_import.consumer.wallabag_v1
+            callback: wallabag_import.consumer.ampq.wallabag_v1
         import_wallabag_v2:
             connection: default
             exchange_options:
@@ -278,4 +278,4 @@ old_sound_rabbit_mq:
                 type: topic
             queue_options:
                 name: 'wallabag.import.wallabag_v2'
-            callback: wallabag_import.consumer.wallabag_v2
+            callback: wallabag_import.consumer.ampq.wallabag_v2

app/config/parameters.yml.dist

@@ -46,3 +46,7 @@ parameters:
     rabbitmq_port: 5672
     rabbitmq_user: guest
     rabbitmq_password: guest
+
+    # Redis processing
+    redis_host: localhost
+    redis_port: 6379

composer.json

@@ -82,7 +82,9 @@
         "ocramius/proxy-manager": "1.*",
         "white-october/pagerfanta-bundle": "^1.0",
         "mouf/nodejs-installer": "~1.0",
-        "php-amqplib/rabbitmq-bundle": "^1.8"
+        "php-amqplib/rabbitmq-bundle": "^1.8",
+        "predis/predis": "^1.0",
+        "javibravo/simpleue": "^1.0"
     },
     "require-dev": {
         "doctrine/doctrine-fixtures-bundle": "~2.2",
@@ -90,7 +92,8 @@
         "sensio/generator-bundle": "^3.0",
         "phpunit/phpunit": "~5.0",
         "symfony/phpunit-bridge": "^3.0",
-        "friendsofphp/php-cs-fixer": "~1.9"
+        "friendsofphp/php-cs-fixer": "~1.9",
+        "m6web/redis-mock": "^2.0"
     },
     "scripts": {
         "post-cmd": [

src/Wallabag/CoreBundle/Command/InstallCommand.php

@@ -322,7 +322,12 @@ class InstallCommand extends ContainerAwareCommand
                 'section' => 'import',
             ],
             [
-                'name' => 'rabbitmq',
+                'name' => 'import_with_redis',
+                'value' => '0',
+                'section' => 'import',
+            ],
+            [
+                'name' => 'import_with_rabbitmq',
                 'value' => '0',
                 'section' => 'import',
             ],

src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php

@@ -96,7 +96,12 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
                 'section' => 'import',
             ],
             [
-                'name' => 'rabbitmq',
+                'name' => 'import_with_redis',
+                'value' => '0',
+                'section' => 'import',
+            ],
+            [
+                'name' => 'import_with_rabbitmq',
                 'value' => '0',
                 'section' => 'import',
             ],

src/Wallabag/CoreBundle/Resources/config/services.yml

@@ -125,3 +125,11 @@ services:
         arguments:
             - "@security.token_storage"
             - "@router"
+
+    wallabag_core.redis.client:
+        class: Predis\Client
+        arguments:
+            -
+                host: '%redis_host%'
+                port: '%redis_port%'
+                schema: tcp

src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace Wallabag\ImportBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Config\Definition\Exception\Exception;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Simpleue\Worker\QueueWorker;
+
+class RedisWorkerCommand extends ContainerAwareCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('wallabag:import:redis-worker')
+            ->setDescription('Launch Redis worker')
+            ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
+        $output->writeln('Waiting for message ...');
+
+        $serviceName = $input->getArgument('serviceName');
+
+        if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
+            throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
+        }
+
+        $worker = new QueueWorker(
+            $this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
+            $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName)
+        );
+
+        $worker->start();
+    }
+}

src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Wallabag\ImportBundle\Consumer\AMPQ;
+namespace Wallabag\ImportBundle\Consumer;
 
 use Doctrine\ORM\EntityManager;
 use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
@@ -12,7 +12,7 @@ use Wallabag\CoreBundle\Entity\Tag;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
 
-class EntryConsumer implements ConsumerInterface
+class AMPQEntryConsumer implements ConsumerInterface
 {
     private $em;
     private $userRepository;
@@ -64,5 +64,7 @@ class EntryConsumer implements ConsumerInterface
 
             return;
         }
+
+        $this->logger->info('Content with url ('.$entry->getUrl().') imported !');
     }
 }

src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php

@@ -0,0 +1,84 @@
+<?php
+
+namespace Wallabag\ImportBundle\Consumer;
+
+use Simpleue\Job\Job;
+use Doctrine\ORM\EntityManager;
+use Wallabag\ImportBundle\Import\AbstractImport;
+use Wallabag\UserBundle\Repository\UserRepository;
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\CoreBundle\Entity\Tag;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
+
+class RedisEntryConsumer implements Job
+{
+    private $em;
+    private $userRepository;
+    private $import;
+    private $logger;
+
+    public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
+    {
+        $this->em = $em;
+        $this->userRepository = $userRepository;
+        $this->import = $import;
+        $this->logger = $logger ?: new NullLogger();
+    }
+
+    /**
+     * Handle one message by one message.
+     *
+     * @param string $job Content of the message (directly from Redis)
+     *
+     * @return bool
+     */
+    public function manage($job)
+    {
+        $storedEntry = json_decode($job, true);
+
+        $user = $this->userRepository->find($storedEntry['userId']);
+
+        // no user? Drop message
+        if (null === $user) {
+            $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
+
+            return false;
+        }
+
+        $this->import->setUser($user);
+
+        $entry = $this->import->parseEntry($storedEntry);
+
+        if (null === $entry) {
+            $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
+
+            return false;
+        }
+
+        try {
+            $this->em->flush();
+
+            // clear only affected entities
+            $this->em->clear(Entry::class);
+            $this->em->clear(Tag::class);
+        } catch (\Exception $e) {
+            $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
+
+            return false;
+        }
+
+        $this->logger->info('Content with url ('.$entry->getUrl().') imported !');
+
+        return true;
+    }
+
+    /**
+     * Should tell if the given job will kill the worker.
+     * We don't want to stop it :).
+     */
+    public function isStopJob($job)
+    {
+        return false;
+    }
+}

src/Wallabag/ImportBundle/Controller/PocketController.php

@@ -20,8 +20,10 @@ class PocketController extends Controller
         $pocket = $this->get('wallabag_import.pocket.import');
         $pocket->setUser($this->getUser());
 
-        if ($this->get('craue_config')->get('rabbitmq')) {
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
-            $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
+            $pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
         }
 
         return $pocket;

src/Wallabag/ImportBundle/Controller/ReadabilityController.php

@@ -20,8 +20,10 @@ class ReadabilityController extends Controller
         $readability = $this->get('wallabag_import.readability.import');
         $readability->setUser($this->getUser());
 
-        if ($this->get('craue_config')->get('rabbitmq')) {
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
-            $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
+            $readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
         }
 
         if ($form->isValid()) {

src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php

@@ -14,8 +14,10 @@ class WallabagV1Controller extends WallabagController
     {
         $service = $this->get('wallabag_import.wallabag_v1.import');
 
-        if ($this->get('craue_config')->get('rabbitmq')) {
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
-            $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
+            $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
         }
 
         return $service;

src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php

@@ -14,8 +14,10 @@ class WallabagV2Controller extends WallabagController
     {
         $service = $this->get('wallabag_import.wallabag_v2.import');
 
-        if ($this->get('craue_config')->get('rabbitmq')) {
+        if ($this->get('craue_config')->get('import_with_rabbitmq')) {
-            $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
+            $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
+        } elseif ($this->get('craue_config')->get('import_with_redis')) {
+            $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
         }
 
         return $service;

src/Wallabag/ImportBundle/Import/AbstractImport.php

@@ -9,7 +9,7 @@ use Wallabag\CoreBundle\Helper\ContentProxy;
 use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Entity\Tag;
 use Wallabag\UserBundle\Entity\User;
-use OldSound\RabbitMqBundle\RabbitMq\Producer;
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
 
 abstract class AbstractImport implements ImportInterface
 {
@@ -35,12 +35,12 @@ abstract class AbstractImport implements ImportInterface
     }
 
     /**
-     * Set RabbitMQ Producer to send each entry to a queue.
+     * Set RabbitMQ/Redis Producer to send each entry to a queue.
      * This method should be called when user has enabled RabbitMQ.
      *
-     * @param Producer $producer
+     * @param ProducerInterface $producer
      */
-    public function setRabbitmqProducer(Producer $producer)
+    public function setProducer(ProducerInterface $producer)
     {
         $this->producer = $producer;
     }

src/Wallabag/ImportBundle/Redis/Producer.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace Wallabag\ImportBundle\Redis;
+
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
+use Simpleue\Queue\RedisQueue;
+
+/**
+ * This is a proxy class for "Simpleue\Queue\RedisQueue".
+ * It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
+ *
+ * It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
+ * kind of class which implements the same interface.
+ * So we can inject either a RabbitMQ producer or a Redis producer with the same signature
+ */
+class Producer implements ProducerInterface
+{
+    private $queue;
+
+    public function __construct(RedisQueue $queue)
+    {
+        $this->queue = $queue;
+    }
+
+    /**
+     * Publish a message in the Redis queue.
+     *
+     * @param string $msgBody
+     * @param string $routingKey           NOT USED
+     * @param array  $additionalProperties NOT USED
+     */
+    public function publish($msgBody, $routingKey = '', $additionalProperties = array())
+    {
+        $this->queue->sendJob($msgBody);
+    }
+}

src/Wallabag/ImportBundle/Resources/config/rabbit.yml

@@ -0,0 +1,30 @@
+# RabbitMQ stuff
+services:
+    wallabag_import.consumer.ampq.pocket:
+        class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.pocket.import"
+            - "@logger"
+    wallabag_import.consumer.ampq.readability:
+        class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.readability.import"
+            - "@logger"
+    wallabag_import.consumer.ampq.wallabag_v1:
+        class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v1.import"
+            - "@logger"
+    wallabag_import.consumer.ampq.wallabag_v2:
+        class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v2.import"
+            - "@logger"

src/Wallabag/ImportBundle/Resources/config/redis.yml

@@ -0,0 +1,81 @@
+# Redis stuff
+services:
+    # readability
+    wallabag_import.queue.redis.readability:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.readability"
+
+    wallabag_import.producer.redis.readability:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.readability"
+
+    wallabag_import.consumer.redis.readability:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.readability.import"
+            - "@logger"
+
+    # pocket
+    wallabag_import.queue.redis.pocket:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.pocket"
+
+    wallabag_import.producer.redis.pocket:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.pocket"
+
+    wallabag_import.consumer.redis.pocket:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.pocket.import"
+            - "@logger"
+
+    # wallabag v1
+    wallabag_import.queue.redis.wallabag_v1:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.wallabag_v1"
+
+    wallabag_import.producer.redis.wallabag_v1:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.wallabag_v1"
+
+    wallabag_import.consumer.redis.wallabag_v1:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v1.import"
+            - "@logger"
+
+    # wallabag v2
+    wallabag_import.queue.redis.wallabag_v2:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.wallabag_v2"
+
+    wallabag_import.producer.redis.wallabag_v2:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.wallabag_v2"
+
+    wallabag_import.consumer.redis.wallabag_v2:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v2.import"
+            - "@logger"

src/Wallabag/ImportBundle/Resources/config/services.yml

@@ -1,33 +1,8 @@
-services:
+imports:
-    wallabag_import.consumer.pocket:
+    - { resource: rabbit.yml }
-        class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
+    - { resource: redis.yml }
-        arguments:
-            - "@doctrine.orm.entity_manager"
-            - "@wallabag_user.user_repository"
-            - "@wallabag_import.pocket.import"
-            - "@logger"
-    wallabag_import.consumer.readability:
-        class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
-        arguments:
-            - "@doctrine.orm.entity_manager"
-            - "@wallabag_user.user_repository"
-            - "@wallabag_import.readability.import"
-            - "@logger"
-    wallabag_import.consumer.wallabag_v1:
-        class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
-        arguments:
-            - "@doctrine.orm.entity_manager"
-            - "@wallabag_user.user_repository"
-            - "@wallabag_import.wallabag_v1.import"
-            - "@logger"
-    wallabag_import.consumer.wallabag_v2:
-        class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
-        arguments:
-            - "@doctrine.orm.entity_manager"
-            - "@wallabag_user.user_repository"
-            - "@wallabag_import.wallabag_v2.import"
-            - "@logger"
 
+services:
     wallabag_import.chain:
         class: Wallabag\ImportBundle\Import\ImportChain
 

tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php

@@ -2,12 +2,12 @@
 
 namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
 
-use Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer;
+use Wallabag\ImportBundle\Consumer\AMPQEntryConsumer;
 use PhpAmqpLib\Message\AMQPMessage;
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Entity\Entry;
 
-class EntryConsumerTest extends \PHPUnit_Framework_TestCase
+class AMPQEntryConsumerTest extends \PHPUnit_Framework_TestCase
 {
     public function testMessageOk()
     {
@@ -112,7 +112,7 @@ JSON;
             ->with(json_decode($body, true))
             ->willReturn($entry);
 
-        $consumer = new EntryConsumer(
+        $consumer = new AMPQEntryConsumer(
             $em,
             $userRepository,
             $import
@@ -157,7 +157,7 @@ JSON;
             ->disableOriginalConstructor()
             ->getMock();
 
-        $consumer = new EntryConsumer(
+        $consumer = new AMPQEntryConsumer(
             $em,
             $userRepository,
             $import
@@ -212,7 +212,7 @@ JSON;
             ->with(json_decode($body, true))
             ->willReturn(null);
 
-        $consumer = new EntryConsumer(
+        $consumer = new AMPQEntryConsumer(
             $em,
             $userRepository,
             $import

tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php

@@ -0,0 +1,224 @@
+<?php
+
+namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
+
+use Wallabag\ImportBundle\Consumer\RedisEntryConsumer;
+use Wallabag\UserBundle\Entity\User;
+use Wallabag\CoreBundle\Entity\Entry;
+
+class RedisEntryConsumerTest extends \PHPUnit_Framework_TestCase
+{
+    public function testMessageOk()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->once())
+            ->method('flush');
+
+        $em
+            ->expects($this->exactly(2))
+            ->method('clear');
+
+        $body = <<<'JSON'
+{
+    "item_id": "1402935436",
+    "resolved_id": "1402935436",
+    "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
+    "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
+    "favorite": "0",
+    "status": "0",
+    "time_added": "1473020899",
+    "time_updated": "1473020899",
+    "time_read": "0",
+    "time_favorited": "0",
+    "sort_id": 0,
+    "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
+    "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
+    "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings.  It all started with a mic drop ...",
+    "is_article": "1",
+    "is_index": "0",
+    "has_video": "0",
+    "has_image": "1",
+    "word_count": "200",
+    "tags": {
+        "ifttt": {
+            "item_id": "1402935436",
+            "tag": "ifttt"
+        },
+        "mashable": {
+            "item_id": "1402935436",
+            "tag": "mashable"
+        }
+    },
+    "authors": {
+        "2484273": {
+            "item_id": "1402935436",
+            "author_id": "2484273",
+            "name": "Adam Rosenberg",
+            "url": "http://mashable.com/author/adam-rosenberg/"
+        }
+    },
+    "image": {
+        "item_id": "1402935436",
+        "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
+        "width": "0",
+        "height": "0"
+    },
+    "images": {
+        "1": {
+            "item_id": "1402935436",
+            "image_id": "1",
+            "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
+            "width": "0",
+            "height": "0",
+            "credit": "Image:  Steve Eichner/NameFace/Sipa USA",
+            "caption": ""
+        }
+    },
+    "userId": 1
+}
+JSON;
+
+        $user = new User();
+        $entry = new Entry($user);
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(1)
+            ->willReturn($user);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $import
+            ->expects($this->once())
+            ->method('setUser')
+            ->with($user);
+
+        $import
+            ->expects($this->once())
+            ->method('parseEntry')
+            ->with(json_decode($body, true))
+            ->willReturn($entry);
+
+        $consumer = new RedisEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $res = $consumer->manage($body);
+
+        $this->assertTrue($res);
+    }
+
+    public function testMessageWithBadUser()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->never())
+            ->method('flush');
+
+        $em
+            ->expects($this->never())
+            ->method('clear');
+
+        $body = '{ "userId": 123 }';
+
+        $user = new User();
+        $entry = new Entry($user);
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(123)
+            ->willReturn(null);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $consumer = new RedisEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $res = $consumer->manage($body);
+
+        $this->assertFalse($res);
+    }
+
+    public function testMessageWithEntryProcessed()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $em
+            ->expects($this->never())
+            ->method('flush');
+
+        $em
+            ->expects($this->never())
+            ->method('clear');
+
+        $body = '{ "userId": 123 }';
+
+        $user = new User();
+
+        $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $userRepository
+            ->expects($this->once())
+            ->method('find')
+            // userId from the body json above
+            ->with(123)
+            ->willReturn($user);
+
+        $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $import
+            ->expects($this->once())
+            ->method('setUser')
+            ->with($user);
+
+        $import
+            ->expects($this->once())
+            ->method('parseEntry')
+            ->with(json_decode($body, true))
+            ->willReturn(null);
+
+        $consumer = new RedisEntryConsumer(
+            $em,
+            $userRepository,
+            $import
+        );
+
+        $res = $consumer->manage($body);
+
+        $this->assertFalse($res);
+    }
+}

tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php

@@ -22,14 +22,29 @@ class PocketControllerTest extends WallabagCoreTestCase
         $this->logInAs('admin');
         $client = $this->getClient();
 
-        $client->getContainer()->get('craue_config')->set('rabbitmq', 1);
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
 
         $crawler = $client->request('GET', '/import/pocket');
 
         $this->assertEquals(200, $client->getResponse()->getStatusCode());
         $this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
 
-        $client->getContainer()->get('craue_config')->set('rabbitmq', 0);
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+    }
+
+    public function testImportPocketWithRedisEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+        $crawler = $client->request('GET', '/import/pocket');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
     }
 
     public function testImportPocketAuthBadToken()

tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php

@@ -24,7 +24,7 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
         $this->logInAs('admin');
         $client = $this->getClient();
 
-        $client->getContainer()->get('craue_config')->set('rabbitmq', 1);
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
 
         $crawler = $client->request('GET', '/import/readability');
 
@@ -32,7 +32,23 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
         $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
         $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
 
-        $client->getContainer()->get('craue_config')->set('rabbitmq', 0);
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+    }
+
+    public function testImportReadabilityWithRedisEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+        $crawler = $client->request('GET', '/import/readability');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
     }
 
     public function testImportReadabilityWithFile()

tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php

@@ -24,7 +24,7 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
         $this->logInAs('admin');
         $client = $this->getClient();
 
-        $client->getContainer()->get('craue_config')->set('rabbitmq', 1);
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
 
         $crawler = $client->request('GET', '/import/wallabag-v1');
 
@@ -32,7 +32,23 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
         $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
         $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
 
-        $client->getContainer()->get('craue_config')->set('rabbitmq', 0);
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+    }
+
+    public function testImportWallabagWithRedisEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+        $crawler = $client->request('GET', '/import/wallabag-v1');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
     }
 
     public function testImportWallabagWithFile()

tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php

@@ -24,7 +24,7 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
         $this->logInAs('admin');
         $client = $this->getClient();
 
-        $client->getContainer()->get('craue_config')->set('rabbitmq', 1);
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
 
         $crawler = $client->request('GET', '/import/wallabag-v2');
 
@@ -32,7 +32,23 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
         $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
         $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
 
-        $client->getContainer()->get('craue_config')->set('rabbitmq', 0);
+        $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+    }
+
+    public function testImportWallabagWithRedisEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+        $crawler = $client->request('GET', '/import/wallabag-v2');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+        $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
     }
 
     public function testImportWallabagWithFile()

tests/Wallabag/ImportBundle/Import/PocketImportTest.php

@@ -9,8 +9,11 @@ use GuzzleHttp\Client;
 use GuzzleHttp\Subscriber\Mock;
 use GuzzleHttp\Message\Response;
 use GuzzleHttp\Stream\Stream;
+use Wallabag\ImportBundle\Redis\Producer;
 use Monolog\Logger;
 use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
 
 class PocketImportTest extends \PHPUnit_Framework_TestCase
 {
@@ -442,7 +445,7 @@ JSON;
             ->with(json_encode($bodyAsArray));
 
         $pocketImport->setClient($client);
-        $pocketImport->setRabbitmqProducer($producer);
+        $pocketImport->setProducer($producer);
         $pocketImport->authorize('wunderbar_code');
 
         $res = $pocketImport->setMarkAsRead(true)->import();
@@ -451,6 +454,87 @@ JSON;
         $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary());
     }
 
+    /**
+     * Will sample results from https://getpocket.com/developer/docs/v3/retrieve.
+     */
+    public function testImportWithRedis()
+    {
+        $client = new Client();
+
+        $body = <<<'JSON'
+{
+    "item_id": "229279689",
+    "resolved_id": "229279689",
+    "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
+    "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
+    "favorite": "1",
+    "status": "1",
+    "time_added": "1473020899",
+    "time_updated": "1473020899",
+    "time_read": "0",
+    "time_favorited": "0",
+    "sort_id": 0,
+    "resolved_title": "The Massive Ryder Cup Preview",
+    "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
+    "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
+    "is_article": "1",
+    "has_video": "0",
+    "has_image": "0",
+    "word_count": "3197"
+}
+JSON;
+
+        $mock = new Mock([
+            new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))),
+            new Response(200, ['Content-Type' => 'application/json'], Stream::factory('
+                {
+                    "status": 1,
+                    "list": {
+                        "229279690": '.$body.'
+                    }
+                }
+            ')),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $pocketImport = $this->getPocketImport();
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = new Entry($this->user);
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $queue = new RedisQueue($redisMock, 'pocket');
+        $producer = new Producer($queue);
+
+        $pocketImport->setClient($client);
+        $pocketImport->setProducer($producer);
+        $pocketImport->authorize('wunderbar_code');
+
+        $res = $pocketImport->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary());
+
+        $this->assertNotEmpty($redisMock->lpop('pocket'));
+    }
+
     public function testImportBadResponse()
     {
         $client = new Client();

tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php

@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
 use Wallabag\ImportBundle\Import\ReadabilityImport;
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
 use Monolog\Logger;
 use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
 
 class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
 {
@@ -152,7 +155,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
             ->expects($this->exactly(2))
             ->method('publish');
 
-        $readabilityImport->setRabbitmqProducer($producer);
+        $readabilityImport->setProducer($producer);
 
         $res = $readabilityImport->setMarkAsRead(true)->import();
 
@@ -160,6 +163,46 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
         $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
     }
 
+    public function testImportWithRedis()
+    {
+        $readabilityImport = $this->getReadabilityImport();
+        $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $queue = new RedisQueue($redisMock, 'readability');
+        $producer = new Producer($queue);
+
+        $readabilityImport->setProducer($producer);
+
+        $res = $readabilityImport->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
+
+        $this->assertNotEmpty($redisMock->lpop('readability'));
+    }
+
     public function testImportBadFile()
     {
         $readabilityImport = $this->getReadabilityImport();

tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php

@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
 use Wallabag\ImportBundle\Import\WallabagV1Import;
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
 use Monolog\Logger;
 use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
 
 class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
 {
@@ -152,7 +155,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
             ->expects($this->exactly(4))
             ->method('publish');
 
-        $wallabagV1Import->setRabbitmqProducer($producer);
+        $wallabagV1Import->setProducer($producer);
 
         $res = $wallabagV1Import->setMarkAsRead(true)->import();
 
@@ -160,6 +163,46 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
         $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary());
     }
 
+    public function testImportWithRedis()
+    {
+        $wallabagV1Import = $this->getWallabagV1Import();
+        $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $queue = new RedisQueue($redisMock, 'wallabag_v1');
+        $producer = new Producer($queue);
+
+        $wallabagV1Import->setProducer($producer);
+
+        $res = $wallabagV1Import->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary());
+
+        $this->assertNotEmpty($redisMock->lpop('wallabag_v1'));
+    }
+
     public function testImportBadFile()
     {
         $wallabagV1Import = $this->getWallabagV1Import();

tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php

@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
 use Wallabag\ImportBundle\Import\WallabagV2Import;
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
 use Monolog\Logger;
 use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
 
 class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
 {
@@ -144,7 +147,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
             ->expects($this->exactly(24))
             ->method('publish');
 
-        $wallabagV2Import->setRabbitmqProducer($producer);
+        $wallabagV2Import->setProducer($producer);
 
         $res = $wallabagV2Import->setMarkAsRead(true)->import();
 
@@ -152,6 +155,42 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
         $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary());
     }
 
+    public function testImportWithRedis()
+    {
+        $wallabagV2Import = $this->getWallabagV2Import();
+        $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->never())
+            ->method('findByUrlAndUserId');
+
+        $this->em
+            ->expects($this->never())
+            ->method('getRepository');
+
+        $this->contentProxy
+            ->expects($this->never())
+            ->method('updateEntry');
+
+        $factory = new RedisMockFactory();
+        $redisMock = $factory->getAdapter('Predis\Client', true);
+
+        $queue = new RedisQueue($redisMock, 'wallabag_v2');
+        $producer = new Producer($queue);
+
+        $wallabagV2Import->setProducer($producer);
+
+        $res = $wallabagV2Import->setMarkAsRead(true)->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary());
+
+        $this->assertNotEmpty($redisMock->lpop('wallabag_v2'));
+    }
+
     public function testImportBadFile()
     {
         $wallabagV1Import = $this->getWallabagV2Import();