Merge pull request #2180 from wallabag/download-pictures

Download pictures

.gitignore

@@ -25,6 +25,8 @@ web/uploads/
 !web/bundles
 web/bundles/*
 !web/bundles/wallabagcore
+/web/assets/images/*
+!web/assets/images/.gitkeep
 
 # Build
 /app/build
@@ -34,7 +36,6 @@ web/bundles/*
 /composer.phar
 
 # Data for wallabag
-data/assets/*
 data/db/wallabag*.sqlite
 
 # Docker container logs and data

app/DoctrineMigrations/Version20161031132655.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace Application\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+use Symfony\Component\DependencyInjection\ContainerAwareInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+class Version20161031132655 extends AbstractMigration implements ContainerAwareInterface
+{
+    /**
+     * @var ContainerInterface
+     */
+    private $container;
+
+    public function setContainer(ContainerInterface $container = null)
+    {
+        $this->container = $container;
+    }
+
+    private function getTable($tableName)
+    {
+        return $this->container->getParameter('database_table_prefix') . $tableName;
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function up(Schema $schema)
+    {
+        $this->addSql("INSERT INTO \"".$this->getTable('craue_config_setting')."\" (name, value, section) VALUES ('download_images_enabled', 0, 'misc')");
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function down(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() == 'sqlite', 'Migration can only be executed safely on \'mysql\' or \'postgresql\'.');
+
+        $this->addSql("DELETE FROM \"".$this->getTable('craue_config_setting')."\" WHERE name = 'download_images_enabled';");
+    }
+}

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml

@@ -29,3 +29,4 @@ piwik_enabled: Aktiver Piwik
 demo_mode_enabled: "Aktiver demo-indstilling? (anvendes kun til wallabags offentlige demo)"
 demo_mode_username: "Demobruger"
 # share_public: Allow public url for entries
+# download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml

@@ -29,3 +29,4 @@ piwik_enabled: Piwik aktivieren
 demo_mode_enabled: "Test-Modus aktivieren? (nur für die öffentliche wallabag-Demo genutzt)"
 demo_mode_username: "Test-Benutzer"
 share_public: Erlaube eine öffentliche URL für Einträge
+# download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml

@@ -29,3 +29,4 @@ piwik_enabled: Enable Piwik
 demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)"
 demo_mode_username: "Demo user"
 share_public: Allow public url for entries
+download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml

@@ -29,3 +29,4 @@ piwik_enabled: Activar Piwik
 demo_mode_enabled: "Activar modo demo (sólo usado para la demo de wallabag)"
 demo_mode_username: "Nombre de usuario demo"
 # share_public: Allow public url for entries
+# download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml

@@ -29,3 +29,4 @@ modify_settings: "اعمال"
 # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)"
 # demo_mode_username: "Demo user"
 # share_public: Allow public url for entries
+# download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml

@@ -29,3 +29,4 @@ piwik_enabled: Activer Piwik
 demo_mode_enabled: "Activer le mode démo ? (utiliser uniquement pour la démo publique de wallabag)"
 demo_mode_username: "Utilisateur de la démo"
 share_public: Autoriser une URL publique pour les articles
+download_images_enabled: Télécharger les images en local

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml

@@ -29,3 +29,4 @@ piwik_enabled: Abilita Piwik
 demo_mode_enabled: "Abilita modalità demo ? (usato solo per la demo pubblica di wallabag)"
 demo_mode_username: "Utente Demo"
 # share_public: Allow public url for entries
+# download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml

@@ -29,3 +29,4 @@ piwik_enabled: Activar Piwik
 demo_mode_enabled: "Activar lo mode demostracion ? (utilizar solament per la demostracion publica de wallabag)"
 demo_mode_username: "Utilizaire de la demostracion"
 # share_public: Allow public url for entries
+# download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml

@@ -29,3 +29,4 @@ piwik_enabled: Włacz Piwik
 demo_mode_enabled: "Włacz tryb demo? (używany wyłącznie dla publicznej demonstracji Wallabag)"
 demo_mode_username: "Użytkownik Demonstracyjny"
 share_public: Zezwalaj na publiczny adres url dla wpisow
+# download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml

@@ -29,3 +29,4 @@ modify_settings: "aplică"
 # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)"
 # demo_mode_username: "Demo user"
 # share_public: Allow public url for entries
+# download_images_enabled: Download images locally

app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml

@@ -29,3 +29,4 @@
 # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)"
 # demo_mode_username: "Demo user"
 # share_public: Allow public url for entries
+# download_images_enabled: Download images locally

app/config/services.yml

@@ -32,13 +32,13 @@ services:
             - { name: twig.extension }
 
     wallabag.locale_listener:
-        class: Wallabag\CoreBundle\EventListener\LocaleListener
+        class: Wallabag\CoreBundle\Event\Listener\LocaleListener
         arguments: ["%kernel.default_locale%"]
         tags:
             - { name: kernel.event_subscriber }
 
     wallabag.user_locale_listener:
-        class: Wallabag\CoreBundle\EventListener\UserLocaleListener
+        class: Wallabag\CoreBundle\Event\Listener\UserLocaleListener
         arguments: ["@session"]
         tags:
             - { name: kernel.event_listener, event: security.interactive_login, method: onInteractiveLogin }

composer.json

@@ -82,7 +82,8 @@
         "white-october/pagerfanta-bundle": "^1.0",
         "php-amqplib/rabbitmq-bundle": "^1.8",
         "predis/predis": "^1.0",
-        "javibravo/simpleue": "^1.0"
+        "javibravo/simpleue": "^1.0",
+        "symfony/dom-crawler": "^3.1"
     },
     "require-dev": {
         "doctrine/doctrine-fixtures-bundle": "~2.2",

src/Wallabag/ApiBundle/Controller/WallabagRestController.php

@@ -14,6 +14,8 @@ use Symfony\Component\Security\Core\Exception\AccessDeniedException;
 use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Entity\Tag;
 use Wallabag\AnnotationBundle\Entity\Annotation;
+use Wallabag\CoreBundle\Event\EntrySavedEvent;
+use Wallabag\CoreBundle\Event\EntryDeletedEvent;
 
 class WallabagRestController extends FOSRestController
 {
@@ -233,9 +235,11 @@ class WallabagRestController extends FOSRestController
 
         $em = $this->getDoctrine()->getManager();
         $em->persist($entry);
-
         $em->flush();
 
+        // entry saved, dispatch event about it!
+        $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+
         $json = $this->get('serializer')->serialize($entry, 'json');
 
         return (new JsonResponse())->setJson($json);
@@ -308,6 +312,9 @@ class WallabagRestController extends FOSRestController
         $this->validateAuthentication();
         $this->validateUserAccess($entry->getUser()->getId());
 
+        // entry deleted, dispatch event about it!
+        $this->get('event_dispatcher')->dispatch(EntryDeletedEvent::NAME, new EntryDeletedEvent($entry));
+
         $em = $this->getDoctrine()->getManager();
         $em->remove($entry);
         $em->flush();

src/Wallabag/CoreBundle/Command/InstallCommand.php

@@ -370,7 +370,7 @@ class InstallCommand extends ContainerAwareCommand
             ],
             [
                 'name' => 'wallabag_url',
-                'value' => 'http://v2.wallabag.org',
+                'value' => '',
                 'section' => 'misc',
             ],
             [
@@ -398,6 +398,11 @@ class InstallCommand extends ContainerAwareCommand
                 'value' => 'wallabag',
                 'section' => 'misc',
             ],
+            [
+                'name' => 'download_images_enabled',
+                'value' => '0',
+                'section' => 'misc',
+            ],
         ];
 
         foreach ($settings as $setting) {

src/Wallabag/CoreBundle/Controller/EntryController.php

@@ -13,6 +13,8 @@ use Wallabag\CoreBundle\Form\Type\EntryFilterType;
 use Wallabag\CoreBundle\Form\Type\EditEntryType;
 use Wallabag\CoreBundle\Form\Type\NewEntryType;
 use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache;
+use Wallabag\CoreBundle\Event\EntrySavedEvent;
+use Wallabag\CoreBundle\Event\EntryDeletedEvent;
 
 class EntryController extends Controller
 {
@@ -81,6 +83,9 @@ class EntryController extends Controller
             $em->persist($entry);
             $em->flush();
 
+            // entry saved, dispatch event about it!
+            $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+
             return $this->redirect($this->generateUrl('homepage'));
         }
 
@@ -107,6 +112,9 @@ class EntryController extends Controller
             $em = $this->getDoctrine()->getManager();
             $em->persist($entry);
             $em->flush();
+
+            // entry saved, dispatch event about it!
+            $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
         }
 
         return $this->redirect($this->generateUrl('homepage'));
@@ -343,6 +351,9 @@ class EntryController extends Controller
         $em->persist($entry);
         $em->flush();
 
+        // entry saved, dispatch event about it!
+        $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+
         return $this->redirect($this->generateUrl('view', ['id' => $entry->getId()]));
     }
 
@@ -431,6 +442,9 @@ class EntryController extends Controller
             UrlGeneratorInterface::ABSOLUTE_PATH
         );
 
+        // entry deleted, dispatch event about it!
+        $this->get('event_dispatcher')->dispatch(EntryDeletedEvent::NAME, new EntryDeletedEvent($entry));
+
         $em = $this->getDoctrine()->getManager();
         $em->remove($entry);
         $em->flush();

src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php

@@ -140,6 +140,11 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
                 'value' => 'wallabag',
                 'section' => 'misc',
             ],
+            [
+                'name' => 'download_images_enabled',
+                'value' => '0',
+                'section' => 'misc',
+            ],
         ];
 
         foreach ($settings as $setting) {
@@ -158,6 +163,6 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
      */
     public function getOrder()
     {
-        return 50;
+        return 29;
     }
 }

src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace Wallabag\CoreBundle\Event;
+
+use Symfony\Component\EventDispatcher\Event;
+use Wallabag\CoreBundle\Entity\Entry;
+
+/**
+ * This event is fired as soon as an entry is deleted.
+ */
+class EntryDeletedEvent extends Event
+{
+    const NAME = 'entry.deleted';
+
+    protected $entry;
+
+    public function __construct(Entry $entry)
+    {
+        $this->entry = $entry;
+    }
+
+    public function getEntry()
+    {
+        return $this->entry;
+    }
+}

src/Wallabag/CoreBundle/Event/EntrySavedEvent.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace Wallabag\CoreBundle\Event;
+
+use Symfony\Component\EventDispatcher\Event;
+use Wallabag\CoreBundle\Entity\Entry;
+
+/**
+ * This event is fired as soon as an entry was saved.
+ */
+class EntrySavedEvent extends Event
+{
+    const NAME = 'entry.saved';
+
+    protected $entry;
+
+    public function __construct(Entry $entry)
+    {
+        $this->entry = $entry;
+    }
+
+    public function getEntry()
+    {
+        return $this->entry;
+    }
+}

src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Wallabag\CoreBundle\EventListener;
+namespace Wallabag\CoreBundle\Event\Listener;
 
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 use Symfony\Component\HttpKernel\Event\GetResponseEvent;

src/Wallabag/CoreBundle/Event/Listener/UserLocaleListener.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Wallabag\CoreBundle\EventListener;
+namespace Wallabag\CoreBundle\Event\Listener;
 
 use Symfony\Component\HttpFoundation\Session\Session;
 use Symfony\Component\Security\Http\Event\InteractiveLoginEvent;

src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php

@@ -0,0 +1,121 @@
+<?php
+
+namespace Wallabag\CoreBundle\Event\Subscriber;
+
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+use Psr\Log\LoggerInterface;
+use Wallabag\CoreBundle\Helper\DownloadImages;
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\CoreBundle\Event\EntrySavedEvent;
+use Wallabag\CoreBundle\Event\EntryDeletedEvent;
+use Doctrine\ORM\EntityManager;
+
+class DownloadImagesSubscriber implements EventSubscriberInterface
+{
+    private $em;
+    private $downloadImages;
+    private $enabled;
+    private $logger;
+
+    public function __construct(EntityManager $em, DownloadImages $downloadImages, $enabled, LoggerInterface $logger)
+    {
+        $this->em = $em;
+        $this->downloadImages = $downloadImages;
+        $this->enabled = $enabled;
+        $this->logger = $logger;
+    }
+
+    public static function getSubscribedEvents()
+    {
+        return [
+            EntrySavedEvent::NAME => 'onEntrySaved',
+            EntryDeletedEvent::NAME => 'onEntryDeleted',
+        ];
+    }
+
+    /**
+     * Download images and updated the data into the entry.
+     *
+     * @param EntrySavedEvent $event
+     */
+    public function onEntrySaved(EntrySavedEvent $event)
+    {
+        if (!$this->enabled) {
+            $this->logger->debug('DownloadImagesSubscriber: disabled.');
+
+            return;
+        }
+
+        $entry = $event->getEntry();
+
+        $html = $this->downloadImages($entry);
+        if (false !== $html) {
+            $this->logger->debug('DownloadImagesSubscriber: updated html.');
+
+            $entry->setContent($html);
+        }
+
+        // update preview picture
+        $previewPicture = $this->downloadPreviewImage($entry);
+        if (false !== $previewPicture) {
+            $this->logger->debug('DownloadImagesSubscriber: update preview picture.');
+
+            $entry->setPreviewPicture($previewPicture);
+        }
+
+        $this->em->persist($entry);
+        $this->em->flush();
+    }
+
+    /**
+     * Remove images related to the entry.
+     *
+     * @param EntryDeletedEvent $event
+     */
+    public function onEntryDeleted(EntryDeletedEvent $event)
+    {
+        if (!$this->enabled) {
+            $this->logger->debug('DownloadImagesSubscriber: disabled.');
+
+            return;
+        }
+
+        $this->downloadImages->removeImages($event->getEntry()->getId());
+    }
+
+    /**
+     * Download all images from the html.
+     *
+     * @todo If we want to add async download, it should be done in that method
+     *
+     * @param Entry $entry
+     *
+     * @return string|false False in case of async
+     */
+    private function downloadImages(Entry $entry)
+    {
+        return $this->downloadImages->processHtml(
+            $entry->getId(),
+            $entry->getContent(),
+            $entry->getUrl()
+        );
+    }
+
+    /**
+     * Download the preview picture.
+     *
+     * @todo If we want to add async download, it should be done in that method
+     *
+     * @param Entry $entry
+     *
+     * @return string|false False in case of async
+     */
+    private function downloadPreviewImage(Entry $entry)
+    {
+        return $this->downloadImages->processSingleImage(
+            $entry->getId(),
+            $entry->getPreviewPicture(),
+            $entry->getUrl()
+        );
+    }
+}

src/Wallabag/CoreBundle/Event/Subscriber/SQLiteCascadeDeleteSubscriber.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Wallabag\CoreBundle\Subscriber;
+namespace Wallabag\CoreBundle\Event\Subscriber;
 
 use Doctrine\Common\EventSubscriber;
 use Doctrine\ORM\Event\LifecycleEventArgs;

src/Wallabag/CoreBundle/Event/Subscriber/TablePrefixSubscriber.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Wallabag\CoreBundle\Subscriber;
+namespace Wallabag\CoreBundle\Event\Subscriber;
 
 use Doctrine\Common\EventSubscriber;
 use Doctrine\ORM\Event\LoadClassMetadataEventArgs;

src/Wallabag/CoreBundle/Helper/ContentProxy.php

@@ -3,7 +3,7 @@
 namespace Wallabag\CoreBundle\Helper;
 
 use Graby\Graby;
-use Psr\Log\LoggerInterface as Logger;
+use Psr\Log\LoggerInterface;
 use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Entity\Tag;
 use Wallabag\CoreBundle\Tools\Utils;
@@ -20,7 +20,7 @@ class ContentProxy
     protected $logger;
     protected $tagRepository;
 
-    public function __construct(Graby $graby, RuleBasedTagger $tagger, TagRepository $tagRepository, Logger $logger)
+    public function __construct(Graby $graby, RuleBasedTagger $tagger, TagRepository $tagRepository, LoggerInterface $logger)
     {
         $this->graby = $graby;
         $this->tagger = $tagger;
@@ -66,6 +66,7 @@ class ContentProxy
         $entry->setUrl($content['url'] ?: $url);
         $entry->setTitle($title);
         $entry->setContent($html);
+
         $entry->setLanguage($content['language']);
         $entry->setMimetype($content['content_type']);
         $entry->setReadingTime(Utils::getReadingTime($html));

src/Wallabag/CoreBundle/Helper/DownloadImages.php

@@ -0,0 +1,233 @@
+<?php
+
+namespace Wallabag\CoreBundle\Helper;
+
+use Psr\Log\LoggerInterface;
+use Symfony\Component\DomCrawler\Crawler;
+use GuzzleHttp\Client;
+use Symfony\Component\HttpFoundation\File\MimeType\MimeTypeExtensionGuesser;
+use Symfony\Component\Finder\Finder;
+
+class DownloadImages
+{
+    const REGENERATE_PICTURES_QUALITY = 80;
+
+    private $client;
+    private $baseFolder;
+    private $logger;
+    private $mimeGuesser;
+    private $wallabagUrl;
+
+    public function __construct(Client $client, $baseFolder, $wallabagUrl, LoggerInterface $logger)
+    {
+        $this->client = $client;
+        $this->baseFolder = $baseFolder;
+        $this->wallabagUrl = rtrim($wallabagUrl, '/');
+        $this->logger = $logger;
+        $this->mimeGuesser = new MimeTypeExtensionGuesser();
+
+        $this->setFolder();
+    }
+
+    /**
+     * Setup base folder where all images are going to be saved.
+     */
+    private function setFolder()
+    {
+        // if folder doesn't exist, attempt to create one and store the folder name in property $folder
+        if (!file_exists($this->baseFolder)) {
+            mkdir($this->baseFolder, 0777, true);
+        }
+    }
+
+    /**
+     * Process the html and extract image from it, save them to local and return the updated html.
+     *
+     * @param int    $entryId ID of the entry
+     * @param string $html
+     * @param string $url     Used as a base path for relative image and folder
+     *
+     * @return string
+     */
+    public function processHtml($entryId, $html, $url)
+    {
+        $crawler = new Crawler($html);
+        $result = $crawler
+            ->filterXpath('//img')
+            ->extract(array('src'));
+
+        $relativePath = $this->getRelativePath($entryId);
+
+        // download and save the image to the folder
+        foreach ($result as $image) {
+            $imagePath = $this->processSingleImage($entryId, $image, $url, $relativePath);
+
+            if (false === $imagePath) {
+                continue;
+            }
+
+            $html = str_replace($image, $imagePath, $html);
+        }
+
+        return $html;
+    }
+
+    /**
+     * Process a single image:
+     *     - retrieve it
+     *     - re-saved it (for security reason)
+     *     - return the new local path.
+     *
+     * @param int    $entryId      ID of the entry
+     * @param string $imagePath    Path to the image to retrieve
+     * @param string $url          Url from where the image were found
+     * @param string $relativePath Relative local path to saved the image
+     *
+     * @return string Relative url to access the image from the web
+     */
+    public function processSingleImage($entryId, $imagePath, $url, $relativePath = null)
+    {
+        if (null === $relativePath) {
+            $relativePath = $this->getRelativePath($entryId);
+        }
+
+        $this->logger->debug('DownloadImages: working on image: '.$imagePath);
+
+        $folderPath = $this->baseFolder.'/'.$relativePath;
+
+        // build image path
+        $absolutePath = $this->getAbsoluteLink($url, $imagePath);
+        if (false === $absolutePath) {
+            $this->logger->error('DownloadImages: Can not determine the absolute path for that image, skipping.');
+
+            return false;
+        }
+
+        try {
+            $res = $this->client->get($absolutePath);
+        } catch (\Exception $e) {
+            $this->logger->error('DownloadImages: Can not retrieve image, skipping.', ['exception' => $e]);
+
+            return false;
+        }
+
+        $ext = $this->mimeGuesser->guess($res->getHeader('content-type'));
+        $this->logger->debug('DownloadImages: Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]);
+        if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) {
+            $this->logger->error('DownloadImages: Processed image with not allowed extension. Skipping '.$imagePath);
+
+            return false;
+        }
+        $hashImage = hash('crc32', $absolutePath);
+        $localPath = $folderPath.'/'.$hashImage.'.'.$ext;
+
+        try {
+            $im = imagecreatefromstring($res->getBody());
+        } catch (\Exception $e) {
+            $im = false;
+        }
+
+        if (false === $im) {
+            $this->logger->error('DownloadImages: Error while regenerating image', ['path' => $localPath]);
+
+            return false;
+        }
+
+        switch ($ext) {
+            case 'gif':
+                $result = imagegif($im, $localPath);
+                $this->logger->debug('DownloadImages: Re-creating gif');
+                break;
+            case 'jpeg':
+            case 'jpg':
+                $result = imagejpeg($im, $localPath, self::REGENERATE_PICTURES_QUALITY);
+                $this->logger->debug('DownloadImages: Re-creating jpg');
+                break;
+            case 'png':
+                $result = imagepng($im, $localPath, ceil(self::REGENERATE_PICTURES_QUALITY / 100 * 9));
+                $this->logger->debug('DownloadImages: Re-creating png');
+        }
+
+        imagedestroy($im);
+
+        return $this->wallabagUrl.'/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext;
+    }
+
+    /**
+     * Remove all images for the given entry id.
+     *
+     * @param int $entryId ID of the entry
+     */
+    public function removeImages($entryId)
+    {
+        $relativePath = $this->getRelativePath($entryId);
+        $folderPath = $this->baseFolder.'/'.$relativePath;
+
+        $finder = new Finder();
+        $finder
+            ->files()
+            ->ignoreDotFiles(true)
+            ->in($folderPath);
+
+        foreach ($finder as $file) {
+            @unlink($file->getRealPath());
+        }
+
+        @rmdir($folderPath);
+    }
+
+    /**
+     * Generate the folder where we are going to save images based on the entry url.
+     *
+     * @param int $entryId ID of the entry
+     *
+     * @return string
+     */
+    private function getRelativePath($entryId)
+    {
+        $hashId = hash('crc32', $entryId);
+        $relativePath = $hashId[0].'/'.$hashId[1].'/'.$hashId;
+        $folderPath = $this->baseFolder.'/'.$relativePath;
+
+        if (!file_exists($folderPath)) {
+            mkdir($folderPath, 0777, true);
+        }
+
+        $this->logger->debug('DownloadImages: Folder used for that Entry id', ['folder' => $folderPath, 'entryId' => $entryId]);
+
+        return $relativePath;
+    }
+
+    /**
+     * Make an $url absolute based on the $base.
+     *
+     * @see Graby->makeAbsoluteStr
+     *
+     * @param string $base Base url
+     * @param string $url  Url to make it absolute
+     *
+     * @return false|string
+     */
+    private function getAbsoluteLink($base, $url)
+    {
+        if (preg_match('!^https?://!i', $url)) {
+            // already absolute
+            return $url;
+        }
+
+        $base = new \SimplePie_IRI($base);
+
+        // remove '//' in URL path (causes URLs not to resolve properly)
+        if (isset($base->ipath)) {
+            $base->ipath = preg_replace('!//+!', '/', $base->ipath);
+        }
+
+        if ($absolute = \SimplePie_IRI::absolutize($base, $url)) {
+            return $absolute->get_uri();
+        }
+
+        $this->logger->error('DownloadImages: Can not make an absolute link', ['base' => $base, 'url' => $url]);
+
+        return false;
+    }
+}

src/Wallabag/CoreBundle/Resources/config/services.yml

@@ -30,7 +30,7 @@ services:
             - "@doctrine"
 
     wallabag_core.subscriber.table_prefix:
-        class: Wallabag\CoreBundle\Subscriber\TablePrefixSubscriber
+        class: Wallabag\CoreBundle\Event\Subscriber\TablePrefixSubscriber
         arguments:
             - "%database_table_prefix%"
         tags:
@@ -131,8 +131,29 @@ services:
             - '%kernel.debug%'
 
     wallabag_core.subscriber.sqlite_cascade_delete:
-        class:  Wallabag\CoreBundle\Subscriber\SQLiteCascadeDeleteSubscriber
+        class: Wallabag\CoreBundle\Event\Subscriber\SQLiteCascadeDeleteSubscriber
         arguments:
             - "@doctrine"
         tags:
             - { name: doctrine.event_subscriber }
+
+    wallabag_core.subscriber.download_images:
+        class: Wallabag\CoreBundle\Event\Subscriber\DownloadImagesSubscriber
+        arguments:
+            - "@doctrine.orm.default_entity_manager"
+            - "@wallabag_core.entry.download_images"
+            - '@=service(''craue_config'').get(''download_images_enabled'')'
+            - "@logger"
+        tags:
+            - { name: kernel.event_subscriber }
+
+    wallabag_core.entry.download_images:
+        class: Wallabag\CoreBundle\Helper\DownloadImages
+        arguments:
+            - "@wallabag_core.entry.download_images.client"
+            - "%kernel.root_dir%/../web/assets/images"
+            - '@=service(''craue_config'').get(''wallabag_url'')'
+            - "@logger"
+
+    wallabag_core.entry.download_images.client:
+        class: GuzzleHttp\Client

src/Wallabag/CoreBundle/Resources/translations/messages.da.yml

@@ -368,6 +368,7 @@ import:
     #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
     worker:
         # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     # firefox:
     #    page_title: 'Import > Firefox'
     #    description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file."
@@ -505,3 +506,8 @@ flashes:
         notice:
             # client_created: 'New client created.'
             # client_deleted: 'Client deleted'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/CoreBundle/Resources/translations/messages.de.yml

@@ -368,6 +368,7 @@ import:
         how_to: 'Bitte wähle deinen Readability Export aus und klicke den unteren Button für das Hochladen und Importieren dessen.'
     worker:
         enabled: "Der Import erfolgt asynchron. Sobald der Import gestartet ist, wird diese Aufgabe extern abgearbeitet. Der aktuelle Service dafür ist:"
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     firefox:
         page_title: 'Aus Firefox importieren'
         description: "Dieser Import wird all deine Lesezeichen aus Firefox importieren. Gehe zu deinen Lesezeichen (Strg+Shift+O), dann auf \"Importen und Sichern\", wähle \"Sichern…\". Du erhälst eine .json Datei."
@@ -505,3 +506,8 @@ flashes:
         notice:
             client_created: 'Neuer Client erstellt.'
             client_deleted: 'Client gelöscht'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/CoreBundle/Resources/translations/messages.en.yml

@@ -368,6 +368,7 @@ import:
         how_to: 'Please select your Readability export and click on the below button to upload and import it.'
     worker:
         enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
+        download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     firefox:
         page_title: 'Import > Firefox'
         description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file."
@@ -471,6 +472,7 @@ flashes:
             rss_updated: 'RSS information updated'
             tagging_rules_updated: 'Tagging rules updated'
             tagging_rules_deleted: 'Tagging rule deleted'
+            # user_added: 'User "%username%" added'
             rss_token_updated: 'RSS token updated'
             annotations_reset: Annotations reset
             tags_reset: Tags reset

src/Wallabag/CoreBundle/Resources/translations/messages.es.yml

@@ -368,6 +368,7 @@ import:
         # how_to: 'Please select your Readability export and click on the below button to upload and import it.'
     worker:
         # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     firefox:
        page_title: 'Importar > Firefox'
        # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file."
@@ -505,3 +506,8 @@ flashes:
         notice:
             client_created: 'Nuevo cliente creado.'
             client_deleted: 'Cliente suprimido'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml

@@ -285,6 +285,7 @@ quickstart:
         paragraph_2: 'ادامه دهید!'
     configure:
         title: 'برنامه را تنظیم کنید'
+        # description: 'In order to have an application which suits you, have a look into the configuration of wallabag.'
         language: 'زبان و نمای برنامه را تغییر دهید'
         rss: 'خوراک آر-اس-اس را فعال کنید'
         tagging_rules: 'قانون‌های برچسب‌گذاری خودکار مقاله‌هایتان را تعریف کنید'
@@ -367,6 +368,7 @@ import:
         # how_to: 'Please select your Readability export and click on the below button to upload and import it.'
     worker:
         # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     firefox:
        page_title: 'درون‌ریزی > Firefox'
        # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file."
@@ -504,3 +506,8 @@ flashes:
         notice:
             # client_created: 'New client created.'
             # client_deleted: 'Client deleted'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml

@@ -368,6 +368,7 @@ import:
         how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer."
     worker:
         enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :"
+        download_images_warning: "Vous avez configuré le téléchagement des images pour vos articles. Combiné à l'import classique, cette opération peut être très très longue (voire échouer). Nous vous conseillons <strong>vivement</strong> d'activer les imports asynchrones."
     firefox:
         page_title: 'Import > Firefox'
         description: "Cet outil va vous permettre d'importer tous vos marques-pages de Firefox. Ouvrez le panneau des marques-pages (Ctrl+Maj+O), puis dans « Importation et sauvegarde », choisissez « Sauvegarde... ». Vous allez récupérer un fichier .json. </p>"
@@ -505,3 +506,8 @@ flashes:
         notice:
             client_created: 'Nouveau client %name% créé'
             client_deleted: 'Client %name% supprimé'
+    user:
+        notice:
+            added: 'Utilisateur "%username%" ajouté'
+            updated: 'Utilisateur "%username%" mis à jour'
+            deleted: 'Utilisateur "%username%" supprimé'

src/Wallabag/CoreBundle/Resources/translations/messages.it.yml

@@ -368,6 +368,7 @@ import:
         # how_to: 'Please select your Readability export and click on the below button to upload and import it.'
     worker:
         # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     firefox:
        page_title: 'Importa da > Firefox'
        # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file."
@@ -505,3 +506,8 @@ flashes:
         notice:
             client_created: 'Nuovo client creato.'
             client_deleted: 'Client eliminato'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml

@@ -368,6 +368,7 @@ import:
         how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar."
     worker:
         enabled: "L'importacion se fa de manièra asincròna. Un còp l'importacion lançada, una aisina externa s'ocuparà dels messatges un per un. Lo servici actual es : "
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     firefox:
         page_title: 'Importar > Firefox'
         description: "Aquesta aisina importarà totas vòstres favorits de Firefox. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file."
@@ -505,3 +506,8 @@ flashes:
         notice:
             client_created: 'Novèl client creat'
             client_deleted: 'Client suprimit'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml

@@ -368,6 +368,7 @@ import:
         how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.'
     worker:
         enabled: "Import jest wykonywany asynchronicznie. Od momentu rozpoczęcia importu, zewnętrzna usługa może zajmować się na raz tylko jednym zadaniem. Bieżącą usługą jest:"
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     firefox:
        page_title: 'Import > Firefox'
        description: "Ten importer zaimportuje wszystkie twoje zakładki z Firefoksa. Idź do twoich zakładek (Ctrl+Shift+O), następnie w \"Import i kopie zapasowe\", wybierz \"Utwórz kopię zapasową...\". Uzyskasz plik .json."
@@ -505,3 +506,8 @@ flashes:
         notice:
             client_created: 'Nowy klient utworzony.'
             client_deleted: 'Klient usunięty'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml

@@ -368,6 +368,7 @@ import:
     #     how_to: 'Please select your Readability export and click on the below button to upload and import it.'
     worker:
         # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     # firefox:
     #    page_title: 'Import > Firefox'
     #    description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file."
@@ -505,3 +506,8 @@ flashes:
         notice:
             # client_created: 'New client created.'
             # client_deleted: 'Client deleted'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml

@@ -109,6 +109,7 @@ config:
         # if_label: 'if'
         # then_tag_as_label: 'then tag as'
         # delete_rule_label: 'delete'
+        # edit_rule_label: 'edit'
         rule_label: 'Kural'
         tags_label: 'Etiketler'
         faq:
@@ -367,6 +368,7 @@ import:
         # how_to: 'Please select your Readability export and click on the below button to upload and import it.'
     worker:
         # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
+        # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We <strong>strongly recommend</strong> to enable asynchronous import to avoid errors."
     firefox:
        page_title: 'İçe Aktar > Firefox'
        # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file."
@@ -504,3 +506,8 @@ flashes:
         notice:
             # client_created: 'New client created.'
             # client_deleted: 'Client deleted'
+    user:
+        notice:
+            # added: 'User "%username%" added'
+            # updated: 'User "%username%" updated'
+            # deleted: 'User "%username%" deleted'

src/Wallabag/ImportBundle/Command/ImportCommand.php

@@ -14,10 +14,10 @@ class ImportCommand extends ContainerAwareCommand
     {
         $this
             ->setName('wallabag:import')
-            ->setDescription('Import entries from a JSON export from a wallabag v1 instance')
+            ->setDescription('Import entries from a JSON export')
             ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate')
             ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file')
-            ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2, firefox or chrome', 'v1')
+            ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, readability, firefox or chrome', 'v1')
             ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false)
         ;
     }
@@ -42,29 +42,35 @@ class ImportCommand extends ContainerAwareCommand
 
         switch ($input->getOption('importer')) {
             case 'v2':
-                $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import');
+                $import = $this->getContainer()->get('wallabag_import.wallabag_v2.import');
                 break;
             case 'firefox':
-                $wallabag = $this->getContainer()->get('wallabag_import.firefox.import');
+                $import = $this->getContainer()->get('wallabag_import.firefox.import');
                 break;
             case 'chrome':
-                $wallabag = $this->getContainer()->get('wallabag_import.chrome.import');
+                $import = $this->getContainer()->get('wallabag_import.chrome.import');
+                break;
+            case 'readability':
+                $import = $this->getContainer()->get('wallabag_import.readability.import');
+                break;
+            case 'instapaper':
+                $import = $this->getContainer()->get('wallabag_import.instapaper.import');
                 break;
             case 'v1':
             default:
-                $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import');
+                $import = $this->getContainer()->get('wallabag_import.wallabag_v1.import');
                 break;
         }
 
-        $wallabag->setMarkAsRead($input->getOption('markAsRead'));
-        $wallabag->setUser($user);
+        $import->setMarkAsRead($input->getOption('markAsRead'));
+        $import->setUser($user);
 
-        $res = $wallabag
+        $res = $import
             ->setFilepath($input->getArgument('filepath'))
             ->import();
 
         if (true === $res) {
-            $summary = $wallabag->getSummary();
+            $summary = $import->getSummary();
             $output->writeln('<info>'.$summary['imported'].' imported</info>');
             $output->writeln('<comment>'.$summary['skipped'].' already saved</comment>');
         }

src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php

@@ -9,6 +9,8 @@ use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Entity\Tag;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Wallabag\CoreBundle\Event\EntrySavedEvent;
 
 abstract class AbstractConsumer
 {
@@ -17,11 +19,12 @@ abstract class AbstractConsumer
     protected $import;
     protected $logger;
 
-    public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
+    public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, EventDispatcherInterface $eventDispatcher, LoggerInterface $logger = null)
     {
         $this->em = $em;
         $this->userRepository = $userRepository;
         $this->import = $import;
+        $this->eventDispatcher = $eventDispatcher;
         $this->logger = $logger ?: new NullLogger();
     }
 
@@ -59,6 +62,9 @@ abstract class AbstractConsumer
         try {
             $this->em->flush();
 
+            // entry saved, dispatch event about it!
+            $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+
             // clear only affected entities
             $this->em->clear(Entry::class);
             $this->em->clear(Tag::class);

src/Wallabag/ImportBundle/Import/AbstractImport.php

@@ -10,12 +10,15 @@ use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Entity\Tag;
 use Wallabag\UserBundle\Entity\User;
 use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Wallabag\CoreBundle\Event\EntrySavedEvent;
 
 abstract class AbstractImport implements ImportInterface
 {
     protected $em;
     protected $logger;
     protected $contentProxy;
+    protected $eventDispatcher;
     protected $producer;
     protected $user;
     protected $markAsRead;
@@ -23,11 +26,12 @@ abstract class AbstractImport implements ImportInterface
     protected $importedEntries = 0;
     protected $queuedEntries = 0;
 
-    public function __construct(EntityManager $em, ContentProxy $contentProxy)
+    public function __construct(EntityManager $em, ContentProxy $contentProxy, EventDispatcherInterface $eventDispatcher)
     {
         $this->em = $em;
         $this->logger = new NullLogger();
         $this->contentProxy = $contentProxy;
+        $this->eventDispatcher = $eventDispatcher;
     }
 
     public function setLogger(LoggerInterface $logger)
@@ -104,6 +108,7 @@ abstract class AbstractImport implements ImportInterface
     protected function parseEntries($entries)
     {
         $i = 1;
+        $entryToBeFlushed = [];
 
         foreach ($entries as $importedEntry) {
             if ($this->markAsRead) {
@@ -116,10 +121,21 @@ abstract class AbstractImport implements ImportInterface
                 continue;
             }
 
+            // store each entry to be flushed so we can trigger the entry.saved event for each of them
+            // entry.saved needs the entry to be persisted in db because it needs it id to generate
+            // images (at least)
+            $entryToBeFlushed[] = $entry;
+
             // flush every 20 entries
             if (($i % 20) === 0) {
                 $this->em->flush();
 
+                foreach ($entryToBeFlushed as $entry) {
+                    $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+                }
+
+                $entryToBeFlushed = [];
+
                 // clear only affected entities
                 $this->em->clear(Entry::class);
                 $this->em->clear(Tag::class);
@@ -128,6 +144,12 @@ abstract class AbstractImport implements ImportInterface
         }
 
         $this->em->flush();
+
+        if (!empty($entryToBeFlushed)) {
+            foreach ($entryToBeFlushed as $entry) {
+                $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+            }
+        }
     }
 
     /**

src/Wallabag/ImportBundle/Import/BrowserImport.php

@@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Import;
 use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\UserBundle\Entity\User;
 use Wallabag\CoreBundle\Helper\ContentProxy;
+use Wallabag\CoreBundle\Event\EntrySavedEvent;
 
 abstract class BrowserImport extends AbstractImport
 {
@@ -81,6 +82,7 @@ abstract class BrowserImport extends AbstractImport
     protected function parseEntries($entries)
     {
         $i = 1;
+        $entryToBeFlushed = [];
 
         foreach ($entries as $importedEntry) {
             if ((array) $importedEntry !== $importedEntry) {
@@ -93,14 +95,29 @@ abstract class BrowserImport extends AbstractImport
                 continue;
             }
 
+            // @see AbstractImport
+            $entryToBeFlushed[] = $entry;
+
             // flush every 20 entries
             if (($i % 20) === 0) {
                 $this->em->flush();
+
+                foreach ($entryToBeFlushed as $entry) {
+                    $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+                }
+
+                $entryToBeFlushed = [];
             }
             ++$i;
         }
 
         $this->em->flush();
+
+        if (!empty($entryToBeFlushed)) {
+            foreach ($entryToBeFlushed as $entry) {
+                $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+            }
+        }
     }
 
     /**

src/Wallabag/ImportBundle/Import/PocketImport.php

@@ -2,8 +2,6 @@
 
 namespace Wallabag\ImportBundle\Import;
 
-use Psr\Log\NullLogger;
-use Doctrine\ORM\EntityManager;
 use GuzzleHttp\Client;
 use GuzzleHttp\Exception\RequestException;
 use Wallabag\CoreBundle\Entity\Entry;
@@ -16,13 +14,6 @@ class PocketImport extends AbstractImport
 
     const NB_ELEMENTS = 5000;
 
-    public function __construct(EntityManager $em, ContentProxy $contentProxy)
-    {
-        $this->em = $em;
-        $this->contentProxy = $contentProxy;
-        $this->logger = new NullLogger();
-    }
-
     /**
      * Only used for test purpose.
      *

src/Wallabag/ImportBundle/Resources/config/rabbit.yml

@@ -6,6 +6,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.pocket.import"
+            - "@event_dispatcher"
             - "@logger"
     wallabag_import.consumer.amqp.readability:
         class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
@@ -13,6 +14,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.readability.import"
+            - "@event_dispatcher"
             - "@logger"
     wallabag_import.consumer.amqp.instapaper:
         class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
@@ -20,6 +22,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.instapaper.import"
+            - "@event_dispatcher"
             - "@logger"
     wallabag_import.consumer.amqp.wallabag_v1:
         class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
@@ -27,6 +30,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.wallabag_v1.import"
+            - "@event_dispatcher"
             - "@logger"
     wallabag_import.consumer.amqp.wallabag_v2:
         class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
@@ -34,6 +38,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.wallabag_v2.import"
+            - "@event_dispatcher"
             - "@logger"
     wallabag_import.consumer.amqp.firefox:
         class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
@@ -41,6 +46,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.firefox.import"
+            - "@event_dispatcher"
             - "@logger"
     wallabag_import.consumer.amqp.chrome:
         class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
@@ -48,4 +54,5 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.chrome.import"
+            - "@event_dispatcher"
             - "@logger"

src/Wallabag/ImportBundle/Resources/config/redis.yml

@@ -18,6 +18,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.readability.import"
+            - "@event_dispatcher"
             - "@logger"
 
     # instapaper
@@ -38,6 +39,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.instapaper.import"
+            - "@event_dispatcher"
             - "@logger"
 
     # pocket
@@ -58,6 +60,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.pocket.import"
+            - "@event_dispatcher"
             - "@logger"
 
     # wallabag v1
@@ -78,6 +81,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.wallabag_v1.import"
+            - "@event_dispatcher"
             - "@logger"
 
     # wallabag v2
@@ -98,6 +102,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.wallabag_v2.import"
+            - "@event_dispatcher"
             - "@logger"
 
     # firefox
@@ -118,6 +123,7 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.firefox.import"
+            - "@event_dispatcher"
             - "@logger"
 
     # chrome
@@ -138,4 +144,5 @@ services:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_user.user_repository"
             - "@wallabag_import.chrome.import"
+            - "@event_dispatcher"
             - "@logger"

src/Wallabag/ImportBundle/Resources/config/services.yml

@@ -20,7 +20,7 @@ services:
         arguments:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
-            - "@craue_config"
+            - "@event_dispatcher"
         calls:
             - [ setClient, [ "@wallabag_import.pocket.client" ] ]
             - [ setLogger, [ "@logger" ]]
@@ -32,6 +32,7 @@ services:
         arguments:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
+            - "@event_dispatcher"
         calls:
             - [ setLogger, [ "@logger" ]]
         tags:
@@ -42,6 +43,7 @@ services:
         arguments:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
+            - "@event_dispatcher"
         calls:
             - [ setLogger, [ "@logger" ]]
         tags:
@@ -52,6 +54,7 @@ services:
         arguments:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
+            - "@event_dispatcher"
         calls:
             - [ setLogger, [ "@logger" ]]
         tags:
@@ -62,6 +65,7 @@ services:
         arguments:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
+            - "@event_dispatcher"
         calls:
             - [ setLogger, [ "@logger" ]]
         tags:
@@ -72,6 +76,7 @@ services:
         arguments:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
+            - "@event_dispatcher"
         calls:
             - [ setLogger, [ "@logger" ]]
         tags:
@@ -81,6 +86,7 @@ services:
         arguments:
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
+            - "@event_dispatcher"
         calls:
             - [ setLogger, [ "@logger" ]]
         tags:

src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig

@@ -6,6 +6,8 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_information.html.twig' %}
+
             <div class="row">
                 <blockquote>{{ import.description|trans|raw }}</blockquote>
                 <p>{{ 'import.chrome.how_to'|trans }}</p>

src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig

@@ -6,6 +6,8 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_information.html.twig' %}
+
             <div class="row">
                 <blockquote>{{ import.description|trans|raw }}</blockquote>
                 <p>{{ 'import.firefox.how_to'|trans }}</p>

src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig

@@ -1,8 +1,15 @@
 {% set redis = craue_setting('import_with_redis') %}
 {% set rabbit = craue_setting('import_with_rabbitmq') %}
+{% set downloadImages = craue_setting('download_images_enabled') %}
 
 {% if redis or rabbit %}
     <div class="card-panel yellow darken-1 black-text">
         {{ 'import.worker.enabled'|trans }} <strong>{% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %}</strong>
     </div>
 {% endif %}
+
+{% if not redis and not rabbit and downloadImages %}
+    <div class="card-panel orange darken-1 black-text">
+        {{ 'import.worker.download_images_warning'|trans|raw }}
+    </div>
+{% endif %}

src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig

@@ -6,6 +6,8 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_information.html.twig' %}
+
             {{ 'import.page_description'|trans }}
             <ul>
                 {% for import in imports %}

src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig

@@ -6,7 +6,7 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
-            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+            {% include 'WallabagImportBundle:Import:_information.html.twig' %}
 
             <div class="row">
                 <blockquote>{{ import.description|trans }}</blockquote>

src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig

@@ -6,7 +6,7 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
-            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+            {% include 'WallabagImportBundle:Import:_information.html.twig' %}
 
             {% if not has_consumer_key %}
                 <div class="card-panel red white-text">

src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig

@@ -6,7 +6,7 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
-            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+            {% include 'WallabagImportBundle:Import:_information.html.twig' %}
 
             <div class="row">
                 <blockquote>{{ import.description|trans }}</blockquote>

src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig

@@ -6,7 +6,7 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
-            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+            {% include 'WallabagImportBundle:Import:_information.html.twig' %}
 
             <div class="row">
                 <blockquote>{{ import.description|trans }}</blockquote>

tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php

@@ -836,4 +836,64 @@ class EntryControllerTest extends WallabagCoreTestCase
         $client->request('GET', '/share/'.$content->getUuid());
         $this->assertEquals(404, $client->getResponse()->getStatusCode());
     }
+
+    public function testNewEntryWithDownloadImagesEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $url = 'http://www.20minutes.fr/montpellier/1952003-20161030-video-car-tombe-panne-rugbymen-perpignan-improvisent-melee-route';
+        $client->getContainer()->get('craue_config')->set('download_images_enabled', 1);
+
+        $crawler = $client->request('GET', '/new');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+
+        $form = $crawler->filter('form[name=entry]')->form();
+
+        $data = [
+            'entry[url]' => $url,
+        ];
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(302, $client->getResponse()->getStatusCode());
+
+        $em = $client->getContainer()
+            ->get('doctrine.orm.entity_manager');
+
+        $entry = $em
+            ->getRepository('WallabagCoreBundle:Entry')
+            ->findByUrlAndUserId($url, $this->getLoggedInUserId());
+
+        $this->assertInstanceOf('Wallabag\CoreBundle\Entity\Entry', $entry);
+        $this->assertEquals($url, $entry->getUrl());
+        $this->assertContains('Perpignan', $entry->getTitle());
+        $this->assertContains('/d9bc0fcd.jpeg', $entry->getContent());
+
+        $client->getContainer()->get('craue_config')->set('download_images_enabled', 0);
+    }
+
+    /**
+     * @depends testNewEntryWithDownloadImagesEnabled
+     */
+    public function testRemoveEntryWithDownloadImagesEnabled()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $url = 'http://www.20minutes.fr/montpellier/1952003-20161030-video-car-tombe-panne-rugbymen-perpignan-improvisent-melee-route';
+        $client->getContainer()->get('craue_config')->set('download_images_enabled', 1);
+
+        $content = $client->getContainer()
+            ->get('doctrine.orm.entity_manager')
+            ->getRepository('WallabagCoreBundle:Entry')
+            ->findByUrlAndUserId($url, $this->getLoggedInUserId());
+
+        $client->request('GET', '/delete/'.$content->getId());
+
+        $this->assertEquals(302, $client->getResponse()->getStatusCode());
+
+        $client->getContainer()->get('craue_config')->set('download_images_enabled', 0);
+    }
 }

tests/Wallabag/CoreBundle/Event/Listener/LocaleListenerTest.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Tests\Wallabag\CoreBundle\EventListener;
+namespace Tests\Wallabag\CoreBundle\Event\Listener;
 
 use Symfony\Component\EventDispatcher\EventDispatcher;
 use Symfony\Component\HttpFoundation\Request;
@@ -9,7 +9,7 @@ use Symfony\Component\HttpFoundation\Session\Storage\MockArraySessionStorage;
 use Symfony\Component\HttpKernel\Event\GetResponseEvent;
 use Symfony\Component\HttpKernel\HttpKernelInterface;
 use Symfony\Component\HttpKernel\KernelEvents;
-use Wallabag\CoreBundle\EventListener\LocaleListener;
+use Wallabag\CoreBundle\Event\Listener\LocaleListener;
 
 class LocaleListenerTest extends \PHPUnit_Framework_TestCase
 {

tests/Wallabag/CoreBundle/Event/Listener/UserLocaleListenerTest.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Tests\Wallabag\CoreBundle\EventListener;
+namespace Tests\Wallabag\CoreBundle\Event\Listener;
 
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Session\Session;
@@ -8,7 +8,7 @@ use Symfony\Component\HttpFoundation\Session\Storage\MockArraySessionStorage;
 use Symfony\Component\Security\Core\Authentication\Token\UsernamePasswordToken;
 use Symfony\Component\Security\Http\Event\InteractiveLoginEvent;
 use Wallabag\CoreBundle\Entity\Config;
-use Wallabag\CoreBundle\EventListener\UserLocaleListener;
+use Wallabag\CoreBundle\Event\Listener\UserLocaleListener;
 use Wallabag\UserBundle\Entity\User;
 
 class UserLocaleListenerTest extends \PHPUnit_Framework_TestCase

tests/Wallabag/CoreBundle/Event/Subscriber/TablePrefixSubscriberTest.php

@@ -1,11 +1,11 @@
 <?php
 
-namespace Tests\Wallabag\CoreBundle\Subscriber;
+namespace Tests\Wallabag\CoreBundle\Event\Subscriber;
 
 use Doctrine\Common\EventManager;
 use Doctrine\ORM\Event\LoadClassMetadataEventArgs;
 use Doctrine\ORM\Mapping\ClassMetadata;
-use Wallabag\CoreBundle\Subscriber\TablePrefixSubscriber;
+use Wallabag\CoreBundle\Event\Subscriber\TablePrefixSubscriber;
 
 class TablePrefixSubscriberTest extends \PHPUnit_Framework_TestCase
 {

tests/Wallabag/CoreBundle/Helper/DownloadImagesTest.php

@@ -0,0 +1,143 @@
+<?php
+
+namespace Tests\Wallabag\CoreBundle\Helper;
+
+use Wallabag\CoreBundle\Helper\DownloadImages;
+use Psr\Log\NullLogger;
+use Monolog\Logger;
+use Monolog\Handler\TestHandler;
+use GuzzleHttp\Client;
+use GuzzleHttp\Subscriber\Mock;
+use GuzzleHttp\Message\Response;
+use GuzzleHttp\Stream\Stream;
+
+class DownloadImagesTest extends \PHPUnit_Framework_TestCase
+{
+    public function testProcessHtml()
+    {
+        $client = new Client();
+
+        $mock = new Mock([
+            new Response(200, ['content-type' => 'image/png'], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $logHandler = new TestHandler();
+        $logger = new Logger('test', array($logHandler));
+
+        $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger);
+
+        $res = $download->processHtml(123, '<div><img src="http://i.imgur.com/T9qgcHc.jpg" /></div>', 'http://imgur.com/gallery/WxtWY');
+
+        $this->assertContains('http://wallabag.io/assets/images/9/b/9b0ead26/c638b4c2.png', $res);
+    }
+
+    public function testProcessHtmlWithBadImage()
+    {
+        $client = new Client();
+
+        $mock = new Mock([
+            new Response(200, ['content-type' => 'application/json'], Stream::factory('')),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $logHandler = new TestHandler();
+        $logger = new Logger('test', array($logHandler));
+
+        $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger);
+        $res = $download->processHtml(123, '<div><img src="http://i.imgur.com/T9qgcHc.jpg" /></div>', 'http://imgur.com/gallery/WxtWY');
+
+        $this->assertContains('http://i.imgur.com/T9qgcHc.jpg', $res, 'Image were not replace because of content-type');
+    }
+
+    public function singleImage()
+    {
+        return [
+            ['image/pjpeg', 'jpeg'],
+            ['image/jpeg', 'jpeg'],
+            ['image/png', 'png'],
+            ['image/gif', 'gif'],
+        ];
+    }
+
+    /**
+     * @dataProvider singleImage
+     */
+    public function testProcessSingleImage($header, $extension)
+    {
+        $client = new Client();
+
+        $mock = new Mock([
+            new Response(200, ['content-type' => $header], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $logHandler = new TestHandler();
+        $logger = new Logger('test', array($logHandler));
+
+        $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger);
+        $res = $download->processSingleImage(123, 'T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY');
+
+        $this->assertContains('/assets/images/9/b/9b0ead26/ebe60399.'.$extension, $res);
+    }
+
+    public function testProcessSingleImageWithBadUrl()
+    {
+        $client = new Client();
+
+        $mock = new Mock([
+            new Response(404, []),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $logHandler = new TestHandler();
+        $logger = new Logger('test', array($logHandler));
+
+        $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger);
+        $res = $download->processSingleImage(123, 'T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY');
+
+        $this->assertFalse($res, 'Image can not be found, so it will not be replaced');
+    }
+
+    public function testProcessSingleImageWithBadImage()
+    {
+        $client = new Client();
+
+        $mock = new Mock([
+            new Response(200, ['content-type' => 'image/png'], Stream::factory('')),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $logHandler = new TestHandler();
+        $logger = new Logger('test', array($logHandler));
+
+        $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger);
+        $res = $download->processSingleImage(123, 'http://i.imgur.com/T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY');
+
+        $this->assertFalse($res, 'Image can not be loaded, so it will not be replaced');
+    }
+
+    public function testProcessSingleImageFailAbsolute()
+    {
+        $client = new Client();
+
+        $mock = new Mock([
+            new Response(200, ['content-type' => 'image/png'], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))),
+        ]);
+
+        $client->getEmitter()->attach($mock);
+
+        $logHandler = new TestHandler();
+        $logger = new Logger('test', array($logHandler));
+
+        $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger);
+        $res = $download->processSingleImage(123, '/i.imgur.com/T9qgcHc.jpg', 'imgur.com/gallery/WxtWY');
+
+        $this->assertFalse($res, 'Absolute image can not be determined, so it will not be replaced');
+    }
+}

tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php

@@ -112,10 +112,19 @@ JSON;
             ->with(json_decode($body, true))
             ->willReturn($entry);
 
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->once())
+            ->method('dispatch');
+
         $consumer = new AMQPEntryConsumer(
             $em,
             $userRepository,
-            $import
+            $import,
+            $dispatcher
         );
 
         $message = new AMQPMessage($body);
@@ -157,10 +166,19 @@ JSON;
             ->disableOriginalConstructor()
             ->getMock();
 
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->never())
+            ->method('dispatch');
+
         $consumer = new AMQPEntryConsumer(
             $em,
             $userRepository,
-            $import
+            $import,
+            $dispatcher
         );
 
         $message = new AMQPMessage($body);
@@ -212,10 +230,19 @@ JSON;
             ->with(json_decode($body, true))
             ->willReturn(null);
 
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->never())
+            ->method('dispatch');
+
         $consumer = new AMQPEntryConsumer(
             $em,
             $userRepository,
-            $import
+            $import,
+            $dispatcher
         );
 
         $message = new AMQPMessage($body);

tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php

@@ -111,10 +111,19 @@ JSON;
             ->with(json_decode($body, true))
             ->willReturn($entry);
 
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->once())
+            ->method('dispatch');
+
         $consumer = new RedisEntryConsumer(
             $em,
             $userRepository,
-            $import
+            $import,
+            $dispatcher
         );
 
         $res = $consumer->manage($body);
@@ -156,10 +165,19 @@ JSON;
             ->disableOriginalConstructor()
             ->getMock();
 
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->never())
+            ->method('dispatch');
+
         $consumer = new RedisEntryConsumer(
             $em,
             $userRepository,
-            $import
+            $import,
+            $dispatcher
         );
 
         $res = $consumer->manage($body);
@@ -211,10 +229,19 @@ JSON;
             ->with(json_decode($body, true))
             ->willReturn(null);
 
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->never())
+            ->method('dispatch');
+
         $consumer = new RedisEntryConsumer(
             $em,
             $userRepository,
-            $import
+            $import,
+            $dispatcher
         );
 
         $res = $consumer->manage($body);

tests/Wallabag/ImportBundle/Import/ChromeImportTest.php

@@ -18,7 +18,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase
     protected $logHandler;
     protected $contentProxy;
 
-    private function getChromeImport($unsetUser = false)
+    private function getChromeImport($unsetUser = false, $dispatched = 0)
     {
         $this->user = new User();
 
@@ -30,7 +30,15 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $wallabag = new ChromeImport($this->em, $this->contentProxy);
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->exactly($dispatched))
+            ->method('dispatch');
+
+        $wallabag = new ChromeImport($this->em, $this->contentProxy, $dispatcher);
 
         $this->logHandler = new TestHandler();
         $logger = new Logger('test', [$this->logHandler]);
@@ -54,7 +62,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImport()
     {
-        $chromeImport = $this->getChromeImport();
+        $chromeImport = $this->getChromeImport(false, 1);
         $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
@@ -87,7 +95,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImportAndMarkAllAsRead()
     {
-        $chromeImport = $this->getChromeImport();
+        $chromeImport = $this->getChromeImport(false, 1);
         $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')

tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php

@@ -18,7 +18,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase
     protected $logHandler;
     protected $contentProxy;
 
-    private function getFirefoxImport($unsetUser = false)
+    private function getFirefoxImport($unsetUser = false, $dispatched = 0)
     {
         $this->user = new User();
 
@@ -30,7 +30,15 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $wallabag = new FirefoxImport($this->em, $this->contentProxy);
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->exactly($dispatched))
+            ->method('dispatch');
+
+        $wallabag = new FirefoxImport($this->em, $this->contentProxy, $dispatcher);
 
         $this->logHandler = new TestHandler();
         $logger = new Logger('test', [$this->logHandler]);
@@ -54,7 +62,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImport()
     {
-        $firefoxImport = $this->getFirefoxImport();
+        $firefoxImport = $this->getFirefoxImport(false, 2);
         $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
@@ -87,7 +95,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImportAndMarkAllAsRead()
     {
-        $firefoxImport = $this->getFirefoxImport();
+        $firefoxImport = $this->getFirefoxImport(false, 1);
         $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')

tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php

@@ -18,7 +18,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase
     protected $logHandler;
     protected $contentProxy;
 
-    private function getInstapaperImport($unsetUser = false)
+    private function getInstapaperImport($unsetUser = false, $dispatched = 0)
     {
         $this->user = new User();
 
@@ -30,7 +30,15 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $import = new InstapaperImport($this->em, $this->contentProxy);
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->exactly($dispatched))
+            ->method('dispatch');
+
+        $import = new InstapaperImport($this->em, $this->contentProxy, $dispatcher);
 
         $this->logHandler = new TestHandler();
         $logger = new Logger('test', [$this->logHandler]);
@@ -54,7 +62,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImport()
     {
-        $instapaperImport = $this->getInstapaperImport();
+        $instapaperImport = $this->getInstapaperImport(false, 3);
         $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
@@ -87,7 +95,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImportAndMarkAllAsRead()
     {
-        $instapaperImport = $this->getInstapaperImport();
+        $instapaperImport = $this->getInstapaperImport(false, 1);
         $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')

tests/Wallabag/ImportBundle/Import/PocketImportTest.php

@@ -24,7 +24,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
     protected $contentProxy;
     protected $logHandler;
 
-    private function getPocketImport($consumerKey = 'ConsumerKey')
+    private function getPocketImport($consumerKey = 'ConsumerKey', $dispatched = 0)
     {
         $this->user = new User();
 
@@ -55,10 +55,15 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
             ->method('getScheduledEntityInsertions')
             ->willReturn([]);
 
-        $pocket = new PocketImport(
-            $this->em,
-            $this->contentProxy
-        );
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->exactly($dispatched))
+            ->method('dispatch');
+
+        $pocket = new PocketImport($this->em, $this->contentProxy, $dispatcher);
         $pocket->setUser($this->user);
 
         $this->logHandler = new TestHandler();
@@ -252,7 +257,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
 
         $client->getEmitter()->attach($mock);
 
-        $pocketImport = $this->getPocketImport();
+        $pocketImport = $this->getPocketImport('ConsumerKey', 1);
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
             ->disableOriginalConstructor()
@@ -339,7 +344,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
 
         $client->getEmitter()->attach($mock);
 
-        $pocketImport = $this->getPocketImport();
+        $pocketImport = $this->getPocketImport('ConsumerKey', 2);
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
             ->disableOriginalConstructor()
@@ -591,7 +596,7 @@ JSON;
 
         $client->getEmitter()->attach($mock);
 
-        $pocketImport = $this->getPocketImport();
+        $pocketImport = $this->getPocketImport('ConsumerKey', 1);
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
             ->disableOriginalConstructor()

tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php

@@ -18,7 +18,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
     protected $logHandler;
     protected $contentProxy;
 
-    private function getReadabilityImport($unsetUser = false)
+    private function getReadabilityImport($unsetUser = false, $dispatched = 0)
     {
         $this->user = new User();
 
@@ -30,7 +30,15 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $wallabag = new ReadabilityImport($this->em, $this->contentProxy);
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->exactly($dispatched))
+            ->method('dispatch');
+
+        $wallabag = new ReadabilityImport($this->em, $this->contentProxy, $dispatcher);
 
         $this->logHandler = new TestHandler();
         $logger = new Logger('test', [$this->logHandler]);
@@ -54,7 +62,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImport()
     {
-        $readabilityImport = $this->getReadabilityImport();
+        $readabilityImport = $this->getReadabilityImport(false, 24);
         $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
@@ -87,7 +95,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImportAndMarkAllAsRead()
     {
-        $readabilityImport = $this->getReadabilityImport();
+        $readabilityImport = $this->getReadabilityImport(false, 1);
         $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability-read.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')

tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php

@@ -18,7 +18,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
     protected $logHandler;
     protected $contentProxy;
 
-    private function getWallabagV1Import($unsetUser = false)
+    private function getWallabagV1Import($unsetUser = false, $dispatched = 0)
     {
         $this->user = new User();
 
@@ -44,7 +44,15 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $wallabag = new WallabagV1Import($this->em, $this->contentProxy);
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->exactly($dispatched))
+            ->method('dispatch');
+
+        $wallabag = new WallabagV1Import($this->em, $this->contentProxy, $dispatcher);
 
         $this->logHandler = new TestHandler();
         $logger = new Logger('test', [$this->logHandler]);
@@ -68,7 +76,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImport()
     {
-        $wallabagV1Import = $this->getWallabagV1Import();
+        $wallabagV1Import = $this->getWallabagV1Import(false, 3);
         $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
@@ -101,7 +109,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImportAndMarkAllAsRead()
     {
-        $wallabagV1Import = $this->getWallabagV1Import();
+        $wallabagV1Import = $this->getWallabagV1Import(false, 3);
         $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1-read.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')

tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php

@@ -18,7 +18,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
     protected $logHandler;
     protected $contentProxy;
 
-    private function getWallabagV2Import($unsetUser = false)
+    private function getWallabagV2Import($unsetUser = false, $dispatched = 0)
     {
         $this->user = new User();
 
@@ -44,7 +44,15 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $wallabag = new WallabagV2Import($this->em, $this->contentProxy);
+        $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $dispatcher
+            ->expects($this->exactly($dispatched))
+            ->method('dispatch');
+
+        $wallabag = new WallabagV2Import($this->em, $this->contentProxy, $dispatcher);
 
         $this->logHandler = new TestHandler();
         $logger = new Logger('test', [$this->logHandler]);
@@ -68,7 +76,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImport()
     {
-        $wallabagV2Import = $this->getWallabagV2Import();
+        $wallabagV2Import = $this->getWallabagV2Import(false, 2);
         $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
@@ -97,7 +105,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImportAndMarkAllAsRead()
     {
-        $wallabagV2Import = $this->getWallabagV2Import();
+        $wallabagV2Import = $this->getWallabagV2Import(false, 2);
         $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2-read.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
@@ -246,7 +254,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
 
     public function testImportWithExceptionFromGraby()
     {
-        $wallabagV2Import = $this->getWallabagV2Import();
+        $wallabagV2Import = $this->getWallabagV2Import(false, 2);
         $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
 
         $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')