Merge pull request #1601 from wallabag/v2-import-from-v2

[#1590] Add JSON import from wallabag v2

docs/en/user/import.rst

@@ -1,8 +1,8 @@
 Migrate to wallabag
 ===================
 
-From wallabag 1.x
------------------
+From wallabag
+-------------
 
 Export your data from your wallabag 1.x
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -15,6 +15,17 @@ On your config page, click on ``JSON export`` in the ``Export your wallabag data
 
 You will have a ``wallabag-export-1-1970-01-01.json`` file.
 
+Export your data from your wallabag 2.x
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On the export sidebar, click on ``JSON``.
+
+.. image:: ../../img/user/export_wllbg_2.png
+:alt: Export from wallabag 2.x
+   :align: center
+
+You will have a ``Unread articles.json`` file.
+
 Import your data into wallabag 2.x
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -24,7 +35,7 @@ Click on  ``Import`` link in the menu, select your export file on your computer
    :alt: Import from wallabag 1.x
    :align: center
 
-All your wallabag 1.x articles will be imported.
+All your wallabag articles will be imported.
 
 From Pocket
 -----------
@@ -46,7 +57,7 @@ Import your data into wallabag 2.x
 
 Click on  ``Import`` link in the menu, on ``Import contents`` in Pocket section and then on ``Connect to Pocket and import data``.
 
-You need to authorize wallabag to interact with your Pocket account. Your data will be imported. Data import can be a demanding process for your server (we need to work on this import to improve it). 
+You need to authorize wallabag to interact with your Pocket account. Your data will be imported. Data import can be a demanding process for your server (we need to work on this import to improve it).
 
 From Instapaper
 ---------------

src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace Wallabag\ImportBundle\Controller;
+
+use Symfony\Bundle\FrameworkBundle\Controller\Controller;
+use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
+use Symfony\Component\HttpFoundation\Request;
+use Wallabag\ImportBundle\Form\Type\UploadImportType;
+
+class WallabagV2Controller extends Controller
+{
+    /**
+     * @Route("/wallabag-v2", name="import_wallabag_v2")
+     */
+    public function indexAction(Request $request)
+    {
+        $form = $this->createForm(UploadImportType::class);
+        $form->handleRequest($request);
+
+        $wallabag = $this->get('wallabag_import.wallabag_v2.import');
+
+        if ($form->isValid()) {
+            $file = $form->get('file')->getData();
+            $name = $this->getUser()->getId().'.json';
+
+            if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+                $res = $wallabag
+                    ->setUser($this->getUser())
+                    ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
+                    ->import();
+
+                $message = 'Import failed, please try again.';
+
+                if (true === $res) {
+                    $summary = $wallabag->getSummary();
+                    $message = 'Import summary: '.$summary['imported'].' imported, '.$summary['skipped'].' already saved.';
+
+                    unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
+                }
+
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    $message
+                );
+
+                return $this->redirect($this->generateUrl('homepage'));
+            } else {
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    'Error while processing import. Please verify your import file.'
+                );
+            }
+        }
+
+        return $this->render('WallabagImportBundle:WallabagV2:index.html.twig', [
+            'form' => $form->createView(),
+            'import' => $wallabag,
+        ]);
+    }
+}

src/Wallabag/ImportBundle/Import/WallabagV1Import.php

@@ -11,12 +11,12 @@ use Wallabag\CoreBundle\Tools\Utils;
 
 class WallabagV1Import implements ImportInterface
 {
-    private $user;
-    private $em;
-    private $logger;
-    private $skippedEntries = 0;
-    private $importedEntries = 0;
-    private $filepath;
+    protected $user;
+    protected $em;
+    protected $logger;
+    protected $skippedEntries = 0;
+    protected $importedEntries = 0;
+    protected $filepath;
 
     public function __construct(EntityManager $em)
     {
@@ -72,13 +72,13 @@ class WallabagV1Import implements ImportInterface
     public function import()
     {
         if (!$this->user) {
-            $this->logger->error('WallabagV1Import: user is not defined');
+            $this->logger->error('WallabagImport: user is not defined');
 
             return false;
         }
 
         if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
-            $this->logger->error('WallabagV1Import: unable to read file', array('filepath' => $this->filepath));
+            $this->logger->error('WallabagImport: unable to read file', array('filepath' => $this->filepath));
 
             return false;
         }
@@ -120,7 +120,7 @@ class WallabagV1Import implements ImportInterface
     /**
      * @param $entries
      */
-    private function parseEntries($entries)
+    protected function parseEntries($entries)
     {
         $i = 1;
 

src/Wallabag/ImportBundle/Import/WallabagV2Import.php

@@ -0,0 +1,75 @@
+<?php
+
+namespace Wallabag\ImportBundle\Import;
+
+use Wallabag\CoreBundle\Entity\Entry;
+
+class WallabagV2Import extends WallabagV1Import implements ImportInterface
+{
+    /**
+     * {@inheritdoc}
+     */
+    public function getName()
+    {
+        return 'wallabag v2';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getUrl()
+    {
+        return 'import_wallabag_v2';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getDescription()
+    {
+        return 'This importer will import all your wallabag v2 articles. On the export sidebar, click on "JSON". You will have a "Unread articles.json" file.';
+    }
+
+    /**
+     * @param $entries
+     */
+    protected function parseEntries($entries)
+    {
+        $i = 1;
+
+        foreach ($entries as $importedEntry) {
+            $existingEntry = $this->em
+                ->getRepository('WallabagCoreBundle:Entry')
+                ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
+
+            if (false !== $existingEntry) {
+                ++$this->skippedEntries;
+                continue;
+            }
+
+            // @see ContentProxy->updateEntry
+            $entry = new Entry($this->user);
+            $entry->setUrl($importedEntry['url']);
+            $entry->setTitle($importedEntry['title']);
+            $entry->setArchived($importedEntry['is_archived']);
+            $entry->setStarred($importedEntry['is_starred']);
+            $entry->setContent($importedEntry['content']);
+            $entry->setReadingTime($importedEntry['reading_time']);
+            $entry->setDomainName($importedEntry['domain_name']);
+            $entry->setMimetype($importedEntry['mimetype']);
+            $entry->setLanguage($importedEntry['language']);
+            $entry->setPreviewPicture($importedEntry['preview_picture']);
+
+            $this->em->persist($entry);
+            ++$this->importedEntries;
+
+            // flush every 20 entries
+            if (($i % 20) === 0) {
+                $this->em->flush();
+            }
+            ++$i;
+        }
+
+        $this->em->flush();
+    }
+}

src/Wallabag/ImportBundle/Resources/config/services.yml

@@ -32,3 +32,12 @@ services:
             - [ setLogger, [ "@logger" ]]
         tags:
             -  { name: wallabag_import.import, alias: wallabag_v1 }
+
+    wallabag_import.wallabag_v2.import:
+        class: Wallabag\ImportBundle\Import\WallabagV2Import
+        arguments:
+            - "@doctrine.orm.entity_manager"
+        calls:
+            - [ setLogger, [ "@logger" ]]
+        tags:
+            -  { name: wallabag_import.import, alias: wallabag_v2 }

src/Wallabag/ImportBundle/Resources/views/WallabagV2/index.html.twig

@@ -0,0 +1,2 @@
+{% extends "WallabagImportBundle:WallabagV1:index.html.twig" %}
+{% block title %}{% trans %}Import > Wallabag v2{% endtrans %}{% endblock %}

src/Wallabag/ImportBundle/Tests/Controller/ImportControllerTest.php

@@ -24,6 +24,6 @@ class ImportControllerTest extends WallabagCoreTestCase
         $crawler = $client->request('GET', '/import/');
 
         $this->assertEquals(200, $client->getResponse()->getStatusCode());
-        $this->assertEquals(2, $crawler->filter('blockquote')->count());
+        $this->assertEquals(3, $crawler->filter('blockquote')->count());
     }
 }

src/Wallabag/ImportBundle/Tests/Controller/WallabagV2ControllerTest.php

@@ -0,0 +1,69 @@
+<?php
+
+namespace Wallabag\ImportBundle\Tests\Controller;
+
+use Wallabag\CoreBundle\Tests\WallabagCoreTestCase;
+use Symfony\Component\HttpFoundation\File\UploadedFile;
+
+class WallabagV2ControllerTest extends WallabagCoreTestCase
+{
+    public function testImportWallabag()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $crawler = $client->request('GET', '/import/wallabag-v2');
+
+        $this->assertEquals(200, $client->getResponse()->getStatusCode());
+        $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+        $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+    }
+
+    public function testImportWallabagWithFile()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $crawler = $client->request('GET', '/import/wallabag-v2');
+        $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+        $file = new UploadedFile(__DIR__.'/../fixtures/wallabag-v2.json', 'wallabag-v2.json');
+
+        $data = array(
+            'upload_import_file[file]' => $file,
+        );
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(302, $client->getResponse()->getStatusCode());
+
+        $crawler = $client->followRedirect();
+
+        $this->assertGreaterThan(1, $alert = $crawler->filter('div.messages.success')->extract(array('_text')));
+        $this->assertContains('Import summary', $alert[0]);
+    }
+
+    public function testImportWallabagWithEmptyFile()
+    {
+        $this->logInAs('admin');
+        $client = $this->getClient();
+
+        $crawler = $client->request('GET', '/import/wallabag-v2');
+        $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
+
+        $file = new UploadedFile(__DIR__.'/../fixtures/test.txt', 'test.txt');
+
+        $data = array(
+            'upload_import_file[file]' => $file,
+        );
+
+        $client->submit($form, $data);
+
+        $this->assertEquals(302, $client->getResponse()->getStatusCode());
+
+        $crawler = $client->followRedirect();
+
+        $this->assertGreaterThan(1, $alert = $crawler->filter('div.messages.success')->extract(array('_text')));
+        $this->assertContains('Import failed, please try again', $alert[0]);
+    }
+}

src/Wallabag/ImportBundle/Tests/Import/WallabagV1ImportTest.php

@@ -21,17 +21,17 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $pocket = new WallabagV1Import($this->em);
+        $wallabag = new WallabagV1Import($this->em);
 
         $this->logHandler = new TestHandler();
         $logger = new Logger('test', array($this->logHandler));
-        $pocket->setLogger($logger);
+        $wallabag->setLogger($logger);
 
         if (false === $unsetUser) {
-            $pocket->setUser($this->user);
+            $wallabag->setUser($this->user);
         }
 
-        return $pocket;
+        return $wallabag;
     }
 
     public function testInit()
@@ -77,7 +77,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
         $this->assertFalse($res);
 
         $records = $this->logHandler->getRecords();
-        $this->assertContains('WallabagV1Import: unable to read file', $records[0]['message']);
+        $this->assertContains('WallabagImport: unable to read file', $records[0]['message']);
         $this->assertEquals('ERROR', $records[0]['level_name']);
     }
 
@@ -91,7 +91,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
         $this->assertFalse($res);
 
         $records = $this->logHandler->getRecords();
-        $this->assertContains('WallabagV1Import: user is not defined', $records[0]['message']);
+        $this->assertContains('WallabagImport: user is not defined', $records[0]['message']);
         $this->assertEquals('ERROR', $records[0]['level_name']);
     }
 }

src/Wallabag/ImportBundle/Tests/Import/WallabagV2ImportTest.php

@@ -0,0 +1,97 @@
+<?php
+
+namespace Wallabag\ImportBundle\Tests\Import;
+
+use Wallabag\ImportBundle\Import\WallabagV2Import;
+use Wallabag\UserBundle\Entity\User;
+use Monolog\Logger;
+use Monolog\Handler\TestHandler;
+
+class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
+{
+    protected $user;
+    protected $em;
+    protected $logHandler;
+
+    private function getWallabagV2Import($unsetUser = false)
+    {
+        $this->user = new User();
+
+        $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $wallabag = new WallabagV2Import($this->em);
+
+        $this->logHandler = new TestHandler();
+        $logger = new Logger('test', array($this->logHandler));
+        $wallabag->setLogger($logger);
+
+        if (false === $unsetUser) {
+            $wallabag->setUser($this->user);
+        }
+
+        return $wallabag;
+    }
+
+    public function testInit()
+    {
+        $wallabagV2Import = $this->getWallabagV2Import();
+
+        $this->assertEquals('wallabag v2', $wallabagV2Import->getName());
+        $this->assertNotEmpty($wallabagV2Import->getUrl());
+        $this->assertContains('This importer will import all your wallabag v2 articles.', $wallabagV2Import->getDescription());
+    }
+
+    public function testImport()
+    {
+        $wallabagV2Import = $this->getWallabagV2Import();
+        $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->exactly(2))
+            ->method('findByUrlAndUserId')
+            ->will($this->onConsecutiveCalls(false, true, false));
+
+        $this->em
+            ->expects($this->any())
+            ->method('getRepository')
+            ->willReturn($entryRepo);
+
+        $res = $wallabagV2Import->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 1, 'imported' => 1], $wallabagV2Import->getSummary());
+    }
+
+    public function testImportBadFile()
+    {
+        $wallabagV1Import = $this->getWallabagV2Import();
+        $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.jsonx');
+
+        $res = $wallabagV1Import->import();
+
+        $this->assertFalse($res);
+
+        $records = $this->logHandler->getRecords();
+        $this->assertContains('WallabagImport: unable to read file', $records[0]['message']);
+        $this->assertEquals('ERROR', $records[0]['level_name']);
+    }
+
+    public function testImportUserNotDefined()
+    {
+        $wallabagV1Import = $this->getWallabagV2Import(true);
+        $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
+
+        $res = $wallabagV1Import->import();
+
+        $this->assertFalse($res);
+
+        $records = $this->logHandler->getRecords();
+        $this->assertContains('WallabagImport: user is not defined', $records[0]['message']);
+        $this->assertEquals('ERROR', $records[0]['level_name']);
+    }
+}