fix #1502 avoid duplicate entry and store pocket url in config

app/config/config.yml

@@ -33,6 +33,14 @@ wallabag_core:
     import:
         allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain']
 
+wallabag_import:
+    importers:
+        pocket_urls:
+            oauth_request: https://getpocket.com/v3/oauth/request
+            auth_authorize: https://getpocket.com/auth/authorize
+            oauth_authorize: https://getpocket.com/v3/oauth/authorize
+            get: https://getpocket.com/v3/get
+
 # Twig Configuration
 twig:
     debug:            "%kernel.debug%"

src/Wallabag/CoreBundle/Controller/EntryController.php

@@ -41,6 +41,7 @@ class EntryController extends Controller
      */
     public function addEntryFormAction(Request $request)
     {
+        $em = $this->getDoctrine()->getManager();
         $entry = new Entry($this->getUser());
 
         $form = $this->createForm(new NewEntryType(), $entry);
@@ -48,6 +49,19 @@ class EntryController extends Controller
         $form->handleRequest($request);
 
         if ($form->isValid()) {
+            $existingEntry = $em
+                ->getRepository('WallabagCoreBundle:Entry')
+                ->findOneByUrl($entry->getUrl());
+
+            if (!is_null($existingEntry)) {
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    'Entry already saved on '.$existingEntry->getCreatedAt()->format('d-m-Y')
+                );
+
+                return $this->redirect($this->generateUrl('view', array('id' => $existingEntry->getId())));
+            }
+
             $this->updateEntry($entry);
             $this->get('session')->getFlashBag()->add(
                 'notice',

src/Wallabag/ImportBundle/DependencyInjection/Configuration.php

@@ -2,6 +2,7 @@
 
 namespace Wallabag\ImportBundle\DependencyInjection;
 
+use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
 use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 use Symfony\Component\Config\Definition\ConfigurationInterface;
 
@@ -12,6 +13,22 @@ class Configuration implements ConfigurationInterface
         $treeBuilder = new TreeBuilder();
         $rootNode = $treeBuilder->root('wallabag_import');
 
+        $rootNode
+            ->children()
+                ->arrayNode('importers')
+                    ->append($this->getURLs())
+                ->end()
+            ->end()
+        ;
+
         return $treeBuilder;
     }
+
+    private function getURLs()
+    {
+        $node = new ArrayNodeDefinition('pocket_urls');
+        $node->prototype('scalar')->end();
+
+        return $node;
+    }
 }

src/Wallabag/ImportBundle/DependencyInjection/WallabagImportExtension.php

@@ -13,6 +13,7 @@ class WallabagImportExtension extends Extension
     {
         $configuration = new Configuration();
         $config = $this->processConfiguration($configuration, $configs);
+        $container->setParameter('wallabag_import.pocket', $config['importers']['pocket_urls']);
 
         $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config'));
         $loader->load('services.yml');

src/Wallabag/ImportBundle/Import/PocketImport.php

@@ -15,13 +15,19 @@ class PocketImport implements ImportInterface
     private $session;
     private $em;
     private $consumerKey;
+    private $skippedEntries;
+    private $importedEntries;
+    private $pocketURL;
 
-    public function __construct($tokenStorage, Session $session, EntityManager $em, $consumerKey)
+    public function __construct($tokenStorage, Session $session, EntityManager $em, $consumerKey, $pocketURL)
     {
         $this->user = $tokenStorage->getToken()->getUser();
         $this->session = $session;
         $this->em = $em;
         $this->consumerKey = $consumerKey;
+        $this->skippedEntries = 0;
+        $this->importedEntries = 0;
+        $this->pocketURL = $pocketURL;
     }
 
     public function getName()
@@ -64,9 +70,25 @@ class PocketImport implements ImportInterface
             return $pocketEntry['resolved_title'];
         } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
             return $pocketEntry['given_title'];
-        } else {
-            return 'Untitled';
         }
+
+        return 'Untitled';
+    }
+
+    /**
+     * Returns the good URL for current entry.
+     *
+     * @param $pocketEntry
+     *
+     * @return string
+     */
+    private function guessURL($pocketEntry)
+    {
+        if (isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '') {
+            return $pocketEntry['resolved_url'];
+        }
+
+        return $pocketEntry['given_url'];
     }
 
     private function assignTagsToEntry(Entry $entry, $tags)
@@ -95,7 +117,20 @@ class PocketImport implements ImportInterface
     {
         foreach ($entries as $pocketEntry) {
             $entry = new Entry($this->user);
-            $entry->setUrl($pocketEntry['given_url']);
+            $url = $this->guessURL($pocketEntry);
+
+            $existingEntry = $this->em
+                ->getRepository('WallabagCoreBundle:Entry')
+                ->findOneByUrl($url);
+
+            if (!is_null($existingEntry)) {
+                ++$this->skippedEntries;
+                continue;
+            }
+
+            $entry->setUrl($url);
+            $entry->setDomainName(parse_url($url, PHP_URL_HOST));
+
             if ($pocketEntry['status'] == 1) {
                 $entry->setArchived(true);
             }
@@ -118,20 +153,20 @@ class PocketImport implements ImportInterface
             }
 
             if (!empty($pocketEntry['tags'])) {
-                $this->assignTagsToEntry($entry, $pocketEntry['tags']);
+                //   $this->assignTagsToEntry($entry, $pocketEntry['tags']);
             }
 
             $this->em->persist($entry);
+            ++$this->importedEntries;
         }
 
-        $this->user->setLastPocketImport(new \DateTime());
         $this->em->flush();
     }
 
     public function oAuthRequest($redirectUri, $callbackUri)
     {
         $client = $this->createClient();
-        $request = $client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
+        $request = $client->createRequest('POST', $this->pocketURL['oauth_request'],
             [
                 'body' => json_encode([
                     'consumer_key' => $this->consumerKey,
@@ -146,14 +181,14 @@ class PocketImport implements ImportInterface
         // store code in session for callback method
         $this->session->set('pocketCode', $values['code']);
 
-        return 'https://getpocket.com/auth/authorize?request_token='.$values['code'].'&redirect_uri='.$callbackUri;
+        return $this->pocketURL['auth_authorize'].'?request_token='.$values['code'].'&redirect_uri='.$callbackUri;
     }
 
     public function oAuthAuthorize()
     {
         $client = $this->createClient();
 
-        $request = $client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
+        $request = $client->createRequest('POST', $this->pocketURL['oauth_authorize'],
             [
                 'body' => json_encode([
                     'consumer_key' => $this->consumerKey,
@@ -170,9 +205,8 @@ class PocketImport implements ImportInterface
     public function import($accessToken)
     {
         $client = $this->createClient();
-        $since = (!is_null($this->user->getLastPocketImport()) ? $this->user->getLastPocketImport()->getTimestamp() : '');
 
-        $request = $client->createRequest('POST', 'https://getpocket.com/v3/get',
+        $request = $client->createRequest('POST', $this->pocketURL['get'],
             [
                 'body' => json_encode([
                     'consumer_key' => $this->consumerKey,
@@ -180,7 +214,6 @@ class PocketImport implements ImportInterface
                     'detailType' => 'complete',
                     'state' => 'all',
                     'sort' => 'oldest',
-                    'since' => $since,
                 ]),
             ]
         );
@@ -192,7 +225,7 @@ class PocketImport implements ImportInterface
 
         $this->session->getFlashBag()->add(
             'notice',
-            count($entries['list']).' entries imported'
+            $this->importedEntries.' entries imported, '.$this->skippedEntries.' already saved.'
         );
     }
 }

src/Wallabag/ImportBundle/Resources/config/services.yml

@@ -6,3 +6,4 @@ services:
             - @session
             - @doctrine.orm.entity_manager
             - %pocket_consumer_key%
+            - %wallabag_import.pocket%

src/Wallabag/UserBundle/Entity/User.php

@@ -84,13 +84,6 @@ class User extends BaseUser implements TwoFactorInterface, TrustedComputerInterf
      */
     private $trusted;
 
-    /**
-     * @var date
-     *
-     * @ORM\Column(name="last_pocket_import", type="datetime", nullable=true)
-     */
-    private $lastPocketImport;
-
     public function __construct()
     {
         parent::__construct();
@@ -247,20 +240,4 @@ class User extends BaseUser implements TwoFactorInterface, TrustedComputerInterf
 
         return false;
     }
-
-    /**
-     * @return date
-     */
-    public function getLastPocketImport()
-    {
-        return $this->lastPocketImport;
-    }
-
-    /**
-     * @param date $lastPocketImport
-     */
-    public function setLastPocketImport($lastPocketImport)
-    {
-        $this->lastPocketImport = $lastPocketImport;
-    }
 }