Keep url in exists endpoint

- Add migration
- Use md5 instead of sha512 (we don't need security here, just a hash)
- Update tests

app/DoctrineMigrations/Version20190401105353.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace Application\Migrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Wallabag\CoreBundle\Doctrine\WallabagMigration;
+
+/**
+ * Add hashed_url in entry.
+ */
+class Version20190401105353 extends WallabagMigration
+{
+    /**
+     * @param Schema $schema
+     */
+    public function up(Schema $schema)
+    {
+        $entryTable = $schema->getTable($this->getTable('entry'));
+
+        $this->skipIf($entryTable->hasColumn('hashed_url'), 'It seems that you already played this migration.');
+
+        $entryTable->addColumn('hashed_url', 'text', [
+            'length' => 32,
+            'notnull' => false,
+        ]);
+
+        // sqlite doesn't have the MD5 function by default
+        if ('sqlite' !== $this->connection->getDatabasePlatform()->getName()) {
+            $this->addSql('UPDATE ' . $this->getTable('entry') . ' SET hashed_url = MD5(url)');
+        }
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function down(Schema $schema)
+    {
+        $entryTable = $schema->getTable($this->getTable('entry'));
+
+        $this->skipIf(!$entryTable->hasColumn('hashed_url'), 'It seems that you already played this migration.');
+
+        $entryTable->dropColumn('hashed_url');
+    }
+}

src/Wallabag/ApiBundle/Controller/EntryRestController.php

@@ -27,10 +27,10 @@ class EntryRestController extends WallabagRestController
      * @ApiDoc(
      *       parameters={
      *          {"name"="return_id", "dataType"="string", "required"=false, "format"="1 or 0", "description"="Set 1 if you want to retrieve ID in case entry(ies) exists, 0 by default"},
-     *          {"name"="url", "dataType"="string", "required"=true, "format"="An url", "description"="Url to check if it exists"},
-     *          {"name"="urls", "dataType"="string", "required"=false, "format"="An array of urls (?urls[]=http...&urls[]=http...)", "description"="Urls (as an array) to check if it exists"}
-     *          {"name"="hashedurl", "dataType"="string", "required"=true, "format"="An url", "description"="Md5 url to check if it exists"},
-     *          {"name"="hashedurls", "dataType"="string", "required"=false, "format"="An array of urls (?urls[]=http...&urls[]=http...)", "description"="Md5 urls (as an array) to check if it exists"}
+     *          {"name"="url", "dataType"="string", "required"=true, "format"="An url", "description"="DEPRECATED, use hashed_url instead"},
+     *          {"name"="urls", "dataType"="string", "required"=false, "format"="An array of urls (?urls[]=http...&urls[]=http...)", "description"="DEPRECATED, use hashed_urls instead"},
+     *          {"name"="hashed_url", "dataType"="string", "required"=true, "format"="An url", "description"="Md5 url to check if it exists"},
+     *          {"name"="hashed_urls", "dataType"="string", "required"=false, "format"="An array of urls (?urls[]=http...&urls[]=http...)", "description"="Md5 urls (as an array) to check if it exists"}
      *       }
      * )
      *
@@ -39,22 +39,18 @@ class EntryRestController extends WallabagRestController
     public function getEntriesExistsAction(Request $request)
     {
         $this->validateAuthentication();
+        $repo = $this->getDoctrine()->getRepository('WallabagCoreBundle:Entry');
 
         $returnId = (null === $request->query->get('return_id')) ? false : (bool) $request->query->get('return_id');
-        $urls = $request->query->get('urls', []);
 
-        $hashedUrls = $request->query->get('hashedurls', []);
+        $urls = $request->query->get('urls', []);
+        $hashedUrls = $request->query->get('hashed_urls', []);
 
         // handle multiple urls first
         if (!empty($hashedUrls)) {
             $results = [];
             foreach ($hashedUrls as $hashedUrl) {
-                $res = $this->getDoctrine()
-                    ->getRepository('WallabagCoreBundle:Entry')
-                    ->findOneBy([
-                        'hashedUrl' => $hashedUrl,
-                        'user' => $this->getUser()->getId(),
-                    ]);
+                $res = $repo->findByHashedUrlAndUserId($hashedUrl, $this->getUser()->getId());
 
                 // $results[$url] = $this->returnExistInformation($res, $returnId);
                 $results[$hashedUrl] = $this->returnExistInformation($res, $returnId);
@@ -63,24 +59,33 @@ class EntryRestController extends WallabagRestController
             return $this->sendResponse($results);
         }
 
-        // let's see if it is a simple url?
-        $hashedUrl = $request->query->get('hashedurl', '');
+        // @deprecated, to be remove in 3.0
+        if (!empty($urls)) {
+            $results = [];
+            foreach ($urls as $url) {
+                $res = $repo->findByUrlAndUserId($url, $this->getUser()->getId());
 
-        // if (empty($url)) {
-        //     throw $this->createAccessDeniedException('URL is empty?, logged user id: ' . $this->getUser()->getId());
-        // }
-
-        if (empty($hashedUrl)) {
-            throw $this->createAccessDeniedException('URL is empty?, logged user id: '.$this->getUser()->getId());
+                $results[$url] = $this->returnExistInformation($res, $returnId);
             }
 
-        $res = $this->getDoctrine()
-            ->getRepository('WallabagCoreBundle:Entry')
-            // ->findByUrlAndUserId($url, $this->getUser()->getId());
-            ->findOneBy([
-                'hashedUrl' => $hashedUrl,
-                'user' => $this->getUser()->getId(),
-            ]);
+            return $this->sendResponse($results);
+        }
+
+        // let's see if it is a simple url?
+        $url = $request->query->get('url', '');
+        $hashedUrl = $request->query->get('hashed_url', '');
+
+        if (empty($url) && empty($hashedUrl)) {
+            throw $this->createAccessDeniedException('URL is empty?, logged user id: ' . $this->getUser()->getId());
+        }
+
+        $method = 'findByUrlAndUserId';
+        if (!empty($hashedUrl)) {
+            $method = 'findByHashedUrlAndUserId';
+            $url = $hashedUrl;
+        }
+
+        $res = $repo->$method($url, $this->getUser()->getId());
 
         return $this->sendResponse(['exists' => $this->returnExistInformation($res, $returnId)]);
     }

src/Wallabag/CoreBundle/Command/GenerateUrlHashesCommand.php

@@ -45,13 +45,13 @@ class GenerateUrlHashesCommand extends ContainerAwareCommand
         } else {
             $users = $this->getDoctrine()->getRepository('WallabagUserBundle:User')->findAll();
 
-            $output->writeln(sprintf('Generating hashed urls for the %d user account entries', count($users)));
+            $output->writeln(sprintf('Generating hashed urls for "%d" users', \count($users)));
 
             foreach ($users as $user) {
-                $output->writeln(sprintf('Processing user %s', $user->getUsername()));
+                $output->writeln(sprintf('Processing user: %s', $user->getUsername()));
                 $this->generateHashedUrls($user);
             }
-            $output->writeln(sprintf('Finished generated hashed urls'));
+            $output->writeln('Finished generated hashed urls');
         }
 
         return 0;
@@ -67,13 +67,20 @@ class GenerateUrlHashesCommand extends ContainerAwareCommand
 
         $entries = $repo->findByUser($user->getId());
 
+        $i = 1;
         foreach ($entries as $entry) {
-            $entry->setHashedUrl(hash('sha512', $entry->getUrl()));
+            $entry->setHashedUrl(hash('md5', $entry->getUrl()));
             $em->persist($entry);
+
+            if (0 === ($i % 20)) {
                 $em->flush();
             }
+            ++$i;
+        }
 
-        $this->output->writeln(sprintf('Generated hashed urls for user %s', $user->getUserName()));
+        $em->flush();
+
+        $this->output->writeln(sprintf('Generated hashed urls for user: %s', $user->getUserName()));
     }
 
     /**

src/Wallabag/CoreBundle/DataFixtures/EntryFixtures.php

@@ -30,7 +30,6 @@ class EntryFixtures extends Fixture implements DependentFixtureInterface
             'entry2' => [
                 'user' => 'admin-user',
                 'url' => 'http://0.0.0.0/entry2',
-                'hashed_url' => hash('md5', 'http://0.0.0.0/entry2'),
                 'reading_time' => 1,
                 'domain' => 'domain.io',
                 'mime' => 'text/html',
@@ -90,6 +89,7 @@ class EntryFixtures extends Fixture implements DependentFixtureInterface
         foreach ($entries as $reference => $item) {
             $entry = new Entry($this->getReference($item['user']));
             $entry->setUrl($item['url']);
+            $entry->setHashedUrl(hash('md5', $item['url']));
             $entry->setReadingTime($item['reading_time']);
             $entry->setDomainName($item['domain']);
             $entry->setMimetype($item['mime']);

src/Wallabag/CoreBundle/Entity/Entry.php

@@ -26,7 +26,7 @@ use Wallabag\UserBundle\Entity\User;
  *     indexes={
  *         @ORM\Index(name="created_at", columns={"created_at"}),
  *         @ORM\Index(name="uid", columns={"uid"}),
- *         @ORM\Index(name="hashedurl", columns={"hashedurl"})
+ *         @ORM\Index(name="hashed_url", columns={"hashed_url"})
  *     }
  * )
  * @ORM\HasLifecycleCallbacks()
@@ -79,7 +79,7 @@ class Entry
     /**
      * @var string
      *
-     * @ORM\Column(name="hashedurl", type="text", nullable=true)
+     * @ORM\Column(name="hashed_url", type="string", length=32, nullable=true)
      */
     private $hashedUrl;
 

src/Wallabag/CoreBundle/Repository/EntryRepository.php

@@ -346,6 +346,30 @@ class EntryRepository extends EntityRepository
         return false;
     }
 
+    /**
+     * Find an entry by its hashed url and its owner.
+     * If it exists, return the entry otherwise return false.
+     *
+     * @param $hashedUrl
+     * @param $userId
+     *
+     * @return Entry|bool
+     */
+    public function findByHashedUrlAndUserId($hashedUrl, $userId)
+    {
+        $res = $this->createQueryBuilder('e')
+            ->where('e.hashedUrl = :hashed_url')->setParameter('hashed_url', urldecode($hashedUrl))
+            ->andWhere('e.user = :user_id')->setParameter('user_id', $userId)
+            ->getQuery()
+            ->getResult();
+
+        if (\count($res)) {
+            return current($res);
+        }
+
+        return false;
+    }
+
     /**
      * Count all entries for a user.
      *

tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php

@@ -971,40 +971,42 @@ class EntryRestControllerTest extends WallabagApiTestCase
         $this->assertGreaterThanOrEqual($now->getTimestamp(), (new \DateTime($content['starred_at']))->getTimestamp());
     }
 
-    public function testGetEntriesExistsWithReturnId()
+    public function dataForEntriesExistWithUrl()
     {
-        $this->client->request('GET', '/api/entries/exists?url=http://0.0.0.0/entry2&return_id=1');
+        $url = hash('md5', 'http://0.0.0.0/entry2');
 
-        $this->assertSame(200, $this->client->getResponse()->getStatusCode());
-
-        $content = json_decode($this->client->getResponse()->getContent(), true);
-
-        // it returns a database id, we don't know it, so we only check it's greater than the lowest possible value
-        $this->assertGreaterThan(1, $content['exists']);
+        return [
+            'with_id' => [
+                'url' => '/api/entries/exists?url=http://0.0.0.0/entry2&return_id=1',
+                'expectedValue' => 2,
+            ],
+            'without_id' => [
+                'url' => '/api/entries/exists?url=http://0.0.0.0/entry2',
+                'expectedValue' => true,
+            ],
+            'hashed_url_with_id' => [
+                'url' => '/api/entries/exists?hashed_url=' . $url . '&return_id=1',
+                'expectedValue' => 2,
+            ],
+            'hashed_url_without_id' => [
+                'url' => '/api/entries/exists?hashed_url=' . $url . '',
+                'expectedValue' => true,
+            ],
+        ];
     }
 
-    public function testGetEntriesExistsWithoutReturnId()
+    /**
+     * @dataProvider dataForEntriesExistWithUrl
+     */
+    public function testGetEntriesExists($url, $expectedValue)
     {
-        $this->client->request('GET', '/api/entries/exists?url=http://0.0.0.0/entry2');
-
-        $this->client->request('GET', '/api/entries/exists?hashedurl=' . hash('md5', 'http://0.0.0.0/entry2'));
+        $this->client->request('GET', $url);
 
         $this->assertSame(200, $this->client->getResponse()->getStatusCode());
 
         $content = json_decode($this->client->getResponse()->getContent(), true);
 
-        $this->assertTrue($content['exists']);
-    }
-
-    public function testGetEntriesExistsWithHash()
-    {
-        $this->client->request('GET', '/api/entries/exists?hashedurl=' . hash('md5', 'http://0.0.0.0/entry2'));
-
-        $this->assertSame(200, $this->client->getResponse()->getStatusCode());
-
-        $content = json_decode($this->client->getResponse()->getContent(), true);
-
-        $this->assertSame(2, $content['exists']);
+        $this->assertSame($expectedValue, $content['exists']);
     }
 
     public function testGetEntriesExistsWithManyUrls()
@@ -1045,42 +1047,37 @@ class EntryRestControllerTest extends WallabagApiTestCase
     {
         $url1 = 'http://0.0.0.0/entry2';
         $url2 = 'http://0.0.0.0/entry10';
-        $this->client->request('GET', '/api/entries/exists?hashedurls[]='.hash('md5',$url1).'&hashedurls[]='.hash('md5',$url2) . '&return_id=1');
+        $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('md5', $url1) . '&hashed_urls[]=' . hash('md5', $url2) . '&return_id=1');
 
         $this->assertSame(200, $this->client->getResponse()->getStatusCode());
 
         $content = json_decode($this->client->getResponse()->getContent(), true);
 
-        $this->assertArrayHasKey($url1, $content);
-        $this->assertArrayHasKey($url2, $content);
-        $this->assertSame(2, $content[$url1]);
-        $this->assertNull($content[$url2]);
-
         $this->assertArrayHasKey(hash('md5', $url1), $content);
         $this->assertArrayHasKey(hash('md5', $url2), $content);
-        $this->assertEquals(2, $content[hash('md5', $url1)]);
-        $this->assertEquals(false, $content[hash('md5', $url2)]);
+        $this->assertSame(2, $content[hash('md5', $url1)]);
+        $this->assertNull($content[hash('md5', $url2)]);
     }
 
     public function testGetEntriesExistsWithManyUrlsHashedReturnBool()
     {
         $url1 = 'http://0.0.0.0/entry2';
         $url2 = 'http://0.0.0.0/entry10';
-        $this->client->request('GET', '/api/entries/exists?hashedurls[]='.hash('md5',$url1).'&hashedurls[]='.hash('md5',$url2));
+        $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('md5', $url1) . '&hashed_urls[]=' . hash('md5', $url2));
 
         $this->assertSame(200, $this->client->getResponse()->getStatusCode());
 
         $content = json_decode($this->client->getResponse()->getContent(), true);
 
-        $this->assertArrayHasKey($url1, $content);
-        $this->assertArrayHasKey($url2, $content);
-        $this->assertTrue($content[$url1]);
-        $this->assertFalse($content[$url2]);
+        $this->assertArrayHasKey(hash('md5', $url1), $content);
+        $this->assertArrayHasKey(hash('md5', $url2), $content);
+        $this->assertTrue($content[hash('md5', $url1)]);
+        $this->assertFalse($content[hash('md5', $url2)]);
     }
 
     public function testGetEntriesExistsWhichDoesNotExists()
     {
-        $this->client->request('GET', '/api/entries/exists?hashedurl='.hash('md5','http://google.com/entry2'));
+        $this->client->request('GET', '/api/entries/exists?hashed_url=' . hash('md5', 'http://google.com/entry2'));
 
         $this->assertSame(200, $this->client->getResponse()->getStatusCode());
 
@@ -1091,7 +1088,7 @@ class EntryRestControllerTest extends WallabagApiTestCase
 
     public function testGetEntriesExistsWithNoUrl()
     {
-        $this->client->request('GET', '/api/entries/exists?hashedurl=');
+        $this->client->request('GET', '/api/entries/exists?hashed_url=');
 
         $this->assertSame(403, $this->client->getResponse()->getStatusCode());
     }

tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php

@@ -22,7 +22,7 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase
             'command' => $command->getName(),
         ]);
 
-        $this->assertContains('Generating hashed urls for the 3 user account entries', $tester->getDisplay());
+        $this->assertContains('Generating hashed urls for "3" users', $tester->getDisplay());
         $this->assertContains('Finished generated hashed urls', $tester->getDisplay());
     }
 
@@ -55,7 +55,7 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase
             'username' => 'admin',
         ]);
 
-        $this->assertContains('Generated hashed urls for user admin', $tester->getDisplay());
+        $this->assertContains('Generated hashed urls for user: admin', $tester->getDisplay());
     }
 
     public function testGenerateUrls()
@@ -88,11 +88,11 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase
             'username' => 'admin',
         ]);
 
-        $this->assertContains('Generated hashed urls for user admin', $tester->getDisplay());
+        $this->assertContains('Generated hashed urls for user: admin', $tester->getDisplay());
 
         $entry = $em->getRepository('WallabagCoreBundle:Entry')->findOneByUrl($url);
 
-        $this->assertEquals($entry->getHashedUrl(), hash('sha512', $url));
+        $this->assertSame($entry->getHashedUrl(), hash('md5', $url));
 
         $query = $em->createQuery('DELETE FROM Wallabag\CoreBundle\Entity\Entry e WHERE e.url = :url');
         $query->setParameter('url', $url);