Reduce imported entries

Down from 2.2 minutes to 1.39 minutes
This commit is contained in:
Jeremy Benoist 2017-05-16 21:17:10 +02:00
parent 74a75f7d43
commit cf05a1ae34
No known key found for this signature in database
GPG key ID: BCA73962457ACC3C
8 changed files with 23 additions and 491 deletions

View file

@ -119,6 +119,7 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
$this->getLoggedInUserId()
);
// empty because it wasn't re-imported
$this->assertEmpty($content->getMimetype(), 'Mimetype for http://www.liberation.fr is empty');
$this->assertEmpty($content->getPreviewPicture(), 'Preview picture for http://www.liberation.fr is empty');
$this->assertEmpty($content->getLanguage(), 'Language for http://www.liberation.fr is empty');

View file

@ -67,14 +67,14 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
public function testImport()
{
$readabilityImport = $this->getReadabilityImport(false, 23);
$readabilityImport = $this->getReadabilityImport(false, 3);
$readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->exactly(23))
$entryRepo->expects($this->exactly(3))
->method('findByUrlAndUserId')
->willReturn(false);
@ -88,14 +88,14 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
->getMock();
$this->contentProxy
->expects($this->exactly(23))
->expects($this->exactly(3))
->method('updateEntry')
->willReturn($entry);
$res = $readabilityImport->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 23, 'queued' => 0], $readabilityImport->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 3, 'queued' => 0], $readabilityImport->getSummary());
}
public function testImportAndMarkAllAsRead()
@ -165,7 +165,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
->getMock();
$producer
->expects($this->exactly(23))
->expects($this->exactly(3))
->method('publish');
$readabilityImport->setProducer($producer);
@ -173,7 +173,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
$res = $readabilityImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 23], $readabilityImport->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 3], $readabilityImport->getSummary());
}
public function testImportWithRedis()
@ -211,7 +211,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
$res = $readabilityImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 23], $readabilityImport->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 3], $readabilityImport->getSummary());
$this->assertNotEmpty($redisMock->lpop('readability'));
}

View file

@ -82,14 +82,14 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
public function testImport()
{
$wallabagV1Import = $this->getWallabagV1Import(false, 3);
$wallabagV1Import = $this->getWallabagV1Import(false, 1);
$wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->exactly(4))
$entryRepo->expects($this->exactly(2))
->method('findByUrlAndUserId')
->will($this->onConsecutiveCalls(false, true, false, false));
@ -103,14 +103,14 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
->getMock();
$this->contentProxy
->expects($this->exactly(3))
->expects($this->exactly(1))
->method('updateEntry')
->willReturn($entry);
$res = $wallabagV1Import->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 1, 'imported' => 3, 'queued' => 0], $wallabagV1Import->getSummary());
$this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $wallabagV1Import->getSummary());
}
public function testImportAndMarkAllAsRead()
@ -180,7 +180,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
->getMock();
$producer
->expects($this->exactly(4))
->expects($this->exactly(2))
->method('publish');
$wallabagV1Import->setProducer($producer);
@ -188,7 +188,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV1Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $wallabagV1Import->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 2], $wallabagV1Import->getSummary());
}
public function testImportWithRedis()
@ -226,7 +226,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV1Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $wallabagV1Import->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 2], $wallabagV1Import->getSummary());
$this->assertNotEmpty($redisMock->lpop('wallabag_v1'));
}

View file

@ -89,7 +89,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->exactly(24))
$entryRepo->expects($this->exactly(6))
->method('findByUrlAndUserId')
->will($this->onConsecutiveCalls(false, true, false));
@ -106,7 +106,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV2Import->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 22, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
$this->assertEquals(['skipped' => 4, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
}
public function testImportAndMarkAllAsRead()
@ -172,7 +172,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
->getMock();
$producer
->expects($this->exactly(24))
->expects($this->exactly(6))
->method('publish');
$wallabagV2Import->setProducer($producer);
@ -180,7 +180,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV2Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $wallabagV2Import->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 6], $wallabagV2Import->getSummary());
}
public function testImportWithRedis()
@ -214,7 +214,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV2Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $wallabagV2Import->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 6], $wallabagV2Import->getSummary());
$this->assertNotEmpty($redisMock->lpop('wallabag_v2'));
}
@ -267,7 +267,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->exactly(24))
$entryRepo->expects($this->exactly(6))
->method('findByUrlAndUserId')
->will($this->onConsecutiveCalls(false, true, false));
@ -284,6 +284,6 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV2Import->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 22, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
$this->assertEquals(['skipped' => 4, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
}
}

View file

@ -1,5 +1,3 @@
[{"href":"https:\/\/developers.google.com\/web\/updates\/2016\/07\/infinite-scroller","description":"Complexities of an Infinite Scroller","extended":"TL;DR: Re-use your DOM elements and remove the ones that are far away from the viewport. Use placeholders to account for delayed data","meta":"21ff61c6f648901168f9e6119f53df7d","hash":"e69b65724cca1c585b446d4c47865d76","time":"2016-10-31T15:57:56Z","shared":"yes","toread":"no","tags":"infinite dom performance scroll"},
{"href":"https:\/\/ma.ttias.be\/varnish-explained\/","description":"Varnish (explained) for PHP developers","extended":"A few months ago, I gave a presentation at LaraconEU in Amsterdam titled \"Varnish for PHP developers\". The generic title of that presentation is actually Varnish Explained and this is a write-up of that presentation, the video and the slides.","meta":"d32ad9fac2ed29da4aec12c562e9afb1","hash":"21dd6bdda8ad62666a2c9e79f6e80f98","time":"2016-10-26T06:43:03Z","shared":"yes","toread":"no","tags":"varnish PHP"},
{"href":"https:\/\/ilia.ws\/files\/nginx_torontophpug.pdf","description":"Nginx Tricks for PHP Developers","extended":"","meta":"9adbb5c4ca6760e335b920800d88c70a","hash":"0189bb08f8bd0122c6544bed4624c546","time":"2016-10-05T07:11:27Z","shared":"yes","toread":"no","tags":"nginx PHP best_practice"},
{"href":"https:\/\/jolicode.com\/blog\/starting-a-mobile-application-with-react-native","description":"Starting a mobile application with React Native","extended":"While preparing our next React Native training, I learnt a lot on the library and discovered an amazing community with a lot of packages.","meta":"bd140bd3e53e3a0b4cb08cdaf64bcbfc","hash":"015fa10cd97f56186420555e52cfab62","time":"2016-09-23T10:58:20Z","shared":"yes","toread":"no","tags":"react-native"},
{"href":"http:\/\/open.blogs.nytimes.com\/2016\/08\/29\/testing-varnish-using-varnishtest\/","description":"Testing Varnish Using Varnishtest","extended":"Varnish ships with the ability to test using the testing tool varnishtest. Varnishtest gives you the ability to write VCL tests you can run on the command line or as part of your build process.","meta":"ca2752a07adea4bab52cd19e8fdbf356","hash":"d3e642cc1274d10e4c12ee31f5dde736","time":"2016-08-30T09:33:24Z","shared":"yes","toread":"no","tags":"varnish test vcl"}]
{"href":"https:\/\/ilia.ws\/files\/nginx_torontophpug.pdf","description":"Nginx Tricks for PHP Developers","extended":"","meta":"9adbb5c4ca6760e335b920800d88c70a","hash":"0189bb08f8bd0122c6544bed4624c546","time":"2016-10-05T07:11:27Z","shared":"yes","toread":"no","tags":"nginx PHP best_practice"}]

View file

@ -17,152 +17,12 @@
"date_added": "2016-09-08T11:55:58+0200",
"favorite": true
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Échecs",
"article__url": "https://fr.wikipedia.org/wiki/Échecs"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "90% des dossiers médicaux des Coréens du sud vendus à des entreprises privées - ZATAZ",
"article__url": "http://www.zataz.com/90-des-dossiers-medicaux-des-coreens-du-sud-vendus-a-des-entreprises-privees/"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Mass Surveillance As Art",
"article__url": "https://www.nationaljournal.com/s/73311/mass-surveillance-art"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "What David Cameron did to the pig, his party is now doing to the country",
"article__url": "http://www.newstatesman.com/2015/09/what-david-cameron-did-pig-his-party-now-doing-country"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "CLICK HERE to support 2016 CES Winner, Revolutionary Auto-Tracking Robot",
"article__url": "https://www.indiegogo.com/projects/2016-ces-winner-revolutionary-auto-tracking-robot"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 1,
"article__title": "No title found",
"article__url": "http://carnetdevol.shost.ca/wordpress/aide-memoire-sur-les-commandes-associees-a-systemd/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Présentation d'Arduino - Tuto Arduino - Le blog d'Eskimon",
"article__url": "http://eskimon.fr/73-arduino-101-presentation"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Lenovo ThinkPad X1 Carbon Ultrabook Review",
"article__url": "http://www.notebookcheck.net/Lenovo-ThinkPad-X1-Carbon-Ultrabook-Review.138033.0.html"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Visitons le Château de Landsberg !",
"article__url": "http://autour-du-mont-sainte-odile.overblog.com/2016/01/visitons-le-chateau-de-landsberg.html"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Contrer les stéréotypes par les livres : “C'est dès l'enfance qu'ils se construisent”",
"article__url": "https://www.actualitte.com/article/monde-edition/contrer-les-stereotypes-par-les-livres-c-est-des-l-enfance-qu-ils-se-construisent/64058"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "[ROM][6.0.1][Layers][N5] TipsyOS official builds {UBER TCs}",
"article__url": "http://forum.xda-developers.com/google-nexus-5/development/rom-tipsyos-official-builds-uber-tcs-t3325989"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Top 15 Podcasts All Web Developers Should Follow - Envato Tuts+ Code Article",
"article__url": "http://code.tutsplus.com/articles/top-15-podcasts-all-web-developers-should-follow--net-14461"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "University of Mississippi",
"article__url": "http://olemiss.edu"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "FinnChristiansen.de Jetzt Dank Lets Encrypt Per HTTPS Erreichbar",
"article__url": "https://www.finnchristiansen.de/2015/12/06/finnchristiansen-de-jetzt-dank-lets-encrypt-per-https-erreichbar/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Le développeur et l'ingénierie logicielle",
"article__url": "http://wemucs.com/le-developpeur-et-lingenierie-logicielle/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "The Role of Methylation in Gene Expression",
"article__url": "http://www.nature.com/scitable/topicpage/the-role-of-methylation-in-gene-expression-1070"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "E-Mail-Adresse kostenlos, FreeMail, De-Mail & Nachrichten",
"article__url": "http://web.de"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "OpenSSH Server on Arch Linux | DominicM test",
"article__url": "http://dominicm.com/openssh-server-arch-linux/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Site Moved | Site Help",
"article__url": "http://g1.com/help/sitemoved.asp"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "#Maroc : le stylo anti-pédophiles EAGLE dAMESYS est moins bien configuré que les faux-lowers Twitter du roi Mohammed VI",
"article__url": "https://reflets.info/maroc-le-stylo-anti-pedophiles-eagle-damesys-est-moins-bien-configure-que-les-faux-lowers-twitter-du-roi-mohammed-vi/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Simple Cloud Infrastructure for Developers",
"article__url": "https://www.digitalocean.com/"
}
],
"recommendations": []

View file

@ -32,38 +32,5 @@
"content": "<span class=\"name\">README.md</span><p>wallabag is a self hostable application allowing you to not miss any content anymore. Click, save, read it when you can. It extracts content so that you can read it when you have time.</p>\n<p>More informations on our website: <a href=\"http://wallabag.org\">wallabag.org</a></p>\n<h2><a class=\"anchor\" href=\"https://github.com/wallabag/wallabag#license\"></a>License</h2>\n<p>Copyright © 2010-2014 Nicolas Lœuillet <a href=\"mailto:nicolas@loeuillet.org\">nicolas@loeuillet.org</a> This work is free. You can redistribute it and/or modify it under the terms of the Do What The Fuck You Want To Public License, Version 2, as published by Sam Hocevar. See the COPYING file for more details.</p>\n",
"user_id": "1",
"tags":""
},
{
"0": "3",
"1": "a self hostable application for saving web pages | wallabag",
"2": "https://www.wallabag.org/",
"3": "1",
"4": "0",
"5": "\n<div class=\"row\">\n<div class=\"col-lg-8 col-md-12 col-xs-12 col-sm-12\">\n<p>wallabag (formerly poche) is a <strong>self hostable application for saving web pages</strong>. Unlike other services, wallabag is free (as in freedom) and open source.</p>\n</div>\n\n</div>\n<div class=\"row\">\n<div class=\"col-lg-8 col-md-12 col-xs-12 col-sm-12\">\n<p>With this application you will not miss content anymore. <strong>Click, save, read it when you want</strong>. It saves the content you select so that you can read it when you have time.</p>\n</div>\n\n</div>\n<div class=\"row\">\n<div class=\"col-lg-6 col-md-12 col-xs-12 col-sm-12\">\n<h2>How it works</h2>\n<p>Thanks to the bookmarklet or <a title=\"Downloads\" href=\"http://www.wallabag.org/downloads/\">third-party applications</a>, you save an article in your wallabag to read it later. Then, when you open your wallabag, <strong>you can comfortably read your articles</strong>.</p>\n<h2>How to use wallabag</h2>\n<p>There are two ways to use wallabag: you can <a href=\"http://www.wallabag.org/frequently-asked-questions/#How_can_I_install_wallabag_and_what_are_the_requirements\">install it</a> on your web server or you can <a href=\"http://app.inthepoche.com\">create an account</a> at Framabag (we install and upgrade wallabag for you).</p>\n</div>\n\n</div>\n",
"6": "1",
"id": "3",
"title": "a self hostable application for saving web pages | wallabag",
"url": "https://www.wallabag.org/",
"is_read": "1",
"is_fav": "0",
"content": "\n<div class=\"row\">\n<div class=\"col-lg-8 col-md-12 col-xs-12 col-sm-12\">\n<p>wallabag (formerly poche) is a <strong>self hostable application for saving web pages</strong>. Unlike other services, wallabag is free (as in freedom) and open source.</p>\n</div>\n\n</div>\n<div class=\"row\">\n<div class=\"col-lg-8 col-md-12 col-xs-12 col-sm-12\">\n<p>With this application you will not miss content anymore. <strong>Click, save, read it when you want</strong>. It saves the content you select so that you can read it when you have time.</p>\n</div>\n\n</div>\n<div class=\"row\">\n<div class=\"col-lg-6 col-md-12 col-xs-12 col-sm-12\">\n<h2>How it works</h2>\n<p>Thanks to the bookmarklet or <a title=\"Downloads\" href=\"http://www.wallabag.org/downloads/\">third-party applications</a>, you save an article in your wallabag to read it later. Then, when you open your wallabag, <strong>you can comfortably read your articles</strong>.</p>\n<h2>How to use wallabag</h2>\n<p>There are two ways to use wallabag: you can <a href=\"http://www.wallabag.org/frequently-asked-questions/#How_can_I_install_wallabag_and_what_are_the_requirements\">install it</a> on your web server or you can <a href=\"http://app.inthepoche.com\">create an account</a> at Framabag (we install and upgrade wallabag for you).</p>\n</div>\n\n</div>\n",
"user_id": "1"
},
{
"0": "4",
"1": "Sans titre",
"2": "http:\/\/www.konradlischka.info\/2016\/01\/blog\/wie-ein-deutsches-start-up-mit-wagniskapital-die-marktluecke-fuer-lokalen-digitaljournalismus-schliessen-will\/",
"3": "0",
"4": "0",
"5": "[unable to retrieve full-text content]",
"6": "1",
"id": "4",
"title": "Sans titre",
"url": "http:\/\/www.konradlischka.info\/2016\/01\/blog\/wie-ein-deutsches-start-up-mit-wagniskapital-die-marktluecke-fuer-lokalen-digitaljournalismus-schliessen-will\/",
"is_read": "0",
"is_fav": "0",
"content": "[unable to retrieve full-text content]",
"user_id": "1",
"tags": ""
}
]

File diff suppressed because one or more lines are too long