mirrors/wallabag
1
0
Fork 0
mirror of https://github.com/wallabag/wallabag.git synced 2024-09-28 14:22:02 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1941 from wallabag/v2-asynchronous-jobs

Browse source 
Use asynchronous jobs for imports
This commit is contained in:
Jeremy Benoist 2016-09-19 07:15:40 +02:00 committed by GitHub
commit da18a4682f
85 changed files with 2905 additions and 466 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.travis.yml
View file

@ -1,5 +1,9 @@
language: php
services:
- rabbitmq
- redis
# faster builds on docker-container setup
sudo: false

1
app/AppKernel.php
View file

@ -38,6 +38,7 @@ class AppKernel extends Kernel
new Wallabag\UserBundle\WallabagUserBundle(),
new Wallabag\ImportBundle\WallabagImportBundle(),
new Wallabag\AnnotationBundle\WallabagAnnotationBundle(),
new OldSound\RabbitMqBundle\OldSoundRabbitMqBundle(),
];
if (in_array($this->getEnvironment(), ['dev', 'test'], true)) {

42
app/DoctrineMigrations/Version20160911214952.php Normal file
View file

@ -0,0 +1,42 @@
<?php
namespace Application\Migrations;
use Doctrine\DBAL\Migrations\AbstractMigration;
use Doctrine\DBAL\Schema\Schema;
use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;
class Version20160911214952 extends AbstractMigration implements ContainerAwareInterface
{
/**
* @var ContainerInterface
*/
private $container;
public function setContainer(ContainerInterface $container = null)
{
$this->container = $container;
}
private function getTable($tableName)
{
return $this->container->getParameter('database_table_prefix') . $tableName;
}
/**
* @param Schema $schema
*/
public function up(Schema $schema)
{
$this->addSql('INSERT INTO `'.$this->getTable('craue_config_setting').'` (`name`, `value`, `section`) VALUES (\'import_with_redis\', \'0\', \'import\')');
$this->addSql('INSERT INTO `'.$this->getTable('craue_config_setting').'` (`name`, `value`, `section`) VALUES (\'import_with_rabbitmq\', \'0\', \'import\')');
}
/**
* @param Schema $schema
*/
public function down(Schema $schema)
{
}
}

46
app/DoctrineMigrations/Version20160916201049.php Normal file
View file

@ -0,0 +1,46 @@
<?php
namespace Application\Migrations;
use Doctrine\DBAL\Migrations\AbstractMigration;
use Doctrine\DBAL\Schema\Schema;
use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;
class Version20160916201049 extends AbstractMigration implements ContainerAwareInterface
{
/**
* @var ContainerInterface
*/
private $container;
public function setContainer(ContainerInterface $container = null)
{
$this->container = $container;
}
private function getTable($tableName)
{
return $this->container->getParameter('database_table_prefix') . $tableName;
}
/**
* @param Schema $schema
*/
public function up(Schema $schema)
{
$this->addSql('ALTER TABLE '.$this->getTable('config').' ADD pocket_consumer_key VARCHAR(255) DEFAULT NULL');
$this->addSql("DELETE FROM `".$this->getTable('craue_config_setting')."` WHERE `name` = 'pocket_consumer_key';");
}
/**
* @param Schema $schema
*/
public function down(Schema $schema)
{
$this->abortIf($this->connection->getDatabasePlatform()->getName() == 'sqlite', 'Migration can only be executed safely on \'mysql\' or \'postgresql\'.');
$this->addSql('ALTER TABLE `'.$this->getTable('config').'` DROP pocket_consumer_key');
$this->addSql("INSERT INTO `".$this->getTable('craue_config_setting')."` (`name`, `value`, `section`) VALUES ('pocket_consumer_key', NULL, 'import')");
}
}

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml
View file

@ -8,7 +8,8 @@ export_csv: Aktiver eksport til CSV
export_json: Aktiver eksport til JSON
export_txt: Aktiver eksport til TXT
export_xml: Aktiver eksport til XML
pocket_consumer_key: Brugers nøgle til Pocket for at importere materialer (https://getpocket.com/developer/docs/authentication)
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
shaarli_url: Shaarli-URL, hvis tjenesten er aktiv
share_diaspora: Aktiver deling til Diaspora
share_mail: Aktiver deling med email

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml
View file

@ -8,7 +8,8 @@ export_csv: CSV-Export aktivieren
export_json: JSON-Export aktivieren
export_txt: TXT-Export aktivieren
export_xml: XML-Export aktivieren
pocket_consumer_key: Consumer-Key für Pocket, um Inhalte zu importieren (https://getpocket.com/developer/docs/authentication)
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
shaarli_url: Shaarli-URL, sofern der Service aktiviert ist
share_diaspora: Teilen zu Diaspora aktiveren
share_mail: Teilen via E-Mail aktiveren

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml
View file

@ -8,7 +8,8 @@ export_csv: Enable CSV export
export_json: Enable JSON export
export_txt: Enable TXT export
export_xml: Enable XML export
pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication)
import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
import_with_redis: Enable Redis to import data asynchronously
shaarli_url: Shaarli URL, if the service is enabled
share_diaspora: Enable share to Diaspora
share_mail: Enable share by email

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml
View file

@ -8,7 +8,8 @@ export_csv: Activar exportación a CSV
export_json: Activar exportación a JSON
export_txt: Activar exportación a TXT
export_xml: Activar exportación a XML
pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication)
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
shaarli_url: Shaarli URL, si el servicio está activado
share_diaspora: Activar compartir con Diaspora
share_mail: Activar compartir con email

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml
View file

@ -8,7 +8,8 @@ export_csv: فعال‌سازی برون‌سپاری به CSV
export_json: فعال‌سازی برون‌سپاری به JSON
export_txt: فعال‌سازی برون‌سپاری به TXT
export_xml: فعال‌سازی برون‌سپاری به XML
pocket_consumer_key: کلید کاربری Pocket برای درون‌ریزی مطالب (https://getpocket.com/developer/docs/authentication)
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
shaarli_url: نشانی Shaarli، اگر فعال بود
share_diaspora: فعال‌سازی هم‌رسانی به Diaspora
share_mail: فعال‌سازی هم‌رسانی با ایمیل

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml
View file

@ -8,7 +8,8 @@ export_csv: Activer l'export CSV
export_json: Activer l'export JSON
export_txt: Activer l'export TXT
export_xml: Activer l'export XML
pocket_consumer_key: Clé d'authentification Pocket pour importer les données (https://getpocket.com/developer/docs/authentication)
import_with_rabbitmq: Activer RabbitMQ pour gérer les imports de façon asynchrone
import_with_redis: Activer Redis pour gérer les imports de façon asynchrone
shaarli_url: URL de Shaarli, si le service Shaarli est activé
share_diaspora: Activer le partage vers Diaspora
share_mail: Activer le partage par email

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml
View file

@ -8,7 +8,8 @@ export_csv: Abilita esportazione CSV
export_json: Abilita esportazione JSON
export_txt: Abilita esportazione TXT
export_xml: Abilita esportazione XML
pocket_consumer_key: Consumer key per Pocket per importare i contenuti (https://getpocket.com/developer/docs/authentication)
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
shaarli_url: Shaarli URL, se il servizio è abilitato
share_diaspora: Abilita la condivisione con Diaspora
share_mail: Abilita la condivisione per email

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml
View file

@ -8,7 +8,8 @@ export_csv: Activar l'expòrt CSV
export_json: Activar l'expòrt JSON
export_txt: Activar l'expòrt TXT
export_xml: Activar l'expòrt XML
pocket_consumer_key: Clau d'autentificacion Pocket per importar las donadas (https://getpocket.com/developer/docs/authentication)
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
shaarli_url: URL de Shaarli, se lo servici Shaarli es activat
share_diaspora: Activar lo partatge cap a Diaspora
share_mail: Activar lo partatge per corrièl

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml
View file

@ -8,7 +8,8 @@ export_csv: Włącz eksport do CSV
export_json: Włącz eksport do JSON
export_txt: Włącz eksport do TXT
export_xml: Włącz eksport do XML
pocket_consumer_key: Klucz klienta Pocket do importu zawartości (https://getpocket.com/developer/docs/authentication)
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
shaarli_url: Adress URL Shaarli, jeżeli usługa jest włączona
share_diaspora: Włącz udostępnianie dla Diaspora
share_mail: Włącz udostępnianie przez email

3
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml
View file

@ -8,7 +8,8 @@ export_csv: Permite exportare CSV
export_json: Permite exportare JSON
export_txt: Permite exportare TXT
export_xml: Permite exportare XML
pocket_consumer_key: Cheie consumator pentru importarea contentului din Pocket (https://getpocket.com/developer/docs/authentication)
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
shaarli_url: Shaarli URL, dacă serviciul este permis
share_diaspora: Permite share către Diaspora
share_mail: Permite share prin email

31
app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml Normal file
View file

@ -0,0 +1,31 @@
# download_pictures: Download pictures on your server
# carrot: Enable share to Carrot
# diaspora_url: Diaspora URL, if the service is enabled
# export_epub: Enable ePub export
# export_mobi: Enable .mobi export
# export_pdf: Enable PDF export
# export_csv: Enable CSV export
# export_json: Enable JSON export
# export_txt: Enable TXT export
# export_xml: Enable XML export
# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
# import_with_redis: Enable Redis to import data asynchronously
# shaarli_url: Shaarli URL, if the service is enabled
# share_diaspora: Enable share to Diaspora
# share_mail: Enable share by email
# share_shaarli: Enable share to Shaarli
# share_twitter: Enable share to Twitter
# show_printlink: Display a link to print content
# wallabag_support_url: Support URL for wallabag
# wallabag_url: URL of *your* wallabag instance
# entry: "article"
# export: "export"
# import: "import"
# misc: "misc"
# modify_settings: "apply"
# piwik_host: Host of your website in Piwik
# piwik_site_id: ID of your website in Piwik
# piwik_enabled: Enable Piwik
# demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)"
# demo_mode_username: "Demo user"
# share_public: Allow public url for entries

64
app/config/config.yml
View file

@ -215,3 +215,67 @@ lexik_maintenance:
response:
code: 503
status: "wallabag Service Temporarily Unavailable"
old_sound_rabbit_mq:
connections:
default:
host: "%rabbitmq_host%"
port: "%rabbitmq_port%"
user: "%rabbitmq_user%"
password: "%rabbitmq_password%"
vhost: /
lazy: true
producers:
import_pocket:
connection: default
exchange_options:
name: 'wallabag.import.pocket'
type: topic
import_readability:
connection: default
exchange_options:
name: 'wallabag.import.readability'
type: topic
import_wallabag_v1:
connection: default
exchange_options:
name: 'wallabag.import.wallabag_v1'
type: topic
import_wallabag_v2:
connection: default
exchange_options:
name: 'wallabag.import.wallabag_v2'
type: topic
consumers:
import_pocket:
connection: default
exchange_options:
name: 'wallabag.import.pocket'
type: topic
queue_options:
name: 'wallabag.import.pocket'
callback: wallabag_import.consumer.amqp.pocket
import_readability:
connection: default
exchange_options:
name: 'wallabag.import.readability'
type: topic
queue_options:
name: 'wallabag.import.readability'
callback: wallabag_import.consumer.amqp.readability
import_wallabag_v1:
connection: default
exchange_options:
name: 'wallabag.import.wallabag_v1'
type: topic
queue_options:
name: 'wallabag.import.wallabag_v1'
callback: wallabag_import.consumer.amqp.wallabag_v1
import_wallabag_v2:
connection: default
exchange_options:
name: 'wallabag.import.wallabag_v2'
type: topic
queue_options:
name: 'wallabag.import.wallabag_v2'
callback: wallabag_import.consumer.amqp.wallabag_v2

12
app/config/parameters.yml.dist
View file

@ -38,3 +38,15 @@ parameters:
fosuser_confirmation: true
from_email: no-reply@wallabag.org
rss_limit: 50
# RabbitMQ processing
rabbitmq_host: localhost
rabbitmq_port: 5672
rabbitmq_user: guest
rabbitmq_password: guest
# Redis processing
redis_host: localhost
redis_port: 6379

2
app/config/parameters_test.yml
View file

@ -5,4 +5,4 @@ parameters:
test_database_name: null
test_database_user: null
test_database_password: null
test_database_path: '%kernel.root_dir%/../data/db/wallabag_testYO.sqlite'
test_database_path: '%kernel.root_dir%/../data/db/wallabag_test.sqlite'

8
composer.json
View file

@ -81,7 +81,10 @@
"lexik/maintenance-bundle": "~2.1",
"ocramius/proxy-manager": "1.*",
"white-october/pagerfanta-bundle": "^1.0",
"mouf/nodejs-installer": "~1.0"
"mouf/nodejs-installer": "~1.0",
"php-amqplib/rabbitmq-bundle": "^1.8",
"predis/predis": "^1.0",
"javibravo/simpleue": "^1.0"
},
"require-dev": {
"doctrine/doctrine-fixtures-bundle": "~2.2",
@ -89,7 +92,8 @@
"sensio/generator-bundle": "^3.0",
"phpunit/phpunit": "~5.0",
"symfony/phpunit-bridge": "^3.0",
"friendsofphp/php-cs-fixer": "~1.9"
"friendsofphp/php-cs-fixer": "~1.9",
"m6web/redis-mock": "^2.0"
},
"scripts": {
"post-cmd": [

13
docker-compose.yml
View file

@ -11,6 +11,7 @@ services:
links:
- php:php
command: nginx -c /nginx.conf
php:
build:
context: docker/php
@ -30,6 +31,7 @@ services:
# If all DBMS are commented out, sqlite will be used as default
# - ./docker/postgres/env
# - ./docker/mariadb/env
#postgres:
# image: postgres:9
# ports:
@ -38,6 +40,7 @@ services:
# - ./docker/data/pgsql:/var/lib/postgresql/data
# env_file:
# - ./docker/postgres/env
#mariadb:
# image: mariadb:10
# ports:
@ -46,3 +49,13 @@ services:
# - ./docker/data/mariadb:/var/lib/mysql
# env_file:
# - ./docker/mariadb/env
rabbitmq:
image: rabbitmq:3-management
ports:
- "15672:15672"
redis:
image: redis
ports:
- "6379:6379"

67
docs/en/developer/rabbitmq.rst Normal file
View file

@ -0,0 +1,67 @@
Install RabbitMQ for asynchronous tasks
=======================================
In order to launch asynchronous tasks (useful for huge imports for example), we can use RabbitMQ.
Requirements
------------
You need to have RabbitMQ installed on your server.
Installation
~~~~~~~~~~~~
.. code:: bash
wget https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
apt-key add rabbitmq-signing-key-public.asc
apt-get update
apt-get install rabbitmq-server
Configuration and launch
~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: bash
rabbitmq-plugins enable rabbitmq_management # (useful to have a web interface, available at http://localhost:15672/ (guest/guest)
rabbitmq-server -detached
Stop RabbitMQ
~~~~~~~~~~~~~
.. code:: bash
rabbitmqctl stop
Configure RabbitMQ in wallabag
------------------------------
Edit your ``parameters.yml`` file to edit RabbitMQ configuration. The default one should be ok:
.. code:: yaml
rabbitmq_host: localhost
rabbitmq_port: 5672
rabbitmq_user: guest
rabbitmq_password: guest
Launch RabbitMQ consumer
------------------------
Depending on which service you want to import from you need to enable one (or many if you want to support many) cron job:
.. code:: bash
# for Pocket import
bin/console rabbitmq:consumer import_pocket -w
# for Readbility import
bin/console rabbitmq:consumer import_readability -w
# for wallabag v1 import
bin/console rabbitmq:consumer import_wallabag_v1 -w
# for wallabag v2 import
bin/console rabbitmq:consumer import_wallabag_v2 -w

62
docs/en/developer/redis.rst Normal file
View file

@ -0,0 +1,62 @@
Install Redis for asynchronous tasks
=======================================
In order to launch asynchronous tasks (useful for huge imports for example), we can use Redis.
Requirements
------------
You need to have Redis installed on your server.
Installation
~~~~~~~~~~~~
.. code:: bash
apt-get install redis-server
Launch
~~~~~~
The server might be already running after installing, if not you can launch it using:
.. code:: bash
redis-server
Configure Redis in wallabag
---------------------------
Edit your ``parameters.yml`` file to edit Redis configuration. The default one should be ok:
.. code:: yaml
redis_host: localhost
redis_port: 6379
Launch Redis consumer
------------------------
Depending on which service you want to import from you need to enable one (or many if you want to support many) cron job:
.. code:: bash
# for Pocket import
bin/console wallabag:import:redis-worker pocket -vv >> /path/to/wallabag/var/logs/redis-pocket.log
# for Readbility import
bin/console wallabag:import:redis-worker readability -vv >> /path/to/wallabag/var/logs/redis-readability.log
# for wallabag v1 import
bin/console wallabag:import:redis-worker wallabag_v1 -vv >> /path/to/wallabag/var/logs/redis-wallabag_v1.log
# for wallabag v2 import
bin/console wallabag:import:redis-worker wallabag_v2 -vv >> /path/to/wallabag/var/logs/redis-wallabag_v2.log
If you want to launch the import only for some messages and not all, you can specify this number (here 12) and the worker will stop right after the 12th message :
.. code:: bash
bin/console wallabag:import:redis-worker pocket -vv --maxIterations=12

9
src/Wallabag/CoreBundle/Command/InstallCommand.php
View file

@ -317,8 +317,13 @@ class InstallCommand extends ContainerAwareCommand
'section' => 'export',
],
[
'name' => 'pocket_consumer_key',
'value' => null,
'name' => 'import_with_redis',
'value' => '0',
'section' => 'import',
],
[
'name' => 'import_with_rabbitmq',
'value' => '0',
'section' => 'import',
],
[

50
src/Wallabag/CoreBundle/Controller/EntryController.php
View file

@ -17,26 +17,35 @@ use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache;
class EntryController extends Controller
{
/**
* @param Entry $entry
* Fetch content and update entry.
* In case it fails, entry will return to avod loosing the data.
*
* @param Entry $entry
* @param string $prefixMessage Should be the translation key: entry_saved or entry_reloaded
*
* @return Entry
*/
private function updateEntry(Entry $entry)
private function updateEntry(Entry $entry, $prefixMessage = 'entry_saved')
{
// put default title in case of fetching content failed
$entry->setTitle('No title found');
$message = 'flashes.entry.notice.'.$prefixMessage;
try {
$entry = $this->get('wallabag_core.content_proxy')->updateEntry($entry, $entry->getUrl());
$em = $this->getDoctrine()->getManager();
$em->persist($entry);
$em->flush();
} catch (\Exception $e) {
$this->get('logger')->error('Error while saving an entry', [
'exception' => $e,
'entry' => $entry,
]);
return false;
$message = 'flashes.entry.notice.'.$prefixMessage.'_failed';
}
return true;
$this->get('session')->getFlashBag()->add('notice', $message);
return $entry;
}
/**
@ -66,12 +75,11 @@ class EntryController extends Controller
return $this->redirect($this->generateUrl('view', ['id' => $existingEntry->getId()]));
}
$message = 'flashes.entry.notice.entry_saved';
if (false === $this->updateEntry($entry)) {
$message = 'flashes.entry.notice.entry_saved_failed';
}
$this->updateEntry($entry);
$this->get('session')->getFlashBag()->add('notice', $message);
$em = $this->getDoctrine()->getManager();
$em->persist($entry);
$em->flush();
return $this->redirect($this->generateUrl('homepage'));
}
@ -95,6 +103,10 @@ class EntryController extends Controller
if (false === $this->checkIfEntryAlreadyExists($entry)) {
$this->updateEntry($entry);
$em = $this->getDoctrine()->getManager();
$em->persist($entry);
$em->flush();
}
return $this->redirect($this->generateUrl('homepage'));
@ -316,15 +328,11 @@ class EntryController extends Controller
{
$this->checkUserAction($entry);
$message = 'flashes.entry.notice.entry_reloaded';
if (false === $this->updateEntry($entry)) {
$message = 'flashes.entry.notice.entry_reload_failed';
}
$this->updateEntry($entry, 'entry_reloaded');
$this->get('session')->getFlashBag()->add(
'notice',
$message
);
$em = $this->getDoctrine()->getManager();
$em->persist($entry);
$em->flush();
return $this->redirect($this->generateUrl('view', ['id' => $entry->getId()]));
}

3
src/Wallabag/CoreBundle/DataFixtures/ORM/LoadConfigData.php
View file

@ -20,6 +20,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
$adminConfig->setItemsPerPage(30);
$adminConfig->setReadingSpeed(1);
$adminConfig->setLanguage('en');
$adminConfig->setPocketConsumerKey('xxxxx');
$manager->persist($adminConfig);
@ -30,6 +31,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
$bobConfig->setItemsPerPage(10);
$bobConfig->setReadingSpeed(1);
$bobConfig->setLanguage('fr');
$bobConfig->setPocketConsumerKey(null);
$manager->persist($bobConfig);
@ -40,6 +42,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface
$emptyConfig->setItemsPerPage(10);
$emptyConfig->setReadingSpeed(1);
$emptyConfig->setLanguage('en');
$emptyConfig->setPocketConsumerKey(null);
$manager->persist($emptyConfig);

9
src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php
View file

@ -91,8 +91,13 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
'section' => 'export',
],
[
'name' => 'pocket_consumer_key',
'value' => null,
'name' => 'import_with_redis',
'value' => '0',
'section' => 'import',
],
[
'name' => 'import_with_rabbitmq',
'value' => '0',
'section' => 'import',
],
[

31
src/Wallabag/CoreBundle/Entity/Config.php
View file

@ -80,6 +80,13 @@ class Config
*/
private $readingSpeed;
/**
* @var string
*
* @ORM\Column(name="pocket_consumer_key", type="string", nullable=true)
*/
private $pocketConsumerKey;
/**
* @ORM\OneToOne(targetEntity="Wallabag\UserBundle\Entity\User", inversedBy="config")
*/
@ -278,6 +285,30 @@ class Config
return $this->readingSpeed;
}
/**
* Set pocketConsumerKey.
*
* @param string $pocketConsumerKey
*
* @return Config
*/
public function setPocketConsumerKey($pocketConsumerKey)
{
$this->pocketConsumerKey = $pocketConsumerKey;
return $this;
}
/**
* Get pocketConsumerKey.
*
* @return string
*/
public function getPocketConsumerKey()
{
return $this->pocketConsumerKey;
}
/**
* @param TaggingRule $rule
*

23
src/Wallabag/CoreBundle/Entity/Entry.php
View file

@ -97,7 +97,7 @@ class Entry
private $content;
/**
* @var date
* @var \DateTime
*
* @ORM\Column(name="created_at", type="datetime")
*
@ -106,7 +106,7 @@ class Entry
private $createdAt;
/**
* @var date
* @var \DateTime
*
* @ORM\Column(name="updated_at", type="datetime")
*
@ -410,7 +410,22 @@ class Entry
}
/**
* @return string
* Set created_at.
* Only used when importing data from an other service.
*
* @param \DateTime $createdAt
*
* @return Entry
*/
public function setCreatedAt(\DateTime $createdAt)
{
$this->createdAt = $createdAt;
return $this;
}
/**
* @return \DateTime
*/
public function getCreatedAt()
{
@ -418,7 +433,7 @@ class Entry
}
/**
* @return string
* @return \DateTime
*/
public function getUpdatedAt()
{

3
src/Wallabag/CoreBundle/Form/Type/ConfigType.php
View file

@ -52,6 +52,9 @@ class ConfigType extends AbstractType
'choices' => array_flip($this->languages),
'label' => 'config.form_settings.language_label',
])
->add('pocket_consumer_key', null, [
'label' => 'config.form_settings.pocket_consumer_key_label',
])
->add('save', SubmitType::class, [
'label' => 'config.form.save',
])

8
src/Wallabag/CoreBundle/Resources/config/services.yml
View file

@ -125,3 +125,11 @@ services:
arguments:
- "@security.token_storage"
- "@router"
wallabag_core.redis.client:
class: Predis\Client
arguments:
-
host: '%redis_host%'
port: '%redis_port%'
schema: tcp

8
src/Wallabag/CoreBundle/Resources/translations/messages.da.yml
View file

@ -68,6 +68,7 @@ config:
# 200_word: 'I read ~200 words per minute'
# 300_word: 'I read ~300 words per minute'
# 400_word: 'I read ~400 words per minute'
pocket_consumer_key_label: Brugers nøgle til Pocket for at importere materialer
form_rss:
description: 'RSS-feeds fra wallabag gør det muligt at læse de artikler, der gemmes i wallabag, med din RSS-læser. Det kræver, at du genererer et token først.'
token_label: 'RSS-Token'
@ -346,6 +347,8 @@ import:
# page_title: 'Import > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
# page_title: 'Developer'
@ -411,10 +414,10 @@ flashes:
notice:
# entry_already_saved: 'Entry already saved on %date%'
# entry_saved: 'Entry saved'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
# entry_updated: 'Entry updated'
# entry_reloaded: 'Entry reloaded'
# entry_reload_failed: 'Failed to reload entry'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Artikel arkiveret'
entry_unarchived: 'Artikel ikke længere arkiveret'
entry_starred: 'Artikel markeret som favorit'
@ -428,6 +431,7 @@ flashes:
# failed: 'Import failed, please try again.'
# failed_on_file: 'Error while processing import. Please verify your import file.'
# summary: 'Import summary: %imported% imported, %skipped% already saved.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
# client_created: 'New client created.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.de.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Ich lese ~200 Wörter pro Minute'
300_word: 'Ich lese ~300 Wörter pro Minute'
400_word: 'Ich lese ~400 Wörter pro Minute'
pocket_consumer_key_label: Consumer-Key für Pocket, um Inhalte zu importieren
form_rss:
description: 'Die RSS-Feeds von wallabag erlauben es dir, deine gespeicherten Artikel mit deinem bevorzugten RSS-Reader zu lesen. Vorher musst du jedoch einen Token erstellen.'
token_label: 'RSS-token'
@ -346,6 +347,8 @@ import:
page_title: 'Aus Readability importieren'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Entwickler'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Eintrag bereits am %date% gespeichert'
entry_saved: 'Eintrag gespeichert'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Eintrag aktualisiert'
entry_reloaded: 'Eintrag neugeladen'
entry_reload_failed: 'Neuladen des Eintrags fehlgeschlagen'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Artikel archiviert'
entry_unarchived: 'Artikel dearchiviert'
entry_starred: 'Artikel favorisiert'
@ -428,6 +431,7 @@ flashes:
failed: 'Import fehlgeschlagen, bitte erneut probieren.'
failed_on_file: 'Fehler während des Imports. Bitte überprüfe deine Import-Datei.'
summary: 'Import-Zusammenfassung: %imported% importiert, %skipped% bereits gespeichert.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Neuer Client erstellt.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.en.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'I read ~200 words per minute'
300_word: 'I read ~300 words per minute'
400_word: 'I read ~400 words per minute'
pocket_consumer_key_label: Consumer key for Pocket to import contents
form_rss:
description: 'RSS feeds provided by wallabag allow you to read your saved articles with your favourite RSS reader. You need to generate a token first.'
token_label: 'RSS token'
@ -346,6 +347,8 @@ import:
page_title: 'Import > Readability'
description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Developer'
@ -413,10 +416,10 @@ flashes:
notice:
entry_already_saved: 'Entry already saved on %date%'
entry_saved: 'Entry saved'
entry_saved_failed: 'Failed to save entry'
entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Entry updated'
entry_reloaded: 'Entry reloaded'
entry_reload_failed: 'Failed to reload entry'
entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Entry archived'
entry_unarchived: 'Entry unarchived'
entry_starred: 'Entry starred'
@ -430,6 +433,7 @@ flashes:
failed: 'Import failed, please try again.'
failed_on_file: 'Error while processing import. Please verify your import file.'
summary: 'Import summary: %imported% imported, %skipped% already saved.'
summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'New client %name% created.'

14
src/Wallabag/CoreBundle/Resources/translations/messages.es.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Leo ~200 palabras por minuto'
300_word: 'Leo ~300 palabras por minuto'
400_word: 'Leo ~400 palabras por minuto'
# pocket_consumer_key_label: Consumer key for Pocket to import contents
form_rss:
description: 'Los feeds RSS de wallabag permiten leer los artículos guardados con su lector RSS favorito. Necesita generar un token primero'
token_label: 'RSS token'
@ -346,6 +347,8 @@ import:
page_title: 'Importar > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Promotor'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Entrada ya guardada por %fecha%'
entry_saved: 'Entrada guardada'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Entrada actualizada'
entry_reloaded: 'Entrada recargada'
entry_reload_failed: 'Entrada recargada reprobada'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Artículo archivado'
entry_unarchived: 'Artículo desarchivado'
entry_starred: 'Artículo guardado en los favoritos'
@ -425,9 +428,10 @@ flashes:
tag_added: 'Etiqueta añadida'
import:
notice:
failed: 'Importación reprobada, por favor inténtelo de nuevo.'
failed_on_file: 'Se ocurre un error por procesar importación. Por favor verifique su archivo importado.'
summary: 'Resúmen importado: %importado% importado, %saltados% ya guardado.'
failed: 'Importación reprobada, por favor inténtelo de nuevo.'
failed_on_file: 'Se ocurre un error por procesar importación. Por favor verifique su archivo importado.'
summary: 'Resúmen importado: %importado% importado, %saltados% ya guardado.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Nuevo cliente creado.'

12
src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'من تقریباً ۲۰۰ واژه را در دقیقه می‌خوانم'
300_word: 'من تقریباً ۳۰۰ واژه را در دقیقه می‌خوانم'
400_word: 'من تقریباً ۴۰۰ واژه را در دقیقه می‌خوانم'
pocket_consumer_key_label: کلید کاربری Pocket برای درون‌ریزی مطالب
form_rss:
description: 'با خوراک آر-اس-اس که wallabag در اختیارتان می‌گذارد، می‌توانید مقاله‌های ذخیره‌شده را در نرم‌افزار آر-اس-اس دلخواه خود بخوانید. برای این کار نخست باید یک کد بسازید.'
token_label: 'کد آر-اس-اس'
@ -344,8 +345,10 @@ import:
description: 'این برنامه همهٔ داده‌های شما را در نسخهٔ ۲ wallabag درون‌ریزی می‌کند. به بخش «همهٔ مقاله‌ها» بروید و در بخش «برون‌ریزی» روی "JSON" کلیک کنید. با این کار شما پرونده‌ای به شکل "All articles.json" دریافت خواهید کرد.'
readability:
page_title: 'درون‌ریزی > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
# page_title: 'Developer'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'این مقاله در تاریخ %date% ذخیره شده بود'
entry_saved: 'مقاله ذخیره شد'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'مقاله به‌روز شد'
entry_reloaded: 'مقاله به‌روز شد'
entry_reload_failed: 'به‌روزرسانی مقاله شکست خورد'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'مقاله بایگانی شد'
entry_unarchived: 'مقاله از بایگانی درآمد'
entry_starred: 'مقاله برگزیده شد'
@ -428,6 +431,7 @@ flashes:
failed: 'درون‌ریزی شکست خورد. لطفاً دوباره تلاش کنید.'
failed_on_file: 'خطا هنگام پردازش پروندهٔ ورودی. آیا پروندهٔ درون‌ریزی شده سالم است؟'
summary: 'گزارش درون‌ریزی: %imported% وارد شد, %skipped% از قبل ذخیره شده بود.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
# client_created: 'New client created.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Je lis environ 200 mots par minute'
300_word: 'Je lis environ 300 mots par minute'
400_word: 'Je lis environ 400 mots par minute'
pocket_consumer_key_label: Clé d'authentification Pocket pour importer les données
form_rss:
description: "Les flux RSS fournis par wallabag vous permettent de lire vos articles sauvegardés dans votre lecteur de flux préféré. Pour pouvoir les utiliser, vous devez d'abord créer un jeton."
token_label: 'Jeton RSS'
@ -346,6 +347,8 @@ import:
page_title: 'Importer > Readability'
description: 'Cet outil va importer toutes vos données de Readability. Sur la page des outils (https://www.readability.com/tools/), cliquez sur "Export your data" dans la section "Data Export". Vous allez recevoir un email avec un lien pour télécharger le json.'
how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer."
worker:
enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :"
developer:
page_title: 'Développeur'
@ -413,10 +416,10 @@ flashes:
notice:
entry_already_saved: 'Article déjà sauvergardé le %date%'
entry_saved: 'Article enregistré'
entry_saved_failed: "L'enregistrement a échoué"
entry_saved_failed: 'Article enregistré mais impossible de récupérer le contenu'
entry_updated: 'Article mis à jour'
entry_reloaded: 'Article rechargé'
entry_reload_failed: "Le rechargement de l'article a échoué"
entry_reload_failed: "Article mis à jour mais impossible de récupérer le contenu"
entry_archived: 'Article marqué comme lu'
entry_unarchived: 'Article marqué comme non lu'
entry_starred: 'Article ajouté dans les favoris'
@ -430,6 +433,7 @@ flashes:
failed: "L'import a échoué, veuillez ré-essayer"
failed_on_file: "Erreur lors du traitement de l'import. Vérifier votre fichier."
summary: "Rapport d'import: %imported% importés, %skipped% déjà présent."
summary_with_queue: "Rapport d'import: %queued% en cours de traitement."
developer:
notice:
client_created: 'Nouveau client %name% créé'

12
src/Wallabag/CoreBundle/Resources/translations/messages.it.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Leggo ~200 parole al minuto'
300_word: 'Leggo ~300 parole al minuto'
400_word: 'Leggo ~400 parole al minuto'
pocket_consumer_key_label: Consumer key per Pocket per importare i contenuti
form_rss:
description: 'I feed RSS generati da wallabag ti permettono di leggere i tuoi contenuti salvati con il tuo lettore di RSS preferito. Prima, devi generare un token.'
token_label: 'RSS token'
@ -343,8 +344,10 @@ import:
description: 'Questo importatore copierà tutti i tuoi dati da un wallabag v2. Vai in "Tutti i contenuti", e, nella sidebar di esportazione, clicca su "JSON". Otterrai un file "Tutti i contenuti.json".'
readability:
page_title: 'Importa da > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Sviluppatori'
@ -410,10 +413,10 @@ flashes:
notice:
entry_already_saved: 'Contenuto già salvato in data %date%'
entry_saved: 'Contenuto salvato'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Contenuto aggiornato'
entry_reloaded: 'Contenuto ricaricato'
entry_reload_failed: 'Errore nel ricaricamento del contenuto'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Contenuto archiviato'
entry_unarchived: 'Contenuto dis-archiviato'
entry_starred: 'Contenuto segnato come preferito'
@ -427,6 +430,7 @@ flashes:
failed: 'Importazione fallita, riprova.'
failed_on_file: 'Errore durante la processazione dei dati da importare. Verifica il tuo file di import.'
summary: 'Sommario di importazione: %imported% importati, %skipped% già salvati.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Nuovo client creato.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml
View file

@ -68,6 +68,7 @@ config:
200_word: "Legissi a l'entorn de 200 mots per minuta"
300_word: "Legissi a l'entorn de 300 mots per minuta"
400_word: "Legissi a l'entorn de 400 mots per minuta"
pocket_consumer_key_label: Clau d'autentificacion Pocket per importar las donadas
form_rss:
description: "Los fluxes RSS fornits per wallabag vos permeton de legir vòstres articles salvagardats dins vòstre lector de fluxes preferit. Per los poder emplegar, vos cal, d'en primièr crear un geton."
token_label: 'Geton RSS'
@ -346,6 +347,8 @@ import:
page_title: 'Importer > Readability'
description: "Aquesta aisina importarà totas vòstres articles de Readability. Sus la pagina de l'aisina (https://www.readability.com/tools/), clicatz sus \"Export your data\" dins la seccion \"Data Export\". Recebretz un corrièl per telecargar un json (qu'acaba pas amb un .json de fach)."
how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar."
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Desvolopador'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Article ja salvargardat lo %date%'
entry_saved: 'Article enregistrat'
entry_saved_failed: "Fracàs de l'enregistrament de l'entrada"
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Article mes a jorn'
entry_reloaded: 'Article recargat'
entry_reload_failed: "Fracàs de l'actualizacion de l'article"
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Article marcat coma legit'
entry_unarchived: 'Article marcat coma pas legit'
entry_starred: 'Article apondut dins los favorits'
@ -428,6 +431,7 @@ flashes:
failed: "L'importacion a fracassat, mercés de tornar ensajar"
failed_on_file: "Errorr pendent du tractament de l'import. Mercés de verificar vòstre fichièr."
summary: "Rapòrt d'import: %imported% importats, %skipped% ja presents."
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Novèl client creat'

8
src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml
View file

@ -68,6 +68,7 @@ config:
200_word: 'Czytam ~200 słów na minutę'
300_word: 'Czytam ~300 słów na minutę'
400_word: 'Czytam ~400 słów na minutę'
pocket_consumer_key_label: Klucz klienta Pocket do importu zawartości
form_rss:
description: 'Kanały RSS prowadzone przez wallabag pozwalają Ci na czytanie twoich zapisanych artykułów w twoium ulubionym czytniku RSS. Musisz najpierw wynegenerować tokena.‌'
token_label: 'Token RSS'
@ -346,6 +347,8 @@ import:
page_title: 'Import > Readability'
description: 'Ten importer, zaimportuje wszystkie twoje artykuły z Readability. Na stronie narzędzi (https://www.readability.com/tools/), kliknij na "Export your data" w sekcji "Data Export". Otrzymach email z plikiem JSON (plik nie będzie zawierał rozszerzenia .json).'
how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
page_title: 'Deweloper'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Wpis już został dodany %date%'
entry_saved: 'Wpis zapisany'
entry_saved_failed: 'Zapis artykułu się nie powiódł'
# entry_saved_failed: 'Entry saved but fetching content failed'
entry_updated: 'Wpis zaktualizowany'
entry_reloaded: 'Wpis ponownie załadowany'
entry_reload_failed: 'Błąd ponownego załadowania'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Wpis dodany do archiwum'
entry_unarchived: 'Wpis usunięty z archiwum'
entry_starred: 'Wpis oznaczony gwiazdką'
@ -428,6 +431,7 @@ flashes:
failed: 'Nieudany import, prosimy spróbować ponownie.'
failed_on_file: 'Błąd podczas ptrzetwarzania pliku. Sprawdż swój importowany plik.'
summary: 'Podsumowanie importu: %imported% zaimportowane, %skipped% już zapisane.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
client_created: 'Nowy klient utworzony.'

8
src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml
View file

@ -68,6 +68,7 @@ config:
# 200_word: 'I read ~200 words per minute'
# 300_word: 'I read ~300 words per minute'
# 400_word: 'I read ~400 words per minute'
pocket_consumer_key_label: Cheie consumator pentru importarea contentului din Pocket
form_rss:
description: 'Feed-urile RSS oferite de wallabag îți permit să-ți citești articolele salvate în reader-ul tău preferat RSS.'
token_label: 'RSS-Token'
@ -346,6 +347,8 @@ import:
# page_title: 'Import > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
# page_title: 'Developer'
@ -411,10 +414,10 @@ flashes:
notice:
# entry_already_saved: 'Entry already saved on %date%'
# entry_saved: 'Entry saved'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
# entry_updated: 'Entry updated'
# entry_reloaded: 'Entry reloaded'
# entry_reload_failed: 'Failed to reload entry'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Articol arhivat'
entry_unarchived: 'Articol dezarhivat'
entry_starred: 'Articol adăugat la favorite'
@ -428,6 +431,7 @@ flashes:
# failed: 'Import failed, please try again.'
# failed_on_file: 'Error while processing import. Please verify your import file.'
# summary: 'Import summary: %imported% imported, %skipped% already saved.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
# client_created: 'New client created.'

12
src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml
View file

@ -68,6 +68,7 @@ config:
# 200_word: 'I read ~200 words per minute'
# 300_word: 'I read ~300 words per minute'
# 400_word: 'I read ~400 words per minute'
# pocket_consumer_key_label: Consumer key for Pocket to import contents
form_rss:
description: 'wallabag RSS akışı kaydetmiş olduğunuz makalelerini favori RSS okuyucunuzda görüntülemenizi sağlar. Bunu yapabilmek için öncelikle belirteç (token) oluşturmalısınız.'
token_label: 'RSS belirteci (token)'
@ -344,8 +345,10 @@ import:
# description: 'This importer will import all your wallabag v2 articles. Go to All articles, then, on the export sidebar, click on "JSON". You will have a "All articles.json" file.'
readability:
page_title: 'İçe Aktar > Readability'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
# description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).'
# how_to: 'Please select your Readability export and click on the below button to upload and import it.'
worker:
# enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:"
developer:
# page_title: 'Developer'
@ -411,10 +414,10 @@ flashes:
notice:
entry_already_saved: 'Entry already saved on %date%'
entry_saved: 'Makale kaydedildi'
# entry_saved_failed: 'Failed to save entry'
# entry_saved_failed: 'Entry saved but fetching content failed'
# entry_updated: 'Entry updated'
entry_reloaded: 'Makale içeriği yenilendi'
# entry_reload_failed: 'Failed to reload entry'
# entry_reload_failed: 'Entry reloaded but fetching content failed'
entry_archived: 'Makale arşivlendi'
entry_unarchived: 'Makale arşivden çıkartıldı'
entry_starred: 'Makale favorilere eklendi'
@ -428,6 +431,7 @@ flashes:
# failed: 'Import failed, please try again.'
# failed_on_file: 'Error while processing import. Please verify your import file.'
# summary: 'Import summary: %imported% imported, %skipped% already saved.'
# summary_with_queue: 'Import summary: %queued% queued.'
developer:
notice:
# client_created: 'New client created.'

12
src/Wallabag/CoreBundle/Resources/views/themes/baggy/Config/index.html.twig
View file

@ -44,6 +44,18 @@
</div>
</fieldset>
<fieldset class="w500p inline">
<div class="row">
{{ form_label(form.config.pocket_consumer_key) }}
{{ form_errors(form.config.pocket_consumer_key) }}
{{ form_widget(form.config.pocket_consumer_key) }}
<p>
&raquo;
<a href="https://getpocket.com/developer/docs/authentication">https://getpocket.com/developer/docs/authentication</a>
</p>
</div>
</fieldset>
{{ form_rest(form.config) }}
</form>

12
src/Wallabag/CoreBundle/Resources/views/themes/material/Config/index.html.twig
View file

@ -62,6 +62,18 @@
</div>
</div>
<div class="row">
<div class="input-field col s12">
{{ form_label(form.config.pocket_consumer_key) }}
{{ form_errors(form.config.pocket_consumer_key) }}
{{ form_widget(form.config.pocket_consumer_key) }}
<p>
&raquo;
<a href="https://getpocket.com/developer/docs/authentication">https://getpocket.com/developer/docs/authentication</a>
</p>
</div>
</div>
{{ form_widget(form.config.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }}
{{ form_rest(form.config) }}
</form>

2
src/Wallabag/CoreBundle/Resources/views/themes/material/layout.html.twig
View file

@ -19,6 +19,8 @@
Materialize.toast('{{ flashMessage|trans }}', 4000);
</script>
{% endfor %}
{{ render(controller("WallabagImportBundle:Import:checkQueue")) }}
{% endblock %}
{% block menu %}

6
src/Wallabag/ImportBundle/Command/ImportCommand.php
View file

@ -26,6 +26,10 @@ class ImportCommand extends ContainerAwareCommand
{
$output->writeln('Start : '.(new \DateTime())->format('d-m-Y G:i:s').' ---');
if (!file_exists($input->getArgument('filepath'))) {
throw new Exception(sprintf('File "%s" not found', $input->getArgument('filepath')));
}
$em = $this->getContainer()->get('doctrine')->getManager();
// Turning off doctrine default logs queries for saving memory
$em->getConnection()->getConfiguration()->setSQLLogger(null);
@ -43,9 +47,9 @@ class ImportCommand extends ContainerAwareCommand
}
$wallabag->setMarkAsRead($input->getOption('markAsRead'));
$wallabag->setUser($user);
$res = $wallabag
->setUser($user)
->setFilepath($input->getArgument('filepath'))
->import();

44
src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php Normal file
View file

@ -0,0 +1,44 @@
<?php
namespace Wallabag\ImportBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Config\Definition\Exception\Exception;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Simpleue\Worker\QueueWorker;
class RedisWorkerCommand extends ContainerAwareCommand
{
protected function configure()
{
$this
->setName('wallabag:import:redis-worker')
->setDescription('Launch Redis worker')
->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false)
;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
$output->writeln('Waiting for message ...');
$serviceName = $input->getArgument('serviceName');
if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
}
$worker = new QueueWorker(
$this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
$this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName),
$input->getOption('maxIterations')
);
$worker->start();
}
}

17
src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php Normal file
View file

@ -0,0 +1,17 @@
<?php
namespace Wallabag\ImportBundle\Consumer;
use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
use PhpAmqpLib\Message\AMQPMessage;
class AMQPEntryConsumer extends AbstractConsumer implements ConsumerInterface
{
/**
* {@inheritdoc}
*/
public function execute(AMQPMessage $msg)
{
return $this->handleMessage($msg->body);
}
}

74
src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php Normal file
View file

@ -0,0 +1,74 @@
<?php
namespace Wallabag\ImportBundle\Consumer;
use Doctrine\ORM\EntityManager;
use Wallabag\ImportBundle\Import\AbstractImport;
use Wallabag\UserBundle\Repository\UserRepository;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\Tag;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
abstract class AbstractConsumer
{
protected $em;
protected $userRepository;
protected $import;
protected $logger;
public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
{
$this->em = $em;
$this->userRepository = $userRepository;
$this->import = $import;
$this->logger = $logger ?: new NullLogger();
}
/**
* Handle a message and save it.
*
* @param string $body Message from the queue (in json)
*
* @return bool
*/
protected function handleMessage($body)
{
$storedEntry = json_decode($body, true);
$user = $this->userRepository->find($storedEntry['userId']);
// no user? Drop message
if (null === $user) {
$this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
return false;
}
$this->import->setUser($user);
$entry = $this->import->parseEntry($storedEntry);
if (null === $entry) {
$this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
return false;
}
try {
$this->em->flush();
// clear only affected entities
$this->em->clear(Entry::class);
$this->em->clear(Tag::class);
} catch (\Exception $e) {
$this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
return false;
}
$this->logger->info('Content with url imported! ('.$entry->getUrl().')');
return true;
}
}

29
src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php Normal file
View file

@ -0,0 +1,29 @@
<?php
namespace Wallabag\ImportBundle\Consumer;
use Simpleue\Job\Job;
class RedisEntryConsumer extends AbstractConsumer implements Job
{
/**
* Handle one message by one message.
*
* @param string $job Content of the message (directly from Redis)
*
* @return bool
*/
public function manage($job)
{
return $this->handleMessage($job);
}
/**
* Should tell if the given job will kill the worker.
* We don't want to stop it :).
*/
public function isStopJob($job)
{
return false;
}
}

62
src/Wallabag/ImportBundle/Controller/ImportController.php
View file

@ -16,4 +16,66 @@ class ImportController extends Controller
'imports' => $this->get('wallabag_import.chain')->getAll(),
]);
}
/**
* Display how many messages are queue (both in Redis and RabbitMQ).
* Only for admins.
*/
public function checkQueueAction()
{
$nbRedisMessages = null;
$nbRabbitMessages = null;
if (!$this->get('security.authorization_checker')->isGranted('ROLE_SUPER_ADMIN')) {
return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [
'nbRedisMessages' => $nbRedisMessages,
'nbRabbitMessages' => $nbRabbitMessages,
]);
}
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket')
+ $this->getTotalMessageInRabbitQueue('readability')
+ $this->getTotalMessageInRabbitQueue('wallabag_v1')
+ $this->getTotalMessageInRabbitQueue('wallabag_v2')
;
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$redis = $this->get('wallabag_core.redis.client');
$nbRedisMessages = $redis->llen('wallabag.import.pocket')
+ $redis->llen('wallabag.import.readability')
+ $redis->llen('wallabag.import.wallabag_v1')
+ $redis->llen('wallabag.import.wallabag_v2')
;
}
return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [
'nbRedisMessages' => $nbRedisMessages,
'nbRabbitMessages' => $nbRabbitMessages,
]);
}
/**
* Count message in RabbitMQ queue.
* It get one message without acking it (so it'll stay in the queue)
* which will include the total of *other* messages in the queue.
* Adding one to that messages will result in the full total message.
*
* @param string $importService The import service related: pocket, readability, wallabag_v1 or wallabag_v2
*
* @return int
*/
private function getTotalMessageInRabbitQueue($importService)
{
$message = $this
->get('old_sound_rabbit_mq.import_'.$importService.'_consumer')
->getChannel()
->basic_get('wallabag.import.'.$importService);
if (null === $message) {
return 0;
}
return $message->delivery_info['message_count'] + 1;
}
}

35
src/Wallabag/ImportBundle/Controller/PocketController.php
View file

@ -10,12 +10,31 @@ use Symfony\Component\Form\Extension\Core\Type\CheckboxType;
class PocketController extends Controller
{
/**
* Return Pocket Import Service with or without RabbitMQ enabled.
*
* @return \Wallabag\ImportBundle\Import\PocketImport
*/
private function getPocketImportService()
{
$pocket = $this->get('wallabag_import.pocket.import');
$pocket->setUser($this->getUser());
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
}
return $pocket;
}
/**
* @Route("/pocket", name="import_pocket")
*/
public function indexAction()
{
$pocket = $this->get('wallabag_import.pocket.import');
$pocket = $this->getPocketImportService();
$form = $this->createFormBuilder($pocket)
->add('mark_as_read', CheckboxType::class, [
'label' => 'import.form.mark_as_read_label',
@ -24,8 +43,8 @@ class PocketController extends Controller
->getForm();
return $this->render('WallabagImportBundle:Pocket:index.html.twig', [
'import' => $this->get('wallabag_import.pocket.import'),
'has_consumer_key' => '' == trim($this->get('craue_config')->get('pocket_consumer_key')) ? false : true,
'import' => $this->getPocketImportService(),
'has_consumer_key' => '' === trim($this->getUser()->getConfig()->getPocketConsumerKey()) ? false : true,
'form' => $form->createView(),
]);
}
@ -35,7 +54,7 @@ class PocketController extends Controller
*/
public function authAction(Request $request)
{
$requestToken = $this->get('wallabag_import.pocket.import')
$requestToken = $this->getPocketImportService()
->getRequestToken($this->generateUrl('import', [], UrlGeneratorInterface::ABSOLUTE_URL));
if (false === $requestToken) {
@ -62,7 +81,7 @@ class PocketController extends Controller
public function callbackAction()
{
$message = 'flashes.import.notice.failed';
$pocket = $this->get('wallabag_import.pocket.import');
$pocket = $this->getPocketImportService();
$markAsRead = $this->get('session')->get('mark_as_read');
$this->get('session')->remove('mark_as_read');
@ -83,6 +102,12 @@ class PocketController extends Controller
'%imported%' => $summary['imported'],
'%skipped%' => $summary['skipped'],
]);
if (0 < $summary['queued']) {
$message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
'%queued%' => $summary['queued'],
]);
}
}
$this->get('session')->getFlashBag()->add(

16
src/Wallabag/ImportBundle/Controller/ReadabilityController.php
View file

@ -18,15 +18,21 @@ class ReadabilityController extends Controller
$form->handleRequest($request);
$readability = $this->get('wallabag_import.readability.import');
$readability->setUser($this->getUser());
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
}
if ($form->isValid()) {
$file = $form->get('file')->getData();
$markAsRead = $form->get('mark_as_read')->getData();
$name = 'readability_'.$this->getUser()->getId().'.json';
if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
$res = $readability
->setUser($this->getUser())
->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
->setMarkAsRead($markAsRead)
->import();
@ -40,6 +46,12 @@ class ReadabilityController extends Controller
'%skipped%' => $summary['skipped'],
]);
if (0 < $summary['queued']) {
$message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
'%queued%' => $summary['queued'],
]);
}
unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
}

10
src/Wallabag/ImportBundle/Controller/WallabagController.php
View file

@ -38,15 +38,15 @@ abstract class WallabagController extends Controller
$form->handleRequest($request);
$wallabag = $this->getImportService();
$wallabag->setUser($this->getUser());
if ($form->isValid()) {
$file = $form->get('file')->getData();
$markAsRead = $form->get('mark_as_read')->getData();
$name = $this->getUser()->getId().'.json';
if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
$res = $wallabag
->setUser($this->getUser())
->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
->setMarkAsRead($markAsRead)
->import();
@ -60,6 +60,12 @@ abstract class WallabagController extends Controller
'%skipped%' => $summary['skipped'],
]);
if (0 < $summary['queued']) {
$message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
'%queued%' => $summary['queued'],
]);
}
unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
}

10
src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
View file

@ -12,7 +12,15 @@ class WallabagV1Controller extends WallabagController
*/
protected function getImportService()
{
return $this->get('wallabag_import.wallabag_v1.import');
$service = $this->get('wallabag_import.wallabag_v1.import');
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
}
return $service;
}
/**

10
src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
View file

@ -12,7 +12,15 @@ class WallabagV2Controller extends WallabagController
*/
protected function getImportService()
{
return $this->get('wallabag_import.wallabag_v2.import');
$service = $this->get('wallabag_import.wallabag_v2.import');
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
}
return $service;
}
/**

1
src/Wallabag/ImportBundle/Form/Type/UploadImportType.php
View file

@ -15,6 +15,7 @@ class UploadImportType extends AbstractType
$builder
->add('file', FileType::class, [
'label' => 'import.form.file_label',
'required' => true,
])
->add('mark_as_read', CheckboxType::class, [
'label' => 'import.form.mark_as_read_label',

144
src/Wallabag/ImportBundle/Import/AbstractImport.php
View file

@ -7,12 +7,21 @@ use Psr\Log\NullLogger;
use Doctrine\ORM\EntityManager;
use Wallabag\CoreBundle\Helper\ContentProxy;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\Tag;
use Wallabag\UserBundle\Entity\User;
use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
abstract class AbstractImport implements ImportInterface
{
protected $em;
protected $logger;
protected $contentProxy;
protected $producer;
protected $user;
protected $markAsRead;
protected $skippedEntries = 0;
protected $importedEntries = 0;
protected $queuedEntries = 0;
public function __construct(EntityManager $em, ContentProxy $contentProxy)
{
@ -26,22 +35,151 @@ abstract class AbstractImport implements ImportInterface
$this->logger = $logger;
}
/**
* Set RabbitMQ/Redis Producer to send each entry to a queue.
* This method should be called when user has enabled RabbitMQ.
*
* @param ProducerInterface $producer
*/
public function setProducer(ProducerInterface $producer)
{
$this->producer = $producer;
}
/**
* Set current user.
* Could the current *connected* user or one retrieve by the consumer.
*
* @param User $user
*/
public function setUser(User $user)
{
$this->user = $user;
}
/**
* Set whether articles must be all marked as read.
*
* @param bool $markAsRead
*/
public function setMarkAsRead($markAsRead)
{
$this->markAsRead = $markAsRead;
return $this;
}
/**
* Get whether articles must be all marked as read.
*/
public function getMarkAsRead()
{
return $this->markAsRead;
}
/**
* Fetch content from the ContentProxy (using graby).
* If it fails return false instead of the updated entry.
* If it fails return the given entry to be saved in all case (to avoid user to loose the content).
*
* @param Entry $entry Entry to update
* @param string $url Url to grab content for
* @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url
*
* @return Entry|false
* @return Entry
*/
protected function fetchContent(Entry $entry, $url, array $content = [])
{
try {
return $this->contentProxy->updateEntry($entry, $url, $content);
} catch (\Exception $e) {
return false;
return $entry;
}
}
/**
* Parse and insert all given entries.
*
* @param $entries
*/
protected function parseEntries($entries)
{
$i = 1;
foreach ($entries as $importedEntry) {
$entry = $this->parseEntry($importedEntry);
if (null === $entry) {
continue;
}
// flush every 20 entries
if (($i % 20) === 0) {
$this->em->flush();
// clear only affected entities
$this->em->clear(Entry::class);
$this->em->clear(Tag::class);
}
++$i;
}
$this->em->flush();
}
/**
* Parse entries and send them to the queue.
* It should just be a simple loop on all item, no call to the database should be done
* to speedup queuing.
*
* Faster parse entries for Producer.
* We don't care to make check at this time. They'll be done by the consumer.
*
* @param array $entries
*/
protected function parseEntriesForProducer(array $entries)
{
foreach ($entries as $importedEntry) {
// set userId for the producer (it won't know which user is connected)
$importedEntry['userId'] = $this->user->getId();
if ($this->markAsRead) {
$importedEntry = $this->setEntryAsRead($importedEntry);
}
++$this->queuedEntries;
$this->producer->publish(json_encode($importedEntry));
}
}
/**
* {@inheritdoc}
*/
public function getSummary()
{
return [
'skipped' => $this->skippedEntries,
'imported' => $this->importedEntries,
'queued' => $this->queuedEntries,
];
}
/**
* Parse one entry.
*
* @param array $importedEntry
*
* @return Entry
*/
abstract public function parseEntry(array $importedEntry);
/**
* Set current imported entry to archived / read.
* Implementation is different accross all imports.
*
* @param array $importedEntry
*
* @return array
*/
abstract protected function setEntryAsRead(array $importedEntry);
}

212
src/Wallabag/ImportBundle/Import/PocketImport.php
View file

@ -6,30 +6,33 @@ use Psr\Log\NullLogger;
use Doctrine\ORM\EntityManager;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Helper\ContentProxy;
use Craue\ConfigBundle\Util\Config;
class PocketImport extends AbstractImport
{
private $user;
private $client;
private $consumerKey;
private $skippedEntries = 0;
private $importedEntries = 0;
private $markAsRead;
protected $accessToken;
private $accessToken;
public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig)
const NB_ELEMENTS = 5000;
public function __construct(EntityManager $em, ContentProxy $contentProxy)
{
$this->user = $tokenStorage->getToken()->getUser();
$this->em = $em;
$this->contentProxy = $contentProxy;
$this->consumerKey = $craueConfig->get('pocket_consumer_key');
$this->logger = new NullLogger();
}
/**
* Only used for test purpose.
*
* @return string
*/
public function getAccessToken()
{
return $this->accessToken;
}
/**
* {@inheritdoc}
*/
@ -66,7 +69,7 @@ class PocketImport extends AbstractImport
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
[
'body' => json_encode([
'consumer_key' => $this->consumerKey,
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'redirect_uri' => $redirectUri,
]),
]
@ -96,7 +99,7 @@ class PocketImport extends AbstractImport
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
[
'body' => json_encode([
'consumer_key' => $this->consumerKey,
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'code' => $code,
]),
]
@ -115,39 +118,23 @@ class PocketImport extends AbstractImport
return true;
}
/**
* Set whether articles must be all marked as read.
*
* @param bool $markAsRead
*/
public function setMarkAsRead($markAsRead)
{
$this->markAsRead = $markAsRead;
return $this;
}
/**
* Get whether articles must be all marked as read.
*/
public function getMarkAsRead()
{
return $this->markAsRead;
}
/**
* {@inheritdoc}
*/
public function import()
public function import($offset = 0)
{
static $run = 0;
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get',
[
'body' => json_encode([
'consumer_key' => $this->consumerKey,
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'access_token' => $this->accessToken,
'detailType' => 'complete',
'state' => 'all',
'sort' => 'oldest',
'sort' => 'newest',
'count' => self::NB_ELEMENTS,
'offset' => $offset,
]),
]
);
@ -162,22 +149,26 @@ class PocketImport extends AbstractImport
$entries = $response->json();
$this->parseEntries($entries['list']);
if ($this->producer) {
$this->parseEntriesForProducer($entries['list']);
} else {
$this->parseEntries($entries['list']);
}
// if we retrieve exactly the amount of items requested it means we can get more
// re-call import and offset item by the amount previous received:
// - first call get 5k offset 0
// - second call get 5k offset 5k
// - and so on
if (count($entries['list']) === self::NB_ELEMENTS) {
++$run;
return $this->import(self::NB_ELEMENTS * $run);
}
return true;
}
/**
* {@inheritdoc}
*/
public function getSummary()
{
return [
'skipped' => $this->skippedEntries,
'imported' => $this->importedEntries,
];
}
/**
* Set the Guzzle client.
*
@ -189,77 +180,74 @@ class PocketImport extends AbstractImport
}
/**
* @see https://getpocket.com/developer/docs/v3/retrieve
* {@inheritdoc}
*
* @param $entries
* @see https://getpocket.com/developer/docs/v3/retrieve
*/
private function parseEntries($entries)
public function parseEntry(array $importedEntry)
{
$i = 1;
$url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
foreach ($entries as $pocketEntry) {
$url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($url, $this->user->getId());
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($url, $this->user->getId());
if (false !== $existingEntry) {
++$this->skippedEntries;
if (false !== $existingEntry) {
++$this->skippedEntries;
continue;
}
$entry = new Entry($this->user);
$entry = $this->fetchContent($entry, $url);
// jump to next entry in case of problem while getting content
if (false === $entry) {
++$this->skippedEntries;
continue;
}
// 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
if ($pocketEntry['status'] == 1 || $this->markAsRead) {
$entry->setArchived(true);
}
// 0 or 1 - 1 If the item is starred
if ($pocketEntry['favorite'] == 1) {
$entry->setStarred(true);
}
$title = 'Untitled';
if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') {
$title = $pocketEntry['resolved_title'];
} elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
$title = $pocketEntry['given_title'];
}
$entry->setTitle($title);
// 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
$entry->setPreviewPicture($pocketEntry['images'][1]['src']);
}
if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
$this->contentProxy->assignTagsToEntry(
$entry,
array_keys($pocketEntry['tags'])
);
}
$this->em->persist($entry);
++$this->importedEntries;
// flush every 20 entries
if (($i % 20) === 0) {
$this->em->flush();
}
++$i;
return;
}
$this->em->flush();
$this->em->clear();
$entry = new Entry($this->user);
$entry->setUrl($url);
// update entry with content (in case fetching failed, the given entry will be return)
$entry = $this->fetchContent($entry, $url);
// 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
$entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead);
// 0 or 1 - 1 If the item is starred
$entry->setStarred($importedEntry['favorite'] == 1);
$title = 'Untitled';
if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') {
$title = $importedEntry['resolved_title'];
} elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') {
$title = $importedEntry['given_title'];
}
$entry->setTitle($title);
// 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
$entry->setPreviewPicture($importedEntry['images'][1]['src']);
}
if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
$this->contentProxy->assignTagsToEntry(
$entry,
array_keys($importedEntry['tags'])
);
}
if (!empty($importedEntry['time_added'])) {
$entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added']));
}
$this->em->persist($entry);
++$this->importedEntries;
return $entry;
}
/**
* {@inheritdoc}
*/
protected function setEntryAsRead(array $importedEntry)
{
$importedEntry['status'] = '1';
return $importedEntry;
}
}

145
src/Wallabag/ImportBundle/Import/ReadabilityImport.php
View file

@ -3,28 +3,10 @@
namespace Wallabag\ImportBundle\Import;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\UserBundle\Entity\User;
class ReadabilityImport extends AbstractImport
{
private $user;
private $skippedEntries = 0;
private $importedEntries = 0;
private $filepath;
private $markAsRead;
/**
* We define the user in a custom call because on the import command there is no logged in user.
* So we can't retrieve user from the `security.token_storage` service.
*
* @param User $user
*/
public function setUser(User $user)
{
$this->user = $user;
return $this;
}
/**
* {@inheritdoc}
@ -62,37 +44,6 @@ class ReadabilityImport extends AbstractImport
return $this;
}
/**
* Set whether articles must be all marked as read.
*
* @param bool $markAsRead
*/
public function setMarkAsRead($markAsRead)
{
$this->markAsRead = $markAsRead;
return $this;
}
/**
* Get whether articles must be all marked as read.
*/
public function getMarkAsRead()
{
return $this->markAsRead;
}
/**
* {@inheritdoc}
*/
public function getSummary()
{
return [
'skipped' => $this->skippedEntries,
'imported' => $this->importedEntries,
];
}
/**
* {@inheritdoc}
*/
@ -116,64 +67,66 @@ class ReadabilityImport extends AbstractImport
return false;
}
if ($this->producer) {
$this->parseEntriesForProducer($data['bookmarks']);
return true;
}
$this->parseEntries($data['bookmarks']);
return true;
}
/**
* Parse and insert all given entries.
*
* @param $entries
* {@inheritdoc}
*/
protected function parseEntries($entries)
public function parseEntry(array $importedEntry)
{
$i = 1;
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
foreach ($entries as $importedEntry) {
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
if (false !== $existingEntry) {
++$this->skippedEntries;
if (false !== $existingEntry) {
++$this->skippedEntries;
continue;
}
$data = [
'title' => $importedEntry['article__title'],
'url' => $importedEntry['article__url'],
'content_type' => '',
'language' => '',
'is_archived' => $importedEntry['archive'] || $this->markAsRead,
'is_starred' => $importedEntry['favorite'],
];
$entry = $this->fetchContent(
new Entry($this->user),
$data['url'],
$data
);
// jump to next entry in case of problem while getting content
if (false === $entry) {
++$this->skippedEntries;
continue;
}
$entry->setArchived($data['is_archived']);
$entry->setStarred($data['is_starred']);
$this->em->persist($entry);
++$this->importedEntries;
// flush every 20 entries
if (($i % 20) === 0) {
$this->em->flush();
}
++$i;
return;
}
$this->em->flush();
$this->em->clear();
$data = [
'title' => $importedEntry['article__title'],
'url' => $importedEntry['article__url'],
'content_type' => '',
'language' => '',
'is_archived' => $importedEntry['archive'] || $this->markAsRead,
'is_starred' => $importedEntry['favorite'],
'created_at' => $importedEntry['date_added'],
];
$entry = new Entry($this->user);
$entry->setUrl($data['url']);
$entry->setTitle($data['title']);
// update entry with content (in case fetching failed, the given entry will be return)
$entry = $this->fetchContent($entry, $data['url'], $data);
$entry->setArchived($data['is_archived']);
$entry->setStarred($data['is_starred']);
$entry->setCreatedAt(new \DateTime($data['created_at']));
$this->em->persist($entry);
++$this->importedEntries;
return $entry;
}
/**
* {@inheritdoc}
*/
protected function setEntryAsRead(array $importedEntry)
{
$importedEntry['archive'] = 1;
return $importedEntry;
}
}

143
src/Wallabag/ImportBundle/Import/WallabagImport.php
View file

@ -3,15 +3,10 @@
namespace Wallabag\ImportBundle\Import;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\UserBundle\Entity\User;
abstract class WallabagImport extends AbstractImport
{
protected $user;
protected $skippedEntries = 0;
protected $importedEntries = 0;
protected $filepath;
protected $markAsRead;
// untitled in all languages from v1
protected $untitled = [
'Untitled',
@ -28,19 +23,6 @@ abstract class WallabagImport extends AbstractImport
'',
];
/**
* We define the user in a custom call because on the import command there is no logged in user.
* So we can't retrieve user from the `security.token_storage` service.
*
* @param User $user
*/
public function setUser(User $user)
{
$this->user = $user;
return $this;
}
/**
* {@inheritdoc}
*/
@ -79,22 +61,17 @@ abstract class WallabagImport extends AbstractImport
return false;
}
if ($this->producer) {
$this->parseEntriesForProducer($data);
return true;
}
$this->parseEntries($data);
return true;
}
/**
* {@inheritdoc}
*/
public function getSummary()
{
return [
'skipped' => $this->skippedEntries,
'imported' => $this->importedEntries,
];
}
/**
* Set file path to the json file.
*
@ -108,85 +85,59 @@ abstract class WallabagImport extends AbstractImport
}
/**
* Set whether articles must be all marked as read.
*
* @param bool $markAsRead
* {@inheritdoc}
*/
public function setMarkAsRead($markAsRead)
public function parseEntry(array $importedEntry)
{
$this->markAsRead = $markAsRead;
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
return $this;
}
if (false !== $existingEntry) {
++$this->skippedEntries;
/**
* Parse and insert all given entries.
*
* @param $entries
*/
protected function parseEntries($entries)
{
$i = 1;
foreach ($entries as $importedEntry) {
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
if (false !== $existingEntry) {
++$this->skippedEntries;
continue;
}
$data = $this->prepareEntry($importedEntry, $this->markAsRead);
$entry = $this->fetchContent(
new Entry($this->user),
$importedEntry['url'],
$data
);
// jump to next entry in case of problem while getting content
if (false === $entry) {
++$this->skippedEntries;
continue;
}
if (array_key_exists('tags', $data)) {
$this->contentProxy->assignTagsToEntry(
$entry,
$data['tags']
);
}
if (isset($importedEntry['preview_picture'])) {
$entry->setPreviewPicture($importedEntry['preview_picture']);
}
$entry->setArchived($data['is_archived']);
$entry->setStarred($data['is_starred']);
$this->em->persist($entry);
++$this->importedEntries;
// flush every 20 entries
if (($i % 20) === 0) {
$this->em->flush();
}
++$i;
return;
}
$this->em->flush();
$this->em->clear();
$data = $this->prepareEntry($importedEntry);
$entry = new Entry($this->user);
$entry->setUrl($data['url']);
$entry->setTitle($data['title']);
// update entry with content (in case fetching failed, the given entry will be return)
$entry = $this->fetchContent($entry, $data['url'], $data);
if (array_key_exists('tags', $data)) {
$this->contentProxy->assignTagsToEntry(
$entry,
$data['tags']
);
}
if (isset($importedEntry['preview_picture'])) {
$entry->setPreviewPicture($importedEntry['preview_picture']);
}
$entry->setArchived($data['is_archived']);
$entry->setStarred($data['is_starred']);
if (!empty($data['created_at'])) {
$entry->setCreatedAt(new \DateTime($data['created_at']));
}
$this->em->persist($entry);
++$this->importedEntries;
return $entry;
}
/**
* This should return a cleaned array for a given entry to be given to `updateEntry`.
*
* @param array $entry Data from the imported file
* @param bool $markAsRead Should we mark as read content?
* @param array $entry Data from the imported file
*
* @return array
*/
abstract protected function prepareEntry($entry = [], $markAsRead = false);
abstract protected function prepareEntry($entry = []);
}

15
src/Wallabag/ImportBundle/Import/WallabagV1Import.php
View file

@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport
/**
* {@inheritdoc}
*/
protected function prepareEntry($entry = [], $markAsRead = false)
protected function prepareEntry($entry = [])
{
$data = [
'title' => $entry['title'],
@ -39,9 +39,10 @@ class WallabagV1Import extends WallabagImport
'url' => $entry['url'],
'content_type' => '',
'language' => '',
'is_archived' => $entry['is_read'] || $markAsRead,
'is_archived' => $entry['is_read'] || $this->markAsRead,
'is_starred' => $entry['is_fav'],
'tags' => '',
'created_at' => '',
];
// force content to be refreshed in case on bad fetch in the v1 installation
@ -56,4 +57,14 @@ class WallabagV1Import extends WallabagImport
return $data;
}
/**
* {@inheritdoc}
*/
protected function setEntryAsRead(array $importedEntry)
{
$importedEntry['is_read'] = 1;
return $importedEntry;
}
}

14
src/Wallabag/ImportBundle/Import/WallabagV2Import.php
View file

@ -31,12 +31,22 @@ class WallabagV2Import extends WallabagImport
/**
* {@inheritdoc}
*/
protected function prepareEntry($entry = [], $markAsRead = false)
protected function prepareEntry($entry = [])
{
return [
'html' => $entry['content'],
'content_type' => $entry['mimetype'],
'is_archived' => ($entry['is_archived'] || $markAsRead),
'is_archived' => ($entry['is_archived'] || $this->markAsRead),
] + $entry;
}
/**
* {@inheritdoc}
*/
protected function setEntryAsRead(array $importedEntry)
{
$importedEntry['is_archived'] = 1;
return $importedEntry;
}
}

36
src/Wallabag/ImportBundle/Redis/Producer.php Normal file
View file

@ -0,0 +1,36 @@
<?php
namespace Wallabag\ImportBundle\Redis;
use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
use Simpleue\Queue\RedisQueue;
/**
* This is a proxy class for "Simpleue\Queue\RedisQueue".
* It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
*
* It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
* kind of class which implements the same interface.
* So we can inject either a RabbitMQ producer or a Redis producer with the same signature
*/
class Producer implements ProducerInterface
{
private $queue;
public function __construct(RedisQueue $queue)
{
$this->queue = $queue;
}
/**
* Publish a message in the Redis queue.
*
* @param string $msgBody
* @param string $routingKey NOT USED
* @param array $additionalProperties NOT USED
*/
public function publish($msgBody, $routingKey = '', $additionalProperties = array())
{
$this->queue->sendJob($msgBody);
}
}

30
src/Wallabag/ImportBundle/Resources/config/rabbit.yml Normal file
View file

@ -0,0 +1,30 @@
# RabbitMQ stuff
services:
wallabag_import.consumer.amqp.pocket:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.pocket.import"
- "@logger"
wallabag_import.consumer.amqp.readability:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.readability.import"
- "@logger"
wallabag_import.consumer.amqp.wallabag_v1:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v1.import"
- "@logger"
wallabag_import.consumer.amqp.wallabag_v2:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v2.import"
- "@logger"

81
src/Wallabag/ImportBundle/Resources/config/redis.yml Normal file
View file

@ -0,0 +1,81 @@
# Redis stuff
services:
# readability
wallabag_import.queue.redis.readability:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.readability"
wallabag_import.producer.redis.readability:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.readability"
wallabag_import.consumer.redis.readability:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.readability.import"
- "@logger"
# pocket
wallabag_import.queue.redis.pocket:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.pocket"
wallabag_import.producer.redis.pocket:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.pocket"
wallabag_import.consumer.redis.pocket:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.pocket.import"
- "@logger"
# wallabag v1
wallabag_import.queue.redis.wallabag_v1:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.wallabag_v1"
wallabag_import.producer.redis.wallabag_v1:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.wallabag_v1"
wallabag_import.consumer.redis.wallabag_v1:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v1.import"
- "@logger"
# wallabag v2
wallabag_import.queue.redis.wallabag_v2:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.wallabag_v2"
wallabag_import.producer.redis.wallabag_v2:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.wallabag_v2"
wallabag_import.consumer.redis.wallabag_v2:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v2.import"
- "@logger"

5
src/Wallabag/ImportBundle/Resources/config/services.yml
View file

@ -1,3 +1,7 @@
imports:
- { resource: rabbit.yml }
- { resource: redis.yml }
services:
wallabag_import.chain:
class: Wallabag\ImportBundle\Import\ImportChain
@ -14,7 +18,6 @@ services:
wallabag_import.pocket.import:
class: Wallabag\ImportBundle\Import\PocketImport
arguments:
- "@security.token_storage"
- "@doctrine.orm.entity_manager"
- "@wallabag_core.content_proxy"
- "@craue_config"

8
src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig Normal file
View file

@ -0,0 +1,8 @@
{% set redis = craue_setting('import_with_redis') %}
{% set rabbit = craue_setting('import_with_rabbitmq') %}
{% if redis or rabbit %}
<div class="card-panel yellow darken-1 black-text">
{{ 'import.worker.enabled'|trans }} <strong>{% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %}</strong>
</div>
{% endif %}

11
src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig Normal file
View file

@ -0,0 +1,11 @@
{% if nbRedisMessages > 0 %}
<script>
Materialize.toast('Messages in queue: {{ nbRedisMessages }}', 4000);
</script>
{% endif %}
{% if nbRabbitMessages > 0 %}
<script>
Materialize.toast('Messages in queue: {{ nbRabbitMessages }}', 4000);
</script>
{% endif %}

12
src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig
View file

@ -6,15 +6,13 @@
<div class="row">
<div class="col s12">
<div class="card-panel settings">
{% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
{% if not has_consumer_key %}
<div class="card-panel red darken-1">
<div class="card-panel red white-text">
{{ 'import.pocket.config_missing.description'|trans }}
{% if is_granted('ROLE_SUPER_ADMIN') %}
{{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('craue_config_settings_modify') ~ '#set-import">', '%keyurle%':'</a>'})|raw }}
{% else %}
{{ 'import.pocket.config_missing.user_message'|trans }}
{% endif %}
{{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('config') ~ '">', '%keyurle%':'</a>'})|raw }}
</div>
{% endif %}
@ -29,7 +27,7 @@
{{ form_label(form.mark_as_read) }}
</div>
</div>
<button class="btn waves-effect waves-light" type="submit" name="action">
<button class="btn waves-effect waves-light" type="submit" name="action" {% if not has_consumer_key %}disabled="disabled"{% endif %}>
{{ 'import.pocket.connect_to_pocket'|trans }}
</button>
</form>

2
src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
View file

@ -6,6 +6,8 @@
<div class="row">
<div class="col s12">
<div class="card-panel settings">
{% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
<div class="row">
<blockquote>{{ import.description|trans }}</blockquote>
<p>{{ 'import.readability.how_to'|trans }}</p>

2
src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig
View file

@ -6,6 +6,8 @@
<div class="row">
<div class="col s12">
<div class="card-panel settings">
{% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
<div class="row">
<blockquote>{{ import.description|trans }}</blockquote>
<p>{{ 'import.wallabag_v1.how_to'|trans }}</p>

6
src/Wallabag/UserBundle/Resources/config/services.yml
View file

@ -14,3 +14,9 @@ services:
- "@router"
tags:
- { name: kernel.event_subscriber }
wallabag_user.user_repository:
class: Wallabag\UserBundle\Repository\UserRepository
factory: [ "@doctrine.orm.default_entity_manager", getRepository ]
arguments:
- WallabagUserBundle:User

86
tests/Wallabag/ImportBundle/Command/ImportCommandTest.php Normal file
View file

@ -0,0 +1,86 @@
<?php
namespace Tests\Wallabag\ImportBundle\Command;
use Symfony\Bundle\FrameworkBundle\Console\Application;
use Symfony\Component\Console\Tester\CommandTester;
use Wallabag\ImportBundle\Command\ImportCommand;
use Tests\Wallabag\CoreBundle\WallabagCoreTestCase;
use M6Web\Component\RedisMock\RedisMockFactory;
class ImportCommandTest extends WallabagCoreTestCase
{
/**
* @expectedException Symfony\Component\Console\Exception\RuntimeException
* @expectedExceptionMessage Not enough arguments
*/
public function testRunImportCommandWithoutArguments()
{
$application = new Application($this->getClient()->getKernel());
$application->add(new ImportCommand());
$command = $application->find('wallabag:import');
$tester = new CommandTester($command);
$tester->execute([
'command' => $command->getName(),
]);
}
/**
* @expectedException Symfony\Component\Config\Definition\Exception\Exception
* @expectedExceptionMessage not found
*/
public function testRunImportCommandWithoutFilepath()
{
$application = new Application($this->getClient()->getKernel());
$application->add(new ImportCommand());
$command = $application->find('wallabag:import');
$tester = new CommandTester($command);
$tester->execute([
'command' => $command->getName(),
'userId' => 1,
'filepath' => 1,
]);
}
/**
* @expectedException Symfony\Component\Config\Definition\Exception\Exception
* @expectedExceptionMessage User with id
*/
public function testRunImportCommandWithoutUserId()
{
$application = new Application($this->getClient()->getKernel());
$application->add(new ImportCommand());
$command = $application->find('wallabag:import');
$tester = new CommandTester($command);
$tester->execute([
'command' => $command->getName(),
'userId' => 0,
'filepath' => './',
]);
}
public function testRunImportCommand()
{
$application = new Application($this->getClient()->getKernel());
$application->add(new ImportCommand());
$command = $application->find('wallabag:import');
$tester = new CommandTester($command);
$tester->execute([
'command' => $command->getName(),
'userId' => 1,
'filepath' => $application->getKernel()->getContainer()->getParameter('kernel.root_dir').'/../tests/Wallabag/ImportBundle/fixtures/wallabag-v2-read.json',
'--importer' => 'v2',
]);
$this->assertContains('imported', $tester->getDisplay());
$this->assertContains('already saved', $tester->getDisplay());
}
}

74
tests/Wallabag/ImportBundle/Command/RedisWorkerCommandTest.php Normal file
View file

@ -0,0 +1,74 @@
<?php
namespace Tests\Wallabag\ImportBundle\Command;
use Symfony\Bundle\FrameworkBundle\Console\Application;
use Symfony\Component\Console\Tester\CommandTester;
use Wallabag\ImportBundle\Command\RedisWorkerCommand;
use Tests\Wallabag\CoreBundle\WallabagCoreTestCase;
use M6Web\Component\RedisMock\RedisMockFactory;
class RedisWorkerCommandTest extends WallabagCoreTestCase
{
/**
* @expectedException Symfony\Component\Console\Exception\RuntimeException
* @expectedExceptionMessage Not enough arguments (missing: "serviceName")
*/
public function testRunRedisWorkerCommandWithoutArguments()
{
$application = new Application($this->getClient()->getKernel());
$application->add(new RedisWorkerCommand());
$command = $application->find('wallabag:import:redis-worker');
$tester = new CommandTester($command);
$tester->execute([
'command' => $command->getName(),
]);
}
/**
* @expectedException Symfony\Component\Config\Definition\Exception\Exception
* @expectedExceptionMessage No queue or consumer found for service name
*/
public function testRunRedisWorkerCommandWithBadService()
{
$application = new Application($this->getClient()->getKernel());
$application->add(new RedisWorkerCommand());
$command = $application->find('wallabag:import:redis-worker');
$tester = new CommandTester($command);
$tester->execute([
'command' => $command->getName(),
'serviceName' => 'YOMONSERVICE',
]);
}
public function testRunRedisWorkerCommand()
{
$application = new Application($this->getClient()->getKernel());
$application->add(new RedisWorkerCommand());
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$application->getKernel()->getContainer()->set('wallabag_core.redis.client', $redisMock);
// put a fake message in the queue so the worker will stop after reading that message
// instead of waiting for others
$redisMock->lpush('wallabag.import.readability', '{}');
$command = $application->find('wallabag:import:redis-worker');
$tester = new CommandTester($command);
$tester->execute([
'command' => $command->getName(),
'serviceName' => 'readability',
'--maxIterations' => 1,
]);
$this->assertContains('Worker started at', $tester->getDisplay());
$this->assertContains('Waiting for message', $tester->getDisplay());
}
}

225
tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php Normal file
View file

@ -0,0 +1,225 @@
<?php
namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
use Wallabag\ImportBundle\Consumer\AMQPEntryConsumer;
use PhpAmqpLib\Message\AMQPMessage;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
class AMQPEntryConsumerTest extends \PHPUnit_Framework_TestCase
{
public function testMessageOk()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->once())
->method('flush');
$em
->expects($this->exactly(2))
->method('clear');
$body = <<<'JSON'
{
"item_id": "1402935436",
"resolved_id": "1402935436",
"given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
"given_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
"favorite": "0",
"status": "0",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 0,
"resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
"resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
"excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...",
"is_article": "1",
"is_index": "0",
"has_video": "0",
"has_image": "1",
"word_count": "200",
"tags": {
"ifttt": {
"item_id": "1402935436",
"tag": "ifttt"
},
"mashable": {
"item_id": "1402935436",
"tag": "mashable"
}
},
"authors": {
"2484273": {
"item_id": "1402935436",
"author_id": "2484273",
"name": "Adam Rosenberg",
"url": "http://mashable.com/author/adam-rosenberg/"
}
},
"image": {
"item_id": "1402935436",
"src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
"width": "0",
"height": "0"
},
"images": {
"1": {
"item_id": "1402935436",
"image_id": "1",
"src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
"width": "0",
"height": "0",
"credit": "Image: Steve Eichner/NameFace/Sipa USA",
"caption": ""
}
},
"userId": 1
}
JSON;
$user = new User();
$entry = new Entry($user);
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(1)
->willReturn($user);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$import
->expects($this->once())
->method('setUser')
->with($user);
$import
->expects($this->once())
->method('parseEntry')
->with(json_decode($body, true))
->willReturn($entry);
$consumer = new AMQPEntryConsumer(
$em,
$userRepository,
$import
);
$message = new AMQPMessage($body);
$consumer->execute($message);
}
public function testMessageWithBadUser()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->never())
->method('flush');
$em
->expects($this->never())
->method('clear');
$body = '{ "userId": 123 }';
$user = new User();
$entry = new Entry($user);
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(123)
->willReturn(null);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$consumer = new AMQPEntryConsumer(
$em,
$userRepository,
$import
);
$message = new AMQPMessage($body);
$consumer->execute($message);
}
public function testMessageWithEntryProcessed()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->never())
->method('flush');
$em
->expects($this->never())
->method('clear');
$body = '{ "userId": 123 }';
$user = new User();
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(123)
->willReturn($user);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$import
->expects($this->once())
->method('setUser')
->with($user);
$import
->expects($this->once())
->method('parseEntry')
->with(json_decode($body, true))
->willReturn(null);
$consumer = new AMQPEntryConsumer(
$em,
$userRepository,
$import
);
$message = new AMQPMessage($body);
$consumer->execute($message);
}
}

225
tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php Normal file
View file

@ -0,0 +1,225 @@
<?php
namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
use Wallabag\ImportBundle\Consumer\RedisEntryConsumer;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
class RedisEntryConsumerTest extends \PHPUnit_Framework_TestCase
{
public function testMessageOk()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->once())
->method('flush');
$em
->expects($this->exactly(2))
->method('clear');
$body = <<<'JSON'
{
"item_id": "1402935436",
"resolved_id": "1402935436",
"given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
"given_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
"favorite": "0",
"status": "0",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 0,
"resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
"resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
"excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...",
"is_article": "1",
"is_index": "0",
"has_video": "0",
"has_image": "1",
"word_count": "200",
"tags": {
"ifttt": {
"item_id": "1402935436",
"tag": "ifttt"
},
"mashable": {
"item_id": "1402935436",
"tag": "mashable"
}
},
"authors": {
"2484273": {
"item_id": "1402935436",
"author_id": "2484273",
"name": "Adam Rosenberg",
"url": "http://mashable.com/author/adam-rosenberg/"
}
},
"image": {
"item_id": "1402935436",
"src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
"width": "0",
"height": "0"
},
"images": {
"1": {
"item_id": "1402935436",
"image_id": "1",
"src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
"width": "0",
"height": "0",
"credit": "Image: Steve Eichner/NameFace/Sipa USA",
"caption": ""
}
},
"userId": 1
}
JSON;
$user = new User();
$entry = new Entry($user);
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(1)
->willReturn($user);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$import
->expects($this->once())
->method('setUser')
->with($user);
$import
->expects($this->once())
->method('parseEntry')
->with(json_decode($body, true))
->willReturn($entry);
$consumer = new RedisEntryConsumer(
$em,
$userRepository,
$import
);
$res = $consumer->manage($body);
$this->assertTrue($res);
}
public function testMessageWithBadUser()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->never())
->method('flush');
$em
->expects($this->never())
->method('clear');
$body = '{ "userId": 123 }';
$user = new User();
$entry = new Entry($user);
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(123)
->willReturn(null);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$consumer = new RedisEntryConsumer(
$em,
$userRepository,
$import
);
$res = $consumer->manage($body);
$this->assertFalse($res);
}
public function testMessageWithEntryProcessed()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->never())
->method('flush');
$em
->expects($this->never())
->method('clear');
$body = '{ "userId": 123 }';
$user = new User();
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(123)
->willReturn($user);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$import
->expects($this->once())
->method('setUser')
->with($user);
$import
->expects($this->once())
->method('parseEntry')
->with(json_decode($body, true))
->willReturn(null);
$consumer = new RedisEntryConsumer(
$em,
$userRepository,
$import
);
$res = $consumer->manage($body);
$this->assertFalse($res);
$this->assertFalse($consumer->isStopJob($body));
}
}

30
tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php
View file

@ -17,6 +17,36 @@ class PocketControllerTest extends WallabagCoreTestCase
$this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
}
public function testImportPocketWithRabbitEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/pocket');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
}
public function testImportPocketWithRedisEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_redis', 1);
$crawler = $client->request('GET', '/import/pocket');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
$client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportPocketAuthBadToken()
{
$this->logInAs('admin');

75
tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php
View file

@ -19,6 +19,74 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
}
public function testImportReadabilityWithRabbitEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/readability');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
}
public function testImportReadabilityBadFile()
{
$this->logInAs('admin');
$client = $this->getClient();
$crawler = $client->request('GET', '/import/readability');
$form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
$data = [
'upload_import_file[file]' => '',
];
$client->submit($form, $data);
$this->assertEquals(200, $client->getResponse()->getStatusCode());
}
public function testImportReadabilityWithRedisEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_redis', 1);
$crawler = $client->request('GET', '/import/readability');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
$file = new UploadedFile(__DIR__.'/../fixtures/readability.json', 'readability.json');
$data = [
'upload_import_file[file]' => $file,
];
$client->submit($form, $data);
$this->assertEquals(302, $client->getResponse()->getStatusCode());
$crawler = $client->followRedirect();
$this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
$this->assertContains('flashes.import.notice.summary', $body[0]);
$this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.readability'));
$client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportReadabilityWithFile()
{
$this->logInAs('admin');
@ -49,6 +117,13 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
$this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
$this->assertContains('flashes.import.notice.summary', $body[0]);
$this->assertNotEmpty($content->getMimetype());
$this->assertNotEmpty($content->getPreviewPicture());
$this->assertNotEmpty($content->getLanguage());
$this->assertEquals(0, count($content->getTags()));
$this->assertInstanceOf(\DateTime::class, $content->getCreatedAt());
$this->assertEquals('2016-08-25', $content->getCreatedAt()->format('Y-m-d'));
}
public function testImportReadabilityWithFileAndMarkAllAsRead()

74
tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php
View file

@ -19,6 +19,74 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
}
public function testImportWallabagWithRabbitEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/wallabag-v1');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
}
public function testImportWallabagBadFile()
{
$this->logInAs('admin');
$client = $this->getClient();
$crawler = $client->request('GET', '/import/wallabag-v1');
$form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
$data = [
'upload_import_file[file]' => '',
];
$client->submit($form, $data);
$this->assertEquals(200, $client->getResponse()->getStatusCode());
}
public function testImportWallabagWithRedisEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_redis', 1);
$crawler = $client->request('GET', '/import/wallabag-v1');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
$file = new UploadedFile(__DIR__.'/../fixtures/wallabag-v1.json', 'wallabag-v1.json');
$data = [
'upload_import_file[file]' => $file,
];
$client->submit($form, $data);
$this->assertEquals(302, $client->getResponse()->getStatusCode());
$crawler = $client->followRedirect();
$this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
$this->assertContains('flashes.import.notice.summary', $body[0]);
$this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.wallabag_v1'));
$client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportWallabagWithFile()
{
$this->logInAs('admin');
@ -56,6 +124,12 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
$this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
$this->assertContains('flashes.import.notice.summary', $body[0]);
$this->assertEmpty($content->getMimetype());
$this->assertEmpty($content->getPreviewPicture());
$this->assertEmpty($content->getLanguage());
$this->assertEquals(1, count($content->getTags()));
$this->assertInstanceOf(\DateTime::class, $content->getCreatedAt());
}
public function testImportWallabagWithFileAndMarkAllAsRead()

76
tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php
View file

@ -19,6 +19,74 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
}
public function testImportWallabagWithRabbitEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/wallabag-v2');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
}
public function testImportWallabagBadFile()
{
$this->logInAs('admin');
$client = $this->getClient();
$crawler = $client->request('GET', '/import/wallabag-v2');
$form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
$data = [
'upload_import_file[file]' => '',
];
$client->submit($form, $data);
$this->assertEquals(200, $client->getResponse()->getStatusCode());
}
public function testImportWallabagWithRedisEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_redis', 1);
$crawler = $client->request('GET', '/import/wallabag-v2');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form();
$file = new UploadedFile(__DIR__.'/../fixtures/wallabag-v2.json', 'wallabag-v2.json');
$data = [
'upload_import_file[file]' => $file,
];
$client->submit($form, $data);
$this->assertEquals(302, $client->getResponse()->getStatusCode());
$crawler = $client->followRedirect();
$this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text']));
$this->assertContains('flashes.import.notice.summary', $body[0]);
$this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.wallabag_v2'));
$client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportWallabagWithFile()
{
$this->logInAs('admin');
@ -50,9 +118,9 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
$this->getLoggedInUserId()
);
$this->assertEmpty($content->getMimetype());
$this->assertEmpty($content->getPreviewPicture());
$this->assertEmpty($content->getLanguage());
$this->assertNotEmpty($content->getMimetype());
$this->assertNotEmpty($content->getPreviewPicture());
$this->assertNotEmpty($content->getLanguage());
$this->assertEquals(0, count($content->getTags()));
$content = $client->getContainer()
@ -67,6 +135,8 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
$this->assertNotEmpty($content->getPreviewPicture());
$this->assertNotEmpty($content->getLanguage());
$this->assertEquals(2, count($content->getTags()));
$this->assertInstanceOf(\DateTime::class, $content->getCreatedAt());
$this->assertEquals('2016-09-08', $content->getCreatedAt()->format('Y-m-d'));
}
public function testImportWallabagWithEmptyFile()

242
tests/Wallabag/ImportBundle/Import/PocketImportTest.php
View file

@ -4,21 +4,17 @@ namespace Tests\Wallabag\ImportBundle\Import;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\Config;
use Wallabag\ImportBundle\Import\PocketImport;
use GuzzleHttp\Client;
use GuzzleHttp\Subscriber\Mock;
use GuzzleHttp\Message\Response;
use GuzzleHttp\Stream\Stream;
use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
class PocketImportMock extends PocketImport
{
public function getAccessToken()
{
return $this->accessToken;
}
}
use Simpleue\Queue\RedisQueue;
use M6Web\Component\RedisMock\RedisMockFactory;
class PocketImportTest extends \PHPUnit_Framework_TestCase
{
@ -32,45 +28,24 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
{
$this->user = new User();
$this->tokenStorage = $this->getMockBuilder('Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface')
->disableOriginalConstructor()
->getMock();
$config = new Config($this->user);
$config->setPocketConsumerKey('xxx');
$token = $this->getMockBuilder('Symfony\Component\Security\Core\Authentication\Token\TokenInterface')
->disableOriginalConstructor()
->getMock();
$this->user->setConfig($config);
$this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy')
->disableOriginalConstructor()
->getMock();
$token->expects($this->once())
->method('getUser')
->willReturn($this->user);
$this->tokenStorage->expects($this->once())
->method('getToken')
->willReturn($token);
$this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$config = $this->getMockBuilder('Craue\ConfigBundle\Util\Config')
->disableOriginalConstructor()
->getMock();
$config->expects($this->any())
->method('get')
->with('pocket_consumer_key')
->willReturn($consumerKey);
$pocket = new PocketImportMock(
$this->tokenStorage,
$pocket = new PocketImport(
$this->em,
$this->contentProxy,
$config
$this->contentProxy
);
$pocket->setUser($this->user);
$this->logHandler = new TestHandler();
$logger = new Logger('test', [$this->logHandler]);
@ -189,10 +164,16 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
"given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
"favorite": "1",
"status": "1",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 0,
"resolved_title": "The Massive Ryder Cup Preview",
"resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
"excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
"is_article": "1",
"is_index": "0",
"has_video": "1",
"has_image": "1",
"word_count": "3197",
@ -236,10 +217,16 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
"given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
"favorite": "1",
"status": "1",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 1,
"resolved_title": "The Massive Ryder Cup Preview",
"resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
"excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
"is_article": "1",
"is_index": "0",
"has_video": "0",
"has_image": "0",
"word_count": "3197"
@ -279,7 +266,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
$res = $pocketImport->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 1, 'imported' => 1], $pocketImport->getSummary());
$this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $pocketImport->getSummary());
}
/**
@ -302,6 +289,11 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
"given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
"favorite": "1",
"status": "1",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 0,
"excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
"is_article": "1",
"has_video": "1",
@ -315,6 +307,11 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
"given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
"favorite": "1",
"status": "0",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 1,
"excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
"is_article": "1",
"has_video": "0",
@ -364,7 +361,174 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
$res = $pocketImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 2], $pocketImport->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 2, 'queued' => 0], $pocketImport->getSummary());
}
/**
* Will sample results from https://getpocket.com/developer/docs/v3/retrieve.
*/
public function testImportWithRabbit()
{
$client = new Client();
$body = <<<'JSON'
{
"item_id": "229279689",
"resolved_id": "229279689",
"given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
"given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
"favorite": "1",
"status": "1",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 0,
"resolved_title": "The Massive Ryder Cup Preview",
"resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
"excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
"is_article": "1",
"has_video": "0",
"has_image": "0",
"word_count": "3197"
}
JSON;
$mock = new Mock([
new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))),
new Response(200, ['Content-Type' => 'application/json'], Stream::factory('
{
"status": 1,
"list": {
"229279690": '.$body.'
}
}
')),
]);
$client->getEmitter()->attach($mock);
$pocketImport = $this->getPocketImport();
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = new Entry($this->user);
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
->disableOriginalConstructor()
->getMock();
$bodyAsArray = json_decode($body, true);
// because with just use `new User()` so it doesn't have an id
$bodyAsArray['userId'] = null;
$producer
->expects($this->once())
->method('publish')
->with(json_encode($bodyAsArray));
$pocketImport->setClient($client);
$pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $pocketImport->getSummary());
}
/**
* Will sample results from https://getpocket.com/developer/docs/v3/retrieve.
*/
public function testImportWithRedis()
{
$client = new Client();
$body = <<<'JSON'
{
"item_id": "229279689",
"resolved_id": "229279689",
"given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
"given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
"favorite": "1",
"status": "1",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 0,
"resolved_title": "The Massive Ryder Cup Preview",
"resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
"excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
"is_article": "1",
"has_video": "0",
"has_image": "0",
"word_count": "3197"
}
JSON;
$mock = new Mock([
new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))),
new Response(200, ['Content-Type' => 'application/json'], Stream::factory('
{
"status": 1,
"list": {
"229279690": '.$body.'
}
}
')),
]);
$client->getEmitter()->attach($mock);
$pocketImport = $this->getPocketImport();
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = new Entry($this->user);
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$queue = new RedisQueue($redisMock, 'pocket');
$producer = new Producer($queue);
$pocketImport->setClient($client);
$pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $pocketImport->getSummary());
$this->assertNotEmpty($redisMock->lpop('pocket'));
}
public function testImportBadResponse()
@ -402,6 +566,8 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
"status": 1,
"list": {
"229279689": {
"status": "1",
"favorite": "1",
"resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview"
}
}
@ -439,6 +605,6 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase
$res = $pocketImport->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 1, 'imported' => 0], $pocketImport->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 1, 'queued' => 0], $pocketImport->getSummary());
}
}

97
tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php
View file

@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
use Wallabag\ImportBundle\Import\ReadabilityImport;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
use Simpleue\Queue\RedisQueue;
use M6Web\Component\RedisMock\RedisMockFactory;
class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
{
@ -58,9 +61,9 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->exactly(2))
$entryRepo->expects($this->exactly(24))
->method('findByUrlAndUserId')
->will($this->onConsecutiveCalls(false, true));
->willReturn(false);
$this->em
->expects($this->any())
@ -72,14 +75,14 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
->getMock();
$this->contentProxy
->expects($this->exactly(1))
->expects($this->exactly(24))
->method('updateEntry')
->willReturn($entry);
$res = $readabilityImport->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 1, 'imported' => 1], $readabilityImport->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 24, 'queued' => 0], $readabilityImport->getSummary());
}
public function testImportAndMarkAllAsRead()
@ -93,7 +96,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
$entryRepo->expects($this->exactly(2))
->method('findByUrlAndUserId')
->will($this->onConsecutiveCalls(false, false));
->will($this->onConsecutiveCalls(false, true));
$this->em
->expects($this->any())
@ -101,7 +104,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
->willReturn($entryRepo);
$this->contentProxy
->expects($this->exactly(2))
->expects($this->exactly(1))
->method('updateEntry')
->willReturn(new Entry($this->user));
@ -117,7 +120,87 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
$this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $readabilityImport->getSummary());
}
public function testImportWithRabbit()
{
$readabilityImport = $this->getReadabilityImport();
$readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
->disableOriginalConstructor()
->getMock();
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
->disableOriginalConstructor()
->getMock();
$producer
->expects($this->exactly(24))
->method('publish');
$readabilityImport->setProducer($producer);
$res = $readabilityImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $readabilityImport->getSummary());
}
public function testImportWithRedis()
{
$readabilityImport = $this->getReadabilityImport();
$readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
->disableOriginalConstructor()
->getMock();
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$queue = new RedisQueue($redisMock, 'readability');
$producer = new Producer($queue);
$readabilityImport->setProducer($producer);
$res = $readabilityImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $readabilityImport->getSummary());
$this->assertNotEmpty($redisMock->lpop('readability'));
}
public function testImportBadFile()

87
tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php
View file

@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
use Wallabag\ImportBundle\Import\WallabagV1Import;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
use Simpleue\Queue\RedisQueue;
use M6Web\Component\RedisMock\RedisMockFactory;
class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
{
@ -79,7 +82,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV1Import->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 1, 'imported' => 3], $wallabagV1Import->getSummary());
$this->assertEquals(['skipped' => 1, 'imported' => 3, 'queued' => 0], $wallabagV1Import->getSummary());
}
public function testImportAndMarkAllAsRead()
@ -117,7 +120,87 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 3], $wallabagV1Import->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 3, 'queued' => 0], $wallabagV1Import->getSummary());
}
public function testImportWithRabbit()
{
$wallabagV1Import = $this->getWallabagV1Import();
$wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
->disableOriginalConstructor()
->getMock();
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
->disableOriginalConstructor()
->getMock();
$producer
->expects($this->exactly(4))
->method('publish');
$wallabagV1Import->setProducer($producer);
$res = $wallabagV1Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $wallabagV1Import->getSummary());
}
public function testImportWithRedis()
{
$wallabagV1Import = $this->getWallabagV1Import();
$wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
->disableOriginalConstructor()
->getMock();
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$queue = new RedisQueue($redisMock, 'wallabag_v1');
$producer = new Producer($queue);
$wallabagV1Import->setProducer($producer);
$res = $wallabagV1Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $wallabagV1Import->getSummary());
$this->assertNotEmpty($redisMock->lpop('wallabag_v1'));
}
public function testImportBadFile()

83
tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php
View file

@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
use Wallabag\ImportBundle\Import\WallabagV2Import;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
use Simpleue\Queue\RedisQueue;
use M6Web\Component\RedisMock\RedisMockFactory;
class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
{
@ -75,7 +78,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV2Import->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 22, 'imported' => 2], $wallabagV2Import->getSummary());
$this->assertEquals(['skipped' => 22, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
}
public function testImportAndMarkAllAsRead()
@ -113,7 +116,79 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 2], $wallabagV2Import->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
}
public function testImportWithRabbit()
{
$wallabagV2Import = $this->getWallabagV2Import();
$wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
->disableOriginalConstructor()
->getMock();
$producer
->expects($this->exactly(24))
->method('publish');
$wallabagV2Import->setProducer($producer);
$res = $wallabagV2Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $wallabagV2Import->getSummary());
}
public function testImportWithRedis()
{
$wallabagV2Import = $this->getWallabagV2Import();
$wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$queue = new RedisQueue($redisMock, 'wallabag_v2');
$producer = new Producer($queue);
$wallabagV2Import->setProducer($producer);
$res = $wallabagV2Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $wallabagV2Import->getSummary());
$this->assertNotEmpty($redisMock->lpop('wallabag_v2'));
}
public function testImportBadFile()
@ -152,7 +227,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV2Import->import();
$this->assertFalse($res);
$this->assertEquals(['skipped' => 0, 'imported' => 0], $wallabagV2Import->getSummary());
$this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 0], $wallabagV2Import->getSummary());
}
public function testImportWithExceptionFromGraby()
@ -181,6 +256,6 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$res = $wallabagV2Import->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 24, 'imported' => 0], $wallabagV2Import->getSummary());
$this->assertEquals(['skipped' => 22, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary());
}
}

167
tests/Wallabag/ImportBundle/fixtures/readability.json
View file

@ -11,14 +11,165 @@
"archive": false
},
{
"article__excerpt": "TL;DR: Re-use your DOM elements and remove the ones that are far away from the viewport. Use placeholders to account for delayed data. Here&#x2019;s a demo and the code for the infinite&hellip;",
"favorite": false,
"date_archived": "2016-08-26T12:21:54",
"article__url": "https://developers.google.com/web/updates/2016/07/infinite-scroller?imm_mid=0e6839&cmp=em-webops-na-na-newsltr_20160805",
"date_added": "2016-08-06T05:35:26",
"date_favorited": null,
"article__title": "Complexities of an infinite scroller | Web Updates",
"archive": true
"article__title": "Réfugiés: l'UE va créer 100 000 places d'accueil dans les Balkans",
"article__url": "http://www.liberation.fr/planete/2015/10/26/refugies-l-ue-va-creer-100-000-places-d-accueil-dans-les-balkans_1408867",
"archive": false,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": false
},
{
"article__title": "No title found",
"article__url": "http://news.nationalgeographic.com/2016/02/160211-albatrosses-mothers-babies-animals-science/&sf20739758=1",
"archive": false,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": true
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Échecs",
"article__url": "https://fr.wikipedia.org/wiki/Échecs"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "90% des dossiers médicaux des Coréens du sud vendus à des entreprises privées - ZATAZ",
"article__url": "http://www.zataz.com/90-des-dossiers-medicaux-des-coreens-du-sud-vendus-a-des-entreprises-privees/"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Mass Surveillance As Art",
"article__url": "https://www.nationaljournal.com/s/73311/mass-surveillance-art"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "What David Cameron did to the pig, his party is now doing to the country",
"article__url": "http://www.newstatesman.com/2015/09/what-david-cameron-did-pig-his-party-now-doing-country"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "CLICK HERE to support 2016 CES Winner, Revolutionary Auto-Tracking Robot",
"article__url": "https://www.indiegogo.com/projects/2016-ces-winner-revolutionary-auto-tracking-robot"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 1,
"article__title": "No title found",
"article__url": "http://carnetdevol.shost.ca/wordpress/aide-memoire-sur-les-commandes-associees-a-systemd/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Présentation d'Arduino - Tuto Arduino - Le blog d'Eskimon",
"article__url": "http://eskimon.fr/73-arduino-101-presentation"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Lenovo ThinkPad X1 Carbon Ultrabook Review",
"article__url": "http://www.notebookcheck.net/Lenovo-ThinkPad-X1-Carbon-Ultrabook-Review.138033.0.html"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Visitons le Château de Landsberg !",
"article__url": "http://autour-du-mont-sainte-odile.overblog.com/2016/01/visitons-le-chateau-de-landsberg.html"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Contrer les stéréotypes par les livres : “C'est dès l'enfance qu'ils se construisent”",
"article__url": "https://www.actualitte.com/article/monde-edition/contrer-les-stereotypes-par-les-livres-c-est-des-l-enfance-qu-ils-se-construisent/64058"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "[ROM][6.0.1][Layers][N5] TipsyOS official builds {UBER TCs}",
"article__url": "http://forum.xda-developers.com/google-nexus-5/development/rom-tipsyos-official-builds-uber-tcs-t3325989"
},
{
"archive": 0,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Top 15 Podcasts All Web Developers Should Follow - Envato Tuts+ Code Article",
"article__url": "http://code.tutsplus.com/articles/top-15-podcasts-all-web-developers-should-follow--net-14461"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "University of Mississippi",
"article__url": "http://olemiss.edu"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "FinnChristiansen.de Jetzt Dank Let’s Encrypt Per HTTPS Erreichbar",
"article__url": "https://www.finnchristiansen.de/2015/12/06/finnchristiansen-de-jetzt-dank-lets-encrypt-per-https-erreichbar/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Le développeur et l'ingénierie logicielle",
"article__url": "http://wemucs.com/le-developpeur-et-lingenierie-logicielle/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "The Role of Methylation in Gene Expression",
"article__url": "http://www.nature.com/scitable/topicpage/the-role-of-methylation-in-gene-expression-1070"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "E-Mail-Adresse kostenlos, FreeMail, De-Mail & Nachrichten",
"article__url": "http://web.de"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "OpenSSH Server on Arch Linux | DominicM test",
"article__url": "http://dominicm.com/openssh-server-arch-linux/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Site Moved | Site Help",
"article__url": "http://g1.com/help/sitemoved.asp"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "#Maroc : le stylo anti-pédophiles EAGLE d’AMESYS est moins bien configuré que les faux-lowers Twitter du roi Mohammed VI",
"article__url": "https://reflets.info/maroc-le-stylo-anti-pedophiles-eagle-damesys-est-moins-bien-configure-que-les-faux-lowers-twitter-du-roi-mohammed-vi/"
},
{
"archive": 1,
"date_added": "2016-09-08T11:55:58+0200",
"favorite": 0,
"article__title": "Simple Cloud Infrastructure for Developers",
"article__url": "https://www.digitalocean.com/"
}
],
"recommendations": []