From 2af48b81749a913a6a05ecddbb3413e2bfb0ed21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 26 Jul 2023 12:49:30 +0200 Subject: [PATCH] Add Shaarli and Pocket HTML imports --- app/config/config.yml | 28 ++ app/config/services.yml | 18 ++ app/config/services_rabbit.yml | 12 + app/config/services_redis.yml | 32 +++ app/config/wallabag.yml | 2 +- phpstan-baseline.neon | 10 + .../ImportBundle/Command/ImportCommand.php | 38 ++- .../Consumer/RabbitMQConsumerTotalProxy.php | 26 +- .../Controller/HtmlController.php | 83 ++++++ .../Controller/ImportController.php | 4 + .../Controller/PocketHtmlController.php | 57 ++++ .../Controller/ShaarliController.php | 57 ++++ .../ImportBundle/Import/HtmlImport.php | 210 +++++++++++++++ .../ImportBundle/Import/PocketHtmlImport.php | 113 ++++++++ .../ImportBundle/Import/ShaarliImport.php | 66 +++++ .../views/PocketHtml/index.html.twig | 45 ++++ .../Resources/views/Shaarli/index.html.twig | 45 ++++ .../Controller/FirefoxControllerTest.php | 6 +- .../Controller/ImportControllerTest.php | 2 +- .../Controller/PocketHtmlControllerTest.php | 168 ++++++++++++ .../Controller/ShaarliControllerTest.php | 168 ++++++++++++ .../Import/PocketHtmlImportTest.php | 254 ++++++++++++++++++ .../ImportBundle/Import/ShaarliImportTest.php | 254 ++++++++++++++++++ .../ImportBundle/fixtures/ril_export.html | 21 ++ .../fixtures/shaarli-bookmarks.html | 13 + translations/messages.en.yml | 8 + 26 files changed, 1730 insertions(+), 10 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/HtmlController.php create mode 100644 src/Wallabag/ImportBundle/Controller/PocketHtmlController.php create mode 100644 src/Wallabag/ImportBundle/Controller/ShaarliController.php create mode 100644 src/Wallabag/ImportBundle/Import/HtmlImport.php create mode 100644 src/Wallabag/ImportBundle/Import/PocketHtmlImport.php create mode 100644 src/Wallabag/ImportBundle/Import/ShaarliImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig create mode 100644 src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig create mode 100644 tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php create mode 100644 tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php create mode 100644 tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php create mode 100644 tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php create mode 100644 tests/Wallabag/ImportBundle/fixtures/ril_export.html create mode 100644 tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html diff --git a/app/config/config.yml b/app/config/config.yml index b31b29e0d..5bf78df15 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -278,6 +278,16 @@ old_sound_rabbit_mq: exchange_options: name: 'wallabag.import.chrome' type: topic + import_shaarli: + connection: default + exchange_options: + name: 'wallabag.import.shaarli' + type: topic + import_pocket_html: + connection: default + exchange_options: + name: 'wallabag.import.pocket_html' + type: topic consumers: import_pocket: connection: default @@ -369,6 +379,24 @@ old_sound_rabbit_mq: name: 'wallabag.import.chrome' callback: wallabag_import.consumer.amqp.chrome qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} + import_shaarli: + connection: default + exchange_options: + name: 'wallabag.import.shaarli' + type: topic + queue_options: + name: 'wallabag.import.shaarli' + callback: wallabag_import.consumer.amqp.shaarli + qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} + import_pocket_html: + connection: default + exchange_options: + name: 'wallabag.import.pocket_html' + type: topic + queue_options: + name: 'wallabag.import.pocket_html' + callback: wallabag_import.consumer.amqp.pocket_html + qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} fos_js_routing: routes_to_expose: diff --git a/app/config/services.yml b/app/config/services.yml index 605e6383f..891182685 100644 --- a/app/config/services.yml +++ b/app/config/services.yml @@ -116,6 +116,16 @@ services: $rabbitMqProducer: '@old_sound_rabbit_mq.import_wallabag_v2_producer' $redisProducer: '@wallabag_import.producer.redis.wallabag_v2' + Wallabag\ImportBundle\Controller\ShaarliController: + arguments: + $rabbitMqProducer: '@old_sound_rabbit_mq.import_shaarli_producer' + $redisProducer: '@wallabag_import.producer.redis.shaarli' + + Wallabag\ImportBundle\Controller\PocketHtmlController: + arguments: + $rabbitMqProducer: '@old_sound_rabbit_mq.import_pocket_html_producer' + $redisProducer: '@wallabag_import.producer.redis.pocket_html' + Wallabag\ImportBundle\: resource: '../../src/Wallabag/ImportBundle/*' exclude: '../../src/Wallabag/ImportBundle/{Consumer,Controller,Redis}' @@ -351,6 +361,14 @@ services: tags: - { name: wallabag_import.import, alias: chrome } + Wallabag\ImportBundle\Import\ShaarliImport: + tags: + - { name: wallabag_import.import, alias: shaarli } + + Wallabag\ImportBundle\Import\PocketHtmlImport: + tags: + - { name: wallabag_import.import, alias: pocket_html } + # to factorize the proximity and bypass translation for prev & next pagerfanta.view.default_wallabag: class: Pagerfanta\View\OptionableView diff --git a/app/config/services_rabbit.yml b/app/config/services_rabbit.yml index 26e02a784..1b7bfc4f9 100644 --- a/app/config/services_rabbit.yml +++ b/app/config/services_rabbit.yml @@ -18,6 +18,8 @@ services: $pinboardConsumer: '@old_sound_rabbit_mq.import_pinboard_consumer' $deliciousConsumer: '@old_sound_rabbit_mq.import_delicious_consumer' $elcuratorConsumer: '@old_sound_rabbit_mq.import_elcurator_consumer' + $shaarliConsumer: '@old_sound_rabbit_mq.import_shaarli_consumer' + $pocketHtmlConsumer: '@old_sound_rabbit_mq.import_pocket_html_consumer' wallabag_import.consumer.amqp.pocket: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -68,3 +70,13 @@ services: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: $import: '@Wallabag\ImportBundle\Import\ChromeImport' + + wallabag_import.consumer.amqp.shaarli: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\ShaarliImport' + + wallabag_import.consumer.amqp.pocket_html: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\PocketHtmlImport' diff --git a/app/config/services_redis.yml b/app/config/services_redis.yml index 02c7eba95..12e60f0bb 100644 --- a/app/config/services_redis.yml +++ b/app/config/services_redis.yml @@ -164,3 +164,35 @@ services: class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer arguments: $import: '@Wallabag\ImportBundle\Import\ChromeImport' + + # shaarli + wallabag_import.queue.redis.shaarli: + class: Simpleue\Queue\RedisQueue + arguments: + $queueName: "wallabag.import.shaarli" + + wallabag_import.producer.redis.shaarli: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.shaarli" + + wallabag_import.consumer.redis.shaarli: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\ShaarliImport' + + # pocket html + wallabag_import.queue.redis.pocket_html: + class: Simpleue\Queue\RedisQueue + arguments: + $queueName: "wallabag.import.pocket_html" + + wallabag_import.producer.redis.pocket_html: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.pocket_html" + + wallabag_import.consumer.redis.pocket_html: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\PocketHtmlImport' diff --git a/app/config/wallabag.yml b/app/config/wallabag.yml index bbbe46939..91541ff58 100644 --- a/app/config/wallabag.yml +++ b/app/config/wallabag.yml @@ -159,5 +159,5 @@ wallabag_core: rule: _all ~ "https?://www\.lemonde\.fr/tiny.*" wallabag_import: - allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv'] + allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv', 'text/html'] resource_dir: "%kernel.project_dir%/web/uploads/import" diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 6ca99bef5..541d75188 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -64,3 +64,13 @@ parameters: message: "#^Call to an undefined method DOMNode\\:\\:getAttribute\\(\\)\\.$#" count: 1 path: tests/Wallabag/CoreBundle/Controller/FeedControllerTest.php + + - + message: "#^Call to an undefined method Wallabag\\\\ImportBundle\\\\Import\\\\ImportInterface\\:\\:setUser\\(\\)\\.$#" + count: 1 + path: src/Wallabag/ImportBundle/Controller/HtmlController.php + + - + message: "#^Call to an undefined method Wallabag\\\\ImportBundle\\\\Import\\\\ImportInterface\\:\\:setFilepath\\(\\)\\.$#" + count: 1 + path: src/Wallabag/ImportBundle/Controller/HtmlController.php diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index 1031e5675..de2a8994a 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -13,10 +13,13 @@ use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInt use Symfony\Component\Security\Core\Authentication\Token\UsernamePasswordToken; use Wallabag\ImportBundle\Import\ChromeImport; use Wallabag\ImportBundle\Import\DeliciousImport; +use Wallabag\ImportBundle\Import\ElcuratorImport; use Wallabag\ImportBundle\Import\FirefoxImport; use Wallabag\ImportBundle\Import\InstapaperImport; use Wallabag\ImportBundle\Import\PinboardImport; +use Wallabag\ImportBundle\Import\PocketHtmlImport; use Wallabag\ImportBundle\Import\ReadabilityImport; +use Wallabag\ImportBundle\Import\ShaarliImport; use Wallabag\ImportBundle\Import\WallabagV1Import; use Wallabag\ImportBundle\Import\WallabagV2Import; use Wallabag\UserBundle\Entity\User; @@ -35,9 +38,26 @@ class ImportCommand extends Command private PinboardImport $pinboardImport; private DeliciousImport $deliciousImport; private WallabagV1Import $wallabagV1Import; + private ElcuratorImport $elcuratorImport; + private ShaarliImport $shaarliImport; + private PocketHtmlImport $pocketHtmlImport; - public function __construct(EntityManagerInterface $entityManager, TokenStorageInterface $tokenStorage, UserRepository $userRepository, WallabagV2Import $wallabagV2Import, FirefoxImport $firefoxImport, ChromeImport $chromeImport, ReadabilityImport $readabilityImport, InstapaperImport $instapaperImport, PinboardImport $pinboardImport, DeliciousImport $deliciousImport, WallabagV1Import $wallabagV1Import) - { + public function __construct( + EntityManagerInterface $entityManager, + TokenStorageInterface $tokenStorage, + UserRepository $userRepository, + WallabagV2Import $wallabagV2Import, + FirefoxImport $firefoxImport, + ChromeImport $chromeImport, + ReadabilityImport $readabilityImport, + InstapaperImport $instapaperImport, + PinboardImport $pinboardImport, + DeliciousImport $deliciousImport, + WallabagV1Import $wallabagV1Import, + ElcuratorImport $elcuratorImport, + ShaarliImport $shaarliImport, + PocketHtmlImport $pocketHtmlImport + ) { $this->entityManager = $entityManager; $this->tokenStorage = $tokenStorage; $this->userRepository = $userRepository; @@ -49,6 +69,9 @@ class ImportCommand extends Command $this->pinboardImport = $pinboardImport; $this->deliciousImport = $deliciousImport; $this->wallabagV1Import = $wallabagV1Import; + $this->elcuratorImport = $elcuratorImport; + $this->shaarliImport = $shaarliImport; + $this->pocketHtmlImport = $pocketHtmlImport; parent::__construct(); } @@ -60,7 +83,7 @@ class ImportCommand extends Command ->setDescription('Import entries from a JSON export') ->addArgument('username', InputArgument::REQUIRED, 'User to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox or chrome', 'v1') + ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox, chrome, elcurator, shaarli or pocket', 'v1') ->addOption('markAsRead', null, InputOption::VALUE_OPTIONAL, 'Mark all entries as read', false) ->addOption('useUserId', null, InputOption::VALUE_NONE, 'Use user id instead of username to find account') ->addOption('disableContentUpdate', null, InputOption::VALUE_NONE, 'Disable fetching updated content from URL') @@ -120,6 +143,15 @@ class ImportCommand extends Command case 'delicious': $import = $this->deliciousImport; break; + case 'elcurator': + $import = $this->elcuratorImport; + break; + case 'shaarli': + $import = $this->shaarliImport; + break; + case 'pocket': + $import = $this->pocketHtmlImport; + break; default: $import = $this->wallabagV1Import; } diff --git a/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php b/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php index 443d167cd..7f53899d9 100644 --- a/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php +++ b/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php @@ -20,9 +20,23 @@ class RabbitMQConsumerTotalProxy private Consumer $pinboardConsumer; private Consumer $deliciousConsumer; private Consumer $elcuratorConsumer; + private Consumer $shaarliConsumer; + private Consumer $pocketHtmlConsumer; - public function __construct(Consumer $pocketConsumer, Consumer $readabilityConsumer, Consumer $wallabagV1Consumer, Consumer $wallabagV2Consumer, Consumer $firefoxConsumer, Consumer $chromeConsumer, Consumer $instapaperConsumer, Consumer $pinboardConsumer, Consumer $deliciousConsumer, Consumer $elcuratorConsumer) - { + public function __construct( + Consumer $pocketConsumer, + Consumer $readabilityConsumer, + Consumer $wallabagV1Consumer, + Consumer $wallabagV2Consumer, + Consumer $firefoxConsumer, + Consumer $chromeConsumer, + Consumer $instapaperConsumer, + Consumer $pinboardConsumer, + Consumer $deliciousConsumer, + Consumer $elcuratorConsumer, + Consumer $shaarliConsumer, + Consumer $pocketHtmlConsumer + ) { $this->pocketConsumer = $pocketConsumer; $this->readabilityConsumer = $readabilityConsumer; $this->wallabagV1Consumer = $wallabagV1Consumer; @@ -33,6 +47,8 @@ class RabbitMQConsumerTotalProxy $this->pinboardConsumer = $pinboardConsumer; $this->deliciousConsumer = $deliciousConsumer; $this->elcuratorConsumer = $elcuratorConsumer; + $this->shaarliConsumer = $shaarliConsumer; + $this->pocketHtmlConsumer = $pocketHtmlConsumer; } /** @@ -77,6 +93,12 @@ class RabbitMQConsumerTotalProxy case 'elcurator': $consumer = $this->elcuratorConsumer; break; + case 'shaarli': + $consumer = $this->shaarliConsumer; + break; + case 'pocket_html': + $consumer = $this->pocketHtmlConsumer; + break; default: return 0; } diff --git a/src/Wallabag/ImportBundle/Controller/HtmlController.php b/src/Wallabag/ImportBundle/Controller/HtmlController.php new file mode 100644 index 000000000..e9515561f --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/HtmlController.php @@ -0,0 +1,83 @@ +createForm(UploadImportType::class); + $form->handleRequest($request); + + $wallabag = $this->getImportService(); + $wallabag->setUser($this->getUser()); + + if ($form->isSubmitted() && $form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = $this->getUser()->getId() . '.html'; + + if (null !== $file && \in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes'), true) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $wallabag + ->setFilepath($this->getParameter('wallabag_import.resource_dir') . '/' . $name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $wallabag->getSummary(); + $message = $translator->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + if (0 < $summary['queued']) { + $message = $translator->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + + unlink($this->getParameter('wallabag_import.resource_dir') . '/' . $name); + } + + $this->addFlash('notice', $message); + + return $this->redirect($this->generateUrl('homepage')); + } + $this->addFlash('notice', 'flashes.import.notice.failed_on_file'); + } + + return $this->render($this->getImportTemplate(), [ + 'form' => $form->createView(), + 'import' => $wallabag, + ]); + } + + /** + * Return the service to handle the import. + * + * @return ImportInterface + */ + abstract protected function getImportService(); + + /** + * Return the template used for the form. + * + * @return string + */ + abstract protected function getImportTemplate(); +} diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 9309d7d4e..55578df9f 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -57,6 +57,8 @@ class ImportController extends AbstractController + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pinboard') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('delicious') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('elcurator') + + $this->rabbitMQConsumerTotalProxy->getTotalMessage('shaarli') + + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pocket_html') ; } catch (\Exception $e) { $rabbitNotInstalled = true; @@ -75,6 +77,8 @@ class ImportController extends AbstractController + $redis->llen('wallabag.import.pinboard') + $redis->llen('wallabag.import.delicious') + $redis->llen('wallabag.import.elcurator') + + $redis->llen('wallabag.import.shaarli') + + $redis->llen('wallabag.import.pocket_html') ; } catch (\Exception $e) { $redisNotInstalled = true; diff --git a/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php b/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php new file mode 100644 index 000000000..7387bbfdc --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php @@ -0,0 +1,57 @@ +pocketHtmlImport = $pocketHtmlImport; + $this->craueConfig = $craueConfig; + $this->rabbitMqProducer = $rabbitMqProducer; + $this->redisProducer = $redisProducer; + } + + /** + * @Route("/pocket_html", name="import_pocket_html") + */ + public function indexAction(Request $request, TranslatorInterface $translator) + { + return parent::indexAction($request, $translator); + } + + /** + * {@inheritdoc} + */ + protected function getImportService() + { + if ($this->craueConfig->get('import_with_rabbitmq')) { + $this->pocketHtmlImport->setProducer($this->rabbitMqProducer); + } elseif ($this->craueConfig->get('import_with_redis')) { + $this->pocketHtmlImport->setProducer($this->redisProducer); + } + + return $this->pocketHtmlImport; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return '@WallabagImport/PocketHtml/index.html.twig'; + } +} diff --git a/src/Wallabag/ImportBundle/Controller/ShaarliController.php b/src/Wallabag/ImportBundle/Controller/ShaarliController.php new file mode 100644 index 000000000..46dfd1473 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/ShaarliController.php @@ -0,0 +1,57 @@ +shaarliImport = $shaarliImport; + $this->craueConfig = $craueConfig; + $this->rabbitMqProducer = $rabbitMqProducer; + $this->redisProducer = $redisProducer; + } + + /** + * @Route("/shaarli", name="import_shaarli") + */ + public function indexAction(Request $request, TranslatorInterface $translator) + { + return parent::indexAction($request, $translator); + } + + /** + * {@inheritdoc} + */ + protected function getImportService() + { + if ($this->craueConfig->get('import_with_rabbitmq')) { + $this->shaarliImport->setProducer($this->rabbitMqProducer); + } elseif ($this->craueConfig->get('import_with_redis')) { + $this->shaarliImport->setProducer($this->redisProducer); + } + + return $this->shaarliImport; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return '@WallabagImport/Shaarli/index.html.twig'; + } +} diff --git a/src/Wallabag/ImportBundle/Import/HtmlImport.php b/src/Wallabag/ImportBundle/Import/HtmlImport.php new file mode 100644 index 000000000..a96107e0b --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/HtmlImport.php @@ -0,0 +1,210 @@ +user) { + $this->logger->error('Wallabag HTML Import: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('Wallabag HTML Import: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $html = new \DOMDocument(); + + libxml_use_internal_errors(true); + $html->loadHTMLFile($this->filepath); + $hrefs = $html->getElementsByTagName('a'); + libxml_use_internal_errors(false); + + if (0 === $hrefs->length) { + $this->logger->error('Wallabag HTML: no entries in imported file'); + + return false; + } + + $entries = []; + foreach ($hrefs as $href) { + $entry = []; + $entry['url'] = $href->getAttribute('href'); + $entry['tags'] = $href->getAttribute('tags'); + $entry['created_at'] = $href->getAttribute('add_date'); + $entries[] = $entry; + } + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + /** + * Set file path to the html file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $url = $importedEntry['url']; + + $existingEntry = $this->em + ->getRepository(Entry::class) + ->findByUrlAndUserId($url, $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return null; + } + + $data = $this->prepareEntry($importedEntry); + + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->updateArchived($data['is_archived']); + $createdAt = new \DateTime(); + $createdAt->setTimestamp($data['created_at']); + $entry->setCreatedAt($createdAt); + + // update entry with content (in case fetching failed, the given entry will be return) + $this->fetchContent($entry, $data['url'], $data); + + if (\array_key_exists('tags', $data)) { + $this->tagsAssigner->assignTagsToEntry( + $entry, + $data['tags'] + ); + } + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * Parse and insert all given entries. + */ + protected function parseEntries(array $entries) + { + $i = 1; + $entryToBeFlushed = []; + + foreach ($entries as $importedEntry) { + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // @see AbstractImport + $entryToBeFlushed[] = $entry; + + // flush every 20 entries + if (0 === ($i % 20)) { + $this->em->flush(); + + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(new EntrySavedEvent($entry), EntrySavedEvent::NAME); + } + + $entryToBeFlushed = []; + } + ++$i; + } + + $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(new EntrySavedEvent($entry), EntrySavedEvent::NAME); + } + } + } + + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + */ + protected function parseEntriesForProducer(array $entries) + { + foreach ($entries as $importedEntry) { + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->queuedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } + + abstract protected function prepareEntry(array $entry = []); +} diff --git a/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php b/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php new file mode 100644 index 000000000..492a1adfc --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php @@ -0,0 +1,113 @@ +user) { + $this->logger->error('Pocket HTML Import: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('Pocket HTML Import: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $html = new \DOMDocument(); + + libxml_use_internal_errors(true); + $html->loadHTMLFile($this->filepath); + $hrefs = $html->getElementsByTagName('a'); + libxml_use_internal_errors(false); + + if (0 === $hrefs->length) { + $this->logger->error('Pocket HTML: no entries in imported file'); + + return false; + } + + $entries = []; + foreach ($hrefs as $href) { + $entry = []; + $entry['url'] = $href->getAttribute('href'); + $entry['tags'] = $href->getAttribute('tags'); + $entry['created_at'] = $href->getAttribute('time_added'); + $entries[] = $entry; + } + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + /** + * {@inheritdoc} + */ + protected function prepareEntry(array $entry = []) + { + $data = [ + 'title' => '', + 'html' => false, + 'url' => $entry['url'], + 'is_archived' => (int) $this->markAsRead, + 'is_starred' => false, + 'tags' => '', + 'created_at' => $entry['created_at'], + ]; + + if (\array_key_exists('tags', $entry) && '' !== $entry['tags']) { + $data['tags'] = $entry['tags']; + } + + return $data; + } +} diff --git a/src/Wallabag/ImportBundle/Import/ShaarliImport.php b/src/Wallabag/ImportBundle/Import/ShaarliImport.php new file mode 100644 index 000000000..b4c9dc3c3 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ShaarliImport.php @@ -0,0 +1,66 @@ + '', + 'html' => false, + 'url' => $entry['url'], + 'is_archived' => (int) $this->markAsRead, + 'is_starred' => false, + 'tags' => '', + 'created_at' => $entry['created_at'], + ]; + + if (\array_key_exists('tags', $entry) && '' !== $entry['tags']) { + $data['tags'] = $entry['tags']; + } + + return $data; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig new file mode 100644 index 000000000..09f2e689f --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig @@ -0,0 +1,45 @@ +{% extends "@WallabagCore/layout.html.twig" %} + +{% block title %}{{ 'import.pocket_html.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include '@WallabagImport/Import/_information.html.twig' %} + +
+
{{ import.description|trans|raw }}
+

{{ 'import.pocket_html.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig new file mode 100644 index 000000000..edb24e468 --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig @@ -0,0 +1,45 @@ +{% extends "@WallabagCore/layout.html.twig" %} + +{% block title %}{{ 'import.shaarli.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include '@WallabagImport/Import/_information.html.twig' %} + +
+
{{ import.description|trans|raw }}
+

{{ 'import.shaarli.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php index 1c30d6b0a..0e2c5b0ed 100644 --- a/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php @@ -123,9 +123,9 @@ class FirefoxControllerTest extends WallabagCoreTestCase ); $this->assertInstanceOf(Entry::class, $content); - $this->assertNotEmpty($content->getMimetype(), 'Mimetype for http://lexpansion.lexpress.fr is ok'); - $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for http://lexpansion.lexpress.fr is ok'); - $this->assertNotEmpty($content->getLanguage(), 'Language for http://lexpansion.lexpress.fr is ok'); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok'); $this->assertCount(3, $content->getTags()); $content = $client->getContainer() diff --git a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php index 06a2a2e21..b5b220b50 100644 --- a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php @@ -24,6 +24,6 @@ class ImportControllerTest extends WallabagCoreTestCase $crawler = $client->request('GET', '/import/'); $this->assertSame(200, $client->getResponse()->getStatusCode()); - $this->assertSame(10, $crawler->filter('blockquote')->count()); + $this->assertSame(12, $crawler->filter('blockquote')->count()); } } diff --git a/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php new file mode 100644 index 000000000..7ae4a247f --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php @@ -0,0 +1,168 @@ +logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_html'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportPocketHtmlWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/pocket_html'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 0); + } + + public function testImportPocketHtmlBadFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_html'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + } + + public function testImportPocketHtmlWithRedisEnabled() + { + $this->checkRedis(); + $this->logInAs('admin'); + $client = $this->getTestClient(); + $client->getContainer()->get(Config::class)->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/pocket_html'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/ril_export.html', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get(Client::class)->lpop('wallabag.import.pocket_html')); + + $client->getContainer()->get(Config::class)->set('import_with_redis', 0); + } + + public function testImportWallabagWithPocketHtmlFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_html'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/ril_export.html', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://www.20minutes.fr/sport/4002755-20220928-tarn-lapins-ravagent-terrain-match-rugby-doit-etre-annule', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok'); + $this->assertCount(3, $content->getTags()); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://www.lemonde.fr/disparitions/article/2018/07/05/le-journaliste-et-cineaste-claude-lanzmann-est-mort_5326313_3382.html', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for https://www.lemonde.fr is ok'); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_html'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/test.html', 'test.html'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php new file mode 100644 index 000000000..8bc9ffb9d --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php @@ -0,0 +1,168 @@ +logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/shaarli'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportShaarliWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/shaarli'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 0); + } + + public function testImportShaarliBadFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/shaarli'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + } + + public function testImportShaarliWithRedisEnabled() + { + $this->checkRedis(); + $this->logInAs('admin'); + $client = $this->getTestClient(); + $client->getContainer()->get(Config::class)->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/shaarli'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/shaarli-bookmarks.html', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get(Client::class)->lpop('wallabag.import.shaarli')); + + $client->getContainer()->get(Config::class)->set('import_with_redis', 0); + } + + public function testImportWallabagWithShaarliFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/shaarli'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/shaarli-bookmarks.html', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://www.20minutes.fr/sport/4002755-20220928-tarn-lapins-ravagent-terrain-match-rugby-doit-etre-annule', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok'); + $this->assertCount(2, $content->getTags()); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://www.lemonde.fr/disparitions/article/2018/07/05/le-journaliste-et-cineaste-claude-lanzmann-est-mort_5326313_3382.html', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for https://www.lemonde.fr is ok'); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/shaarli'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/test.html', 'test.html'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php new file mode 100644 index 000000000..6ff5e13a0 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php @@ -0,0 +1,254 @@ +getPocketHtmlImport(); + + $this->assertSame('Pocket HTML', $pocketHtmlImport->getName()); + $this->assertNotEmpty($pocketHtmlImport->getUrl()); + $this->assertSame('import.pocket_html.description', $pocketHtmlImport->getDescription()); + } + + public function testImport() + { + $pocketHtmlImport = $this->getPocketHtmlImport(false, 2); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(2)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $pocketHtmlImport->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 2, 'queued' => 0], $pocketHtmlImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $pocketHtmlImport = $this->getPocketHtmlImport(false, 1); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return (bool) $persistedEntry->isArchived(); + })); + + $res = $pocketHtmlImport + ->setMarkAsRead(true) + ->import(); + + $this->assertTrue($res); + + $this->assertSame(['skipped' => 1, 'imported' => 1, 'queued' => 0], $pocketHtmlImport->getSummary()); + } + + public function testImportWithRabbit() + { + $pocketHtmlImport = $this->getPocketHtmlImport(); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder(\OldSound\RabbitMqBundle\RabbitMq\Producer::class) + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(2)) + ->method('publish'); + + $pocketHtmlImport->setProducer($producer); + + $res = $pocketHtmlImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $pocketHtmlImport->getSummary()); + } + + public function testImportWithRedis() + { + $pocketHtmlImport = $this->getPocketHtmlImport(); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter(Client::class, true); + + $queue = new RedisQueue($redisMock, 'pocket_html'); + $producer = new Producer($queue); + + $pocketHtmlImport->setProducer($producer); + + $res = $pocketHtmlImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $pocketHtmlImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('pocket_html')); + } + + public function testImportBadFile() + { + $pocketHtmlImport = $this->getPocketHtmlImport(); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/wallabag-v1.jsonx'); + + $res = $pocketHtmlImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Pocket HTML Import: unable to read file', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $pocketHtmlImport = $this->getPocketHtmlImport(true); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $res = $pocketHtmlImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Pocket HTML Import: user is not defined', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + private function getPocketHtmlImport($unsetUser = false, $dispatched = 0) + { + $this->user = new User(); + + $this->em = $this->getMockBuilder(EntityManager::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder(ContentProxy::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->tagsAssigner = $this->getMockBuilder(TagsAssigner::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher = $this->getMockBuilder(EventDispatcher::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + + $wallabag = new PocketHtmlImport($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher, $logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } +} diff --git a/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php b/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php new file mode 100644 index 000000000..04f8223dd --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php @@ -0,0 +1,254 @@ +getShaarliImport(); + + $this->assertSame('Shaarli', $shaarliImport->getName()); + $this->assertNotEmpty($shaarliImport->getUrl()); + $this->assertSame('import.shaarli.description', $shaarliImport->getDescription()); + } + + public function testImport() + { + $shaarliImport = $this->getShaarliImport(false, 2); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(2)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $shaarliImport->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 2, 'queued' => 0], $shaarliImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $shaarliImport = $this->getShaarliImport(false, 1); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return (bool) $persistedEntry->isArchived(); + })); + + $res = $shaarliImport + ->setMarkAsRead(true) + ->import(); + + $this->assertTrue($res); + + $this->assertSame(['skipped' => 1, 'imported' => 1, 'queued' => 0], $shaarliImport->getSummary()); + } + + public function testImportWithRabbit() + { + $shaarliImport = $this->getShaarliImport(); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder(\OldSound\RabbitMqBundle\RabbitMq\Producer::class) + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(2)) + ->method('publish'); + + $shaarliImport->setProducer($producer); + + $res = $shaarliImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $shaarliImport->getSummary()); + } + + public function testImportWithRedis() + { + $shaarliImport = $this->getShaarliImport(); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter(Client::class, true); + + $queue = new RedisQueue($redisMock, 'shaarli'); + $producer = new Producer($queue); + + $shaarliImport->setProducer($producer); + + $res = $shaarliImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $shaarliImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('shaarli')); + } + + public function testImportBadFile() + { + $shaarliImport = $this->getShaarliImport(); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/wallabag-v1.jsonx'); + + $res = $shaarliImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Wallabag HTML Import: unable to read file', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $shaarliImport = $this->getShaarliImport(true); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $res = $shaarliImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Wallabag HTML Import: user is not defined', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + private function getShaarliImport($unsetUser = false, $dispatched = 0) + { + $this->user = new User(); + + $this->em = $this->getMockBuilder(EntityManager::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder(ContentProxy::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->tagsAssigner = $this->getMockBuilder(TagsAssigner::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher = $this->getMockBuilder(EventDispatcher::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + + $wallabag = new ShaarliImport($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher, $logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } +} diff --git a/tests/Wallabag/ImportBundle/fixtures/ril_export.html b/tests/Wallabag/ImportBundle/fixtures/ril_export.html new file mode 100644 index 000000000..310c092dc --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/ril_export.html @@ -0,0 +1,21 @@ + + + + + + Pocket Export + + +

Unread

+ + +

Read Archive

+ + + diff --git a/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html b/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html new file mode 100644 index 000000000..ad401ea74 --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html @@ -0,0 +1,13 @@ + + + +Bookmarks +

Shaarli export of all bookmarks on Mon, 17 Jul 23 14:31:25 +0200

+

+

The Legacy of Firefox OS. In the two years or so since Mozilla… | by Ben Francis | Medium +
In the two years or so since Mozilla announced the end of Firefox OS as a Mozilla-run project, the B2G source code has found its way into a surprising number of commercial products. +
Template Filters — Eleventy +

+ diff --git a/translations/messages.en.yml b/translations/messages.en.yml index 5462280c7..19b85863e 100644 --- a/translations/messages.en.yml +++ b/translations/messages.en.yml @@ -530,6 +530,14 @@ import: page_title: Import > del.icio.us description: This importer will import all your Delicious bookmarks. Since 2021, you can export again your data from it using the export page (https://del.icio.us/export). Choose the "JSON" format and download it (like "delicious_export.2021.02.06_21.10.json"). how_to: Please select your Delicious export and click on the button below to upload and import it. + shaarli: + page_title: Import > Shaarli + description: This importer will import all your Shaarli bookmarks. Just go to the Tools section, then into "Export database", choose your bookmarks and export them. You will obtain a HTML file. + how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched. + pocket_html: + page_title: Import > Pocket HTML + description: This importer will import all your Pocket bookmarks (via HTML export). Just go to https://getpocket.com/export, then export the HTML file. An HTML file will be downloaded (like "ril_export.html"). + how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched. developer: page_title: API clients management welcome_message: Welcome to the wallabag API